From 8824ff7c9f25a47aa893512fba9174ac44c2ab4f Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 21 Oct 2026 16:27:15 -0400 Subject: [PATCH 01/31] feat(iceberg): scan Iceberg tables in Ballista --- .github/workflows/rust.yml | 2 + Cargo.lock | 1125 ++++++++++++++++- Cargo.toml | 8 + docs/source/index.rst | 1 + docs/source/user-guide/iceberg.md | 108 ++ integrations/iceberg/Cargo.toml | 62 + .../examples/standalone-iceberg-scan.rs | 130 ++ integrations/iceberg/src/lib.rs | 82 ++ integrations/iceberg/src/logical_codec.rs | 260 ++++ integrations/iceberg/src/physical_codec.rs | 177 +++ integrations/iceberg/src/wire.rs | 205 +++ integrations/iceberg/tests/codec.rs | 305 +++++ integrations/iceberg/tests/common/mod.rs | 158 +++ integrations/iceberg/tests/common/rustfs.rs | 144 +++ integrations/iceberg/tests/object_storage.rs | 39 + integrations/iceberg/tests/scan.rs | 32 + 16 files changed, 2809 insertions(+), 29 deletions(-) create mode 100644 docs/source/user-guide/iceberg.md create mode 100644 integrations/iceberg/Cargo.toml create mode 100644 integrations/iceberg/examples/standalone-iceberg-scan.rs create mode 100644 integrations/iceberg/src/lib.rs create mode 100644 integrations/iceberg/src/logical_codec.rs create mode 100644 integrations/iceberg/src/physical_codec.rs create mode 100644 integrations/iceberg/src/wire.rs create mode 100644 integrations/iceberg/tests/codec.rs create mode 100644 integrations/iceberg/tests/common/mod.rs create mode 100644 integrations/iceberg/tests/common/rustfs.rs create mode 100644 integrations/iceberg/tests/object_storage.rs create mode 100644 integrations/iceberg/tests/scan.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 0a3ae40345..6c5d3a3c78 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -29,6 +29,7 @@ on: - "ballista-cli/**" - "benchmarks/**" - "examples/**" + - "integrations/**" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" @@ -46,6 +47,7 @@ on: - "ballista-cli/**" - "benchmarks/**" - "examples/**" + - "integrations/**" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" diff --git a/Cargo.lock b/Cargo.lock index 902aef8084..37edfb2b96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,41 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.8.12" @@ -31,6 +66,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -132,6 +173,33 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.11.3", + "log", + "miniz_oxide 0.9.1", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.28.0", + "thiserror 2.0.19", + "uuid", + "zstd", +] + [[package]] name = "approx" version = "0.5.1" @@ -159,6 +227,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + [[package]] name = "arraydeque" version = "0.5.1" @@ -250,7 +324,7 @@ dependencies = [ "serde", "serde_json", "snap", - "strum_macros", + "strum_macros 0.28.0", "uuid", "zstd", ] @@ -417,6 +491,7 @@ version = "59.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" dependencies = [ + "serde", "serde_core", "serde_json", ] @@ -452,6 +527,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + [[package]] name = "astral-tokio-tar" version = "0.6.4" @@ -480,6 +561,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -524,6 +616,22 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + [[package]] name = "atoi" version = "2.0.0" @@ -577,7 +685,7 @@ dependencies = [ "fastrand", "hex", "http 1.5.0", - "sha1", + "sha1 0.10.7", "time", "tokio", "tracing", @@ -1003,6 +1111,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers 0.3.0", + "tokio", +] + [[package]] name = "ballista" version = "55.0.0" @@ -1096,7 +1215,7 @@ dependencies = [ "dirs 7.0.0", "dotparser", "futures", - "gloo-timers", + "gloo-timers 0.4.0", "js-sys", "mimalloc", "prometheus-parse", @@ -1398,8 +1517,15 @@ dependencies = [ "num-bigint 0.4.8", "num-integer", "num-traits", + "serde", ] +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + [[package]] name = "bit-set" version = "0.5.3" @@ -1483,6 +1609,16 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + [[package]] name = "bollard" version = "0.21.1" @@ -1556,6 +1692,29 @@ dependencies = [ "time", ] +[[package]] +name = "bon" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "brotli" version = "8.0.4" @@ -1604,6 +1763,12 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.12.1" @@ -1692,8 +1857,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-link", ] @@ -1734,6 +1901,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clap" version = "2.34.0" @@ -2004,6 +2181,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2147,6 +2334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -2200,6 +2388,15 @@ dependencies = [ "linktime-proc-macro", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -2209,14 +2406,48 @@ dependencies = [ "cmov", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -2232,17 +2463,52 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn 2.0.119", ] +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -3105,6 +3371,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -3177,7 +3474,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3191,6 +3488,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + [[package]] name = "docker_credential" version = "1.4.0" @@ -3305,6 +3611,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -3312,7 +3629,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3340,6 +3657,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fancy-regex" version = "0.11.0" @@ -3356,6 +3693,18 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -3613,12 +3962,34 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "glob" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "gloo-timers" version = "0.4.0" @@ -3752,7 +4123,13 @@ dependencies = [ [[package]] name = "heck" -version = "0.5.0" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" @@ -3991,6 +4368,154 @@ dependencies = [ "cc", ] +[[package]] +name = "iceberg" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "iceberg-ballista" +version = "55.0.0" +dependencies = [ + "arrow-schema", + "async-trait", + "ballista", + "ballista-core", + "datafusion", + "datafusion-proto", + "futures", + "iceberg", + "iceberg-catalog-rest", + "iceberg-datafusion", + "iceberg-storage-opendal", + "serde", + "serde_json", + "tempfile", + "testcontainers", + "tokio", +] + +[[package]] +name = "iceberg-catalog-rest" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "async-trait", + "chrono", + "http 1.5.0", + "iceberg", + "itertools 0.13.0", + "reqwest 0.12.28", + "serde", + "serde_derive", + "serde_json", + "tokio", + "tracing", + "typed-builder", + "uuid", +] + +[[package]] +name = "iceberg-datafusion" +version = "0.12.0" +source = "git+https://github.com/apache/datafusion-iceberg?rev=b641e5ebe9e92967499ae7e440ba10d7797712df#b641e5ebe9e92967499ae7e440ba10d7797712df" +dependencies = [ + "async-trait", + "bytes", + "dashmap", + "datafusion", + "futures", + "iceberg", + "parquet", + "serde", + "tokio", + "typetag", + "uuid", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "iceberg-property-macro", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -4132,6 +4657,15 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "insta" version = "1.49.0" @@ -4150,7 +4684,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "darling", + "darling 0.23.0", "indoc", "proc-macro2", "quote", @@ -4179,6 +4713,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -4242,10 +4785,14 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", + "jiff-tzdb-platform", + "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -4269,6 +4816,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -4696,12 +5258,38 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "multimap" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -4761,7 +5349,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4786,6 +5374,7 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -4929,7 +5518,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.39.4", "rand 0.10.3", "reqwest 0.12.28", "ring", @@ -4964,6 +5553,147 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-fs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http 1.5.0", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http 1.5.0", + "http-body 1.1.0", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http 1.5.0", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-fs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http 1.5.0", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -4994,6 +5724,40 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + [[package]] name = "outref" version = "0.5.2" @@ -5034,6 +5798,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -5362,6 +6132,18 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.14.0" @@ -5411,6 +6193,16 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.3", +] + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -5453,6 +6245,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + [[package]] name = "procfs" version = "0.17.0" @@ -5521,11 +6326,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.12.1", "log", "multimap", "petgraph", - "prettyplease", + "prettyplease 0.2.37", "prost", "prost-types", "pulldown-cmark", @@ -5542,7 +6347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.119", @@ -5607,6 +6412,12 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + [[package]] name = "quick-xml" version = "0.39.4" @@ -5617,6 +6428,26 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.11" @@ -5671,7 +6502,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5756,6 +6587,9 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] [[package]] name = "rand_core" @@ -5813,7 +6647,7 @@ dependencies = [ "lru 0.18.5", "palette", "serde", - "strum", + "strum 0.28.0", "thiserror 2.0.19", "unicode-segmentation", "unicode-truncate", @@ -5889,7 +6723,7 @@ dependencies = [ "line-clipping", "ratatui-core", "serde", - "strum", + "strum 0.28.0", "time", "unicode-segmentation", "unicode-width 0.2.2", @@ -6042,6 +6876,74 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http 1.5.0", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http 1.5.0", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "hex", + "hmac", + "http 1.5.0", + "jiff", + "log", + "percent-encoding", + "sha1 0.11.0", + "sha2 0.11.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + [[package]] name = "reqwest" version = "0.12.28" @@ -6080,7 +6982,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", ] @@ -6094,6 +6996,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", + "futures-util", "h2", "http 1.5.0", "http-body 1.1.0", @@ -6115,12 +7018,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -6138,6 +7043,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + [[package]] name = "rstest" version = "0.27.0" @@ -6167,6 +7082,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -6211,7 +7136,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6270,7 +7195,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6445,6 +7370,25 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.229" @@ -6563,7 +7507,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "syn 2.0.119", @@ -6593,6 +7537,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -6732,6 +7687,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqlparser" version = "0.62.0" @@ -6832,13 +7793,34 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros", + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -6864,7 +7846,7 @@ dependencies = [ "pbjson", "pbjson-build", "pbjson-types", - "prettyplease", + "prettyplease 0.2.37", "prost", "prost-build", "prost-types", @@ -6979,6 +7961,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -6992,10 +7980,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7382,7 +8370,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "quote", "syn 2.0.119", @@ -7405,7 +8393,7 @@ version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" dependencies = [ - "prettyplease", + "prettyplease 0.2.37", "proc-macro2", "prost-build", "prost-types", @@ -7589,12 +8577,62 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "typify" version = "0.5.0" @@ -7692,6 +8730,16 @@ dependencies = [ "rustc-std-workspace-core", ] +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -7909,6 +8957,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -8034,7 +9095,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8312,6 +9373,12 @@ dependencies = [ "hashlink", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index ee642f93d7..1908b263ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "benchmarks", "chaos-testing", "examples", + "integrations/iceberg", ] resolver = "3" @@ -66,6 +67,13 @@ datafusion-proto = "55" datafusion-proto-common = "55" datafusion-substrait = "55" +# Not released yet. Keep `iceberg` at the revision iceberg-datafusion's own +# workspace pins, so cargo resolves a single iceberg crate. +iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg", rev = "b641e5ebe9e92967499ae7e440ba10d7797712df" } +iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } + ctor = { version = "1.0" } insta = "1.47" itertools = "0.15" diff --git a/docs/source/index.rst b/docs/source/index.rst index 06ee8223bd..3c6680d798 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,7 @@ Table of content user-guide/faq user-guide/extending-components user-guide/extensions-example + user-guide/iceberg .. toctree:: :maxdepth: 1 diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md new file mode 100644 index 0000000000..4b985664a7 --- /dev/null +++ b/docs/source/user-guide/iceberg.md @@ -0,0 +1,108 @@ + + +# Apache Iceberg + +The `iceberg-ballista` crate (`integrations/iceberg`) lets Ballista scan +[Apache Iceberg](https://iceberg.apache.org/) tables. On top of +[iceberg-datafusion](https://github.com/apache/datafusion-iceberg), it adds the +codecs that send Iceberg tables and scans between the client, the scheduler and +the executors. + +The crate is not published yet. It depends on unreleased revisions of +iceberg-datafusion and iceberg-rust, and needs Rust 1.95. + +## Usage + +Register the codecs on the session config, then mount an Iceberg catalog: + +```rust +use std::sync::Arc; + +use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use datafusion::execution::SessionStateBuilder; +use datafusion::prelude::{SessionConfig, SessionContext}; +use iceberg_ballista::register_iceberg_codecs; +use iceberg_datafusion::IcebergCatalogProvider; + +let state = SessionStateBuilder::new() + .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_default_features() + .build(); +let ctx = SessionContext::standalone_with_state(state).await?; + +ctx.register_catalog("rest", Arc::new(IcebergCatalogProvider::try_new(catalog).await?)); +ctx.sql("SELECT * FROM rest.ns.t").await?.show().await?; +``` + +A single table registered as an `IcebergCatalogTableProvider` works the same +way. Either kind reads the table's latest version in each query. + +For time travel, register an `IcebergStaticTableProvider`, which keeps one +version: `IcebergStaticTableProvider::try_new_from_table_snapshot(table, snapshot_id)`. + +The example registers both on a table in a temporary local warehouse, and shows +a commit reaching the live table but not the static one: + +```shell +cargo run -p iceberg-ballista --example standalone-iceberg-scan +``` + +## How it works + +Ballista sends the client's plan to the scheduler, and each stage's plan to +the executors, as bytes. The codecs replace each live Iceberg table with one +version of it: the location of its metadata file and its serialized `FileIO`. +Metadata files never change, so every node rebuilds exactly that version. + +- **A catalog-backed table** is sent as the version its catalog serves when the + client sends the query. Each query asks again, so each reads the latest + version. +- **A static table** is sent as the version it holds. +- **A scan** is sent as the version the scheduler planned it against. + +Tables sent from the client also carry the metadata itself, so the scheduler +needn't read the file. Scans sent to executors carry only the location, which +keeps each task's plan small. + +Only the client talks to a catalog; the scheduler and executors need no catalog +configuration. + +## Limitations + +- **A table read twice in one query can see two versions.** Each scan of a + catalog-backed table, such as each side of a self-join, asks the catalog + separately, so a commit that lands in between is seen by only one of them. +- **Catalog-backed tables need a multi-threaded client runtime.** Sending a + query loads them while blocking the calling thread, which can hang a + single-threaded tokio runtime (`#[tokio::test]`'s default), so there it fails + with an error. Static tables work on either. +- **Reads only.** `INSERT INTO` an Iceberg table fails with "isn't supported + on Ballista yet". +- **No metadata tables.** Tables such as `t$snapshots` can't be sent to the + scheduler yet. +- **One task per scan.** Iceberg scans do not split into partitions yet, so one + executor reads the whole table. Operators above the scan still run in + parallel. +- **Storage credentials travel in plain text.** Each table's serialized + `FileIO` includes them, so the links between client, scheduler and executors + must be trusted. +- **Only standalone mode is tested.** A separately deployed scheduler and + executors must link `iceberg-ballista` and register the codecs too (see + [Extending Components](extending-components.md)). diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml new file mode 100644 index 0000000000..8a097ff01a --- /dev/null +++ b/integrations/iceberg/Cargo.toml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iceberg-ballista" +description = "Apache Iceberg table scans for Ballista" +license = { workspace = true } +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +keywords = ["iceberg", "ballista", "datafusion", "distributed"] +categories = ["database"] +edition = { workspace = true } +# Required by the iceberg crates; the rest of the workspace still builds on 1.94. +rust-version = "1.95" +# crates.io rejects the git dependencies on iceberg and iceberg-datafusion. +publish = false + +[package.metadata.cargo-machete] +# Not imported: enables the `serde` feature, to send a scan's Arrow schema. +ignored = ["arrow-schema"] + +[dependencies] +arrow-schema = { version = "59.2.0", features = ["serde"] } +async-trait = { workspace = true } +ballista-core = { workspace = true } +datafusion = { workspace = true } +datafusion-proto = { workspace = true } +futures = { workspace = true } +iceberg = { workspace = true } +iceberg-datafusion = { workspace = true } +# Storage for rebuilt tables: S3 and local files by default. +iceberg-storage-opendal = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread"] } + +[dev-dependencies] +ballista = { workspace = true, features = ["standalone"] } +iceberg-catalog-rest = { workspace = true } +tempfile = { workspace = true } +testcontainers = { version = "0.28", features = ["http_wait_plain"] } +tokio = { workspace = true, features = ["macros"] } + +[features] +# Tests that need docker. +testcontainers = [] diff --git a/integrations/iceberg/examples/standalone-iceberg-scan.rs b/integrations/iceberg/examples/standalone-iceberg-scan.rs new file mode 100644 index 0000000000..f2f49868a8 --- /dev/null +++ b/integrations/iceberg/examples/standalone-iceberg-scan.rs @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +//! Scans an Iceberg table through a standalone Ballista cluster: live, at its +//! latest version, and static, at an older one. +//! +//! The table is created in a temporary local warehouse, so the example needs +//! nothing running: +//! +//! ```text +//! cargo run -p iceberg-ballista --example standalone-iceberg-scan +//! ``` + +use std::collections::HashMap; +use std::sync::Arc; + +use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use datafusion::common::Result; +use datafusion::execution::SessionStateBuilder; +use datafusion::prelude::{SessionConfig, SessionContext}; +use iceberg::io::LocalFsStorageFactory; +use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; +use iceberg_ballista::register_iceberg_codecs; +use iceberg_datafusion::{ + IcebergCatalogTableProvider, IcebergStaticTableProvider, to_datafusion_error, +}; + +#[tokio::main] +async fn main() -> Result<()> { + let warehouse = tempfile::tempdir()?; + let catalog: Arc = Arc::new( + MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + warehouse.path().display().to_string(), + )]), + ) + .await + .map_err(to_datafusion_error)?, + ); + + let namespace = NamespaceIdent::new("ns".to_string()); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .map_err(to_datafusion_error)?; + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "city", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .map_err(to_datafusion_error)?; + let creation = TableCreation::builder() + .name("trips".to_string()) + .schema(schema) + .build(); + catalog + .create_table(&namespace, creation) + .await + .map_err(to_datafusion_error)?; + + // A catalog-backed table reads the table's latest version in every query, + // so it sees commits made after it was registered: a live read. Ballista + // can't write to Iceberg yet, so plain DataFusion commits through it. + let trips = Arc::new( + IcebergCatalogTableProvider::try_new(catalog, namespace, "trips").await?, + ); + let writer = SessionContext::new(); + writer.register_table("trips", trips.clone())?; + let insert = async |values: &str| -> Result<()> { + let sql = format!("INSERT INTO trips VALUES {values}"); + writer.sql(&sql).await?.collect().await?; + Ok(()) + }; + insert("(1, 'Paris'), (2, 'Oslo')").await?; + + // Start standalone Ballista with the Iceberg codecs. + let state = SessionStateBuilder::new() + .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_default_features() + .build(); + let ctx = SessionContext::standalone_with_state(state).await?; + ctx.register_table("trips", trips.clone())?; + + // A static table always reads the version it was made from: time travel. + let table = trips + .catalog() + .load_table(trips.table_ident()) + .await + .map_err(to_datafusion_error)?; + let v1 = IcebergStaticTableProvider::try_new_from_table(table).await?; + ctx.register_table("trips_v1", Arc::new(v1))?; + + ctx.sql("SELECT * FROM trips ORDER BY id") + .await? + .show() + .await?; + insert("(3, 'Lima')").await?; + // Now includes Lima. + ctx.sql("SELECT * FROM trips ORDER BY id") + .await? + .show() + .await?; + // Still only Paris and Oslo. + ctx.sql("SELECT * FROM trips_v1 ORDER BY id") + .await? + .show() + .await?; + Ok(()) +} diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs new file mode 100644 index 0000000000..a13faf76bf --- /dev/null +++ b/integrations/iceberg/src/lib.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Apache Iceberg table scans for Ballista. +//! +//! Ballista sends plans between nodes as bytes, so these codecs send each Iceberg +//! table and scan as one version of the table, which any node can rebuild +//! without a catalog. A catalog-backed table is sent as the version its catalog +//! serves when the query is sent, a static one as the version it holds. Only +//! the client talks to a catalog. +//! +//! Storage comes from [`iceberg_storage_opendal`]: S3 and local files by +//! default, other stores with its `opendal-*` features. +//! +//! ```no_run +//! # use std::sync::Arc; +//! # use ballista::prelude::{SessionConfigExt, SessionContextExt}; +//! # use datafusion::execution::SessionStateBuilder; +//! # use datafusion::prelude::{SessionConfig, SessionContext}; +//! # use iceberg_ballista::register_iceberg_codecs; +//! # use iceberg_datafusion::IcebergCatalogProvider; +//! # async fn example(catalog: Arc) -> datafusion::error::Result<()> { +//! let state = SessionStateBuilder::new() +//! .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) +//! .with_default_features() +//! .build(); +//! let ctx = SessionContext::standalone_with_state(state).await?; +//! +//! ctx.register_catalog("rest", Arc::new(IcebergCatalogProvider::try_new(catalog).await?)); +//! ctx.sql("SELECT * FROM rest.ns.t").await?.show().await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! # Limitations +//! +//! - Reads only. +//! - Each scan runs as a single task. +//! - Catalog-backed tables need a multi-threaded tokio runtime on the client. +//! - A catalog-backed table read twice in one query can see two versions. +//! - No metadata tables such as `t$snapshots`. +//! - Storage credentials travel in plain text. + +mod logical_codec; +mod physical_codec; +mod wire; + +use std::sync::Arc; + +use ballista_core::extension::SessionConfigExt; +use datafusion::prelude::SessionConfig; +// Linked for the storage it registers; nothing here calls it. +use iceberg_storage_opendal as _; + +pub use crate::logical_codec::IcebergLogicalCodec; +pub use crate::physical_codec::IcebergPhysicalCodec; + +/// Wraps `config`'s Ballista codecs in the Iceberg ones. +/// +/// In standalone mode the scheduler and executor share the client's config; a +/// separately deployed scheduler and executors need this on their configs too. +pub fn register_iceberg_codecs(config: SessionConfig) -> SessionConfig { + let logical = IcebergLogicalCodec::new(config.ballista_logical_extension_codec()); + let physical = IcebergPhysicalCodec::new(config.ballista_physical_extension_codec()); + config + .with_ballista_logical_extension_codec(Arc::new(logical)) + .with_ballista_physical_extension_codec(Arc::new(physical)) +} diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs new file mode 100644 index 0000000000..39b9ef4fc3 --- /dev/null +++ b/integrations/iceberg/src/logical_codec.rs @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Sends Iceberg tables from the client to the scheduler. + +use std::sync::Arc; + +use async_trait::async_trait; +use ballista_core::serde::BallistaLogicalExtensionCodec; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::{Session, TableProvider}; +use datafusion::common::{Result, TableReference, exec_err, not_impl_err}; +use datafusion::datasource::TableType; +use datafusion::datasource::file_format::FileFormatFactory; +use datafusion::datasource::source::DataSourceExec; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::dml::InsertOp; +use datafusion::logical_expr::{ + AggregateUDF, Expr, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, + TableProviderFilterPushDown, WindowUDF, +}; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_proto::logical_plan::LogicalExtensionCodec; +use iceberg::table::Table; +use iceberg_datafusion::read::IcebergDataSource; +use iceberg_datafusion::{ + IcebergCatalogTableProvider, IcebergStaticTableProvider, to_datafusion_error, +}; +use serde::{Deserialize, Serialize}; +use tokio::runtime::{Handle, RuntimeFlavor}; + +use crate::wire::{self, Frame, TAG_DELEGATED, TableWire}; + +#[derive(Debug, Serialize, Deserialize)] +struct ProviderWire { + table: TableWire, + snapshot_id: Option, +} + +/// The scheduler's side of an Iceberg table: one version of it, read with the +/// schema the client planned against. +#[derive(Debug)] +struct PinnedTable { + table: Table, + snapshot_id: Option, + schema: SchemaRef, +} + +#[async_trait] +impl TableProvider for PinnedTable { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> Result> { + let source = IcebergDataSource::builder(self.table.clone(), self.schema.clone()) + .with_snapshot_id(self.snapshot_id) + .with_projection(projection) + .with_filters(filters) + .with_fetch(limit) + .build()?; + Ok(DataSourceExec::from_data_source(source)) + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> Result> { + Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) + } + + async fn insert_into( + &self, + _state: &dyn Session, + _input: Arc, + _insert_op: InsertOp, + ) -> Result> { + not_impl_err!("INSERT INTO an Iceberg table isn't supported on Ballista yet") + } +} + +/// A [`LogicalExtensionCodec`] for [`IcebergStaticTableProvider`] and +/// [`IcebergCatalogTableProvider`] that delegates everything else to an inner +/// codec. +#[derive(Debug)] +pub struct IcebergLogicalCodec { + inner: Arc, +} + +impl IcebergLogicalCodec { + /// Creates a codec that delegates everything but Iceberg tables to `inner`. + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +impl Default for IcebergLogicalCodec { + fn default() -> Self { + Self::new(Arc::new(BallistaLogicalExtensionCodec::default())) + } +} + +impl LogicalExtensionCodec for IcebergLogicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[LogicalPlan], + ctx: &TaskContext, + ) -> Result { + self.inner.try_decode(buf, inputs, ctx) + } + + fn try_encode(&self, node: &Extension, buf: &mut Vec) -> Result<()> { + self.inner.try_encode(node, buf) + } + + fn try_decode_table_provider( + &self, + buf: &[u8], + table_ref: &TableReference, + schema: SchemaRef, + ctx: &TaskContext, + ) -> Result> { + let ProviderWire { table, snapshot_id } = match wire::decode(buf)? { + Frame::Iceberg(provider) => provider, + Frame::Delegated(rest) => { + return self + .inner + .try_decode_table_provider(rest, table_ref, schema, ctx); + } + }; + Ok(Arc::new(PinnedTable { + table: table.load()?, + snapshot_id, + schema, + })) + } + + fn try_encode_table_provider( + &self, + table_ref: &TableReference, + node: Arc, + buf: &mut Vec, + ) -> Result<()> { + let payload = if let Some(provider) = + node.downcast_ref::() + { + ProviderWire { + table: TableWire::with_metadata(provider.table())?, + snapshot_id: provider.snapshot_id(), + } + } else if let Some(provider) = node.downcast_ref::() + { + // Loading blocks this thread, and a REST catalog's pooled + // connections may need this thread's runtime to answer. + if let Ok(RuntimeFlavor::CurrentThread) = + Handle::try_current().map(|handle| handle.runtime_flavor()) + { + return exec_err!( + "Iceberg table {table_ref} is catalog-backed, so sending the query \ + loads it, which can hang on a single-threaded tokio runtime; run \ + the client on a multi-threaded runtime, or register the table as \ + an IcebergStaticTableProvider" + ); + } + let catalog = provider.catalog().clone(); + let ident = provider.table_ident().clone(); + let table = wire::block_on(async move { catalog.load_table(&ident).await })? + .map_err(to_datafusion_error)?; + ProviderWire { + table: TableWire::with_metadata(&table)?, + snapshot_id: None, + } + } else { + buf.push(TAG_DELEGATED); + return self.inner.try_encode_table_provider(table_ref, node, buf); + }; + wire::encode(buf, &payload) + } + + fn try_decode_file_format( + &self, + buf: &[u8], + ctx: &TaskContext, + ) -> Result> { + self.inner.try_decode_file_format(buf, ctx) + } + + fn try_encode_file_format( + &self, + buf: &mut Vec, + node: Arc, + ) -> Result<()> { + self.inner.try_encode_file_format(buf, node) + } + + fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udf(name, buf) + } + + fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udf(node, buf) + } + + fn try_decode_higher_order_function( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.inner.try_decode_higher_order_function(name, buf) + } + + fn try_encode_higher_order_function( + &self, + node: &HigherOrderUDF, + buf: &mut Vec, + ) -> Result<()> { + self.inner.try_encode_higher_order_function(node, buf) + } + + fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udaf(name, buf) + } + + fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udaf(node, buf) + } + + fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udwf(name, buf) + } + + fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udwf(node, buf) + } +} diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs new file mode 100644 index 0000000000..3419b98d09 --- /dev/null +++ b/integrations/iceberg/src/physical_codec.rs @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Sends Iceberg scans from the scheduler to executors. + +use std::sync::Arc; + +use ballista_core::serde::BallistaPhysicalExtensionCodec; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::Result; +use datafusion::datasource::source::DataSourceExec; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; +use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_proto::physical_plan::{ + PhysicalExtensionCodec, PhysicalProtoConverterExtension, +}; +use iceberg::expr::Predicate; +use iceberg_datafusion::read::IcebergDataSource; +use serde::{Deserialize, Serialize}; + +use crate::wire::{self, Frame, TAG_DELEGATED, TableWire}; + +#[derive(Debug, Serialize, Deserialize)] +struct ScanWire { + table: TableWire, + snapshot_id: Option, + /// The scan's output schema, already projected. + schema: SchemaRef, + predicate: Option, + fetch: Option, +} + +/// A [`PhysicalExtensionCodec`] for scans of an [`IcebergDataSource`] that +/// delegates everything else to an inner codec. +#[derive(Debug)] +pub struct IcebergPhysicalCodec { + inner: Arc, +} + +impl IcebergPhysicalCodec { + /// Creates a codec that delegates everything but Iceberg scans to `inner`. + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +impl Default for IcebergPhysicalCodec { + fn default() -> Self { + Self::new(Arc::new(BallistaPhysicalExtensionCodec::default())) + } +} + +impl PhysicalExtensionCodec for IcebergPhysicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result> { + let scan: ScanWire = match wire::decode(buf)? { + Frame::Iceberg(scan) => scan, + Frame::Delegated(rest) => { + return self.inner.try_decode(rest, inputs, ctx, proto_converter); + } + }; + let source = IcebergDataSource::builder(scan.table.load()?, scan.schema) + .with_snapshot_id(scan.snapshot_id) + .with_predicate(scan.predicate) + .with_fetch(scan.fetch) + .build()?; + Ok(DataSourceExec::from_data_source(source)) + } + + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + let source = node + .downcast_ref::() + .and_then(|exec| exec.data_source().downcast_ref::()); + match source { + Some(source) => wire::encode( + buf, + &ScanWire { + table: TableWire::new(source.table())?, + snapshot_id: source.snapshot_id(), + schema: source.schema(), + predicate: source.predicate().cloned(), + fetch: source.fetch(), + }, + ), + None => { + buf.push(TAG_DELEGATED); + self.inner.try_encode(node, buf, proto_converter) + } + } + } + + fn try_decode_expr( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &PhysicalExprDecodeCtx<'_>, + ) -> Result> { + self.inner.try_decode_expr(buf, inputs, ctx) + } + + fn try_encode_expr( + &self, + node: &Arc, + buf: &mut Vec, + ctx: &PhysicalExprEncodeCtx<'_>, + ) -> Result<()> { + self.inner.try_encode_expr(node, buf, ctx) + } + + fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udf(name, buf) + } + + fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udf(node, buf) + } + + fn try_decode_higher_order_function( + &self, + name: &str, + buf: &[u8], + ) -> Result> { + self.inner.try_decode_higher_order_function(name, buf) + } + + fn try_encode_higher_order_function( + &self, + node: &HigherOrderUDF, + buf: &mut Vec, + ) -> Result<()> { + self.inner.try_encode_higher_order_function(node, buf) + } + + fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udaf(name, buf) + } + + fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udaf(node, buf) + } + + fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udwf(name, buf) + } + + fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { + self.inner.try_encode_udwf(node, buf) + } +} diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs new file mode 100644 index 0000000000..71094afb27 --- /dev/null +++ b/integrations/iceberg/src/wire.rs @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! What the codecs send in place of a live [`Table`], and how they frame it. + +use std::collections::VecDeque; +use std::fmt; +use std::sync::{Arc, LazyLock, Mutex}; + +use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err}; +use iceberg::io::FileIO; +use iceberg::spec::{TableMetadata, TableMetadataRef}; +use iceberg::table::Table; +use iceberg::{Runtime, TableIdent}; +use iceberg_datafusion::to_datafusion_error; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use tokio::runtime::{Handle, RuntimeFlavor}; + +/// Runs the codecs' I/O and owns the tables they rebuild: Iceberg plans scans on +/// a table's runtime, which must outlive the table, and a codec's caller may not. +static RUNTIME: LazyLock = LazyLock::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("iceberg-ballista") + .enable_all() + .build() + .expect("failed to build the iceberg-ballista runtime") +}); + +/// Runs `fut` on [`RUNTIME`] and waits for it, from the synchronous codec API. +pub(crate) fn block_on( + fut: impl Future + Send + 'static, +) -> Result { + let wait = || futures::executor::block_on(RUNTIME.spawn(fut)); + let joined = match Handle::try_current().map(|handle| handle.runtime_flavor()) { + Ok(RuntimeFlavor::MultiThread) => tokio::task::block_in_place(wait), + _ => wait(), + }; + joined.map_err(|e| DataFusionError::External(Box::new(e))) +} + +/// Bounds memory rather than tuning hits: a query touches a few table versions, +/// but each can hold megabytes of metadata. +const TABLE_CACHE_CAPACITY: usize = 16; + +/// Tables rebuilt by [`TableWire::load`], least recently used first, so each +/// process rebuilds a version once. Metadata files never change, so entries +/// can't go stale. +static TABLES: LazyLock>> = + LazyLock::new(|| Mutex::new(VecDeque::new())); + +/// A [`Table`] as its metadata file and `FileIO`, from which any node can +/// rebuild exactly that version without a catalog. +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct TableWire { + identifier: TableIdent, + metadata_location: String, + file_io: serde_json::Value, + readonly: bool, + /// The content of `metadata_location`, sent from the client so the + /// scheduler needn't read it; scans sent to executors leave it out. + #[serde(default, skip_serializing_if = "Option::is_none")] + metadata: Option, +} + +impl fmt::Debug for TableWire { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TableWire") + .field("identifier", &self.identifier) + .field("metadata_location", &self.metadata_location) + .field("file_io", &"") + .field("readonly", &self.readonly) + .finish_non_exhaustive() + } +} + +impl TableWire { + pub(crate) fn new(table: &Table) -> Result { + let file_io = table + .file_io() + .serialize_all() + .map_err(to_datafusion_error)?; + Ok(Self { + identifier: table.identifier().clone(), + metadata_location: table + .metadata_location_result() + .map_err(to_datafusion_error)? + .to_string(), + file_io: serde_json::from_slice(&file_io).map_err(json_err)?, + readonly: table.readonly(), + metadata: None, + }) + } + + /// Like [`TableWire::new`], also carrying the table's metadata. + pub(crate) fn with_metadata(table: &Table) -> Result { + Ok(Self { + metadata: Some(table.metadata_ref()), + ..Self::new(table)? + }) + } + + /// Rebuilds the table, or returns it from [`TABLES`]. + pub(crate) fn load(mut self) -> Result { + // Keyed without the metadata, which its location determines. + let metadata = self.metadata.take(); + if let Some(table) = self.touch(&mut TABLES.lock().unwrap()) { + return Ok(table); + } + let table = + block_on(self.clone().read(metadata))?.map_err(to_datafusion_error)?; + let mut tables = TABLES.lock().unwrap(); + // Another thread may have rebuilt it meanwhile. + if let Some(table) = self.touch(&mut tables) { + return Ok(table); + } + if tables.len() == TABLE_CACHE_CAPACITY { + tables.pop_front(); + } + tables.push_back((self.clone(), table.clone())); + Ok(table) + } + + /// Moves this table's entry, if any, to the back of `tables` and returns it. + fn touch(&self, tables: &mut VecDeque<(TableWire, Table)>) -> Option { + let entry = tables.remove(tables.iter().position(|(wire, _)| wire == self)?)?; + let table = entry.1.clone(); + tables.push_back(entry); + Some(table) + } + + async fn read(self, metadata: Option) -> iceberg::Result
{ + let file_io = FileIO::deserialize_all(&serde_json::to_vec(&self.file_io)?)?; + let metadata = match metadata { + Some(metadata) => metadata, + None => Arc::new( + TableMetadata::read_from(&file_io, &self.metadata_location).await?, + ), + }; + Table::builder() + .metadata(metadata) + .metadata_location(self.metadata_location) + .identifier(self.identifier) + .file_io(file_io) + .readonly(self.readonly) + .runtime(Runtime::new(&RUNTIME)) + .build() + } +} + +/// Leads a blob that belongs to the inner codec. +pub(crate) const TAG_DELEGATED: u8 = 0; +/// Leads a blob that belongs to this crate, JSON-encoded. Also the version of +/// that JSON: a change that older decoders can't read takes the next tag. +pub(crate) const TAG_ICEBERG: u8 = 1; + +/// Appends `payload` to `buf` as an Iceberg blob. +pub(crate) fn encode( + buf: &mut Vec, + payload: &T, +) -> Result<(), DataFusionError> { + buf.push(TAG_ICEBERG); + serde_json::to_writer(buf, payload).map_err(json_err) +} + +/// A blob's payload: this crate's, decoded, or the inner codec's bytes. +pub(crate) enum Frame<'a, T> { + Iceberg(T), + Delegated(&'a [u8]), +} + +/// Splits `buf` by its leading tag, decoding an Iceberg payload as `T`. +pub(crate) fn decode( + buf: &[u8], +) -> Result, DataFusionError> { + match buf.split_first() { + Some((&TAG_ICEBERG, rest)) => serde_json::from_slice(rest) + .map(Frame::Iceberg) + .map_err(|e| exec_datafusion_err!("invalid iceberg-ballista payload: {e}")), + Some((&TAG_DELEGATED, rest)) => Ok(Frame::Delegated(rest)), + Some((tag, _)) => exec_err!( + "unknown iceberg-ballista codec tag {tag}: was the plan encoded by \ + another version of iceberg-ballista?" + ), + None => exec_err!("empty iceberg-ballista codec buffer"), + } +} + +fn json_err(e: serde_json::Error) -> DataFusionError { + DataFusionError::External(Box::new(e)) +} diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs new file mode 100644 index 0000000000..25f27b38c9 --- /dev/null +++ b/integrations/iceberg/tests/codec.rs @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Encodes and decodes plans with the Iceberg codecs, as Ballista does between +//! client, scheduler and executors. + +mod common; + +#[cfg(test)] +mod iceberg_nodes { + use std::sync::Arc; + + use datafusion::assert_batches_eq; + use datafusion::catalog::TableProvider; + use datafusion::common::{Result, TableReference}; + use datafusion::datasource::source::DataSourceExec; + use datafusion::execution::TaskContext; + use datafusion::logical_expr::dml::InsertOp; + use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::{collect, displayable}; + use datafusion::prelude::{SessionContext, col, lit}; + use datafusion_proto::logical_plan::LogicalExtensionCodec; + use datafusion_proto::physical_plan::{ + DefaultPhysicalProtoConverter, PhysicalExtensionCodec, + }; + use iceberg_ballista::{IcebergLogicalCodec, IcebergPhysicalCodec}; + use iceberg_datafusion::IcebergStaticTableProvider; + use iceberg_datafusion::read::IcebergDataSource; + + use crate::common::{catalog_table, insert, memory_catalog}; + + #[tokio::test] + async fn should_keep_scan_snapshot_projection_filter_and_fetch() -> Result<()> { + let dir = tempfile::tempdir()?; + let catalog = memory_catalog(dir.path()).await; + let first = insert(&catalog, "(1, 'a'), (2, 'b'), (3, 'c')").await; + let first = first.metadata().current_snapshot_id().unwrap(); + let table = insert(&catalog, "(4, 'd')").await; + let provider = + IcebergStaticTableProvider::try_new_from_table_snapshot(table, first).await?; + let ctx = SessionContext::new(); + let filters = [col("id").gt(lit(1))]; + let scan = provider + .scan(&ctx.state(), Some(&vec![1]), &filters, Some(1)) + .await?; + + let codec = IcebergPhysicalCodec::default(); + let mut buf = Vec::new(); + codec.try_encode(scan, &mut buf, &DefaultPhysicalProtoConverter {})?; + let decoded = codec.try_decode( + &buf, + &[], + &ctx.task_ctx(), + &DefaultPhysicalProtoConverter {}, + )?; + + let source = decoded + .downcast_ref::() + .and_then(|exec| exec.data_source().downcast_ref::()) + .unwrap(); + assert_eq!(source.snapshot_id(), Some(first)); + assert_eq!( + displayable(decoded.as_ref()).one_line().to_string(), + "DataSourceExec: format=iceberg, projection=[name], predicate=id > 1, fetch=1\n" + ); + let result = collect(decoded, ctx.task_ctx()).await?; + let expected = ["+------+", "| name |", "+------+", "| b |", "+------+"]; + assert_batches_eq!(expected, &result); + + Ok(()) + } + + // Multi-threaded: encoding a catalog-backed table loads it. + #[tokio::test(flavor = "multi_thread")] + async fn should_pin_catalog_table_when_encoded() -> Result<()> { + let dir = tempfile::tempdir()?; + let catalog = memory_catalog(dir.path()).await; + let provider = catalog_table(&catalog).await; + let schema = provider.schema(); + let codec = IcebergLogicalCodec::default(); + let table = TableReference::bare("t"); + + let pinned = insert(&catalog, "(1, 'a')").await; + let mut buf = Vec::new(); + codec.try_encode_table_provider(&table, Arc::new(provider), &mut buf)?; + insert(&catalog, "(2, 'b')").await; + // Decoding uses the metadata the client sent, not the file. + std::fs::remove_file(pinned.metadata_location().unwrap())?; + let decoded = codec.try_decode_table_provider( + &buf, + &table, + schema.clone(), + &TaskContext::default(), + )?; + + let ctx = SessionContext::new(); + let input = Arc::new(EmptyExec::new(schema)); + let err = decoded + .insert_into(&ctx.state(), input, InsertOp::Append) + .await + .unwrap_err(); + assert_eq!( + err.to_string(), + "This feature is not implemented: INSERT INTO an Iceberg table isn't \ + supported on Ballista yet" + ); + + ctx.register_table("t", decoded)?; + let result = ctx.sql("SELECT * FROM t").await?.collect().await?; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "+----+------+", + ]; + assert_batches_eq!(expected, &result); + + Ok(()) + } + + #[tokio::test] + async fn should_refuse_catalog_table_on_single_threaded_runtime() -> Result<()> { + let dir = tempfile::tempdir()?; + let provider = catalog_table(&memory_catalog(dir.path()).await).await; + + let err = IcebergLogicalCodec::default() + .try_encode_table_provider( + &TableReference::bare("t"), + Arc::new(provider), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!( + err.to_string(), + "Execution error: Iceberg table t is catalog-backed, so sending the query \ + loads it, which can hang on a single-threaded tokio runtime; run the \ + client on a multi-threaded runtime, or register the table as an \ + IcebergStaticTableProvider" + ); + + Ok(()) + } +} + +#[cfg(test)] +mod other_nodes { + use std::sync::Arc; + + use datafusion::arrow::datatypes::Schema; + use datafusion::catalog::TableProvider; + use datafusion::common::{Result, TableReference}; + use datafusion::datasource::empty::EmptyTable; + use datafusion::execution::TaskContext; + use datafusion::logical_expr::{Extension, LogicalPlan}; + use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::{ExecutionPlan, displayable}; + use datafusion_proto::logical_plan::LogicalExtensionCodec; + use datafusion_proto::physical_plan::{ + DefaultPhysicalProtoConverter, PhysicalExtensionCodec, + PhysicalProtoConverterExtension, + }; + use iceberg_ballista::{IcebergLogicalCodec, IcebergPhysicalCodec}; + + /// An inner codec that encodes anything as `inner`, and decodes only that. + #[derive(Debug)] + struct Inner; + + impl PhysicalExtensionCodec for Inner { + fn try_decode( + &self, + buf: &[u8], + _inputs: &[Arc], + _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result> { + assert_eq!(buf, b"inner"); + Ok(Arc::new(EmptyExec::new(Arc::new(Schema::empty())))) + } + + fn try_encode( + &self, + _node: Arc, + buf: &mut Vec, + _proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + buf.extend_from_slice(b"inner"); + Ok(()) + } + } + + impl LogicalExtensionCodec for Inner { + fn try_decode( + &self, + _buf: &[u8], + _inputs: &[LogicalPlan], + _ctx: &TaskContext, + ) -> Result { + unreachable!() + } + + fn try_encode(&self, _node: &Extension, _buf: &mut Vec) -> Result<()> { + unreachable!() + } + + fn try_decode_table_provider( + &self, + buf: &[u8], + _table_ref: &TableReference, + schema: Arc, + _ctx: &TaskContext, + ) -> Result> { + assert_eq!(buf, b"inner"); + Ok(Arc::new(EmptyTable::new(schema))) + } + + fn try_encode_table_provider( + &self, + _table_ref: &TableReference, + _node: Arc, + buf: &mut Vec, + ) -> Result<()> { + buf.extend_from_slice(b"inner"); + Ok(()) + } + } + + #[test] + fn should_delegate_non_iceberg_nodes_to_inner_codec() -> Result<()> { + let ctx = TaskContext::default(); + let schema = Arc::new(Schema::empty()); + + let physical = IcebergPhysicalCodec::new(Arc::new(Inner)); + let mut buf = Vec::new(); + let plan = Arc::new(EmptyExec::new(schema.clone())); + physical.try_encode(plan, &mut buf, &DefaultPhysicalProtoConverter {})?; + assert_eq!(buf, b"\0inner"); + let decoded = + physical.try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {})?; + assert_eq!( + displayable(decoded.as_ref()).one_line().to_string(), + "EmptyExec\n" + ); + + let logical = IcebergLogicalCodec::new(Arc::new(Inner)); + let table = TableReference::bare("t"); + let mut buf = Vec::new(); + let provider = Arc::new(EmptyTable::new(schema.clone())); + logical.try_encode_table_provider(&table, provider, &mut buf)?; + assert_eq!(buf, b"\0inner"); + let decoded = + logical.try_decode_table_provider(&buf, &table, schema.clone(), &ctx)?; + assert_eq!( + format!("{decoded:?}"), + format!("{:?}", EmptyTable::new(schema)) + ); + + Ok(()) + } + + #[test] + fn should_reject_malformed_buffers() { + let codec = IcebergPhysicalCodec::new(Arc::new(Inner)); + let decode = |buf: &[u8]| { + codec + .try_decode( + buf, + &[], + &TaskContext::default(), + &DefaultPhysicalProtoConverter {}, + ) + .unwrap_err() + .to_string() + }; + + assert_eq!( + decode(b""), + "Execution error: empty iceberg-ballista codec buffer" + ); + assert_eq!( + decode(b"\x07inner"), + "Execution error: unknown iceberg-ballista codec tag 7: was the plan \ + encoded by another version of iceberg-ballista?" + ); + assert_eq!( + decode(b"\x01inner"), + "Execution error: invalid iceberg-ballista payload: expected value at line 1 \ + column 1" + ); + } +} diff --git a/integrations/iceberg/tests/common/mod.rs b/integrations/iceberg/tests/common/mod.rs new file mode 100644 index 0000000000..7466336e49 --- /dev/null +++ b/integrations/iceberg/tests/common/mod.rs @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Helpers shared by the integration tests. + +#![allow(dead_code)] + +#[cfg(feature = "testcontainers")] +pub mod rustfs; + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use datafusion::assert_batches_eq; +use datafusion::error::Result; +use datafusion::execution::SessionStateBuilder; +use datafusion::prelude::{SessionConfig, SessionContext}; +use iceberg::io::LocalFsStorageFactory; +use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; +use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; +use iceberg::table::Table; +use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; +use iceberg_ballista::register_iceberg_codecs; +use iceberg_datafusion::{ + IcebergCatalogProvider, IcebergCatalogTableProvider, IcebergStaticTableProvider, +}; + +/// A memory catalog with its warehouse in `dir`, holding an empty [`create_table`]. +pub async fn memory_catalog(dir: &Path) -> Arc { + let catalog = MemoryCatalogBuilder::default() + .with_storage_factory(Arc::new(LocalFsStorageFactory)) + .load( + "memory", + HashMap::from([( + MEMORY_CATALOG_WAREHOUSE.to_string(), + dir.display().to_string(), + )]), + ) + .await + .unwrap(); + create_table(&catalog).await; + Arc::new(catalog) +} + +/// Creates the empty table `ns.t (id bigint, name string)`. +pub async fn create_table(catalog: &dyn Catalog) { + let namespace = NamespaceIdent::new("ns".to_string()); + catalog + .create_namespace(&namespace, HashMap::new()) + .await + .unwrap(); + let schema = Schema::builder() + .with_fields(vec![ + NestedField::required(1, "id", Type::Primitive(PrimitiveType::Long)).into(), + NestedField::optional(2, "name", Type::Primitive(PrimitiveType::String)) + .into(), + ]) + .build() + .unwrap(); + let creation = TableCreation::builder() + .name("t".to_string()) + .schema(schema) + .build(); + catalog.create_table(&namespace, creation).await.unwrap(); +} + +/// `ns.t`, read through `catalog` on every query. +pub async fn catalog_table(catalog: &Arc) -> IcebergCatalogTableProvider { + let namespace = NamespaceIdent::new("ns".to_string()); + IcebergCatalogTableProvider::try_new(catalog.clone(), namespace, "t") + .await + .unwrap() +} + +/// Commits `values` to `ns.t` with plain DataFusion, and returns the table as +/// of that commit. +pub async fn insert(catalog: &Arc, values: &str) -> Table { + let ctx = SessionContext::new(); + ctx.register_table("t", Arc::new(catalog_table(catalog).await)) + .unwrap(); + ctx.sql(&format!("INSERT INTO t VALUES {values}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + catalog + .load_table(&TableIdent::from_strs(["ns", "t"]).unwrap()) + .await + .unwrap() +} + +/// Commits to an empty `ns.t` twice, and checks that standalone Ballista reads +/// `catalog` mounted at the latest commit and a static table at the first. +pub async fn assert_standalone_scans(catalog: Arc) -> Result<()> { + let first = insert(&catalog, "(1, 'a'), (2, NULL)").await; + + let state = SessionStateBuilder::new() + .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_default_features() + .build(); + let ctx = SessionContext::standalone_with_state(state).await?; + let mount = IcebergCatalogProvider::try_new(catalog.clone()).await?; + ctx.register_catalog("iceberg", Arc::new(mount)); + let v1 = IcebergStaticTableProvider::try_new_from_table(first).await?; + ctx.register_table("t_v1", Arc::new(v1))?; + + insert(&catalog, "(3, 'c')").await; + + let result = ctx + .sql("SELECT * FROM iceberg.ns.t ORDER BY id") + .await? + .collect() + .await?; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 2 | |", + "| 3 | c |", + "+----+------+", + ]; + assert_batches_eq!(expected, &result); + + let result = ctx + .sql("SELECT * FROM t_v1 ORDER BY id") + .await? + .collect() + .await?; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 2 | |", + "+----+------+", + ]; + assert_batches_eq!(expected, &result); + + Ok(()) +} diff --git a/integrations/iceberg/tests/common/rustfs.rs b/integrations/iceberg/tests/common/rustfs.rs new file mode 100644 index 0000000000..84134f0c9b --- /dev/null +++ b/integrations/iceberg/tests/common/rustfs.rs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! An Iceberg REST catalog storing its tables in RustFS. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use iceberg::{Catalog, CatalogBuilder}; +use iceberg_catalog_rest::RestCatalogBuilder; +use iceberg_storage_opendal::OpenDalStorageFactory; +use testcontainers::core::wait::HttpWaitStrategy; +use testcontainers::core::{CmdWaitFor, ExecCommand, IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, Image, ImageExt}; + +const S3_PORT: u16 = 9000; +const REST_PORT: u16 = 8181; +const BUCKET: &str = "warehouse"; +const ACCESS_KEY: &str = "admin"; +const SECRET_KEY: &str = "password"; +const REGION: &str = "us-east-1"; + +/// RustFS and a REST catalog in front of it, removed on drop. +pub struct RestCatalog { + pub catalog: Arc, + _s3: ContainerAsync, + _rest: ContainerAsync, +} + +impl RestCatalog { + pub async fn start() -> Self { + // Docker names are global, and tests run concurrently. + let suffix = unique_suffix(); + let network = format!("iceberg-ballista-{suffix}"); + // Also the hostname the catalog reaches RustFS on. + let s3_host = format!("iceberg-ballista-rustfs-{suffix}"); + + let s3 = GenericImage::new("rustfs/rustfs", "1.0.0") + .with_exposed_port(S3_PORT.tcp()) + // RustFS logs only warnings by default, so wait on its health check. + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new("/health") + .with_port(S3_PORT.tcp()) + .with_expected_status_code(200u16), + )) + .with_env_var("RUSTFS_ACCESS_KEY", ACCESS_KEY) + .with_env_var("RUSTFS_SECRET_KEY", SECRET_KEY) + .with_network(&network) + .with_container_name(&s3_host) + .with_startup_timeout(Duration::from_secs(120)) + .start() + .await + .unwrap(); + // A top-level directory of RustFS's data volume is a bucket. + s3.exec( + ExecCommand::new(["mkdir", &format!("/data/{BUCKET}")]) + .with_cmd_ready_condition(CmdWaitFor::exit_code(0)), + ) + .await + .unwrap(); + + let rest = GenericImage::new("apache/iceberg-rest-fixture", "1.10.1") + .with_exposed_port(REST_PORT.tcp()) + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new("/v1/config") + .with_port(REST_PORT.tcp()) + .with_expected_status_code(200u16), + )) + .with_network(&network) + .with_env_var("AWS_ACCESS_KEY_ID", ACCESS_KEY) + .with_env_var("AWS_SECRET_ACCESS_KEY", SECRET_KEY) + .with_env_var("AWS_REGION", REGION) + .with_env_var( + "CATALOG_CATALOG__IMPL", + "org.apache.iceberg.jdbc.JdbcCatalog", + ) + .with_env_var( + "CATALOG_URI", + "jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory", + ) + .with_env_var("CATALOG_WAREHOUSE", format!("s3://{BUCKET}/")) + .with_env_var("CATALOG_IO__IMPL", "org.apache.iceberg.aws.s3.S3FileIO") + .with_env_var("CATALOG_S3_ENDPOINT", format!("http://{s3_host}:{S3_PORT}")) + // Virtual-hosted style would need a `.` DNS alias, + // which testcontainers can't add. + .with_env_var("CATALOG_S3_PATH__STYLE__ACCESS", "true") + .with_startup_timeout(Duration::from_secs(120)) + .start() + .await + .unwrap(); + + let props = HashMap::from([ + ("uri".to_string(), endpoint(&rest, REST_PORT).await), + ("s3.endpoint".to_string(), endpoint(&s3, S3_PORT).await), + ("s3.access-key-id".to_string(), ACCESS_KEY.to_string()), + ("s3.secret-access-key".to_string(), SECRET_KEY.to_string()), + ("s3.region".to_string(), REGION.to_string()), + ("s3.path-style-access".to_string(), "true".to_string()), + ]); + let catalog = RestCatalogBuilder::default() + .with_storage_factory(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .load("rest", props) + .await + .unwrap(); + Self { + catalog: Arc::new(catalog), + _s3: s3, + _rest: rest, + } + } +} + +/// `http://:` of a container port, from this process. +async fn endpoint(container: &ContainerAsync, port: u16) -> String { + let host = container.get_host().await.unwrap(); + let port = container.get_host_port_ipv4(port.tcp()).await.unwrap(); + format!("http://{host}:{port}") +} + +fn unique_suffix() -> String { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("{}-{}", std::process::id(), nanos % 1_000_000_000) +} diff --git a/integrations/iceberg/tests/object_storage.rs b/integrations/iceberg/tests/object_storage.rs new file mode 100644 index 0000000000..9a5326e5ba --- /dev/null +++ b/integrations/iceberg/tests/object_storage.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Scans Iceberg tables in S3, through a REST catalog, with standalone +//! Ballista. +//! +//! Tests depend on RustFS and Iceberg REST catalog testcontainers, which +//! require docker to run. + +mod common; + +#[cfg(test)] +#[cfg(feature = "testcontainers")] +mod standalone { + use crate::common::rustfs::RestCatalog; + use crate::common::{assert_standalone_scans, create_table}; + + #[tokio::test(flavor = "multi_thread")] + async fn should_scan_s3_tables_through_a_rest_catalog() + -> datafusion::error::Result<()> { + let rest = RestCatalog::start().await; + create_table(rest.catalog.as_ref()).await; + assert_standalone_scans(rest.catalog.clone()).await + } +} diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs new file mode 100644 index 0000000000..d2359f0cc3 --- /dev/null +++ b/integrations/iceberg/tests/scan.rs @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Scans Iceberg tables with standalone Ballista. + +mod common; + +#[cfg(test)] +mod standalone { + use crate::common::{assert_standalone_scans, memory_catalog}; + + // Multi-threaded: sending a query loads its catalog-backed tables. + #[tokio::test(flavor = "multi_thread")] + async fn should_scan_latest_and_older_versions() -> datafusion::error::Result<()> { + let dir = tempfile::tempdir()?; + assert_standalone_scans(memory_catalog(dir.path()).await).await + } +} From 3e64e7a550b1a27a603a57878da609f14c6fe723 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 22 Oct 2026 00:07:51 -0400 Subject: [PATCH 02/31] refactor(iceberg): rename the crate to ballista-iceberg --- Cargo.lock | 44 +++++++++---------- docs/source/user-guide/iceberg.md | 8 ++-- integrations/iceberg/Cargo.toml | 4 +- .../examples/standalone-iceberg-scan.rs | 4 +- integrations/iceberg/src/lib.rs | 4 +- integrations/iceberg/src/wire.rs | 12 ++--- integrations/iceberg/tests/codec.rs | 12 ++--- integrations/iceberg/tests/common/mod.rs | 2 +- integrations/iceberg/tests/common/rustfs.rs | 4 +- 9 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 601de4fb42..1347e6f3a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,6 +1358,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "ballista-iceberg" +version = "55.0.0" +dependencies = [ + "arrow-schema", + "async-trait", + "ballista", + "ballista-core", + "datafusion", + "datafusion-proto", + "futures", + "iceberg", + "iceberg-catalog-rest", + "iceberg-datafusion", + "iceberg-storage-opendal", + "serde", + "serde_json", + "tempfile", + "testcontainers", + "tokio", +] + [[package]] name = "ballista-result-service" version = "55.0.0" @@ -4426,28 +4448,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "iceberg-ballista" -version = "55.0.0" -dependencies = [ - "arrow-schema", - "async-trait", - "ballista", - "ballista-core", - "datafusion", - "datafusion-proto", - "futures", - "iceberg", - "iceberg-catalog-rest", - "iceberg-datafusion", - "iceberg-storage-opendal", - "serde", - "serde_json", - "tempfile", - "testcontainers", - "tokio", -] - [[package]] name = "iceberg-catalog-rest" version = "0.10.1" diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 4b985664a7..dec7c02d9c 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -19,7 +19,7 @@ # Apache Iceberg -The `iceberg-ballista` crate (`integrations/iceberg`) lets Ballista scan +The `ballista-iceberg` crate (`integrations/iceberg`) lets Ballista scan [Apache Iceberg](https://iceberg.apache.org/) tables. On top of [iceberg-datafusion](https://github.com/apache/datafusion-iceberg), it adds the codecs that send Iceberg tables and scans between the client, the scheduler and @@ -38,7 +38,7 @@ use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; -use iceberg_ballista::register_iceberg_codecs; +use ballista_iceberg::register_iceberg_codecs; use iceberg_datafusion::IcebergCatalogProvider; let state = SessionStateBuilder::new() @@ -61,7 +61,7 @@ The example registers both on a table in a temporary local warehouse, and shows a commit reaching the live table but not the static one: ```shell -cargo run -p iceberg-ballista --example standalone-iceberg-scan +cargo run -p ballista-iceberg --example standalone-iceberg-scan ``` ## How it works @@ -104,5 +104,5 @@ configuration. `FileIO` includes them, so the links between client, scheduler and executors must be trusted. - **Only standalone mode is tested.** A separately deployed scheduler and - executors must link `iceberg-ballista` and register the codecs too (see + executors must link `ballista-iceberg` and register the codecs too (see [Extending Components](extending-components.md)). diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index 8a097ff01a..b8114fa414 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -16,8 +16,8 @@ # under the License. [package] -name = "iceberg-ballista" -description = "Apache Iceberg table scans for Ballista" +name = "ballista-iceberg" +description = "Apache Iceberg support for Ballista" license = { workspace = true } version = { workspace = true } homepage = { workspace = true } diff --git a/integrations/iceberg/examples/standalone-iceberg-scan.rs b/integrations/iceberg/examples/standalone-iceberg-scan.rs index f2f49868a8..e466720ea0 100644 --- a/integrations/iceberg/examples/standalone-iceberg-scan.rs +++ b/integrations/iceberg/examples/standalone-iceberg-scan.rs @@ -21,13 +21,14 @@ //! nothing running: //! //! ```text -//! cargo run -p iceberg-ballista --example standalone-iceberg-scan +//! cargo run -p ballista-iceberg --example standalone-iceberg-scan //! ``` use std::collections::HashMap; use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use ballista_iceberg::register_iceberg_codecs; use datafusion::common::Result; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; @@ -35,7 +36,6 @@ use iceberg::io::LocalFsStorageFactory; use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation}; -use iceberg_ballista::register_iceberg_codecs; use iceberg_datafusion::{ IcebergCatalogTableProvider, IcebergStaticTableProvider, to_datafusion_error, }; diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index a13faf76bf..1ac6ab6f7b 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Apache Iceberg table scans for Ballista. +//! Apache Iceberg support for Ballista. //! //! Ballista sends plans between nodes as bytes, so these codecs send each Iceberg //! table and scan as one version of the table, which any node can rebuild @@ -31,7 +31,7 @@ //! # use ballista::prelude::{SessionConfigExt, SessionContextExt}; //! # use datafusion::execution::SessionStateBuilder; //! # use datafusion::prelude::{SessionConfig, SessionContext}; -//! # use iceberg_ballista::register_iceberg_codecs; +//! # use ballista_iceberg::register_iceberg_codecs; //! # use iceberg_datafusion::IcebergCatalogProvider; //! # async fn example(catalog: Arc) -> datafusion::error::Result<()> { //! let state = SessionStateBuilder::new() diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index 71094afb27..6cc9bcfc4a 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -35,10 +35,10 @@ use tokio::runtime::{Handle, RuntimeFlavor}; /// a table's runtime, which must outlive the table, and a codec's caller may not. static RUNTIME: LazyLock = LazyLock::new(|| { tokio::runtime::Builder::new_multi_thread() - .thread_name("iceberg-ballista") + .thread_name("ballista-iceberg") .enable_all() .build() - .expect("failed to build the iceberg-ballista runtime") + .expect("failed to build the ballista-iceberg runtime") }); /// Runs `fut` on [`RUNTIME`] and waits for it, from the synchronous codec API. @@ -190,13 +190,13 @@ pub(crate) fn decode( match buf.split_first() { Some((&TAG_ICEBERG, rest)) => serde_json::from_slice(rest) .map(Frame::Iceberg) - .map_err(|e| exec_datafusion_err!("invalid iceberg-ballista payload: {e}")), + .map_err(|e| exec_datafusion_err!("invalid ballista-iceberg payload: {e}")), Some((&TAG_DELEGATED, rest)) => Ok(Frame::Delegated(rest)), Some((tag, _)) => exec_err!( - "unknown iceberg-ballista codec tag {tag}: was the plan encoded by \ - another version of iceberg-ballista?" + "unknown ballista-iceberg codec tag {tag}: was the plan encoded by \ + another version of ballista-iceberg?" ), - None => exec_err!("empty iceberg-ballista codec buffer"), + None => exec_err!("empty ballista-iceberg codec buffer"), } } diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index 25f27b38c9..4f0db1f6bc 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -24,6 +24,7 @@ mod common; mod iceberg_nodes { use std::sync::Arc; + use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec}; use datafusion::assert_batches_eq; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference}; @@ -37,7 +38,6 @@ mod iceberg_nodes { use datafusion_proto::physical_plan::{ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, }; - use iceberg_ballista::{IcebergLogicalCodec, IcebergPhysicalCodec}; use iceberg_datafusion::IcebergStaticTableProvider; use iceberg_datafusion::read::IcebergDataSource; @@ -161,6 +161,7 @@ mod iceberg_nodes { mod other_nodes { use std::sync::Arc; + use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec}; use datafusion::arrow::datatypes::Schema; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference}; @@ -174,7 +175,6 @@ mod other_nodes { DefaultPhysicalProtoConverter, PhysicalExtensionCodec, PhysicalProtoConverterExtension, }; - use iceberg_ballista::{IcebergLogicalCodec, IcebergPhysicalCodec}; /// An inner codec that encodes anything as `inner`, and decodes only that. #[derive(Debug)] @@ -289,16 +289,16 @@ mod other_nodes { assert_eq!( decode(b""), - "Execution error: empty iceberg-ballista codec buffer" + "Execution error: empty ballista-iceberg codec buffer" ); assert_eq!( decode(b"\x07inner"), - "Execution error: unknown iceberg-ballista codec tag 7: was the plan \ - encoded by another version of iceberg-ballista?" + "Execution error: unknown ballista-iceberg codec tag 7: was the plan \ + encoded by another version of ballista-iceberg?" ); assert_eq!( decode(b"\x01inner"), - "Execution error: invalid iceberg-ballista payload: expected value at line 1 \ + "Execution error: invalid ballista-iceberg payload: expected value at line 1 \ column 1" ); } diff --git a/integrations/iceberg/tests/common/mod.rs b/integrations/iceberg/tests/common/mod.rs index 7466336e49..170920ea39 100644 --- a/integrations/iceberg/tests/common/mod.rs +++ b/integrations/iceberg/tests/common/mod.rs @@ -27,6 +27,7 @@ use std::path::Path; use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; +use ballista_iceberg::register_iceberg_codecs; use datafusion::assert_batches_eq; use datafusion::error::Result; use datafusion::execution::SessionStateBuilder; @@ -36,7 +37,6 @@ use iceberg::memory::{MEMORY_CATALOG_WAREHOUSE, MemoryCatalogBuilder}; use iceberg::spec::{NestedField, PrimitiveType, Schema, Type}; use iceberg::table::Table; use iceberg::{Catalog, CatalogBuilder, NamespaceIdent, TableCreation, TableIdent}; -use iceberg_ballista::register_iceberg_codecs; use iceberg_datafusion::{ IcebergCatalogProvider, IcebergCatalogTableProvider, IcebergStaticTableProvider, }; diff --git a/integrations/iceberg/tests/common/rustfs.rs b/integrations/iceberg/tests/common/rustfs.rs index 84134f0c9b..14e7973825 100644 --- a/integrations/iceberg/tests/common/rustfs.rs +++ b/integrations/iceberg/tests/common/rustfs.rs @@ -47,9 +47,9 @@ impl RestCatalog { pub async fn start() -> Self { // Docker names are global, and tests run concurrently. let suffix = unique_suffix(); - let network = format!("iceberg-ballista-{suffix}"); + let network = format!("ballista-iceberg-{suffix}"); // Also the hostname the catalog reaches RustFS on. - let s3_host = format!("iceberg-ballista-rustfs-{suffix}"); + let s3_host = format!("ballista-iceberg-rustfs-{suffix}"); let s3 = GenericImage::new("rustfs/rustfs", "1.0.0") .with_exposed_port(S3_PORT.tcp()) From 8ef54d69b4b654b9529994c9d91695350f29ab65 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 26 Oct 2026 06:35:43 -0400 Subject: [PATCH 03/31] feat(iceberg): pin catalog-backed tables when a query is planned register_iceberg(state) wraps the session's catalogs so that DataFusion's async table lookup, which runs once per table a query names, loads each catalog-backed Iceberg table and returns it at that version. Every scan of a table in one query reads the same version, and the logical codec no longer loads tables, so it no longer blocks the client or needs a multi-threaded runtime. --- docs/source/user-guide/iceberg.md | 26 +-- .../examples/standalone-iceberg-scan.rs | 8 +- integrations/iceberg/src/catalog.rs | 198 ++++++++++++++++++ integrations/iceberg/src/lib.rs | 32 ++- integrations/iceberg/src/logical_codec.rs | 106 ++-------- integrations/iceberg/tests/codec.rs | 72 +++---- integrations/iceberg/tests/common/mod.rs | 6 +- integrations/iceberg/tests/object_storage.rs | 2 +- integrations/iceberg/tests/scan.rs | 3 +- 9 files changed, 292 insertions(+), 161 deletions(-) create mode 100644 integrations/iceberg/src/catalog.rs diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index dec7c02d9c..d27b74fb3b 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -30,7 +30,7 @@ iceberg-datafusion and iceberg-rust, and needs Rust 1.95. ## Usage -Register the codecs on the session config, then mount an Iceberg catalog: +Ready the session with `register_iceberg`, then mount an Iceberg catalog: ```rust use std::sync::Arc; @@ -38,14 +38,14 @@ use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; -use ballista_iceberg::register_iceberg_codecs; +use ballista_iceberg::register_iceberg; use iceberg_datafusion::IcebergCatalogProvider; let state = SessionStateBuilder::new() - .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_config(SessionConfig::new_with_ballista()) .with_default_features() .build(); -let ctx = SessionContext::standalone_with_state(state).await?; +let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; ctx.register_catalog("rest", Arc::new(IcebergCatalogProvider::try_new(catalog).await?)); ctx.sql("SELECT * FROM rest.ns.t").await?.show().await?; @@ -71,9 +71,11 @@ the executors, as bytes. The codecs replace each live Iceberg table with one version of it: the location of its metadata file and its serialized `FileIO`. Metadata files never change, so every node rebuilds exactly that version. -- **A catalog-backed table** is sent as the version its catalog serves when the - client sends the query. Each query asks again, so each reads the latest - version. +- **A catalog-backed table** is pinned to the version its catalog serves when + the client plans the query. DataFusion looks up each table a query names + once, so every scan of it, such as each side of a self-join, reads that + version, and each new query reads the latest one. A `DataFrame` planned once + and run twice reads the same version both times. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. @@ -86,13 +88,6 @@ configuration. ## Limitations -- **A table read twice in one query can see two versions.** Each scan of a - catalog-backed table, such as each side of a self-join, asks the catalog - separately, so a commit that lands in between is seen by only one of them. -- **Catalog-backed tables need a multi-threaded client runtime.** Sending a - query loads them while blocking the calling thread, which can hang a - single-threaded tokio runtime (`#[tokio::test]`'s default), so there it fails - with an error. Static tables work on either. - **Reads only.** `INSERT INTO` an Iceberg table fails with "isn't supported on Ballista yet". - **No metadata tables.** Tables such as `t$snapshots` can't be sent to the @@ -104,5 +99,6 @@ configuration. `FileIO` includes them, so the links between client, scheduler and executors must be trusted. - **Only standalone mode is tested.** A separately deployed scheduler and - executors must link `ballista-iceberg` and register the codecs too (see + executors must link `ballista-iceberg` and register the codecs with + `register_iceberg_codecs` (see [Extending Components](extending-components.md)). diff --git a/integrations/iceberg/examples/standalone-iceberg-scan.rs b/integrations/iceberg/examples/standalone-iceberg-scan.rs index e466720ea0..f4546b3152 100644 --- a/integrations/iceberg/examples/standalone-iceberg-scan.rs +++ b/integrations/iceberg/examples/standalone-iceberg-scan.rs @@ -28,7 +28,7 @@ use std::collections::HashMap; use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; -use ballista_iceberg::register_iceberg_codecs; +use ballista_iceberg::register_iceberg; use datafusion::common::Result; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; @@ -94,12 +94,12 @@ async fn main() -> Result<()> { }; insert("(1, 'Paris'), (2, 'Oslo')").await?; - // Start standalone Ballista with the Iceberg codecs. + // Start standalone Ballista with Iceberg support. let state = SessionStateBuilder::new() - .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_config(SessionConfig::new_with_ballista()) .with_default_features() .build(); - let ctx = SessionContext::standalone_with_state(state).await?; + let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; ctx.register_table("trips", trips.clone())?; // A static table always reads the version it was made from: time travel. diff --git a/integrations/iceberg/src/catalog.rs b/integrations/iceberg/src/catalog.rs new file mode 100644 index 0000000000..6e07029ea6 --- /dev/null +++ b/integrations/iceberg/src/catalog.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Pins catalog-backed tables to one version while a query is planned. + +use std::sync::Arc; + +use async_trait::async_trait; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::{ + CatalogProvider, CatalogProviderList, SchemaProvider, Session, TableProvider, +}; +use datafusion::common::{Result, not_impl_err}; +use datafusion::datasource::TableType; +use datafusion::datasource::source::DataSourceExec; +use datafusion::logical_expr::dml::InsertOp; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; +use datafusion::physical_plan::ExecutionPlan; +use iceberg::table::Table; +use iceberg_datafusion::read::IcebergDataSource; +use iceberg_datafusion::{IcebergCatalogTableProvider, to_datafusion_error}; + +/// A session's catalogs, serving each catalog-backed Iceberg table as a +/// [`PinnedTable`]. +/// +/// DataFusion looks up each table a query names once, while planning it, so +/// every scan of a table in the query reads the same version, and encoding the +/// plan needs no I/O. +#[derive(Debug)] +pub(crate) struct PinningCatalogList(pub(crate) Arc); + +impl CatalogProviderList for PinningCatalogList { + fn register_catalog( + &self, + name: String, + catalog: Arc, + ) -> Option> { + self.0.register_catalog(name, catalog) + } + + fn catalog_names(&self) -> Vec { + self.0.catalog_names() + } + + fn catalog(&self, name: &str) -> Option> { + let catalog = self.0.catalog(name)?; + Some(Arc::new(PinningCatalog(catalog))) + } +} + +#[derive(Debug)] +struct PinningCatalog(Arc); + +impl CatalogProvider for PinningCatalog { + fn schema_names(&self) -> Vec { + self.0.schema_names() + } + + fn schema(&self, name: &str) -> Option> { + let schema = self.0.schema(name)?; + Some(Arc::new(PinningSchema(schema))) + } + + fn register_schema( + &self, + name: &str, + schema: Arc, + ) -> Result>> { + self.0.register_schema(name, schema) + } + + fn deregister_schema( + &self, + name: &str, + cascade: bool, + ) -> Result>> { + self.0.deregister_schema(name, cascade) + } +} + +#[derive(Debug)] +struct PinningSchema(Arc); + +#[async_trait] +impl SchemaProvider for PinningSchema { + fn owner_name(&self) -> Option<&str> { + self.0.owner_name() + } + + fn table_names(&self) -> Vec { + self.0.table_names() + } + + async fn table(&self, name: &str) -> Result>> { + let Some(table) = self.0.table(name).await? else { + return Ok(None); + }; + let Some(provider) = table.downcast_ref::() else { + return Ok(Some(table)); + }; + let pinned = provider + .catalog() + .load_table(provider.table_ident()) + .await + .map_err(to_datafusion_error)?; + Ok(Some(Arc::new(PinnedTable { + table: pinned, + snapshot_id: None, + schema: provider.schema(), + }))) + } + + async fn table_type(&self, name: &str) -> Result> { + self.0.table_type(name).await + } + + fn register_table( + &self, + name: String, + table: Arc, + ) -> Result>> { + self.0.register_table(name, table) + } + + fn deregister_table(&self, name: &str) -> Result>> { + self.0.deregister_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + self.0.table_exist(name) + } +} + +/// One version of an Iceberg table, read with the schema the client planned +/// against. +#[derive(Debug)] +pub(crate) struct PinnedTable { + pub(crate) table: Table, + pub(crate) snapshot_id: Option, + pub(crate) schema: SchemaRef, +} + +#[async_trait] +impl TableProvider for PinnedTable { + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + filters: &[Expr], + limit: Option, + ) -> Result> { + let source = IcebergDataSource::builder(self.table.clone(), self.schema.clone()) + .with_snapshot_id(self.snapshot_id) + .with_projection(projection) + .with_filters(filters) + .with_fetch(limit) + .build()?; + Ok(DataSourceExec::from_data_source(source)) + } + + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> Result> { + Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) + } + + async fn insert_into( + &self, + _state: &dyn Session, + _input: Arc, + _insert_op: InsertOp, + ) -> Result> { + not_impl_err!("INSERT INTO an Iceberg table isn't supported on Ballista yet") + } +} diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 1ac6ab6f7b..a47e524320 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -20,8 +20,8 @@ //! Ballista sends plans between nodes as bytes, so these codecs send each Iceberg //! table and scan as one version of the table, which any node can rebuild //! without a catalog. A catalog-backed table is sent as the version its catalog -//! serves when the query is sent, a static one as the version it holds. Only -//! the client talks to a catalog. +//! served when the query was planned, a static one as the version it holds. +//! Only the client talks to a catalog. //! //! Storage comes from [`iceberg_storage_opendal`]: S3 and local files by //! default, other stores with its `opendal-*` features. @@ -31,14 +31,14 @@ //! # use ballista::prelude::{SessionConfigExt, SessionContextExt}; //! # use datafusion::execution::SessionStateBuilder; //! # use datafusion::prelude::{SessionConfig, SessionContext}; -//! # use ballista_iceberg::register_iceberg_codecs; +//! # use ballista_iceberg::register_iceberg; //! # use iceberg_datafusion::IcebergCatalogProvider; //! # async fn example(catalog: Arc) -> datafusion::error::Result<()> { //! let state = SessionStateBuilder::new() -//! .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) +//! .with_config(SessionConfig::new_with_ballista()) //! .with_default_features() //! .build(); -//! let ctx = SessionContext::standalone_with_state(state).await?; +//! let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; //! //! ctx.register_catalog("rest", Arc::new(IcebergCatalogProvider::try_new(catalog).await?)); //! ctx.sql("SELECT * FROM rest.ns.t").await?.show().await?; @@ -50,11 +50,10 @@ //! //! - Reads only. //! - Each scan runs as a single task. -//! - Catalog-backed tables need a multi-threaded tokio runtime on the client. -//! - A catalog-backed table read twice in one query can see two versions. //! - No metadata tables such as `t$snapshots`. //! - Storage credentials travel in plain text. +mod catalog; mod logical_codec; mod physical_codec; mod wire; @@ -62,17 +61,32 @@ mod wire; use std::sync::Arc; use ballista_core::extension::SessionConfigExt; +use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::prelude::SessionConfig; // Linked for the storage it registers; nothing here calls it. use iceberg_storage_opendal as _; +use crate::catalog::PinningCatalogList; + pub use crate::logical_codec::IcebergLogicalCodec; pub use crate::physical_codec::IcebergPhysicalCodec; +/// Readies a client's session for Iceberg: registers the codecs, and pins +/// each catalog-backed table a query names to the version its catalog serves +/// when the query is planned. +pub fn register_iceberg(mut state: SessionState) -> SessionState { + *state.config_mut() = register_iceberg_codecs(state.config().clone()); + let catalogs = PinningCatalogList(state.catalog_list().clone()); + SessionStateBuilder::new_from_existing(state) + .with_catalog_list(Arc::new(catalogs)) + .build() +} + /// Wraps `config`'s Ballista codecs in the Iceberg ones. /// -/// In standalone mode the scheduler and executor share the client's config; a -/// separately deployed scheduler and executors need this on their configs too. +/// The client registers them with [`register_iceberg`]. In standalone mode the +/// scheduler and executor share the client's config; a separately deployed +/// scheduler and executors need this on their configs. pub fn register_iceberg_codecs(config: SessionConfig) -> SessionConfig { let logical = IcebergLogicalCodec::new(config.ballista_logical_extension_codec()); let physical = IcebergPhysicalCodec::new(config.ballista_physical_extension_codec()); diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 39b9ef4fc3..6087e9142b 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -19,30 +19,20 @@ use std::sync::Arc; -use async_trait::async_trait; use ballista_core::serde::BallistaLogicalExtensionCodec; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::catalog::{Session, TableProvider}; -use datafusion::common::{Result, TableReference, exec_err, not_impl_err}; -use datafusion::datasource::TableType; +use datafusion::catalog::TableProvider; +use datafusion::common::{Result, TableReference, exec_err}; use datafusion::datasource::file_format::FileFormatFactory; -use datafusion::datasource::source::DataSourceExec; use datafusion::execution::TaskContext; -use datafusion::logical_expr::dml::InsertOp; use datafusion::logical_expr::{ - AggregateUDF, Expr, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, - TableProviderFilterPushDown, WindowUDF, + AggregateUDF, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, }; -use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::logical_plan::LogicalExtensionCodec; -use iceberg::table::Table; -use iceberg_datafusion::read::IcebergDataSource; -use iceberg_datafusion::{ - IcebergCatalogTableProvider, IcebergStaticTableProvider, to_datafusion_error, -}; +use iceberg_datafusion::{IcebergCatalogTableProvider, IcebergStaticTableProvider}; use serde::{Deserialize, Serialize}; -use tokio::runtime::{Handle, RuntimeFlavor}; +use crate::catalog::PinnedTable; use crate::wire::{self, Frame, TAG_DELEGATED, TableWire}; #[derive(Debug, Serialize, Deserialize)] @@ -51,61 +41,11 @@ struct ProviderWire { snapshot_id: Option, } -/// The scheduler's side of an Iceberg table: one version of it, read with the -/// schema the client planned against. -#[derive(Debug)] -struct PinnedTable { - table: Table, - snapshot_id: Option, - schema: SchemaRef, -} - -#[async_trait] -impl TableProvider for PinnedTable { - fn schema(&self) -> SchemaRef { - self.schema.clone() - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - _state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> Result> { - let source = IcebergDataSource::builder(self.table.clone(), self.schema.clone()) - .with_snapshot_id(self.snapshot_id) - .with_projection(projection) - .with_filters(filters) - .with_fetch(limit) - .build()?; - Ok(DataSourceExec::from_data_source(source)) - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> Result> { - Ok(vec![TableProviderFilterPushDown::Inexact; filters.len()]) - } - - async fn insert_into( - &self, - _state: &dyn Session, - _input: Arc, - _insert_op: InsertOp, - ) -> Result> { - not_impl_err!("INSERT INTO an Iceberg table isn't supported on Ballista yet") - } -} - /// A [`LogicalExtensionCodec`] for [`IcebergStaticTableProvider`] and -/// [`IcebergCatalogTableProvider`] that delegates everything else to an inner -/// codec. +/// catalog-backed tables pinned by [`register_iceberg`] that delegates +/// everything else to an inner codec. +/// +/// [`register_iceberg`]: crate::register_iceberg #[derive(Debug)] pub struct IcebergLogicalCodec { inner: Arc, @@ -173,28 +113,16 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { table: TableWire::with_metadata(provider.table())?, snapshot_id: provider.snapshot_id(), } - } else if let Some(provider) = node.downcast_ref::() - { - // Loading blocks this thread, and a REST catalog's pooled - // connections may need this thread's runtime to answer. - if let Ok(RuntimeFlavor::CurrentThread) = - Handle::try_current().map(|handle| handle.runtime_flavor()) - { - return exec_err!( - "Iceberg table {table_ref} is catalog-backed, so sending the query \ - loads it, which can hang on a single-threaded tokio runtime; run \ - the client on a multi-threaded runtime, or register the table as \ - an IcebergStaticTableProvider" - ); - } - let catalog = provider.catalog().clone(); - let ident = provider.table_ident().clone(); - let table = wire::block_on(async move { catalog.load_table(&ident).await })? - .map_err(to_datafusion_error)?; + } else if let Some(pinned) = node.downcast_ref::() { ProviderWire { - table: TableWire::with_metadata(&table)?, - snapshot_id: None, + table: TableWire::with_metadata(&pinned.table)?, + snapshot_id: pinned.snapshot_id, } + } else if node.is::() { + return exec_err!( + "Iceberg table {table_ref} is catalog-backed but wasn't pinned to a \ + version: build the client's session with ballista_iceberg::register_iceberg" + ); } else { buf.push(TAG_DELEGATED); return self.inner.try_encode_table_provider(table_ref, node, buf); diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index 4f0db1f6bc..90ae3b5ed5 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -24,16 +24,18 @@ mod common; mod iceberg_nodes { use std::sync::Arc; - use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec}; + use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, register_iceberg}; use datafusion::assert_batches_eq; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference}; use datafusion::datasource::source::DataSourceExec; - use datafusion::execution::TaskContext; - use datafusion::logical_expr::dml::InsertOp; - use datafusion::physical_plan::empty::EmptyExec; + use datafusion::execution::SessionStateBuilder; use datafusion::physical_plan::{collect, displayable}; use datafusion::prelude::{SessionContext, col, lit}; + use datafusion_proto::bytes::{ + logical_plan_from_bytes_with_extension_codec, + logical_plan_to_bytes_with_extension_codec, + }; use datafusion_proto::logical_plan::LogicalExtensionCodec; use datafusion_proto::physical_plan::{ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, @@ -84,43 +86,34 @@ mod iceberg_nodes { Ok(()) } - // Multi-threaded: encoding a catalog-backed table loads it. - #[tokio::test(flavor = "multi_thread")] - async fn should_pin_catalog_table_when_encoded() -> Result<()> { + #[tokio::test] + async fn should_pin_catalog_tables_when_planned() -> Result<()> { let dir = tempfile::tempdir()?; let catalog = memory_catalog(dir.path()).await; - let provider = catalog_table(&catalog).await; - let schema = provider.schema(); + let state = SessionStateBuilder::new().with_default_features().build(); + let ctx = SessionContext::new_with_state(register_iceberg(state)); + ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; let codec = IcebergLogicalCodec::default(); - let table = TableReference::bare("t"); + let send = async |sql: &str| { + let plan = ctx.state().create_logical_plan(sql).await?; + logical_plan_to_bytes_with_extension_codec(&plan, &codec) + }; let pinned = insert(&catalog, "(1, 'a')").await; - let mut buf = Vec::new(); - codec.try_encode_table_provider(&table, Arc::new(provider), &mut buf)?; + let select = send("SELECT a.id, b.name FROM t a CROSS JOIN t b").await?; + let insert_c = send("INSERT INTO t VALUES (3, 'c')").await?; insert(&catalog, "(2, 'b')").await; // Decoding uses the metadata the client sent, not the file. std::fs::remove_file(pinned.metadata_location().unwrap())?; - let decoded = codec.try_decode_table_provider( - &buf, - &table, - schema.clone(), - &TaskContext::default(), - )?; - - let ctx = SessionContext::new(); - let input = Arc::new(EmptyExec::new(schema)); - let err = decoded - .insert_into(&ctx.state(), input, InsertOp::Append) - .await - .unwrap_err(); - assert_eq!( - err.to_string(), - "This feature is not implemented: INSERT INTO an Iceberg table isn't \ - supported on Ballista yet" - ); - ctx.register_table("t", decoded)?; - let result = ctx.sql("SELECT * FROM t").await?.collect().await?; + let run = async |bytes: &[u8]| { + let plan = logical_plan_from_bytes_with_extension_codec( + bytes, + &ctx.task_ctx(), + &codec, + )?; + ctx.execute_logical_plan(plan).await?.collect().await + }; let expected = [ "+----+------+", "| id | name |", @@ -128,13 +121,18 @@ mod iceberg_nodes { "| 1 | a |", "+----+------+", ]; - assert_batches_eq!(expected, &result); + assert_batches_eq!(expected, &run(&select).await?); + assert_eq!( + run(&insert_c).await.unwrap_err().to_string(), + "This feature is not implemented: INSERT INTO an Iceberg table isn't \ + supported on Ballista yet" + ); Ok(()) } #[tokio::test] - async fn should_refuse_catalog_table_on_single_threaded_runtime() -> Result<()> { + async fn should_refuse_unpinned_catalog_table() -> Result<()> { let dir = tempfile::tempdir()?; let provider = catalog_table(&memory_catalog(dir.path()).await).await; @@ -147,10 +145,8 @@ mod iceberg_nodes { .unwrap_err(); assert_eq!( err.to_string(), - "Execution error: Iceberg table t is catalog-backed, so sending the query \ - loads it, which can hang on a single-threaded tokio runtime; run the \ - client on a multi-threaded runtime, or register the table as an \ - IcebergStaticTableProvider" + "Execution error: Iceberg table t is catalog-backed but wasn't pinned to \ + a version: build the client's session with ballista_iceberg::register_iceberg" ); Ok(()) diff --git a/integrations/iceberg/tests/common/mod.rs b/integrations/iceberg/tests/common/mod.rs index 170920ea39..0a98b8c7d9 100644 --- a/integrations/iceberg/tests/common/mod.rs +++ b/integrations/iceberg/tests/common/mod.rs @@ -27,7 +27,7 @@ use std::path::Path; use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; -use ballista_iceberg::register_iceberg_codecs; +use ballista_iceberg::register_iceberg; use datafusion::assert_batches_eq; use datafusion::error::Result; use datafusion::execution::SessionStateBuilder; @@ -112,10 +112,10 @@ pub async fn assert_standalone_scans(catalog: Arc) -> Result<()> { let first = insert(&catalog, "(1, 'a'), (2, NULL)").await; let state = SessionStateBuilder::new() - .with_config(register_iceberg_codecs(SessionConfig::new_with_ballista())) + .with_config(SessionConfig::new_with_ballista()) .with_default_features() .build(); - let ctx = SessionContext::standalone_with_state(state).await?; + let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; let mount = IcebergCatalogProvider::try_new(catalog.clone()).await?; ctx.register_catalog("iceberg", Arc::new(mount)); let v1 = IcebergStaticTableProvider::try_new_from_table(first).await?; diff --git a/integrations/iceberg/tests/object_storage.rs b/integrations/iceberg/tests/object_storage.rs index 9a5326e5ba..dd35b2fefa 100644 --- a/integrations/iceberg/tests/object_storage.rs +++ b/integrations/iceberg/tests/object_storage.rs @@ -29,7 +29,7 @@ mod standalone { use crate::common::rustfs::RestCatalog; use crate::common::{assert_standalone_scans, create_table}; - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn should_scan_s3_tables_through_a_rest_catalog() -> datafusion::error::Result<()> { let rest = RestCatalog::start().await; diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index d2359f0cc3..9d4936ab86 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -23,8 +23,7 @@ mod common; mod standalone { use crate::common::{assert_standalone_scans, memory_catalog}; - // Multi-threaded: sending a query loads its catalog-backed tables. - #[tokio::test(flavor = "multi_thread")] + #[tokio::test] async fn should_scan_latest_and_older_versions() -> datafusion::error::Result<()> { let dir = tempfile::tempdir()?; assert_standalone_scans(memory_catalog(dir.path()).await).await From abfeaeaf276f220df8369d3195d4e52dcb361eda Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 14:22:27 -0400 Subject: [PATCH 04/31] feat(iceberg): mount Iceberg catalogs on the scheduler for SQL clients Flight SQL clients send SQL that the scheduler plans, so they hold no catalog. iceberg_session_builder builds the scheduler's sessions with configured catalogs mounted and readied with register_iceberg, so their tables are pinned while the scheduler plans. register_iceberg now wraps the codecs itself and leaves an already-ready session unchanged, so register_iceberg_codecs is gone. The docs now say a separately deployed scheduler and executors set the Iceberg codecs as their codec overrides, and that a view keeps the version from when it was created. --- Cargo.lock | 4 + docs/source/user-guide/iceberg.md | 49 ++++++-- integrations/iceberg/Cargo.toml | 4 + integrations/iceberg/src/lib.rs | 51 ++++++-- integrations/iceberg/tests/flight_sql.rs | 144 +++++++++++++++++++++++ integrations/iceberg/tests/scan.rs | 43 ++++++- 6 files changed, 272 insertions(+), 23 deletions(-) create mode 100644 integrations/iceberg/tests/flight_sql.rs diff --git a/Cargo.lock b/Cargo.lock index 1347e6f3a7..659a1f4b51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1362,10 +1362,13 @@ dependencies = [ name = "ballista-iceberg" version = "55.0.0" dependencies = [ + "arrow-flight", "arrow-schema", "async-trait", "ballista", "ballista-core", + "ballista-executor", + "ballista-scheduler", "datafusion", "datafusion-proto", "futures", @@ -1378,6 +1381,7 @@ dependencies = [ "tempfile", "testcontainers", "tokio", + "tonic", ] [[package]] diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index d27b74fb3b..0042c6e3fe 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -64,6 +64,31 @@ a commit reaching the live table but not the static one: cargo run -p ballista-iceberg --example standalone-iceberg-scan ``` +## SQL clients + +Flight SQL clients (JDBC, ADBC, BI tools) send SQL to the scheduler, which plans +it, so they hold no catalog. To let them query Iceberg tables, mount catalogs +on the scheduler with `iceberg_session_builder`, and give the scheduler and +executors the Iceberg codecs: + +```rust +use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder}; + +let rest = IcebergCatalogProvider::try_new(catalog).await?; +let mut config = SchedulerConfig::default() + .with_flight_sql(true) + .with_override_session_builder(Arc::new(iceberg_session_builder([( + "rest".to_string(), + Arc::new(rest) as _, + )]))); +config.override_logical_codec = Some(Arc::new(IcebergLogicalCodec::default())); +config.override_physical_codec = Some(Arc::new(IcebergPhysicalCodec::default())); +``` + +Executors set the same two codecs on their `ExecutorProcessConfig`. A client can +then run `SELECT * FROM rest.ns.t`. Tables are pinned the same way as on the +client, while the scheduler plans the query. + ## How it works Ballista sends the client's plan to the scheduler, and each stage's plan to @@ -74,8 +99,7 @@ Metadata files never change, so every node rebuilds exactly that version. - **A catalog-backed table** is pinned to the version its catalog serves when the client plans the query. DataFusion looks up each table a query names once, so every scan of it, such as each side of a self-join, reads that - version, and each new query reads the latest one. A `DataFrame` planned once - and run twice reads the same version both times. + version, and each new query reads the latest one. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. @@ -83,8 +107,8 @@ Tables sent from the client also carry the metadata itself, so the scheduler needn't read the file. Scans sent to executors carry only the location, which keeps each task's plan small. -Only the client talks to a catalog; the scheduler and executors need no catalog -configuration. +Only the node that plans a query talks to a catalog: the client, or the +scheduler for SQL clients. Executors never need catalog configuration. ## Limitations @@ -98,7 +122,16 @@ configuration. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. -- **Only standalone mode is tested.** A separately deployed scheduler and - executors must link `ballista-iceberg` and register the codecs with - `register_iceberg_codecs` (see - [Extending Components](extending-components.md)). +- **A view keeps the version from when it was created.** DataFusion stores a + view as a planned query, and tables are pinned when a query is planned. The + same goes for a `DataFrame` or a Flight SQL prepared statement run more than + once: each run reads the version from when it was planned. To read newer + commits, recreate the view or plan the query again. +- **A separately deployed scheduler and executors** must link + `ballista-iceberg` and set the Iceberg codecs as their + `override_logical_codec` and `override_physical_codec` (see + [Extending Components](extending-components.md)). The tests cover standalone + mode and an in-process scheduler serving Flight SQL. +- **A mounted catalog lists its tables once.** `IcebergCatalogProvider` reads + the table names when it is built, so a table created afterwards isn't + visible until the catalog is mounted again. diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index b8114fa414..edbcfea76a 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -51,11 +51,15 @@ serde_json = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } [dev-dependencies] +arrow-flight = { workspace = true } ballista = { workspace = true, features = ["standalone"] } +ballista-executor = { workspace = true } +ballista-scheduler = { workspace = true, features = ["flight-sql"] } iceberg-catalog-rest = { workspace = true } tempfile = { workspace = true } testcontainers = { version = "0.28", features = ["http_wait_plain"] } tokio = { workspace = true, features = ["macros"] } +tonic = { workspace = true } [features] # Tests that need docker. diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index a47e524320..5163563499 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -21,7 +21,8 @@ //! table and scan as one version of the table, which any node can rebuild //! without a catalog. A catalog-backed table is sent as the version its catalog //! served when the query was planned, a static one as the version it holds. -//! Only the client talks to a catalog. +//! Only the node that plans a query talks to a catalog: the client, or, for +//! Flight SQL clients, a scheduler built with [`iceberg_session_builder`]. //! //! Storage comes from [`iceberg_storage_opendal`]: S3 and local files by //! default, other stores with its `opendal-*` features. @@ -51,6 +52,8 @@ //! - Reads only. //! - Each scan runs as a single task. //! - No metadata tables such as `t$snapshots`. +//! - A view, or a `DataFrame` or prepared statement reused across runs, reads +//! the version from when it was planned. //! - Storage credentials travel in plain text. mod catalog; @@ -61,6 +64,9 @@ mod wire; use std::sync::Arc; use ballista_core::extension::SessionConfigExt; +use ballista_core::utils::default_session_builder; +use datafusion::catalog::CatalogProvider; +use datafusion::common::Result; use datafusion::execution::{SessionState, SessionStateBuilder}; use datafusion::prelude::SessionConfig; // Linked for the storage it registers; nothing here calls it. @@ -73,24 +79,43 @@ pub use crate::physical_codec::IcebergPhysicalCodec; /// Readies a client's session for Iceberg: registers the codecs, and pins /// each catalog-backed table a query names to the version its catalog serves -/// when the query is planned. +/// when the query is planned. A session that is already ready is returned +/// unchanged. pub fn register_iceberg(mut state: SessionState) -> SessionState { - *state.config_mut() = register_iceberg_codecs(state.config().clone()); + if state.catalog_list().is::() { + return state; + } + let config = state.config_mut(); + let logical = IcebergLogicalCodec::new(config.ballista_logical_extension_codec()); + let physical = IcebergPhysicalCodec::new(config.ballista_physical_extension_codec()); + *config = config + .clone() + .with_ballista_logical_extension_codec(Arc::new(logical)) + .with_ballista_physical_extension_codec(Arc::new(physical)); let catalogs = PinningCatalogList(state.catalog_list().clone()); SessionStateBuilder::new_from_existing(state) .with_catalog_list(Arc::new(catalogs)) .build() } -/// Wraps `config`'s Ballista codecs in the Iceberg ones. +/// Builds a scheduler's sessions with `catalogs` mounted and readied with +/// [`register_iceberg`], for queries the scheduler plans itself: those from +/// Flight SQL clients, which hold no catalog of their own. /// -/// The client registers them with [`register_iceberg`]. In standalone mode the -/// scheduler and executor share the client's config; a separately deployed -/// scheduler and executors need this on their configs. -pub fn register_iceberg_codecs(config: SessionConfig) -> SessionConfig { - let logical = IcebergLogicalCodec::new(config.ballista_logical_extension_codec()); - let physical = IcebergPhysicalCodec::new(config.ballista_physical_extension_codec()); - config - .with_ballista_logical_extension_codec(Arc::new(logical)) - .with_ballista_physical_extension_codec(Arc::new(physical)) +/// Pass it to `SchedulerConfig::with_override_session_builder`. The scheduler +/// and executors also need the Iceberg codecs, set as their +/// `override_logical_codec` and `override_physical_codec`. +pub fn iceberg_session_builder( + catalogs: impl IntoIterator)>, +) -> impl Fn(SessionConfig) -> Result + Send + Sync + 'static { + let catalogs: Vec<_> = catalogs.into_iter().collect(); + move |config| { + let state = register_iceberg(default_session_builder(config)?); + for (name, catalog) in &catalogs { + state + .catalog_list() + .register_catalog(name.clone(), catalog.clone()); + } + Ok(state) + } } diff --git a/integrations/iceberg/tests/flight_sql.rs b/integrations/iceberg/tests/flight_sql.rs new file mode 100644 index 0000000000..eeef8af793 --- /dev/null +++ b/integrations/iceberg/tests/flight_sql.rs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Queries Iceberg tables through a scheduler's Flight SQL frontend, as a SQL +//! client with no catalog of its own does. + +mod common; + +#[cfg(test)] +mod flight_sql { + use std::error::Error; + use std::sync::Arc; + + use arrow_flight::sql::client::FlightSqlServiceClient; + use ballista_core::config::TaskSchedulingPolicy; + use ballista_core::serde::BallistaCodec; + use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient; + use ballista_core::utils::default_config_producer; + use ballista_iceberg::{ + IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder, + }; + use ballista_scheduler::cluster::BallistaCluster; + use ballista_scheduler::config::SchedulerConfig; + use ballista_scheduler::metrics::default_metrics_collector; + use ballista_scheduler::scheduler_process::start_grpc_service_with_listener; + use ballista_scheduler::scheduler_server::SchedulerServer; + use datafusion::arrow::array::RecordBatch; + use datafusion::assert_batches_eq; + use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; + use futures::TryStreamExt; + use iceberg_datafusion::IcebergCatalogProvider; + use tonic::transport::Channel; + + use crate::common::{insert, memory_catalog}; + + type Result = std::result::Result>; + + #[tokio::test] + async fn should_query_scheduler_catalogs_from_a_sql_client() -> Result { + let dir = tempfile::tempdir()?; + let catalog = memory_catalog(dir.path()).await; + insert(&catalog, "(1, 'a'), (2, NULL)").await; + let mount = IcebergCatalogProvider::try_new(catalog.clone()).await?; + let mut client = + start_cluster(vec![("iceberg".to_string(), Arc::new(mount) as _)]).await?; + + let query = "SELECT * FROM iceberg.ns.t ORDER BY id"; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 2 | |", + "+----+------+", + ]; + assert_batches_eq!(expected, &run(&mut client, query).await?); + + insert(&catalog, "(3, 'c')").await; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 2 | |", + "| 3 | c |", + "+----+------+", + ]; + assert_batches_eq!(expected, &run(&mut client, query).await?); + + Ok(()) + } + + /// Starts a scheduler serving Flight SQL with `catalogs` mounted, plus one + /// executor, and returns a Flight SQL client connected to it. + async fn start_cluster( + catalogs: Vec<(String, Arc)>, + ) -> Result> { + let codec = || { + BallistaCodec::new( + Arc::new(IcebergLogicalCodec::default()), + Arc::new(IcebergPhysicalCodec::default()), + ) + }; + let cluster = BallistaCluster::new_memory( + "localhost:50050", + Arc::new(iceberg_session_builder(catalogs)), + Arc::new(default_config_producer), + ); + let config = SchedulerConfig::default() + .with_scheduler_policy(TaskSchedulingPolicy::PullStaged) + .with_flight_sql(true); + let mut scheduler: SchedulerServer = + SchedulerServer::new( + "localhost:50050".to_owned(), + cluster, + codec(), + Arc::new(config), + default_metrics_collector()?, + ); + scheduler.init().await?; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let url = format!("http://{}", listener.local_addr()?); + tokio::spawn(start_grpc_service_with_listener(listener, scheduler)); + + let channel = Channel::from_shared(url)?.connect().await?; + ballista_executor::new_standalone_executor( + SchedulerGrpcClient::new(channel.clone()), + 2, + codec(), + ) + .await?; + let mut client = FlightSqlServiceClient::new(channel); + client.handshake("anonymous", "").await?; + Ok(client) + } + + async fn run( + client: &mut FlightSqlServiceClient, + query: &str, + ) -> Result> { + let info = client.execute(query.to_string(), None).await?; + let mut batches = Vec::new(); + for endpoint in info.endpoint { + let ticket = endpoint.ticket.ok_or("endpoint has no ticket")?; + batches.extend(client.do_get(ticket).await?.try_collect::>().await?); + } + Ok(batches) + } +} diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index 9d4936ab86..ec8b13f149 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -21,11 +21,50 @@ mod common; #[cfg(test)] mod standalone { - use crate::common::{assert_standalone_scans, memory_catalog}; + use std::sync::Arc; + + use ballista::prelude::{SessionConfigExt, SessionContextExt}; + use ballista_iceberg::register_iceberg; + use datafusion::assert_batches_eq; + use datafusion::error::Result; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::{SessionConfig, SessionContext}; + + use crate::common::{assert_standalone_scans, catalog_table, insert, memory_catalog}; #[tokio::test] - async fn should_scan_latest_and_older_versions() -> datafusion::error::Result<()> { + async fn should_scan_latest_and_older_versions() -> Result<()> { let dir = tempfile::tempdir()?; assert_standalone_scans(memory_catalog(dir.path()).await).await } + + // A known limitation: a view stores its planned query, and tables are + // pinned when a query is planned. + #[tokio::test] + async fn should_keep_a_views_version() -> Result<()> { + let dir = tempfile::tempdir()?; + let catalog = memory_catalog(dir.path()).await; + let state = SessionStateBuilder::new() + .with_config(SessionConfig::new_with_ballista()) + .with_default_features() + .build(); + let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; + ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; + + insert(&catalog, "(1, 'a')").await; + ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + insert(&catalog, "(2, 'b')").await; + + let result = ctx.sql("SELECT * FROM v").await?.collect().await?; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "+----+------+", + ]; + assert_batches_eq!(expected, &result); + + Ok(()) + } } From 6ed3d0b0274611a29e6ee177e2fc60fbef07a304 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 15:06:33 -0400 Subject: [PATCH 05/31] fix(iceberg): refuse metadata tables with a clear error A metadata table such as t$snapshots reached the inner codec, which failed with an internal error suggesting a DataFusion bug. The logical codec now refuses it as not supported on Ballista yet. --- integrations/iceberg/src/logical_codec.rs | 7 ++++- integrations/iceberg/tests/codec.rs | 34 ++++++++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 6087e9142b..59170e5e2d 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -22,13 +22,14 @@ use std::sync::Arc; use ballista_core::serde::BallistaLogicalExtensionCodec; use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::TableProvider; -use datafusion::common::{Result, TableReference, exec_err}; +use datafusion::common::{Result, TableReference, exec_err, not_impl_err}; use datafusion::datasource::file_format::FileFormatFactory; use datafusion::execution::TaskContext; use datafusion::logical_expr::{ AggregateUDF, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, }; use datafusion_proto::logical_plan::LogicalExtensionCodec; +use iceberg_datafusion::metadata::IcebergMetadataTableProvider; use iceberg_datafusion::{IcebergCatalogTableProvider, IcebergStaticTableProvider}; use serde::{Deserialize, Serialize}; @@ -123,6 +124,10 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { "Iceberg table {table_ref} is catalog-backed but wasn't pinned to a \ version: build the client's session with ballista_iceberg::register_iceberg" ); + } else if node.is::() { + return not_impl_err!( + "Iceberg metadata table {table_ref} isn't supported on Ballista yet" + ); } else { buf.push(TAG_DELEGATED); return self.inner.try_encode_table_provider(table_ref, node, buf); diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index 90ae3b5ed5..be0181ee4d 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -40,7 +40,9 @@ mod iceberg_nodes { use datafusion_proto::physical_plan::{ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, }; + use iceberg::inspect::MetadataTableType; use iceberg_datafusion::IcebergStaticTableProvider; + use iceberg_datafusion::metadata::IcebergMetadataTableProvider; use iceberg_datafusion::read::IcebergDataSource; use crate::common::{catalog_table, insert, memory_catalog}; @@ -132,22 +134,34 @@ mod iceberg_nodes { } #[tokio::test] - async fn should_refuse_unpinned_catalog_table() -> Result<()> { + async fn should_refuse_unpinned_and_metadata_tables() -> Result<()> { let dir = tempfile::tempdir()?; - let provider = catalog_table(&memory_catalog(dir.path()).await).await; + let catalog = memory_catalog(dir.path()).await; + let table = insert(&catalog, "(1, 'a')").await; + let encode = |provider: Arc| { + IcebergLogicalCodec::default() + .try_encode_table_provider( + &TableReference::bare("t"), + provider, + &mut Vec::new(), + ) + .unwrap_err() + .to_string() + }; - let err = IcebergLogicalCodec::default() - .try_encode_table_provider( - &TableReference::bare("t"), - Arc::new(provider), - &mut Vec::new(), - ) - .unwrap_err(); assert_eq!( - err.to_string(), + encode(Arc::new(catalog_table(&catalog).await)), "Execution error: Iceberg table t is catalog-backed but wasn't pinned to \ a version: build the client's session with ballista_iceberg::register_iceberg" ); + assert_eq!( + encode(Arc::new(IcebergMetadataTableProvider::new( + table, + MetadataTableType::Snapshots + ))), + "This feature is not implemented: Iceberg metadata table t isn't supported \ + on Ballista yet" + ); Ok(()) } From 4749413ab7cd309acda00c4fb32e891a997f8961 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 15:16:09 -0400 Subject: [PATCH 06/31] feat(iceberg): wrap the scheduler's own session builder iceberg_session_builder now takes the scheduler's existing session builder and wraps it, so a scheduler that sets up its sessions itself (for example to register object stores) keeps that setup. Also explains the dead_code allow on the shared test helpers. --- docs/source/user-guide/iceberg.md | 13 +++++++------ integrations/iceberg/src/lib.rs | 16 +++++++++------- integrations/iceberg/tests/common/mod.rs | 1 + integrations/iceberg/tests/flight_sql.rs | 4 ++-- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 0042c6e3fe..3081ae809c 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -68,19 +68,20 @@ cargo run -p ballista-iceberg --example standalone-iceberg-scan Flight SQL clients (JDBC, ADBC, BI tools) send SQL to the scheduler, which plans it, so they hold no catalog. To let them query Iceberg tables, mount catalogs -on the scheduler with `iceberg_session_builder`, and give the scheduler and -executors the Iceberg codecs: +on the scheduler with `iceberg_session_builder`, which wraps the scheduler's +session builder, and give the scheduler and executors the Iceberg codecs: ```rust +use ballista_core::utils::default_session_builder; use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder}; let rest = IcebergCatalogProvider::try_new(catalog).await?; let mut config = SchedulerConfig::default() .with_flight_sql(true) - .with_override_session_builder(Arc::new(iceberg_session_builder([( - "rest".to_string(), - Arc::new(rest) as _, - )]))); + .with_override_session_builder(Arc::new(iceberg_session_builder( + default_session_builder, + [("rest".to_string(), Arc::new(rest) as _)], + ))); config.override_logical_codec = Some(Arc::new(IcebergLogicalCodec::default())); config.override_physical_codec = Some(Arc::new(IcebergPhysicalCodec::default())); ``` diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 5163563499..e3c03e75dc 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -64,7 +64,6 @@ mod wire; use std::sync::Arc; use ballista_core::extension::SessionConfigExt; -use ballista_core::utils::default_session_builder; use datafusion::catalog::CatalogProvider; use datafusion::common::Result; use datafusion::execution::{SessionState, SessionStateBuilder}; @@ -98,19 +97,22 @@ pub fn register_iceberg(mut state: SessionState) -> SessionState { .build() } -/// Builds a scheduler's sessions with `catalogs` mounted and readied with -/// [`register_iceberg`], for queries the scheduler plans itself: those from -/// Flight SQL clients, which hold no catalog of their own. +/// Wraps a scheduler's session builder, `base`, so its sessions have +/// `catalogs` mounted and are readied with [`register_iceberg`], for queries +/// the scheduler plans itself: those from Flight SQL clients, which hold no +/// catalog of their own. Pass `ballista_core::utils::default_session_builder` +/// as `base` if the scheduler has no session builder of its own. /// -/// Pass it to `SchedulerConfig::with_override_session_builder`. The scheduler -/// and executors also need the Iceberg codecs, set as their +/// Pass the result to `SchedulerConfig::with_override_session_builder`. The +/// scheduler and executors also need the Iceberg codecs, set as their /// `override_logical_codec` and `override_physical_codec`. pub fn iceberg_session_builder( + base: impl Fn(SessionConfig) -> Result + Send + Sync + 'static, catalogs: impl IntoIterator)>, ) -> impl Fn(SessionConfig) -> Result + Send + Sync + 'static { let catalogs: Vec<_> = catalogs.into_iter().collect(); move |config| { - let state = register_iceberg(default_session_builder(config)?); + let state = register_iceberg(base(config)?); for (name, catalog) in &catalogs { state .catalog_list() diff --git a/integrations/iceberg/tests/common/mod.rs b/integrations/iceberg/tests/common/mod.rs index 0a98b8c7d9..ef8607056b 100644 --- a/integrations/iceberg/tests/common/mod.rs +++ b/integrations/iceberg/tests/common/mod.rs @@ -17,6 +17,7 @@ //! Helpers shared by the integration tests. +// Each test file compiles this module separately and uses only some of it. #![allow(dead_code)] #[cfg(feature = "testcontainers")] diff --git a/integrations/iceberg/tests/flight_sql.rs b/integrations/iceberg/tests/flight_sql.rs index eeef8af793..2629174962 100644 --- a/integrations/iceberg/tests/flight_sql.rs +++ b/integrations/iceberg/tests/flight_sql.rs @@ -29,7 +29,7 @@ mod flight_sql { use ballista_core::config::TaskSchedulingPolicy; use ballista_core::serde::BallistaCodec; use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient; - use ballista_core::utils::default_config_producer; + use ballista_core::utils::{default_config_producer, default_session_builder}; use ballista_iceberg::{ IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder, }; @@ -97,7 +97,7 @@ mod flight_sql { }; let cluster = BallistaCluster::new_memory( "localhost:50050", - Arc::new(iceberg_session_builder(catalogs)), + Arc::new(iceberg_session_builder(default_session_builder, catalogs)), Arc::new(default_config_producer), ); let config = SchedulerConfig::default() From 43477b2478d53aac0bfca4edc5aad1eccf45ad86 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 15:17:52 -0400 Subject: [PATCH 07/31] docs(iceberg): say why PinnedTable isn't a static table provider --- integrations/iceberg/src/catalog.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integrations/iceberg/src/catalog.rs b/integrations/iceberg/src/catalog.rs index 6e07029ea6..671434e00d 100644 --- a/integrations/iceberg/src/catalog.rs +++ b/integrations/iceberg/src/catalog.rs @@ -147,6 +147,12 @@ impl SchemaProvider for PinningSchema { /// One version of an Iceberg table, read with the schema the client planned /// against. +/// +/// Not an [`IcebergStaticTableProvider`]: that derives its schema from the +/// table, which may have evolved since planning, and is built asynchronously, +/// which a codec's decode can't await. +/// +/// [`IcebergStaticTableProvider`]: iceberg_datafusion::IcebergStaticTableProvider #[derive(Debug)] pub(crate) struct PinnedTable { pub(crate) table: Table, From 5df010959549d7fecb00726709b891eec3f94cae Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 15:26:22 -0400 Subject: [PATCH 08/31] feat(iceberg): take and return the scheduler's SessionBuilder shape iceberg_session_builder now takes and returns Arc, the shape SchedulerConfig stores, so an existing builder passes straight in and the result goes straight to with_override_session_builder. The Flight SQL docs now link to mounting Iceberg catalogs. --- docs/source/user-guide/flightsql.md | 3 ++- docs/source/user-guide/iceberg.md | 9 +++++---- integrations/iceberg/src/lib.rs | 16 ++++++++++------ integrations/iceberg/tests/flight_sql.rs | 2 +- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/docs/source/user-guide/flightsql.md b/docs/source/user-guide/flightsql.md index 6b8de768cd..a7c680fd29 100644 --- a/docs/source/user-guide/flightsql.md +++ b/docs/source/user-guide/flightsql.md @@ -213,7 +213,8 @@ front of the scheduler and leave the frontend as-is. - Embedders can pre-populate the catalog for every session through the scheduler's `SessionBuilder`, so users do not have to re-run `CREATE EXTERNAL TABLE` on each connection. See - [Extending Ballista components](extending-components.md). + [Extending Ballista components](extending-components.md), and + [Apache Iceberg](iceberg.md#sql-clients) for mounting Iceberg catalogs. ## Limitations diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 3081ae809c..3ec133c245 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -69,7 +69,8 @@ cargo run -p ballista-iceberg --example standalone-iceberg-scan Flight SQL clients (JDBC, ADBC, BI tools) send SQL to the scheduler, which plans it, so they hold no catalog. To let them query Iceberg tables, mount catalogs on the scheduler with `iceberg_session_builder`, which wraps the scheduler's -session builder, and give the scheduler and executors the Iceberg codecs: +session builder (`default_session_builder` here, or your own), and give the +scheduler and executors the Iceberg codecs: ```rust use ballista_core::utils::default_session_builder; @@ -78,10 +79,10 @@ use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_sessio let rest = IcebergCatalogProvider::try_new(catalog).await?; let mut config = SchedulerConfig::default() .with_flight_sql(true) - .with_override_session_builder(Arc::new(iceberg_session_builder( - default_session_builder, + .with_override_session_builder(iceberg_session_builder( + Arc::new(default_session_builder), [("rest".to_string(), Arc::new(rest) as _)], - ))); + )); config.override_logical_codec = Some(Arc::new(IcebergLogicalCodec::default())); config.override_physical_codec = Some(Arc::new(IcebergPhysicalCodec::default())); ``` diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index e3c03e75dc..ddca784456 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -97,21 +97,25 @@ pub fn register_iceberg(mut state: SessionState) -> SessionState { .build() } +/// A scheduler's session builder, the shape `SchedulerConfig` stores. +pub type SessionBuilder = + Arc Result + Send + Sync>; + /// Wraps a scheduler's session builder, `base`, so its sessions have /// `catalogs` mounted and are readied with [`register_iceberg`], for queries /// the scheduler plans itself: those from Flight SQL clients, which hold no -/// catalog of their own. Pass `ballista_core::utils::default_session_builder` -/// as `base` if the scheduler has no session builder of its own. +/// catalog of their own. Pass `Arc::new(default_session_builder)` as `base` if +/// the scheduler has no session builder of its own. /// /// Pass the result to `SchedulerConfig::with_override_session_builder`. The /// scheduler and executors also need the Iceberg codecs, set as their /// `override_logical_codec` and `override_physical_codec`. pub fn iceberg_session_builder( - base: impl Fn(SessionConfig) -> Result + Send + Sync + 'static, + base: SessionBuilder, catalogs: impl IntoIterator)>, -) -> impl Fn(SessionConfig) -> Result + Send + Sync + 'static { +) -> SessionBuilder { let catalogs: Vec<_> = catalogs.into_iter().collect(); - move |config| { + Arc::new(move |config| { let state = register_iceberg(base(config)?); for (name, catalog) in &catalogs { state @@ -119,5 +123,5 @@ pub fn iceberg_session_builder( .register_catalog(name.clone(), catalog.clone()); } Ok(state) - } + }) } diff --git a/integrations/iceberg/tests/flight_sql.rs b/integrations/iceberg/tests/flight_sql.rs index 2629174962..14b617b4c5 100644 --- a/integrations/iceberg/tests/flight_sql.rs +++ b/integrations/iceberg/tests/flight_sql.rs @@ -97,7 +97,7 @@ mod flight_sql { }; let cluster = BallistaCluster::new_memory( "localhost:50050", - Arc::new(iceberg_session_builder(default_session_builder, catalogs)), + iceberg_session_builder(Arc::new(default_session_builder), catalogs), Arc::new(default_config_producer), ); let config = SchedulerConfig::default() From ec064cebe30911b2230178421251ce241b05fbcf Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 27 Oct 2026 15:36:48 -0400 Subject: [PATCH 09/31] fix(iceberg): keep the session ID in register_iceberg SessionStateBuilder::new_from_existing gives a session a new ID, so register_iceberg now keeps the original, as Ballista's own share_file_statistics_cache does. Also makes the SessionBuilder alias private: ballista-scheduler already exports one. --- integrations/iceberg/src/lib.rs | 6 ++++-- integrations/iceberg/tests/codec.rs | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index ddca784456..5d55a4e9a1 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -92,14 +92,16 @@ pub fn register_iceberg(mut state: SessionState) -> SessionState { .with_ballista_logical_extension_codec(Arc::new(logical)) .with_ballista_physical_extension_codec(Arc::new(physical)); let catalogs = PinningCatalogList(state.catalog_list().clone()); + // `new_from_existing` would otherwise give the session a new ID. + let session_id = state.session_id().to_string(); SessionStateBuilder::new_from_existing(state) + .with_session_id(session_id) .with_catalog_list(Arc::new(catalogs)) .build() } /// A scheduler's session builder, the shape `SchedulerConfig` stores. -pub type SessionBuilder = - Arc Result + Send + Sync>; +type SessionBuilder = Arc Result + Send + Sync>; /// Wraps a scheduler's session builder, `base`, so its sessions have /// `catalogs` mounted and are readied with [`register_iceberg`], for queries diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index be0181ee4d..ca3ceef90d 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -133,6 +133,15 @@ mod iceberg_nodes { Ok(()) } + #[test] + fn should_keep_the_session_id() { + let state = SessionStateBuilder::new() + .with_session_id("session_0".to_string()) + .build(); + + assert_eq!(register_iceberg(state).session_id(), "session_0"); + } + #[tokio::test] async fn should_refuse_unpinned_and_metadata_tables() -> Result<()> { let dir = tempfile::tempdir()?; From de73e6baabcaef2d2ed20d8331e693ed6efd290b Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 30 Oct 2026 15:44:54 -0400 Subject: [PATCH 10/31] Frame Iceberg payloads with an "iceberg" prefix Payloads that aren't Iceberg's now pass to the inner codec untouched, so codecs for other table formats that follow the same convention can be stacked in any order. Document the convention in the user guide. --- docs/source/user-guide/iceberg.md | 85 ++++++++++++++++++++++ integrations/iceberg/src/logical_codec.rs | 3 +- integrations/iceberg/src/physical_codec.rs | 7 +- integrations/iceberg/src/wire.rs | 27 +++---- integrations/iceberg/tests/codec.rs | 38 +++------- 5 files changed, 111 insertions(+), 49 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 3ec133c245..6548427769 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -112,6 +112,91 @@ keeps each task's plan small. Only the node that plans a query talks to a catalog: the client, or the scheduler for SQL clients. Executors never need catalog configuration. +## Combining with other table formats + +The Iceberg codecs wrap the codecs a session already has, so one query can +read Iceberg tables alongside tables of another format, such as Delta or Hudi, +whose codecs are maintained elsewhere. Any codec that follows the same +convention can be stacked with these in any order: + +1. **Wrap an inner codec.** Take the session's existing codec in the + constructor rather than Ballista's default, and pass it everything that + isn't yours. +2. **Prefix your own bytes with a unique name.** Iceberg payloads start with + `iceberg`. Choose a name for your format, such as `delta`, that no other + codec uses and that doesn't start another codec's name. +3. **Pass everything else through untouched.** When encoding something that + isn't yours, write only what the inner codec writes, without adding a + marker. When decoding, if the bytes don't start with your prefix, hand all + of them to the inner codec. + +A logical codec for a hypothetical Delta integration would look like this: + +```rust +/// Rule 2: the name that starts every payload this codec writes. +const PREFIX: &[u8] = b"delta"; + +#[derive(Debug)] +pub struct DeltaLogicalCodec { + inner: Arc, +} + +impl DeltaLogicalCodec { + /// Rule 1: wrap whatever codec the session already has. + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +impl LogicalExtensionCodec for DeltaLogicalCodec { + fn try_encode_table_provider( + &self, + table_ref: &TableReference, + node: Arc, + buf: &mut Vec, + ) -> Result<()> { + let Some(table) = node.downcast_ref::() else { + // Rule 3: not ours, so write only what the inner codec writes. + return self.inner.try_encode_table_provider(table_ref, node, buf); + }; + buf.extend_from_slice(PREFIX); + encode_delta_table(table, buf) + } + + fn try_decode_table_provider( + &self, + buf: &[u8], + table_ref: &TableReference, + schema: SchemaRef, + ctx: &TaskContext, + ) -> Result> { + match buf.strip_prefix(PREFIX) { + Some(payload) => decode_delta_table(payload, schema), + // Rule 3: not ours, so hand over every byte unchanged. + None => self.inner.try_decode_table_provider(buf, table_ref, schema, ctx), + } + } + + // Every other method forwards to `self.inner`, for example: + fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { + self.inner.try_decode_udf(name, buf) + } + // ... +} +``` + +A physical codec follows the same pattern in `try_encode` and `try_decode` +for its scan. Because each codec only claims bytes with its own prefix, the +codecs can be stacked in either order: + +```rust +let logical = IcebergLogicalCodec::new(Arc::new(DeltaLogicalCodec::new( + Arc::new(BallistaLogicalExtensionCodec::default()), +))); +``` + +Every node still needs every codec that a query's tables use. + ## Limitations - **Reads only.** `INSERT INTO` an Iceberg table fails with "isn't supported diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 59170e5e2d..82eabd498f 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -34,7 +34,7 @@ use iceberg_datafusion::{IcebergCatalogTableProvider, IcebergStaticTableProvider use serde::{Deserialize, Serialize}; use crate::catalog::PinnedTable; -use crate::wire::{self, Frame, TAG_DELEGATED, TableWire}; +use crate::wire::{self, Frame, TableWire}; #[derive(Debug, Serialize, Deserialize)] struct ProviderWire { @@ -129,7 +129,6 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { "Iceberg metadata table {table_ref} isn't supported on Ballista yet" ); } else { - buf.push(TAG_DELEGATED); return self.inner.try_encode_table_provider(table_ref, node, buf); }; wire::encode(buf, &payload) diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index 3419b98d09..bc8385a9c5 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -36,7 +36,7 @@ use iceberg::expr::Predicate; use iceberg_datafusion::read::IcebergDataSource; use serde::{Deserialize, Serialize}; -use crate::wire::{self, Frame, TAG_DELEGATED, TableWire}; +use crate::wire::{self, Frame, TableWire}; #[derive(Debug, Serialize, Deserialize)] struct ScanWire { @@ -110,10 +110,7 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { fetch: source.fetch(), }, ), - None => { - buf.push(TAG_DELEGATED); - self.inner.try_encode(node, buf, proto_converter) - } + None => self.inner.try_encode(node, buf, proto_converter), } } diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index 6cc9bcfc4a..0ede773dc4 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -21,7 +21,7 @@ use std::collections::VecDeque; use std::fmt; use std::sync::{Arc, LazyLock, Mutex}; -use datafusion::common::{DataFusionError, exec_datafusion_err, exec_err}; +use datafusion::common::{DataFusionError, exec_datafusion_err}; use iceberg::io::FileIO; use iceberg::spec::{TableMetadata, TableMetadataRef}; use iceberg::table::Table; @@ -162,18 +162,18 @@ impl TableWire { } } -/// Leads a blob that belongs to the inner codec. -pub(crate) const TAG_DELEGATED: u8 = 0; -/// Leads a blob that belongs to this crate, JSON-encoded. Also the version of -/// that JSON: a change that older decoders can't read takes the next tag. -pub(crate) const TAG_ICEBERG: u8 = 1; +/// Leads a blob that belongs to this crate, JSON-encoded. Any other blob +/// belongs to the inner codec and passes through untouched, so codecs for +/// other formats can be stacked in any order, as long as each has its own +/// prefix. +const PREFIX: &[u8] = b"iceberg"; /// Appends `payload` to `buf` as an Iceberg blob. pub(crate) fn encode( buf: &mut Vec, payload: &T, ) -> Result<(), DataFusionError> { - buf.push(TAG_ICEBERG); + buf.extend_from_slice(PREFIX); serde_json::to_writer(buf, payload).map_err(json_err) } @@ -183,20 +183,15 @@ pub(crate) enum Frame<'a, T> { Delegated(&'a [u8]), } -/// Splits `buf` by its leading tag, decoding an Iceberg payload as `T`. +/// Decodes `buf` as `T` if it has this crate's prefix. pub(crate) fn decode( buf: &[u8], ) -> Result, DataFusionError> { - match buf.split_first() { - Some((&TAG_ICEBERG, rest)) => serde_json::from_slice(rest) + match buf.strip_prefix(PREFIX) { + Some(rest) => serde_json::from_slice(rest) .map(Frame::Iceberg) .map_err(|e| exec_datafusion_err!("invalid ballista-iceberg payload: {e}")), - Some((&TAG_DELEGATED, rest)) => Ok(Frame::Delegated(rest)), - Some((tag, _)) => exec_err!( - "unknown ballista-iceberg codec tag {tag}: was the plan encoded by \ - another version of ballista-iceberg?" - ), - None => exec_err!("empty ballista-iceberg codec buffer"), + None => Ok(Frame::Delegated(buf)), } } diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index ca3ceef90d..fcd6a17396 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -267,7 +267,7 @@ mod other_nodes { let mut buf = Vec::new(); let plan = Arc::new(EmptyExec::new(schema.clone())); physical.try_encode(plan, &mut buf, &DefaultPhysicalProtoConverter {})?; - assert_eq!(buf, b"\0inner"); + assert_eq!(buf, b"inner"); let decoded = physical.try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {})?; assert_eq!( @@ -280,7 +280,7 @@ mod other_nodes { let mut buf = Vec::new(); let provider = Arc::new(EmptyTable::new(schema.clone())); logical.try_encode_table_provider(&table, provider, &mut buf)?; - assert_eq!(buf, b"\0inner"); + assert_eq!(buf, b"inner"); let decoded = logical.try_decode_table_provider(&buf, &table, schema.clone(), &ctx)?; assert_eq!( @@ -292,31 +292,17 @@ mod other_nodes { } #[test] - fn should_reject_malformed_buffers() { - let codec = IcebergPhysicalCodec::new(Arc::new(Inner)); - let decode = |buf: &[u8]| { - codec - .try_decode( - buf, - &[], - &TaskContext::default(), - &DefaultPhysicalProtoConverter {}, - ) - .unwrap_err() - .to_string() - }; - - assert_eq!( - decode(b""), - "Execution error: empty ballista-iceberg codec buffer" - ); - assert_eq!( - decode(b"\x07inner"), - "Execution error: unknown ballista-iceberg codec tag 7: was the plan \ - encoded by another version of ballista-iceberg?" - ); + fn should_reject_malformed_payload() { + let err = IcebergPhysicalCodec::new(Arc::new(Inner)) + .try_decode( + b"iceberginner", + &[], + &TaskContext::default(), + &DefaultPhysicalProtoConverter {}, + ) + .unwrap_err(); assert_eq!( - decode(b"\x01inner"), + err.to_string(), "Execution error: invalid ballista-iceberg payload: expected value at line 1 \ column 1" ); From f7af624b89c826e85c0d073f94e40e1f74aeb21c Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 30 Oct 2026 15:55:10 -0400 Subject: [PATCH 11/31] cleanup --- integrations/iceberg/src/logical_codec.rs | 13 +++++------- integrations/iceberg/src/physical_codec.rs | 9 +++----- integrations/iceberg/src/wire.rs | 24 +++++++++------------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 82eabd498f..eb199ebbe6 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -34,7 +34,7 @@ use iceberg_datafusion::{IcebergCatalogTableProvider, IcebergStaticTableProvider use serde::{Deserialize, Serialize}; use crate::catalog::PinnedTable; -use crate::wire::{self, Frame, TableWire}; +use crate::wire::{self, TableWire}; #[derive(Debug, Serialize, Deserialize)] struct ProviderWire { @@ -86,13 +86,10 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { schema: SchemaRef, ctx: &TaskContext, ) -> Result> { - let ProviderWire { table, snapshot_id } = match wire::decode(buf)? { - Frame::Iceberg(provider) => provider, - Frame::Delegated(rest) => { - return self - .inner - .try_decode_table_provider(rest, table_ref, schema, ctx); - } + let Some(ProviderWire { table, snapshot_id }) = wire::decode(buf)? else { + return self + .inner + .try_decode_table_provider(buf, table_ref, schema, ctx); }; Ok(Arc::new(PinnedTable { table: table.load()?, diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index bc8385a9c5..7428e686ce 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -36,7 +36,7 @@ use iceberg::expr::Predicate; use iceberg_datafusion::read::IcebergDataSource; use serde::{Deserialize, Serialize}; -use crate::wire::{self, Frame, TableWire}; +use crate::wire::{self, TableWire}; #[derive(Debug, Serialize, Deserialize)] struct ScanWire { @@ -76,11 +76,8 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { ctx: &TaskContext, proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let scan: ScanWire = match wire::decode(buf)? { - Frame::Iceberg(scan) => scan, - Frame::Delegated(rest) => { - return self.inner.try_decode(rest, inputs, ctx, proto_converter); - } + let Some(scan) = wire::decode::(buf)? else { + return self.inner.try_decode(buf, inputs, ctx, proto_converter); }; let source = IcebergDataSource::builder(scan.table.load()?, scan.schema) .with_snapshot_id(scan.snapshot_id) diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index 0ede773dc4..dcfc897e95 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -177,22 +177,18 @@ pub(crate) fn encode( serde_json::to_writer(buf, payload).map_err(json_err) } -/// A blob's payload: this crate's, decoded, or the inner codec's bytes. -pub(crate) enum Frame<'a, T> { - Iceberg(T), - Delegated(&'a [u8]), -} - -/// Decodes `buf` as `T` if it has this crate's prefix. +/// Decodes `buf` as `T` if it's an Iceberg blob, or returns `None` if it +/// belongs to the inner codec. pub(crate) fn decode( buf: &[u8], -) -> Result, DataFusionError> { - match buf.strip_prefix(PREFIX) { - Some(rest) => serde_json::from_slice(rest) - .map(Frame::Iceberg) - .map_err(|e| exec_datafusion_err!("invalid ballista-iceberg payload: {e}")), - None => Ok(Frame::Delegated(buf)), - } +) -> Result, DataFusionError> { + buf.strip_prefix(PREFIX) + .map(|rest| { + serde_json::from_slice(rest).map_err(|e| { + exec_datafusion_err!("invalid ballista-iceberg payload: {e}") + }) + }) + .transpose() } fn json_err(e: serde_json::Error) -> DataFusionError { From 1105c5952550a8c028306b256fc3500e8ae3fde8 Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 1 Nov 2026 22:02:52 -0500 Subject: [PATCH 12/31] Move ballista-iceberg out of the root workspace, add with_iceberg ballista-iceberg is now its own workspace, excluded from Ballista's like python/, with its own Cargo.lock and CI workflow. Ballista's root Cargo.lock matches main again, so a DataFusion upgrade that iceberg-rust doesn't support yet doesn't block Ballista's build or release. SchedulerConfigExt::with_iceberg (the `scheduler` feature) sets the Iceberg codecs and mounts catalogs on the scheduler's sessions, and ExecutorProcessConfigExt::with_iceberg (the `executor` feature) sets the codecs on an executor. Both wrap whatever the config already has. They replace iceberg_session_builder. --- .github/workflows/iceberg.yml | 69 + .github/workflows/rust.yml | 2 - Cargo.lock | 1105 +-- Cargo.toml | 10 +- docs/source/user-guide/iceberg.md | 44 +- integrations/iceberg/Cargo.lock | 6728 +++++++++++++++++ integrations/iceberg/Cargo.toml | 68 +- .../examples/standalone-iceberg-scan.rs | 2 +- integrations/iceberg/src/executor.rs | 45 + integrations/iceberg/src/lib.rs | 43 +- integrations/iceberg/src/scheduler.rs | 72 + integrations/iceberg/tests/flight_sql.rs | 48 +- 12 files changed, 7031 insertions(+), 1205 deletions(-) create mode 100644 .github/workflows/iceberg.yml create mode 100644 integrations/iceberg/Cargo.lock create mode 100644 integrations/iceberg/src/executor.rs create mode 100644 integrations/iceberg/src/scheduler.rs diff --git a/.github/workflows/iceberg.yml b/.github/workflows/iceberg.yml new file mode 100644 index 0000000000..ef35f0d749 --- /dev/null +++ b/.github/workflows/iceberg.yml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# ballista-iceberg is its own workspace, outside Ballista's, so it doesn't +# block Ballista's build or release; this workflow checks it separately. +name: Ballista Iceberg + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + branches: [ "main" ] + paths: + - "ballista/**" + - "integrations/iceberg/**" + - ".github/workflows/iceberg.yml" + - "!**/*.md" + pull_request: + paths: + - "ballista/**" + - "integrations/iceberg/**" + - ".github/workflows/iceberg.yml" + - "!**/*.md" + workflow_dispatch: + +defaults: + run: + working-directory: integrations/iceberg + +jobs: + test: + name: check and test ballista-iceberg + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 1 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + - uses: swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 #2.9.3 + with: + workspaces: integrations/iceberg + - name: Check formatting + run: cargo fmt --check + - name: Run clippy + run: | + rustup component add clippy + cargo clippy --all-targets --all-features --locked -- -D warnings + - name: Run tests + run: cargo test --all-features --locked diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8c5c9459e4..0dd43fd4c7 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -29,7 +29,6 @@ on: - "ballista-cli/**" - "benchmarks/**" - "examples/**" - - "integrations/**" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" @@ -47,7 +46,6 @@ on: - "ballista-cli/**" - "benchmarks/**" - "examples/**" - - "integrations/**" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" diff --git a/Cargo.lock b/Cargo.lock index d946657304..4c8096239a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,41 +8,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common 0.1.7", - "generic-array", -] - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", -] - -[[package]] -name = "aes-gcm" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - [[package]] name = "ahash" version = "0.8.12" @@ -66,12 +31,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "aliasable" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" - [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -173,33 +132,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "apache-avro" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" -dependencies = [ - "bigdecimal", - "bon", - "crc32fast", - "digest 0.11.3", - "log", - "miniz_oxide 0.9.1", - "num-bigint 0.4.8", - "ouroboros", - "quad-rand", - "rand 0.10.3", - "regex-lite", - "serde", - "serde_bytes", - "serde_json", - "snap", - "strum 0.28.0", - "thiserror 2.0.19", - "uuid", - "zstd", -] - [[package]] name = "approx" version = "0.5.1" @@ -227,12 +159,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "array-init" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" - [[package]] name = "arraydeque" version = "0.5.1" @@ -324,7 +250,7 @@ dependencies = [ "serde", "serde_json", "snap", - "strum_macros 0.28.0", + "strum_macros", "uuid", "zstd", ] @@ -491,7 +417,6 @@ version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" dependencies = [ - "serde", "serde_core", "serde_json", ] @@ -527,12 +452,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "as-any" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" - [[package]] name = "astral-tokio-tar" version = "0.6.4" @@ -561,17 +480,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-lock" -version = "3.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" -dependencies = [ - "event-listener", - "event-listener-strategy", - "pin-project-lite", -] - [[package]] name = "async-recursion" version = "1.1.1" @@ -616,22 +524,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "asyncband" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" -dependencies = [ - "hashbrown 0.17.1", - "slab", -] - -[[package]] -name = "asyncband" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" - [[package]] name = "atoi" version = "2.0.0" @@ -685,7 +577,7 @@ dependencies = [ "fastrand", "hex", "http 1.5.0", - "sha1 0.10.7", + "sha1", "time", "tokio", "tracing", @@ -1111,17 +1003,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "backon" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" -dependencies = [ - "fastrand", - "gloo-timers 0.3.0", - "tokio", -] - [[package]] name = "ballista" version = "55.0.0" @@ -1216,7 +1097,7 @@ dependencies = [ "dirs 7.0.0", "dotparser", "futures", - "gloo-timers 0.4.0", + "gloo-timers", "js-sys", "mimalloc", "prometheus-parse", @@ -1359,32 +1240,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "ballista-iceberg" -version = "55.0.0" -dependencies = [ - "arrow-flight", - "arrow-schema", - "async-trait", - "ballista", - "ballista-core", - "ballista-executor", - "ballista-scheduler", - "datafusion", - "datafusion-proto", - "futures", - "iceberg", - "iceberg-catalog-rest", - "iceberg-datafusion", - "iceberg-storage-opendal", - "serde", - "serde_json", - "tempfile", - "testcontainers", - "tokio", - "tonic", -] - [[package]] name = "ballista-result-service" version = "55.0.0" @@ -1544,15 +1399,8 @@ dependencies = [ "num-bigint 0.4.8", "num-integer", "num-traits", - "serde", ] -[[package]] -name = "bimap" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" - [[package]] name = "bit-set" version = "0.5.3" @@ -1636,16 +1484,6 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "bnum" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" -dependencies = [ - "serde", - "serde-big-array", -] - [[package]] name = "bollard" version = "0.21.1" @@ -1719,29 +1557,6 @@ dependencies = [ "time", ] -[[package]] -name = "bon" -version = "3.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" -dependencies = [ - "bon-macros", -] - -[[package]] -name = "bon-macros" -version = "3.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" -dependencies = [ - "darling 0.24.1", - "ident_case", - "prettyplease 0.3.0", - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "brotli" version = "8.0.4" @@ -1790,12 +1605,6 @@ version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" @@ -1884,10 +1693,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", - "js-sys", "num-traits", "serde", - "wasm-bindgen", "windows-link", ] @@ -1928,16 +1735,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout", -] - [[package]] name = "clap" version = "2.34.0" @@ -2208,16 +2005,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" -[[package]] -name = "crc-fast" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" -dependencies = [ - "digest 0.10.7", - "spin", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -2361,7 +2148,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] @@ -2415,15 +2201,6 @@ dependencies = [ "linktime-proc-macro", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -2433,48 +2210,14 @@ dependencies = [ "cmov", ] -[[package]] -name = "darling" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" -dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", -] - [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", -] - -[[package]] -name = "darling" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" -dependencies = [ - "darling_core 0.24.1", - "darling_macro 0.24.1", -] - -[[package]] -name = "darling_core" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 2.0.119", + "darling_core", + "darling_macro", ] [[package]] @@ -2490,52 +2233,17 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "darling_core" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" -dependencies = [ - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 3.0.3", -] - -[[package]] -name = "darling_macro" -version = "0.20.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" -dependencies = [ - "darling_core 0.20.11", - "quote", - "syn 2.0.119", -] - [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.23.0", + "darling_core", "quote", "syn 2.0.119", ] -[[package]] -name = "darling_macro" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" -dependencies = [ - "darling_core 0.24.1", - "quote", - "syn 3.0.3", -] - [[package]] name = "dashmap" version = "6.2.1" @@ -3398,37 +3106,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "derive_builder" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" -dependencies = [ - "darling 0.20.11", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "derive_builder_macro" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" -dependencies = [ - "derive_builder_core", - "syn 2.0.119", -] - [[package]] name = "derive_more" version = "2.1.1" @@ -3515,15 +3192,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "dlv-list" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" -dependencies = [ - "const-random", -] - [[package]] name = "docker_credential" version = "1.4.0" @@ -3638,17 +3306,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "erased-serde" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" -dependencies = [ - "serde", - "serde_core", - "typeid", -] - [[package]] name = "errno" version = "0.3.14" @@ -3684,26 +3341,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "event-listener" -version = "5.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" -dependencies = [ - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - [[package]] name = "fancy-regex" version = "0.11.0" @@ -3720,18 +3357,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd2e7510819d6fbf51a5545c8f922716ecfb14df168a3242f7d33e0239efe6a1" -[[package]] -name = "fastnum" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" -dependencies = [ - "bnum", - "num-integer", - "num-traits", - "serde", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -3989,34 +3614,12 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ghash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" -dependencies = [ - "opaque-debug", - "polyval", -] - [[package]] name = "glob" version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "gloo-timers" version = "0.4.0" @@ -4148,12 +3751,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -4395,132 +3992,6 @@ dependencies = [ "cc", ] -[[package]] -name = "iceberg" -version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" -dependencies = [ - "aes-gcm", - "anyhow", - "apache-avro", - "array-init", - "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-ord", - "arrow-schema", - "arrow-select", - "arrow-string", - "as-any", - "async-trait", - "backon", - "base64 0.22.1", - "bimap", - "bytes", - "chrono", - "crc32fast", - "derive_builder", - "fastnum", - "flate2", - "fnv", - "form_urlencoded", - "futures", - "iceberg-property-macro", - "itertools 0.13.0", - "moka", - "murmur3", - "once_cell", - "ordered-float 4.6.0", - "parquet", - "rand 0.9.5", - "reqwest 0.12.28", - "roaring", - "serde", - "serde_bytes", - "serde_derive", - "serde_json", - "serde_repr", - "serde_with", - "strum 0.27.2", - "tokio", - "tracing", - "typed-builder", - "typetag", - "url", - "uuid", - "zeroize", - "zstd", -] - -[[package]] -name = "iceberg-catalog-rest" -version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" -dependencies = [ - "async-trait", - "chrono", - "http 1.5.0", - "iceberg", - "itertools 0.13.0", - "reqwest 0.12.28", - "serde", - "serde_derive", - "serde_json", - "tokio", - "tracing", - "typed-builder", - "uuid", -] - -[[package]] -name = "iceberg-datafusion" -version = "0.12.0" -source = "git+https://github.com/apache/datafusion-iceberg?rev=b641e5ebe9e92967499ae7e440ba10d7797712df#b641e5ebe9e92967499ae7e440ba10d7797712df" -dependencies = [ - "async-trait", - "bytes", - "dashmap", - "datafusion", - "futures", - "iceberg", - "parquet", - "serde", - "tokio", - "typetag", - "uuid", -] - -[[package]] -name = "iceberg-property-macro" -version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "iceberg-storage-opendal" -version = "0.10.1" -source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "cfg-if", - "futures", - "iceberg", - "iceberg-property-macro", - "opendal", - "reqsign-aws-v4", - "reqsign-core", - "serde", - "typetag", - "url", -] - [[package]] name = "icu_collections" version = "2.2.0" @@ -4662,15 +4133,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] - [[package]] name = "insta" version = "1.49.0" @@ -4689,7 +4151,7 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971" dependencies = [ - "darling 0.23.0", + "darling", "indoc", "proc-macro2", "quote", @@ -4718,15 +4180,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "inventory" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" -dependencies = [ - "rustversion", -] - [[package]] name = "ipnet" version = "2.12.0" @@ -4790,14 +4243,10 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", - "jiff-tzdb-platform", - "js-sys", "log", "portable-atomic", "portable-atomic-util", "serde_core", - "wasm-bindgen", - "windows-link", ] [[package]] @@ -4821,21 +4270,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "jiff-tzdb" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - [[package]] name = "jni" version = "0.22.4" @@ -5272,38 +4706,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "moka" -version = "0.12.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" -dependencies = [ - "async-lock", - "crossbeam-channel", - "crossbeam-epoch", - "crossbeam-utils", - "equivalent", - "event-listener", - "futures-util", - "parking_lot", - "portable-atomic", - "smallvec", - "tagptr", - "uuid", -] - [[package]] name = "multimap" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "murmur3" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" - [[package]] name = "nibble_vec" version = "0.1.0" @@ -5388,7 +4796,6 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", - "serde", ] [[package]] @@ -5532,7 +4939,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml 0.39.4", + "quick-xml", "rand 0.10.3", "reqwest 0.12.28", "ring", @@ -5567,147 +4974,6 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - -[[package]] -name = "opendal" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" -dependencies = [ - "ctor", - "opendal-core", - "opendal-http-transport-reqwest", - "opendal-layer-concurrent-limit", - "opendal-layer-logging", - "opendal-layer-retry", - "opendal-layer-timeout", - "opendal-service-fs", - "opendal-service-s3", -] - -[[package]] -name = "opendal-core" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" -dependencies = [ - "anyhow", - "asyncband 0.6.7", - "base64 0.23.1", - "bytes", - "futures", - "http 1.5.0", - "jiff", - "log", - "md-5 0.11.0", - "percent-encoding", - "quick-xml 0.41.0", - "reqsign-core", - "serde", - "serde_json", - "tokio", - "url", - "uuid", - "web-time", -] - -[[package]] -name = "opendal-http-transport-reqwest" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" -dependencies = [ - "bytes", - "futures", - "http 1.5.0", - "http-body 1.1.0", - "opendal-core", - "reqwest 0.13.5", -] - -[[package]] -name = "opendal-layer-concurrent-limit" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" -dependencies = [ - "asyncband 0.6.7", - "futures", - "http 1.5.0", - "opendal-core", -] - -[[package]] -name = "opendal-layer-logging" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" -dependencies = [ - "log", - "opendal-core", -] - -[[package]] -name = "opendal-layer-retry" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" -dependencies = [ - "backon", - "log", - "opendal-core", -] - -[[package]] -name = "opendal-layer-timeout" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" -dependencies = [ - "opendal-core", - "tokio", -] - -[[package]] -name = "opendal-service-fs" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" -dependencies = [ - "bytes", - "log", - "opendal-core", - "serde", - "tokio", - "xattr", -] - -[[package]] -name = "opendal-service-s3" -version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" -dependencies = [ - "base64 0.23.1", - "bytes", - "crc-fast", - "http 1.5.0", - "log", - "md-5 0.11.0", - "opendal-core", - "quick-xml 0.41.0", - "reqsign-aws-v4", - "reqsign-core", - "reqsign-file-read-tokio", - "serde", - "url", -] - [[package]] name = "openssl-probe" version = "0.2.1" @@ -5738,40 +5004,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "ordered-multimap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" -dependencies = [ - "dlv-list", - "hashbrown 0.14.5", -] - -[[package]] -name = "ouroboros" -version = "0.18.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" -dependencies = [ - "aliasable", - "ouroboros_macro", - "static_assertions", -] - -[[package]] -name = "ouroboros_macro" -version = "0.18.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "proc-macro2-diagnostics", - "quote", - "syn 2.0.119", -] - [[package]] name = "outref" version = "0.5.2" @@ -5812,12 +5044,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.5" @@ -6146,18 +5372,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", - "universal-hash", -] - [[package]] name = "portable-atomic" version = "1.14.0" @@ -6207,16 +5421,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "prettyplease" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" -dependencies = [ - "proc-macro2", - "syn 3.0.3", -] - [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -6259,19 +5463,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "proc-macro2-diagnostics" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "version_check", - "yansi", -] - [[package]] name = "procfs" version = "0.17.0" @@ -6344,7 +5535,7 @@ dependencies = [ "log", "multimap", "petgraph", - "prettyplease 0.2.37", + "prettyplease", "prost", "prost-types", "pulldown-cmark", @@ -6426,12 +5617,6 @@ dependencies = [ "pulldown-cmark", ] -[[package]] -name = "quad-rand" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" - [[package]] name = "quick-xml" version = "0.39.4" @@ -6442,26 +5627,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quick-xml" -version = "0.41.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quinn" version = "0.11.11" @@ -6601,9 +5766,6 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] [[package]] name = "rand_core" @@ -6661,7 +5823,7 @@ dependencies = [ "lru 0.18.5", "palette", "serde", - "strum 0.28.0", + "strum", "thiserror 2.0.19", "unicode-segmentation", "unicode-truncate", @@ -6737,7 +5899,7 @@ dependencies = [ "line-clipping", "ratatui-core", "serde", - "strum 0.28.0", + "strum", "time", "unicode-segmentation", "unicode-width 0.2.2", @@ -6890,74 +6052,6 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" -[[package]] -name = "reqsign-aws-core" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" -dependencies = [ - "bytes", - "form_urlencoded", - "hex", - "http 1.5.0", - "log", - "percent-encoding", - "quick-xml 0.42.0", - "reqsign-core", - "rust-ini", - "serde", - "serde_json", - "serde_urlencoded", - "sha1 0.11.0", -] - -[[package]] -name = "reqsign-aws-v4" -version = "3.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" -dependencies = [ - "bytes", - "http 1.5.0", - "log", - "quick-xml 0.42.0", - "reqsign-aws-core", - "reqsign-core", - "serde", -] - -[[package]] -name = "reqsign-core" -version = "3.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" -dependencies = [ - "anyhow", - "asyncband 0.7.3", - "base64 0.23.1", - "bytes", - "hex", - "hmac", - "http 1.5.0", - "jiff", - "log", - "percent-encoding", - "sha1 0.11.0", - "sha2 0.11.0", - "windows-sys 0.61.2", -] - -[[package]] -name = "reqsign-file-read-tokio" -version = "3.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" -dependencies = [ - "anyhow", - "reqsign-core", - "tokio", -] - [[package]] name = "reqwest" version = "0.12.28" @@ -6996,7 +6090,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.4.2", + "wasm-streams", "web-sys", ] @@ -7010,7 +6104,6 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", - "futures-util", "h2", "http 1.5.0", "http-body 1.1.0", @@ -7032,14 +6125,12 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.5.0", "web-sys", ] @@ -7057,16 +6148,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "roaring" -version = "0.11.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" -dependencies = [ - "bytemuck", - "byteorder", -] - [[package]] name = "rstest" version = "0.28.0" @@ -7096,16 +6177,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "rust-ini" -version = "0.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - [[package]] name = "rustc-hash" version = "2.1.3" @@ -7384,25 +6455,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-big-array" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" -dependencies = [ - "serde", -] - -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.229" @@ -7521,7 +6573,7 @@ version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling 0.23.0", + "darling", "proc-macro2", "quote", "syn 2.0.119", @@ -7551,17 +6603,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -7701,12 +6742,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "spin" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" - [[package]] name = "sqlparser" version = "0.62.0" @@ -7807,34 +6842,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "strum" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" -dependencies = [ - "strum_macros 0.27.2", -] - [[package]] name = "strum" version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" dependencies = [ - "strum_macros 0.28.0", -] - -[[package]] -name = "strum_macros" -version = "0.27.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.119", + "strum_macros", ] [[package]] @@ -7860,7 +6874,7 @@ dependencies = [ "pbjson", "pbjson-build", "pbjson-types", - "prettyplease 0.2.37", + "prettyplease", "prost", "prost-build", "prost-types", @@ -7975,12 +6989,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tagptr" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" - [[package]] name = "tap" version = "1.0.1" @@ -8384,7 +7392,7 @@ version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4" dependencies = [ - "prettyplease 0.2.37", + "prettyplease", "proc-macro2", "quote", "syn 2.0.119", @@ -8407,7 +7415,7 @@ version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0e0e10c4980146d75d5acfe4f7a9822278130ca2025c0dc25c82f7307d63ac" dependencies = [ - "prettyplease 0.2.37", + "prettyplease", "proc-macro2", "prost-build", "prost-types", @@ -8593,62 +7601,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" -[[package]] -name = "typed-builder" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "typeid" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" - [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" -[[package]] -name = "typetag" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" -dependencies = [ - "erased-serde", - "inventory", - "once_cell", - "serde", - "typetag-impl", -] - -[[package]] -name = "typetag-impl" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "typify" version = "0.5.0" @@ -8746,16 +7704,6 @@ dependencies = [ "rustc-std-workspace-core", ] -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common 0.1.7", - "subtle", -] - [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -8973,19 +7921,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "wasm-streams" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "web-sys" version = "0.3.103" @@ -9389,12 +8324,6 @@ dependencies = [ "hashlink", ] -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 2a444ffc77..7b1a511b4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [workspace] -exclude = ["dev/msrvcheck", "python"] +exclude = ["dev/msrvcheck", "integrations/iceberg", "python"] members = [ "ballista-cli", "ballista/api-types", @@ -30,7 +30,6 @@ members = [ "benchmarks", "chaos-testing", "examples", - "integrations/iceberg", ] resolver = "3" @@ -67,13 +66,6 @@ datafusion-proto = "55" datafusion-proto-common = "55" datafusion-substrait = "55" -# Not released yet. Keep `iceberg` at the revision iceberg-datafusion's own -# workspace pins, so cargo resolves a single iceberg crate. -iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg", rev = "b641e5ebe9e92967499ae7e440ba10d7797712df" } -iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } - ctor = { version = "1.0" } insta = "1.47" itertools = "0.15" diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 6548427769..347ebce0f6 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -26,7 +26,11 @@ codecs that send Iceberg tables and scans between the client, the scheduler and the executors. The crate is not published yet. It depends on unreleased revisions of -iceberg-datafusion and iceberg-rust, and needs Rust 1.95. +iceberg-datafusion and iceberg-rust, and needs Rust 1.95. It is its own +workspace, with its own `Cargo.lock` and CI, outside Ballista's, so a +DataFusion upgrade in Ballista that iceberg-rust doesn't support yet doesn't +block Ballista's build or release. Build and test it from +`integrations/iceberg`. ## Usage @@ -61,33 +65,36 @@ The example registers both on a table in a temporary local warehouse, and shows a commit reaching the live table but not the static one: ```shell -cargo run -p ballista-iceberg --example standalone-iceberg-scan +cd integrations/iceberg +cargo run --example standalone-iceberg-scan ``` ## SQL clients Flight SQL clients (JDBC, ADBC, BI tools) send SQL to the scheduler, which plans it, so they hold no catalog. To let them query Iceberg tables, mount catalogs -on the scheduler with `iceberg_session_builder`, which wraps the scheduler's -session builder (`default_session_builder` here, or your own), and give the -scheduler and executors the Iceberg codecs: +on the scheduler with `SchedulerConfigExt::with_iceberg` (the `scheduler` +feature), which also sets the Iceberg codecs: ```rust -use ballista_core::utils::default_session_builder; -use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder}; +use ballista_iceberg::SchedulerConfigExt; let rest = IcebergCatalogProvider::try_new(catalog).await?; -let mut config = SchedulerConfig::default() +let config = SchedulerConfig::default() .with_flight_sql(true) - .with_override_session_builder(iceberg_session_builder( - Arc::new(default_session_builder), - [("rest".to_string(), Arc::new(rest) as _)], - )); -config.override_logical_codec = Some(Arc::new(IcebergLogicalCodec::default())); -config.override_physical_codec = Some(Arc::new(IcebergPhysicalCodec::default())); + .with_iceberg([("rest", Arc::new(rest) as _)]); ``` -Executors set the same two codecs on their `ExecutorProcessConfig`. A client can +Executors set the codecs with `ExecutorProcessConfigExt::with_iceberg` (the +`executor` feature): + +```rust +use ballista_iceberg::ExecutorProcessConfigExt; + +let config = ExecutorProcessConfig::default().with_iceberg(); +``` + +Both wrap any session builder or codecs the config already has. A client can then run `SELECT * FROM rest.ns.t`. Tables are pinned the same way as on the client, while the scheduler plans the query. @@ -215,10 +222,9 @@ Every node still needs every codec that a query's tables use. once: each run reads the version from when it was planned. To read newer commits, recreate the view or plan the query again. - **A separately deployed scheduler and executors** must link - `ballista-iceberg` and set the Iceberg codecs as their - `override_logical_codec` and `override_physical_codec` (see - [Extending Components](extending-components.md)). The tests cover standalone - mode and an in-process scheduler serving Flight SQL. + `ballista-iceberg` and be readied with `with_iceberg` (see + [SQL clients](#sql-clients)). The tests cover standalone mode and an + in-process scheduler serving Flight SQL. - **A mounted catalog lists its tables once.** `IcebergCatalogProvider` reads the table names when it is built, so a table created afterwards isn't visible until the catalog is mounted again. diff --git a/integrations/iceberg/Cargo.lock b/integrations/iceberg/Cargo.lock new file mode 100644 index 0000000000..ff381735f2 --- /dev/null +++ b/integrations/iceberg/Cargo.lock @@ -0,0 +1,6728 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.11.3", + "log", + "miniz_oxide 0.9.1", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.28.0", + "thiserror", + "uuid", + "zstd", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4087686b4b0a3427190bae57a1d9a478dbb2d40c5dc1bd6e2b6d797913bdd348" +dependencies = [ + "object", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b543210778ac15d5f7a10153dc39e099f1eee756363589a842029e6c5c76e423" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64 0.23.1", + "bytes", + "futures", + "once_cell", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "arrow-ipc" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap 2.14.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" +dependencies = [ + "serde", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "astral-tokio-tar" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "async-compression" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "ballista" +version = "55.0.0" +dependencies = [ + "async-trait", + "ballista-core", + "ballista-executor", + "ballista-scheduler", + "datafusion", + "log", + "tokio", + "url", +] + +[[package]] +name = "ballista-api-types" +version = "55.0.0" +dependencies = [ + "serde", + "utoipa", +] + +[[package]] +name = "ballista-core" +version = "55.0.0" +dependencies = [ + "arrow-flight", + "async-trait", + "chrono", + "clap", + "datafusion", + "datafusion-proto", + "datafusion-proto-common", + "futures", + "log", + "lru", + "object_store", + "parking_lot", + "pastey", + "prost", + "rand 0.10.3", + "rustc_version", + "serde", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "tonic-prost-build", + "url", + "utoipa", + "uuid", +] + +[[package]] +name = "ballista-executor" +version = "55.0.0" +dependencies = [ + "arrow", + "arrow-flight", + "async-trait", + "ballista-core", + "bytesize", + "dashmap", + "datafusion", + "datafusion-proto", + "futures", + "libc", + "log", + "lru", + "memory-stats", + "parking_lot", + "serde", + "sysinfo", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "uuid", +] + +[[package]] +name = "ballista-flight-sql" +version = "55.0.0" +dependencies = [ + "arrow", + "arrow-flight", + "async-trait", + "ballista-core", + "dashmap", + "datafusion", + "futures", + "log", + "prost", + "tokio", + "tonic", + "uuid", +] + +[[package]] +name = "ballista-history" +version = "55.0.0" +dependencies = [ + "ballista-api-types", + "log", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "ballista-iceberg" +version = "55.0.0" +dependencies = [ + "arrow-flight", + "arrow-schema", + "async-trait", + "ballista", + "ballista-core", + "ballista-executor", + "ballista-scheduler", + "datafusion", + "datafusion-proto", + "futures", + "iceberg", + "iceberg-catalog-rest", + "iceberg-datafusion", + "iceberg-storage-opendal", + "serde", + "serde_json", + "tempfile", + "testcontainers", + "tokio", + "tonic", +] + +[[package]] +name = "ballista-scheduler" +version = "55.0.0" +dependencies = [ + "arrow-flight", + "async-trait", + "axum", + "ballista-api-types", + "ballista-core", + "ballista-flight-sql", + "ballista-history", + "clap", + "dashmap", + "datafusion", + "datafusion-proto", + "ferroid", + "futures", + "http", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "prost", + "rand 0.10.3", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tonic", + "tower-http 0.7.1", + "tracing", + "tracing-appender", + "tracing-subscriber", + "url-escape", + "utoipa", + "uuid", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bollard" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" +dependencies = [ + "async-stream", + "base64 0.22.1", + "bitflags 2.13.1", + "bollard-buildkit-proto", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "num", + "pin-project-lite", + "rand 0.10.3", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-buildkit-proto" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5c97450e79c7c565302dd92e86b08823b47550fcb4fc5ce910194d1b087a1a3" +dependencies = [ + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "bollard-stubs" +version = "1.53.1-rc.29.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +dependencies = [ + "base64 0.22.1", + "bollard-buildkit-proto", + "bytes", + "prost", + "serde", + "serde_json", + "serde_repr", + "time", +] + +[[package]] +name = "bon" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214f5df094ce551a10a30ffb9c70243d61f121a3d985a6495933e181dee6a7d2" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2706da6c749998cc555d04184909608956a69ad3f8ab1a076fbc867b4a3c3ce5" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.3", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap 2.14.0", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "percent-encoding", +] + +[[package]] +name = "datafusion-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.15.0", + "libc", + "log", + "num-traits", + "object_store", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "itertools 0.15.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" + +[[package]] +name = "datafusion-execution" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "bytes", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "pin-project-lite", + "rand 0.9.5", + "tempfile", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "indexmap 2.14.0", + "itertools 0.15.0", + "recursive", + "serde", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.14.0", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-functions" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" +dependencies = [ + "arrow", + "arrow-buffer", + "base64 0.23.1", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.15.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.15.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" +dependencies = [ + "datafusion-doc", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "datafusion-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.14.0", + "itertools 0.15.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "datafusion-proto-models", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.15.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-proto-models", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.15.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "itertools 0.15.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.0", + "itertools 0.15.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" +dependencies = [ + "arrow", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "datafusion-proto-models", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + +[[package]] +name = "datafusion-proto-models" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost", +] + +[[package]] +name = "datafusion-pruning" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap 2.14.0", + "log", + "recursive", + "regex", + "sqlparser", + "stacker", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "docker_credential" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "futures", + "portable-atomic", + "rand 0.10.3", + "tokio", + "web-time", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.1", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.9", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-named-pipe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fab3637d6b04a8037af8a266fdf6cf92ea957e8c53981a2bf6136572531025bf" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float", + "parquet", + "rand 0.9.5", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd", +] + +[[package]] +name = "iceberg-catalog-rest" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "async-trait", + "chrono", + "http", + "iceberg", + "itertools 0.13.0", + "reqwest 0.12.28", + "serde", + "serde_derive", + "serde_json", + "tokio", + "tracing", + "typed-builder", + "uuid", +] + +[[package]] +name = "iceberg-datafusion" +version = "0.12.0" +source = "git+https://github.com/apache/datafusion-iceberg?rev=b641e5ebe9e92967499ae7e440ba10d7797712df#b641e5ebe9e92967499ae7e440ba10d7797712df" +dependencies = [ + "async-trait", + "bytes", + "dashmap", + "datafusion", + "futures", + "iceberg", + "parquet", + "serde", + "tokio", + "typetag", + "uuid", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3#5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "iceberg-property-macro", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "liblzma" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45aec2360b3933207e27908049d8e4df4e476b58180afb1e56b2a4fb72efe4ba" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a046c7f353ba30f810545151e04f63545833803f5b86ee3ddf1517247fe560a5" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5 0.10.6", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.3", + "reqwest 0.12.28", + "ring", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-fs", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-fs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "59.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.23.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "object_store", + "ring", + "seq-macro", + "simdutf8", + "snap", + "tokio", + "twox-hash", + "zstd", +] + +[[package]] +name = "parse-display" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78deb158fb1d73b29efb4b7e9b9860b78059c670de06bd28df8d0b458ded0eb" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e95a50d1084dab562913062c4c34bb204b68fc6ec38a1395909ff5aaaf4f10a" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn 2.0.119", +] + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.14.0", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.3", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck 0.5.0", + "itertools 0.13.0", + "log", + "multimap", + "petgraph", + "prettyplease 0.2.37", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "psm" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645dbe486e346d9b5de3ef16ede18c26e6c70ad97418f4874b8b1889d6e761ea" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags 2.13.1", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.59.0", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "hex", + "hmac", + "http", + "jiff", + "log", + "percent-encoding", + "sha1", + "sha2", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "640c8cdd92b6b12f5bcb1803ca3bbf5ab96e5e6b6b96b9ab77dabe9e880b3190" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn 2.0.119", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "testcontainers" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2bbe381afaaa58ea610c5fc3ffb2184063a32b3e358a179f0b4865dd59934a" +dependencies = [ + "astral-tokio-tar", + "async-trait", + "bollard", + "bytes", + "docker_credential", + "either", + "etcetera", + "ferroid", + "futures", + "http", + "itertools 0.14.0", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "reqwest 0.13.5", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-util", + "url", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0e0e10c4980146d75d5acfe4f7a9822278130ca2025c0dc25c82f7307d63ac" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.0", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +dependencies = [ + "bitflags 2.13.1", + "bytes", + "http", + "percent-encoding", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", + "serde_derive", +] + +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utoipa" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765fe27aeff71012a3f90fa474cf1df863a7d0dd81ed25de4e63fff2544994f" +dependencies = [ + "indexmap 2.14.0", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-gen" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d935f1c83fdf8b88f09bbe050d0cea78ea4afee6d7b0317403228c1200b2c8ab" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index edbcfea76a..4ec947e737 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -18,49 +18,65 @@ [package] name = "ballista-iceberg" description = "Apache Iceberg support for Ballista" -license = { workspace = true } -version = { workspace = true } -homepage = { workspace = true } -repository = { workspace = true } -authors = { workspace = true } +license = "Apache-2.0" +version = "55.0.0" +homepage = "https://datafusion.apache.org/ballista/" +repository = "https://github.com/apache/datafusion-ballista" +authors = ["Apache DataFusion "] keywords = ["iceberg", "ballista", "datafusion", "distributed"] categories = ["database"] -edition = { workspace = true } -# Required by the iceberg crates; the rest of the workspace still builds on 1.94. +edition = "2024" +# Required by the iceberg crates; Ballista itself still builds on 1.94. rust-version = "1.95" # crates.io rejects the git dependencies on iceberg and iceberg-datafusion. publish = false +# Its own workspace, outside Ballista's, so a DataFusion upgrade in Ballista +# that iceberg-rust doesn't support yet doesn't block Ballista's build or +# release. Built and tested by .github/workflows/iceberg.yml. +[workspace] + [package.metadata.cargo-machete] # Not imported: enables the `serde` feature, to send a scan's Arrow schema. ignored = ["arrow-schema"] [dependencies] arrow-schema = { version = "59.2.0", features = ["serde"] } -async-trait = { workspace = true } -ballista-core = { workspace = true } -datafusion = { workspace = true } -datafusion-proto = { workspace = true } -futures = { workspace = true } -iceberg = { workspace = true } -iceberg-datafusion = { workspace = true } +async-trait = "0.1" +ballista-core = { path = "../../ballista/core", version = "55.0.0" } +ballista-executor = { path = "../../ballista/executor", version = "55.0.0", default-features = false, optional = true } +ballista-scheduler = { path = "../../ballista/scheduler", version = "55.0.0", default-features = false, optional = true } +datafusion = "55" +datafusion-proto = "55" +futures = "0.3" +# Not released yet. Keep `iceberg` at the revision iceberg-datafusion's own +# workspace pins, so cargo resolves a single iceberg crate. +iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg", rev = "b641e5ebe9e92967499ae7e440ba10d7797712df" } # Storage for rebuilt tables: S3 and local files by default. -iceberg-storage-opendal = { workspace = true } -serde = { workspace = true, features = ["derive", "rc"] } -serde_json = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread"] } +iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +serde = { version = "1.0", features = ["derive", "rc"] } +serde_json = "1" +tokio = { version = "1", features = ["rt-multi-thread"] } [dev-dependencies] -arrow-flight = { workspace = true } -ballista = { workspace = true, features = ["standalone"] } -ballista-executor = { workspace = true } -ballista-scheduler = { workspace = true, features = ["flight-sql"] } -iceberg-catalog-rest = { workspace = true } -tempfile = { workspace = true } +arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } +ballista = { path = "../../ballista/client", version = "55.0.0", features = ["standalone"] } +ballista-scheduler = { path = "../../ballista/scheduler", version = "55.0.0", features = ["flight-sql"] } +iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } +tempfile = "3.27" testcontainers = { version = "0.28", features = ["http_wait_plain"] } -tokio = { workspace = true, features = ["macros"] } -tonic = { workspace = true } +tokio = { version = "1", features = ["macros"] } +tonic = "0.14" [features] +# `ExecutorProcessConfigExt`, to ready an executor for Iceberg. +executor = ["dep:ballista-executor"] +# `SchedulerConfigExt`, to ready a scheduler for Iceberg. +scheduler = ["dep:ballista-scheduler"] # Tests that need docker. testcontainers = [] + +[[test]] +name = "flight_sql" +required-features = ["executor", "scheduler"] diff --git a/integrations/iceberg/examples/standalone-iceberg-scan.rs b/integrations/iceberg/examples/standalone-iceberg-scan.rs index f4546b3152..43a86e15c6 100644 --- a/integrations/iceberg/examples/standalone-iceberg-scan.rs +++ b/integrations/iceberg/examples/standalone-iceberg-scan.rs @@ -21,7 +21,7 @@ //! nothing running: //! //! ```text -//! cargo run -p ballista-iceberg --example standalone-iceberg-scan +//! cargo run --example standalone-iceberg-scan //! ``` use std::collections::HashMap; diff --git a/integrations/iceberg/src/executor.rs b/integrations/iceberg/src/executor.rs new file mode 100644 index 0000000000..432aa6c29b --- /dev/null +++ b/integrations/iceberg/src/executor.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Readies a separately deployed executor for Iceberg. + +use std::sync::Arc; + +use ballista_executor::executor_process::ExecutorProcessConfig; + +use crate::{IcebergLogicalCodec, IcebergPhysicalCodec}; + +/// Readies an [`ExecutorProcessConfig`] for Iceberg. +pub trait ExecutorProcessConfigExt { + /// Sets the Iceberg codecs, wrapping any codecs already set. + fn with_iceberg(self) -> Self; +} + +impl ExecutorProcessConfigExt for ExecutorProcessConfig { + fn with_iceberg(mut self) -> Self { + let logical = self.override_logical_codec.take(); + let physical = self.override_physical_codec.take(); + self.override_logical_codec = Some(Arc::new( + logical.map_or_else(IcebergLogicalCodec::default, IcebergLogicalCodec::new), + )); + self.override_physical_codec = Some(Arc::new( + physical + .map_or_else(IcebergPhysicalCodec::default, IcebergPhysicalCodec::new), + )); + self + } +} diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 5d55a4e9a1..5ff6eef1d2 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -22,7 +22,9 @@ //! without a catalog. A catalog-backed table is sent as the version its catalog //! served when the query was planned, a static one as the version it holds. //! Only the node that plans a query talks to a catalog: the client, or, for -//! Flight SQL clients, a scheduler built with [`iceberg_session_builder`]. +//! Flight SQL clients, a scheduler readied with `SchedulerConfigExt` (the +//! `scheduler` feature). Executors are readied with `ExecutorProcessConfigExt` +//! (the `executor` feature). //! //! Storage comes from [`iceberg_storage_opendal`]: S3 and local files by //! default, other stores with its `opendal-*` features. @@ -57,24 +59,29 @@ //! - Storage credentials travel in plain text. mod catalog; +#[cfg(feature = "executor")] +mod executor; mod logical_codec; mod physical_codec; +#[cfg(feature = "scheduler")] +mod scheduler; mod wire; use std::sync::Arc; use ballista_core::extension::SessionConfigExt; -use datafusion::catalog::CatalogProvider; -use datafusion::common::Result; use datafusion::execution::{SessionState, SessionStateBuilder}; -use datafusion::prelude::SessionConfig; // Linked for the storage it registers; nothing here calls it. use iceberg_storage_opendal as _; use crate::catalog::PinningCatalogList; +#[cfg(feature = "executor")] +pub use crate::executor::ExecutorProcessConfigExt; pub use crate::logical_codec::IcebergLogicalCodec; pub use crate::physical_codec::IcebergPhysicalCodec; +#[cfg(feature = "scheduler")] +pub use crate::scheduler::SchedulerConfigExt; /// Readies a client's session for Iceberg: registers the codecs, and pins /// each catalog-backed table a query names to the version its catalog serves @@ -99,31 +106,3 @@ pub fn register_iceberg(mut state: SessionState) -> SessionState { .with_catalog_list(Arc::new(catalogs)) .build() } - -/// A scheduler's session builder, the shape `SchedulerConfig` stores. -type SessionBuilder = Arc Result + Send + Sync>; - -/// Wraps a scheduler's session builder, `base`, so its sessions have -/// `catalogs` mounted and are readied with [`register_iceberg`], for queries -/// the scheduler plans itself: those from Flight SQL clients, which hold no -/// catalog of their own. Pass `Arc::new(default_session_builder)` as `base` if -/// the scheduler has no session builder of its own. -/// -/// Pass the result to `SchedulerConfig::with_override_session_builder`. The -/// scheduler and executors also need the Iceberg codecs, set as their -/// `override_logical_codec` and `override_physical_codec`. -pub fn iceberg_session_builder( - base: SessionBuilder, - catalogs: impl IntoIterator)>, -) -> SessionBuilder { - let catalogs: Vec<_> = catalogs.into_iter().collect(); - Arc::new(move |config| { - let state = register_iceberg(base(config)?); - for (name, catalog) in &catalogs { - state - .catalog_list() - .register_catalog(name.clone(), catalog.clone()); - } - Ok(state) - }) -} diff --git a/integrations/iceberg/src/scheduler.rs b/integrations/iceberg/src/scheduler.rs new file mode 100644 index 0000000000..5d705da0eb --- /dev/null +++ b/integrations/iceberg/src/scheduler.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Readies a separately deployed scheduler for Iceberg. + +use std::sync::Arc; + +use ballista_core::utils::default_session_builder; +use ballista_scheduler::config::SchedulerConfig; +use datafusion::catalog::CatalogProvider; + +use crate::{IcebergLogicalCodec, IcebergPhysicalCodec, register_iceberg}; + +/// Readies a [`SchedulerConfig`] for Iceberg. +pub trait SchedulerConfigExt { + /// Sets the Iceberg codecs, wrapping any codecs already set, and mounts + /// `catalogs` in each session, readied with [`register_iceberg`]. The + /// scheduler plans queries from Flight SQL clients itself, so they can + /// then query those catalogs. + fn with_iceberg>( + self, + catalogs: impl IntoIterator)>, + ) -> Self; +} + +impl SchedulerConfigExt for SchedulerConfig { + fn with_iceberg>( + mut self, + catalogs: impl IntoIterator)>, + ) -> Self { + let logical = self.override_logical_codec.take(); + let physical = self.override_physical_codec.take(); + self.override_logical_codec = Some(Arc::new( + logical.map_or_else(IcebergLogicalCodec::default, IcebergLogicalCodec::new), + )); + self.override_physical_codec = Some(Arc::new( + physical + .map_or_else(IcebergPhysicalCodec::default, IcebergPhysicalCodec::new), + )); + let base = self + .override_session_builder + .take() + .unwrap_or_else(|| Arc::new(default_session_builder)); + let catalogs: Vec<(String, _)> = catalogs + .into_iter() + .map(|(name, catalog)| (name.into(), catalog)) + .collect(); + self.with_override_session_builder(Arc::new(move |config| { + let state = register_iceberg(base(config)?); + for (name, catalog) in &catalogs { + state + .catalog_list() + .register_catalog(name.clone(), catalog.clone()); + } + Ok(state) + })) + } +} diff --git a/integrations/iceberg/tests/flight_sql.rs b/integrations/iceberg/tests/flight_sql.rs index 14b617b4c5..c730032c81 100644 --- a/integrations/iceberg/tests/flight_sql.rs +++ b/integrations/iceberg/tests/flight_sql.rs @@ -29,14 +29,13 @@ mod flight_sql { use ballista_core::config::TaskSchedulingPolicy; use ballista_core::serde::BallistaCodec; use ballista_core::serde::protobuf::scheduler_grpc_client::SchedulerGrpcClient; - use ballista_core::utils::{default_config_producer, default_session_builder}; - use ballista_iceberg::{ - IcebergLogicalCodec, IcebergPhysicalCodec, iceberg_session_builder, - }; + use ballista_executor::executor_process::ExecutorProcessConfig; + use ballista_iceberg::{ExecutorProcessConfigExt, SchedulerConfigExt}; use ballista_scheduler::cluster::BallistaCluster; use ballista_scheduler::config::SchedulerConfig; - use ballista_scheduler::metrics::default_metrics_collector; - use ballista_scheduler::scheduler_process::start_grpc_service_with_listener; + use ballista_scheduler::scheduler_process::{ + create_scheduler, start_grpc_service_with_listener, + }; use ballista_scheduler::scheduler_server::SchedulerServer; use datafusion::arrow::array::RecordBatch; use datafusion::assert_batches_eq; @@ -89,39 +88,32 @@ mod flight_sql { async fn start_cluster( catalogs: Vec<(String, Arc)>, ) -> Result> { - let codec = || { - BallistaCodec::new( - Arc::new(IcebergLogicalCodec::default()), - Arc::new(IcebergPhysicalCodec::default()), - ) - }; - let cluster = BallistaCluster::new_memory( - "localhost:50050", - iceberg_session_builder(Arc::new(default_session_builder), catalogs), - Arc::new(default_config_producer), - ); let config = SchedulerConfig::default() .with_scheduler_policy(TaskSchedulingPolicy::PullStaged) - .with_flight_sql(true); - let mut scheduler: SchedulerServer = - SchedulerServer::new( - "localhost:50050".to_owned(), - cluster, - codec(), - Arc::new(config), - default_metrics_collector()?, - ); - scheduler.init().await?; + .with_flight_sql(true) + .with_iceberg(catalogs); + let cluster = BallistaCluster::new_from_config(&config).await?; + let scheduler: SchedulerServer = + create_scheduler(cluster, Arc::new(config)).await?; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; let url = format!("http://{}", listener.local_addr()?); tokio::spawn(start_grpc_service_with_listener(listener, scheduler)); let channel = Channel::from_shared(url)?.connect().await?; + let executor = ExecutorProcessConfig::default().with_iceberg(); + let codec = BallistaCodec::new( + executor + .override_logical_codec + .expect("set by with_iceberg"), + executor + .override_physical_codec + .expect("set by with_iceberg"), + ); ballista_executor::new_standalone_executor( SchedulerGrpcClient::new(channel.clone()), 2, - codec(), + codec, ) .await?; let mut client = FlightSqlServiceClient::new(channel); From 74521cabb7d4b8491bfab246d3d517c0814e68ac Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 1 Nov 2026 22:34:20 -0500 Subject: [PATCH 13/31] Limit the Iceberg workflow's token to reading contents --- .github/workflows/iceberg.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/iceberg.yml b/.github/workflows/iceberg.yml index ef35f0d749..3cbe60d50d 100644 --- a/.github/workflows/iceberg.yml +++ b/.github/workflows/iceberg.yml @@ -40,6 +40,9 @@ on: - "!**/*.md" workflow_dispatch: +permissions: + contents: read + defaults: run: working-directory: integrations/iceberg From 2519c7a03148cdef822144cf4b70e3e9b2aff394 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 3 Nov 2026 08:27:49 -0500 Subject: [PATCH 14/31] Compose the Iceberg codecs after the session's codecs --- docs/source/user-guide/iceberg.md | 91 ++------------ integrations/iceberg/src/executor.rs | 24 ++-- integrations/iceberg/src/lib.rs | 12 +- integrations/iceberg/src/logical_codec.rs | 115 ++++------------- integrations/iceberg/src/physical_codec.rs | 140 ++++++--------------- integrations/iceberg/src/scheduler.rs | 24 ++-- integrations/iceberg/src/wire.rs | 27 +--- integrations/iceberg/tests/codec.rs | 123 +++++++++--------- 8 files changed, 175 insertions(+), 381 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 347ebce0f6..f3803d2fca 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -94,7 +94,7 @@ use ballista_iceberg::ExecutorProcessConfigExt; let config = ExecutorProcessConfig::default().with_iceberg(); ``` -Both wrap any session builder or codecs the config already has. A client can +Both keep any session builder or codecs the config already has. A client can then run `SELECT * FROM rest.ns.t`. Tables are pinned the same way as on the client, while the scheduler plans the query. @@ -121,88 +121,13 @@ scheduler for SQL clients. Executors never need catalog configuration. ## Combining with other table formats -The Iceberg codecs wrap the codecs a session already has, so one query can -read Iceberg tables alongside tables of another format, such as Delta or Hudi, -whose codecs are maintained elsewhere. Any codec that follows the same -convention can be stacked with these in any order: - -1. **Wrap an inner codec.** Take the session's existing codec in the - constructor rather than Ballista's default, and pass it everything that - isn't yours. -2. **Prefix your own bytes with a unique name.** Iceberg payloads start with - `iceberg`. Choose a name for your format, such as `delta`, that no other - codec uses and that doesn't start another codec's name. -3. **Pass everything else through untouched.** When encoding something that - isn't yours, write only what the inner codec writes, without adding a - marker. When decoding, if the bytes don't start with your prefix, hand all - of them to the inner codec. - -A logical codec for a hypothetical Delta integration would look like this: - -```rust -/// Rule 2: the name that starts every payload this codec writes. -const PREFIX: &[u8] = b"delta"; - -#[derive(Debug)] -pub struct DeltaLogicalCodec { - inner: Arc, -} - -impl DeltaLogicalCodec { - /// Rule 1: wrap whatever codec the session already has. - pub fn new(inner: Arc) -> Self { - Self { inner } - } -} - -impl LogicalExtensionCodec for DeltaLogicalCodec { - fn try_encode_table_provider( - &self, - table_ref: &TableReference, - node: Arc, - buf: &mut Vec, - ) -> Result<()> { - let Some(table) = node.downcast_ref::() else { - // Rule 3: not ours, so write only what the inner codec writes. - return self.inner.try_encode_table_provider(table_ref, node, buf); - }; - buf.extend_from_slice(PREFIX); - encode_delta_table(table, buf) - } - - fn try_decode_table_provider( - &self, - buf: &[u8], - table_ref: &TableReference, - schema: SchemaRef, - ctx: &TaskContext, - ) -> Result> { - match buf.strip_prefix(PREFIX) { - Some(payload) => decode_delta_table(payload, schema), - // Rule 3: not ours, so hand over every byte unchanged. - None => self.inner.try_decode_table_provider(buf, table_ref, schema, ctx), - } - } - - // Every other method forwards to `self.inner`, for example: - fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udf(name, buf) - } - // ... -} -``` - -A physical codec follows the same pattern in `try_encode` and `try_decode` -for its scan. Because each codec only claims bytes with its own prefix, the -codecs can be stacked in either order: - -```rust -let logical = IcebergLogicalCodec::new(Arc::new(DeltaLogicalCodec::new( - Arc::new(BallistaLogicalExtensionCodec::default()), -))); -``` - -Every node still needs every codec that a query's tables use. +`register_iceberg` and `with_iceberg` add the Iceberg codecs after the codecs +already set, with `ComposedLogicalExtensionCodec` and DataFusion's +`ComposedPhysicalExtensionCodec` (see +[Combining Codecs](extending-components.md#combining-codecs)). To read tables +of another format too, such as Delta, set its codecs before calling them. +Clients, the scheduler and executors must all set the same codecs in the same +order. ## Limitations diff --git a/integrations/iceberg/src/executor.rs b/integrations/iceberg/src/executor.rs index 432aa6c29b..edfc0aa7b4 100644 --- a/integrations/iceberg/src/executor.rs +++ b/integrations/iceberg/src/executor.rs @@ -19,27 +19,31 @@ use std::sync::Arc; +use ballista_core::serde::{ + BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, +}; use ballista_executor::executor_process::ExecutorProcessConfig; use crate::{IcebergLogicalCodec, IcebergPhysicalCodec}; /// Readies an [`ExecutorProcessConfig`] for Iceberg. pub trait ExecutorProcessConfigExt { - /// Sets the Iceberg codecs, wrapping any codecs already set. + /// Adds the Iceberg codecs after any codecs already set. fn with_iceberg(self) -> Self; } impl ExecutorProcessConfigExt for ExecutorProcessConfig { fn with_iceberg(mut self) -> Self { - let logical = self.override_logical_codec.take(); - let physical = self.override_physical_codec.take(); - self.override_logical_codec = Some(Arc::new( - logical.map_or_else(IcebergLogicalCodec::default, IcebergLogicalCodec::new), - )); - self.override_physical_codec = Some(Arc::new( - physical - .map_or_else(IcebergPhysicalCodec::default, IcebergPhysicalCodec::new), - )); + let logical = self + .override_logical_codec + .take() + .unwrap_or_else(|| Arc::new(BallistaLogicalExtensionCodec::default())); + let physical = self + .override_physical_codec + .take() + .unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())); + self.override_logical_codec = Some(IcebergLogicalCodec::after(logical)); + self.override_physical_codec = Some(IcebergPhysicalCodec::after(physical)); self } } diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 5ff6eef1d2..8ef9098ff3 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -83,7 +83,8 @@ pub use crate::physical_codec::IcebergPhysicalCodec; #[cfg(feature = "scheduler")] pub use crate::scheduler::SchedulerConfigExt; -/// Readies a client's session for Iceberg: registers the codecs, and pins +/// Readies a client's session for Iceberg: adds the Iceberg codecs after the +/// session's codecs, and pins /// each catalog-backed table a query names to the version its catalog serves /// when the query is planned. A session that is already ready is returned /// unchanged. @@ -92,12 +93,13 @@ pub fn register_iceberg(mut state: SessionState) -> SessionState { return state; } let config = state.config_mut(); - let logical = IcebergLogicalCodec::new(config.ballista_logical_extension_codec()); - let physical = IcebergPhysicalCodec::new(config.ballista_physical_extension_codec()); + let logical = IcebergLogicalCodec::after(config.ballista_logical_extension_codec()); + let physical = + IcebergPhysicalCodec::after(config.ballista_physical_extension_codec()); *config = config .clone() - .with_ballista_logical_extension_codec(Arc::new(logical)) - .with_ballista_physical_extension_codec(Arc::new(physical)); + .with_ballista_logical_extension_codec(logical) + .with_ballista_physical_extension_codec(physical); let catalogs = PinningCatalogList(state.catalog_list().clone()); // `new_from_existing` would otherwise give the session a new ID. let session_id = state.session_id().to_string(); diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index eb199ebbe6..77fb691045 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -19,15 +19,12 @@ use std::sync::Arc; -use ballista_core::serde::BallistaLogicalExtensionCodec; +use ballista_core::serde::ComposedLogicalExtensionCodec; use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference, exec_err, not_impl_err}; -use datafusion::datasource::file_format::FileFormatFactory; use datafusion::execution::TaskContext; -use datafusion::logical_expr::{ - AggregateUDF, Extension, HigherOrderUDF, LogicalPlan, ScalarUDF, WindowUDF, -}; +use datafusion::logical_expr::{Extension, LogicalPlan}; use datafusion_proto::logical_plan::LogicalExtensionCodec; use iceberg_datafusion::metadata::IcebergMetadataTableProvider; use iceberg_datafusion::{IcebergCatalogTableProvider, IcebergStaticTableProvider}; @@ -43,54 +40,48 @@ struct ProviderWire { } /// A [`LogicalExtensionCodec`] for [`IcebergStaticTableProvider`] and -/// catalog-backed tables pinned by [`register_iceberg`] that delegates -/// everything else to an inner codec. +/// catalog-backed tables pinned by [`register_iceberg`]. It fails on anything +/// else, so compose it after other codecs with +/// [`ComposedLogicalExtensionCodec`], as [`register_iceberg`] does. /// /// [`register_iceberg`]: crate::register_iceberg -#[derive(Debug)] -pub struct IcebergLogicalCodec { - inner: Arc, -} +#[derive(Debug, Default)] +pub struct IcebergLogicalCodec; impl IcebergLogicalCodec { - /// Creates a codec that delegates everything but Iceberg tables to `inner`. - pub fn new(inner: Arc) -> Self { - Self { inner } - } -} - -impl Default for IcebergLogicalCodec { - fn default() -> Self { - Self::new(Arc::new(BallistaLogicalExtensionCodec::default())) + /// Composes `codec` with this codec after it. + pub(crate) fn after( + codec: Arc, + ) -> Arc { + Arc::new(ComposedLogicalExtensionCodec::new(vec![ + codec, + Arc::new(Self), + ])) } } impl LogicalExtensionCodec for IcebergLogicalCodec { fn try_decode( &self, - buf: &[u8], - inputs: &[LogicalPlan], - ctx: &TaskContext, + _buf: &[u8], + _inputs: &[LogicalPlan], + _ctx: &TaskContext, ) -> Result { - self.inner.try_decode(buf, inputs, ctx) + not_impl_err!("ballista-iceberg has no extension nodes") } - fn try_encode(&self, node: &Extension, buf: &mut Vec) -> Result<()> { - self.inner.try_encode(node, buf) + fn try_encode(&self, _node: &Extension, _buf: &mut Vec) -> Result<()> { + not_impl_err!("ballista-iceberg has no extension nodes") } fn try_decode_table_provider( &self, buf: &[u8], - table_ref: &TableReference, + _table_ref: &TableReference, schema: SchemaRef, - ctx: &TaskContext, + _ctx: &TaskContext, ) -> Result> { - let Some(ProviderWire { table, snapshot_id }) = wire::decode(buf)? else { - return self - .inner - .try_decode_table_provider(buf, table_ref, schema, ctx); - }; + let ProviderWire { table, snapshot_id } = wire::decode(buf)?; Ok(Arc::new(PinnedTable { table: table.load()?, snapshot_id, @@ -126,64 +117,8 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { "Iceberg metadata table {table_ref} isn't supported on Ballista yet" ); } else { - return self.inner.try_encode_table_provider(table_ref, node, buf); + return not_impl_err!("{table_ref} isn't an Iceberg table"); }; wire::encode(buf, &payload) } - - fn try_decode_file_format( - &self, - buf: &[u8], - ctx: &TaskContext, - ) -> Result> { - self.inner.try_decode_file_format(buf, ctx) - } - - fn try_encode_file_format( - &self, - buf: &mut Vec, - node: Arc, - ) -> Result<()> { - self.inner.try_encode_file_format(buf, node) - } - - fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udf(name, buf) - } - - fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udf(node, buf) - } - - fn try_decode_higher_order_function( - &self, - name: &str, - buf: &[u8], - ) -> Result> { - self.inner.try_decode_higher_order_function(name, buf) - } - - fn try_encode_higher_order_function( - &self, - node: &HigherOrderUDF, - buf: &mut Vec, - ) -> Result<()> { - self.inner.try_encode_higher_order_function(node, buf) - } - - fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udaf(name, buf) - } - - fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udaf(node, buf) - } - - fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udwf(name, buf) - } - - fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udwf(node, buf) - } } diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index 7428e686ce..24e1d711c9 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -19,18 +19,14 @@ use std::sync::Arc; -use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::common::Result; +use datafusion::common::{Result, not_impl_err}; use datafusion::datasource::source::DataSourceExec; use datafusion::execution::TaskContext; -use datafusion::logical_expr::{AggregateUDF, HigherOrderUDF, ScalarUDF, WindowUDF}; -use datafusion::physical_expr::PhysicalExpr; -use datafusion::physical_expr_common::physical_expr::proto_decode::PhysicalExprDecodeCtx; -use datafusion::physical_expr_common::physical_expr::proto_encode::PhysicalExprEncodeCtx; use datafusion::physical_plan::ExecutionPlan; use datafusion_proto::physical_plan::{ - PhysicalExtensionCodec, PhysicalProtoConverterExtension, + ComposedPhysicalExtensionCodec, PhysicalExtensionCodec, + PhysicalProtoConverterExtension, }; use iceberg::expr::Predicate; use iceberg_datafusion::read::IcebergDataSource; @@ -48,23 +44,23 @@ struct ScanWire { fetch: Option, } -/// A [`PhysicalExtensionCodec`] for scans of an [`IcebergDataSource`] that -/// delegates everything else to an inner codec. -#[derive(Debug)] -pub struct IcebergPhysicalCodec { - inner: Arc, -} +/// A [`PhysicalExtensionCodec`] for scans of an [`IcebergDataSource`]. It +/// fails on anything else, so compose it after other codecs with +/// [`ComposedPhysicalExtensionCodec`], as [`register_iceberg`] does. +/// +/// [`register_iceberg`]: crate::register_iceberg +#[derive(Debug, Default)] +pub struct IcebergPhysicalCodec; impl IcebergPhysicalCodec { - /// Creates a codec that delegates everything but Iceberg scans to `inner`. - pub fn new(inner: Arc) -> Self { - Self { inner } - } -} - -impl Default for IcebergPhysicalCodec { - fn default() -> Self { - Self::new(Arc::new(BallistaPhysicalExtensionCodec::default())) + /// Composes `codec` with this codec after it. + pub(crate) fn after( + codec: Arc, + ) -> Arc { + Arc::new(ComposedPhysicalExtensionCodec::new(vec![ + codec, + Arc::new(Self), + ])) } } @@ -72,13 +68,11 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { fn try_decode( &self, buf: &[u8], - inputs: &[Arc], - ctx: &TaskContext, - proto_converter: &dyn PhysicalProtoConverterExtension, + _inputs: &[Arc], + _ctx: &TaskContext, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result> { - let Some(scan) = wire::decode::(buf)? else { - return self.inner.try_decode(buf, inputs, ctx, proto_converter); - }; + let scan: ScanWire = wire::decode(buf)?; let source = IcebergDataSource::builder(scan.table.load()?, scan.schema) .with_snapshot_id(scan.snapshot_id) .with_predicate(scan.predicate) @@ -91,81 +85,23 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { &self, node: Arc, buf: &mut Vec, - proto_converter: &dyn PhysicalProtoConverterExtension, + _proto_converter: &dyn PhysicalProtoConverterExtension, ) -> Result<()> { - let source = node + let Some(source) = node .downcast_ref::() - .and_then(|exec| exec.data_source().downcast_ref::()); - match source { - Some(source) => wire::encode( - buf, - &ScanWire { - table: TableWire::new(source.table())?, - snapshot_id: source.snapshot_id(), - schema: source.schema(), - predicate: source.predicate().cloned(), - fetch: source.fetch(), - }, - ), - None => self.inner.try_encode(node, buf, proto_converter), - } - } - - fn try_decode_expr( - &self, - buf: &[u8], - inputs: &[Arc], - ctx: &PhysicalExprDecodeCtx<'_>, - ) -> Result> { - self.inner.try_decode_expr(buf, inputs, ctx) - } - - fn try_encode_expr( - &self, - node: &Arc, - buf: &mut Vec, - ctx: &PhysicalExprEncodeCtx<'_>, - ) -> Result<()> { - self.inner.try_encode_expr(node, buf, ctx) - } - - fn try_decode_udf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udf(name, buf) - } - - fn try_encode_udf(&self, node: &ScalarUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udf(node, buf) - } - - fn try_decode_higher_order_function( - &self, - name: &str, - buf: &[u8], - ) -> Result> { - self.inner.try_decode_higher_order_function(name, buf) - } - - fn try_encode_higher_order_function( - &self, - node: &HigherOrderUDF, - buf: &mut Vec, - ) -> Result<()> { - self.inner.try_encode_higher_order_function(node, buf) - } - - fn try_decode_udaf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udaf(name, buf) - } - - fn try_encode_udaf(&self, node: &AggregateUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udaf(node, buf) - } - - fn try_decode_udwf(&self, name: &str, buf: &[u8]) -> Result> { - self.inner.try_decode_udwf(name, buf) - } - - fn try_encode_udwf(&self, node: &WindowUDF, buf: &mut Vec) -> Result<()> { - self.inner.try_encode_udwf(node, buf) + .and_then(|exec| exec.data_source().downcast_ref::()) + else { + return not_impl_err!("{} isn't an Iceberg scan", node.name()); + }; + wire::encode( + buf, + &ScanWire { + table: TableWire::new(source.table())?, + snapshot_id: source.snapshot_id(), + schema: source.schema(), + predicate: source.predicate().cloned(), + fetch: source.fetch(), + }, + ) } } diff --git a/integrations/iceberg/src/scheduler.rs b/integrations/iceberg/src/scheduler.rs index 5d705da0eb..7d9b2de1a3 100644 --- a/integrations/iceberg/src/scheduler.rs +++ b/integrations/iceberg/src/scheduler.rs @@ -19,6 +19,9 @@ use std::sync::Arc; +use ballista_core::serde::{ + BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, +}; use ballista_core::utils::default_session_builder; use ballista_scheduler::config::SchedulerConfig; use datafusion::catalog::CatalogProvider; @@ -27,7 +30,7 @@ use crate::{IcebergLogicalCodec, IcebergPhysicalCodec, register_iceberg}; /// Readies a [`SchedulerConfig`] for Iceberg. pub trait SchedulerConfigExt { - /// Sets the Iceberg codecs, wrapping any codecs already set, and mounts + /// Adds the Iceberg codecs after any codecs already set, and mounts /// `catalogs` in each session, readied with [`register_iceberg`]. The /// scheduler plans queries from Flight SQL clients itself, so they can /// then query those catalogs. @@ -42,15 +45,16 @@ impl SchedulerConfigExt for SchedulerConfig { mut self, catalogs: impl IntoIterator)>, ) -> Self { - let logical = self.override_logical_codec.take(); - let physical = self.override_physical_codec.take(); - self.override_logical_codec = Some(Arc::new( - logical.map_or_else(IcebergLogicalCodec::default, IcebergLogicalCodec::new), - )); - self.override_physical_codec = Some(Arc::new( - physical - .map_or_else(IcebergPhysicalCodec::default, IcebergPhysicalCodec::new), - )); + let logical = self + .override_logical_codec + .take() + .unwrap_or_else(|| Arc::new(BallistaLogicalExtensionCodec::default())); + let physical = self + .override_physical_codec + .take() + .unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())); + self.override_logical_codec = Some(IcebergLogicalCodec::after(logical)); + self.override_physical_codec = Some(IcebergPhysicalCodec::after(physical)); let base = self .override_session_builder .take() diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index dcfc897e95..b7632f3ca5 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! What the codecs send in place of a live [`Table`], and how they frame it. +//! What the codecs send in place of a live [`Table`], and how they encode it. use std::collections::VecDeque; use std::fmt; @@ -162,33 +162,18 @@ impl TableWire { } } -/// Leads a blob that belongs to this crate, JSON-encoded. Any other blob -/// belongs to the inner codec and passes through untouched, so codecs for -/// other formats can be stacked in any order, as long as each has its own -/// prefix. -const PREFIX: &[u8] = b"iceberg"; - -/// Appends `payload` to `buf` as an Iceberg blob. +/// Writes `payload` to `buf` as JSON. pub(crate) fn encode( buf: &mut Vec, payload: &T, ) -> Result<(), DataFusionError> { - buf.extend_from_slice(PREFIX); serde_json::to_writer(buf, payload).map_err(json_err) } -/// Decodes `buf` as `T` if it's an Iceberg blob, or returns `None` if it -/// belongs to the inner codec. -pub(crate) fn decode( - buf: &[u8], -) -> Result, DataFusionError> { - buf.strip_prefix(PREFIX) - .map(|rest| { - serde_json::from_slice(rest).map_err(|e| { - exec_datafusion_err!("invalid ballista-iceberg payload: {e}") - }) - }) - .transpose() +/// Reads a payload written by [`encode`]. +pub(crate) fn decode(buf: &[u8]) -> Result { + serde_json::from_slice(buf) + .map_err(|e| exec_datafusion_err!("invalid ballista-iceberg payload: {e}")) } fn json_err(e: serde_json::Error) -> DataFusionError { diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index fcd6a17396..e33f87db33 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -24,7 +24,8 @@ mod common; mod iceberg_nodes { use std::sync::Arc; - use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, register_iceberg}; + use ballista_core::extension::SessionConfigExt; + use ballista_iceberg::{IcebergLogicalCodec, register_iceberg}; use datafusion::assert_batches_eq; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference}; @@ -37,9 +38,7 @@ mod iceberg_nodes { logical_plan_to_bytes_with_extension_codec, }; use datafusion_proto::logical_plan::LogicalExtensionCodec; - use datafusion_proto::physical_plan::{ - DefaultPhysicalProtoConverter, PhysicalExtensionCodec, - }; + use datafusion_proto::physical_plan::DefaultPhysicalProtoConverter; use iceberg::inspect::MetadataTableType; use iceberg_datafusion::IcebergStaticTableProvider; use iceberg_datafusion::metadata::IcebergMetadataTableProvider; @@ -56,13 +55,15 @@ mod iceberg_nodes { let table = insert(&catalog, "(4, 'd')").await; let provider = IcebergStaticTableProvider::try_new_from_table_snapshot(table, first).await?; - let ctx = SessionContext::new(); + let state = SessionStateBuilder::new().with_default_features().build(); + let ctx = SessionContext::new_with_state(register_iceberg(state)); let filters = [col("id").gt(lit(1))]; let scan = provider .scan(&ctx.state(), Some(&vec![1]), &filters, Some(1)) .await?; - let codec = IcebergPhysicalCodec::default(); + // The session's codecs: Ballista's, then Iceberg's. + let codec = ctx.state().config().ballista_physical_extension_codec(); let mut buf = Vec::new(); codec.try_encode(scan, &mut buf, &DefaultPhysicalProtoConverter {})?; let decoded = codec.try_decode( @@ -95,10 +96,10 @@ mod iceberg_nodes { let state = SessionStateBuilder::new().with_default_features().build(); let ctx = SessionContext::new_with_state(register_iceberg(state)); ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; - let codec = IcebergLogicalCodec::default(); + let codec = ctx.state().config().ballista_logical_extension_codec(); let send = async |sql: &str| { let plan = ctx.state().create_logical_plan(sql).await?; - logical_plan_to_bytes_with_extension_codec(&plan, &codec) + logical_plan_to_bytes_with_extension_codec(&plan, codec.as_ref()) }; let pinned = insert(&catalog, "(1, 'a')").await; @@ -112,7 +113,7 @@ mod iceberg_nodes { let plan = logical_plan_from_bytes_with_extension_codec( bytes, &ctx.task_ctx(), - &codec, + codec.as_ref(), )?; ctx.execute_logical_plan(plan).await?.collect().await }; @@ -148,7 +149,7 @@ mod iceberg_nodes { let catalog = memory_catalog(dir.path()).await; let table = insert(&catalog, "(1, 'a')").await; let encode = |provider: Arc| { - IcebergLogicalCodec::default() + IcebergLogicalCodec .try_encode_table_provider( &TableReference::bare("t"), provider, @@ -180,48 +181,25 @@ mod iceberg_nodes { mod other_nodes { use std::sync::Arc; - use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec}; + use ballista_core::extension::SessionConfigExt; + use ballista_iceberg::{IcebergLogicalCodec, IcebergPhysicalCodec, register_iceberg}; use datafusion::arrow::datatypes::Schema; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference}; use datafusion::datasource::empty::EmptyTable; - use datafusion::execution::TaskContext; + use datafusion::execution::{SessionStateBuilder, TaskContext}; use datafusion::logical_expr::{Extension, LogicalPlan}; use datafusion::physical_plan::empty::EmptyExec; - use datafusion::physical_plan::{ExecutionPlan, displayable}; + use datafusion::prelude::SessionConfig; use datafusion_proto::logical_plan::LogicalExtensionCodec; use datafusion_proto::physical_plan::{ DefaultPhysicalProtoConverter, PhysicalExtensionCodec, - PhysicalProtoConverterExtension, }; - /// An inner codec that encodes anything as `inner`, and decodes only that. + /// A codec that encodes any table as `inner`, and decodes only that. #[derive(Debug)] struct Inner; - impl PhysicalExtensionCodec for Inner { - fn try_decode( - &self, - buf: &[u8], - _inputs: &[Arc], - _ctx: &TaskContext, - _proto_converter: &dyn PhysicalProtoConverterExtension, - ) -> Result> { - assert_eq!(buf, b"inner"); - Ok(Arc::new(EmptyExec::new(Arc::new(Schema::empty())))) - } - - fn try_encode( - &self, - _node: Arc, - buf: &mut Vec, - _proto_converter: &dyn PhysicalProtoConverterExtension, - ) -> Result<()> { - buf.extend_from_slice(b"inner"); - Ok(()) - } - } - impl LogicalExtensionCodec for Inner { fn try_decode( &self, @@ -259,30 +237,27 @@ mod other_nodes { } #[test] - fn should_delegate_non_iceberg_nodes_to_inner_codec() -> Result<()> { - let ctx = TaskContext::default(); + fn should_keep_the_session_codecs_first() -> Result<()> { + let config = + SessionConfig::new().with_ballista_logical_extension_codec(Arc::new(Inner)); + let state = SessionStateBuilder::new().with_config(config).build(); + let codec = register_iceberg(state) + .config() + .ballista_logical_extension_codec(); + let table = TableReference::bare("t"); let schema = Arc::new(Schema::empty()); - let physical = IcebergPhysicalCodec::new(Arc::new(Inner)); - let mut buf = Vec::new(); - let plan = Arc::new(EmptyExec::new(schema.clone())); - physical.try_encode(plan, &mut buf, &DefaultPhysicalProtoConverter {})?; - assert_eq!(buf, b"inner"); - let decoded = - physical.try_decode(&buf, &[], &ctx, &DefaultPhysicalProtoConverter {})?; - assert_eq!( - displayable(decoded.as_ref()).one_line().to_string(), - "EmptyExec\n" - ); - - let logical = IcebergLogicalCodec::new(Arc::new(Inner)); - let table = TableReference::bare("t"); let mut buf = Vec::new(); let provider = Arc::new(EmptyTable::new(schema.clone())); - logical.try_encode_table_provider(&table, provider, &mut buf)?; - assert_eq!(buf, b"inner"); - let decoded = - logical.try_decode_table_provider(&buf, &table, schema.clone(), &ctx)?; + codec.try_encode_table_provider(&table, provider, &mut buf)?; + // Position 0, the default, isn't written; then the blob, as field 2. + assert_eq!(buf, b"\x12\x05inner"); + let decoded = codec.try_decode_table_provider( + &buf, + &table, + schema.clone(), + &TaskContext::default(), + )?; assert_eq!( format!("{decoded:?}"), format!("{:?}", EmptyTable::new(schema)) @@ -291,11 +266,39 @@ mod other_nodes { Ok(()) } + #[test] + fn should_refuse_other_nodes() { + let schema = Arc::new(Schema::empty()); + let logical = IcebergLogicalCodec + .try_encode_table_provider( + &TableReference::bare("t"), + Arc::new(EmptyTable::new(schema.clone())), + &mut Vec::new(), + ) + .unwrap_err(); + assert_eq!( + logical.to_string(), + "This feature is not implemented: t isn't an Iceberg table" + ); + + let physical = IcebergPhysicalCodec + .try_encode( + Arc::new(EmptyExec::new(schema)), + &mut Vec::new(), + &DefaultPhysicalProtoConverter {}, + ) + .unwrap_err(); + assert_eq!( + physical.to_string(), + "This feature is not implemented: EmptyExec isn't an Iceberg scan" + ); + } + #[test] fn should_reject_malformed_payload() { - let err = IcebergPhysicalCodec::new(Arc::new(Inner)) + let err = IcebergPhysicalCodec .try_decode( - b"iceberginner", + b"inner", &[], &TaskContext::default(), &DefaultPhysicalProtoConverter {}, From 97d465d70be1ff294e29026c6c55497923db100b Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 3 Nov 2026 08:42:47 -0500 Subject: [PATCH 15/31] Fall back to Ballista's codecs in IcebergLogicalCodec::after --- integrations/iceberg/src/executor.rs | 19 ++++--------------- integrations/iceberg/src/lib.rs | 12 ++++++------ integrations/iceberg/src/logical_codec.rs | 10 +++++++--- integrations/iceberg/src/physical_codec.rs | 7 +++++-- integrations/iceberg/src/scheduler.rs | 17 ++++------------- 5 files changed, 26 insertions(+), 39 deletions(-) diff --git a/integrations/iceberg/src/executor.rs b/integrations/iceberg/src/executor.rs index edfc0aa7b4..5dafad2bde 100644 --- a/integrations/iceberg/src/executor.rs +++ b/integrations/iceberg/src/executor.rs @@ -17,11 +17,6 @@ //! Readies a separately deployed executor for Iceberg. -use std::sync::Arc; - -use ballista_core::serde::{ - BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, -}; use ballista_executor::executor_process::ExecutorProcessConfig; use crate::{IcebergLogicalCodec, IcebergPhysicalCodec}; @@ -34,16 +29,10 @@ pub trait ExecutorProcessConfigExt { impl ExecutorProcessConfigExt for ExecutorProcessConfig { fn with_iceberg(mut self) -> Self { - let logical = self - .override_logical_codec - .take() - .unwrap_or_else(|| Arc::new(BallistaLogicalExtensionCodec::default())); - let physical = self - .override_physical_codec - .take() - .unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())); - self.override_logical_codec = Some(IcebergLogicalCodec::after(logical)); - self.override_physical_codec = Some(IcebergPhysicalCodec::after(physical)); + let logical = IcebergLogicalCodec::after(self.override_logical_codec.take()); + let physical = IcebergPhysicalCodec::after(self.override_physical_codec.take()); + self.override_logical_codec = Some(logical); + self.override_physical_codec = Some(physical); self } } diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 8ef9098ff3..663c785724 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -84,18 +84,18 @@ pub use crate::physical_codec::IcebergPhysicalCodec; pub use crate::scheduler::SchedulerConfigExt; /// Readies a client's session for Iceberg: adds the Iceberg codecs after the -/// session's codecs, and pins -/// each catalog-backed table a query names to the version its catalog serves -/// when the query is planned. A session that is already ready is returned -/// unchanged. +/// session's codecs, and pins each catalog-backed table a query names to the +/// version its catalog serves when the query is planned. A session that is +/// already ready is returned unchanged. pub fn register_iceberg(mut state: SessionState) -> SessionState { if state.catalog_list().is::() { return state; } let config = state.config_mut(); - let logical = IcebergLogicalCodec::after(config.ballista_logical_extension_codec()); + let logical = + IcebergLogicalCodec::after(Some(config.ballista_logical_extension_codec())); let physical = - IcebergPhysicalCodec::after(config.ballista_physical_extension_codec()); + IcebergPhysicalCodec::after(Some(config.ballista_physical_extension_codec())); *config = config .clone() .with_ballista_logical_extension_codec(logical) diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 77fb691045..477bea5d32 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -19,7 +19,9 @@ use std::sync::Arc; -use ballista_core::serde::ComposedLogicalExtensionCodec; +use ballista_core::serde::{ + BallistaLogicalExtensionCodec, ComposedLogicalExtensionCodec, +}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::catalog::TableProvider; use datafusion::common::{Result, TableReference, exec_err, not_impl_err}; @@ -49,10 +51,12 @@ struct ProviderWire { pub struct IcebergLogicalCodec; impl IcebergLogicalCodec { - /// Composes `codec` with this codec after it. + /// Composes `codec`, or Ballista's when `None`, with this codec after it. pub(crate) fn after( - codec: Arc, + codec: Option>, ) -> Arc { + let codec = + codec.unwrap_or_else(|| Arc::new(BallistaLogicalExtensionCodec::default())); Arc::new(ComposedLogicalExtensionCodec::new(vec![ codec, Arc::new(Self), diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index 24e1d711c9..97f41db4b3 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -19,6 +19,7 @@ use std::sync::Arc; +use ballista_core::serde::BallistaPhysicalExtensionCodec; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, not_impl_err}; use datafusion::datasource::source::DataSourceExec; @@ -53,10 +54,12 @@ struct ScanWire { pub struct IcebergPhysicalCodec; impl IcebergPhysicalCodec { - /// Composes `codec` with this codec after it. + /// Composes `codec`, or Ballista's when `None`, with this codec after it. pub(crate) fn after( - codec: Arc, + codec: Option>, ) -> Arc { + let codec = + codec.unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())); Arc::new(ComposedPhysicalExtensionCodec::new(vec![ codec, Arc::new(Self), diff --git a/integrations/iceberg/src/scheduler.rs b/integrations/iceberg/src/scheduler.rs index 7d9b2de1a3..9dccfb35ab 100644 --- a/integrations/iceberg/src/scheduler.rs +++ b/integrations/iceberg/src/scheduler.rs @@ -19,9 +19,6 @@ use std::sync::Arc; -use ballista_core::serde::{ - BallistaLogicalExtensionCodec, BallistaPhysicalExtensionCodec, -}; use ballista_core::utils::default_session_builder; use ballista_scheduler::config::SchedulerConfig; use datafusion::catalog::CatalogProvider; @@ -45,16 +42,10 @@ impl SchedulerConfigExt for SchedulerConfig { mut self, catalogs: impl IntoIterator)>, ) -> Self { - let logical = self - .override_logical_codec - .take() - .unwrap_or_else(|| Arc::new(BallistaLogicalExtensionCodec::default())); - let physical = self - .override_physical_codec - .take() - .unwrap_or_else(|| Arc::new(BallistaPhysicalExtensionCodec::default())); - self.override_logical_codec = Some(IcebergLogicalCodec::after(logical)); - self.override_physical_codec = Some(IcebergPhysicalCodec::after(physical)); + let logical = IcebergLogicalCodec::after(self.override_logical_codec.take()); + let physical = IcebergPhysicalCodec::after(self.override_physical_codec.take()); + self.override_logical_codec = Some(logical); + self.override_physical_codec = Some(physical); let base = self .override_session_builder .take() From e7d19b97573a3582fa58bdda460e06d55e91e9ed Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 3 Nov 2026 13:03:35 -0500 Subject: [PATCH 16/31] Correct the Iceberg docs on how the codecs are added --- docs/source/user-guide/iceberg.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index f3803d2fca..b659e2836e 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -74,7 +74,7 @@ cargo run --example standalone-iceberg-scan Flight SQL clients (JDBC, ADBC, BI tools) send SQL to the scheduler, which plans it, so they hold no catalog. To let them query Iceberg tables, mount catalogs on the scheduler with `SchedulerConfigExt::with_iceberg` (the `scheduler` -feature), which also sets the Iceberg codecs: +feature), which also adds the Iceberg codecs: ```rust use ballista_iceberg::SchedulerConfigExt; @@ -85,7 +85,7 @@ let config = SchedulerConfig::default() .with_iceberg([("rest", Arc::new(rest) as _)]); ``` -Executors set the codecs with `ExecutorProcessConfigExt::with_iceberg` (the +Executors add the codecs with `ExecutorProcessConfigExt::with_iceberg` (the `executor` feature): ```rust @@ -94,7 +94,8 @@ use ballista_iceberg::ExecutorProcessConfigExt; let config = ExecutorProcessConfig::default().with_iceberg(); ``` -Both keep any session builder or codecs the config already has. A client can +Both keep any codecs the config already has, and the scheduler keeps its +session builder, mounting the catalogs on each session it builds. A client can then run `SELECT * FROM rest.ns.t`. Tables are pinned the same way as on the client, while the scheduler plans the query. @@ -125,9 +126,10 @@ scheduler for SQL clients. Executors never need catalog configuration. already set, with `ComposedLogicalExtensionCodec` and DataFusion's `ComposedPhysicalExtensionCodec` (see [Combining Codecs](extending-components.md#combining-codecs)). To read tables -of another format too, such as Delta, set its codecs before calling them. -Clients, the scheduler and executors must all set the same codecs in the same -order. +of another format too, such as Delta, set its codecs first: on the session's +`SessionConfig` before calling `register_iceberg`, and on `SchedulerConfig` and +`ExecutorProcessConfig` before calling `with_iceberg`. Clients, the scheduler +and executors must all set the same codecs in the same order. ## Limitations From 9025298d97dcf2f9ebf47dd8f05e0d75355f97bd Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 3 Nov 2026 15:42:52 -0500 Subject: [PATCH 17/31] Re-pin a view's Iceberg tables each time a query names it --- docs/source/user-guide/iceberg.md | 11 +- integrations/iceberg/src/catalog.rs | 122 ++++++++++++++++++---- integrations/iceberg/src/lib.rs | 4 +- integrations/iceberg/src/logical_codec.rs | 1 + integrations/iceberg/tests/scan.rs | 23 +++- 5 files changed, 132 insertions(+), 29 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index b659e2836e..db4bb53c83 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -110,6 +110,8 @@ Metadata files never change, so every node rebuilds exactly that version. the client plans the query. DataFusion looks up each table a query names once, so every scan of it, such as each side of a self-join, reads that version, and each new query reads the latest one. +- **A view** stores the plan it was created with, so each query that names it + pins its catalog-backed tables again, and it reads their latest versions. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. @@ -143,11 +145,10 @@ and executors must all set the same codecs in the same order. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. -- **A view keeps the version from when it was created.** DataFusion stores a - view as a planned query, and tables are pinned when a query is planned. The - same goes for a `DataFrame` or a Flight SQL prepared statement run more than - once: each run reads the version from when it was planned. To read newer - commits, recreate the view or plan the query again. +- **A reused `DataFrame` or prepared statement keeps its version.** Tables are + pinned when a query is planned, so a `DataFrame` or a Flight SQL prepared + statement run more than once reads the version from when it was planned. To + read newer commits, plan the query again. - **A separately deployed scheduler and executors** must link `ballista-iceberg` and be readied with `with_iceberg` (see [SQL clients](#sql-clients)). The tests cover standalone mode and an diff --git a/integrations/iceberg/src/catalog.rs b/integrations/iceberg/src/catalog.rs index 671434e00d..2574e0fa68 100644 --- a/integrations/iceberg/src/catalog.rs +++ b/integrations/iceberg/src/catalog.rs @@ -17,29 +17,38 @@ //! Pins catalog-backed tables to one version while a query is planned. +use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::catalog::view::ViewTable; use datafusion::catalog::{ CatalogProvider, CatalogProviderList, SchemaProvider, Session, TableProvider, }; -use datafusion::common::{Result, not_impl_err}; -use datafusion::datasource::TableType; +use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; +use datafusion::common::{DataFusionError, Result, not_impl_err}; use datafusion::datasource::source::DataSourceExec; +use datafusion::datasource::{DefaultTableSource, TableType, provider_as_source}; use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; +use datafusion::logical_expr::{ + Expr, LogicalPlan, TableProviderFilterPushDown, TableScan, +}; use datafusion::physical_plan::ExecutionPlan; +use futures::future::try_join_all; +use iceberg::TableIdent; use iceberg::table::Table; use iceberg_datafusion::read::IcebergDataSource; use iceberg_datafusion::{IcebergCatalogTableProvider, to_datafusion_error}; /// A session's catalogs, serving each catalog-backed Iceberg table as a -/// [`PinnedTable`]. +/// [`PinnedTable`], and each view with its Iceberg tables pinned again. /// /// DataFusion looks up each table a query names once, while planning it, so /// every scan of a table in the query reads the same version, and encoding the -/// plan needs no I/O. +/// plan needs no I/O. A view stores the plan it was created with, so its tables +/// are pinned again each time a query names it, and it reads the latest +/// versions, as it would in DataFusion. #[derive(Debug)] pub(crate) struct PinningCatalogList(pub(crate) Arc); @@ -109,19 +118,16 @@ impl SchemaProvider for PinningSchema { let Some(table) = self.0.table(name).await? else { return Ok(None); }; - let Some(provider) = table.downcast_ref::() else { - return Ok(Some(table)); - }; - let pinned = provider - .catalog() - .load_table(provider.table_ident()) - .await - .map_err(to_datafusion_error)?; - Ok(Some(Arc::new(PinnedTable { - table: pinned, - snapshot_id: None, - schema: provider.schema(), - }))) + if let Some(provider) = table.downcast_ref::() { + let table = load(provider).await?; + return Ok(Some(Arc::new(PinnedTable::pin(provider, table)))); + } + if let Some(view) = table.downcast_ref::() + && let Some(repinned) = repin(view).await? + { + return Ok(Some(Arc::new(repinned))); + } + Ok(Some(table)) } async fn table_type(&self, name: &str) -> Result> { @@ -145,6 +151,70 @@ impl SchemaProvider for PinningSchema { } } +/// A catalog-backed table: its catalog, by address, and its identifier. +type TableKey = (usize, TableIdent); + +fn key(provider: &IcebergCatalogTableProvider) -> TableKey { + let catalog = Arc::as_ptr(provider.catalog()).cast::<()>() as usize; + (catalog, provider.table_ident().clone()) +} + +async fn load(provider: &IcebergCatalogTableProvider) -> Result
{ + provider + .catalog() + .load_table(provider.table_ident()) + .await + .map_err(to_datafusion_error) +} + +/// The catalog-backed table a scan's pinned table was pinned from, if any. +fn origin(scan: &TableScan) -> Option<&IcebergCatalogTableProvider> { + let source = scan.source.downcast_ref::()?; + let pinned = source.table_provider.downcast_ref::()?; + pinned.origin.as_ref() +} + +/// Returns `view` with each catalog-backed Iceberg table it reads pinned to +/// the version its catalog serves now, loading each table once, or `None` if +/// it reads none. DataFusion inlines a view into any plan built over it, so a +/// view's plan holds tables, never other views. +async fn repin(view: &ViewTable) -> Result> { + let mut providers = HashMap::new(); + view.logical_plan().apply_with_subqueries(|node| { + if let LogicalPlan::TableScan(scan) = node + && let Some(origin) = origin(scan) + { + providers + .entry(key(origin)) + .or_insert_with(|| origin.clone()); + } + Ok(TreeNodeRecursion::Continue) + })?; + if providers.is_empty() { + return Ok(None); + } + let loads = providers.into_iter().map(|(key, provider)| async move { + Ok::<_, DataFusionError>((key, load(&provider).await?)) + }); + let tables: HashMap<_, _> = try_join_all(loads).await?.into_iter().collect(); + let plan = view + .logical_plan() + .clone() + .transform_up_with_subqueries(|node| { + let LogicalPlan::TableScan(mut scan) = node else { + return Ok(Transformed::no(node)); + }; + let Some(origin) = origin(&scan) else { + return Ok(Transformed::no(LogicalPlan::TableScan(scan))); + }; + let table = tables[&key(origin)].clone(); + scan.source = provider_as_source(Arc::new(PinnedTable::pin(origin, table))); + Ok(Transformed::yes(LogicalPlan::TableScan(scan))) + })? + .data; + Ok(Some(ViewTable::new(plan, view.definition().cloned()))) +} + /// One version of an Iceberg table, read with the schema the client planned /// against. /// @@ -158,6 +228,22 @@ pub(crate) struct PinnedTable { pub(crate) table: Table, pub(crate) snapshot_id: Option, pub(crate) schema: SchemaRef, + /// The catalog-backed table it was pinned from, to pin it again. Not sent + /// between nodes. + pub(crate) origin: Option, +} + +impl PinnedTable { + /// `table`, a version of `provider` loaded from its catalog, read with the + /// provider's schema, which it caches when it is created. + fn pin(provider: &IcebergCatalogTableProvider, table: Table) -> Self { + Self { + table, + snapshot_id: None, + schema: provider.schema(), + origin: Some(provider.clone()), + } + } } #[async_trait] diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 663c785724..6a75c2bdaf 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -54,8 +54,8 @@ //! - Reads only. //! - Each scan runs as a single task. //! - No metadata tables such as `t$snapshots`. -//! - A view, or a `DataFrame` or prepared statement reused across runs, reads -//! the version from when it was planned. +//! - A `DataFrame` or prepared statement reused across runs reads the version +//! from when it was planned. //! - Storage credentials travel in plain text. mod catalog; diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 477bea5d32..3445eb7f2b 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -90,6 +90,7 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { table: table.load()?, snapshot_id, schema, + origin: None, })) } diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index ec8b13f149..8e0e77bcd4 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -38,10 +38,8 @@ mod standalone { assert_standalone_scans(memory_catalog(dir.path()).await).await } - // A known limitation: a view stores its planned query, and tables are - // pinned when a query is planned. #[tokio::test] - async fn should_keep_a_views_version() -> Result<()> { + async fn should_read_a_views_latest_version() -> Result<()> { let dir = tempfile::tempdir()?; let catalog = memory_catalog(dir.path()).await; let state = SessionStateBuilder::new() @@ -53,17 +51,34 @@ mod standalone { insert(&catalog, "(1, 'a')").await; ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; + // A view over a view, reading `t` again in a subquery. + ctx.sql("CREATE VIEW w AS SELECT name FROM v WHERE id IN (SELECT id FROM t)") + .await?; insert(&catalog, "(2, 'b')").await; - let result = ctx.sql("SELECT * FROM v").await?.collect().await?; + let result = ctx + .sql("SELECT * FROM v ORDER BY id") + .await? + .collect() + .await?; let expected = [ "+----+------+", "| id | name |", "+----+------+", "| 1 | a |", + "| 2 | b |", "+----+------+", ]; assert_batches_eq!(expected, &result); + let result = ctx + .sql("SELECT * FROM w ORDER BY name") + .await? + .collect() + .await?; + let expected = [ + "+------+", "| name |", "+------+", "| a |", "| b |", "+------+", + ]; + assert_batches_eq!(expected, &result); Ok(()) } From 0b399889d9313cd89d6bcbc79bece9235c49511a Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 5 Nov 2026 20:09:33 -0500 Subject: [PATCH 18/31] Move view re-pinning to a follow-up PR Reverts 9025298d so it can be reviewed on its own. Views keep the version from when they were created, documented as a limitation. --- docs/source/user-guide/iceberg.md | 11 +- integrations/iceberg/src/catalog.rs | 122 ++++------------------ integrations/iceberg/src/lib.rs | 4 +- integrations/iceberg/src/logical_codec.rs | 1 - integrations/iceberg/tests/scan.rs | 23 +--- 5 files changed, 29 insertions(+), 132 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index db4bb53c83..b659e2836e 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -110,8 +110,6 @@ Metadata files never change, so every node rebuilds exactly that version. the client plans the query. DataFusion looks up each table a query names once, so every scan of it, such as each side of a self-join, reads that version, and each new query reads the latest one. -- **A view** stores the plan it was created with, so each query that names it - pins its catalog-backed tables again, and it reads their latest versions. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. @@ -145,10 +143,11 @@ and executors must all set the same codecs in the same order. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. -- **A reused `DataFrame` or prepared statement keeps its version.** Tables are - pinned when a query is planned, so a `DataFrame` or a Flight SQL prepared - statement run more than once reads the version from when it was planned. To - read newer commits, plan the query again. +- **A view keeps the version from when it was created.** DataFusion stores a + view as a planned query, and tables are pinned when a query is planned. The + same goes for a `DataFrame` or a Flight SQL prepared statement run more than + once: each run reads the version from when it was planned. To read newer + commits, recreate the view or plan the query again. - **A separately deployed scheduler and executors** must link `ballista-iceberg` and be readied with `with_iceberg` (see [SQL clients](#sql-clients)). The tests cover standalone mode and an diff --git a/integrations/iceberg/src/catalog.rs b/integrations/iceberg/src/catalog.rs index 2574e0fa68..671434e00d 100644 --- a/integrations/iceberg/src/catalog.rs +++ b/integrations/iceberg/src/catalog.rs @@ -17,38 +17,29 @@ //! Pins catalog-backed tables to one version while a query is planned. -use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; -use datafusion::catalog::view::ViewTable; use datafusion::catalog::{ CatalogProvider, CatalogProviderList, SchemaProvider, Session, TableProvider, }; -use datafusion::common::tree_node::{Transformed, TreeNodeRecursion}; -use datafusion::common::{DataFusionError, Result, not_impl_err}; +use datafusion::common::{Result, not_impl_err}; +use datafusion::datasource::TableType; use datafusion::datasource::source::DataSourceExec; -use datafusion::datasource::{DefaultTableSource, TableType, provider_as_source}; use datafusion::logical_expr::dml::InsertOp; -use datafusion::logical_expr::{ - Expr, LogicalPlan, TableProviderFilterPushDown, TableScan, -}; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown}; use datafusion::physical_plan::ExecutionPlan; -use futures::future::try_join_all; -use iceberg::TableIdent; use iceberg::table::Table; use iceberg_datafusion::read::IcebergDataSource; use iceberg_datafusion::{IcebergCatalogTableProvider, to_datafusion_error}; /// A session's catalogs, serving each catalog-backed Iceberg table as a -/// [`PinnedTable`], and each view with its Iceberg tables pinned again. +/// [`PinnedTable`]. /// /// DataFusion looks up each table a query names once, while planning it, so /// every scan of a table in the query reads the same version, and encoding the -/// plan needs no I/O. A view stores the plan it was created with, so its tables -/// are pinned again each time a query names it, and it reads the latest -/// versions, as it would in DataFusion. +/// plan needs no I/O. #[derive(Debug)] pub(crate) struct PinningCatalogList(pub(crate) Arc); @@ -118,16 +109,19 @@ impl SchemaProvider for PinningSchema { let Some(table) = self.0.table(name).await? else { return Ok(None); }; - if let Some(provider) = table.downcast_ref::() { - let table = load(provider).await?; - return Ok(Some(Arc::new(PinnedTable::pin(provider, table)))); - } - if let Some(view) = table.downcast_ref::() - && let Some(repinned) = repin(view).await? - { - return Ok(Some(Arc::new(repinned))); - } - Ok(Some(table)) + let Some(provider) = table.downcast_ref::() else { + return Ok(Some(table)); + }; + let pinned = provider + .catalog() + .load_table(provider.table_ident()) + .await + .map_err(to_datafusion_error)?; + Ok(Some(Arc::new(PinnedTable { + table: pinned, + snapshot_id: None, + schema: provider.schema(), + }))) } async fn table_type(&self, name: &str) -> Result> { @@ -151,70 +145,6 @@ impl SchemaProvider for PinningSchema { } } -/// A catalog-backed table: its catalog, by address, and its identifier. -type TableKey = (usize, TableIdent); - -fn key(provider: &IcebergCatalogTableProvider) -> TableKey { - let catalog = Arc::as_ptr(provider.catalog()).cast::<()>() as usize; - (catalog, provider.table_ident().clone()) -} - -async fn load(provider: &IcebergCatalogTableProvider) -> Result
{ - provider - .catalog() - .load_table(provider.table_ident()) - .await - .map_err(to_datafusion_error) -} - -/// The catalog-backed table a scan's pinned table was pinned from, if any. -fn origin(scan: &TableScan) -> Option<&IcebergCatalogTableProvider> { - let source = scan.source.downcast_ref::()?; - let pinned = source.table_provider.downcast_ref::()?; - pinned.origin.as_ref() -} - -/// Returns `view` with each catalog-backed Iceberg table it reads pinned to -/// the version its catalog serves now, loading each table once, or `None` if -/// it reads none. DataFusion inlines a view into any plan built over it, so a -/// view's plan holds tables, never other views. -async fn repin(view: &ViewTable) -> Result> { - let mut providers = HashMap::new(); - view.logical_plan().apply_with_subqueries(|node| { - if let LogicalPlan::TableScan(scan) = node - && let Some(origin) = origin(scan) - { - providers - .entry(key(origin)) - .or_insert_with(|| origin.clone()); - } - Ok(TreeNodeRecursion::Continue) - })?; - if providers.is_empty() { - return Ok(None); - } - let loads = providers.into_iter().map(|(key, provider)| async move { - Ok::<_, DataFusionError>((key, load(&provider).await?)) - }); - let tables: HashMap<_, _> = try_join_all(loads).await?.into_iter().collect(); - let plan = view - .logical_plan() - .clone() - .transform_up_with_subqueries(|node| { - let LogicalPlan::TableScan(mut scan) = node else { - return Ok(Transformed::no(node)); - }; - let Some(origin) = origin(&scan) else { - return Ok(Transformed::no(LogicalPlan::TableScan(scan))); - }; - let table = tables[&key(origin)].clone(); - scan.source = provider_as_source(Arc::new(PinnedTable::pin(origin, table))); - Ok(Transformed::yes(LogicalPlan::TableScan(scan))) - })? - .data; - Ok(Some(ViewTable::new(plan, view.definition().cloned()))) -} - /// One version of an Iceberg table, read with the schema the client planned /// against. /// @@ -228,22 +158,6 @@ pub(crate) struct PinnedTable { pub(crate) table: Table, pub(crate) snapshot_id: Option, pub(crate) schema: SchemaRef, - /// The catalog-backed table it was pinned from, to pin it again. Not sent - /// between nodes. - pub(crate) origin: Option, -} - -impl PinnedTable { - /// `table`, a version of `provider` loaded from its catalog, read with the - /// provider's schema, which it caches when it is created. - fn pin(provider: &IcebergCatalogTableProvider, table: Table) -> Self { - Self { - table, - snapshot_id: None, - schema: provider.schema(), - origin: Some(provider.clone()), - } - } } #[async_trait] diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 6a75c2bdaf..663c785724 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -54,8 +54,8 @@ //! - Reads only. //! - Each scan runs as a single task. //! - No metadata tables such as `t$snapshots`. -//! - A `DataFrame` or prepared statement reused across runs reads the version -//! from when it was planned. +//! - A view, or a `DataFrame` or prepared statement reused across runs, reads +//! the version from when it was planned. //! - Storage credentials travel in plain text. mod catalog; diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 3445eb7f2b..477bea5d32 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -90,7 +90,6 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { table: table.load()?, snapshot_id, schema, - origin: None, })) } diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index 8e0e77bcd4..ec8b13f149 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -38,8 +38,10 @@ mod standalone { assert_standalone_scans(memory_catalog(dir.path()).await).await } + // A known limitation: a view stores its planned query, and tables are + // pinned when a query is planned. #[tokio::test] - async fn should_read_a_views_latest_version() -> Result<()> { + async fn should_keep_a_views_version() -> Result<()> { let dir = tempfile::tempdir()?; let catalog = memory_catalog(dir.path()).await; let state = SessionStateBuilder::new() @@ -51,34 +53,17 @@ mod standalone { insert(&catalog, "(1, 'a')").await; ctx.sql("CREATE VIEW v AS SELECT * FROM t").await?; - // A view over a view, reading `t` again in a subquery. - ctx.sql("CREATE VIEW w AS SELECT name FROM v WHERE id IN (SELECT id FROM t)") - .await?; insert(&catalog, "(2, 'b')").await; - let result = ctx - .sql("SELECT * FROM v ORDER BY id") - .await? - .collect() - .await?; + let result = ctx.sql("SELECT * FROM v").await?.collect().await?; let expected = [ "+----+------+", "| id | name |", "+----+------+", "| 1 | a |", - "| 2 | b |", "+----+------+", ]; assert_batches_eq!(expected, &result); - let result = ctx - .sql("SELECT * FROM w ORDER BY name") - .await? - .collect() - .await?; - let expected = [ - "+------+", "| name |", "+------+", "| a |", "| b |", "+------+", - ]; - assert_batches_eq!(expected, &result); Ok(()) } From eb78175a3ca297a1016ab4e19d74b65968991839 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 5 Nov 2026 20:13:54 -0500 Subject: [PATCH 19/31] Pin ballista-iceberg's Ballista crates to a commit on main Depend on the Ballista crates at a commit that includes #2752 instead of the crates beside it, and stop running the workflow on changes under ballista/, so a dependency bump in Ballista can't break this build. --- .github/workflows/iceberg.yml | 7 +++---- integrations/iceberg/Cargo.lock | 7 +++++++ integrations/iceberg/Cargo.toml | 13 ++++++++----- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/iceberg.yml b/.github/workflows/iceberg.yml index 3cbe60d50d..64a72a55f1 100644 --- a/.github/workflows/iceberg.yml +++ b/.github/workflows/iceberg.yml @@ -16,8 +16,9 @@ # under the License. -# ballista-iceberg is its own workspace, outside Ballista's, so it doesn't -# block Ballista's build or release; this workflow checks it separately. +# ballista-iceberg is its own workspace, outside Ballista's, and depends on a +# pinned commit of Ballista, so it doesn't block Ballista's build or release; +# this workflow checks it separately. name: Ballista Iceberg concurrency: @@ -28,13 +29,11 @@ on: push: branches: [ "main" ] paths: - - "ballista/**" - "integrations/iceberg/**" - ".github/workflows/iceberg.yml" - "!**/*.md" pull_request: paths: - - "ballista/**" - "integrations/iceberg/**" - ".github/workflows/iceberg.yml" - "!**/*.md" diff --git a/integrations/iceberg/Cargo.lock b/integrations/iceberg/Cargo.lock index ff381735f2..145d554862 100644 --- a/integrations/iceberg/Cargo.lock +++ b/integrations/iceberg/Cargo.lock @@ -668,6 +668,7 @@ dependencies = [ [[package]] name = "ballista" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "async-trait", "ballista-core", @@ -682,6 +683,7 @@ dependencies = [ [[package]] name = "ballista-api-types" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "serde", "utoipa", @@ -690,6 +692,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "arrow-flight", "async-trait", @@ -722,6 +725,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "arrow", "arrow-flight", @@ -750,6 +754,7 @@ dependencies = [ [[package]] name = "ballista-flight-sql" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "arrow", "arrow-flight", @@ -768,6 +773,7 @@ dependencies = [ [[package]] name = "ballista-history" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "ballista-api-types", "log", @@ -805,6 +811,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" +source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" dependencies = [ "arrow-flight", "async-trait", diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index 4ec947e737..c2f2059a5c 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -43,9 +43,12 @@ ignored = ["arrow-schema"] [dependencies] arrow-schema = { version = "59.2.0", features = ["serde"] } async-trait = "0.1" -ballista-core = { path = "../../ballista/core", version = "55.0.0" } -ballista-executor = { path = "../../ballista/executor", version = "55.0.0", default-features = false, optional = true } -ballista-scheduler = { path = "../../ballista/scheduler", version = "55.0.0", default-features = false, optional = true } +# A commit on main with ComposedLogicalExtensionCodec (#2752), not the crates +# in this repo, so a change to Ballista doesn't break this build. Move to +# crates.io once a release includes #2752. +ballista-core = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0" } +ballista-executor = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", default-features = false, optional = true } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", default-features = false, optional = true } datafusion = "55" datafusion-proto = "55" futures = "0.3" @@ -61,8 +64,8 @@ tokio = { version = "1", features = ["rt-multi-thread"] } [dev-dependencies] arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } -ballista = { path = "../../ballista/client", version = "55.0.0", features = ["standalone"] } -ballista-scheduler = { path = "../../ballista/scheduler", version = "55.0.0", features = ["flight-sql"] } +ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = ["standalone"] } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = ["flight-sql"] } iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } tempfile = "3.27" testcontainers = { version = "0.28", features = ["http_wait_plain"] } From 8f731f5118bc77655fe99ae0484f866ea53b3ca4 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 5 Nov 2026 20:13:54 -0500 Subject: [PATCH 20/31] Refuse to encode an Iceberg scan with more than one partition Ballista restricts each task to its partitions only for the scans it knows, so each task would read every partition of an Iceberg scan and return duplicate rows. Fail loudly once a dependency bump starts splitting scans. --- integrations/iceberg/src/physical_codec.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index 97f41db4b3..184388ff80 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -24,7 +24,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::{Result, not_impl_err}; use datafusion::datasource::source::DataSourceExec; use datafusion::execution::TaskContext; -use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion_proto::physical_plan::{ ComposedPhysicalExtensionCodec, PhysicalExtensionCodec, PhysicalProtoConverterExtension, @@ -96,6 +96,15 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { else { return not_impl_err!("{} isn't an Iceberg scan", node.name()); }; + // Ballista restricts each task to its partitions only for the scans it + // knows, such as file scans, so each task would read every partition + // of this one and rows would come back duplicated. + let partitions = node.output_partitioning().partition_count(); + if partitions > 1 { + return not_impl_err!( + "Iceberg scans with {partitions} partitions aren't supported on Ballista yet" + ); + } wire::encode( buf, &ScanWire { From f11230e131561941e9ca983fde4885a86c0f2aab Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 5 Nov 2026 20:13:54 -0500 Subject: [PATCH 21/31] Document the catalog wrapper, the partition guard and the Ballista pin --- docs/source/user-guide/iceberg.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index b659e2836e..fe87e4dca6 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -27,7 +27,8 @@ the executors. The crate is not published yet. It depends on unreleased revisions of iceberg-datafusion and iceberg-rust, and needs Rust 1.95. It is its own -workspace, with its own `Cargo.lock` and CI, outside Ballista's, so a +workspace, with its own `Cargo.lock` and CI, outside Ballista's, and depends +on a pinned commit of Ballista rather than the crates beside it, so a DataFusion upgrade in Ballista that iceberg-rust doesn't support yet doesn't block Ballista's build or release. Build and test it from `integrations/iceberg`. @@ -139,7 +140,8 @@ and executors must all set the same codecs in the same order. scheduler yet. - **One task per scan.** Iceberg scans do not split into partitions yet, so one executor reads the whole table. Operators above the scan still run in - parallel. + parallel. A scan reporting more than one partition fails to encode, rather + than being read in full by every task. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. @@ -152,6 +154,9 @@ and executors must all set the same codecs in the same order. `ballista-iceberg` and be readied with `with_iceberg` (see [SQL clients](#sql-clients)). The tests cover standalone mode and an in-process scheduler serving Flight SQL. +- **`ctx.catalog(name)` returns a wrapper.** `register_iceberg` wraps each + catalog to pin its tables, so the result doesn't downcast to the type you + registered. Keep the `Arc` you registered to reach it. - **A mounted catalog lists its tables once.** `IcebergCatalogProvider` reads the table names when it is built, so a table created afterwards isn't visible until the catalog is mounted again. From a75a72126a72b9e28dfc7c41d53519c4504fe5b1 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 5 Nov 2026 20:19:26 -0500 Subject: [PATCH 22/31] Format ballista-iceberg's Cargo.toml with taplo --- integrations/iceberg/Cargo.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index c2f2059a5c..7e121b05e7 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -64,8 +64,12 @@ tokio = { version = "1", features = ["rt-multi-thread"] } [dev-dependencies] arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } -ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = ["standalone"] } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = ["flight-sql"] } +ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = [ + "standalone", +] } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = [ + "flight-sql", +] } iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } tempfile = "3.27" testcontainers = { version = "0.28", features = ["http_wait_plain"] } From 56cd7bd3b5af7795a0365afe95f8c9f28fb59dfb Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 8 Nov 2026 13:29:22 -0500 Subject: [PATCH 23/31] Send Iceberg tables without their metadata A table's metadata grows with each snapshot, and every scan of it carried a copy, so a self-join on a table with a long history could exceed the gRPC message limit. Send only the metadata file's location: the table cache keeps the extra read to once per version. --- docs/source/user-guide/iceberg.md | 6 ++-- integrations/iceberg/src/logical_codec.rs | 4 +-- integrations/iceberg/src/wire.rs | 34 +++++------------------ integrations/iceberg/tests/codec.rs | 4 +-- 4 files changed, 13 insertions(+), 35 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index fe87e4dca6..89c8a01432 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -114,9 +114,9 @@ Metadata files never change, so every node rebuilds exactly that version. - **A static table** is sent as the version it holds. - **A scan** is sent as the version the scheduler planned it against. -Tables sent from the client also carry the metadata itself, so the scheduler -needn't read the file. Scans sent to executors carry only the location, which -keeps each task's plan small. +Only the location is sent, not the metadata, so a plan's size doesn't grow +with the table's history. Each node reads a version's metadata file when it +first rebuilds it, and keeps up to 16 rebuilt versions. Only the node that plans a query talks to a catalog: the client, or the scheduler for SQL clients. Executors never need catalog configuration. diff --git a/integrations/iceberg/src/logical_codec.rs b/integrations/iceberg/src/logical_codec.rs index 477bea5d32..23cd8a7b91 100644 --- a/integrations/iceberg/src/logical_codec.rs +++ b/integrations/iceberg/src/logical_codec.rs @@ -103,12 +103,12 @@ impl LogicalExtensionCodec for IcebergLogicalCodec { node.downcast_ref::() { ProviderWire { - table: TableWire::with_metadata(provider.table())?, + table: TableWire::new(provider.table())?, snapshot_id: provider.snapshot_id(), } } else if let Some(pinned) = node.downcast_ref::() { ProviderWire { - table: TableWire::with_metadata(&pinned.table)?, + table: TableWire::new(&pinned.table)?, snapshot_id: pinned.snapshot_id, } } else if node.is::() { diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index b7632f3ca5..ac7bd79afc 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -19,11 +19,11 @@ use std::collections::VecDeque; use std::fmt; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::{LazyLock, Mutex}; use datafusion::common::{DataFusionError, exec_datafusion_err}; use iceberg::io::FileIO; -use iceberg::spec::{TableMetadata, TableMetadataRef}; +use iceberg::spec::TableMetadata; use iceberg::table::Table; use iceberg::{Runtime, TableIdent}; use iceberg_datafusion::to_datafusion_error; @@ -71,10 +71,6 @@ pub(crate) struct TableWire { metadata_location: String, file_io: serde_json::Value, readonly: bool, - /// The content of `metadata_location`, sent from the client so the - /// scheduler needn't read it; scans sent to executors leave it out. - #[serde(default, skip_serializing_if = "Option::is_none")] - metadata: Option, } impl fmt::Debug for TableWire { @@ -102,27 +98,15 @@ impl TableWire { .to_string(), file_io: serde_json::from_slice(&file_io).map_err(json_err)?, readonly: table.readonly(), - metadata: None, - }) - } - - /// Like [`TableWire::new`], also carrying the table's metadata. - pub(crate) fn with_metadata(table: &Table) -> Result { - Ok(Self { - metadata: Some(table.metadata_ref()), - ..Self::new(table)? }) } /// Rebuilds the table, or returns it from [`TABLES`]. - pub(crate) fn load(mut self) -> Result { - // Keyed without the metadata, which its location determines. - let metadata = self.metadata.take(); + pub(crate) fn load(self) -> Result { if let Some(table) = self.touch(&mut TABLES.lock().unwrap()) { return Ok(table); } - let table = - block_on(self.clone().read(metadata))?.map_err(to_datafusion_error)?; + let table = block_on(self.clone().read())?.map_err(to_datafusion_error)?; let mut tables = TABLES.lock().unwrap(); // Another thread may have rebuilt it meanwhile. if let Some(table) = self.touch(&mut tables) { @@ -143,14 +127,10 @@ impl TableWire { Some(table) } - async fn read(self, metadata: Option) -> iceberg::Result
{ + async fn read(self) -> iceberg::Result
{ let file_io = FileIO::deserialize_all(&serde_json::to_vec(&self.file_io)?)?; - let metadata = match metadata { - Some(metadata) => metadata, - None => Arc::new( - TableMetadata::read_from(&file_io, &self.metadata_location).await?, - ), - }; + let metadata = + TableMetadata::read_from(&file_io, &self.metadata_location).await?; Table::builder() .metadata(metadata) .metadata_location(self.metadata_location) diff --git a/integrations/iceberg/tests/codec.rs b/integrations/iceberg/tests/codec.rs index e33f87db33..58252db792 100644 --- a/integrations/iceberg/tests/codec.rs +++ b/integrations/iceberg/tests/codec.rs @@ -102,12 +102,10 @@ mod iceberg_nodes { logical_plan_to_bytes_with_extension_codec(&plan, codec.as_ref()) }; - let pinned = insert(&catalog, "(1, 'a')").await; + insert(&catalog, "(1, 'a')").await; let select = send("SELECT a.id, b.name FROM t a CROSS JOIN t b").await?; let insert_c = send("INSERT INTO t VALUES (3, 'c')").await?; insert(&catalog, "(2, 'b')").await; - // Decoding uses the metadata the client sent, not the file. - std::fs::remove_file(pinned.metadata_location().unwrap())?; let run = async |bytes: &[u8]| { let plan = logical_plan_from_bytes_with_extension_codec( From aa0473fb2a3757fb18594a1019385e3a709d3424 Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 8 Nov 2026 13:29:22 -0500 Subject: [PATCH 24/31] Say when the Iceberg table cache rebuilds a version again --- integrations/iceberg/src/wire.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/integrations/iceberg/src/wire.rs b/integrations/iceberg/src/wire.rs index ac7bd79afc..05383c0ca9 100644 --- a/integrations/iceberg/src/wire.rs +++ b/integrations/iceberg/src/wire.rs @@ -57,9 +57,11 @@ pub(crate) fn block_on( /// but each can hold megabytes of metadata. const TABLE_CACHE_CAPACITY: usize = 16; -/// Tables rebuilt by [`TableWire::load`], least recently used first, so each -/// process rebuilds a version once. Metadata files never change, so entries -/// can't go stale. +/// Tables rebuilt by [`TableWire::load`], least recently used first. Metadata +/// files never change, so entries can't go stale. Entries are keyed by the +/// table's `FileIO` too, so a process rebuilds a version once only while its +/// storage credentials stay the same: a catalog that vends new credentials on +/// each load makes each query rebuild it. static TABLES: LazyLock>> = LazyLock::new(|| Mutex::new(VecDeque::new())); From abcf538e69f5edb7a3ee9651666f2fddd65bfeef Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 8 Nov 2026 13:29:22 -0500 Subject: [PATCH 25/31] Document that encrypted Iceberg tables aren't supported --- docs/source/user-guide/iceberg.md | 2 ++ integrations/iceberg/src/lib.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 89c8a01432..d41866e0a2 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -142,6 +142,8 @@ and executors must all set the same codecs in the same order. executor reads the whole table. Operators above the scan still run in parallel. A scan reporting more than one partition fails to encode, rather than being read in full by every task. +- **No encrypted tables.** A table with `encryption.key-id` set fails to load + on the scheduler and executors, which have no key management client. - **Storage credentials travel in plain text.** Each table's serialized `FileIO` includes them, so the links between client, scheduler and executors must be trusted. diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 663c785724..6b2fe76887 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -56,6 +56,7 @@ //! - No metadata tables such as `t$snapshots`. //! - A view, or a `DataFrame` or prepared statement reused across runs, reads //! the version from when it was planned. +//! - No encrypted tables. //! - Storage credentials travel in plain text. mod catalog; From 7755c0f1d6c1c52aae8ba36675df0abcfec61a8b Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 8 Nov 2026 13:29:35 -0500 Subject: [PATCH 26/31] Document calling with_iceberg once, with every catalog --- docs/source/user-guide/iceberg.md | 4 +++- integrations/iceberg/src/executor.rs | 2 +- integrations/iceberg/src/scheduler.rs | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index d41866e0a2..3ffca84d8d 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -96,7 +96,9 @@ let config = ExecutorProcessConfig::default().with_iceberg(); ``` Both keep any codecs the config already has, and the scheduler keeps its -session builder, mounting the catalogs on each session it builds. A client can +session builder, mounting the catalogs on each session it builds. Call each +once, passing the scheduler every catalog: a second call adds the codecs +again, so the scheduler's would no longer match the executors'. A client can then run `SELECT * FROM rest.ns.t`. Tables are pinned the same way as on the client, while the scheduler plans the query. diff --git a/integrations/iceberg/src/executor.rs b/integrations/iceberg/src/executor.rs index 5dafad2bde..1d5adcf04e 100644 --- a/integrations/iceberg/src/executor.rs +++ b/integrations/iceberg/src/executor.rs @@ -23,7 +23,7 @@ use crate::{IcebergLogicalCodec, IcebergPhysicalCodec}; /// Readies an [`ExecutorProcessConfig`] for Iceberg. pub trait ExecutorProcessConfigExt { - /// Adds the Iceberg codecs after any codecs already set. + /// Adds the Iceberg codecs after any codecs already set. Call it once. fn with_iceberg(self) -> Self; } diff --git a/integrations/iceberg/src/scheduler.rs b/integrations/iceberg/src/scheduler.rs index 9dccfb35ab..2c3493df34 100644 --- a/integrations/iceberg/src/scheduler.rs +++ b/integrations/iceberg/src/scheduler.rs @@ -31,6 +31,9 @@ pub trait SchedulerConfigExt { /// `catalogs` in each session, readied with [`register_iceberg`]. The /// scheduler plans queries from Flight SQL clients itself, so they can /// then query those catalogs. + /// + /// Call it once, with every catalog: a second call adds the codecs again, + /// and the scheduler's codecs no longer match the executors'. fn with_iceberg>( self, catalogs: impl IntoIterator)>, From cd54fd58f02a9ec6e11a17bf8438f15b2317f1df Mon Sep 17 00:00:00 2001 From: Noah Date: Sun, 8 Nov 2026 13:29:35 -0500 Subject: [PATCH 27/31] Correct which operators above an Iceberg scan run in parallel --- docs/source/user-guide/iceberg.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 3ffca84d8d..49d8563982 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -141,9 +141,8 @@ and executors must all set the same codecs in the same order. - **No metadata tables.** Tables such as `t$snapshots` can't be sent to the scheduler yet. - **One task per scan.** Iceberg scans do not split into partitions yet, so one - executor reads the whole table. Operators above the scan still run in - parallel. A scan reporting more than one partition fails to encode, rather - than being read in full by every task. + executor reads the whole table. Operators after the first shuffle still run + in parallel. A scan reporting more than one partition fails to encode. - **No encrypted tables.** A table with `encryption.key-id` set fails to load on the scheduler and executors, which have no key management client. - **Storage credentials travel in plain text.** Each table's serialized From 8c19489c8ded98bf9a48894fbe237fcf524fe3c2 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 10 Nov 2026 20:36:58 -0500 Subject: [PATCH 28/31] Pin ballista-iceberg to a Ballista commit with #2721 --- integrations/iceberg/Cargo.lock | 44 ++++++++++++++------------------- integrations/iceberg/Cargo.toml | 17 +++++++------ 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/integrations/iceberg/Cargo.lock b/integrations/iceberg/Cargo.lock index 145d554862..1ecfb58b49 100644 --- a/integrations/iceberg/Cargo.lock +++ b/integrations/iceberg/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "ballista" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "async-trait", "ballista-core", @@ -683,7 +683,7 @@ dependencies = [ [[package]] name = "ballista-api-types" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "serde", "utoipa", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "arrow-flight", "async-trait", @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "arrow", "arrow-flight", @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "ballista-flight-sql" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "arrow", "arrow-flight", @@ -773,7 +773,7 @@ dependencies = [ [[package]] name = "ballista-history" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "ballista-api-types", "log", @@ -811,7 +811,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d151e3275081f0506cb252d45e10553841c3ba9c#d151e3275081f0506cb252d45e10553841c3ba9c" +source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" dependencies = [ "arrow-flight", "async-trait", @@ -820,6 +820,7 @@ dependencies = [ "ballista-core", "ballista-flight-sql", "ballista-history", + "blake3", "clap", "dashmap", "datafusion", @@ -2487,7 +2488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3812,7 +3813,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4445,8 +4446,8 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", - "itertools 0.13.0", + "heck 0.4.1", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -4467,7 +4468,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.119", @@ -4602,7 +4603,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4986,7 +4987,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5044,7 +5045,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5599,7 +5600,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6353,7 +6354,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6472,15 +6473,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.61.2" diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index 7e121b05e7..3adb26d7cf 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -43,12 +43,13 @@ ignored = ["arrow-schema"] [dependencies] arrow-schema = { version = "59.2.0", features = ["serde"] } async-trait = "0.1" -# A commit on main with ComposedLogicalExtensionCodec (#2752), not the crates -# in this repo, so a change to Ballista doesn't break this build. Move to -# crates.io once a release includes #2752. -ballista-core = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0" } -ballista-executor = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", default-features = false, optional = true } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", default-features = false, optional = true } +# A commit on main with ComposedLogicalExtensionCodec (#2752) and the +# partition-sliceable resolver (#2721), not the crates in this repo, so a +# change to Ballista doesn't break this build. Move to crates.io once a +# release includes both. +ballista-core = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0" } +ballista-executor = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", default-features = false, optional = true } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", default-features = false, optional = true } datafusion = "55" datafusion-proto = "55" futures = "0.3" @@ -64,10 +65,10 @@ tokio = { version = "1", features = ["rt-multi-thread"] } [dev-dependencies] arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } -ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = [ +ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", features = [ "standalone", ] } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d151e3275081f0506cb252d45e10553841c3ba9c", version = "55.0.0", features = [ +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", features = [ "flight-sql", ] } iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } From 42c909b434e0b61bd8d1334d2292a080939be524 Mon Sep 17 00:00:00 2001 From: Noah Date: Sat, 14 Nov 2026 12:27:31 -0500 Subject: [PATCH 29/31] Mark Iceberg support experimental, document split-union limits, bump pins - Mark the docs page and the crate docs experimental. - Document that a UNION ALL on a stage the adaptive planner splits can fail (a union of Iceberg scans, #2852) or return rows more than once (branches that merge their input, such as under ORDER BY, #2854), and that the static planner avoids both. - Test a split-stage UNION ALL of Iceberg scans under both planners. - Pin Ballista to the last commit on main before DataFusion 56, which iceberg-datafusion doesn't support yet, and iceberg-datafusion to its latest main. iceberg-rust stays at the revision iceberg-datafusion pins. --- docs/source/user-guide/iceberg.md | 16 +++++- integrations/iceberg/Cargo.lock | 19 +++---- integrations/iceberg/Cargo.toml | 17 ++++--- integrations/iceberg/src/lib.rs | 5 ++ integrations/iceberg/tests/scan.rs | 79 +++++++++++++++++++++++++++++- 5 files changed, 117 insertions(+), 19 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 49d8563982..b3ce6c93b5 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -17,7 +17,7 @@ under the License. --> -# Apache Iceberg +# Apache Iceberg (experimental) The `ballista-iceberg` crate (`integrations/iceberg`) lets Ballista scan [Apache Iceberg](https://iceberg.apache.org/) tables. On top of @@ -33,6 +33,8 @@ DataFusion upgrade in Ballista that iceberg-rust doesn't support yet doesn't block Ballista's build or release. Build and test it from `integrations/iceberg`. +Iceberg support is experimental. Its behavior and API may change. + ## Usage Ready the session with `register_iceberg`, then mount an Iceberg catalog: @@ -143,6 +145,18 @@ and executors must all set the same codecs in the same order. - **One task per scan.** Iceberg scans do not split into partitions yet, so one executor reads the whole table. Operators after the first shuffle still run in parallel. A scan reporting more than one partition fails to encode. +- **A `UNION ALL` can fail or return wrong rows when the adaptive planner + splits its stage between tasks.** That happens with + `ballista.scheduler.max_partitions_per_task` set, or when the executor has + fewer free task slots than the union has partitions. A union of Iceberg scans + fails with an error that suggests `SchedulerConfig::partition_sliceable_resolver`, + which standalone mode can't set + ([#2852](https://github.com/apache/datafusion-ballista/pull/2852)). A union + whose branches each merge their input into one partition, as with an + `ORDER BY` over the union, a `LIMIT` in a branch or aggregates without + `GROUP BY`, returns some rows more than once, whatever the table format + ([#2854](https://github.com/apache/datafusion-ballista/issues/2854)). Setting + `ballista.planner.adaptive.enabled` to `false` avoids both. - **No encrypted tables.** A table with `encryption.key-id` set fails to load on the scheduler and executors, which have no key management client. - **Storage credentials travel in plain text.** Each table's serialized diff --git a/integrations/iceberg/Cargo.lock b/integrations/iceberg/Cargo.lock index 1ecfb58b49..165cec1ded 100644 --- a/integrations/iceberg/Cargo.lock +++ b/integrations/iceberg/Cargo.lock @@ -668,7 +668,7 @@ dependencies = [ [[package]] name = "ballista" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "async-trait", "ballista-core", @@ -683,7 +683,7 @@ dependencies = [ [[package]] name = "ballista-api-types" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "serde", "utoipa", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "ballista-core" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "arrow-flight", "async-trait", @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "ballista-executor" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "arrow", "arrow-flight", @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "ballista-flight-sql" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "arrow", "arrow-flight", @@ -773,7 +773,7 @@ dependencies = [ [[package]] name = "ballista-history" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "ballista-api-types", "log", @@ -811,7 +811,7 @@ dependencies = [ [[package]] name = "ballista-scheduler" version = "55.0.0" -source = "git+https://github.com/apache/datafusion-ballista?rev=d98720804369917151052e1c280a1dae711817fb#d98720804369917151052e1c280a1dae711817fb" +source = "git+https://github.com/apache/datafusion-ballista?rev=2f4938ef05c38f8e68f944d6aed22f2f5714a9ed#2f4938ef05c38f8e68f944d6aed22f2f5714a9ed" dependencies = [ "arrow-flight", "async-trait", @@ -836,6 +836,7 @@ dependencies = [ "rand 0.10.3", "serde", "serde_json", + "socket2", "tokio", "tokio-stream", "tonic", @@ -3150,7 +3151,7 @@ dependencies = [ [[package]] name = "iceberg-datafusion" version = "0.12.0" -source = "git+https://github.com/apache/datafusion-iceberg?rev=b641e5ebe9e92967499ae7e440ba10d7797712df#b641e5ebe9e92967499ae7e440ba10d7797712df" +source = "git+https://github.com/apache/datafusion-iceberg?rev=82cea1462bff959021b881311c9ef9f4f22328af#82cea1462bff959021b881311c9ef9f4f22328af" dependencies = [ "async-trait", "bytes", @@ -4446,7 +4447,7 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap", diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index 3adb26d7cf..4e84c21510 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -45,18 +45,19 @@ arrow-schema = { version = "59.2.0", features = ["serde"] } async-trait = "0.1" # A commit on main with ComposedLogicalExtensionCodec (#2752) and the # partition-sliceable resolver (#2721), not the crates in this repo, so a -# change to Ballista doesn't break this build. Move to crates.io once a -# release includes both. -ballista-core = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0" } -ballista-executor = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", default-features = false, optional = true } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", default-features = false, optional = true } +# change to Ballista doesn't break this build. It is the last commit before +# Ballista moved to DataFusion 56, which iceberg-datafusion doesn't support +# yet. Move to crates.io once a release includes both. +ballista-core = { git = "https://github.com/apache/datafusion-ballista", rev = "2f4938ef05c38f8e68f944d6aed22f2f5714a9ed", version = "55.0.0" } +ballista-executor = { git = "https://github.com/apache/datafusion-ballista", rev = "2f4938ef05c38f8e68f944d6aed22f2f5714a9ed", version = "55.0.0", default-features = false, optional = true } +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "2f4938ef05c38f8e68f944d6aed22f2f5714a9ed", version = "55.0.0", default-features = false, optional = true } datafusion = "55" datafusion-proto = "55" futures = "0.3" # Not released yet. Keep `iceberg` at the revision iceberg-datafusion's own # workspace pins, so cargo resolves a single iceberg crate. iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } -iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg", rev = "b641e5ebe9e92967499ae7e440ba10d7797712df" } +iceberg-datafusion = { git = "https://github.com/apache/datafusion-iceberg", rev = "82cea1462bff959021b881311c9ef9f4f22328af" } # Storage for rebuilt tables: S3 and local files by default. iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } serde = { version = "1.0", features = ["derive", "rc"] } @@ -65,10 +66,10 @@ tokio = { version = "1", features = ["rt-multi-thread"] } [dev-dependencies] arrow-flight = { version = "59.2.0", features = ["flight-sql-experimental"] } -ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", features = [ +ballista = { git = "https://github.com/apache/datafusion-ballista", rev = "2f4938ef05c38f8e68f944d6aed22f2f5714a9ed", version = "55.0.0", features = [ "standalone", ] } -ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "d98720804369917151052e1c280a1dae711817fb", version = "55.0.0", features = [ +ballista-scheduler = { git = "https://github.com/apache/datafusion-ballista", rev = "2f4938ef05c38f8e68f944d6aed22f2f5714a9ed", version = "55.0.0", features = [ "flight-sql", ] } iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", rev = "5b82ab99aeaf50bf4f5b3b776ac9cd6380a5f3e3" } diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 6b2fe76887..82fda9c6c0 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -17,6 +17,8 @@ //! Apache Iceberg support for Ballista. //! +//! This crate is experimental. Its behavior and API may change. +//! //! Ballista sends plans between nodes as bytes, so these codecs send each Iceberg //! table and scan as one version of the table, which any node can rebuild //! without a catalog. A catalog-backed table is sent as the version its catalog @@ -58,6 +60,9 @@ //! the version from when it was planned. //! - No encrypted tables. //! - Storage credentials travel in plain text. +//! - A `UNION ALL` can fail, or with an `ORDER BY` return rows more than once, +//! when the adaptive planner splits its stage between tasks. Disabling the +//! adaptive planner avoids it. mod catalog; #[cfg(feature = "executor")] diff --git a/integrations/iceberg/tests/scan.rs b/integrations/iceberg/tests/scan.rs index ec8b13f149..2eed8d97b0 100644 --- a/integrations/iceberg/tests/scan.rs +++ b/integrations/iceberg/tests/scan.rs @@ -21,14 +21,18 @@ mod common; #[cfg(test)] mod standalone { + use std::path::Path; use std::sync::Arc; use ballista::prelude::{SessionConfigExt, SessionContextExt}; + use ballista_core::config::{ + BALLISTA_ADAPTIVE_PLANNER_ENABLED, BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, + }; use ballista_iceberg::register_iceberg; - use datafusion::assert_batches_eq; use datafusion::error::Result; use datafusion::execution::SessionStateBuilder; use datafusion::prelude::{SessionConfig, SessionContext}; + use datafusion::{assert_batches_eq, assert_batches_sorted_eq}; use crate::common::{assert_standalone_scans, catalog_table, insert, memory_catalog}; @@ -67,4 +71,77 @@ mod standalone { Ok(()) } + + /// Standalone Ballista over `ns.t` holding `(1, 'a'), (2, 'b')`, splitting + /// every stage into one task per partition, so each branch of a union runs + /// in a task of its own. + async fn split_stage_context(dir: &Path, adaptive: bool) -> Result { + let catalog = memory_catalog(dir).await; + insert(&catalog, "(1, 'a'), (2, 'b')").await; + let config = SessionConfig::new_with_ballista() + .set_str(BALLISTA_ADAPTIVE_PLANNER_ENABLED, &adaptive.to_string()) + .set_str(BALLISTA_SCHEDULER_MAX_PARTITIONS_PER_TASK, "1"); + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .build(); + let ctx = SessionContext::standalone_with_state(register_iceberg(state)).await?; + ctx.register_table("t", Arc::new(catalog_table(&catalog).await))?; + Ok(ctx) + } + + const UNION: &str = "SELECT id, name FROM t UNION ALL SELECT id, name FROM t"; + + #[tokio::test] + async fn should_union_scans_on_a_split_stage_with_the_static_planner() -> Result<()> { + let dir = tempfile::tempdir()?; + let ctx = split_stage_context(dir.path(), false).await?; + let expected = [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 1 | a |", + "| 1 | a |", + "| 2 | b |", + "| 2 | b |", + "+----+------+", + ]; + + let result = ctx.sql(UNION).await?.collect().await?; + assert_batches_sorted_eq!(expected, &result); + + // Each branch is sorted before the union, which the adaptive planner + // gets wrong too (#2854). + let sorted = format!("{UNION} ORDER BY id"); + let result = ctx.sql(&sorted).await?.collect().await?; + assert_batches_eq!(expected, &result); + + Ok(()) + } + + // A known limitation: the adaptive planner keeps both branches in the + // union's stage, and the task that owns one branch can't restrict the + // other branch's Iceberg scan to none of its partitions (#2852). + #[tokio::test] + async fn should_refuse_a_union_of_scans_on_a_split_stage_with_the_adaptive_planner() + -> Result<()> { + let dir = tempfile::tempdir()?; + let ctx = split_stage_context(dir.path(), true).await?; + + let error = ctx.sql(UNION).await?.collect().await.unwrap_err(); + // The message starts with the job's id. + let error = error.to_string(); + let (_, cause) = error.split_once("DataFusion error: ").unwrap(); + assert_eq!( + cause, + "Error during planning: cannot restrict `DataSourceExec: format=iceberg, \ + projection=[id, name]` to partitions [] of its 1: the scheduler does not \ + know how to restrict a `DataSourceExec`, and left alone it would number \ + its partitions from zero in every task. A node of your own can be \ + restricted by `SchedulerConfig::partition_sliceable_resolver`, for \ + instance by implementing `PartitionSliceable` and calling it from there." + ); + + Ok(()) + } } From e5ec4322cf6640bb9645d7c39b841ec2ecbecced Mon Sep 17 00:00:00 2001 From: Noah Date: Sat, 14 Nov 2026 12:38:31 -0500 Subject: [PATCH 30/31] Correct the multi-partition guard's reason and a stale Rust version note Since #2721 the scheduler refuses a stage split over a scan it can't restrict, so the guard keeps a query from failing only on a busy cluster rather than from returning duplicate rows. Ballista main now needs Rust 1.95 too. Say what #2852 does in the union limitation. --- docs/source/user-guide/iceberg.md | 4 ++-- integrations/iceberg/Cargo.toml | 2 +- integrations/iceberg/src/physical_codec.rs | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index b3ce6c93b5..265bab0be1 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -150,8 +150,8 @@ and executors must all set the same codecs in the same order. `ballista.scheduler.max_partitions_per_task` set, or when the executor has fewer free task slots than the union has partitions. A union of Iceberg scans fails with an error that suggests `SchedulerConfig::partition_sliceable_resolver`, - which standalone mode can't set - ([#2852](https://github.com/apache/datafusion-ballista/pull/2852)). A union + which standalone mode can't set; + [#2852](https://github.com/apache/datafusion-ballista/pull/2852) fixes it. A union whose branches each merge their input into one partition, as with an `ORDER BY` over the union, a `LIMIT` in a branch or aggregates without `GROUP BY`, returns some rows more than once, whatever the table format diff --git a/integrations/iceberg/Cargo.toml b/integrations/iceberg/Cargo.toml index 4e84c21510..8c6ff44297 100644 --- a/integrations/iceberg/Cargo.toml +++ b/integrations/iceberg/Cargo.toml @@ -26,7 +26,7 @@ authors = ["Apache DataFusion "] keywords = ["iceberg", "ballista", "datafusion", "distributed"] categories = ["database"] edition = "2024" -# Required by the iceberg crates; Ballista itself still builds on 1.94. +# Required by the iceberg crates. rust-version = "1.95" # crates.io rejects the git dependencies on iceberg and iceberg-datafusion. publish = false diff --git a/integrations/iceberg/src/physical_codec.rs b/integrations/iceberg/src/physical_codec.rs index 184388ff80..dd6990e81c 100644 --- a/integrations/iceberg/src/physical_codec.rs +++ b/integrations/iceberg/src/physical_codec.rs @@ -96,9 +96,9 @@ impl PhysicalExtensionCodec for IcebergPhysicalCodec { else { return not_impl_err!("{} isn't an Iceberg scan", node.name()); }; - // Ballista restricts each task to its partitions only for the scans it - // knows, such as file scans, so each task would read every partition - // of this one and rows would come back duplicated. + // Ballista can't restrict this scan to a task's partitions, so the job + // would fail whenever its stage is split between tasks, which depends + // on how busy the cluster is. Refuse it every time instead. let partitions = node.output_partitioning().partition_count(); if partitions > 1 { return not_impl_err!( From 50a7dac7ed06aabf112882d8657d9e214e568fd8 Mon Sep 17 00:00:00 2001 From: Noah Date: Sat, 14 Nov 2026 13:22:14 -0500 Subject: [PATCH 31/31] Say a split UNION fails too, not only UNION ALL --- docs/source/user-guide/iceberg.md | 4 ++-- integrations/iceberg/src/lib.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/iceberg.md b/docs/source/user-guide/iceberg.md index 265bab0be1..82e2b64c14 100644 --- a/docs/source/user-guide/iceberg.md +++ b/docs/source/user-guide/iceberg.md @@ -145,8 +145,8 @@ and executors must all set the same codecs in the same order. - **One task per scan.** Iceberg scans do not split into partitions yet, so one executor reads the whole table. Operators after the first shuffle still run in parallel. A scan reporting more than one partition fails to encode. -- **A `UNION ALL` can fail or return wrong rows when the adaptive planner - splits its stage between tasks.** That happens with +- **A `UNION` or `UNION ALL` can fail or return wrong rows when the adaptive + planner splits its stage between tasks.** That happens with `ballista.scheduler.max_partitions_per_task` set, or when the executor has fewer free task slots than the union has partitions. A union of Iceberg scans fails with an error that suggests `SchedulerConfig::partition_sliceable_resolver`, diff --git a/integrations/iceberg/src/lib.rs b/integrations/iceberg/src/lib.rs index 82fda9c6c0..68d520d2ab 100644 --- a/integrations/iceberg/src/lib.rs +++ b/integrations/iceberg/src/lib.rs @@ -60,9 +60,9 @@ //! the version from when it was planned. //! - No encrypted tables. //! - Storage credentials travel in plain text. -//! - A `UNION ALL` can fail, or with an `ORDER BY` return rows more than once, -//! when the adaptive planner splits its stage between tasks. Disabling the -//! adaptive planner avoids it. +//! - A `UNION` or `UNION ALL` can fail, or with an `ORDER BY` return rows more +//! than once, when the adaptive planner splits its stage between tasks. +//! Disabling the adaptive planner avoids it. mod catalog; #[cfg(feature = "executor")]