From 2d22fc560e007ac81e68522f7fb0e1672c5904a4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:04:13 -0700 Subject: [PATCH 1/2] feat: add datafusion-spark-cli, a SQL shell with the Spark functions apache/datafusion#27337 removed `datafusion-cli --spark`, so there is no longer a way to try the Spark-compatible functions interactively. Add a `datafusion-spark-cli` binary that uses the `datafusion-cli` library for the shell, with the Spark SQL dialect and functions always enabled and ANSI mode on by default, as in Spark 4. The crate is not published, and is not a default workspace member, so a plain `cargo build` or `cargo test` does not build its dependencies. A new workflow builds it and runs a smoke test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/cli.yml | 35 + AGENTS.md | 2 + Cargo.lock | 1653 +++++++++++++++++++++- Cargo.toml | 5 +- README.md | 2 + crates/cli/Cargo.toml | 31 + crates/cli/src/main.rs | 249 ++++ crates/spark/README.md | 13 +- crates/sqllogictest/test_files/README.md | 7 +- 9 files changed, 1924 insertions(+), 73 deletions(-) create mode 100644 .github/workflows/cli.yml create mode 100644 crates/cli/Cargo.toml create mode 100644 crates/cli/src/main.rs diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml new file mode 100644 index 00000000..892e4fba --- /dev/null +++ b/.github/workflows/cli.yml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +name: CLI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + smoke-test: + name: datafusion-spark-cli smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain install stable --profile minimal --no-self-update && rustup default stable + - uses: Swatinem/rust-cache@v2 + - run: cargo build -p datafusion-spark-cli + # A Spark function, and ANSI mode on by default and off with `--ansi false`. + - name: Run queries + run: | + set -euo pipefail + cli=target/debug/datafusion-spark-cli + $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \ + | grep -x '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b' + if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then + echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" && exit 1 + fi + $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS int)) AS a" \ + | grep -x -- '-2147483648' diff --git a/AGENTS.md b/AGENTS.md index 48ba53c5..06a5a868 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,8 @@ DataFusion expressions that emulate Apache Spark's behavior. - `crates/spark` – the `datafusion-spark` crate - `crates/sqllogictest` – `.slt` tests for the Spark functions +- `crates/cli` – `datafusion-spark-cli`, a SQL shell for trying the functions (not a default + workspace member, so build it with `-p datafusion-spark-cli`) Before opening a PR, run `cargo fmt --all`, `cargo clippy --all-targets --all-features -- -D warnings`, `cargo test -p datafusion-spark` and `cargo test -p datafusion-spark-sqllogictest`. diff --git a/Cargo.lock b/Cargo.lock index 461a5116..1d6d8020 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -112,7 +112,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -123,7 +123,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -135,6 +135,15 @@ dependencies = [ "object", ] +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayvec" version = "0.7.8" @@ -436,18 +445,418 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-config" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16ec023cac0d0c0432e606a2d14f04eb70d09c71a739a406141e7be6b923ab95" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de8930c11353c72ae54aa24dc94d692df74d59814839cbaf8069d63dbcb4c519" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.121.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d414ba8f00c2b4b506893ded67827e63be88b5e6cce71b7c0ac8f96ce09bc5" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db26187b124bcc637641f7963d5200c5ec2409ea5b7730dd7e014ef4e0ab6f19" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "fastrand", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.126.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c667da7839295125cfe23280f798c07ba694a9fed191af804edb4bda09ac7f10" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a4393e6addc3380632887b0998d06ad5f099ab5c478be0a3eeedecf8604161" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19af609dd5edf8b056cd0bb001c706c8b3b1458055328b334ec13d7f46f6aaa8" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45f9fc77ffe92c92c341165a01a565bd30c39ecf5aa8c0719b41719009604f9" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a416257d7b1b0f32a517a470a8c1dd354d8c4aa697ccab81a11a1e6cfb7418e3" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "tracing", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bc688673e282f42d3152dcd9cacc1a9a75734d7cd39d02e58989276fc750f3c" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0467a2414e84b1b01e37452896fb3562b93926e81c619eba6ecf27ce9e682a0e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7ef30d39e131651359b60a1c560de9a9502a4c3754172ab2d6e8734fe21bcba" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff4bf014c512840b2bba80a20417b723cbd6a0a65c02e2d81f50b4268aedfc7" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http", + "http-body", + "http-body-util", + "itoa", + "pin-project-lite", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "590fdc1a70546b2f7f2061c42eaf173fb6f26d935330d069edc04dbd8f01d924" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "base64" version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bigdecimal" version = "0.4.11" @@ -486,7 +895,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.1", ] [[package]] @@ -540,6 +949,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.6.1" @@ -586,7 +1005,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -598,6 +1017,7 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", + "serde", "windows-link", ] @@ -678,6 +1098,24 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "cmov" version = "0.5.4" @@ -690,6 +1128,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" version = "8.0.1" @@ -730,7 +1178,7 @@ dependencies = [ "encode_unicode", "libc", "unicode-width 0.2.3", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -765,12 +1213,31 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -795,6 +1262,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.5" @@ -936,7 +1413,7 @@ dependencies = [ [[package]] name = "datafusion" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -989,7 +1466,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1013,7 +1490,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1034,21 +1511,50 @@ dependencies = [ ] [[package]] -name = "datafusion-common" +name = "datafusion-cli" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", - "arrow-ipc", - "arrow-schema", + "async-trait", + "aws-config", + "aws-credential-types", "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.1", - "indexmap", - "itertools 0.15.0", - "libc", - "log", + "clap", + "datafusion", + "datafusion-common", + "datafusion-spark 56.0.0 (git+https://github.com/apache/datafusion?branch=branch-56)", + "dirs", + "env_logger", + "futures", + "log", + "mimalloc", + "object_store", + "parking_lot 0.13.0", + "parquet", + "regex", + "rustyline", + "tokio", + "url", +] + +[[package]] +name = "datafusion-common" +version = "56.0.0" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "hex", + "indexmap", + "itertools 0.15.0", + "libc", + "log", "num-traits", "object_store", "parquet", @@ -1062,7 +1568,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "futures", "log", @@ -1072,7 +1578,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-compression", @@ -1108,7 +1614,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-ipc", @@ -1131,7 +1637,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-avro", @@ -1154,7 +1660,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1176,7 +1682,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1198,7 +1704,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1229,12 +1735,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" [[package]] name = "datafusion-execution" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-buffer", @@ -1248,6 +1754,7 @@ dependencies = [ "log", "object_store", "parking_lot 0.13.0", + "parquet", "pin-project-lite", "rand 0.9.5", "tempfile", @@ -1259,7 +1766,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1282,7 +1789,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1293,7 +1800,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-buffer", @@ -1325,7 +1832,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-select", @@ -1347,7 +1854,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1358,7 +1865,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-ord", @@ -1382,7 +1889,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1397,7 +1904,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1413,7 +1920,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1422,7 +1929,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "datafusion-doc", "quote", @@ -1432,7 +1939,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "chrono", @@ -1451,7 +1958,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1474,7 +1981,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1488,7 +1995,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "chrono", @@ -1505,7 +2012,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1525,7 +2032,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-data", @@ -1562,7 +2069,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "datafusion-common", @@ -1577,7 +2084,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow-schema", "async-trait", @@ -1615,7 +2122,7 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "serde_json", - "sha1", + "sha1 0.11.0", "sha2", "tokio", "twox-hash", @@ -1625,7 +2132,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "base64", @@ -1646,12 +2153,24 @@ dependencies = [ "percent-encoding", "rand 0.9.5", "serde_json", - "sha1", + "sha1 0.11.0", "sha2", "twox-hash", "url", ] +[[package]] +name = "datafusion-spark-cli" +version = "56.0.0" +dependencies = [ + "clap", + "datafusion", + "datafusion-cli", + "datafusion-spark 56.0.0", + "env_logger", + "tokio", +] + [[package]] name = "datafusion-spark-sqllogictest" version = "56.0.0" @@ -1667,7 +2186,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "arrow-schema", @@ -1687,7 +2206,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" version = "56.0.0" -source = "git+https://github.com/apache/datafusion?branch=branch-56#3f197fe0bc6365756bb88ef2e51c01631da913ca" +source = "git+https://github.com/apache/datafusion?branch=branch-56#cce6374e6256b3b8892ec60674578cf32bc3a32c" dependencies = [ "arrow", "async-trait", @@ -1709,6 +2228,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "digest" version = "0.10.7" @@ -1731,6 +2256,27 @@ dependencies = [ "ctutils", ] +[[package]] +name = "dirs" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -1742,6 +2288,12 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "educe" version = "0.6.0" @@ -1766,6 +2318,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "endian-type" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" + [[package]] name = "enum-ordinalize" version = "4.4.2" @@ -1786,6 +2344,29 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.3" @@ -1799,9 +2380,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + [[package]] name = "escape8259" version = "0.5.4" @@ -1847,6 +2434,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -1877,6 +2470,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.34" @@ -1982,8 +2581,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -2005,9 +2606,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -2016,6 +2619,25 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -2066,6 +2688,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" version = "1.5.0" @@ -2076,6 +2716,35 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + [[package]] name = "humantime" version = "2.4.0" @@ -2091,6 +2760,67 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2242,6 +2972,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2272,6 +3008,90 @@ version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" +[[package]] +name = "jiff" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175dc66334db3b9ab9985bf8b61cfed50dc16d20964bb47f9e479adcae984a7f" +dependencies = [ + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e599b80292e597019b8b3e392de3f4bea2959ddfb8c7dff60e035e9e311a2c1b" +dependencies = [ + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7b1af7547d47354bc438d797e3854601bde8f8d185780c5ba4ab27afa54f1ba" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -2388,6 +3208,24 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "libtest-mimic" version = "0.8.2" @@ -2436,6 +3274,12 @@ version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "lz4_flex" version = "0.14.0" @@ -2472,15 +3316,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] -name = "miniz_oxide" -version = "0.9.1" +name = "mimalloc" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" dependencies = [ - "adler2", + "libmimalloc-sys", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20521a903f02b0a573cd6840e9d0a5a0cc043d361fe924092cd70f8556cc9c4a" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + [[package]] name = "nix" version = "0.31.3" @@ -2522,6 +3395,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-integer" version = "0.1.47" @@ -2557,17 +3436,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", + "aws-lc-rs", + "base64", "bytes", "chrono", + "crc-fast", + "form_urlencoded", "futures-channel", "futures-core", "futures-util", "http", + "http-body-util", "humantime", + "hyper", "itertools 0.15.0", + "md-5 0.11.0", "nix", "parking_lot 0.12.5", "percent-encoding", + "quick-xml", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", "thiserror", "tokio", "tracing", @@ -2575,7 +3468,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2596,6 +3489,24 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.4.0" @@ -2683,6 +3594,7 @@ dependencies = [ "num-integer", "num-traits", "object_store", + "ring", "seq-macro", "simdutf8", "snap", @@ -2793,6 +3705,15 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.6" @@ -2802,6 +3723,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2830,6 +3757,73 @@ dependencies = [ "cc", ] +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -2851,6 +3845,16 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radix_trie" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.8" @@ -2927,6 +3931,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.12.0" @@ -2985,6 +3998,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.13.1" @@ -3008,12 +4031,77 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -3033,7 +4121,82 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -3042,6 +4205,26 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rustyline" +version = "19.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e1d6bd81c2d5feafd2419138df9183c45a10a492c4e873b8b0acb8eb2b549e" +dependencies = [ + "bitflags", + "clipboard-win", + "home", + "libc", + "log", + "memchr", + "nix", + "radix_trie", + "unicode-segmentation", + "unicode-width 0.2.3", + "utf8parse", + "windows-sys 0.61.2", +] + [[package]] name = "ryu" version = "1.0.23" @@ -3057,12 +4240,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -3118,6 +4333,29 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha1" version = "0.11.0" @@ -3125,7 +4363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -3136,7 +4374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -3146,12 +4384,32 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "simd-adler32" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3188,6 +4446,22 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqllogictest" version = "0.29.1" @@ -3251,7 +4525,7 @@ dependencies = [ "cfg-if", "libc", "psm", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3282,6 +4556,12 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -3304,6 +4584,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -3325,7 +4614,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3348,6 +4637,36 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -3377,6 +4696,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.54.1" @@ -3384,8 +4709,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", + "libc", + "mio", + "parking_lot 0.12.5", "pin-project-lite", + "signal-hook-registry", + "socket2", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -3399,6 +4730,16 @@ dependencies = [ "syn 3.0.10", ] +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.19" @@ -3420,10 +4761,56 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", "pin-project-lite", + "sync_wrapper", "tokio", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.44" @@ -3455,6 +4842,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "twox-hash" version = "2.1.5" @@ -3500,6 +4893,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -3512,6 +4911,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.1.0" @@ -3550,6 +4955,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -3560,6 +4971,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3630,6 +5050,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.107" @@ -3650,6 +5083,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3672,7 +5114,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3740,6 +5182,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -3749,6 +5200,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -3761,6 +5276,12 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.4" @@ -3825,6 +5346,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/Cargo.toml b/Cargo.toml index 8428bc24..76f0d58f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,10 @@ [workspace] resolver = "2" -members = ["crates/spark", "crates/sqllogictest"] +members = ["crates/cli", "crates/spark", "crates/sqllogictest"] +# The CLI pulls in datafusion-cli and its object store dependencies, so a plain +# `cargo build` or `cargo test` at the root skips it. `--workspace` includes it. +default-members = ["crates/spark", "crates/sqllogictest"] [workspace.package] authors = ["Apache DataFusion "] diff --git a/README.md b/README.md index 08ecc021..dabfa799 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,8 @@ for more. - `crates/spark` – the `datafusion-spark` crate - `crates/sqllogictest` – SQL logic tests for the Spark functions +- `crates/cli` – `datafusion-spark-cli`, a SQL shell with the Spark functions registered + (`cargo run -p datafusion-spark-cli`) - `.ai/skills` – agent skills for working on this repository (see `AGENTS.md`) ## Building and testing diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml new file mode 100644 index 00000000..9792126d --- /dev/null +++ b/crates/cli/Cargo.toml @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +[package] +name = "datafusion-spark-cli" +description = "Command line SQL shell for DataFusion with the Spark-compatible functions and SQL dialect" +version = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } +edition = { workspace = true } +rust-version = { workspace = true } +# Not published to crates.io yet. +publish = false + +[lints] +workspace = true + +[[bin]] +name = "datafusion-spark-cli" +path = "src/main.rs" + +[dependencies] +clap = { version = "4.5.60", features = ["derive"] } +datafusion = { workspace = true, features = ["sql", "parquet", "nested_expressions"] } +datafusion-cli = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-spark = { path = "../spark", features = ["core"] } +env_logger = "0.11" +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs new file mode 100644 index 00000000..bc8c75e4 --- /dev/null +++ b/crates/cli/src/main.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible functions +//! registered and the Spark SQL dialect enabled. +//! +//! The shell itself (the REPL, running files and commands, output formats and +//! object stores) comes from the `datafusion-cli` library. + +use std::env; +use std::path::Path; +use std::process::ExitCode; +use std::sync::Arc; + +use clap::{ArgAction, Parser}; +use datafusion::common::config::{ConfigNonZeroUsize, Dialect}; +use datafusion::common::{DataFusionError, Result}; +use datafusion::config::ConfigOptions; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::logical_expr::ExplainFormat; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_cli::catalog::DynamicObjectStoreCatalog; +use datafusion_cli::exec; +use datafusion_cli::object_storage::instrumented::{ + InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry, +}; +use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location}; +use datafusion_cli::print_format::PrintFormat; +use datafusion_cli::print_options::{MaxRows, PrintOptions}; +use datafusion_spark::SessionStateBuilderSpark; + +#[derive(Debug, Parser)] +#[clap(version, about, long_about = None)] +struct Args { + #[clap( + short = 'c', + long, + num_args = 0.., + help = "Execute the given command string(s), then exit", + value_parser(parse_command) + )] + command: Vec, + + #[clap( + short, + long, + num_args = 0.., + help = "Execute commands from file(s), then exit", + value_parser(parse_valid_file) + )] + file: Vec, + + #[clap( + short = 'r', + long, + num_args = 0.., + help = "Run the provided files on startup, before the REPL", + value_parser(parse_valid_file), + conflicts_with = "file" + )] + rc: Vec, + + #[clap( + short = 'p', + long, + help = "Path to your data, default to current directory", + value_parser(parse_valid_data_dir) + )] + data_path: Option, + + #[clap( + short = 'b', + long, + help = "The batch size of each query, or use DataFusion default", + value_parser(parse_batch_size) + )] + batch_size: Option, + + #[clap( + long, + default_value_t = true, + action = ArgAction::Set, + help = "Evaluate with ANSI mode on, like Spark 4's default \ + `spark.sql.ansi.enabled = true`. Pass `--ansi false` for Spark's legacy behavior" + )] + ansi: bool, + + #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] + format: PrintFormat, + + #[clap( + short, + long, + help = "Reduce printing other than the results and work quietly" + )] + quiet: bool, + + #[clap( + long, + help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", + default_value = "40" + )] + maxrows: MaxRows, + + #[clap(long, help = "Enables console syntax highlighting")] + color: bool, +} + +impl Args { + /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from stdin. + fn repl_mode(&self) -> bool { + self.command.is_empty() && self.file.is_empty() + } + + /// Whether stdin carries the session's SQL, so it cannot also back a + /// `LOCATION '/dev/stdin'` table. + fn reads_sql_from_stdin(&self) -> bool { + self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f)) + } +} + +#[tokio::main] +async fn main() -> ExitCode { + if let Err(e) = main_inner().await { + println!("Error: {e}"); + return ExitCode::FAILURE; + } + ExitCode::SUCCESS +} + +async fn main_inner() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + if !args.quiet { + println!( + "DataFusion Spark CLI v{} (ANSI mode {})", + env!("CARGO_PKG_VERSION"), + if args.ansi { "on" } else { "off" } + ); + } + + if let Some(path) = &args.data_path { + env::set_current_dir(Path::new(path))?; + } + + let instrumented_registry = Arc::new( + InstrumentedObjectStoreRegistry::new() + .with_profile_mode(InstrumentedObjectStoreMode::Disabled), + ); + let runtime_env = RuntimeEnvBuilder::new() + .with_object_store_registry(Arc::clone(&instrumented_registry) as _) + .build_arc()?; + let state = SessionStateBuilder::new() + .with_config(session_config(&args)?) + .with_runtime_env(runtime_env) + .with_default_features() + .with_spark_features() + .build(); + + // Query files by their path or URL, such as `SELECT * FROM 'data.parquet'`, as + // `datafusion-cli` does. + let ctx = SessionContext::new_with_state(state).enable_url_table(); + ctx.refresh_catalogs().await?; + ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new( + ctx.state().catalog_list().clone(), + ctx.state_weak_ref(), + ))); + + let mut print_options = PrintOptions { + format: args.format, + quiet: args.quiet, + maxrows: args.maxrows, + color: args.color, + instrumented_registry, + }; + + if args.repl_mode() { + if !args.rc.is_empty() { + exec::exec_from_files(&ctx, args.rc, &print_options).await?; + } + return exec::exec_from_repl(&ctx, &mut print_options) + .await + .map_err(|e| DataFusionError::External(Box::new(e))); + } + if !args.file.is_empty() { + exec::exec_from_files(&ctx, args.file.clone(), &print_options).await?; + } + if !args.command.is_empty() { + exec::exec_from_commands(&ctx, args.command.clone(), &print_options).await?; + } + Ok(()) +} + +fn session_config(args: &Args) -> Result { + // Read `DATAFUSION_*` environment variables, then apply the command line. + let mut options = ConfigOptions::from_env()?; + options.sql_parser.dialect = Dialect::Spark; + options.execution.enable_ansi_mode = args.ansi; + if let Some(batch_size) = args.batch_size { + options.execution.batch_size = ConfigNonZeroUsize::try_new(batch_size)?; + } + // Defaults that `datafusion-cli` also uses, unless set in the environment. + if env::var_os("DATAFUSION_EXPLAIN_FORMAT").is_none() { + options.explain.format = ExplainFormat::Tree; + } + if env::var_os("DATAFUSION_FORMAT_NULL").is_none() { + options.format.null = String::from("NULL"); + } + + let mut config = SessionConfig::from(options).with_information_schema(true); + if args.reads_sql_from_stdin() { + config = config.with_extension(Arc::new(StdinCarriesCommands)); + } + Ok(config) +} + +fn parse_valid_file(path: &str) -> Result { + if Path::new(path).is_file() { + Ok(path.to_string()) + } else { + Err(format!("Invalid file '{path}'")) + } +} + +fn parse_valid_data_dir(dir: &str) -> Result { + if Path::new(dir).is_dir() { + Ok(dir.to_string()) + } else { + Err(format!("Invalid data directory '{dir}'")) + } +} + +fn parse_batch_size(size: &str) -> Result { + match size.parse::() { + Ok(size) if size > 0 => Ok(size), + _ => Err(format!("Invalid batch size '{size}'")), + } +} + +fn parse_command(command: &str) -> Result { + if command.is_empty() { + Err("-c flag expects only non empty commands".to_string()) + } else { + Ok(command.to_string()) + } +} diff --git a/crates/spark/README.md b/crates/spark/README.md index f0a24128..914d5ed5 100644 --- a/crates/spark/README.md +++ b/crates/spark/README.md @@ -66,17 +66,18 @@ use datafusion_spark::expr_fn::sha2; let expr = sha2(col("my_data"), lit(256)); ``` -## Using the DataFusion CLI +## Trying the functions in a SQL shell -Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register -the Spark-compatible scalar, aggregate, and window functions for the session: +`datafusion-spark-cli` is the DataFusion CLI with the Spark SQL dialect enabled +and the Spark-compatible scalar, aggregate, and window functions registered. It +lives in [`crates/cli`](../cli) in this repository and is not published yet: ```shell -datafusion-cli --spark +cargo run -p datafusion-spark-cli ``` -The option is disabled by default because Spark-compatible functions can -override DataFusion functions with the same name. +The Spark functions override DataFusion functions with the same name. ANSI mode +is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior. ## Things to know diff --git a/crates/sqllogictest/test_files/README.md b/crates/sqllogictest/test_files/README.md index f06180e1..f1684c67 100644 --- a/crates/sqllogictest/test_files/README.md +++ b/crates/sqllogictest/test_files/README.md @@ -39,9 +39,10 @@ statement ok set datafusion.execution.enable_ansi_mode = false; ``` -Run `datafusion-cli --spark` to explore Spark-compatible functions -interactively. This enables the Spark SQL dialect and registers the -Spark-compatible function library for the CLI session. +Run `cargo run -p datafusion-spark-cli` to explore Spark-compatible functions +interactively. The CLI enables the Spark SQL dialect and registers the +Spark-compatible function library. Unlike these test files, it starts with +ANSI mode on, as Spark 4 does. Pass `--ansi false` to turn it off. ### Finding Test Cases From f0cf6de160bd2df74d3109fd640996202f7d6183 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 05:51:04 -0700 Subject: [PATCH 2/2] fix(cli): resolve -f and -r paths before changing to the data path With -p the CLI changed directory before opening the -f and -r files, so a relative path ran the file of that name in the data directory, or panicked if there was none. Make those paths absolute first, open the files ourselves so a failure is an error rather than a panic, and accept -f /dev/stdin. CI now also runs the REPL from stdin with a Spark-only type, checks the ARITHMETIC_OVERFLOW message, and runs -f and -r with a data path. The READMEs show that --ansi false goes after -- with cargo run. --- .github/workflows/cli.yml | 20 ++++++++++- crates/cli/src/main.rs | 46 +++++++++++++++++++++--- crates/spark/README.md | 7 +++- crates/sqllogictest/test_files/README.md | 3 +- 4 files changed, 68 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 892e4fba..074770bf 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -28,8 +28,26 @@ jobs: cli=target/debug/datafusion-spark-cli $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \ | grep -x '529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b' - if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then + if out=$($cli -q -c "SELECT abs(cast(-2147483648 AS int))" 2>&1); then echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" && exit 1 fi + grep -F '[ARITHMETIC_OVERFLOW]' <<< "$out" $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS int)) AS a" \ | grep -x -- '-2147483648' + # SQL read from stdin by the REPL. `LONG` is a type in the Spark dialect only. + - name: Run the REPL from stdin + run: | + set -euo pipefail + echo "SELECT CAST(1 AS LONG) AS l;" \ + | target/debug/datafusion-spark-cli -q --format csv | grep -x '1' + # `-f` and `-r` paths are relative to the working directory, not to `-p`. + - name: Run files with a data path + run: | + set -euo pipefail + cli=$PWD/target/debug/datafusion-spark-cli + dir=$(mktemp -d) && cd "$dir" && mkdir data + echo "SELECT 'cwd' AS s;" > q.sql + echo "SELECT 'data' AS s;" > data/q.sql + $cli -q --format csv -p data -f q.sql | grep -x 'cwd' + echo "" | $cli -q --format csv -p data -r q.sql | grep -x 'cwd' + echo "SELECT 'stdin' AS s;" | $cli -q --format csv -p data -f /dev/stdin | grep -x 'stdin' diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index bc8c75e4..e2f0772f 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -9,7 +9,9 @@ //! object stores) comes from the `datafusion-cli` library. use std::env; -use std::path::Path; +use std::fs::File; +use std::io::BufReader; +use std::path::{Path, absolute}; use std::process::ExitCode; use std::sync::Arc; @@ -132,7 +134,7 @@ async fn main() -> ExitCode { async fn main_inner() -> Result<()> { env_logger::init(); - let args = Args::parse(); + let mut args = Args::parse(); if !args.quiet { println!( @@ -143,6 +145,10 @@ async fn main_inner() -> Result<()> { } if let Some(path) = &args.data_path { + // `-f` and `-r` paths are relative to where the CLI was started, not to + // the data directory. + args.file = absolute_paths(&args.file)?; + args.rc = absolute_paths(&args.rc)?; env::set_current_dir(Path::new(path))?; } @@ -179,14 +185,14 @@ async fn main_inner() -> Result<()> { if args.repl_mode() { if !args.rc.is_empty() { - exec::exec_from_files(&ctx, args.rc, &print_options).await?; + exec_files(&ctx, &args.rc, &print_options).await?; } return exec::exec_from_repl(&ctx, &mut print_options) .await .map_err(|e| DataFusionError::External(Box::new(e))); } if !args.file.is_empty() { - exec::exec_from_files(&ctx, args.file.clone(), &print_options).await?; + exec_files(&ctx, &args.file, &print_options).await?; } if !args.command.is_empty() { exec::exec_from_commands(&ctx, args.command.clone(), &print_options).await?; @@ -194,6 +200,35 @@ async fn main_inner() -> Result<()> { Ok(()) } +/// Makes each path absolute, leaving the names of stdin as they are. +fn absolute_paths(paths: &[String]) -> Result> { + paths + .iter() + .map(|path| { + if is_stdin_location(path) { + return Ok(path.clone()); + } + Ok(absolute(path)?.to_string_lossy().into_owned()) + }) + .collect() +} + +/// Runs the SQL in each file in turn. Unlike `exec::exec_from_files`, which panics, +/// a file that cannot be opened is an error. +async fn exec_files( + ctx: &SessionContext, + files: &[String], + print_options: &PrintOptions, +) -> Result<()> { + for path in files { + let file = File::open(path).map_err(|e| { + DataFusionError::Execution(format!("Cannot open file '{path}': {e}")) + })?; + exec::exec_from_lines(ctx, &mut BufReader::new(file), print_options).await?; + } + Ok(()) +} + fn session_config(args: &Args) -> Result { // Read `DATAFUSION_*` environment variables, then apply the command line. let mut options = ConfigOptions::from_env()?; @@ -218,7 +253,8 @@ fn session_config(args: &Args) -> Result { } fn parse_valid_file(path: &str) -> Result { - if Path::new(path).is_file() { + // stdin is a pipe or a terminal, not a regular file. + if is_stdin_location(path) || Path::new(path).is_file() { Ok(path.to_string()) } else { Err(format!("Invalid file '{path}'")) diff --git a/crates/spark/README.md b/crates/spark/README.md index be2f195f..592f4c2d 100644 --- a/crates/spark/README.md +++ b/crates/spark/README.md @@ -84,7 +84,12 @@ cargo run -p datafusion-spark-cli ``` The Spark functions override DataFusion functions with the same name. ANSI mode -is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior. +is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy behavior, +after `--` so that `cargo` passes it to the CLI: + +```shell +cargo run -p datafusion-spark-cli -- --ansi false +``` ## Things to know diff --git a/crates/sqllogictest/test_files/README.md b/crates/sqllogictest/test_files/README.md index 305932b1..1b1e5a0c 100644 --- a/crates/sqllogictest/test_files/README.md +++ b/crates/sqllogictest/test_files/README.md @@ -42,7 +42,8 @@ set datafusion.execution.enable_ansi_mode = false; Run `cargo run -p datafusion-spark-cli` to explore Spark-compatible functions interactively. The CLI enables the Spark SQL dialect and registers the Spark-compatible function library. Unlike these test files, it starts with -ANSI mode on, as Spark 4 does. Pass `--ansi false` to turn it off. +ANSI mode on, as Spark 4 does. To turn it off, pass `--ansi false` after `--`, +so that `cargo` passes it to the CLI: `cargo run -p datafusion-spark-cli -- --ansi false`. ### Finding Test Cases