From fe0cf04fbb5ef88e5ba7a019eefec7dd8ea15cf3 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:07:50 -0500 Subject: [PATCH 1/2] feat: add clickhouse.client.compression database option Add a database option to choose the transport compression used between the driver and ClickHouse: "lz4" (the default, unchanged) or "none". ClickHouse compresses a query's response on a single thread, which can cap one query's throughput far below what a fast network carries; "none" removes that cap. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++ src/lib.rs | 14 ++++++++ src/options.rs | 50 +++++++++++++++++++++++++++ tests/it/compression.rs | 76 +++++++++++++++++++++++++++++++++++++++++ tests/it/main.rs | 1 + 5 files changed, 147 insertions(+) create mode 100644 tests/it/compression.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b508748..1ad56a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* Added the `clickhouse.client.compression` database option to choose the transport compression: + `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, + so `"none"` can be much faster on fast networks. + ## [0.1.2] - 2026-11-06 ### Fixed diff --git a/src/lib.rs b/src/lib.rs index 26b5bed..449213d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver { password: None, product_info: self.product_info.clone(), settings: HashMap::new(), + compression: None, }; for (key, value) in opts { @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase { password: Option, product_info: ProductInfo, settings: HashMap, + /// `None` keeps the `clickhouse` crate's default (LZ4). + compression: Option, } impl Database for ClickhouseDatabase { @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase { // to a different value at a lower level; `AugmentedClient` covers that. .with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION")); + if let Some(compression) = self.compression { + client = client.with_compression(compression); + } + if let Some(url) = &self.url { client = client.with_url(url.to_string()); } @@ -583,6 +590,9 @@ impl ClickhouseDatabase { options::PRODUCT_INFO => { self.product_info = value.try_into()?; } + options::COMPRESSION => { + self.compression = Some(options::parse_compression(&value.try_string(key)?)?); + } other => { return Err(Error::with_message_and_status( format!("unknown database option {other:?}"), @@ -605,6 +615,10 @@ impl ClickhouseDatabase { match key { options::PRODUCT_INFO => Ok(self.product_info.to_string()), + options::COMPRESSION => Ok(options::compression_name( + self.compression.unwrap_or(clickhouse::Compression::Lz4), + ) + .to_string()), other => Err(Error::with_message_and_status( format!("unknown database option {other:?}"), Status::InvalidArguments, diff --git a/src/options.rs b/src/options.rs index 1412dfc..9644b4e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id"; /// as Arrow binary strings instead. pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string"; +/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`. +/// +/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1` +/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver. +/// With `"none"`, data is sent as is. +/// +/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput +/// of one query to well below what a fast network carries. On fast links, `"none"` is often +/// faster; on slow links, `"lz4"` sends fewer bytes. +/// +/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates. +/// The current value may be read back with [`Optionable::get_option_string()`]. +/// +/// # Example +/// ```rust +/// use adbc_clickhouse::ClickhouseDriver; +/// +/// use adbc_core::{Driver, Optionable}; +/// +/// let mut driver = ClickhouseDriver::init(); +/// +/// let db = driver +/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())]) +/// .unwrap(); +/// +/// assert_eq!( +/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(), +/// "none" +/// ); +/// ``` +pub const COMPRESSION: &str = "clickhouse.client.compression"; + +pub(crate) fn parse_compression(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "none" => Ok(clickhouse::Compression::None), + "lz4" => Ok(clickhouse::Compression::Lz4), + _ => Err(Error::with_message_and_status( + format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"), + Status::InvalidArguments, + )), + } +} + +pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str { + match compression { + clickhouse::Compression::None => "none", + _ => "lz4", + } +} + /// Prefix for passing arbitrary [ClickHouse settings] through as driver options. /// /// An option key of the form `clickhouse.setting.` maps directly to the diff --git a/tests/it/compression.rs b/tests/it/compression.rs new file mode 100644 index 0000000..c9cad29 --- /dev/null +++ b/tests/it/compression.rs @@ -0,0 +1,76 @@ +//! Tests for the `clickhouse.client.compression` database option. + +use crate::test_driver; +use adbc_clickhouse::options; +use adbc_core::error::Status; +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; + +/// Sum `number` over enough rows to span several response blocks. +fn sum_numbers(compression: Option<&str>) -> u64 { + let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())]; + if let Some(compression) = compression { + opts.push((options::COMPRESSION.into(), compression.into())); + } + let db = test_driver().new_database_with_opts(opts).unwrap(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT number FROM system.numbers LIMIT 1000000") + .unwrap(); + + statement + .execute() + .unwrap() + .collect::, _>>() + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) + .sum() +} + +#[test] +fn compression_none_and_lz4_return_the_same_data() { + let expected = 999_999 * 1_000_000 / 2; + assert_eq!(sum_numbers(None), expected); + assert_eq!(sum_numbers(Some("lz4")), expected); + assert_eq!(sum_numbers(Some("none")), expected); + assert_eq!(sum_numbers(Some("NONE")), expected); +} + +#[test] +fn compression_read_back() { + let mut driver = test_driver(); + + let db = driver.new_database().unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "lz4" + ); + + let db = driver + .new_database_with_opts([(options::COMPRESSION.into(), "none".into())]) + .unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "none" + ); +} + +#[test] +fn compression_rejects_unknown_values() { + let err = test_driver() + .new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())]) + .err() + .expect("expected an error for an unknown compression method"); + assert_eq!(err.status, Status::InvalidArguments); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index f990b70..45ebac8 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -11,6 +11,7 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use std::sync::Arc; mod bulk_ingest; +mod compression; mod database; mod get_table_schema; mod params; From c177c415abf262614bf58a943f9cfd787f085dd2 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:09:19 -0500 Subject: [PATCH 2/2] feat: implement Statement::execute_schema() with DESCRIBE Return the result schema of a query without running it, by sending DESCRIBE () and mapping the columns with the same type mapping as Connection::get_table_schema(). Bound parameters are sent with the DESCRIBE. The schema requests get their own query IDs: the statement's query ID is reused by its data query, and ClickHouse rejects a request whose query ID is still held by an earlier request that is being cleaned up. The settings query starts only after DESCRIBE has finished, because both carry the connection's session ID and ClickHouse rejects overlapping requests on one session. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + src/schema.rs | 44 ++++++++++++++ src/statement.rs | 46 +++++++++++++-- tests/it/execute_schema.rs | 117 +++++++++++++++++++++++++++++++++++++ tests/it/main.rs | 1 + 5 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 tests/it/execute_schema.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad56a8..41d5cd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added the `clickhouse.client.compression` database option to choose the transport compression: `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, so `"none"` can be much faster on fast networks. +* Implemented `Statement::execute_schema()` with `DESCRIBE ()`, using the same type mapping + as `Connection::get_table_schema()`. Bound parameters are sent with the `DESCRIBE`. ## [0.1.2] - 2026-11-06 diff --git a/src/schema.rs b/src/schema.rs index e62a9e8..ae6f035 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,6 +1,7 @@ use adbc_core::error::{Error, Status}; use arrow_schema::{DataType, Field, Fields, Schema, TimeUnit}; use clickhouse::Client; +use clickhouse::query::Query; use clickhouse_types::DataTypeNode; use clickhouse_types::data_types::{DateTimePrecision, DecimalType, EnumType}; use serde::{Deserialize, Deserializer}; @@ -166,6 +167,49 @@ pub async fn of_table( Ok(Schema::new(fields)) } +/// Describe the result columns of a query prepared as `DESCRIBE ()`. +/// +/// The columns are mapped with the same type mapping (and the same server settings) as +/// [`of_table()`], so the schema matches the Arrow stream ClickHouse returns for the query. +pub async fn of_query(client: &Client, query: Query) -> adbc_core::error::Result { + let mut cursor = query.fetch::().map_err(|e| { + Error::with_message_and_status( + format!("error beginning fetch of query schema: {e}"), + Status::Internal, + ) + })?; + + let mut columns = Vec::new(); + while let Some(column) = cursor.next().await.map_err(|e| { + Error::with_message_and_status( + format!("error retrieving query schema: {e}"), + Status::Internal, + ) + })? { + columns.push(column); + } + drop(cursor); + + // Both requests carry the connection's session ID, and ClickHouse rejects a request that + // overlaps another on the same session, so finish `DESCRIBE` before querying the settings. + let settings = Settings::query(client).await?; + + let mut fields = Vec::with_capacity(columns.len()); + for column in &columns { + let field = column_to_arrow(column, &settings).map_err(|e| Error { + message: format!( + "error mapping query column {:?}: {}", + column.name, e.message + ), + ..e + })?; + + fields.push(field); + } + + Ok(Schema::new(fields)) +} + fn column_to_arrow(column: &Column, settings: &Settings) -> Result { let ch_type = DataTypeNode::new(&column.r#type).map_err(|e| { Error::with_message_and_status( diff --git a/src/statement.rs b/src/statement.rs index ef4bf2b..baee75a 100644 --- a/src/statement.rs +++ b/src/statement.rs @@ -23,7 +23,7 @@ use clickhouse::query::Query; use crate::options::OptionValueExt; use crate::writer::ArrowStreamWriter; -use crate::{AugmentedClient, Result, TokioContext, options, random_id}; +use crate::{AugmentedClient, Result, TokioContext, options, random_id, schema}; use clickhouse::_priv::sql_escape_identifier; use clickhouse_ext_arrow::ArrowQueryExt; @@ -251,13 +251,47 @@ impl Statement for ClickhouseStatement { execute_streaming_ingest(&self.client, &self.tokio, ingest, stream) } - /// # Not Implemented - /// Currently will return an error with [`Status::NotImplemented`]. + /// Get the schema of the query's result set without executing the query. /// - /// See the tracking issue for details and subscribe for updates: - /// + /// This runs `DESCRIBE ()` on the server and maps the result columns with the same + /// type mapping as [`Connection::get_table_schema()`][adbc_core::Connection::get_table_schema], + /// so the schema matches the one [`execute()`][Self::execute] returns. Bound parameters + /// (from [`bind()`][Self::bind]) are sent with the `DESCRIBE`. + /// + /// Returns [`Status::InvalidState`] if no query is set, or if the query is an `INSERT` + /// that would stream bound data. fn execute_schema(&mut self) -> adbc_core::error::Result { - err_unimplemented!("ClickhouseStatement::execute_schema()") + let StatementMode::SqlQuery(sql) = &self.mode else { + return Err(Error::with_message_and_status( + "execute_schema requires a SQL query", + Status::InvalidState, + )); + }; + if is_streaming_insert(sql) { + return Err(Error::with_message_and_status( + "execute_schema requires a query that returns a result set", + Status::InvalidState, + )); + } + + let _guard = self.tokio.enter(); + + // This statement's query ID is reused for its eventual data query. Give each metadata + // request its own ID, so ClickHouse does not reject the data query (or the second + // metadata request) while an earlier request with the same ID is still being cleaned up. + let mut schema_client = Client::clone(&self.client); + schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema")); + let mut settings_client = schema_client.clone(); + settings_client.set_setting(options::as_setting::QUERY_ID, random_id("schema_settings")); + + // Raw so a literal `?` is not treated as a bind placeholder. + let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})")); + if let Some(BindType::Single(batch)) = &self.bind { + query = bind_record_batch(query, batch.clone())?; + } + + self.tokio + .block_on(schema::of_query(&settings_client, query)) } /// # Not Implemented diff --git a/tests/it/execute_schema.rs b/tests/it/execute_schema.rs new file mode 100644 index 0000000..fbfb34b --- /dev/null +++ b/tests/it/execute_schema.rs @@ -0,0 +1,117 @@ +//! Tests for `Statement::execute_schema()`. + +use crate::test_database; +use adbc_core::error::Status; +use adbc_core::{Connection, Database, Statement}; +use arrow_array::{RecordBatch, create_array}; +use arrow_schema::{DataType, Field, Schema}; + +/// Columns of several types, including nullable, decimal and low-cardinality ones. +const QUERY: &str = "SELECT number, toString(number) AS s, toNullable(toInt32(number)) AS n, \ + toDecimal64(number, 2) AS d64, toDecimal128(number, 4) AS d128, toDate32('2024-01-01') AS d32, \ + toDateTime64('2024-01-01 00:00:00', 3, 'UTC') AS ts, toFloat64(number) / 3 AS f, \ + toLowCardinality(toString(number % 3)) AS lc, [number, number + 1] AS arr \ + FROM system.numbers LIMIT 3"; + +/// `execute_schema()` annotates each field with its ClickHouse type (`CLICKHOUSE:type` metadata), +/// as `get_table_schema()` does; the Arrow stream from `execute()` carries no field metadata. +fn without_metadata(schema: &Schema) -> Schema { + Schema::new( + schema + .fields() + .iter() + .map(|f| f.as_ref().clone().with_metadata(Default::default())) + .collect::>(), + ) +} + +#[test] +fn execute_schema_matches_execute() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let described = statement.execute_schema().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let executed = statement.execute().unwrap().schema(); + + assert_eq!(without_metadata(&described), *executed); + assert_eq!( + described.field_with_name("lc").unwrap().metadata()["CLICKHOUSE:type"], + "LowCardinality(String)" + ); +} + +#[test] +fn execute_after_execute_schema() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + // The data query reuses the statement's query ID; the schema requests must not collide with it. + for _ in 0..3 { + let described = statement.execute_schema().unwrap(); + let reader = statement.execute().unwrap(); + assert_eq!(without_metadata(&described), *reader.schema()); + let rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); + assert_eq!(rows, 3); + } +} + +#[test] +fn execute_schema_with_bound_parameters() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT {x:Int32} AS x, {s:String} AS s") + .unwrap(); + + let params = RecordBatch::try_new( + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + .into(), + vec![create_array!(Int32, [7]), create_array!(Utf8, ["seven"])], + ) + .unwrap(); + statement.bind(params).unwrap(); + + assert_eq!( + without_metadata(&statement.execute_schema().unwrap()), + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + ); +} + +#[test] +fn execute_schema_without_query() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert_eq!(err.status, Status::InvalidState); +} + +#[test] +fn execute_schema_reports_server_errors() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT * FROM adbc_no_such_table_for_execute_schema") + .unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert!( + err.message.contains("UNKNOWN_TABLE") || err.message.contains("doesn't exist"), + "unexpected error: {}", + err.message + ); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index 45ebac8..97dde9c 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -13,6 +13,7 @@ use std::sync::Arc; mod bulk_ingest; mod compression; mod database; +mod execute_schema; mod get_table_schema; mod params; mod settings;