diff --git a/CHANGELOG.md b/CHANGELOG.md index b508748..41d5cd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* Added the `clickhouse.client.compression` database option to choose the transport compression: + `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, + so `"none"` can be much faster on fast networks. +* Implemented `Statement::execute_schema()` with `DESCRIBE ()`, using the same type mapping + as `Connection::get_table_schema()`. Bound parameters are sent with the `DESCRIBE`. + ## [0.1.2] - 2026-11-06 ### Fixed diff --git a/src/lib.rs b/src/lib.rs index 26b5bed..449213d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver { password: None, product_info: self.product_info.clone(), settings: HashMap::new(), + compression: None, }; for (key, value) in opts { @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase { password: Option, product_info: ProductInfo, settings: HashMap, + /// `None` keeps the `clickhouse` crate's default (LZ4). + compression: Option, } impl Database for ClickhouseDatabase { @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase { // to a different value at a lower level; `AugmentedClient` covers that. .with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION")); + if let Some(compression) = self.compression { + client = client.with_compression(compression); + } + if let Some(url) = &self.url { client = client.with_url(url.to_string()); } @@ -583,6 +590,9 @@ impl ClickhouseDatabase { options::PRODUCT_INFO => { self.product_info = value.try_into()?; } + options::COMPRESSION => { + self.compression = Some(options::parse_compression(&value.try_string(key)?)?); + } other => { return Err(Error::with_message_and_status( format!("unknown database option {other:?}"), @@ -605,6 +615,10 @@ impl ClickhouseDatabase { match key { options::PRODUCT_INFO => Ok(self.product_info.to_string()), + options::COMPRESSION => Ok(options::compression_name( + self.compression.unwrap_or(clickhouse::Compression::Lz4), + ) + .to_string()), other => Err(Error::with_message_and_status( format!("unknown database option {other:?}"), Status::InvalidArguments, diff --git a/src/options.rs b/src/options.rs index 1412dfc..9644b4e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id"; /// as Arrow binary strings instead. pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string"; +/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`. +/// +/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1` +/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver. +/// With `"none"`, data is sent as is. +/// +/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput +/// of one query to well below what a fast network carries. On fast links, `"none"` is often +/// faster; on slow links, `"lz4"` sends fewer bytes. +/// +/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates. +/// The current value may be read back with [`Optionable::get_option_string()`]. +/// +/// # Example +/// ```rust +/// use adbc_clickhouse::ClickhouseDriver; +/// +/// use adbc_core::{Driver, Optionable}; +/// +/// let mut driver = ClickhouseDriver::init(); +/// +/// let db = driver +/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())]) +/// .unwrap(); +/// +/// assert_eq!( +/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(), +/// "none" +/// ); +/// ``` +pub const COMPRESSION: &str = "clickhouse.client.compression"; + +pub(crate) fn parse_compression(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "none" => Ok(clickhouse::Compression::None), + "lz4" => Ok(clickhouse::Compression::Lz4), + _ => Err(Error::with_message_and_status( + format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"), + Status::InvalidArguments, + )), + } +} + +pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str { + match compression { + clickhouse::Compression::None => "none", + _ => "lz4", + } +} + /// Prefix for passing arbitrary [ClickHouse settings] through as driver options. /// /// An option key of the form `clickhouse.setting.` maps directly to the diff --git a/src/schema.rs b/src/schema.rs index e62a9e8..ae6f035 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,6 +1,7 @@ use adbc_core::error::{Error, Status}; use arrow_schema::{DataType, Field, Fields, Schema, TimeUnit}; use clickhouse::Client; +use clickhouse::query::Query; use clickhouse_types::DataTypeNode; use clickhouse_types::data_types::{DateTimePrecision, DecimalType, EnumType}; use serde::{Deserialize, Deserializer}; @@ -166,6 +167,49 @@ pub async fn of_table( Ok(Schema::new(fields)) } +/// Describe the result columns of a query prepared as `DESCRIBE ()`. +/// +/// The columns are mapped with the same type mapping (and the same server settings) as +/// [`of_table()`], so the schema matches the Arrow stream ClickHouse returns for the query. +pub async fn of_query(client: &Client, query: Query) -> adbc_core::error::Result { + let mut cursor = query.fetch::().map_err(|e| { + Error::with_message_and_status( + format!("error beginning fetch of query schema: {e}"), + Status::Internal, + ) + })?; + + let mut columns = Vec::new(); + while let Some(column) = cursor.next().await.map_err(|e| { + Error::with_message_and_status( + format!("error retrieving query schema: {e}"), + Status::Internal, + ) + })? { + columns.push(column); + } + drop(cursor); + + // Both requests carry the connection's session ID, and ClickHouse rejects a request that + // overlaps another on the same session, so finish `DESCRIBE` before querying the settings. + let settings = Settings::query(client).await?; + + let mut fields = Vec::with_capacity(columns.len()); + for column in &columns { + let field = column_to_arrow(column, &settings).map_err(|e| Error { + message: format!( + "error mapping query column {:?}: {}", + column.name, e.message + ), + ..e + })?; + + fields.push(field); + } + + Ok(Schema::new(fields)) +} + fn column_to_arrow(column: &Column, settings: &Settings) -> Result { let ch_type = DataTypeNode::new(&column.r#type).map_err(|e| { Error::with_message_and_status( diff --git a/src/statement.rs b/src/statement.rs index ef4bf2b..baee75a 100644 --- a/src/statement.rs +++ b/src/statement.rs @@ -23,7 +23,7 @@ use clickhouse::query::Query; use crate::options::OptionValueExt; use crate::writer::ArrowStreamWriter; -use crate::{AugmentedClient, Result, TokioContext, options, random_id}; +use crate::{AugmentedClient, Result, TokioContext, options, random_id, schema}; use clickhouse::_priv::sql_escape_identifier; use clickhouse_ext_arrow::ArrowQueryExt; @@ -251,13 +251,47 @@ impl Statement for ClickhouseStatement { execute_streaming_ingest(&self.client, &self.tokio, ingest, stream) } - /// # Not Implemented - /// Currently will return an error with [`Status::NotImplemented`]. + /// Get the schema of the query's result set without executing the query. /// - /// See the tracking issue for details and subscribe for updates: - /// + /// This runs `DESCRIBE ()` on the server and maps the result columns with the same + /// type mapping as [`Connection::get_table_schema()`][adbc_core::Connection::get_table_schema], + /// so the schema matches the one [`execute()`][Self::execute] returns. Bound parameters + /// (from [`bind()`][Self::bind]) are sent with the `DESCRIBE`. + /// + /// Returns [`Status::InvalidState`] if no query is set, or if the query is an `INSERT` + /// that would stream bound data. fn execute_schema(&mut self) -> adbc_core::error::Result { - err_unimplemented!("ClickhouseStatement::execute_schema()") + let StatementMode::SqlQuery(sql) = &self.mode else { + return Err(Error::with_message_and_status( + "execute_schema requires a SQL query", + Status::InvalidState, + )); + }; + if is_streaming_insert(sql) { + return Err(Error::with_message_and_status( + "execute_schema requires a query that returns a result set", + Status::InvalidState, + )); + } + + let _guard = self.tokio.enter(); + + // This statement's query ID is reused for its eventual data query. Give each metadata + // request its own ID, so ClickHouse does not reject the data query (or the second + // metadata request) while an earlier request with the same ID is still being cleaned up. + let mut schema_client = Client::clone(&self.client); + schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema")); + let mut settings_client = schema_client.clone(); + settings_client.set_setting(options::as_setting::QUERY_ID, random_id("schema_settings")); + + // Raw so a literal `?` is not treated as a bind placeholder. + let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})")); + if let Some(BindType::Single(batch)) = &self.bind { + query = bind_record_batch(query, batch.clone())?; + } + + self.tokio + .block_on(schema::of_query(&settings_client, query)) } /// # Not Implemented diff --git a/tests/it/compression.rs b/tests/it/compression.rs new file mode 100644 index 0000000..c9cad29 --- /dev/null +++ b/tests/it/compression.rs @@ -0,0 +1,76 @@ +//! Tests for the `clickhouse.client.compression` database option. + +use crate::test_driver; +use adbc_clickhouse::options; +use adbc_core::error::Status; +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; + +/// Sum `number` over enough rows to span several response blocks. +fn sum_numbers(compression: Option<&str>) -> u64 { + let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())]; + if let Some(compression) = compression { + opts.push((options::COMPRESSION.into(), compression.into())); + } + let db = test_driver().new_database_with_opts(opts).unwrap(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT number FROM system.numbers LIMIT 1000000") + .unwrap(); + + statement + .execute() + .unwrap() + .collect::, _>>() + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) + .sum() +} + +#[test] +fn compression_none_and_lz4_return_the_same_data() { + let expected = 999_999 * 1_000_000 / 2; + assert_eq!(sum_numbers(None), expected); + assert_eq!(sum_numbers(Some("lz4")), expected); + assert_eq!(sum_numbers(Some("none")), expected); + assert_eq!(sum_numbers(Some("NONE")), expected); +} + +#[test] +fn compression_read_back() { + let mut driver = test_driver(); + + let db = driver.new_database().unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "lz4" + ); + + let db = driver + .new_database_with_opts([(options::COMPRESSION.into(), "none".into())]) + .unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "none" + ); +} + +#[test] +fn compression_rejects_unknown_values() { + let err = test_driver() + .new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())]) + .err() + .expect("expected an error for an unknown compression method"); + assert_eq!(err.status, Status::InvalidArguments); +} diff --git a/tests/it/execute_schema.rs b/tests/it/execute_schema.rs new file mode 100644 index 0000000..fbfb34b --- /dev/null +++ b/tests/it/execute_schema.rs @@ -0,0 +1,117 @@ +//! Tests for `Statement::execute_schema()`. + +use crate::test_database; +use adbc_core::error::Status; +use adbc_core::{Connection, Database, Statement}; +use arrow_array::{RecordBatch, create_array}; +use arrow_schema::{DataType, Field, Schema}; + +/// Columns of several types, including nullable, decimal and low-cardinality ones. +const QUERY: &str = "SELECT number, toString(number) AS s, toNullable(toInt32(number)) AS n, \ + toDecimal64(number, 2) AS d64, toDecimal128(number, 4) AS d128, toDate32('2024-01-01') AS d32, \ + toDateTime64('2024-01-01 00:00:00', 3, 'UTC') AS ts, toFloat64(number) / 3 AS f, \ + toLowCardinality(toString(number % 3)) AS lc, [number, number + 1] AS arr \ + FROM system.numbers LIMIT 3"; + +/// `execute_schema()` annotates each field with its ClickHouse type (`CLICKHOUSE:type` metadata), +/// as `get_table_schema()` does; the Arrow stream from `execute()` carries no field metadata. +fn without_metadata(schema: &Schema) -> Schema { + Schema::new( + schema + .fields() + .iter() + .map(|f| f.as_ref().clone().with_metadata(Default::default())) + .collect::>(), + ) +} + +#[test] +fn execute_schema_matches_execute() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let described = statement.execute_schema().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let executed = statement.execute().unwrap().schema(); + + assert_eq!(without_metadata(&described), *executed); + assert_eq!( + described.field_with_name("lc").unwrap().metadata()["CLICKHOUSE:type"], + "LowCardinality(String)" + ); +} + +#[test] +fn execute_after_execute_schema() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + // The data query reuses the statement's query ID; the schema requests must not collide with it. + for _ in 0..3 { + let described = statement.execute_schema().unwrap(); + let reader = statement.execute().unwrap(); + assert_eq!(without_metadata(&described), *reader.schema()); + let rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); + assert_eq!(rows, 3); + } +} + +#[test] +fn execute_schema_with_bound_parameters() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT {x:Int32} AS x, {s:String} AS s") + .unwrap(); + + let params = RecordBatch::try_new( + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + .into(), + vec![create_array!(Int32, [7]), create_array!(Utf8, ["seven"])], + ) + .unwrap(); + statement.bind(params).unwrap(); + + assert_eq!( + without_metadata(&statement.execute_schema().unwrap()), + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + ); +} + +#[test] +fn execute_schema_without_query() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert_eq!(err.status, Status::InvalidState); +} + +#[test] +fn execute_schema_reports_server_errors() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT * FROM adbc_no_such_table_for_execute_schema") + .unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert!( + err.message.contains("UNKNOWN_TABLE") || err.message.contains("doesn't exist"), + "unexpected error: {}", + err.message + ); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index f990b70..97dde9c 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -11,7 +11,9 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use std::sync::Arc; mod bulk_ingest; +mod compression; mod database; +mod execute_schema; mod get_table_schema; mod params; mod settings;