From fe0cf04fbb5ef88e5ba7a019eefec7dd8ea15cf3 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:07:50 -0500 Subject: [PATCH] feat: add clickhouse.client.compression database option Add a database option to choose the transport compression used between the driver and ClickHouse: "lz4" (the default, unchanged) or "none". ClickHouse compresses a query's response on a single thread, which can cap one query's throughput far below what a fast network carries; "none" removes that cap. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++ src/lib.rs | 14 ++++++++ src/options.rs | 50 +++++++++++++++++++++++++++ tests/it/compression.rs | 76 +++++++++++++++++++++++++++++++++++++++++ tests/it/main.rs | 1 + 5 files changed, 147 insertions(+) create mode 100644 tests/it/compression.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b508748..1ad56a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* Added the `clickhouse.client.compression` database option to choose the transport compression: + `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, + so `"none"` can be much faster on fast networks. + ## [0.1.2] - 2026-11-06 ### Fixed diff --git a/src/lib.rs b/src/lib.rs index 26b5bed..449213d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver { password: None, product_info: self.product_info.clone(), settings: HashMap::new(), + compression: None, }; for (key, value) in opts { @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase { password: Option, product_info: ProductInfo, settings: HashMap, + /// `None` keeps the `clickhouse` crate's default (LZ4). + compression: Option, } impl Database for ClickhouseDatabase { @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase { // to a different value at a lower level; `AugmentedClient` covers that. .with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION")); + if let Some(compression) = self.compression { + client = client.with_compression(compression); + } + if let Some(url) = &self.url { client = client.with_url(url.to_string()); } @@ -583,6 +590,9 @@ impl ClickhouseDatabase { options::PRODUCT_INFO => { self.product_info = value.try_into()?; } + options::COMPRESSION => { + self.compression = Some(options::parse_compression(&value.try_string(key)?)?); + } other => { return Err(Error::with_message_and_status( format!("unknown database option {other:?}"), @@ -605,6 +615,10 @@ impl ClickhouseDatabase { match key { options::PRODUCT_INFO => Ok(self.product_info.to_string()), + options::COMPRESSION => Ok(options::compression_name( + self.compression.unwrap_or(clickhouse::Compression::Lz4), + ) + .to_string()), other => Err(Error::with_message_and_status( format!("unknown database option {other:?}"), Status::InvalidArguments, diff --git a/src/options.rs b/src/options.rs index 1412dfc..9644b4e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id"; /// as Arrow binary strings instead. pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string"; +/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`. +/// +/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1` +/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver. +/// With `"none"`, data is sent as is. +/// +/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput +/// of one query to well below what a fast network carries. On fast links, `"none"` is often +/// faster; on slow links, `"lz4"` sends fewer bytes. +/// +/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates. +/// The current value may be read back with [`Optionable::get_option_string()`]. +/// +/// # Example +/// ```rust +/// use adbc_clickhouse::ClickhouseDriver; +/// +/// use adbc_core::{Driver, Optionable}; +/// +/// let mut driver = ClickhouseDriver::init(); +/// +/// let db = driver +/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())]) +/// .unwrap(); +/// +/// assert_eq!( +/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(), +/// "none" +/// ); +/// ``` +pub const COMPRESSION: &str = "clickhouse.client.compression"; + +pub(crate) fn parse_compression(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "none" => Ok(clickhouse::Compression::None), + "lz4" => Ok(clickhouse::Compression::Lz4), + _ => Err(Error::with_message_and_status( + format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"), + Status::InvalidArguments, + )), + } +} + +pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str { + match compression { + clickhouse::Compression::None => "none", + _ => "lz4", + } +} + /// Prefix for passing arbitrary [ClickHouse settings] through as driver options. /// /// An option key of the form `clickhouse.setting.` maps directly to the diff --git a/tests/it/compression.rs b/tests/it/compression.rs new file mode 100644 index 0000000..c9cad29 --- /dev/null +++ b/tests/it/compression.rs @@ -0,0 +1,76 @@ +//! Tests for the `clickhouse.client.compression` database option. + +use crate::test_driver; +use adbc_clickhouse::options; +use adbc_core::error::Status; +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; + +/// Sum `number` over enough rows to span several response blocks. +fn sum_numbers(compression: Option<&str>) -> u64 { + let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())]; + if let Some(compression) = compression { + opts.push((options::COMPRESSION.into(), compression.into())); + } + let db = test_driver().new_database_with_opts(opts).unwrap(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT number FROM system.numbers LIMIT 1000000") + .unwrap(); + + statement + .execute() + .unwrap() + .collect::, _>>() + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) + .sum() +} + +#[test] +fn compression_none_and_lz4_return_the_same_data() { + let expected = 999_999 * 1_000_000 / 2; + assert_eq!(sum_numbers(None), expected); + assert_eq!(sum_numbers(Some("lz4")), expected); + assert_eq!(sum_numbers(Some("none")), expected); + assert_eq!(sum_numbers(Some("NONE")), expected); +} + +#[test] +fn compression_read_back() { + let mut driver = test_driver(); + + let db = driver.new_database().unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "lz4" + ); + + let db = driver + .new_database_with_opts([(options::COMPRESSION.into(), "none".into())]) + .unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "none" + ); +} + +#[test] +fn compression_rejects_unknown_values() { + let err = test_driver() + .new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())]) + .err() + .expect("expected an error for an unknown compression method"); + assert_eq!(err.status, Status::InvalidArguments); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index f990b70..45ebac8 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -11,6 +11,7 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use std::sync::Arc; mod bulk_ingest; +mod compression; mod database; mod get_table_schema; mod params;