Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

* Added the `clickhouse.client.compression` database option to choose the transport compression:
`"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread,
so `"none"` can be much faster on fast networks.

## [0.1.2] - 2026-11-06

### Fixed
Expand Down
14 changes: 14 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver {
password: None,
product_info: self.product_info.clone(),
settings: HashMap::new(),
compression: None,
};

for (key, value) in opts {
Expand Down Expand Up @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase {
password: Option<String>,
product_info: ProductInfo,
settings: HashMap<String, String>,
/// `None` keeps the `clickhouse` crate's default (LZ4).
compression: Option<clickhouse::Compression>,
}

impl Database for ClickhouseDatabase {
Expand Down Expand Up @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase {
// to a different value at a lower level; `AugmentedClient` covers that.
.with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION"));

if let Some(compression) = self.compression {
client = client.with_compression(compression);
}

if let Some(url) = &self.url {
client = client.with_url(url.to_string());
}
Expand Down Expand Up @@ -583,6 +590,9 @@ impl ClickhouseDatabase {
options::PRODUCT_INFO => {
self.product_info = value.try_into()?;
}
options::COMPRESSION => {
self.compression = Some(options::parse_compression(&value.try_string(key)?)?);
}
other => {
return Err(Error::with_message_and_status(
format!("unknown database option {other:?}"),
Expand All @@ -605,6 +615,10 @@ impl ClickhouseDatabase {

match key {
options::PRODUCT_INFO => Ok(self.product_info.to_string()),
options::COMPRESSION => Ok(options::compression_name(
self.compression.unwrap_or(clickhouse::Compression::Lz4),
)
.to_string()),
other => Err(Error::with_message_and_status(
format!("unknown database option {other:?}"),
Status::InvalidArguments,
Expand Down
50 changes: 50 additions & 0 deletions src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id";
/// as Arrow binary strings instead.
pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string";

/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`.
///
/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1`
/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver.
/// With `"none"`, data is sent as is.
///
/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput
/// of one query to well below what a fast network carries. On fast links, `"none"` is often
/// faster; on slow links, `"lz4"` sends fewer bytes.
///
/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates.
/// The current value may be read back with [`Optionable::get_option_string()`].
///
/// # Example
/// ```rust
/// use adbc_clickhouse::ClickhouseDriver;
///
/// use adbc_core::{Driver, Optionable};
///
/// let mut driver = ClickhouseDriver::init();
///
/// let db = driver
/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())])
/// .unwrap();
///
/// assert_eq!(
/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(),
/// "none"
/// );
/// ```
pub const COMPRESSION: &str = "clickhouse.client.compression";

pub(crate) fn parse_compression(value: &str) -> Result<clickhouse::Compression, Error> {
match value.to_ascii_lowercase().as_str() {
"none" => Ok(clickhouse::Compression::None),
"lz4" => Ok(clickhouse::Compression::Lz4),
_ => Err(Error::with_message_and_status(
format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"),
Status::InvalidArguments,
)),
}
}

pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str {
match compression {
clickhouse::Compression::None => "none",
_ => "lz4",
}
}

/// Prefix for passing arbitrary [ClickHouse settings] through as driver options.
///
/// An option key of the form `clickhouse.setting.<setting_name>` maps directly to the
Expand Down
76 changes: 76 additions & 0 deletions tests/it/compression.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
//! Tests for the `clickhouse.client.compression` database option.

use crate::test_driver;
use adbc_clickhouse::options;
use adbc_core::error::Status;
use adbc_core::options::OptionDatabase;
use adbc_core::{Connection, Database, Driver, Optionable, Statement};
use arrow_array::RecordBatch;
use arrow_array::cast::AsArray;
use arrow_array::types::UInt64Type;

/// Sum `number` over enough rows to span several response blocks.
fn sum_numbers(compression: Option<&str>) -> u64 {
let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())];
if let Some(compression) = compression {
opts.push((options::COMPRESSION.into(), compression.into()));
}
let db = test_driver().new_database_with_opts(opts).unwrap();
let mut conn = db.new_connection().unwrap();
let mut statement = conn.new_statement().unwrap();
statement
.set_sql_query("SELECT number FROM system.numbers LIMIT 1000000")
.unwrap();

statement
.execute()
.unwrap()
.collect::<Result<Vec<RecordBatch>, _>>()
.unwrap()
.iter()
.flat_map(|batch| {
batch
.column(0)
.as_primitive::<UInt64Type>()
.values()
.to_vec()
})
.sum()
}

#[test]
fn compression_none_and_lz4_return_the_same_data() {
let expected = 999_999 * 1_000_000 / 2;
assert_eq!(sum_numbers(None), expected);
assert_eq!(sum_numbers(Some("lz4")), expected);
assert_eq!(sum_numbers(Some("none")), expected);
assert_eq!(sum_numbers(Some("NONE")), expected);
}

#[test]
fn compression_read_back() {
let mut driver = test_driver();

let db = driver.new_database().unwrap();
assert_eq!(
db.get_option_string(options::COMPRESSION.into()).unwrap(),
"lz4"
);

let db = driver
.new_database_with_opts([(options::COMPRESSION.into(), "none".into())])
.unwrap();
assert_eq!(
db.get_option_string(options::COMPRESSION.into()).unwrap(),
"none"
);
}

#[test]
fn compression_rejects_unknown_values() {
let err = test_driver()
.new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())])
.err()
.expect("expected an error for an unknown compression method");
assert_eq!(err.status, Status::InvalidArguments);
}
1 change: 1 addition & 0 deletions tests/it/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit};
use std::sync::Arc;

mod bulk_ingest;
mod compression;
mod database;
mod get_table_schema;
mod params;
Expand Down