Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

* Added the `clickhouse.client.compression` database option to choose the transport compression:
`"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread,
so `"none"` can be much faster on fast networks.
* Implemented `Statement::execute_schema()` with `DESCRIBE (<query>)`, using the same type mapping
as `Connection::get_table_schema()`. Bound parameters are sent with the `DESCRIBE`.

## [0.1.2] - 2026-11-06

### Fixed
Expand Down
14 changes: 14 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver {
password: None,
product_info: self.product_info.clone(),
settings: HashMap::new(),
compression: None,
};

for (key, value) in opts {
Expand Down Expand Up @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase {
password: Option<String>,
product_info: ProductInfo,
settings: HashMap<String, String>,
/// `None` keeps the `clickhouse` crate's default (LZ4).
compression: Option<clickhouse::Compression>,
}

impl Database for ClickhouseDatabase {
Expand Down Expand Up @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase {
// to a different value at a lower level; `AugmentedClient` covers that.
.with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION"));

if let Some(compression) = self.compression {
client = client.with_compression(compression);
}

if let Some(url) = &self.url {
client = client.with_url(url.to_string());
}
Expand Down Expand Up @@ -583,6 +590,9 @@ impl ClickhouseDatabase {
options::PRODUCT_INFO => {
self.product_info = value.try_into()?;
}
options::COMPRESSION => {
self.compression = Some(options::parse_compression(&value.try_string(key)?)?);
}
other => {
return Err(Error::with_message_and_status(
format!("unknown database option {other:?}"),
Expand All @@ -605,6 +615,10 @@ impl ClickhouseDatabase {

match key {
options::PRODUCT_INFO => Ok(self.product_info.to_string()),
options::COMPRESSION => Ok(options::compression_name(
self.compression.unwrap_or(clickhouse::Compression::Lz4),
)
.to_string()),
other => Err(Error::with_message_and_status(
format!("unknown database option {other:?}"),
Status::InvalidArguments,
Expand Down
50 changes: 50 additions & 0 deletions src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id";
/// as Arrow binary strings instead.
pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string";

/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`.
///
/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1`
/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver.
/// With `"none"`, data is sent as is.
///
/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput
/// of one query to well below what a fast network carries. On fast links, `"none"` is often
/// faster; on slow links, `"lz4"` sends fewer bytes.
///
/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates.
/// The current value may be read back with [`Optionable::get_option_string()`].
///
/// # Example
/// ```rust
/// use adbc_clickhouse::ClickhouseDriver;
///
/// use adbc_core::{Driver, Optionable};
///
/// let mut driver = ClickhouseDriver::init();
///
/// let db = driver
/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())])
/// .unwrap();
///
/// assert_eq!(
/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(),
/// "none"
/// );
/// ```
pub const COMPRESSION: &str = "clickhouse.client.compression";

pub(crate) fn parse_compression(value: &str) -> Result<clickhouse::Compression, Error> {
match value.to_ascii_lowercase().as_str() {
"none" => Ok(clickhouse::Compression::None),
"lz4" => Ok(clickhouse::Compression::Lz4),
_ => Err(Error::with_message_and_status(
format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"),
Status::InvalidArguments,
)),
}
}

pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str {
match compression {
clickhouse::Compression::None => "none",
_ => "lz4",
}
}

/// Prefix for passing arbitrary [ClickHouse settings] through as driver options.
///
/// An option key of the form `clickhouse.setting.<setting_name>` maps directly to the
Expand Down
44 changes: 44 additions & 0 deletions src/schema.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use adbc_core::error::{Error, Status};
use arrow_schema::{DataType, Field, Fields, Schema, TimeUnit};
use clickhouse::Client;
use clickhouse::query::Query;
use clickhouse_types::DataTypeNode;
use clickhouse_types::data_types::{DateTimePrecision, DecimalType, EnumType};
use serde::{Deserialize, Deserializer};
Expand Down Expand Up @@ -166,6 +167,49 @@ pub async fn of_table(
Ok(Schema::new(fields))
}

/// Describe the result columns of a query prepared as `DESCRIBE (<query>)`.
///
/// The columns are mapped with the same type mapping (and the same server settings) as
/// [`of_table()`], so the schema matches the Arrow stream ClickHouse returns for the query.
pub async fn of_query(client: &Client, query: Query) -> adbc_core::error::Result<Schema> {
let mut cursor = query.fetch::<Column>().map_err(|e| {
Error::with_message_and_status(
format!("error beginning fetch of query schema: {e}"),
Status::Internal,
)
})?;

let mut columns = Vec::new();
while let Some(column) = cursor.next().await.map_err(|e| {
Error::with_message_and_status(
format!("error retrieving query schema: {e}"),
Status::Internal,
)
})? {
columns.push(column);
}
drop(cursor);

// Both requests carry the connection's session ID, and ClickHouse rejects a request that
// overlaps another on the same session, so finish `DESCRIBE` before querying the settings.
let settings = Settings::query(client).await?;

let mut fields = Vec::with_capacity(columns.len());
for column in &columns {
let field = column_to_arrow(column, &settings).map_err(|e| Error {
message: format!(
"error mapping query column {:?}: {}",
column.name, e.message
),
..e
})?;

fields.push(field);
}

Ok(Schema::new(fields))
}

fn column_to_arrow(column: &Column, settings: &Settings) -> Result<Field> {
let ch_type = DataTypeNode::new(&column.r#type).map_err(|e| {
Error::with_message_and_status(
Expand Down
46 changes: 40 additions & 6 deletions src/statement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use clickhouse::query::Query;

use crate::options::OptionValueExt;
use crate::writer::ArrowStreamWriter;
use crate::{AugmentedClient, Result, TokioContext, options, random_id};
use crate::{AugmentedClient, Result, TokioContext, options, random_id, schema};
use clickhouse::_priv::sql_escape_identifier;
use clickhouse_ext_arrow::ArrowQueryExt;

Expand Down Expand Up @@ -251,13 +251,47 @@ impl Statement for ClickhouseStatement {
execute_streaming_ingest(&self.client, &self.tokio, ingest, stream)
}

/// # Not Implemented
/// Currently will return an error with [`Status::NotImplemented`].
/// Get the schema of the query's result set without executing the query.
///
/// See the tracking issue for details and subscribe for updates:
/// <https://github.com/ClickHouse/adbc_clickhouse/issues/16>
/// This runs `DESCRIBE (<query>)` on the server and maps the result columns with the same
/// type mapping as [`Connection::get_table_schema()`][adbc_core::Connection::get_table_schema],
/// so the schema matches the one [`execute()`][Self::execute] returns. Bound parameters
/// (from [`bind()`][Self::bind]) are sent with the `DESCRIBE`.
///
/// Returns [`Status::InvalidState`] if no query is set, or if the query is an `INSERT`
/// that would stream bound data.
fn execute_schema(&mut self) -> adbc_core::error::Result<Schema> {
err_unimplemented!("ClickhouseStatement::execute_schema()")
let StatementMode::SqlQuery(sql) = &self.mode else {
return Err(Error::with_message_and_status(
"execute_schema requires a SQL query",
Status::InvalidState,
));
};
if is_streaming_insert(sql) {
return Err(Error::with_message_and_status(
"execute_schema requires a query that returns a result set",
Status::InvalidState,
));
}

let _guard = self.tokio.enter();

// This statement's query ID is reused for its eventual data query. Give each metadata
// request its own ID, so ClickHouse does not reject the data query (or the second
// metadata request) while an earlier request with the same ID is still being cleaned up.
let mut schema_client = Client::clone(&self.client);
schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema"));
let mut settings_client = schema_client.clone();
settings_client.set_setting(options::as_setting::QUERY_ID, random_id("schema_settings"));

// Raw so a literal `?` is not treated as a bind placeholder.
let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})"));
if let Some(BindType::Single(batch)) = &self.bind {
query = bind_record_batch(query, batch.clone())?;
}

self.tokio
.block_on(schema::of_query(&settings_client, query))
}

/// # Not Implemented
Expand Down
76 changes: 76 additions & 0 deletions tests/it/compression.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
//! Tests for the `clickhouse.client.compression` database option.

use crate::test_driver;
use adbc_clickhouse::options;
use adbc_core::error::Status;
use adbc_core::options::OptionDatabase;
use adbc_core::{Connection, Database, Driver, Optionable, Statement};
use arrow_array::RecordBatch;
use arrow_array::cast::AsArray;
use arrow_array::types::UInt64Type;

/// Sum `number` over enough rows to span several response blocks.
fn sum_numbers(compression: Option<&str>) -> u64 {
let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())];
if let Some(compression) = compression {
opts.push((options::COMPRESSION.into(), compression.into()));
}
let db = test_driver().new_database_with_opts(opts).unwrap();
let mut conn = db.new_connection().unwrap();
let mut statement = conn.new_statement().unwrap();
statement
.set_sql_query("SELECT number FROM system.numbers LIMIT 1000000")
.unwrap();

statement
.execute()
.unwrap()
.collect::<Result<Vec<RecordBatch>, _>>()
.unwrap()
.iter()
.flat_map(|batch| {
batch
.column(0)
.as_primitive::<UInt64Type>()
.values()
.to_vec()
})
.sum()
}

#[test]
fn compression_none_and_lz4_return_the_same_data() {
let expected = 999_999 * 1_000_000 / 2;
assert_eq!(sum_numbers(None), expected);
assert_eq!(sum_numbers(Some("lz4")), expected);
assert_eq!(sum_numbers(Some("none")), expected);
assert_eq!(sum_numbers(Some("NONE")), expected);
}

#[test]
fn compression_read_back() {
let mut driver = test_driver();

let db = driver.new_database().unwrap();
assert_eq!(
db.get_option_string(options::COMPRESSION.into()).unwrap(),
"lz4"
);

let db = driver
.new_database_with_opts([(options::COMPRESSION.into(), "none".into())])
.unwrap();
assert_eq!(
db.get_option_string(options::COMPRESSION.into()).unwrap(),
"none"
);
}

#[test]
fn compression_rejects_unknown_values() {
let err = test_driver()
.new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())])
.err()
.expect("expected an error for an unknown compression method");
assert_eq!(err.status, Status::InvalidArguments);
}
Loading