From fe0cf04fbb5ef88e5ba7a019eefec7dd8ea15cf3 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:07:50 -0500 Subject: [PATCH 1/4] feat: add clickhouse.client.compression database option Add a database option to choose the transport compression used between the driver and ClickHouse: "lz4" (the default, unchanged) or "none". ClickHouse compresses a query's response on a single thread, which can cap one query's throughput far below what a fast network carries; "none" removes that cap. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++ src/lib.rs | 14 ++++++++ src/options.rs | 50 +++++++++++++++++++++++++++ tests/it/compression.rs | 76 +++++++++++++++++++++++++++++++++++++++++ tests/it/main.rs | 1 + 5 files changed, 147 insertions(+) create mode 100644 tests/it/compression.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b508748..1ad56a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +* Added the `clickhouse.client.compression` database option to choose the transport compression: + `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, + so `"none"` can be much faster on fast networks. + ## [0.1.2] - 2026-11-06 ### Fixed diff --git a/src/lib.rs b/src/lib.rs index 26b5bed..449213d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -242,6 +242,7 @@ impl Driver for ClickhouseDriver { password: None, product_info: self.product_info.clone(), settings: HashMap::new(), + compression: None, }; for (key, value) in opts { @@ -356,6 +357,8 @@ pub struct ClickhouseDatabase { password: Option, product_info: ProductInfo, settings: HashMap, + /// `None` keeps the `clickhouse` crate's default (LZ4). + compression: Option, } impl Database for ClickhouseDatabase { @@ -386,6 +389,10 @@ impl Database for ClickhouseDatabase { // to a different value at a lower level; `AugmentedClient` covers that. .with_product_info("adbc_clickhouse", env!("CARGO_PKG_VERSION")); + if let Some(compression) = self.compression { + client = client.with_compression(compression); + } + if let Some(url) = &self.url { client = client.with_url(url.to_string()); } @@ -583,6 +590,9 @@ impl ClickhouseDatabase { options::PRODUCT_INFO => { self.product_info = value.try_into()?; } + options::COMPRESSION => { + self.compression = Some(options::parse_compression(&value.try_string(key)?)?); + } other => { return Err(Error::with_message_and_status( format!("unknown database option {other:?}"), @@ -605,6 +615,10 @@ impl ClickhouseDatabase { match key { options::PRODUCT_INFO => Ok(self.product_info.to_string()), + options::COMPRESSION => Ok(options::compression_name( + self.compression.unwrap_or(clickhouse::Compression::Lz4), + ) + .to_string()), other => Err(Error::with_message_and_status( format!("unknown database option {other:?}"), Status::InvalidArguments, diff --git a/src/options.rs b/src/options.rs index 1412dfc..9644b4e 100644 --- a/src/options.rs +++ b/src/options.rs @@ -164,6 +164,56 @@ pub const QUERY_ID: &str = "clickhouse.client.query_id"; /// as Arrow binary strings instead. pub const OUTPUT_STRING_AS_STRING: &str = "clickhouse.client.output_string_as_string"; +/// Transport compression between the driver and ClickHouse: `"lz4"` (the default) or `"none"`. +/// +/// With `"lz4"`, ClickHouse compresses each query response into LZ4 blocks (the HTTP `compress=1` +/// parameter) and the driver decompresses them; `INSERT` data is compressed by the driver. +/// With `"none"`, data is sent as is. +/// +/// ClickHouse compresses a query's response on a single thread, so LZ4 can limit the throughput +/// of one query to well below what a fast network carries. On fast links, `"none"` is often +/// faster; on slow links, `"lz4"` sends fewer bytes. +/// +/// May be set on [`ClickhouseDatabase`] only, and applies to every connection it creates. +/// The current value may be read back with [`Optionable::get_option_string()`]. +/// +/// # Example +/// ```rust +/// use adbc_clickhouse::ClickhouseDriver; +/// +/// use adbc_core::{Driver, Optionable}; +/// +/// let mut driver = ClickhouseDriver::init(); +/// +/// let db = driver +/// .new_database_with_opts([(adbc_clickhouse::options::COMPRESSION.into(), "none".into())]) +/// .unwrap(); +/// +/// assert_eq!( +/// db.get_option_string(adbc_clickhouse::options::COMPRESSION.into()).unwrap(), +/// "none" +/// ); +/// ``` +pub const COMPRESSION: &str = "clickhouse.client.compression"; + +pub(crate) fn parse_compression(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "none" => Ok(clickhouse::Compression::None), + "lz4" => Ok(clickhouse::Compression::Lz4), + _ => Err(Error::with_message_and_status( + format!("invalid value for {COMPRESSION:?}: {value:?} (expected \"lz4\" or \"none\")"), + Status::InvalidArguments, + )), + } +} + +pub(crate) fn compression_name(compression: clickhouse::Compression) -> &'static str { + match compression { + clickhouse::Compression::None => "none", + _ => "lz4", + } +} + /// Prefix for passing arbitrary [ClickHouse settings] through as driver options. /// /// An option key of the form `clickhouse.setting.` maps directly to the diff --git a/tests/it/compression.rs b/tests/it/compression.rs new file mode 100644 index 0000000..c9cad29 --- /dev/null +++ b/tests/it/compression.rs @@ -0,0 +1,76 @@ +//! Tests for the `clickhouse.client.compression` database option. + +use crate::test_driver; +use adbc_clickhouse::options; +use adbc_core::error::Status; +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use arrow_array::RecordBatch; +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; + +/// Sum `number` over enough rows to span several response blocks. +fn sum_numbers(compression: Option<&str>) -> u64 { + let mut opts = vec![(OptionDatabase::Uri, "http://localhost:8123/".into())]; + if let Some(compression) = compression { + opts.push((options::COMPRESSION.into(), compression.into())); + } + let db = test_driver().new_database_with_opts(opts).unwrap(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT number FROM system.numbers LIMIT 1000000") + .unwrap(); + + statement + .execute() + .unwrap() + .collect::, _>>() + .unwrap() + .iter() + .flat_map(|batch| { + batch + .column(0) + .as_primitive::() + .values() + .to_vec() + }) + .sum() +} + +#[test] +fn compression_none_and_lz4_return_the_same_data() { + let expected = 999_999 * 1_000_000 / 2; + assert_eq!(sum_numbers(None), expected); + assert_eq!(sum_numbers(Some("lz4")), expected); + assert_eq!(sum_numbers(Some("none")), expected); + assert_eq!(sum_numbers(Some("NONE")), expected); +} + +#[test] +fn compression_read_back() { + let mut driver = test_driver(); + + let db = driver.new_database().unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "lz4" + ); + + let db = driver + .new_database_with_opts([(options::COMPRESSION.into(), "none".into())]) + .unwrap(); + assert_eq!( + db.get_option_string(options::COMPRESSION.into()).unwrap(), + "none" + ); +} + +#[test] +fn compression_rejects_unknown_values() { + let err = test_driver() + .new_database_with_opts([(options::COMPRESSION.into(), "zstd".into())]) + .err() + .expect("expected an error for an unknown compression method"); + assert_eq!(err.status, Status::InvalidArguments); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index f990b70..45ebac8 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -11,6 +11,7 @@ use arrow_schema::{DataType, Field, Schema, TimeUnit}; use std::sync::Arc; mod bulk_ingest; +mod compression; mod database; mod get_table_schema; mod params; From c177c415abf262614bf58a943f9cfd787f085dd2 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:09:19 -0500 Subject: [PATCH 2/4] feat: implement Statement::execute_schema() with DESCRIBE Return the result schema of a query without running it, by sending DESCRIBE () and mapping the columns with the same type mapping as Connection::get_table_schema(). Bound parameters are sent with the DESCRIBE. The schema requests get their own query IDs: the statement's query ID is reused by its data query, and ClickHouse rejects a request whose query ID is still held by an earlier request that is being cleaned up. The settings query starts only after DESCRIBE has finished, because both carry the connection's session ID and ClickHouse rejects overlapping requests on one session. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + src/schema.rs | 44 ++++++++++++++ src/statement.rs | 46 +++++++++++++-- tests/it/execute_schema.rs | 117 +++++++++++++++++++++++++++++++++++++ tests/it/main.rs | 1 + 5 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 tests/it/execute_schema.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ad56a8..41d5cd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added the `clickhouse.client.compression` database option to choose the transport compression: `"lz4"` (the default) or `"none"`. ClickHouse compresses each query response on a single thread, so `"none"` can be much faster on fast networks. +* Implemented `Statement::execute_schema()` with `DESCRIBE ()`, using the same type mapping + as `Connection::get_table_schema()`. Bound parameters are sent with the `DESCRIBE`. ## [0.1.2] - 2026-11-06 diff --git a/src/schema.rs b/src/schema.rs index e62a9e8..ae6f035 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,6 +1,7 @@ use adbc_core::error::{Error, Status}; use arrow_schema::{DataType, Field, Fields, Schema, TimeUnit}; use clickhouse::Client; +use clickhouse::query::Query; use clickhouse_types::DataTypeNode; use clickhouse_types::data_types::{DateTimePrecision, DecimalType, EnumType}; use serde::{Deserialize, Deserializer}; @@ -166,6 +167,49 @@ pub async fn of_table( Ok(Schema::new(fields)) } +/// Describe the result columns of a query prepared as `DESCRIBE ()`. +/// +/// The columns are mapped with the same type mapping (and the same server settings) as +/// [`of_table()`], so the schema matches the Arrow stream ClickHouse returns for the query. +pub async fn of_query(client: &Client, query: Query) -> adbc_core::error::Result { + let mut cursor = query.fetch::().map_err(|e| { + Error::with_message_and_status( + format!("error beginning fetch of query schema: {e}"), + Status::Internal, + ) + })?; + + let mut columns = Vec::new(); + while let Some(column) = cursor.next().await.map_err(|e| { + Error::with_message_and_status( + format!("error retrieving query schema: {e}"), + Status::Internal, + ) + })? { + columns.push(column); + } + drop(cursor); + + // Both requests carry the connection's session ID, and ClickHouse rejects a request that + // overlaps another on the same session, so finish `DESCRIBE` before querying the settings. + let settings = Settings::query(client).await?; + + let mut fields = Vec::with_capacity(columns.len()); + for column in &columns { + let field = column_to_arrow(column, &settings).map_err(|e| Error { + message: format!( + "error mapping query column {:?}: {}", + column.name, e.message + ), + ..e + })?; + + fields.push(field); + } + + Ok(Schema::new(fields)) +} + fn column_to_arrow(column: &Column, settings: &Settings) -> Result { let ch_type = DataTypeNode::new(&column.r#type).map_err(|e| { Error::with_message_and_status( diff --git a/src/statement.rs b/src/statement.rs index ef4bf2b..baee75a 100644 --- a/src/statement.rs +++ b/src/statement.rs @@ -23,7 +23,7 @@ use clickhouse::query::Query; use crate::options::OptionValueExt; use crate::writer::ArrowStreamWriter; -use crate::{AugmentedClient, Result, TokioContext, options, random_id}; +use crate::{AugmentedClient, Result, TokioContext, options, random_id, schema}; use clickhouse::_priv::sql_escape_identifier; use clickhouse_ext_arrow::ArrowQueryExt; @@ -251,13 +251,47 @@ impl Statement for ClickhouseStatement { execute_streaming_ingest(&self.client, &self.tokio, ingest, stream) } - /// # Not Implemented - /// Currently will return an error with [`Status::NotImplemented`]. + /// Get the schema of the query's result set without executing the query. /// - /// See the tracking issue for details and subscribe for updates: - /// + /// This runs `DESCRIBE ()` on the server and maps the result columns with the same + /// type mapping as [`Connection::get_table_schema()`][adbc_core::Connection::get_table_schema], + /// so the schema matches the one [`execute()`][Self::execute] returns. Bound parameters + /// (from [`bind()`][Self::bind]) are sent with the `DESCRIBE`. + /// + /// Returns [`Status::InvalidState`] if no query is set, or if the query is an `INSERT` + /// that would stream bound data. fn execute_schema(&mut self) -> adbc_core::error::Result { - err_unimplemented!("ClickhouseStatement::execute_schema()") + let StatementMode::SqlQuery(sql) = &self.mode else { + return Err(Error::with_message_and_status( + "execute_schema requires a SQL query", + Status::InvalidState, + )); + }; + if is_streaming_insert(sql) { + return Err(Error::with_message_and_status( + "execute_schema requires a query that returns a result set", + Status::InvalidState, + )); + } + + let _guard = self.tokio.enter(); + + // This statement's query ID is reused for its eventual data query. Give each metadata + // request its own ID, so ClickHouse does not reject the data query (or the second + // metadata request) while an earlier request with the same ID is still being cleaned up. + let mut schema_client = Client::clone(&self.client); + schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema")); + let mut settings_client = schema_client.clone(); + settings_client.set_setting(options::as_setting::QUERY_ID, random_id("schema_settings")); + + // Raw so a literal `?` is not treated as a bind placeholder. + let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})")); + if let Some(BindType::Single(batch)) = &self.bind { + query = bind_record_batch(query, batch.clone())?; + } + + self.tokio + .block_on(schema::of_query(&settings_client, query)) } /// # Not Implemented diff --git a/tests/it/execute_schema.rs b/tests/it/execute_schema.rs new file mode 100644 index 0000000..fbfb34b --- /dev/null +++ b/tests/it/execute_schema.rs @@ -0,0 +1,117 @@ +//! Tests for `Statement::execute_schema()`. + +use crate::test_database; +use adbc_core::error::Status; +use adbc_core::{Connection, Database, Statement}; +use arrow_array::{RecordBatch, create_array}; +use arrow_schema::{DataType, Field, Schema}; + +/// Columns of several types, including nullable, decimal and low-cardinality ones. +const QUERY: &str = "SELECT number, toString(number) AS s, toNullable(toInt32(number)) AS n, \ + toDecimal64(number, 2) AS d64, toDecimal128(number, 4) AS d128, toDate32('2024-01-01') AS d32, \ + toDateTime64('2024-01-01 00:00:00', 3, 'UTC') AS ts, toFloat64(number) / 3 AS f, \ + toLowCardinality(toString(number % 3)) AS lc, [number, number + 1] AS arr \ + FROM system.numbers LIMIT 3"; + +/// `execute_schema()` annotates each field with its ClickHouse type (`CLICKHOUSE:type` metadata), +/// as `get_table_schema()` does; the Arrow stream from `execute()` carries no field metadata. +fn without_metadata(schema: &Schema) -> Schema { + Schema::new( + schema + .fields() + .iter() + .map(|f| f.as_ref().clone().with_metadata(Default::default())) + .collect::>(), + ) +} + +#[test] +fn execute_schema_matches_execute() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let described = statement.execute_schema().unwrap(); + + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let executed = statement.execute().unwrap().schema(); + + assert_eq!(without_metadata(&described), *executed); + assert_eq!( + described.field_with_name("lc").unwrap().metadata()["CLICKHOUSE:type"], + "LowCardinality(String)" + ); +} + +#[test] +fn execute_after_execute_schema() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + // The data query reuses the statement's query ID; the schema requests must not collide with it. + for _ in 0..3 { + let described = statement.execute_schema().unwrap(); + let reader = statement.execute().unwrap(); + assert_eq!(without_metadata(&described), *reader.schema()); + let rows: usize = reader.map(|batch| batch.unwrap().num_rows()).sum(); + assert_eq!(rows, 3); + } +} + +#[test] +fn execute_schema_with_bound_parameters() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT {x:Int32} AS x, {s:String} AS s") + .unwrap(); + + let params = RecordBatch::try_new( + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + .into(), + vec![create_array!(Int32, [7]), create_array!(Utf8, ["seven"])], + ) + .unwrap(); + statement.bind(params).unwrap(); + + assert_eq!( + without_metadata(&statement.execute_schema().unwrap()), + Schema::new(vec![ + Field::new("x", DataType::Int32, false), + Field::new("s", DataType::Utf8, false), + ]) + ); +} + +#[test] +fn execute_schema_without_query() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert_eq!(err.status, Status::InvalidState); +} + +#[test] +fn execute_schema_reports_server_errors() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT * FROM adbc_no_such_table_for_execute_schema") + .unwrap(); + let err = statement.execute_schema().unwrap_err(); + assert!( + err.message.contains("UNKNOWN_TABLE") || err.message.contains("doesn't exist"), + "unexpected error: {}", + err.message + ); +} diff --git a/tests/it/main.rs b/tests/it/main.rs index 45ebac8..97dde9c 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -13,6 +13,7 @@ use std::sync::Arc; mod bulk_ingest; mod compression; mod database; +mod execute_schema; mod get_table_schema; mod params; mod settings; From dc4cefc840ea8150db398b24737bc7d3357a71e6 Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:11:36 -0500 Subject: [PATCH 3/4] fix: map Date per output_format_arrow_date_as_uint16 ClickHouse 26.2 added output_format_arrow_date_as_uint16, default 0, and now writes Date columns as Arrow date32 in ArrowStream. The declared schema still said UInt16, so get_table_schema() and execute_schema() disagreed with the data execute() returns. Read the setting with the other Arrow format settings and map Date to Date32 unless it is 1. Servers without the setting (26.1 and older) always write UInt16, so a missing setting keeps the old mapping. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ agents.md | 2 ++ src/schema.rs | 26 +++++++++++++++++++++++++- tests/it/execute_schema.rs | 35 ++++++++++++++++++++++++++++++++++- tests/it/get_table_schema.rs | 4 ++-- 5 files changed, 69 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d5cd7..329e9ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Implemented `Statement::execute_schema()` with `DESCRIBE ()`, using the same type mapping as `Connection::get_table_schema()`. Bound parameters are sent with the `DESCRIBE`. +### Fixed + +* Map ClickHouse `Date` to Arrow `Date32` when `output_format_arrow_date_as_uint16` is `0`, the default + since ClickHouse 26.2, so `Connection::get_table_schema()` and `Statement::execute_schema()` match the + data `Statement::execute()` returns. Older servers, which always write `Date` as `UInt16`, are unchanged. + ## [0.1.2] - 2026-11-06 ### Fixed diff --git a/agents.md b/agents.md index 7f1b98d..bf4a0ce 100644 --- a/agents.md +++ b/agents.md @@ -32,6 +32,8 @@ A primer for AI agents working on this repository. Read this before doing substa - **`arrow-ipc::StreamDecoder::finish()` only inspects state, doesn't mutate it.** Safe to call multiple times. It also catches every partial-data / truncated-stream scenario with `IpcError("Unexpected End of Stream")`, so this driver doesn't need to re-validate that. - **`SchemaRef = Arc`.** Existing codebase convention is `Schema::new(...).into()` / `Schema::empty().into()`, *not* `Arc::new(Schema::...)`. The `Arc::new` form is only used here for `Runtime` and `Client`. +- **`Date` in `ArrowStream` depends on the server version.** ClickHouse 26.2 added `output_format_arrow_date_as_uint16` (default `0` → Arrow `date32`); 26.1 and older lack the setting and always write `uint16` (checked against the 25.8, 26.1, 26.2 and 26.10 Docker images). Declared schemas (`get_table_schema`, `execute_schema`) must follow it via `Settings::query()`. + ## Related local repos worth referencing - `/Users/josemunoz/repos/clickhouse-rs` — the underlying ClickHouse client. Useful files: `src/response.rs` (HTTP error filtering), `src/cursors/*` (fused stream behavior), `src/query.rs` (`fetch_bytes` / `execute`). diff --git a/src/schema.rs b/src/schema.rs index ae6f035..030e688 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -39,6 +39,12 @@ struct Settings { /// #[serde(deserialize_with = "deserialize_bool_setting")] use_64_bit_indexes_for_dictionary: bool, + /// + /// + /// Added in ClickHouse 26.2 with default `0` (`Date` as Arrow `date32`). Older servers lack the + /// setting and always write `Date` as `UInt16`, so a missing setting means `true`. + #[serde(deserialize_with = "deserialize_bool_setting_default_true")] + date_as_uint16: bool, } impl Default for Settings { @@ -57,6 +63,7 @@ impl Default for Settings { low_cardinality_as_dictionary: false, use_signed_indexes_for_dictionary: false, use_64_bit_indexes_for_dictionary: false, + date_as_uint16: true, } } } @@ -71,6 +78,7 @@ impl Settings { "low_cardinality_as_dictionary", "use_signed_indexes_for_dictionary", "use_64_bit_indexes_for_dictionary", + "date_as_uint16", ]; let mut sql = "SELECT ".to_string(); @@ -284,7 +292,8 @@ fn ch_type_to_arrow( // a canonical extension type in Arrow: https://arrow.apache.org/docs/format/CanonicalExtensions.html#uuid // DataTypeNode::UUID => {} // https://github.com/ClickHouse/ClickHouse/blob/3196ab525aa6f1fef8d367db7610b765f7737f01/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp#L86 - DataTypeNode::Date => Ok(DataType::UInt16), + DataTypeNode::Date if settings.date_as_uint16 => Ok(DataType::UInt16), + DataTypeNode::Date => Ok(DataType::Date32), DataTypeNode::Date32 => Ok(DataType::Date32), // https://github.com/ClickHouse/ClickHouse/blob/3196ab525aa6f1fef8d367db7610b765f7737f01/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp#L87C1-L87C114 DataTypeNode::DateTime(_) => Ok(DataType::UInt32), @@ -491,3 +500,18 @@ where _ => Err(Error::custom(format!("expected \"0\" or \"1\", got {s:?}"))), } } + +/// Like [`deserialize_bool_setting`], but a missing setting means `true`. +fn deserialize_bool_setting_default_true<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + use serde::de::Error; + + match Option::::deserialize(deserializer)?.as_deref() { + None => Ok(true), + Some("0") => Ok(false), + Some("1") => Ok(true), + Some(s) => Err(Error::custom(format!("expected \"0\" or \"1\", got {s:?}"))), + } +} diff --git a/tests/it/execute_schema.rs b/tests/it/execute_schema.rs index fbfb34b..1b0a5f1 100644 --- a/tests/it/execute_schema.rs +++ b/tests/it/execute_schema.rs @@ -2,13 +2,14 @@ use crate::test_database; use adbc_core::error::Status; +use adbc_core::options::OptionConnection; use adbc_core::{Connection, Database, Statement}; use arrow_array::{RecordBatch, create_array}; use arrow_schema::{DataType, Field, Schema}; /// Columns of several types, including nullable, decimal and low-cardinality ones. const QUERY: &str = "SELECT number, toString(number) AS s, toNullable(toInt32(number)) AS n, \ - toDecimal64(number, 2) AS d64, toDecimal128(number, 4) AS d128, toDate32('2024-01-01') AS d32, \ + toDecimal64(number, 2) AS d64, toDecimal128(number, 4) AS d128, toDate('2024-01-01') AS d, toDate32('2024-01-01') AS d32, \ toDateTime64('2024-01-01 00:00:00', 3, 'UTC') AS ts, toFloat64(number) / 3 AS f, \ toLowCardinality(toString(number % 3)) AS lc, [number, number + 1] AS arr \ FROM system.numbers LIMIT 3"; @@ -115,3 +116,35 @@ fn execute_schema_reports_server_errors() { err.message ); } + +#[test] +fn execute_schema_date_follows_date_as_uint16() { + let db = test_database(); + for (setting, expected) in [("0", DataType::Date32), ("1", DataType::UInt16)] { + let mut conn = db + .new_connection_with_opts([( + OptionConnection::Other( + "clickhouse.setting.output_format_arrow_date_as_uint16".into(), + ), + setting.into(), + )]) + .unwrap(); + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT toDate('2024-01-02') AS d") + .unwrap(); + + let described = statement.execute_schema().unwrap(); + let executed = statement.execute().unwrap().schema(); + assert_eq!( + described.field(0).data_type(), + &expected, + "setting {setting}" + ); + assert_eq!( + executed.field(0).data_type(), + &expected, + "setting {setting}" + ); + } +} diff --git a/tests/it/get_table_schema.rs b/tests/it/get_table_schema.rs index ae56b15..43dfa06 100644 --- a/tests/it/get_table_schema.rs +++ b/tests/it/get_table_schema.rs @@ -312,7 +312,7 @@ Schema { }, Field { name: \"time_date\", - data_type: UInt16, + data_type: Date32, metadata: { \"CLICKHOUSE:type\": \"Date\", }, @@ -392,7 +392,7 @@ Schema { }, Field { name: \"chrono_date\", - data_type: UInt16, + data_type: Date32, metadata: { \"CLICKHOUSE:type\": \"Date\", }, From faa2c87512ca78bd8031b0cc6cd15e02f3e4f6cd Mon Sep 17 00:00:00 2001 From: Andrew Witten Date: Thu, 12 Nov 2026 16:13:53 -0500 Subject: [PATCH 4/4] fix: retry requests rejected with SESSION_IS_LOCKED A connection's requests share one ClickHouse session, and ClickHouse runs one request at a time per session. When a result set is dropped before it is fully read, the server is still finishing that query, so the next query on the same connection failed with SESSION_IS_LOCKED (code 373) every time. The error arrives before any data, so retry execute() and execute_schema() for up to about five seconds (5 ms doubling to 200 ms). A bind given as a stream cannot be replayed, so those requests are not retried. In the new regression test the follow-up query succeeds after 30-40 ms. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 ++ agents.md | 2 ++ src/statement.rs | 84 +++++++++++++++++++++++++++++++-------------- tests/it/main.rs | 1 + tests/it/session.rs | 36 +++++++++++++++++++ 5 files changed, 101 insertions(+), 25 deletions(-) create mode 100644 tests/it/session.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 329e9ac..56698d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Map ClickHouse `Date` to Arrow `Date32` when `output_format_arrow_date_as_uint16` is `0`, the default since ClickHouse 26.2, so `Connection::get_table_schema()` and `Statement::execute_schema()` match the data `Statement::execute()` returns. Older servers, which always write `Date` as `UInt16`, are unchanged. +* Retry `Statement::execute()` and `Statement::execute_schema()` for up to about five seconds when + ClickHouse rejects them with `SESSION_IS_LOCKED`, which happened on any query sent right after a + result set on the same connection was dropped before it was fully read. ## [0.1.2] - 2026-11-06 diff --git a/agents.md b/agents.md index bf4a0ce..6c5bc3d 100644 --- a/agents.md +++ b/agents.md @@ -34,6 +34,8 @@ A primer for AI agents working on this repository. Read this before doing substa - **`Date` in `ArrowStream` depends on the server version.** ClickHouse 26.2 added `output_format_arrow_date_as_uint16` (default `0` → Arrow `date32`); 26.1 and older lack the setting and always write `uint16` (checked against the 25.8, 26.1, 26.2 and 26.10 Docker images). Declared schemas (`get_table_schema`, `execute_schema`) must follow it via `Settings::query()`. +- **A session runs one HTTP request at a time.** A request arriving while another holds the session fails with `SESSION_IS_LOCKED` (373) before any data, e.g. the next query after a result set is dropped unread (the server is still finishing it). `retry_if_session_locked()` in `statement.rs` covers `execute()`/`execute_schema()`; repro in `tests/it/session.rs`. + ## Related local repos worth referencing - `/Users/josemunoz/repos/clickhouse-rs` — the underlying ClickHouse client. Useful files: `src/response.rs` (HTTP error filtering), `src/cursors/*` (fused stream behavior), `src/query.rs` (`fetch_bytes` / `execute`). diff --git a/src/statement.rs b/src/statement.rs index baee75a..61f6386 100644 --- a/src/statement.rs +++ b/src/statement.rs @@ -276,22 +276,25 @@ impl Statement for ClickhouseStatement { let _guard = self.tokio.enter(); - // This statement's query ID is reused for its eventual data query. Give each metadata - // request its own ID, so ClickHouse does not reject the data query (or the second - // metadata request) while an earlier request with the same ID is still being cleaned up. - let mut schema_client = Client::clone(&self.client); - schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema")); - let mut settings_client = schema_client.clone(); - settings_client.set_setting(options::as_setting::QUERY_ID, random_id("schema_settings")); - - // Raw so a literal `?` is not treated as a bind placeholder. - let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})")); - if let Some(BindType::Single(batch)) = &self.bind { - query = bind_record_batch(query, batch.clone())?; - } + retry_if_session_locked(|| { + // This statement's query ID is reused for its eventual data query. Give each metadata + // request its own ID, so ClickHouse does not reject the data query (or the second + // metadata request) while an earlier request with the same ID is still being cleaned up. + let mut schema_client = Client::clone(&self.client); + schema_client.set_setting(options::as_setting::QUERY_ID, random_id("schema")); + let mut settings_client = schema_client.clone(); + settings_client + .set_setting(options::as_setting::QUERY_ID, random_id("schema_settings")); + + // Raw so a literal `?` is not treated as a bind placeholder. + let mut query = schema_client.query_raw(&format!("DESCRIBE ({sql})")); + if let Some(BindType::Single(batch)) = &self.bind { + query = bind_record_batch(query, batch.clone())?; + } - self.tokio - .block_on(schema::of_query(&settings_client, query)) + self.tokio + .block_on(schema::of_query(&settings_client, query)) + }) } /// # Not Implemented @@ -989,18 +992,49 @@ fn fetch_blocking( // but that results in a larger generated future type let _guard = tokio.enter(); - // Raw so a literal `?` is not treated as a bind placeholder; - // parameters are bound server-side via `Query::param()`. - let mut query = client.query_raw(sql); - query = bind_query(query, bind)?; + let fetch = |bind: Option| -> Result { + // Raw so a literal `?` is not treated as a bind placeholder; + // parameters are bound server-side via `Query::param()`. + let mut query = client.query_raw(sql); + query = bind_query(query, bind)?; - let cursor = query.fetch_arrow().map_err(|e| { - Error::with_message_and_status(format!("error executing query: {e}"), Status::Internal) - })?; + let cursor = query.fetch_arrow().map_err(|e| { + Error::with_message_and_status(format!("error executing query: {e}"), Status::Internal) + })?; + + tokio + .block_on(ArrowStreamReader::begin(tokio.clone(), cursor)) + .map_err(Into::into) + }; - tokio - .block_on(ArrowStreamReader::begin(tokio.clone(), cursor)) - .map_err(Into::into) + match bind { + // A streamed bind cannot be replayed. + Some(BindType::Stream(stream)) => fetch(Some(BindType::Stream(stream))), + Some(BindType::Single(batch)) => { + retry_if_session_locked(|| fetch(Some(BindType::Single(batch.clone())))) + } + None => retry_if_session_locked(|| fetch(None)), + } +} + +/// ClickHouse runs one request at a time per session and rejects a request that arrives while +/// another one holds the session (`SESSION_IS_LOCKED`, code 373). That happens, for example, when +/// a result set is dropped before it is fully read: the server is still finishing that query when +/// the next request arrives. The error comes before any data, so retry for about five seconds. +fn retry_if_session_locked(mut request: impl FnMut() -> Result) -> Result { + const RETRY_LIMIT: u32 = 30; + let mut attempt = 0; + loop { + match request() { + Err(e) if attempt < RETRY_LIMIT && e.message.contains("SESSION_IS_LOCKED") => { + // 5, 10, 20, ... ms, capped at 200 ms. + let delay_ms = (5u64 << attempt.min(6)).min(200); + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + attempt += 1; + } + result => return result, + } + } } fn execute_blocking( diff --git a/tests/it/main.rs b/tests/it/main.rs index 97dde9c..a327396 100644 --- a/tests/it/main.rs +++ b/tests/it/main.rs @@ -16,6 +16,7 @@ mod database; mod execute_schema; mod get_table_schema; mod params; +mod session; mod settings; // NOTE: tests run with the `current-thread` runtime by default. diff --git a/tests/it/session.rs b/tests/it/session.rs new file mode 100644 index 0000000..d0ac3c8 --- /dev/null +++ b/tests/it/session.rs @@ -0,0 +1,36 @@ +//! Requests on one connection share a ClickHouse session, which runs one request at a time. + +use crate::test_database; +use adbc_core::{Connection, Database, Statement}; +use std::time::{Duration, Instant}; + +#[test] +fn query_after_dropping_an_unfinished_result() { + let db = test_database(); + let mut conn = db.new_connection().unwrap(); + + for _ in 0..5 { + let mut statement = conn.new_statement().unwrap(); + statement + .set_sql_query("SELECT number, toString(number) FROM system.numbers LIMIT 50000000") + .unwrap(); + let mut reader = statement.execute().unwrap(); + reader.next().unwrap().unwrap(); + drop(reader); + + // The server may still hold the session for the abandoned query. + let started = Instant::now(); + let mut statement = conn.new_statement().unwrap(); + statement.set_sql_query("SELECT 1 AS one").unwrap(); + let schema = statement.execute_schema().unwrap(); + assert_eq!(schema.field(0).name(), "one"); + let rows: usize = statement + .execute() + .unwrap() + .map(|batch| batch.unwrap().num_rows()) + .sum(); + assert_eq!(rows, 1); + println!("follow-up queries took {:?}", started.elapsed()); + assert!(started.elapsed() < Duration::from_secs(5)); + } +}