From c42c6452ff6afc5ebc68ea734689ea09d24e436f Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 16 Jun 2026 14:01:46 -0400 Subject: [PATCH 01/15] feat: implement execute_partitions and read_partition Add ADBC partitioned execution so clients can split a Substrait or SQL query into one independently-executable partition per DataFusion output partition and read each on a fresh connection. - descriptor codec: [u32 target_partitions][u32 index][u8 kind][query] - DataFusionReader::from_stream for single-partition streams - execute_partitions: emit one descriptor per output partition, encoding the session's target_partitions - read_partition: re-plan with target_partitions pinned to the descriptor so partition i denotes the same slice regardless of executor CPU count Prepared statements return NOT_IMPLEMENTED (no portable plan serialization without datafusion-proto); clients fall back to a single partition. Also fix a latent bug: rebuilding the SessionState with the default config re-enabled create_default_catalog_and_schema, which on build() overwrote the carried catalog and dropped providers registered via ContextInit. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 211 +++++++++++++++++++++++++++++++++++++-- tests/test_datafusion.rs | 106 ++++++++++++++++++++ 2 files changed, 310 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 46d8e7c..5cb30ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ use datafusion::dataframe::DataFrameWriteOptions; use datafusion::datasource::MemTable; use datafusion::logical_expr::LogicalPlan; use datafusion::logical_expr::dml::InsertOp; +use datafusion::physical_plan::ExecutionPlanProperties; use datafusion::prelude::*; use datafusion_substrait::logical_plan::consumer::from_substrait_plan; use datafusion_substrait::substrait::proto::Plan; @@ -258,6 +259,21 @@ impl DataFusionReader { schema: schema.into(), }) } + + /// Construct a reader directly from a single-partition stream, bypassing the + /// `DataFrame` path. Used by `read_partition`, which executes one partition of an + /// already-built physical plan. + pub fn from_stream( + runtime: Arc, + stream: datafusion::execution::SendableRecordBatchStream, + schema: SchemaRef, + ) -> Self { + Self { + runtime, + stream, + schema, + } + } } impl Iterator for DataFusionReader { @@ -678,11 +694,59 @@ impl Connection for DataFusionConnection { fn read_partition( &self, - _partition: impl AsRef<[u8]>, + partition: impl AsRef<[u8]>, ) -> Result> { - Err(ErrorHelper::not_implemented() - .message("read_partition") - .to_adbc()) + let (target_partitions, index, query) = decode_descriptor(partition.as_ref())?; + self.runtime.block_on(async { + // Pin target_partitions so the physical plan matches the one execute_partitions + // counted; DataFusion planning is deterministic only when this is held fixed. + let state = + datafusion::execution::session_state::SessionStateBuilder::new_from_existing( + self.ctx.state(), + ) + .with_config( + self.ctx + .copied_config() + .with_target_partitions(target_partitions) + // The carried-over catalog list already holds the registered providers; + // recreating the default catalog here would wipe them. + .with_create_default_catalog_and_schema(false), + ) + .build(); + + let plan = match query { + DescQuery::Substrait(bytes) => { + let proto = Plan::decode(bytes.as_slice()).map_err(|e| { + ErrorHelper::invalid_argument() + .message(e.to_string()) + .to_adbc() + })?; + from_substrait_plan(&state, &proto) + .await + .map_err(ErrorHelper::from_datafusion)? + } + DescQuery::Sql(sql) => state + .create_logical_plan(&sql) + .await + .map_err(ErrorHelper::from_datafusion)?, + }; + register_object_store_for_plan(&self.ctx, &plan) + .await + .map_err(ErrorHelper::from_datafusion)?; + let schema = plan.schema().as_arrow().clone(); + let physical = state + .create_physical_plan(&plan) + .await + .map_err(ErrorHelper::from_datafusion)?; + let stream = physical + .execute(index as usize, state.task_ctx()) + .map_err(ErrorHelper::from_datafusion)?; + Ok(Box::new(DataFusionReader::from_stream( + self.runtime.clone(), + stream, + schema.into(), + )) as Box) + }) } } @@ -714,6 +778,65 @@ impl QueryState { } } +/// A query reconstructed from a partition descriptor. `Prepared` plans are not +/// representable here: they have no portable serialization without `datafusion-proto`. +enum DescQuery { + Substrait(Vec), + Sql(String), +} + +/// Serialize a query for embedding in a partition descriptor. Returns the kind byte +/// (0 = Substrait, 1 = SQL) and the query bytes. `Prepared` statements are rejected +/// with `NOT_IMPLEMENTED` so clients fall back to single-partition execution. +fn encode_query(query: &QueryState) -> adbc_core::error::Result<(u8, Vec)> { + match query { + QueryState::Substrait(plan) => Ok((0, plan.encode_to_vec())), + QueryState::Sql(sql) => Ok((1, sql.clone().into_bytes())), + QueryState::Prepared(_) => Err(ErrorHelper::not_implemented() + .message("partitioned execution of prepared statements") + .to_adbc()), + } +} + +/// Build a self-contained partition descriptor: +/// `[u32 LE target_partitions][u32 LE index][u8 kind][query bytes...]`. +/// +/// `target_partitions` is encoded so `read_partition` can pin it and re-plan into the +/// identical physical partitioning — see the determinism note on `read_partition`. +fn encode_descriptor(target_partitions: u32, index: u32, kind: u8, query: &[u8]) -> Vec { + let mut out = Vec::with_capacity(9 + query.len()); + out.extend_from_slice(&target_partitions.to_le_bytes()); + out.extend_from_slice(&index.to_le_bytes()); + out.push(kind); + out.extend_from_slice(query); + out +} + +/// Inverse of [`encode_descriptor`]. Returns `(target_partitions, index, query)`. +fn decode_descriptor(bytes: &[u8]) -> adbc_core::error::Result<(usize, u32, DescQuery)> { + if bytes.len() < 9 { + return Err(ErrorHelper::invalid_argument() + .message("short partition descriptor") + .to_adbc()); + } + let target_partitions = u32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; + let index = u32::from_le_bytes(bytes[4..8].try_into().unwrap()); + let query = match bytes[8] { + 0 => DescQuery::Substrait(bytes[9..].to_vec()), + 1 => DescQuery::Sql(String::from_utf8(bytes[9..].to_vec()).map_err(|e| { + ErrorHelper::invalid_argument() + .message(e.to_string()) + .to_adbc() + })?), + other => { + return Err(ErrorHelper::invalid_argument() + .format(format_args!("unknown descriptor kind {other}")) + .to_adbc()); + } + }; + Ok((target_partitions, index, query)) +} + pub struct DataFusionStatement { runtime: Arc, ctx: Arc, @@ -1031,9 +1154,33 @@ impl Statement for DataFusionStatement { } fn execute_partitions(&mut self) -> adbc_core::error::Result { - Err(ErrorHelper::not_implemented() - .message("execute_partitions") - .to_adbc()) + let query = self.query.as_ref().ok_or_else(|| { + ErrorHelper::invalid_state() + .message("no query or Substrait plan has been set") + .to_adbc() + })?; + self.runtime.block_on(async { + // Plan logically (registers object store) and build the physical plan so we + // can count its natural output partitions. + let df = query.execute(&self.ctx).await?; + let schema = df.schema().as_arrow().clone(); + let physical = df + .create_physical_plan() + .await + .map_err(ErrorHelper::from_datafusion)?; + let n = physical.output_partitioning().partition_count() as u32; + // Capture target_partitions so read_partition re-plans identically. + let target_partitions = self.ctx.copied_config().target_partitions() as u32; + let (kind, query_bytes) = encode_query(query)?; + let partitions = (0..n) + .map(|i| encode_descriptor(target_partitions, i, kind, &query_bytes)) + .collect::>(); + Ok(adbc_core::PartitionedResult { + partitions, + schema, + rows_affected: -1, + }) + }) } fn get_parameter_schema(&self) -> adbc_core::error::Result { @@ -1148,3 +1295,53 @@ impl Statement for DataFusionStatement { #[cfg(feature = "ffi")] adbc_ffi::export_driver!(AdbcDriverDatafusionInit, DataFusionDriver); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn descriptor_round_trip_sql() { + let bytes = encode_descriptor(8, 3, 1, b"SELECT 1"); + let (target_partitions, index, query) = decode_descriptor(&bytes).unwrap(); + assert_eq!(target_partitions, 8); + assert_eq!(index, 3); + match query { + DescQuery::Sql(sql) => assert_eq!(sql, "SELECT 1"), + _ => panic!("expected SQL"), + } + } + + #[test] + fn descriptor_round_trip_substrait() { + let plan_bytes = vec![1u8, 2, 3, 4, 5]; + let bytes = encode_descriptor(4, 0, 0, &plan_bytes); + let (target_partitions, index, query) = decode_descriptor(&bytes).unwrap(); + assert_eq!(target_partitions, 4); + assert_eq!(index, 0); + match query { + DescQuery::Substrait(b) => assert_eq!(b, plan_bytes), + _ => panic!("expected Substrait"), + } + } + + #[test] + fn decode_rejects_short_descriptor() { + assert!(decode_descriptor(&[0u8; 8]).is_err()); + } + + #[test] + fn decode_rejects_unknown_kind() { + let bytes = encode_descriptor(1, 0, 9, b"x"); + assert!(decode_descriptor(&bytes).is_err()); + } + + #[test] + fn encode_query_rejects_prepared() { + let plan = LogicalPlan::EmptyRelation(datafusion::logical_expr::EmptyRelation { + produce_one_row: false, + schema: Arc::new(datafusion::common::DFSchema::empty()), + }); + assert!(encode_query(&QueryState::Prepared(plan)).is_err()); + } +} diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index 8753d72..99b72db 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -422,3 +422,109 @@ async fn test_running_in_async() { assert_eq!(batch.num_rows(), 3); assert_eq!(batch.num_columns(), 2); } + +/// A single-partition keyed table; a `GROUP BY k` over it hash-repartitions to +/// `target_partitions`, giving a plan whose output partition count is driven by config. +fn keyed_table(n_keys: i32) -> datafusion::error::Result { + let schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let keys: ArrayRef = Arc::new(Int32Array::from((0..n_keys).collect::>())); + let vals: ArrayRef = Arc::new(Int32Array::from( + (0..n_keys).map(|i| i * 10).collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), vec![keys, vals])?; + MemTable::try_new(schema, vec![vec![batch]]) +} + +/// A connection whose session pins `target_partitions` to `n` and has table `t` registered. +fn connection_with_target_partitions(n: usize) -> DataFusionConnection { + let init: ContextInit = Arc::new(move |ctx, _opts| { + let config = SessionConfig::new() + .with_information_schema(true) + .with_target_partitions(n); + *ctx = SessionContext::new_with_config(config); + ctx.register_table("t", Arc::new(keyed_table(8)?))?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + let database = driver.new_database().unwrap(); + database.new_connection().unwrap() +} + +const GROUP_BY_QUERY: &str = "SELECT k, sum(v) AS s FROM t GROUP BY k"; + +/// Collect the `k` column of every row produced by reading the given descriptor. +fn read_partition_keys(connection: &DataFusionConnection, descriptor: &[u8]) -> Vec { + let reader = connection.read_partition(descriptor).unwrap(); + let mut keys = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..col.len() { + keys.push(col.value(i)); + } + } + keys +} + +#[test] +fn test_execute_partitions_returns_one_descriptor_per_output_partition() { + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + + let result = statement.execute_partitions().unwrap(); + + // Hash-repartitioned aggregate => one ADBC partition per target partition. + assert!( + result.partitions.len() > 1, + "expected N > 1 partitions, got {}", + result.partitions.len() + ); + assert_eq!(result.rows_affected, -1); + + // The union of all partitions reproduces every key exactly once. + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_read_partition_pins_target_partitions() { + // Plan with target_partitions = 4. + let mut planner = connection_with_target_partitions(4); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + assert!(result.partitions.len() > 1); + + // Execute on a connection whose default target_partitions differs (1). Pinning the + // descriptor's value is what keeps partition `i` meaningful and in range here. + let executor = connection_with_target_partitions(1); + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&executor, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_execute_partitions_prepared_not_implemented() { + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement.prepare().unwrap(); + + let err = statement.execute_partitions().unwrap_err(); + assert_eq!(err.status, adbc_core::error::Status::NotImplemented); +} From 12771eedd948ae9879c983c06940d880ad1b78c3 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 09:39:51 -0400 Subject: [PATCH 02/15] feat: serialize physical plans into partition descriptors execute_partitions serializes the already-built physical plan into each partition descriptor (datafusion-proto); read_partition deserializes it and executes one partition. The descriptor carries the plan, not the source query, so a reader runs the exact plan the planner built without re-planning. datafusion-proto serializes built-in nodes on its own. A provider that emits custom ExecutionPlan nodes registers a codec via DataFusionDriver::with_codec so those nodes round-trip; with no codec the default codec covers built-in nodes only, and a plan containing a node the codec cannot encode fails execute_partitions. Prepared statements partition like any other query, since the physical plan is serialized regardless of source. Descriptor layout: [version(1)][index(4)][serialized plan]. read_partition caches the deserialized plan per connection, keyed by plan bytes and bounded, so a connection reading several partitions of one plan decodes it once. Co-Authored-By: Claude Opus 4.8 --- Cargo.lock | 39 ++++ Cargo.toml | 1 + README.md | 39 ++++ src/lib.rs | 277 ++++++++++++----------- tests/test_datafusion.rs | 22 +- tests/test_proto_partitions.rs | 390 +++++++++++++++++++++++++++++++++ 6 files changed, 634 insertions(+), 134 deletions(-) create mode 100644 tests/test_proto_partitions.rs diff --git a/Cargo.lock b/Cargo.lock index 597be8e..d88ff46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,6 +16,7 @@ dependencies = [ "aws-config", "aws-credential-types", "datafusion", + "datafusion-proto", "datafusion-substrait", "driverbase", "futures", @@ -1923,6 +1924,44 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +dependencies = [ + "arrow", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "object_store", + "prost", + "rand 0.9.4", +] + +[[package]] +name = "datafusion-proto-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "53.1.0" diff --git a/Cargo.toml b/Cargo.toml index cb37905..9b9bd45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ async-trait = "0.1" aws-config = "1.8" aws-credential-types = "1.2" datafusion = { version = "53", default-features = false, features = ["avro", "compression", "crypto_expressions", "datetime_expressions", "nested_expressions", "parquet", "recursive_protection", "regex_expressions", "sql", "unicode_expressions"] } +datafusion-proto = { version = "53", default-features = false } datafusion-substrait = { version = "53", default-features = false } driverbase = { git = "https://github.com/adbc-drivers/driverbase-rs.git" } futures = "0.3.32" diff --git a/README.md b/README.md index 56b5fee..6db1035 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,45 @@ various platforms from the [Columnar](https://columnar.tech) CDN in the future. See [Building](#building) to build the drivers yourself. +## Partitioned execution + +ADBC partitioned execution splits a result set into independent pieces that can be read +in parallel. One caller runs `execute_partitions`, which plans the query and returns one +opaque **descriptor** per output partition; each descriptor is then handed to +`read_partition`, which produces that partition's rows. The two halves need not run in the +same place: descriptors can be read back on the same connection, on other threads, or +shipped to other processes or machines (for example, a distributed engine fanning each +descriptor out to a worker). A descriptor is self-contained — the only thing that travels +between the planner and the reader. + +`execute_partitions` serializes the already-built **physical plan** into each descriptor +([`datafusion-proto`](https://docs.rs/datafusion-proto)), and `read_partition` +**deserializes** it and executes one partition. Because the reader runs the exact plan the +planner built, every reader agrees on the partition layout and the reader needs only the +plan bytes (plus any registered providers and codec) — not the original query text or the +planner's session configuration. + +### Custom nodes + +`datafusion-proto` serializes built-in nodes with no configuration. A provider that emits +custom `ExecutionPlan` nodes must register a `PhysicalExtensionCodec` alongside the +`ContextInit` hook so those nodes round-trip: + +```rust +let driver = DataFusionDriver::new_with_context_init(handle, context_init).with_codec(codec); +``` + +The custom node must be reconstructable from its encoded bytes plus the hook-built session. +With no codec the driver uses the default codec, which covers built-in nodes only; a plan +containing a custom node the codec cannot encode **fails** `execute_partitions` rather than +falling back to a re-plan path. + +`read_partition` caches a deserialized plan per connection, keyed by the descriptor's +plan bytes, so a connection that reads several partitions of the same plan pays the +decode cost once per distinct plan rather than once per partition. + +See `tests/test_proto_partitions.rs` for a worked custom provider + codec example. + ## Building See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/src/lib.rs b/src/lib.rs index 5cb30ce..0757144 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ use datafusion::dataframe::DataFrameWriteOptions; use datafusion::datasource::MemTable; use datafusion::logical_expr::LogicalPlan; use datafusion::logical_expr::dml::InsertOp; -use datafusion::physical_plan::ExecutionPlanProperties; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::*; use datafusion_substrait::logical_plan::consumer::from_substrait_plan; use datafusion_substrait::substrait::proto::Plan; @@ -76,6 +76,17 @@ pub type ContextInit = Arc< dyn Fn(&mut SessionContext, &mut DatabaseOpts) -> datafusion::error::Result<()> + Send + Sync, >; +/// Optional extension codec for serializing custom `ExecutionPlan` nodes. +/// +/// Partitioned execution serializes the physical plan into each partition descriptor +/// (`datafusion-proto`) so `read_partition` can execute it without the original query. +/// `datafusion-proto` serializes built-in DataFusion nodes on its own; a provider that +/// emits custom `ExecutionPlan` nodes registers a codec here so those nodes can be encoded +/// and reconstructed from their bytes plus the hook-built session. With no codec the driver +/// uses the default codec, which handles built-in nodes only — a plan containing a custom +/// node the codec cannot encode fails `execute_partitions`. +pub type PhysicalCodec = Arc; + impl ErrorHelper { fn from_datafusion(err: datafusion::error::DataFusionError) -> DriverError { match err { @@ -260,9 +271,7 @@ impl DataFusionReader { }) } - /// Construct a reader directly from a single-partition stream, bypassing the - /// `DataFrame` path. Used by `read_partition`, which executes one partition of an - /// already-built physical plan. + /// Construct a reader directly from a single-partition stream. pub fn from_stream( runtime: Arc, stream: datafusion::execution::SendableRecordBatchStream, @@ -294,6 +303,7 @@ impl RecordBatchReader for DataFusionReader { pub struct DataFusionDriver { handle: Option, context_init: ContextInit, + codec: Option, } impl DataFusionDriver { @@ -309,9 +319,22 @@ impl DataFusionDriver { Self { handle, context_init, + codec: None, } } + /// Register an extension codec for serializing custom `ExecutionPlan` nodes in partition + /// descriptors (see [`PhysicalCodec`]). Chains onto any constructor; only needed when a + /// provider emits custom nodes: + /// + /// ```ignore + /// let driver = DataFusionDriver::new_with_context_init(handle, init).with_codec(codec); + /// ``` + pub fn with_codec(mut self, codec: PhysicalCodec) -> Self { + self.codec = Some(codec); + self + } + fn new_database_with_database_opts( &self, database_opts: &mut DatabaseOpts, @@ -330,6 +353,7 @@ impl DataFusionDriver { Ok(DataFusionDatabase { handle: self.handle.clone(), ctx: Arc::new(ctx), + codec: self.codec.clone(), }) } } @@ -369,6 +393,7 @@ impl Driver for DataFusionDriver { pub struct DataFusionDatabase { handle: Option, ctx: Arc, + codec: Option, } impl Optionable for DataFusionDatabase { @@ -426,6 +451,8 @@ impl Database for DataFusionDatabase { Ok(DataFusionConnection { runtime: Arc::new(runtime), ctx: self.ctx.clone(), + codec: self.codec.clone(), + plan_cache: Arc::new(PlanCache::default()), }) } @@ -448,6 +475,8 @@ impl Database for DataFusionDatabase { let mut connection = DataFusionConnection { runtime: Arc::new(runtime), ctx: self.ctx.clone(), + codec: self.codec.clone(), + plan_cache: Arc::new(PlanCache::default()), }; for (key, value) in opts { @@ -458,9 +487,22 @@ impl Database for DataFusionDatabase { } } +/// Cache of deserialized physical plans, keyed by descriptor plan bytes. +/// +/// `read_partition` deserializes a given plan once and reuses it across every partition +/// index. When a connection is shared across an executor's task slots (as connectors are +/// expected to do), this collapses N per-task deserializes into one per distinct plan. +type PlanCache = parking_lot::Mutex, Arc>>; + +/// Bound on distinct cached plans, so a long-lived connection running many different +/// queries does not accumulate plans without limit. On overflow the cache is cleared. +const PLAN_CACHE_CAP: usize = 16; + pub struct DataFusionConnection { runtime: Arc, ctx: Arc, + codec: Option, + plan_cache: Arc, } impl Optionable for DataFusionConnection { @@ -599,6 +641,7 @@ impl Connection for DataFusionConnection { Ok(DataFusionStatement { runtime: self.runtime.clone(), ctx: self.ctx.clone(), + codec: self.codec.clone(), query: None, bound: None, ingest: BulkIngestState::new(), @@ -696,55 +739,51 @@ impl Connection for DataFusionConnection { &self, partition: impl AsRef<[u8]>, ) -> Result> { - let (target_partitions, index, query) = decode_descriptor(partition.as_ref())?; + let (index, plan_bytes) = decode_descriptor(partition.as_ref())?; self.runtime.block_on(async { - // Pin target_partitions so the physical plan matches the one execute_partitions - // counted; DataFusion planning is deterministic only when this is held fixed. - let state = - datafusion::execution::session_state::SessionStateBuilder::new_from_existing( - self.ctx.state(), - ) - .with_config( - self.ctx - .copied_config() - .with_target_partitions(target_partitions) - // The carried-over catalog list already holds the registered providers; - // recreating the default catalog here would wipe them. - .with_create_default_catalog_and_schema(false), - ) - .build(); - - let plan = match query { - DescQuery::Substrait(bytes) => { - let proto = Plan::decode(bytes.as_slice()).map_err(|e| { - ErrorHelper::invalid_argument() - .message(e.to_string()) - .to_adbc() - })?; - from_substrait_plan(&state, &proto) - .await - .map_err(ErrorHelper::from_datafusion)? + // Deserialize the physical plan the driver already built and execute one + // partition — no logical/physical re-planning. + let task_ctx = self.ctx.task_ctx(); + let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; + let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = + self.codec.as_deref().unwrap_or(&default_codec); + // Deserialize once per distinct plan and reuse across partition indices; a + // shared connection then pays the decode cost once per executor, not per task. + let physical = { + let mut cache = self.plan_cache.lock(); + if let Some(plan) = cache.get(plan_bytes.as_slice()) { + plan.clone() + } else { + let plan = + datafusion_proto::bytes::physical_plan_from_bytes_with_extension_codec( + &plan_bytes, + task_ctx.as_ref(), + codec, + ) + .map_err(ErrorHelper::from_datafusion)?; + if cache.len() >= PLAN_CACHE_CAP { + cache.clear(); + } + cache.insert(plan_bytes.clone(), plan.clone()); + plan } - DescQuery::Sql(sql) => state - .create_logical_plan(&sql) - .await - .map_err(ErrorHelper::from_datafusion)?, }; - register_object_store_for_plan(&self.ctx, &plan) - .await - .map_err(ErrorHelper::from_datafusion)?; - let schema = plan.schema().as_arrow().clone(); - let physical = state - .create_physical_plan(&plan) - .await - .map_err(ErrorHelper::from_datafusion)?; + let n = physical.output_partitioning().partition_count(); + if index as usize >= n { + return Err(ErrorHelper::invalid_argument() + .format(format_args!( + "partition index {index} out of range (plan has {n} partitions)" + )) + .to_adbc()); + } + let schema = physical.schema(); let stream = physical - .execute(index as usize, state.task_ctx()) + .execute(index as usize, task_ctx) .map_err(ErrorHelper::from_datafusion)?; Ok(Box::new(DataFusionReader::from_stream( self.runtime.clone(), stream, - schema.into(), + schema, )) as Box) }) } @@ -778,68 +817,57 @@ impl QueryState { } } -/// A query reconstructed from a partition descriptor. `Prepared` plans are not -/// representable here: they have no portable serialization without `datafusion-proto`. -enum DescQuery { - Substrait(Vec), - Sql(String), -} +/// Current partition-descriptor format version. Descriptors are opaque and version-local: +/// the same driver build both produces and consumes them, so older layouts are not decoded. +/// The version byte lets the format evolve (e.g. a future descriptor-shrink layout) without +/// silently misreading an old payload. +const DESCRIPTOR_VERSION: u8 = 1; -/// Serialize a query for embedding in a partition descriptor. Returns the kind byte -/// (0 = Substrait, 1 = SQL) and the query bytes. `Prepared` statements are rejected -/// with `NOT_IMPLEMENTED` so clients fall back to single-partition execution. -fn encode_query(query: &QueryState) -> adbc_core::error::Result<(u8, Vec)> { - match query { - QueryState::Substrait(plan) => Ok((0, plan.encode_to_vec())), - QueryState::Sql(sql) => Ok((1, sql.clone().into_bytes())), - QueryState::Prepared(_) => Err(ErrorHelper::not_implemented() - .message("partitioned execution of prepared statements") - .to_adbc()), - } -} +/// Fixed descriptor header length: `version(1) + index(4)`. +const DESCRIPTOR_HEADER_LEN: usize = 5; + +/// Size above which a serialized plan payload is logged as a warning. The full plan is +/// copied into every one of the N partition descriptors, so a large plan is paid N times +/// over until the descriptor-shrink work ships the plan once. Operator-only plans are a +/// few KB and never trip this; the threshold mainly catches plans that inline data (e.g. a +/// custom node embedding its rows) or very large queries. +const PROTO_DESCRIPTOR_WARN_BYTES: usize = 8 << 20; // 8 MiB /// Build a self-contained partition descriptor: -/// `[u32 LE target_partitions][u32 LE index][u8 kind][query bytes...]`. +/// `[u8 version][u32 LE index][serialized physical plan...]`. /// -/// `target_partitions` is encoded so `read_partition` can pin it and re-plan into the -/// identical physical partitioning — see the determinism note on `read_partition`. -fn encode_descriptor(target_partitions: u32, index: u32, kind: u8, query: &[u8]) -> Vec { - let mut out = Vec::with_capacity(9 + query.len()); - out.extend_from_slice(&target_partitions.to_le_bytes()); +/// The payload is the full `datafusion-proto`-serialized physical plan; `read_partition` +/// deserializes it and executes the given partition index. +fn encode_descriptor(index: u32, plan_bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(DESCRIPTOR_HEADER_LEN + plan_bytes.len()); + out.push(DESCRIPTOR_VERSION); out.extend_from_slice(&index.to_le_bytes()); - out.push(kind); - out.extend_from_slice(query); + out.extend_from_slice(plan_bytes); out } -/// Inverse of [`encode_descriptor`]. Returns `(target_partitions, index, query)`. -fn decode_descriptor(bytes: &[u8]) -> adbc_core::error::Result<(usize, u32, DescQuery)> { - if bytes.len() < 9 { +/// Inverse of [`encode_descriptor`]. Returns `(index, plan_bytes)`. +fn decode_descriptor(bytes: &[u8]) -> adbc_core::error::Result<(u32, Vec)> { + if bytes.len() < DESCRIPTOR_HEADER_LEN { return Err(ErrorHelper::invalid_argument() .message("short partition descriptor") .to_adbc()); } - let target_partitions = u32::from_le_bytes(bytes[0..4].try_into().unwrap()) as usize; - let index = u32::from_le_bytes(bytes[4..8].try_into().unwrap()); - let query = match bytes[8] { - 0 => DescQuery::Substrait(bytes[9..].to_vec()), - 1 => DescQuery::Sql(String::from_utf8(bytes[9..].to_vec()).map_err(|e| { - ErrorHelper::invalid_argument() - .message(e.to_string()) - .to_adbc() - })?), - other => { - return Err(ErrorHelper::invalid_argument() - .format(format_args!("unknown descriptor kind {other}")) - .to_adbc()); - } - }; - Ok((target_partitions, index, query)) + let version = bytes[0]; + if version != DESCRIPTOR_VERSION { + return Err(ErrorHelper::invalid_argument() + .format(format_args!("unsupported descriptor version {version}")) + .to_adbc()); + } + let index = u32::from_le_bytes(bytes[1..5].try_into().unwrap()); + let plan_bytes = bytes[DESCRIPTOR_HEADER_LEN..].to_vec(); + Ok((index, plan_bytes)) } pub struct DataFusionStatement { runtime: Arc, ctx: Arc, + codec: Option, query: Option, bound: Option>, ingest: BulkIngestState, @@ -1169,11 +1197,28 @@ impl Statement for DataFusionStatement { .await .map_err(ErrorHelper::from_datafusion)?; let n = physical.output_partitioning().partition_count() as u32; - // Capture target_partitions so read_partition re-plans identically. - let target_partitions = self.ctx.copied_config().target_partitions() as u32; - let (kind, query_bytes) = encode_query(query)?; + + // Serialize the already-built physical plan so read_partition deserializes it + // instead of re-planning. The default codec covers built-in nodes; a registered + // codec additionally covers a provider's custom nodes. A node neither can encode + // fails the query (no re-plan fallback). + let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; + let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = + self.codec.as_deref().unwrap_or(&default_codec); + let plan_bytes = datafusion_proto::bytes::physical_plan_to_bytes_with_extension_codec( + physical, codec, + ) + .map_err(ErrorHelper::from_datafusion)?; + if plan_bytes.len() > PROTO_DESCRIPTOR_WARN_BYTES { + // Until the descriptor-shrink work lands, the full plan rides each of the N + // descriptors. Warn when that gets large. + log::warn!( + "partition descriptor carries a {}-byte plan across {n} partitions", + plan_bytes.len(), + ); + } let partitions = (0..n) - .map(|i| encode_descriptor(target_partitions, i, kind, &query_bytes)) + .map(|i| encode_descriptor(i, &plan_bytes)) .collect::>(); Ok(adbc_core::PartitionedResult { partitions, @@ -1301,47 +1346,23 @@ mod tests { use super::*; #[test] - fn descriptor_round_trip_sql() { - let bytes = encode_descriptor(8, 3, 1, b"SELECT 1"); - let (target_partitions, index, query) = decode_descriptor(&bytes).unwrap(); - assert_eq!(target_partitions, 8); - assert_eq!(index, 3); - match query { - DescQuery::Sql(sql) => assert_eq!(sql, "SELECT 1"), - _ => panic!("expected SQL"), - } - } - - #[test] - fn descriptor_round_trip_substrait() { - let plan_bytes = vec![1u8, 2, 3, 4, 5]; - let bytes = encode_descriptor(4, 0, 0, &plan_bytes); - let (target_partitions, index, query) = decode_descriptor(&bytes).unwrap(); - assert_eq!(target_partitions, 4); - assert_eq!(index, 0); - match query { - DescQuery::Substrait(b) => assert_eq!(b, plan_bytes), - _ => panic!("expected Substrait"), - } + fn descriptor_round_trip() { + let plan_bytes = vec![9u8, 8, 7, 6]; + let bytes = encode_descriptor(2, &plan_bytes); + let (index, payload) = decode_descriptor(&bytes).unwrap(); + assert_eq!(index, 2); + assert_eq!(payload, plan_bytes); } #[test] fn decode_rejects_short_descriptor() { - assert!(decode_descriptor(&[0u8; 8]).is_err()); + assert!(decode_descriptor(&[DESCRIPTOR_VERSION; DESCRIPTOR_HEADER_LEN - 1]).is_err()); } #[test] - fn decode_rejects_unknown_kind() { - let bytes = encode_descriptor(1, 0, 9, b"x"); + fn decode_rejects_unknown_version() { + let mut bytes = encode_descriptor(0, b"x"); + bytes[0] = DESCRIPTOR_VERSION + 1; assert!(decode_descriptor(&bytes).is_err()); } - - #[test] - fn encode_query_rejects_prepared() { - let plan = LogicalPlan::EmptyRelation(datafusion::logical_expr::EmptyRelation { - produce_one_row: false, - schema: Arc::new(datafusion::common::DFSchema::empty()), - }); - assert!(encode_query(&QueryState::Prepared(plan)).is_err()); - } } diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index 99b72db..7191abe 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -499,7 +499,7 @@ fn test_execute_partitions_returns_one_descriptor_per_output_partition() { } #[test] -fn test_read_partition_pins_target_partitions() { +fn test_read_partition_executes_serialized_plan() { // Plan with target_partitions = 4. let mut planner = connection_with_target_partitions(4); let mut statement = planner.new_statement().unwrap(); @@ -507,8 +507,9 @@ fn test_read_partition_pins_target_partitions() { let result = statement.execute_partitions().unwrap(); assert!(result.partitions.len() > 1); - // Execute on a connection whose default target_partitions differs (1). Pinning the - // descriptor's value is what keeps partition `i` meaningful and in range here. + // Execute on a connection whose default target_partitions differs (1). The descriptor + // carries the already-built physical plan, so its partitioning is fixed regardless of the + // executor's config — partition `i` stays meaningful and in range. let executor = connection_with_target_partitions(1); let mut keys = Vec::new(); for descriptor in &result.partitions { @@ -519,12 +520,21 @@ fn test_read_partition_pins_target_partitions() { } #[test] -fn test_execute_partitions_prepared_not_implemented() { +fn test_execute_partitions_prepared() { + // The proto path serializes the built physical plan regardless of source, so a prepared + // statement partitions like any other query. let mut connection = connection_with_target_partitions(4); let mut statement = connection.new_statement().unwrap(); statement.set_sql_query(GROUP_BY_QUERY).unwrap(); statement.prepare().unwrap(); - let err = statement.execute_partitions().unwrap_err(); - assert_eq!(err.status, adbc_core::error::Status::NotImplemented); + let result = statement.execute_partitions().unwrap(); + assert!(result.partitions.len() > 1); + + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); } diff --git a/tests/test_proto_partitions.rs b/tests/test_proto_partitions.rs new file mode 100644 index 0000000..88cb82b --- /dev/null +++ b/tests/test_proto_partitions.rs @@ -0,0 +1,390 @@ +// Copyright (c) 2025-2026 ADBC Drivers Contributors +// +// This file has been modified from its original version, which is +// under the Apache License: +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Proto path tests: a custom `TableProvider` emits a custom `ExecutionPlan`, and a +//! `PhysicalExtensionCodec` round-trips it. `execute_partitions` serializes the physical +//! plan; a *fresh* connection's `read_partition` reproduces the rows by deserializing the +//! plan — the descriptor carries only the serialized plan, so re-planning is structurally +//! impossible and a passing test proves the deserialize path. Also covers the hard error +//! when a custom node cannot be encoded (no re-plan fallback). + +use std::any::Any; +use std::fmt; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use adbc_core::{Connection, Database, Driver, Statement}; +use adbc_driver_datafusion::{ContextInit, DataFusionConnection, DataFusionDriver, PhysicalCodec}; +use arrow_array::{ArrayRef, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use async_trait::async_trait; +use datafusion::catalog::{Session, TableProvider}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::{Expr, TableType}; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::memory::MemoryStream; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + SendableRecordBatchStream, +}; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; + +/// Single `Int32` column named `v`. +fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)])) +} + +/// A custom scan node: holds the full data and a partition count; partition `i` emits a +/// round-robin slice (`data[i], data[i + n], ...`). Carries everything needed to rebuild +/// itself from bytes, so it survives proto round-trip via [`CustomCodec`]. +#[derive(Debug)] +struct CustomExec { + data: Vec, + partitions: usize, + schema: SchemaRef, + props: Arc, +} + +impl CustomExec { + fn new(data: Vec, partitions: usize) -> Self { + let schema = schema(); + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Self { + data, + partitions, + schema, + props, + } + } +} + +impl DisplayAs for CustomExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "CustomExec(partitions={})", self.partitions) + } +} + +impl ExecutionPlan for CustomExec { + fn name(&self) -> &str { + "CustomExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> DFResult> { + Ok(self) + } + + fn execute( + &self, + partition: usize, + _context: Arc, + ) -> DFResult { + let mut vals = Vec::new(); + let mut i = partition; + while i < self.data.len() { + vals.push(self.data[i]); + i += self.partitions; + } + let col: ArrayRef = Arc::new(Int32Array::from(vals)); + let batch = RecordBatch::try_new(self.schema.clone(), vec![col])?; + Ok(Box::pin(MemoryStream::try_new( + vec![batch], + self.schema.clone(), + None, + )?)) + } +} + +/// A table provider whose `scan` returns a [`CustomExec`]. +#[derive(Debug)] +struct CustomProvider { + data: Vec, + partitions: usize, +} + +#[async_trait] +impl TableProvider for CustomProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + schema() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> DFResult> { + Ok(Arc::new(CustomExec::new( + self.data.clone(), + self.partitions, + ))) + } +} + +/// Codec for [`CustomExec`]: encodes `partitions` then the data as little-endian i32s. +#[derive(Debug)] +struct CustomCodec; + +impl PhysicalExtensionCodec for CustomCodec { + fn try_decode( + &self, + buf: &[u8], + _inputs: &[Arc], + _ctx: &TaskContext, + ) -> DFResult> { + if buf.len() < 8 { + return Err(DataFusionError::Internal("short CustomExec buffer".into())); + } + let partitions = u64::from_le_bytes(buf[0..8].try_into().unwrap()) as usize; + let data: Vec = buf[8..] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + Ok(Arc::new(CustomExec::new(data, partitions))) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> DFResult<()> { + let exec = node + .as_any() + .downcast_ref::() + .ok_or_else(|| DataFusionError::Internal("not a CustomExec".into()))?; + buf.extend_from_slice(&(exec.partitions as u64).to_le_bytes()); + for v in &exec.data { + buf.extend_from_slice(&v.to_le_bytes()); + } + Ok(()) + } +} + +/// A codec that refuses to encode, to exercise the hard error when a custom node cannot +/// be serialized (no re-plan fallback). +#[derive(Debug)] +struct FailingCodec; + +impl PhysicalExtensionCodec for FailingCodec { + fn try_decode( + &self, + _buf: &[u8], + _inputs: &[Arc], + _ctx: &TaskContext, + ) -> DFResult> { + Err(DataFusionError::Internal("decode unsupported".into())) + } + + fn try_encode(&self, _node: Arc, _buf: &mut Vec) -> DFResult<()> { + Err(DataFusionError::Internal("encode unsupported".into())) + } +} + +/// Delegates to [`CustomCodec`] but counts `try_decode` calls, to prove the per-connection +/// plan cache deserializes a given plan only once. +#[derive(Debug)] +struct CountingCodec { + decodes: Arc, +} + +impl PhysicalExtensionCodec for CountingCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + ) -> DFResult> { + self.decodes.fetch_add(1, Ordering::SeqCst); + CustomCodec.try_decode(buf, inputs, ctx) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> DFResult<()> { + CustomCodec.try_encode(node, buf) + } +} + +const VALUES: i32 = 8; +const N_PARTITIONS: usize = 3; +const QUERY: &str = "SELECT v FROM ct"; + +/// Build a connection whose `ct` table is the custom provider, with the given codec (if any). +fn custom_connection(codec: Option) -> DataFusionConnection { + let init: ContextInit = Arc::new(|ctx, _opts| { + ctx.register_table( + "ct", + Arc::new(CustomProvider { + data: (0..VALUES).collect(), + partitions: N_PARTITIONS, + }), + )?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + if let Some(codec) = codec { + driver = driver.with_codec(codec); + } + let database = driver.new_database().unwrap(); + database.new_connection().unwrap() +} + +/// Read every `v` value produced by the descriptor. +fn read_values(connection: &DataFusionConnection, descriptor: &[u8]) -> Vec { + let reader = connection.read_partition(descriptor).unwrap(); + let mut out = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..col.len() { + out.push(col.value(i)); + } + } + out +} + +#[test] +fn proto_round_trips_custom_plan_without_replanning() { + let mut planner = custom_connection(Some(Arc::new(CustomCodec))); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + N_PARTITIONS, + "one descriptor per custom output partition" + ); + + // A FRESH connection — also codec-equipped, as a separate executor would be — reproduces + // the rows purely from the serialized plan (no SQL string rides the descriptor). + let executor = custom_connection(Some(Arc::new(CustomCodec))); + let mut values = Vec::new(); + for descriptor in &result.partitions { + values.extend(read_values(&executor, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); +} + +#[test] +fn custom_node_without_codec_errors() { + // No codec: the default codec cannot encode the custom node, so execute_partitions fails + // rather than silently falling back to a re-plan path. + let mut connection = custom_connection(None); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + assert!( + statement.execute_partitions().is_err(), + "custom node + no codec must error, not fall back" + ); +} + +#[test] +fn codec_encode_failure_errors() { + // The codec can't encode this plan; execute_partitions errors the whole query (no + // re-plan fallback). + let mut connection = custom_connection(Some(Arc::new(FailingCodec))); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + assert!( + statement.execute_partitions().is_err(), + "failed proto encode must error, not fall back" + ); +} + +#[test] +fn read_partition_rejects_out_of_range_index() { + let mut planner = custom_connection(Some(Arc::new(CustomCodec))); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + + // Corrupt the index (bytes 1..5) to an out-of-range partition. + let mut bad = result.partitions[0].clone(); + bad[1..5].copy_from_slice(&u32::to_le_bytes(99)); + + let executor = custom_connection(Some(Arc::new(CustomCodec))); + let err = match executor.read_partition(&bad) { + Ok(_) => panic!("expected out-of-range index to be rejected"), + Err(e) => e, + }; + assert_eq!(err.status, adbc_core::error::Status::InvalidArguments); +} + +#[test] +fn plan_cache_deserializes_once_across_partitions() { + let decodes = Arc::new(AtomicUsize::new(0)); + let codec: PhysicalCodec = Arc::new(CountingCodec { + decodes: decodes.clone(), + }); + let mut connection = custom_connection(Some(codec)); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), N_PARTITIONS); + + // Read every partition on the SAME connection: the plan is identical across them, so + // the cache should deserialize it once and reuse it for the rest. + let mut values = Vec::new(); + for descriptor in &result.partitions { + values.extend(read_values(&connection, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); + + assert_eq!( + decodes.load(Ordering::SeqCst), + 1, + "plan should be deserialized once and cached across partition indices" + ); +} From 9b6a30ab7045562b8cdfbb60912b9dd7ba20e16a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 12:43:15 -0400 Subject: [PATCH 03/15] refactor: make DataFusionReader::from_stream pub(crate) Only read_partition constructs a reader from a stream; narrowing the visibility keeps it out of the crate's public API surface. Co-Authored-By: Claude Opus 4.8 --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0757144..bf5dfd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -272,7 +272,7 @@ impl DataFusionReader { } /// Construct a reader directly from a single-partition stream. - pub fn from_stream( + pub(crate) fn from_stream( runtime: Arc, stream: datafusion::execution::SendableRecordBatchStream, schema: SchemaRef, From e1fd3c0cc07f2f54232789e8c9f3d08c6f08e5e2 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 12:44:12 -0400 Subject: [PATCH 04/15] test: assert exact partition count in execute_partitions test The comment claimed one descriptor per target partition but the assertion only checked len() > 1. Hash-repartitioning to target_partitions=4 yields a deterministic 4 output partitions, so assert the exact count. Co-Authored-By: Claude Opus 4.8 --- tests/test_datafusion.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index 7191abe..d193a8a 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -481,10 +481,11 @@ fn test_execute_partitions_returns_one_descriptor_per_output_partition() { let result = statement.execute_partitions().unwrap(); - // Hash-repartitioned aggregate => one ADBC partition per target partition. - assert!( - result.partitions.len() > 1, - "expected N > 1 partitions, got {}", + // Hash-repartitioned aggregate => one ADBC partition per target partition (4). + assert_eq!( + result.partitions.len(), + 4, + "expected one descriptor per target partition, got {}", result.partitions.len() ); assert_eq!(result.rows_affected, -1); From bd8e0710aade77b5f7574c54095b7eb4830f3e85 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 14:29:04 -0400 Subject: [PATCH 05/15] docs: drop references to abandoned descriptor-shrink follow-up The descriptor-shrink idea (ship the plan once, descriptors reference it) cannot stay within the ADBC self-contained-partition contract, so it is not planned. Remove the comments that referenced it; keep the substantive notes on format versioning and the N-copy plan cost. Co-Authored-By: Claude Opus 4.8 --- src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index bf5dfd4..fc5408b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -819,8 +819,7 @@ impl QueryState { /// Current partition-descriptor format version. Descriptors are opaque and version-local: /// the same driver build both produces and consumes them, so older layouts are not decoded. -/// The version byte lets the format evolve (e.g. a future descriptor-shrink layout) without -/// silently misreading an old payload. +/// The version byte lets the format evolve without silently misreading an old payload. const DESCRIPTOR_VERSION: u8 = 1; /// Fixed descriptor header length: `version(1) + index(4)`. @@ -828,9 +827,8 @@ const DESCRIPTOR_HEADER_LEN: usize = 5; /// Size above which a serialized plan payload is logged as a warning. The full plan is /// copied into every one of the N partition descriptors, so a large plan is paid N times -/// over until the descriptor-shrink work ships the plan once. Operator-only plans are a -/// few KB and never trip this; the threshold mainly catches plans that inline data (e.g. a -/// custom node embedding its rows) or very large queries. +/// over. Operator-only plans are a few KB and never trip this; the threshold mainly catches +/// plans that inline data (e.g. a custom node embedding its rows) or very large queries. const PROTO_DESCRIPTOR_WARN_BYTES: usize = 8 << 20; // 8 MiB /// Build a self-contained partition descriptor: @@ -1210,8 +1208,7 @@ impl Statement for DataFusionStatement { ) .map_err(ErrorHelper::from_datafusion)?; if plan_bytes.len() > PROTO_DESCRIPTOR_WARN_BYTES { - // Until the descriptor-shrink work lands, the full plan rides each of the N - // descriptors. Warn when that gets large. + // The full plan is copied into each of the N descriptors; warn when large. log::warn!( "partition descriptor carries a {}-byte plan across {n} partitions", plan_bytes.len(), From 600f8da0a6bd5001c90235d80107f19f3ac62580 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 14:53:20 -0400 Subject: [PATCH 06/15] feat: add partition_mode option for shuffling plans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Producing one output partition of a plan that shuffles (hash repartition for joins/grouped aggregates, or a sort) requires reading all input, so executing partitions independently re-runs the pre-shuffle pipeline once per partition — usually slower than a single execution. Add the statement option datafusion.partition_mode: - multi (default): natural partitions, warn when the plan shuffles - auto: coalesce to one partition when the plan shuffles, else natural - single: always coalesce to one partition Collapsing wraps the plan in CoalescePartitionsExec so the single reader runs the whole pipeline once. Shuffle detection walks the physical plan for Repartition/Sort/SortPreservingMerge nodes. Co-Authored-By: Claude Opus 4.8 --- README.md | 23 ++++++++ src/lib.rs | 101 +++++++++++++++++++++++++++++++++++ tests/test_datafusion.rs | 110 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 234 insertions(+) diff --git a/README.md b/README.md index 6db1035..ca4ca92 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,29 @@ planner built, every reader agrees on the partition layout and the reader needs plan bytes (plus any registered providers and codec) — not the original query text or the planner's session configuration. +### Shuffling plans + +Partitioned execution shines when each output partition reads independent input — a +partitioned scan, or filters/projections over one. When the plan **shuffles** (a hash +repartition for a join or grouped aggregate, or a sort), producing one output partition +requires reading *all* input, so executing partitions independently re-runs the whole +pre-shuffle pipeline once per partition — usually slower than a single execution. + +The statement option `datafusion.partition_mode` controls this: + +| Value | Behavior | +| --- | --- | +| `multi` (default) | One descriptor per natural output partition; logs a warning if the plan shuffles. | +| `auto` | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | +| `single` | Always collapse to a single partition. | + +```rust +statement.set_option( + OptionStatement::Other("datafusion.partition_mode".into()), + OptionValue::String("auto".into()), +)?; +``` + ### Custom nodes `datafusion-proto` serializes built-in nodes with no configuration. A provider that emits diff --git a/src/lib.rs b/src/lib.rs index fc5408b..4b2c4d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ use datafusion::dataframe::DataFrameWriteOptions; use datafusion::datasource::MemTable; use datafusion::logical_expr::LogicalPlan; use datafusion::logical_expr::dml::InsertOp; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::*; use datafusion_substrait::logical_plan::consumer::from_substrait_plan; @@ -642,6 +643,7 @@ impl Connection for DataFusionConnection { runtime: self.runtime.clone(), ctx: self.ctx.clone(), codec: self.codec.clone(), + partition_mode: PartitionMode::default(), query: None, bound: None, ingest: BulkIngestState::new(), @@ -862,10 +864,76 @@ fn decode_descriptor(bytes: &[u8]) -> adbc_core::error::Result<(u32, Vec)> { Ok((index, plan_bytes)) } +/// Statement option selecting the `execute_partitions` strategy. String value, one of +/// `auto`, `single`, or `multi` (see [`PartitionMode`]). +const OPTION_PARTITION_MODE: &str = "datafusion.partition_mode"; + +/// How `execute_partitions` splits a query into descriptors. +/// +/// Producing one output partition of a plan that *shuffles* (a hash repartition for joins +/// and grouped aggregates, or a sort) requires reading every input partition. Executing +/// each such partition independently — the point of partitioned execution — therefore +/// re-runs the whole pre-shuffle pipeline once per partition and is usually slower than a +/// single execution. This mode controls how that case is handled. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum PartitionMode { + /// One descriptor per natural output partition; warn when the plan shuffles. Default — + /// preserves the partition count the caller's plan produced. + #[default] + Multi, + /// Collapse to a single coalesced partition when the plan shuffles, otherwise one + /// descriptor per natural output partition. + Auto, + /// Always collapse to a single coalesced partition. + Single, +} + +impl PartitionMode { + fn from_option(value: &str) -> adbc_core::error::Result { + match value { + "multi" => Ok(Self::Multi), + "auto" => Ok(Self::Auto), + "single" => Ok(Self::Single), + other => Err(ErrorHelper::invalid_argument() + .format(format_args!( + "unknown {OPTION_PARTITION_MODE} '{other}' (expected auto, single, or multi)" + )) + .to_adbc()), + } + } + + fn as_option(self) -> &'static str { + match self { + Self::Multi => "multi", + Self::Auto => "auto", + Self::Single => "single", + } + } +} + +/// True if executing a single output partition of `plan` would re-read all input partitions, +/// i.e. the plan contains a repartition or sort shuffle. Such plans are poor candidates for +/// distributed partitioned execution (each partition re-runs the pre-shuffle pipeline). +fn plan_has_shuffle(plan: &Arc) -> bool { + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; + + let node = plan.as_any(); + if node.is::() + || node.is::() + || node.is::() + { + return true; + } + plan.children().into_iter().any(plan_has_shuffle) +} + pub struct DataFusionStatement { runtime: Arc, ctx: Arc, codec: Option, + partition_mode: PartitionMode, query: Option, bound: Option>, ingest: BulkIngestState, @@ -893,6 +961,15 @@ impl Optionable for DataFusionStatement { .message("temporary tables are not supported") .to_adbc()), }, + OPTION_PARTITION_MODE => match value { + OptionValue::String(v) => { + self.partition_mode = PartitionMode::from_option(&v)?; + Ok(()) + } + _ => Err(ErrorHelper::set_invalid_option(&key, &value) + .message("must be a string") + .to_adbc()), + }, _ => Err(ErrorHelper::set_unknown_option(&key).to_adbc()), } } @@ -905,6 +982,7 @@ impl Optionable for DataFusionStatement { .format(format_args!("{key:?} has not been set")) .to_adbc()), }, + OPTION_PARTITION_MODE => Ok(self.partition_mode.as_option().to_string()), _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), } } @@ -1194,8 +1272,31 @@ impl Statement for DataFusionStatement { .create_physical_plan() .await .map_err(ErrorHelper::from_datafusion)?; + let natural_n = physical.output_partitioning().partition_count() as u32; + + // A plan that shuffles re-runs its pre-shuffle pipeline once per partition when + // executed partition-by-partition. Depending on partition_mode, collapse such a + // plan to a single coalesced partition, or keep the natural partitions and warn. + let shuffle = plan_has_shuffle(&physical); + let collapse = matches!(self.partition_mode, PartitionMode::Single) + || (matches!(self.partition_mode, PartitionMode::Auto) && shuffle); + + let physical = if collapse && natural_n > 1 { + Arc::new(CoalescePartitionsExec::new(physical)) as Arc + } else { + physical + }; let n = physical.output_partitioning().partition_count() as u32; + if shuffle && !collapse && n > 1 { + log::warn!( + "execute_partitions: plan contains a shuffle (repartition/sort), so each of \ + the {n} partitions re-runs the pre-shuffle pipeline; this is often slower \ + than a single execution. Set {OPTION_PARTITION_MODE}=auto or =single to \ + coalesce into one partition." + ); + } + // Serialize the already-built physical plan so read_partition deserializes it // instead of re-planning. The default codec covers built-in nodes; a registered // codec additionally covers a provider's custom nodes. A node neither can encode diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index d193a8a..b051f67 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -539,3 +539,113 @@ fn test_execute_partitions_prepared() { keys.sort(); assert_eq!(keys, (0..8).collect::>()); } + +const PARTITION_MODE: &str = "datafusion.partition_mode"; + +/// A connection with a two-partition in-memory table `p` (keys 0..4 and 4..8) and no +/// repartitioning forced, so `SELECT * FROM p` is a shuffle-free two-partition scan. +fn connection_with_two_partition_table() -> DataFusionConnection { + let init: ContextInit = Arc::new(|ctx, _opts| { + let schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let part = |lo: i32, hi: i32| { + let keys: ArrayRef = Arc::new(Int32Array::from((lo..hi).collect::>())); + let vals: ArrayRef = + Arc::new(Int32Array::from((lo..hi).map(|i| i * 10).collect::>())); + RecordBatch::try_new(schema.clone(), vec![keys, vals]).unwrap() + }; + // Two batch groups => two table partitions. + let table = MemTable::try_new(schema.clone(), vec![vec![part(0, 4)], vec![part(4, 8)]])?; + ctx.register_table("p", Arc::new(table))?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + let database = driver.new_database().unwrap(); + database.new_connection().unwrap() +} + +#[test] +fn test_partition_mode_single_collapses_to_one() { + // GROUP_BY_QUERY hash-repartitions to 4 partitions; single mode must coalesce to one. + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("single".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), 1, "single mode => one descriptor"); + + // The lone descriptor reproduces every key — the coalesced plan runs the whole query. + let mut keys = read_partition_keys(&connection, &result.partitions[0]); + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_partition_mode_auto_collapses_shuffle() { + // Aggregate has a shuffle => auto coalesces to one partition. + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("auto".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + 1, + "auto mode collapses a shuffling plan to one partition" + ); +} + +#[test] +fn test_partition_mode_auto_keeps_shuffle_free_partitions() { + // A plain two-partition scan has no shuffle => auto keeps the natural partitions. + let mut connection = connection_with_two_partition_table(); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query("SELECT k, v FROM p").unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("auto".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + 2, + "auto keeps natural partitions when the plan does not shuffle" + ); + + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_partition_mode_rejects_unknown_value() { + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + let err = statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("nonsense".to_string()), + ) + .unwrap_err(); + assert_eq!(err.status, adbc_core::error::Status::InvalidArguments); +} From 88ca5beea38d3a642f76bb9edc25d2d4a942c522 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 26 Jun 2026 15:01:19 -0400 Subject: [PATCH 07/15] docs: note partition_mode=single overlaps with execute single mode returns the same one-partition result as calling execute directly, just through the partition API. Document the overlap so callers prefer execute unless they are built around execute_partitions/read_partition. Co-Authored-By: Claude Opus 4.8 --- README.md | 4 ++++ src/lib.rs | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ca4ca92..7a54c5b 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,10 @@ statement.set_option( )?; ``` +`single` produces the same one-partition result as calling `execute` directly — it just +delivers it through the partition API. Prefer `execute` unless your caller is built around +`execute_partitions`/`read_partition` and wants uniform handling. + ### Custom nodes `datafusion-proto` serializes built-in nodes with no configuration. A provider that emits diff --git a/src/lib.rs b/src/lib.rs index 4b2c4d9..7ddceec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -884,7 +884,10 @@ enum PartitionMode { /// Collapse to a single coalesced partition when the plan shuffles, otherwise one /// descriptor per natural output partition. Auto, - /// Always collapse to a single coalesced partition. + /// Always collapse to a single coalesced partition. This yields the same one-partition + /// result as calling [`Statement::execute`] directly, just delivered through the + /// partition API; prefer `execute` unless the caller is built around + /// `execute_partitions`/`read_partition`. Single, } From 15133f72708f8fc080d10a8f9524fe0fa16ddab1 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 29 Jun 2026 07:50:30 -0400 Subject: [PATCH 08/15] feat: default partition_mode to auto; trim PR-flagged docs Address review on PR #32: - Switch the default PartitionMode from Multi to Auto. Auto collapses a shuffling plan to a single partition instead of re-running the pre-shuffle pipeline once per partition, which is the safer default. Tests that relied on the old Multi default now request partition_mode=multi explicitly. - Apply the suggested trim to the with_codec doc comment. - Fix the garbled "A node neither can encode fails the query" comment. - Trim comments that restated the code (DESCRIPTOR_VERSION, PROTO_DESCRIPTOR_WARN_BYTES, read_partition cache, execute_partitions). - Summarize the README partitioned-execution section, moving implementation detail into the source. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 43 +++++++++++------------------------ src/lib.rs | 48 +++++++++++++++------------------------- tests/test_datafusion.rs | 25 +++++++++++++++++++-- 3 files changed, 54 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 7a54c5b..a1c6f55 100644 --- a/README.md +++ b/README.md @@ -50,61 +50,44 @@ shipped to other processes or machines (for example, a distributed engine fannin descriptor out to a worker). A descriptor is self-contained — the only thing that travels between the planner and the reader. -`execute_partitions` serializes the already-built **physical plan** into each descriptor -([`datafusion-proto`](https://docs.rs/datafusion-proto)), and `read_partition` -**deserializes** it and executes one partition. Because the reader runs the exact plan the -planner built, every reader agrees on the partition layout and the reader needs only the -plan bytes (plus any registered providers and codec) — not the original query text or the -planner's session configuration. +A descriptor carries the planner's physical plan, so a reader runs exactly the plan the +planner built — it needs no query text or session configuration, only the descriptor (plus +any providers and codec the plan references). ### Shuffling plans -Partitioned execution shines when each output partition reads independent input — a -partitioned scan, or filters/projections over one. When the plan **shuffles** (a hash -repartition for a join or grouped aggregate, or a sort), producing one output partition -requires reading *all* input, so executing partitions independently re-runs the whole +Partitioned execution is fast when each output partition reads independent input. When the +plan **shuffles** (a hash repartition for a join or grouped aggregate, or a sort), every +output partition reads *all* input, so executing partitions independently re-runs the pre-shuffle pipeline once per partition — usually slower than a single execution. The statement option `datafusion.partition_mode` controls this: | Value | Behavior | | --- | --- | -| `multi` (default) | One descriptor per natural output partition; logs a warning if the plan shuffles. | -| `auto` | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | +| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | +| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles. | | `single` | Always collapse to a single partition. | ```rust statement.set_option( OptionStatement::Other("datafusion.partition_mode".into()), - OptionValue::String("auto".into()), + OptionValue::String("multi".into()), )?; ``` -`single` produces the same one-partition result as calling `execute` directly — it just -delivers it through the partition API. Prefer `execute` unless your caller is built around -`execute_partitions`/`read_partition` and wants uniform handling. +`single` delivers the same one-partition result as `execute`, through the partition API. ### Custom nodes -`datafusion-proto` serializes built-in nodes with no configuration. A provider that emits -custom `ExecutionPlan` nodes must register a `PhysicalExtensionCodec` alongside the -`ContextInit` hook so those nodes round-trip: +A provider that emits custom `ExecutionPlan` nodes must register a `PhysicalExtensionCodec` +so those nodes round-trip into descriptors; without one, a plan containing a custom node +fails `execute_partitions`. See `tests/test_proto_partitions.rs` for a worked example. ```rust let driver = DataFusionDriver::new_with_context_init(handle, context_init).with_codec(codec); ``` -The custom node must be reconstructable from its encoded bytes plus the hook-built session. -With no codec the driver uses the default codec, which covers built-in nodes only; a plan -containing a custom node the codec cannot encode **fails** `execute_partitions` rather than -falling back to a re-plan path. - -`read_partition` caches a deserialized plan per connection, keyed by the descriptor's -plan bytes, so a connection that reads several partitions of the same plan pays the -decode cost once per distinct plan rather than once per partition. - -See `tests/test_proto_partitions.rs` for a worked custom provider + codec example. - ## Building See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/src/lib.rs b/src/lib.rs index 7ddceec..f4d19c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -325,12 +325,7 @@ impl DataFusionDriver { } /// Register an extension codec for serializing custom `ExecutionPlan` nodes in partition - /// descriptors (see [`PhysicalCodec`]). Chains onto any constructor; only needed when a - /// provider emits custom nodes: - /// - /// ```ignore - /// let driver = DataFusionDriver::new_with_context_init(handle, init).with_codec(codec); - /// ``` + /// descriptors (see [`PhysicalCodec`]). pub fn with_codec(mut self, codec: PhysicalCodec) -> Self { self.codec = Some(codec); self @@ -743,14 +738,12 @@ impl Connection for DataFusionConnection { ) -> Result> { let (index, plan_bytes) = decode_descriptor(partition.as_ref())?; self.runtime.block_on(async { - // Deserialize the physical plan the driver already built and execute one - // partition — no logical/physical re-planning. let task_ctx = self.ctx.task_ctx(); let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = self.codec.as_deref().unwrap_or(&default_codec); - // Deserialize once per distinct plan and reuse across partition indices; a - // shared connection then pays the decode cost once per executor, not per task. + // Cache the deserialized plan per distinct payload: a connection reading several + // partitions of one plan pays the proto decode once, not once per partition. let physical = { let mut cache = self.plan_cache.lock(); if let Some(plan) = cache.get(plan_bytes.as_slice()) { @@ -819,18 +812,16 @@ impl QueryState { } } -/// Current partition-descriptor format version. Descriptors are opaque and version-local: -/// the same driver build both produces and consumes them, so older layouts are not decoded. -/// The version byte lets the format evolve without silently misreading an old payload. +/// Partition-descriptor format version. The same driver build produces and consumes +/// descriptors, so the byte exists only to reject a payload from a different layout. const DESCRIPTOR_VERSION: u8 = 1; /// Fixed descriptor header length: `version(1) + index(4)`. const DESCRIPTOR_HEADER_LEN: usize = 5; -/// Size above which a serialized plan payload is logged as a warning. The full plan is -/// copied into every one of the N partition descriptors, so a large plan is paid N times -/// over. Operator-only plans are a few KB and never trip this; the threshold mainly catches -/// plans that inline data (e.g. a custom node embedding its rows) or very large queries. +/// Warn when a serialized plan exceeds this size: the plan is copied into all N descriptors, +/// so a large payload is paid N times. Operator-only plans are a few KB; this mainly catches +/// plans that inline data. const PROTO_DESCRIPTOR_WARN_BYTES: usize = 8 << 20; // 8 MiB /// Build a self-contained partition descriptor: @@ -877,12 +868,13 @@ const OPTION_PARTITION_MODE: &str = "datafusion.partition_mode"; /// single execution. This mode controls how that case is handled. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] enum PartitionMode { - /// One descriptor per natural output partition; warn when the plan shuffles. Default — - /// preserves the partition count the caller's plan produced. - #[default] + /// One descriptor per natural output partition; warn when the plan shuffles. Preserves + /// the partition count the caller's plan produced even when that is slower. Multi, /// Collapse to a single coalesced partition when the plan shuffles, otherwise one - /// descriptor per natural output partition. + /// descriptor per natural output partition. Default — never silently re-runs a shuffle + /// once per partition. + #[default] Auto, /// Always collapse to a single coalesced partition. This yields the same one-partition /// result as calling [`Statement::execute`] directly, just delivered through the @@ -923,9 +915,7 @@ fn plan_has_shuffle(plan: &Arc) -> bool { use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; let node = plan.as_any(); - if node.is::() - || node.is::() - || node.is::() + if node.is::() || node.is::() || node.is::() { return true; } @@ -1267,8 +1257,7 @@ impl Statement for DataFusionStatement { .to_adbc() })?; self.runtime.block_on(async { - // Plan logically (registers object store) and build the physical plan so we - // can count its natural output partitions. + // Plan logically (registers object store), then build the physical plan. let df = query.execute(&self.ctx).await?; let schema = df.schema().as_arrow().clone(); let physical = df @@ -1300,10 +1289,9 @@ impl Statement for DataFusionStatement { ); } - // Serialize the already-built physical plan so read_partition deserializes it - // instead of re-planning. The default codec covers built-in nodes; a registered - // codec additionally covers a provider's custom nodes. A node neither can encode - // fails the query (no re-plan fallback). + // Serialize the physical plan into each descriptor; read_partition deserializes + // rather than re-plans. The default codec covers built-in nodes, a registered + // codec also covers custom ones; a node neither can encode fails the query. let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = self.codec.as_deref().unwrap_or(&default_codec); diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index b051f67..47195ce 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -478,6 +478,14 @@ fn test_execute_partitions_returns_one_descriptor_per_output_partition() { let mut connection = connection_with_target_partitions(4); let mut statement = connection.new_statement().unwrap(); statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + // GROUP_BY_QUERY shuffles, so the default `auto` mode would collapse it; force `multi` + // to exercise one descriptor per natural partition. + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); let result = statement.execute_partitions().unwrap(); @@ -505,6 +513,12 @@ fn test_read_partition_executes_serialized_plan() { let mut planner = connection_with_target_partitions(4); let mut statement = planner.new_statement().unwrap(); statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); let result = statement.execute_partitions().unwrap(); assert!(result.partitions.len() > 1); @@ -527,6 +541,12 @@ fn test_execute_partitions_prepared() { let mut connection = connection_with_target_partitions(4); let mut statement = connection.new_statement().unwrap(); statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); statement.prepare().unwrap(); let result = statement.execute_partitions().unwrap(); @@ -552,8 +572,9 @@ fn connection_with_two_partition_table() -> DataFusionConnection { ])); let part = |lo: i32, hi: i32| { let keys: ArrayRef = Arc::new(Int32Array::from((lo..hi).collect::>())); - let vals: ArrayRef = - Arc::new(Int32Array::from((lo..hi).map(|i| i * 10).collect::>())); + let vals: ArrayRef = Arc::new(Int32Array::from( + (lo..hi).map(|i| i * 10).collect::>(), + )); RecordBatch::try_new(schema.clone(), vec![keys, vals]).unwrap() }; // Two batch groups => two table partitions. From 9d31c3288b67355565b0ee9f3dd4c137a2b5a4f5 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 29 Jun 2026 07:54:41 -0400 Subject: [PATCH 09/15] Update license on new test file --- tests/test_proto_partitions.rs | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/test_proto_partitions.rs b/tests/test_proto_partitions.rs index 88cb82b..bf40e01 100644 --- a/tests/test_proto_partitions.rs +++ b/tests/test_proto_partitions.rs @@ -1,24 +1,16 @@ -// Copyright (c) 2025-2026 ADBC Drivers Contributors +// Copyright (c) 2026 ADBC Drivers Contributors // -// This file has been modified from its original version, which is -// under the Apache License: +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. //! Proto path tests: a custom `TableProvider` emits a custom `ExecutionPlan`, and a //! `PhysicalExtensionCodec` round-trips it. `execute_partitions` serializes the physical From 7270e17e5ab54b8da8ed2498ee4518f9cd440fa9 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 29 Jun 2026 08:04:56 -0400 Subject: [PATCH 10/15] docs: correct and trim codec doc comments The codec serializes more than ExecutionPlan nodes (also UDFs and exprs); reword the PhysicalCodec type doc, with_codec, and the README accordingly, and trim the surrounding prose. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 8 ++++---- src/lib.rs | 16 +++++----------- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a1c6f55..1368aed 100644 --- a/README.md +++ b/README.md @@ -78,11 +78,11 @@ statement.set_option( `single` delivers the same one-partition result as `execute`, through the partition API. -### Custom nodes +### Custom extensions -A provider that emits custom `ExecutionPlan` nodes must register a `PhysicalExtensionCodec` -so those nodes round-trip into descriptors; without one, a plan containing a custom node -fails `execute_partitions`. See `tests/test_proto_partitions.rs` for a worked example. +A provider emitting custom physical-plan extensions (`ExecutionPlan` nodes, UDFs, exprs) +must register a `PhysicalExtensionCodec` so they round-trip into descriptors; without one, +such a plan fails `execute_partitions`. See `tests/test_proto_partitions.rs` for an example. ```rust let driver = DataFusionDriver::new_with_context_init(handle, context_init).with_codec(codec); diff --git a/src/lib.rs b/src/lib.rs index f4d19c0..3a803cb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -77,15 +77,9 @@ pub type ContextInit = Arc< dyn Fn(&mut SessionContext, &mut DatabaseOpts) -> datafusion::error::Result<()> + Send + Sync, >; -/// Optional extension codec for serializing custom `ExecutionPlan` nodes. -/// -/// Partitioned execution serializes the physical plan into each partition descriptor -/// (`datafusion-proto`) so `read_partition` can execute it without the original query. -/// `datafusion-proto` serializes built-in DataFusion nodes on its own; a provider that -/// emits custom `ExecutionPlan` nodes registers a codec here so those nodes can be encoded -/// and reconstructed from their bytes plus the hook-built session. With no codec the driver -/// uses the default codec, which handles built-in nodes only — a plan containing a custom -/// node the codec cannot encode fails `execute_partitions`. +/// Optional codec for serializing custom physical-plan extensions (`ExecutionPlan` nodes, +/// UDFs, exprs) into partition descriptors. Without one, a plan containing a custom +/// extension fails `execute_partitions`. pub type PhysicalCodec = Arc; impl ErrorHelper { @@ -324,8 +318,8 @@ impl DataFusionDriver { } } - /// Register an extension codec for serializing custom `ExecutionPlan` nodes in partition - /// descriptors (see [`PhysicalCodec`]). + /// Register an extension codec for serializing custom physical-plan extensions + /// (`ExecutionPlan` nodes, UDFs, exprs) in partition descriptors (see [`PhysicalCodec`]). pub fn with_codec(mut self, codec: PhysicalCodec) -> Self { self.codec = Some(codec); self From b183f4ea6abf3fd209837f5fca206534d132165c Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 29 Jun 2026 10:08:20 -0400 Subject: [PATCH 11/15] docs: standardize no-query message and trim two comments - Use "no query has been set" at all seven call sites (some said "no query or Substrait plan has been set"). - Trim the execute_partitions comments: keep the non-obvious note that query.execute registers object stores, and drop the re-planning aside that assumed knowledge of an approach not used here. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3a803cb..c864156 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1183,7 +1183,7 @@ impl Statement for DataFusionStatement { Some(q) => q.execute(&self.ctx).await?, None => { return Err(ErrorHelper::invalid_state() - .message("no query or Substrait plan has been set") + .message("no query has been set") .to_adbc()); } }; @@ -1209,7 +1209,7 @@ impl Statement for DataFusionStatement { Some(q) => q.execute(&self.ctx).await?, None => { return Err(ErrorHelper::invalid_state() - .message("no query or Substrait plan has been set") + .message("no query has been set") .to_adbc()); } }; @@ -1247,11 +1247,11 @@ impl Statement for DataFusionStatement { fn execute_partitions(&mut self) -> adbc_core::error::Result { let query = self.query.as_ref().ok_or_else(|| { ErrorHelper::invalid_state() - .message("no query or Substrait plan has been set") + .message("no query has been set") .to_adbc() })?; self.runtime.block_on(async { - // Plan logically (registers object store), then build the physical plan. + // query.execute also registers the plan's object stores, required before execution. let df = query.execute(&self.ctx).await?; let schema = df.schema().as_arrow().clone(); let physical = df @@ -1283,9 +1283,9 @@ impl Statement for DataFusionStatement { ); } - // Serialize the physical plan into each descriptor; read_partition deserializes - // rather than re-plans. The default codec covers built-in nodes, a registered - // codec also covers custom ones; a node neither can encode fails the query. + // Serialize the physical plan into each descriptor. The default codec covers + // built-in nodes, a registered codec also covers custom ones; a node neither + // can encode fails the query. let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = self.codec.as_deref().unwrap_or(&default_codec); From 965b1335bc54c88ea31771067149bbb6f86ebc55 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 30 Jun 2026 08:25:34 -0400 Subject: [PATCH 12/15] docs: move partitioned-execution docs to datafusion.md Relocate the Partitioned execution section out of the README and into docs/datafusion.md, the rendered user-facing docs. Merge the derived-driver pattern (custom TableProvider/codec via a downstream cdylib) into the existing Custom extensions subsection. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 49 ------------------ docs/datafusion.md | 123 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 1368aed..56b5fee 100644 --- a/README.md +++ b/README.md @@ -39,55 +39,6 @@ various platforms from the [Columnar](https://columnar.tech) CDN in the future. See [Building](#building) to build the drivers yourself. -## Partitioned execution - -ADBC partitioned execution splits a result set into independent pieces that can be read -in parallel. One caller runs `execute_partitions`, which plans the query and returns one -opaque **descriptor** per output partition; each descriptor is then handed to -`read_partition`, which produces that partition's rows. The two halves need not run in the -same place: descriptors can be read back on the same connection, on other threads, or -shipped to other processes or machines (for example, a distributed engine fanning each -descriptor out to a worker). A descriptor is self-contained — the only thing that travels -between the planner and the reader. - -A descriptor carries the planner's physical plan, so a reader runs exactly the plan the -planner built — it needs no query text or session configuration, only the descriptor (plus -any providers and codec the plan references). - -### Shuffling plans - -Partitioned execution is fast when each output partition reads independent input. When the -plan **shuffles** (a hash repartition for a join or grouped aggregate, or a sort), every -output partition reads *all* input, so executing partitions independently re-runs the -pre-shuffle pipeline once per partition — usually slower than a single execution. - -The statement option `datafusion.partition_mode` controls this: - -| Value | Behavior | -| --- | --- | -| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | -| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles. | -| `single` | Always collapse to a single partition. | - -```rust -statement.set_option( - OptionStatement::Other("datafusion.partition_mode".into()), - OptionValue::String("multi".into()), -)?; -``` - -`single` delivers the same one-partition result as `execute`, through the partition API. - -### Custom extensions - -A provider emitting custom physical-plan extensions (`ExecutionPlan` nodes, UDFs, exprs) -must register a `PhysicalExtensionCodec` so they round-trip into descriptors; without one, -such a plan fails `execute_partitions`. See `tests/test_proto_partitions.rs` for an example. - -```rust -let driver = DataFusionDriver::new_with_context_init(handle, context_init).with_codec(codec); -``` - ## Building See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/docs/datafusion.md b/docs/datafusion.md index 36e349c..3a3c169 100644 --- a/docs/datafusion.md +++ b/docs/datafusion.md @@ -86,6 +86,129 @@ SELECT `Breed Name`, `Lifespan` {{ footnotes|safe }} +## Partitioned execution + +ADBC partitioned execution splits a result set into independent pieces that can be read +in parallel. One caller runs `execute_partitions`, which plans the query and returns one +opaque **descriptor** per output partition; each descriptor is then handed to +`read_partition`, which produces that partition's rows. The two halves need not run in the +same place: descriptors can be read back on the same connection, on other threads, or +shipped to other processes or machines (for example, a distributed engine fanning each +descriptor out to a worker). A descriptor is self-contained — the only thing that travels +between the planner and the reader. + +A descriptor carries the planner's physical plan, so a reader runs exactly the plan the +planner built — it needs no query text or session configuration, only the descriptor (plus +any providers and codec the plan references). + +### Shuffling plans + +Partitioned execution is fast when each output partition reads independent input. When the +plan **shuffles** (a hash repartition for a join or grouped aggregate, or a sort), every +output partition reads *all* input, so executing partitions independently re-runs the +pre-shuffle pipeline once per partition — usually slower than a single execution. + +The statement option `datafusion.partition_mode` controls this: + +| Value | Behavior | +| --- | --- | +| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | +| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles. | +| `single` | Always collapse to a single partition. | + +```rust +statement.set_option( + OptionStatement::Other("datafusion.partition_mode".into()), + OptionValue::String("multi".into()), +)?; +``` + +`single` delivers the same one-partition result as `execute`, through the partition API. + +### Custom extensions + +The pre-packaged driver exposes the built-in DataFusion engine. To add your own +[`TableProvider`], catalog, object store, UDFs, or custom `ExecutionPlan` nodes, build a +*derived driver*: a small Rust crate that depends on this one, bakes your extensions in, and +exports its own ADBC entrypoint as a `cdylib`. A driver manager then loads your shared +library exactly like any other ADBC driver — the Python examples above work unchanged once +`driver=` points at your `.so`/`.dylib`/`.dll`. + +Two hooks customize the driver: + +- **`new_with_context_init`** — runs a closure against each database's `SessionContext`. This + is where you register table providers, catalogs, object stores, and UDFs. +- **`with_codec`** — registers a `PhysicalExtensionCodec`. Needed *only* when your plans + contain custom physical-plan extensions (custom `ExecutionPlan` nodes, UDFs, exprs), so + they round-trip into partition descriptors; without one, such a plan fails + `execute_partitions`. A provider that emits only standard DataFusion plan nodes does not + need a codec. See `tests/test_proto_partitions.rs` for a codec example. + +The C entrypoint constructs the driver via `Default` (it takes no arguments), so bake your +extensions into a `Default` impl on a thin newtype. The `Driver` trait is just +`new_database` / `new_database_with_opts`, which delegate to the inner driver: + +```rust +use std::sync::Arc; +use adbc_core::Driver; +use adbc_driver_datafusion::{DataFusionDatabase, DataFusionDriver}; + +struct MyDriver(DataFusionDriver); + +impl Default for MyDriver { + fn default() -> Self { + // Register custom extensions on every database's SessionContext. + let context_init = Arc::new(|ctx: &mut datafusion::prelude::SessionContext, _opts| { + ctx.register_table("my_table", Arc::new(MyTableProvider::new()))?; + Ok(()) + }); + + let driver = DataFusionDriver::new_with_context_init(None, context_init) + .with_codec(Arc::new(MyCodec::default())); // only for custom plan nodes + + MyDriver(driver) + } +} + +impl Driver for MyDriver { + type DatabaseType = DataFusionDatabase; + + fn new_database(&mut self) -> adbc_core::error::Result { + self.0.new_database() + } + + fn new_database_with_opts( + &mut self, + opts: impl IntoIterator< + Item = (adbc_core::options::OptionDatabase, adbc_core::options::OptionValue), + >, + ) -> adbc_core::error::Result { + self.0.new_database_with_opts(opts) + } +} + +adbc_ffi::export_driver!(MyDatafusionDriverInit, MyDriver); +``` + +Build it with `crate-type = ["cdylib"]` and load your entrypoint: + +```python +from adbc_driver_manager import dbapi + +dbapi.connect( + driver="/path/to/libmy_datafusion_driver.so", + entrypoint="MyDatafusionDriverInit", +) +``` + +Because extensions are Rust types compiled into the driver, this derived-driver approach is +the supported path for custom providers and codecs — there is no way to inject them through +a driver manager's options at load time. Ordinary partitioned execution over built-in and +registered sources works through any driver manager without a derived driver, since +descriptors are opaque bytes the manager simply hands back to `read_partition`. + +[`TableProvider`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProvider.html + ## Compatibility {{ compatibility_info|safe }} From 58890a122e1b352df8642f6832a6efb3a9bce81c Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 1 Jul 2026 09:52:57 +0900 Subject: [PATCH 13/15] simplify docs --- docs/datafusion.md | 129 +++++---------------------------------------- 1 file changed, 13 insertions(+), 116 deletions(-) diff --git a/docs/datafusion.md b/docs/datafusion.md index 3a3c169..5876b8d 100644 --- a/docs/datafusion.md +++ b/docs/datafusion.md @@ -86,128 +86,25 @@ SELECT `Breed Name`, `Lifespan` {{ footnotes|safe }} -## Partitioned execution - -ADBC partitioned execution splits a result set into independent pieces that can be read -in parallel. One caller runs `execute_partitions`, which plans the query and returns one -opaque **descriptor** per output partition; each descriptor is then handed to -`read_partition`, which produces that partition's rows. The two halves need not run in the -same place: descriptors can be read back on the same connection, on other threads, or -shipped to other processes or machines (for example, a distributed engine fanning each -descriptor out to a worker). A descriptor is self-contained — the only thing that travels -between the planner and the reader. - -A descriptor carries the planner's physical plan, so a reader runs exactly the plan the -planner built — it needs no query text or session configuration, only the descriptor (plus -any providers and codec the plan references). - -### Shuffling plans - -Partitioned execution is fast when each output partition reads independent input. When the -plan **shuffles** (a hash repartition for a join or grouped aggregate, or a sort), every -output partition reads *all* input, so executing partitions independently re-runs the -pre-shuffle pipeline once per partition — usually slower than a single execution. - -The statement option `datafusion.partition_mode` controls this: - -| Value | Behavior | -| --- | --- | -| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions. | -| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles. | -| `single` | Always collapse to a single partition. | - -```rust -statement.set_option( - OptionStatement::Other("datafusion.partition_mode".into()), - OptionValue::String("multi".into()), -)?; -``` +### Partitioned Execution -`single` delivers the same one-partition result as `execute`, through the partition API. - -### Custom extensions - -The pre-packaged driver exposes the built-in DataFusion engine. To add your own -[`TableProvider`], catalog, object store, UDFs, or custom `ExecutionPlan` nodes, build a -*derived driver*: a small Rust crate that depends on this one, bakes your extensions in, and -exports its own ADBC entrypoint as a `cdylib`. A driver manager then loads your shared -library exactly like any other ADBC driver — the Python examples above work unchanged once -`driver=` points at your `.so`/`.dylib`/`.dll`. - -Two hooks customize the driver: - -- **`new_with_context_init`** — runs a closure against each database's `SessionContext`. This - is where you register table providers, catalogs, object stores, and UDFs. -- **`with_codec`** — registers a `PhysicalExtensionCodec`. Needed *only* when your plans - contain custom physical-plan extensions (custom `ExecutionPlan` nodes, UDFs, exprs), so - they round-trip into partition descriptors; without one, such a plan fails - `execute_partitions`. A provider that emits only standard DataFusion plan nodes does not - need a codec. See `tests/test_proto_partitions.rs` for a codec example. - -The C entrypoint constructs the driver via `Default` (it takes no arguments), so bake your -extensions into a `Default` impl on a thin newtype. The `Driver` trait is just -`new_database` / `new_database_with_opts`, which delegate to the inner driver: - -```rust -use std::sync::Arc; -use adbc_core::Driver; -use adbc_driver_datafusion::{DataFusionDatabase, DataFusionDriver}; - -struct MyDriver(DataFusionDriver); - -impl Default for MyDriver { - fn default() -> Self { - // Register custom extensions on every database's SessionContext. - let context_init = Arc::new(|ctx: &mut datafusion::prelude::SessionContext, _opts| { - ctx.register_table("my_table", Arc::new(MyTableProvider::new()))?; - Ok(()) - }); - - let driver = DataFusionDriver::new_with_context_init(None, context_init) - .with_codec(Arc::new(MyCodec::default())); // only for custom plan nodes - - MyDriver(driver) - } -} - -impl Driver for MyDriver { - type DatabaseType = DataFusionDatabase; - - fn new_database(&mut self) -> adbc_core::error::Result { - self.0.new_database() - } - - fn new_database_with_opts( - &mut self, - opts: impl IntoIterator< - Item = (adbc_core::options::OptionDatabase, adbc_core::options::OptionValue), - >, - ) -> adbc_core::error::Result { - self.0.new_database_with_opts(opts) - } -} - -adbc_ffi::export_driver!(MyDatafusionDriverInit, MyDriver); -``` +This driver supports ADBC's partitioned execution; one partition is generated per output partition. Each partition contains the physical plan, and can be distributed across CPUs or physical machines. If the plan contains a shuffle, however, distributed execution will likely be slower, as each partition must read the entire output. -Build it with `crate-type = ["cdylib"]` and load your entrypoint: +The statement option `datafusion.partition_mode` controls what to do if a shuffle is detected: -```python -from adbc_driver_manager import dbapi +| Value | Behavior | +|------------------|-------------------------------------------------------------------------------------| +| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions | +| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles | +| `single` | Always collapse to a single partition (effectively the same as regular execution) | -dbapi.connect( - driver="/path/to/libmy_datafusion_driver.so", - entrypoint="MyDatafusionDriverInit", -) -``` +### Extending DataFusion -Because extensions are Rust types compiled into the driver, this derived-driver approach is -the supported path for custom providers and codecs — there is no way to inject them through -a driver manager's options at load time. Ordinary partitioned execution over built-in and -registered sources works through any driver manager without a derived driver, since -descriptors are opaque bytes the manager simply hands back to `read_partition`. +It is possible to customize the embedded DataFusion by depending on this crate, then using hooks to customize the DataFusion `SessionContext` and register `PhysicalExtensionCodec`s. The customized driver can then be built into a shared library and distributed as an ADBC driver. For more, see: -[`TableProvider`]: https://docs.rs/datafusion/latest/datafusion/catalog/trait.TableProvider.html +- `DataFusionDriver::new_with_context_init`, which accepts a callback to modify the `SessionContext` +- `DataFusionDriver::with_codec`, which accepts a `PhysicalExtensionCodec` used to serialize/deserialize plans in partitioned execution +- The [`adbc_ffi`](https://crates.io/crates/adbc_ffi) crate, which provides helpers to export a Rust ADBC driver as a shared library ## Compatibility From 23570c2af42ff2ab479207fdfc2076f6ea50214a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 1 Jul 2026 08:49:47 -0400 Subject: [PATCH 14/15] perf: share plan cache across connections via database Move the deserialized-physical-plan cache from DataFusionConnection up to DataFusionDatabase and clone the Arc into each connection. Connectors open one connection per task but cache one database per executor, so a per-connection cache was written once and never reused. Database-scoping it collapses N per-task plan deserializes into one per distinct plan per executor. Also swap the clear-all-on-overflow HashMap for an LRU (cap 16 -> 64) so overflow evicts only the least-recently-used plan instead of wiping hot entries. Co-Authored-By: Claude Opus 4.8 (1M context) --- Cargo.lock | 15 +++++++++++++++ Cargo.toml | 1 + src/lib.rs | 44 +++++++++++++++++++++++++++++--------------- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d88ff46..3eb428d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,7 @@ dependencies = [ "driverbase", "futures", "log", + "lru", "object_store", "parking_lot", "prost", @@ -2399,6 +2400,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -2884,6 +2890,15 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index 9b9bd45..1e438e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,6 +43,7 @@ datafusion-substrait = { version = "53", default-features = false } driverbase = { git = "https://github.com/adbc-drivers/driverbase-rs.git" } futures = "0.3.32" log = "0.4" +lru = "0.18" object_store = { version = "0.13", features = ["aws", "gcp", "http"] } parking_lot = "0.12" prost = "0.14.3" diff --git a/src/lib.rs b/src/lib.rs index c864156..3894e3e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,8 +40,11 @@ use prost::Message; use std::collections::HashMap; use std::fmt::Debug; use std::future::Future; +use std::num::NonZeroUsize; use std::sync::Arc; +use lru::LruCache; + use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::{ArrowError, SchemaRef}; @@ -344,6 +347,7 @@ impl DataFusionDriver { handle: self.handle.clone(), ctx: Arc::new(ctx), codec: self.codec.clone(), + plan_cache: Arc::new(new_plan_cache()), }) } } @@ -384,6 +388,9 @@ pub struct DataFusionDatabase { handle: Option, ctx: Arc, codec: Option, + /// Shared by every connection this database opens, so all of an executor's + /// per-task connections reuse one deserialized plan per distinct plan bytes. + plan_cache: Arc, } impl Optionable for DataFusionDatabase { @@ -442,7 +449,7 @@ impl Database for DataFusionDatabase { runtime: Arc::new(runtime), ctx: self.ctx.clone(), codec: self.codec.clone(), - plan_cache: Arc::new(PlanCache::default()), + plan_cache: Arc::clone(&self.plan_cache), }) } @@ -466,7 +473,7 @@ impl Database for DataFusionDatabase { runtime: Arc::new(runtime), ctx: self.ctx.clone(), codec: self.codec.clone(), - plan_cache: Arc::new(PlanCache::default()), + plan_cache: Arc::clone(&self.plan_cache), }; for (key, value) in opts { @@ -480,13 +487,22 @@ impl Database for DataFusionDatabase { /// Cache of deserialized physical plans, keyed by descriptor plan bytes. /// /// `read_partition` deserializes a given plan once and reuses it across every partition -/// index. When a connection is shared across an executor's task slots (as connectors are -/// expected to do), this collapses N per-task deserializes into one per distinct plan. -type PlanCache = parking_lot::Mutex, Arc>>; - -/// Bound on distinct cached plans, so a long-lived connection running many different -/// queries does not accumulate plans without limit. On overflow the cache is cleared. -const PLAN_CACHE_CAP: usize = 16; +/// index. The cache lives on [`DataFusionDatabase`] and is shared (via `Arc`) by every +/// connection opened from it. Since connectors open one connection per task but cache one +/// database per executor, this collapses N per-task deserializes into one per distinct plan +/// per executor. +type PlanCache = parking_lot::Mutex, Arc>>; + +/// Bound on distinct cached plans, so a database serving many different queries does not +/// accumulate plans without limit. On overflow the least-recently-used plan is evicted; +/// already-handed-out `Arc`s stay alive until their tasks finish. +const PLAN_CACHE_CAP: usize = 64; + +fn new_plan_cache() -> PlanCache { + parking_lot::Mutex::new(LruCache::new( + NonZeroUsize::new(PLAN_CACHE_CAP).expect("PLAN_CACHE_CAP is nonzero"), + )) +} pub struct DataFusionConnection { runtime: Arc, @@ -736,8 +752,9 @@ impl Connection for DataFusionConnection { let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = self.codec.as_deref().unwrap_or(&default_codec); - // Cache the deserialized plan per distinct payload: a connection reading several - // partitions of one plan pays the proto decode once, not once per partition. + // Cache the deserialized plan per distinct payload. The cache is shared across + // all connections from this database, so every task reading a partition of the + // same plan pays the proto decode once, not once per task. let physical = { let mut cache = self.plan_cache.lock(); if let Some(plan) = cache.get(plan_bytes.as_slice()) { @@ -750,10 +767,7 @@ impl Connection for DataFusionConnection { codec, ) .map_err(ErrorHelper::from_datafusion)?; - if cache.len() >= PLAN_CACHE_CAP { - cache.clear(); - } - cache.insert(plan_bytes.clone(), plan.clone()); + cache.put(plan_bytes.clone(), plan.clone()); plan } }; From 9b51df2b13d20d81c563ef3ee818db1c02082fac Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 1 Jul 2026 09:05:09 -0400 Subject: [PATCH 15/15] feat: expose plan-deserialize counter for testing/eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a database-scoped AtomicU64 counting physical-plan deserializations (plan-cache misses), shared into every connection alongside the plan cache. Exposed as the read-only int option OPTION_PLAN_DESERIALIZE_COUNT on both database and connection. This is for testing and evaluation only, not part of the ADBC option contract: it lets a caller assert the shared cache works — deserializes equal the number of distinct plans, not the number of partitions read. Add a test that plans on one connection and reads each partition on a separate connection from the same database, asserting one deserialize. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib.rs | 31 ++++++++++++++-- tests/test_proto_partitions.rs | 66 ++++++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 9 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3894e3e..1de941f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -42,6 +42,7 @@ use std::fmt::Debug; use std::future::Future; use std::num::NonZeroUsize; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use lru::LruCache; @@ -348,6 +349,7 @@ impl DataFusionDriver { ctx: Arc::new(ctx), codec: self.codec.clone(), plan_cache: Arc::new(new_plan_cache()), + plan_deserializations: Arc::new(AtomicU64::new(0)), }) } } @@ -391,6 +393,9 @@ pub struct DataFusionDatabase { /// Shared by every connection this database opens, so all of an executor's /// per-task connections reuse one deserialized plan per distinct plan bytes. plan_cache: Arc, + /// Count of plan deserializations (cache misses) across all connections. + /// See [`OPTION_PLAN_DESERIALIZE_COUNT`]; used for testing and evaluation. + plan_deserializations: Arc, } impl Optionable for DataFusionDatabase { @@ -426,7 +431,12 @@ impl Optionable for DataFusionDatabase { } fn get_option_int(&self, key: Self::Option) -> adbc_core::error::Result { - Err(ErrorHelper::get_unknown_option(&key).to_adbc()) + match key.as_ref() { + OPTION_PLAN_DESERIALIZE_COUNT => { + Ok(self.plan_deserializations.load(Ordering::Relaxed) as i64) + } + _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), + } } fn get_option_double(&self, key: Self::Option) -> adbc_core::error::Result { @@ -450,6 +460,7 @@ impl Database for DataFusionDatabase { ctx: self.ctx.clone(), codec: self.codec.clone(), plan_cache: Arc::clone(&self.plan_cache), + plan_deserializations: Arc::clone(&self.plan_deserializations), }) } @@ -474,6 +485,7 @@ impl Database for DataFusionDatabase { ctx: self.ctx.clone(), codec: self.codec.clone(), plan_cache: Arc::clone(&self.plan_cache), + plan_deserializations: Arc::clone(&self.plan_deserializations), }; for (key, value) in opts { @@ -504,11 +516,20 @@ fn new_plan_cache() -> PlanCache { )) } +/// Read-only integer option: number of physical plans this database has deserialized +/// from partition descriptors (i.e. plan-cache misses across all its connections). +/// +/// Intended primarily for testing and evaluation: it lets a caller confirm the plan cache +/// is working — e.g. asserting deserializations equal the number of distinct plans rather +/// than the number of partitions read. It is not part of the ADBC option contract. +pub const OPTION_PLAN_DESERIALIZE_COUNT: &str = "adbc.datafusion.plan_deserialize_count"; + pub struct DataFusionConnection { runtime: Arc, ctx: Arc, codec: Option, plan_cache: Arc, + plan_deserializations: Arc, } impl Optionable for DataFusionConnection { @@ -609,7 +630,12 @@ impl Optionable for DataFusionConnection { } fn get_option_int(&self, key: Self::Option) -> adbc_core::error::Result { - Err(ErrorHelper::get_unknown_option(&key).to_adbc()) + match key.as_ref() { + OPTION_PLAN_DESERIALIZE_COUNT => { + Ok(self.plan_deserializations.load(Ordering::Relaxed) as i64) + } + _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), + } } fn get_option_double(&self, key: Self::Option) -> adbc_core::error::Result { @@ -767,6 +793,7 @@ impl Connection for DataFusionConnection { codec, ) .map_err(ErrorHelper::from_datafusion)?; + self.plan_deserializations.fetch_add(1, Ordering::Relaxed); cache.put(plan_bytes.clone(), plan.clone()); plan } diff --git a/tests/test_proto_partitions.rs b/tests/test_proto_partitions.rs index bf40e01..082d1f3 100644 --- a/tests/test_proto_partitions.rs +++ b/tests/test_proto_partitions.rs @@ -24,8 +24,12 @@ use std::fmt; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; -use adbc_core::{Connection, Database, Driver, Statement}; -use adbc_driver_datafusion::{ContextInit, DataFusionConnection, DataFusionDriver, PhysicalCodec}; +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use adbc_driver_datafusion::{ + ContextInit, DataFusionConnection, DataFusionDatabase, DataFusionDriver, + OPTION_PLAN_DESERIALIZE_COUNT, PhysicalCodec, +}; use arrow_array::{ArrayRef, Int32Array, RecordBatch}; use arrow_schema::{DataType, Field, Schema, SchemaRef}; use async_trait::async_trait; @@ -217,7 +221,7 @@ impl PhysicalExtensionCodec for FailingCodec { } } -/// Delegates to [`CustomCodec`] but counts `try_decode` calls, to prove the per-connection +/// Delegates to [`CustomCodec`] but counts `try_decode` calls, to prove the shared /// plan cache deserializes a given plan only once. #[derive(Debug)] struct CountingCodec { @@ -244,8 +248,8 @@ const VALUES: i32 = 8; const N_PARTITIONS: usize = 3; const QUERY: &str = "SELECT v FROM ct"; -/// Build a connection whose `ct` table is the custom provider, with the given codec (if any). -fn custom_connection(codec: Option) -> DataFusionConnection { +/// Build a database whose `ct` table is the custom provider, with the given codec (if any). +fn custom_database(codec: Option) -> DataFusionDatabase { let init: ContextInit = Arc::new(|ctx, _opts| { ctx.register_table( "ct", @@ -260,8 +264,12 @@ fn custom_connection(codec: Option) -> DataFusionConnection { if let Some(codec) = codec { driver = driver.with_codec(codec); } - let database = driver.new_database().unwrap(); - database.new_connection().unwrap() + driver.new_database().unwrap() +} + +/// Build a connection whose `ct` table is the custom provider, with the given codec (if any). +fn custom_connection(codec: Option) -> DataFusionConnection { + custom_database(codec).new_connection().unwrap() } /// Read every `v` value produced by the descriptor. @@ -380,3 +388,47 @@ fn plan_cache_deserializes_once_across_partitions() { "plan should be deserialized once and cached across partition indices" ); } + +#[test] +fn plan_cache_is_shared_across_connections_from_one_database() { + // Mirrors the connector's real shape: plan on one connection, then read each partition + // on a SEPARATE connection (as per-task connections would). All connections come from + // one database, so the database-scoped cache should still deserialize the plan once. + let decodes = Arc::new(AtomicUsize::new(0)); + let codec: PhysicalCodec = Arc::new(CountingCodec { + decodes: decodes.clone(), + }); + let database = custom_database(Some(codec)); + + let mut planner = database.new_connection().unwrap(); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), N_PARTITIONS); + + let mut values = Vec::new(); + for descriptor in &result.partitions { + // A fresh connection per partition — the connection-level cache would miss every time. + let executor = database.new_connection().unwrap(); + values.extend(read_values(&executor, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); + + assert_eq!( + decodes.load(Ordering::SeqCst), + 1, + "shared cache should deserialize the plan once across per-task connections" + ); + + // The driver's own counter (exposed for testing/evaluation) agrees. + let count = database + .get_option_int(OptionDatabase::Other( + OPTION_PLAN_DESERIALIZE_COUNT.to_string(), + )) + .unwrap(); + assert_eq!( + count, 1, + "plan-deserialize counter should equal distinct plans" + ); +}