diff --git a/Cargo.lock b/Cargo.lock index 597be8e..3eb428d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,10 +16,12 @@ dependencies = [ "aws-config", "aws-credential-types", "datafusion", + "datafusion-proto", "datafusion-substrait", "driverbase", "futures", "log", + "lru", "object_store", "parking_lot", "prost", @@ -1923,6 +1925,44 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a387aaef949dc16bb6abc81bd1af850ec7449183aef011214f9724957495738" +dependencies = [ + "arrow", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "object_store", + "prost", + "rand 0.9.4", +] + +[[package]] +name = "datafusion-proto-common" +version = "53.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e614c7c53a9c304c6a850b821010bb492e57300311835f1180613f9d2c63d9" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + [[package]] name = "datafusion-pruning" version = "53.1.0" @@ -2360,6 +2400,11 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -2845,6 +2890,15 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +[[package]] +name = "lru" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a860605968fce16869fd239cf4237a82f3ac470723415db603b0e8b6c8d4fb9" +dependencies = [ + "hashbrown 0.17.1", +] + [[package]] name = "lru-slab" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index cb37905..1e438e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,10 +38,12 @@ async-trait = "0.1" aws-config = "1.8" aws-credential-types = "1.2" datafusion = { version = "53", default-features = false, features = ["avro", "compression", "crypto_expressions", "datetime_expressions", "nested_expressions", "parquet", "recursive_protection", "regex_expressions", "sql", "unicode_expressions"] } +datafusion-proto = { version = "53", default-features = false } datafusion-substrait = { version = "53", default-features = false } driverbase = { git = "https://github.com/adbc-drivers/driverbase-rs.git" } futures = "0.3.32" log = "0.4" +lru = "0.18" object_store = { version = "0.13", features = ["aws", "gcp", "http"] } parking_lot = "0.12" prost = "0.14.3" diff --git a/docs/datafusion.md b/docs/datafusion.md index 36e349c..5876b8d 100644 --- a/docs/datafusion.md +++ b/docs/datafusion.md @@ -86,6 +86,26 @@ SELECT `Breed Name`, `Lifespan` {{ footnotes|safe }} +### Partitioned Execution + +This driver supports ADBC's partitioned execution; one partition is generated per output partition. Each partition contains the physical plan, and can be distributed across CPUs or physical machines. If the plan contains a shuffle, however, distributed execution will likely be slower, as each partition must read the entire output. + +The statement option `datafusion.partition_mode` controls what to do if a shuffle is detected: + +| Value | Behavior | +|------------------|-------------------------------------------------------------------------------------| +| `auto` (default) | Collapse to a single partition when the plan shuffles, otherwise natural partitions | +| `multi` | One descriptor per natural output partition; logs a warning if the plan shuffles | +| `single` | Always collapse to a single partition (effectively the same as regular execution) | + +### Extending DataFusion + +It is possible to customize the embedded DataFusion by depending on this crate, then using hooks to customize the DataFusion `SessionContext` and register `PhysicalExtensionCodec`s. The customized driver can then be built into a shared library and distributed as an ADBC driver. For more, see: + +- `DataFusionDriver::new_with_context_init`, which accepts a callback to modify the `SessionContext` +- `DataFusionDriver::with_codec`, which accepts a `PhysicalExtensionCodec` used to serialize/deserialize plans in partitioned execution +- The [`adbc_ffi`](https://crates.io/crates/adbc_ffi) crate, which provides helpers to export a Rust ADBC driver as a shared library + ## Compatibility {{ compatibility_info|safe }} diff --git a/src/lib.rs b/src/lib.rs index 46d8e7c..1de941f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,8 @@ use datafusion::dataframe::DataFrameWriteOptions; use datafusion::datasource::MemTable; use datafusion::logical_expr::LogicalPlan; use datafusion::logical_expr::dml::InsertOp; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use datafusion::prelude::*; use datafusion_substrait::logical_plan::consumer::from_substrait_plan; use datafusion_substrait::substrait::proto::Plan; @@ -38,7 +40,11 @@ use prost::Message; use std::collections::HashMap; use std::fmt::Debug; use std::future::Future; +use std::num::NonZeroUsize; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use lru::LruCache; use arrow_array::{RecordBatch, RecordBatchReader}; use arrow_schema::{ArrowError, SchemaRef}; @@ -75,6 +81,11 @@ pub type ContextInit = Arc< dyn Fn(&mut SessionContext, &mut DatabaseOpts) -> datafusion::error::Result<()> + Send + Sync, >; +/// Optional codec for serializing custom physical-plan extensions (`ExecutionPlan` nodes, +/// UDFs, exprs) into partition descriptors. Without one, a plan containing a custom +/// extension fails `execute_partitions`. +pub type PhysicalCodec = Arc; + impl ErrorHelper { fn from_datafusion(err: datafusion::error::DataFusionError) -> DriverError { match err { @@ -258,6 +269,19 @@ impl DataFusionReader { schema: schema.into(), }) } + + /// Construct a reader directly from a single-partition stream. + pub(crate) fn from_stream( + runtime: Arc, + stream: datafusion::execution::SendableRecordBatchStream, + schema: SchemaRef, + ) -> Self { + Self { + runtime, + stream, + schema, + } + } } impl Iterator for DataFusionReader { @@ -278,6 +302,7 @@ impl RecordBatchReader for DataFusionReader { pub struct DataFusionDriver { handle: Option, context_init: ContextInit, + codec: Option, } impl DataFusionDriver { @@ -293,9 +318,17 @@ impl DataFusionDriver { Self { handle, context_init, + codec: None, } } + /// Register an extension codec for serializing custom physical-plan extensions + /// (`ExecutionPlan` nodes, UDFs, exprs) in partition descriptors (see [`PhysicalCodec`]). + pub fn with_codec(mut self, codec: PhysicalCodec) -> Self { + self.codec = Some(codec); + self + } + fn new_database_with_database_opts( &self, database_opts: &mut DatabaseOpts, @@ -314,6 +347,9 @@ impl DataFusionDriver { Ok(DataFusionDatabase { handle: self.handle.clone(), ctx: Arc::new(ctx), + codec: self.codec.clone(), + plan_cache: Arc::new(new_plan_cache()), + plan_deserializations: Arc::new(AtomicU64::new(0)), }) } } @@ -353,6 +389,13 @@ impl Driver for DataFusionDriver { pub struct DataFusionDatabase { handle: Option, ctx: Arc, + codec: Option, + /// Shared by every connection this database opens, so all of an executor's + /// per-task connections reuse one deserialized plan per distinct plan bytes. + plan_cache: Arc, + /// Count of plan deserializations (cache misses) across all connections. + /// See [`OPTION_PLAN_DESERIALIZE_COUNT`]; used for testing and evaluation. + plan_deserializations: Arc, } impl Optionable for DataFusionDatabase { @@ -388,7 +431,12 @@ impl Optionable for DataFusionDatabase { } fn get_option_int(&self, key: Self::Option) -> adbc_core::error::Result { - Err(ErrorHelper::get_unknown_option(&key).to_adbc()) + match key.as_ref() { + OPTION_PLAN_DESERIALIZE_COUNT => { + Ok(self.plan_deserializations.load(Ordering::Relaxed) as i64) + } + _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), + } } fn get_option_double(&self, key: Self::Option) -> adbc_core::error::Result { @@ -410,6 +458,9 @@ impl Database for DataFusionDatabase { Ok(DataFusionConnection { runtime: Arc::new(runtime), ctx: self.ctx.clone(), + codec: self.codec.clone(), + plan_cache: Arc::clone(&self.plan_cache), + plan_deserializations: Arc::clone(&self.plan_deserializations), }) } @@ -432,6 +483,9 @@ impl Database for DataFusionDatabase { let mut connection = DataFusionConnection { runtime: Arc::new(runtime), ctx: self.ctx.clone(), + codec: self.codec.clone(), + plan_cache: Arc::clone(&self.plan_cache), + plan_deserializations: Arc::clone(&self.plan_deserializations), }; for (key, value) in opts { @@ -442,9 +496,40 @@ impl Database for DataFusionDatabase { } } +/// Cache of deserialized physical plans, keyed by descriptor plan bytes. +/// +/// `read_partition` deserializes a given plan once and reuses it across every partition +/// index. The cache lives on [`DataFusionDatabase`] and is shared (via `Arc`) by every +/// connection opened from it. Since connectors open one connection per task but cache one +/// database per executor, this collapses N per-task deserializes into one per distinct plan +/// per executor. +type PlanCache = parking_lot::Mutex, Arc>>; + +/// Bound on distinct cached plans, so a database serving many different queries does not +/// accumulate plans without limit. On overflow the least-recently-used plan is evicted; +/// already-handed-out `Arc`s stay alive until their tasks finish. +const PLAN_CACHE_CAP: usize = 64; + +fn new_plan_cache() -> PlanCache { + parking_lot::Mutex::new(LruCache::new( + NonZeroUsize::new(PLAN_CACHE_CAP).expect("PLAN_CACHE_CAP is nonzero"), + )) +} + +/// Read-only integer option: number of physical plans this database has deserialized +/// from partition descriptors (i.e. plan-cache misses across all its connections). +/// +/// Intended primarily for testing and evaluation: it lets a caller confirm the plan cache +/// is working — e.g. asserting deserializations equal the number of distinct plans rather +/// than the number of partitions read. It is not part of the ADBC option contract. +pub const OPTION_PLAN_DESERIALIZE_COUNT: &str = "adbc.datafusion.plan_deserialize_count"; + pub struct DataFusionConnection { runtime: Arc, ctx: Arc, + codec: Option, + plan_cache: Arc, + plan_deserializations: Arc, } impl Optionable for DataFusionConnection { @@ -545,7 +630,12 @@ impl Optionable for DataFusionConnection { } fn get_option_int(&self, key: Self::Option) -> adbc_core::error::Result { - Err(ErrorHelper::get_unknown_option(&key).to_adbc()) + match key.as_ref() { + OPTION_PLAN_DESERIALIZE_COUNT => { + Ok(self.plan_deserializations.load(Ordering::Relaxed) as i64) + } + _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), + } } fn get_option_double(&self, key: Self::Option) -> adbc_core::error::Result { @@ -583,6 +673,8 @@ impl Connection for DataFusionConnection { Ok(DataFusionStatement { runtime: self.runtime.clone(), ctx: self.ctx.clone(), + codec: self.codec.clone(), + partition_mode: PartitionMode::default(), query: None, bound: None, ingest: BulkIngestState::new(), @@ -678,11 +770,52 @@ impl Connection for DataFusionConnection { fn read_partition( &self, - _partition: impl AsRef<[u8]>, + partition: impl AsRef<[u8]>, ) -> Result> { - Err(ErrorHelper::not_implemented() - .message("read_partition") - .to_adbc()) + let (index, plan_bytes) = decode_descriptor(partition.as_ref())?; + self.runtime.block_on(async { + let task_ctx = self.ctx.task_ctx(); + let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; + let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = + self.codec.as_deref().unwrap_or(&default_codec); + // Cache the deserialized plan per distinct payload. The cache is shared across + // all connections from this database, so every task reading a partition of the + // same plan pays the proto decode once, not once per task. + let physical = { + let mut cache = self.plan_cache.lock(); + if let Some(plan) = cache.get(plan_bytes.as_slice()) { + plan.clone() + } else { + let plan = + datafusion_proto::bytes::physical_plan_from_bytes_with_extension_codec( + &plan_bytes, + task_ctx.as_ref(), + codec, + ) + .map_err(ErrorHelper::from_datafusion)?; + self.plan_deserializations.fetch_add(1, Ordering::Relaxed); + cache.put(plan_bytes.clone(), plan.clone()); + plan + } + }; + let n = physical.output_partitioning().partition_count(); + if index as usize >= n { + return Err(ErrorHelper::invalid_argument() + .format(format_args!( + "partition index {index} out of range (plan has {n} partitions)" + )) + .to_adbc()); + } + let schema = physical.schema(); + let stream = physical + .execute(index as usize, task_ctx) + .map_err(ErrorHelper::from_datafusion)?; + Ok(Box::new(DataFusionReader::from_stream( + self.runtime.clone(), + stream, + schema, + )) as Box) + }) } } @@ -714,9 +847,121 @@ impl QueryState { } } +/// Partition-descriptor format version. The same driver build produces and consumes +/// descriptors, so the byte exists only to reject a payload from a different layout. +const DESCRIPTOR_VERSION: u8 = 1; + +/// Fixed descriptor header length: `version(1) + index(4)`. +const DESCRIPTOR_HEADER_LEN: usize = 5; + +/// Warn when a serialized plan exceeds this size: the plan is copied into all N descriptors, +/// so a large payload is paid N times. Operator-only plans are a few KB; this mainly catches +/// plans that inline data. +const PROTO_DESCRIPTOR_WARN_BYTES: usize = 8 << 20; // 8 MiB + +/// Build a self-contained partition descriptor: +/// `[u8 version][u32 LE index][serialized physical plan...]`. +/// +/// The payload is the full `datafusion-proto`-serialized physical plan; `read_partition` +/// deserializes it and executes the given partition index. +fn encode_descriptor(index: u32, plan_bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(DESCRIPTOR_HEADER_LEN + plan_bytes.len()); + out.push(DESCRIPTOR_VERSION); + out.extend_from_slice(&index.to_le_bytes()); + out.extend_from_slice(plan_bytes); + out +} + +/// Inverse of [`encode_descriptor`]. Returns `(index, plan_bytes)`. +fn decode_descriptor(bytes: &[u8]) -> adbc_core::error::Result<(u32, Vec)> { + if bytes.len() < DESCRIPTOR_HEADER_LEN { + return Err(ErrorHelper::invalid_argument() + .message("short partition descriptor") + .to_adbc()); + } + let version = bytes[0]; + if version != DESCRIPTOR_VERSION { + return Err(ErrorHelper::invalid_argument() + .format(format_args!("unsupported descriptor version {version}")) + .to_adbc()); + } + let index = u32::from_le_bytes(bytes[1..5].try_into().unwrap()); + let plan_bytes = bytes[DESCRIPTOR_HEADER_LEN..].to_vec(); + Ok((index, plan_bytes)) +} + +/// Statement option selecting the `execute_partitions` strategy. String value, one of +/// `auto`, `single`, or `multi` (see [`PartitionMode`]). +const OPTION_PARTITION_MODE: &str = "datafusion.partition_mode"; + +/// How `execute_partitions` splits a query into descriptors. +/// +/// Producing one output partition of a plan that *shuffles* (a hash repartition for joins +/// and grouped aggregates, or a sort) requires reading every input partition. Executing +/// each such partition independently — the point of partitioned execution — therefore +/// re-runs the whole pre-shuffle pipeline once per partition and is usually slower than a +/// single execution. This mode controls how that case is handled. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +enum PartitionMode { + /// One descriptor per natural output partition; warn when the plan shuffles. Preserves + /// the partition count the caller's plan produced even when that is slower. + Multi, + /// Collapse to a single coalesced partition when the plan shuffles, otherwise one + /// descriptor per natural output partition. Default — never silently re-runs a shuffle + /// once per partition. + #[default] + Auto, + /// Always collapse to a single coalesced partition. This yields the same one-partition + /// result as calling [`Statement::execute`] directly, just delivered through the + /// partition API; prefer `execute` unless the caller is built around + /// `execute_partitions`/`read_partition`. + Single, +} + +impl PartitionMode { + fn from_option(value: &str) -> adbc_core::error::Result { + match value { + "multi" => Ok(Self::Multi), + "auto" => Ok(Self::Auto), + "single" => Ok(Self::Single), + other => Err(ErrorHelper::invalid_argument() + .format(format_args!( + "unknown {OPTION_PARTITION_MODE} '{other}' (expected auto, single, or multi)" + )) + .to_adbc()), + } + } + + fn as_option(self) -> &'static str { + match self { + Self::Multi => "multi", + Self::Auto => "auto", + Self::Single => "single", + } + } +} + +/// True if executing a single output partition of `plan` would re-read all input partitions, +/// i.e. the plan contains a repartition or sort shuffle. Such plans are poor candidates for +/// distributed partitioned execution (each partition re-runs the pre-shuffle pipeline). +fn plan_has_shuffle(plan: &Arc) -> bool { + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::sorts::sort::SortExec; + use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; + + let node = plan.as_any(); + if node.is::() || node.is::() || node.is::() + { + return true; + } + plan.children().into_iter().any(plan_has_shuffle) +} + pub struct DataFusionStatement { runtime: Arc, ctx: Arc, + codec: Option, + partition_mode: PartitionMode, query: Option, bound: Option>, ingest: BulkIngestState, @@ -744,6 +989,15 @@ impl Optionable for DataFusionStatement { .message("temporary tables are not supported") .to_adbc()), }, + OPTION_PARTITION_MODE => match value { + OptionValue::String(v) => { + self.partition_mode = PartitionMode::from_option(&v)?; + Ok(()) + } + _ => Err(ErrorHelper::set_invalid_option(&key, &value) + .message("must be a string") + .to_adbc()), + }, _ => Err(ErrorHelper::set_unknown_option(&key).to_adbc()), } } @@ -756,6 +1010,7 @@ impl Optionable for DataFusionStatement { .format(format_args!("{key:?} has not been set")) .to_adbc()), }, + OPTION_PARTITION_MODE => Ok(self.partition_mode.as_option().to_string()), _ => Err(ErrorHelper::get_unknown_option(&key).to_adbc()), } } @@ -969,7 +1224,7 @@ impl Statement for DataFusionStatement { Some(q) => q.execute(&self.ctx).await?, None => { return Err(ErrorHelper::invalid_state() - .message("no query or Substrait plan has been set") + .message("no query has been set") .to_adbc()); } }; @@ -995,7 +1250,7 @@ impl Statement for DataFusionStatement { Some(q) => q.execute(&self.ctx).await?, None => { return Err(ErrorHelper::invalid_state() - .message("no query or Substrait plan has been set") + .message("no query has been set") .to_adbc()); } }; @@ -1031,9 +1286,70 @@ impl Statement for DataFusionStatement { } fn execute_partitions(&mut self) -> adbc_core::error::Result { - Err(ErrorHelper::not_implemented() - .message("execute_partitions") - .to_adbc()) + let query = self.query.as_ref().ok_or_else(|| { + ErrorHelper::invalid_state() + .message("no query has been set") + .to_adbc() + })?; + self.runtime.block_on(async { + // query.execute also registers the plan's object stores, required before execution. + let df = query.execute(&self.ctx).await?; + let schema = df.schema().as_arrow().clone(); + let physical = df + .create_physical_plan() + .await + .map_err(ErrorHelper::from_datafusion)?; + let natural_n = physical.output_partitioning().partition_count() as u32; + + // A plan that shuffles re-runs its pre-shuffle pipeline once per partition when + // executed partition-by-partition. Depending on partition_mode, collapse such a + // plan to a single coalesced partition, or keep the natural partitions and warn. + let shuffle = plan_has_shuffle(&physical); + let collapse = matches!(self.partition_mode, PartitionMode::Single) + || (matches!(self.partition_mode, PartitionMode::Auto) && shuffle); + + let physical = if collapse && natural_n > 1 { + Arc::new(CoalescePartitionsExec::new(physical)) as Arc + } else { + physical + }; + let n = physical.output_partitioning().partition_count() as u32; + + if shuffle && !collapse && n > 1 { + log::warn!( + "execute_partitions: plan contains a shuffle (repartition/sort), so each of \ + the {n} partitions re-runs the pre-shuffle pipeline; this is often slower \ + than a single execution. Set {OPTION_PARTITION_MODE}=auto or =single to \ + coalesce into one partition." + ); + } + + // Serialize the physical plan into each descriptor. The default codec covers + // built-in nodes, a registered codec also covers custom ones; a node neither + // can encode fails the query. + let default_codec = datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec {}; + let codec: &dyn datafusion_proto::physical_plan::PhysicalExtensionCodec = + self.codec.as_deref().unwrap_or(&default_codec); + let plan_bytes = datafusion_proto::bytes::physical_plan_to_bytes_with_extension_codec( + physical, codec, + ) + .map_err(ErrorHelper::from_datafusion)?; + if plan_bytes.len() > PROTO_DESCRIPTOR_WARN_BYTES { + // The full plan is copied into each of the N descriptors; warn when large. + log::warn!( + "partition descriptor carries a {}-byte plan across {n} partitions", + plan_bytes.len(), + ); + } + let partitions = (0..n) + .map(|i| encode_descriptor(i, &plan_bytes)) + .collect::>(); + Ok(adbc_core::PartitionedResult { + partitions, + schema, + rows_affected: -1, + }) + }) } fn get_parameter_schema(&self) -> adbc_core::error::Result { @@ -1148,3 +1464,29 @@ impl Statement for DataFusionStatement { #[cfg(feature = "ffi")] adbc_ffi::export_driver!(AdbcDriverDatafusionInit, DataFusionDriver); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn descriptor_round_trip() { + let plan_bytes = vec![9u8, 8, 7, 6]; + let bytes = encode_descriptor(2, &plan_bytes); + let (index, payload) = decode_descriptor(&bytes).unwrap(); + assert_eq!(index, 2); + assert_eq!(payload, plan_bytes); + } + + #[test] + fn decode_rejects_short_descriptor() { + assert!(decode_descriptor(&[DESCRIPTOR_VERSION; DESCRIPTOR_HEADER_LEN - 1]).is_err()); + } + + #[test] + fn decode_rejects_unknown_version() { + let mut bytes = encode_descriptor(0, b"x"); + bytes[0] = DESCRIPTOR_VERSION + 1; + assert!(decode_descriptor(&bytes).is_err()); + } +} diff --git a/tests/test_datafusion.rs b/tests/test_datafusion.rs index 8753d72..47195ce 100644 --- a/tests/test_datafusion.rs +++ b/tests/test_datafusion.rs @@ -422,3 +422,251 @@ async fn test_running_in_async() { assert_eq!(batch.num_rows(), 3); assert_eq!(batch.num_columns(), 2); } + +/// A single-partition keyed table; a `GROUP BY k` over it hash-repartitions to +/// `target_partitions`, giving a plan whose output partition count is driven by config. +fn keyed_table(n_keys: i32) -> datafusion::error::Result { + let schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let keys: ArrayRef = Arc::new(Int32Array::from((0..n_keys).collect::>())); + let vals: ArrayRef = Arc::new(Int32Array::from( + (0..n_keys).map(|i| i * 10).collect::>(), + )); + let batch = RecordBatch::try_new(schema.clone(), vec![keys, vals])?; + MemTable::try_new(schema, vec![vec![batch]]) +} + +/// A connection whose session pins `target_partitions` to `n` and has table `t` registered. +fn connection_with_target_partitions(n: usize) -> DataFusionConnection { + let init: ContextInit = Arc::new(move |ctx, _opts| { + let config = SessionConfig::new() + .with_information_schema(true) + .with_target_partitions(n); + *ctx = SessionContext::new_with_config(config); + ctx.register_table("t", Arc::new(keyed_table(8)?))?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + let database = driver.new_database().unwrap(); + database.new_connection().unwrap() +} + +const GROUP_BY_QUERY: &str = "SELECT k, sum(v) AS s FROM t GROUP BY k"; + +/// Collect the `k` column of every row produced by reading the given descriptor. +fn read_partition_keys(connection: &DataFusionConnection, descriptor: &[u8]) -> Vec { + let reader = connection.read_partition(descriptor).unwrap(); + let mut keys = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..col.len() { + keys.push(col.value(i)); + } + } + keys +} + +#[test] +fn test_execute_partitions_returns_one_descriptor_per_output_partition() { + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + // GROUP_BY_QUERY shuffles, so the default `auto` mode would collapse it; force `multi` + // to exercise one descriptor per natural partition. + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + + // Hash-repartitioned aggregate => one ADBC partition per target partition (4). + assert_eq!( + result.partitions.len(), + 4, + "expected one descriptor per target partition, got {}", + result.partitions.len() + ); + assert_eq!(result.rows_affected, -1); + + // The union of all partitions reproduces every key exactly once. + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_read_partition_executes_serialized_plan() { + // Plan with target_partitions = 4. + let mut planner = connection_with_target_partitions(4); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); + let result = statement.execute_partitions().unwrap(); + assert!(result.partitions.len() > 1); + + // Execute on a connection whose default target_partitions differs (1). The descriptor + // carries the already-built physical plan, so its partitioning is fixed regardless of the + // executor's config — partition `i` stays meaningful and in range. + let executor = connection_with_target_partitions(1); + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&executor, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_execute_partitions_prepared() { + // The proto path serializes the built physical plan regardless of source, so a prepared + // statement partitions like any other query. + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("multi".to_string()), + ) + .unwrap(); + statement.prepare().unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert!(result.partitions.len() > 1); + + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +const PARTITION_MODE: &str = "datafusion.partition_mode"; + +/// A connection with a two-partition in-memory table `p` (keys 0..4 and 4..8) and no +/// repartitioning forced, so `SELECT * FROM p` is a shuffle-free two-partition scan. +fn connection_with_two_partition_table() -> DataFusionConnection { + let init: ContextInit = Arc::new(|ctx, _opts| { + let schema: SchemaRef = Arc::new(Schema::new(vec![ + Field::new("k", DataType::Int32, false), + Field::new("v", DataType::Int32, false), + ])); + let part = |lo: i32, hi: i32| { + let keys: ArrayRef = Arc::new(Int32Array::from((lo..hi).collect::>())); + let vals: ArrayRef = Arc::new(Int32Array::from( + (lo..hi).map(|i| i * 10).collect::>(), + )); + RecordBatch::try_new(schema.clone(), vec![keys, vals]).unwrap() + }; + // Two batch groups => two table partitions. + let table = MemTable::try_new(schema.clone(), vec![vec![part(0, 4)], vec![part(4, 8)]])?; + ctx.register_table("p", Arc::new(table))?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + let database = driver.new_database().unwrap(); + database.new_connection().unwrap() +} + +#[test] +fn test_partition_mode_single_collapses_to_one() { + // GROUP_BY_QUERY hash-repartitions to 4 partitions; single mode must coalesce to one. + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("single".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), 1, "single mode => one descriptor"); + + // The lone descriptor reproduces every key — the coalesced plan runs the whole query. + let mut keys = read_partition_keys(&connection, &result.partitions[0]); + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_partition_mode_auto_collapses_shuffle() { + // Aggregate has a shuffle => auto coalesces to one partition. + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(GROUP_BY_QUERY).unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("auto".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + 1, + "auto mode collapses a shuffling plan to one partition" + ); +} + +#[test] +fn test_partition_mode_auto_keeps_shuffle_free_partitions() { + // A plain two-partition scan has no shuffle => auto keeps the natural partitions. + let mut connection = connection_with_two_partition_table(); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query("SELECT k, v FROM p").unwrap(); + statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("auto".to_string()), + ) + .unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + 2, + "auto keeps natural partitions when the plan does not shuffle" + ); + + let mut keys = Vec::new(); + for descriptor in &result.partitions { + keys.extend(read_partition_keys(&connection, descriptor)); + } + keys.sort(); + assert_eq!(keys, (0..8).collect::>()); +} + +#[test] +fn test_partition_mode_rejects_unknown_value() { + let mut connection = connection_with_target_partitions(4); + let mut statement = connection.new_statement().unwrap(); + let err = statement + .set_option( + OptionStatement::Other(PARTITION_MODE.to_string()), + OptionValue::String("nonsense".to_string()), + ) + .unwrap_err(); + assert_eq!(err.status, adbc_core::error::Status::InvalidArguments); +} diff --git a/tests/test_proto_partitions.rs b/tests/test_proto_partitions.rs new file mode 100644 index 0000000..082d1f3 --- /dev/null +++ b/tests/test_proto_partitions.rs @@ -0,0 +1,434 @@ +// Copyright (c) 2026 ADBC Drivers Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Proto path tests: a custom `TableProvider` emits a custom `ExecutionPlan`, and a +//! `PhysicalExtensionCodec` round-trips it. `execute_partitions` serializes the physical +//! plan; a *fresh* connection's `read_partition` reproduces the rows by deserializing the +//! plan — the descriptor carries only the serialized plan, so re-planning is structurally +//! impossible and a passing test proves the deserialize path. Also covers the hard error +//! when a custom node cannot be encoded (no re-plan fallback). + +use std::any::Any; +use std::fmt; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use adbc_core::options::OptionDatabase; +use adbc_core::{Connection, Database, Driver, Optionable, Statement}; +use adbc_driver_datafusion::{ + ContextInit, DataFusionConnection, DataFusionDatabase, DataFusionDriver, + OPTION_PLAN_DESERIALIZE_COUNT, PhysicalCodec, +}; +use arrow_array::{ArrayRef, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use async_trait::async_trait; +use datafusion::catalog::{Session, TableProvider}; +use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::execution::TaskContext; +use datafusion::logical_expr::{Expr, TableType}; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::memory::MemoryStream; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, + SendableRecordBatchStream, +}; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; + +/// Single `Int32` column named `v`. +fn schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new("v", DataType::Int32, false)])) +} + +/// A custom scan node: holds the full data and a partition count; partition `i` emits a +/// round-robin slice (`data[i], data[i + n], ...`). Carries everything needed to rebuild +/// itself from bytes, so it survives proto round-trip via [`CustomCodec`]. +#[derive(Debug)] +struct CustomExec { + data: Vec, + partitions: usize, + schema: SchemaRef, + props: Arc, +} + +impl CustomExec { + fn new(data: Vec, partitions: usize) -> Self { + let schema = schema(); + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Self { + data, + partitions, + schema, + props, + } + } +} + +impl DisplayAs for CustomExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "CustomExec(partitions={})", self.partitions) + } +} + +impl ExecutionPlan for CustomExec { + fn name(&self) -> &str { + "CustomExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> DFResult> { + Ok(self) + } + + fn execute( + &self, + partition: usize, + _context: Arc, + ) -> DFResult { + let mut vals = Vec::new(); + let mut i = partition; + while i < self.data.len() { + vals.push(self.data[i]); + i += self.partitions; + } + let col: ArrayRef = Arc::new(Int32Array::from(vals)); + let batch = RecordBatch::try_new(self.schema.clone(), vec![col])?; + Ok(Box::pin(MemoryStream::try_new( + vec![batch], + self.schema.clone(), + None, + )?)) + } +} + +/// A table provider whose `scan` returns a [`CustomExec`]. +#[derive(Debug)] +struct CustomProvider { + data: Vec, + partitions: usize, +} + +#[async_trait] +impl TableProvider for CustomProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + schema() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + _projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> DFResult> { + Ok(Arc::new(CustomExec::new( + self.data.clone(), + self.partitions, + ))) + } +} + +/// Codec for [`CustomExec`]: encodes `partitions` then the data as little-endian i32s. +#[derive(Debug)] +struct CustomCodec; + +impl PhysicalExtensionCodec for CustomCodec { + fn try_decode( + &self, + buf: &[u8], + _inputs: &[Arc], + _ctx: &TaskContext, + ) -> DFResult> { + if buf.len() < 8 { + return Err(DataFusionError::Internal("short CustomExec buffer".into())); + } + let partitions = u64::from_le_bytes(buf[0..8].try_into().unwrap()) as usize; + let data: Vec = buf[8..] + .chunks_exact(4) + .map(|c| i32::from_le_bytes(c.try_into().unwrap())) + .collect(); + Ok(Arc::new(CustomExec::new(data, partitions))) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> DFResult<()> { + let exec = node + .as_any() + .downcast_ref::() + .ok_or_else(|| DataFusionError::Internal("not a CustomExec".into()))?; + buf.extend_from_slice(&(exec.partitions as u64).to_le_bytes()); + for v in &exec.data { + buf.extend_from_slice(&v.to_le_bytes()); + } + Ok(()) + } +} + +/// A codec that refuses to encode, to exercise the hard error when a custom node cannot +/// be serialized (no re-plan fallback). +#[derive(Debug)] +struct FailingCodec; + +impl PhysicalExtensionCodec for FailingCodec { + fn try_decode( + &self, + _buf: &[u8], + _inputs: &[Arc], + _ctx: &TaskContext, + ) -> DFResult> { + Err(DataFusionError::Internal("decode unsupported".into())) + } + + fn try_encode(&self, _node: Arc, _buf: &mut Vec) -> DFResult<()> { + Err(DataFusionError::Internal("encode unsupported".into())) + } +} + +/// Delegates to [`CustomCodec`] but counts `try_decode` calls, to prove the shared +/// plan cache deserializes a given plan only once. +#[derive(Debug)] +struct CountingCodec { + decodes: Arc, +} + +impl PhysicalExtensionCodec for CountingCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + ) -> DFResult> { + self.decodes.fetch_add(1, Ordering::SeqCst); + CustomCodec.try_decode(buf, inputs, ctx) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> DFResult<()> { + CustomCodec.try_encode(node, buf) + } +} + +const VALUES: i32 = 8; +const N_PARTITIONS: usize = 3; +const QUERY: &str = "SELECT v FROM ct"; + +/// Build a database whose `ct` table is the custom provider, with the given codec (if any). +fn custom_database(codec: Option) -> DataFusionDatabase { + let init: ContextInit = Arc::new(|ctx, _opts| { + ctx.register_table( + "ct", + Arc::new(CustomProvider { + data: (0..VALUES).collect(), + partitions: N_PARTITIONS, + }), + )?; + Ok(()) + }); + let mut driver = DataFusionDriver::new_with_context_init(None, init); + if let Some(codec) = codec { + driver = driver.with_codec(codec); + } + driver.new_database().unwrap() +} + +/// Build a connection whose `ct` table is the custom provider, with the given codec (if any). +fn custom_connection(codec: Option) -> DataFusionConnection { + custom_database(codec).new_connection().unwrap() +} + +/// Read every `v` value produced by the descriptor. +fn read_values(connection: &DataFusionConnection, descriptor: &[u8]) -> Vec { + let reader = connection.read_partition(descriptor).unwrap(); + let mut out = Vec::new(); + for batch in reader { + let batch = batch.unwrap(); + let col = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..col.len() { + out.push(col.value(i)); + } + } + out +} + +#[test] +fn proto_round_trips_custom_plan_without_replanning() { + let mut planner = custom_connection(Some(Arc::new(CustomCodec))); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + let result = statement.execute_partitions().unwrap(); + assert_eq!( + result.partitions.len(), + N_PARTITIONS, + "one descriptor per custom output partition" + ); + + // A FRESH connection — also codec-equipped, as a separate executor would be — reproduces + // the rows purely from the serialized plan (no SQL string rides the descriptor). + let executor = custom_connection(Some(Arc::new(CustomCodec))); + let mut values = Vec::new(); + for descriptor in &result.partitions { + values.extend(read_values(&executor, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); +} + +#[test] +fn custom_node_without_codec_errors() { + // No codec: the default codec cannot encode the custom node, so execute_partitions fails + // rather than silently falling back to a re-plan path. + let mut connection = custom_connection(None); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + assert!( + statement.execute_partitions().is_err(), + "custom node + no codec must error, not fall back" + ); +} + +#[test] +fn codec_encode_failure_errors() { + // The codec can't encode this plan; execute_partitions errors the whole query (no + // re-plan fallback). + let mut connection = custom_connection(Some(Arc::new(FailingCodec))); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + + assert!( + statement.execute_partitions().is_err(), + "failed proto encode must error, not fall back" + ); +} + +#[test] +fn read_partition_rejects_out_of_range_index() { + let mut planner = custom_connection(Some(Arc::new(CustomCodec))); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + + // Corrupt the index (bytes 1..5) to an out-of-range partition. + let mut bad = result.partitions[0].clone(); + bad[1..5].copy_from_slice(&u32::to_le_bytes(99)); + + let executor = custom_connection(Some(Arc::new(CustomCodec))); + let err = match executor.read_partition(&bad) { + Ok(_) => panic!("expected out-of-range index to be rejected"), + Err(e) => e, + }; + assert_eq!(err.status, adbc_core::error::Status::InvalidArguments); +} + +#[test] +fn plan_cache_deserializes_once_across_partitions() { + let decodes = Arc::new(AtomicUsize::new(0)); + let codec: PhysicalCodec = Arc::new(CountingCodec { + decodes: decodes.clone(), + }); + let mut connection = custom_connection(Some(codec)); + let mut statement = connection.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), N_PARTITIONS); + + // Read every partition on the SAME connection: the plan is identical across them, so + // the cache should deserialize it once and reuse it for the rest. + let mut values = Vec::new(); + for descriptor in &result.partitions { + values.extend(read_values(&connection, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); + + assert_eq!( + decodes.load(Ordering::SeqCst), + 1, + "plan should be deserialized once and cached across partition indices" + ); +} + +#[test] +fn plan_cache_is_shared_across_connections_from_one_database() { + // Mirrors the connector's real shape: plan on one connection, then read each partition + // on a SEPARATE connection (as per-task connections would). All connections come from + // one database, so the database-scoped cache should still deserialize the plan once. + let decodes = Arc::new(AtomicUsize::new(0)); + let codec: PhysicalCodec = Arc::new(CountingCodec { + decodes: decodes.clone(), + }); + let database = custom_database(Some(codec)); + + let mut planner = database.new_connection().unwrap(); + let mut statement = planner.new_statement().unwrap(); + statement.set_sql_query(QUERY).unwrap(); + let result = statement.execute_partitions().unwrap(); + assert_eq!(result.partitions.len(), N_PARTITIONS); + + let mut values = Vec::new(); + for descriptor in &result.partitions { + // A fresh connection per partition — the connection-level cache would miss every time. + let executor = database.new_connection().unwrap(); + values.extend(read_values(&executor, descriptor)); + } + values.sort(); + assert_eq!(values, (0..VALUES).collect::>()); + + assert_eq!( + decodes.load(Ordering::SeqCst), + 1, + "shared cache should deserialize the plan once across per-task connections" + ); + + // The driver's own counter (exposed for testing/evaluation) agrees. + let count = database + .get_option_int(OptionDatabase::Other( + OPTION_PLAN_DESERIALIZE_COUNT.to_string(), + )) + .unwrap(); + assert_eq!( + count, 1, + "plan-deserialize counter should equal distinct plans" + ); +}