From 8228957519fb85039ba61f091469b9fb40e4605f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 19 Nov 2026 18:22:39 +0100 Subject: [PATCH 1/2] Do not propagate config extension options that have their default value Workers rebuild config extensions from their Default implementation, so options left at their default do not need to travel in the headers of every task. Adds a config_propagation benchmark. --- datafusion/distributed-benchmarks/Cargo.toml | 4 + .../benches/config_propagation.rs | 198 ++++++++++++++++++ .../distributed/src/config_extension_ext.rs | 75 ++++--- 3 files changed, 245 insertions(+), 32 deletions(-) create mode 100644 datafusion/distributed-benchmarks/benches/config_propagation.rs diff --git a/datafusion/distributed-benchmarks/Cargo.toml b/datafusion/distributed-benchmarks/Cargo.toml index 796ff66b5..9d4949ec8 100644 --- a/datafusion/distributed-benchmarks/Cargo.toml +++ b/datafusion/distributed-benchmarks/Cargo.toml @@ -70,3 +70,7 @@ harness = false [[bench]] name = "over_distribution" harness = false + +[[bench]] +name = "config_propagation" +harness = false diff --git a/datafusion/distributed-benchmarks/benches/config_propagation.rs b/datafusion/distributed-benchmarks/benches/config_propagation.rs new file mode 100644 index 000000000..7359e6060 --- /dev/null +++ b/datafusion/distributed-benchmarks/benches/config_propagation.rs @@ -0,0 +1,198 @@ +//! # Config propagation +//! +//! ```shell +//! cargo bench -p datafusion-distributed-benchmarks --bench config_propagation -- --save-baseline main +//! cargo bench -p datafusion-distributed-benchmarks --bench config_propagation -- --baseline main +//! ``` +//! +//! Every task the coordinator sends to a worker carries the session's config extensions as gRPC +//! headers, which the worker parses back into its own session. This runs a query that fans out to +//! many tasks over localhost workers, each producing a single row, so the time is dominated by +//! dispatching tasks. To stress the headers, the session carries a user config extension next to +//! `DistributedConfig` with large string options, integer, float and boolean options, and options +//! with long names. `defaults` leaves every option at its default; `few_overrides` changes a +//! handful of options of each kind, as a typical session would. +//! An untimed preflight reports the config headers a worker received per task. + +use criterion::{Criterion, criterion_group, criterion_main}; +use datafusion::common::extensions_options; +use datafusion::config::{ConfigExtension, ExtensionOptions}; +use datafusion::error::Result; +use datafusion::execution::SessionState; +use datafusion::physical_plan::execute_stream; +use datafusion::prelude::SessionContext; +use datafusion_distributed::test_utils::localhost::start_localhost_context; +use datafusion_distributed::test_utils::routing::{ + URLEmitterExtensionCodec, URLEmitterFunction, url_emitter_desired_task_count, + url_emitter_scale_up_leaf_node, +}; +use datafusion_distributed::{DistributedExt, WorkerQueryContext}; +use futures::TryStreamExt; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tokio::runtime::Builder as RuntimeBuilder; + +const NUM_WORKERS: usize = 32; +const QUERY: &str = "SELECT * FROM url_emitter(128, 128, 'config_propagation')"; +const CONFIG_HEADER_PREFIX: &str = "x-datafusion-distributed-config-"; +/// Size of each [LargeOptions] string value, both default and overridden. With every option +/// sent, the config headers stay just under the 16 KiB HTTP/2 header list limit that hyper +/// applies by default; beyond it the worker rejects the task. +const STRING_BYTES: usize = 1024; +/// Shared name of the [LargeOptions] options with long keys, followed by their index. +const LONG_KEY: &str = "option_with_a_deliberately_long_name_to_stress_the_header_keys"; + +/// Scenario name and how many [LargeOptions] options of each kind are changed from their default. +const SCENARIOS: [(&str, usize); 2] = [("defaults", 0), ("few_overrides", 4)]; + +fn config_propagation(c: &mut Criterion) { + let rt = RuntimeBuilder::new_multi_thread() + .enable_all() + .build() + .expect("tokio runtime"); + let mut group = c.benchmark_group("config_propagation"); + group.sample_size(10); + group.warm_up_time(Duration::from_secs(1)); + + for (scenario, overridden) in SCENARIOS { + // (headers, bytes) of the config headers received by the last task a worker started. + let received = Arc::new(Mutex::new((0, 0))); + let worker_received = Arc::clone(&received); + let worker = move |ctx: WorkerQueryContext| { + let received = Arc::clone(&worker_received); + async move { build_state(ctx, &received) } + }; + + let (mut ctx, _workers, _) = rt.block_on(start_localhost_context(NUM_WORKERS, worker)); + ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); + ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); + ctx.set_distributed_user_codec(URLEmitterExtensionCodec); + ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); + let mut options = LargeOptions::default(); + for i in 0..overridden { + let string = "y".repeat(STRING_BYTES); + options.set(&format!("string_{i:02}"), &string).unwrap(); + options.set(&format!("integer_{i:02}"), "1").unwrap(); + options.set(&format!("float_{i:02}"), "1.5").unwrap(); + options.set(&format!("boolean_{i:02}"), "true").unwrap(); + options.set(&format!("{LONG_KEY}_{i:02}"), "1").unwrap(); + } + ctx.set_distributed_option_extension(options); + + let rows = rt.block_on(run_query(&ctx)).unwrap(); + let (headers, bytes) = *received.lock().unwrap(); + println!("preflight {scenario}: rows={rows} config_headers={headers} bytes={bytes}"); + + group.bench_function(scenario, |b| { + b.iter(|| rt.block_on(run_query(&ctx)).unwrap()) + }); + } + group.finish(); +} + +/// Plans and executes the query, draining the output without keeping it, and returns its row +/// count. +async fn run_query(ctx: &SessionContext) -> Result { + let plan = ctx.sql(QUERY).await?.create_physical_plan().await?; + execute_stream(plan, ctx.task_ctx())? + .try_fold(0, |rows, batch| async move { Ok(rows + batch.num_rows()) }) + .await +} + +fn build_state(ctx: WorkerQueryContext, received: &Mutex<(usize, usize)>) -> Result { + let config_headers = ctx + .headers + .iter() + .filter(|(k, _)| k.as_str().starts_with(CONFIG_HEADER_PREFIX)); + *received.lock().unwrap() = config_headers.fold((0, 0), |(headers, bytes), (k, v)| { + (headers + 1, bytes + k.as_str().len() + v.len()) + }); + Ok(ctx + .builder + .with_distributed_user_codec(URLEmitterExtensionCodec) + .with_distributed_option_extension_from_headers::(&ctx.headers)? + .build()) +} + +extensions_options! { + /// A user config extension with many options of every kind, propagated to workers on every task. + pub struct LargeOptions { + pub string_00: String, default = "x".repeat(STRING_BYTES) + pub string_01: String, default = "x".repeat(STRING_BYTES) + pub string_02: String, default = "x".repeat(STRING_BYTES) + pub string_03: String, default = "x".repeat(STRING_BYTES) + pub string_04: String, default = "x".repeat(STRING_BYTES) + pub string_05: String, default = "x".repeat(STRING_BYTES) + pub integer_00: usize, default = 0 + pub integer_01: usize, default = 0 + pub integer_02: usize, default = 0 + pub integer_03: usize, default = 0 + pub integer_04: usize, default = 0 + pub integer_05: usize, default = 0 + pub integer_06: usize, default = 0 + pub integer_07: usize, default = 0 + pub integer_08: usize, default = 0 + pub integer_09: usize, default = 0 + pub integer_10: usize, default = 0 + pub integer_11: usize, default = 0 + pub integer_12: usize, default = 0 + pub integer_13: usize, default = 0 + pub integer_14: usize, default = 0 + pub integer_15: usize, default = 0 + pub float_00: f64, default = 0.0 + pub float_01: f64, default = 0.0 + pub float_02: f64, default = 0.0 + pub float_03: f64, default = 0.0 + pub float_04: f64, default = 0.0 + pub float_05: f64, default = 0.0 + pub float_06: f64, default = 0.0 + pub float_07: f64, default = 0.0 + pub float_08: f64, default = 0.0 + pub float_09: f64, default = 0.0 + pub float_10: f64, default = 0.0 + pub float_11: f64, default = 0.0 + pub float_12: f64, default = 0.0 + pub float_13: f64, default = 0.0 + pub float_14: f64, default = 0.0 + pub float_15: f64, default = 0.0 + pub boolean_00: bool, default = false + pub boolean_01: bool, default = false + pub boolean_02: bool, default = false + pub boolean_03: bool, default = false + pub boolean_04: bool, default = false + pub boolean_05: bool, default = false + pub boolean_06: bool, default = false + pub boolean_07: bool, default = false + pub boolean_08: bool, default = false + pub boolean_09: bool, default = false + pub boolean_10: bool, default = false + pub boolean_11: bool, default = false + pub boolean_12: bool, default = false + pub boolean_13: bool, default = false + pub boolean_14: bool, default = false + pub boolean_15: bool, default = false + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_00: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_01: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_02: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_03: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_04: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_05: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_06: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_07: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_08: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_09: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_10: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_11: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_12: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_13: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_14: usize, default = 0 + pub option_with_a_deliberately_long_name_to_stress_the_header_keys_15: usize, default = 0 + } +} + +impl ConfigExtension for LargeOptions { + const PREFIX: &'static str = "large"; +} + +criterion_group!(benches, config_propagation); +criterion_main!(benches); diff --git a/datafusion/distributed/src/config_extension_ext.rs b/datafusion/distributed/src/config_extension_ext.rs index 6776d463e..b59a1922c 100644 --- a/datafusion/distributed/src/config_extension_ext.rs +++ b/datafusion/distributed/src/config_extension_ext.rs @@ -1,4 +1,4 @@ -use datafusion::common::{DataFusionError, HashSet, internal_datafusion_err}; +use datafusion::common::{DataFusionError, HashMap, HashSet, internal_datafusion_err}; use datafusion::config::ConfigExtension; use datafusion::prelude::SessionConfig; use http::{HeaderMap, HeaderName}; @@ -13,12 +13,7 @@ pub(crate) fn set_distributed_option_extension( t: T, ) { cfg.options_mut().extensions.insert(t); - let mut propagation_ctx = match cfg.get_extension::() { - None => ConfigExtensionPropagationContext::default(), - Some(prev) => prev.as_ref().clone(), - }; - propagation_ctx.prefixes.push(T::PREFIX); - cfg.set_extension(Arc::new(propagation_ctx)); + register_for_propagation::(cfg); } pub(crate) fn set_distributed_option_extension_from_headers<'a, T: ConfigExtension + Default>( @@ -39,12 +34,7 @@ pub(crate) fn set_distributed_option_extension_from_headers<'a, T: ConfigExtensi } } - let mut propagation_ctx = match cfg.get_extension::() { - None => ConfigExtensionPropagationContext::default(), - Some(prev) => prev.as_ref().clone(), - }; - propagation_ctx.prefixes.push(T::PREFIX); - cfg.set_extension(Arc::new(propagation_ctx)); + register_for_propagation::(cfg); // If the config extension existed before, we want to modify instead of adding a new one from // scratch. If not, we'll start from scratch with a new one. @@ -87,9 +77,30 @@ pub(crate) fn set_distributed_option_extension_from_headers<'a, T: ConfigExtensi }) } +/// Config extensions that should be propagated over the wire, keyed by their prefix. Each one +/// carries the values of its [Default] implementation, so that entries that were not changed from +/// their default do not need to be sent: the receiving end starts from [Default] as well. #[derive(Clone, Debug, Default)] struct ConfigExtensionPropagationContext { - prefixes: Vec<&'static str>, + defaults_by_prefix: HashMap<&'static str, HashMap>>, +} + +fn register_for_propagation(cfg: &mut SessionConfig) { + let mut propagation_ctx = match cfg.get_extension::() { + None => ConfigExtensionPropagationContext::default(), + Some(prev) => prev.as_ref().clone(), + }; + propagation_ctx + .defaults_by_prefix + .entry(T::PREFIX) + .or_insert_with(|| { + T::default() + .entries() + .into_iter() + .map(|entry| (entry.key, entry.value)) + .collect() + }); + cfg.set_extension(Arc::new(propagation_ctx)); } pub(crate) fn get_config_extension_propagation_headers( @@ -102,17 +113,20 @@ pub(crate) fn get_config_extension_propagation_headers( .get_extension::() .unwrap_or_default(); - if prefixes_to_send.prefixes.is_empty() { + if prefixes_to_send.defaults_by_prefix.is_empty() { return Ok(HeaderMap::new()); } let mut headers = HeaderMap::new(); for (prefix, extension) in cfg.options().extensions.iter() { - if !prefixes_to_send.prefixes.contains(&prefix) { + let Some(defaults) = prefixes_to_send.defaults_by_prefix.get(prefix) else { continue; - } + }; for entry in extension.entries() { + if defaults.get(&entry.key) == Some(&entry.value) { + continue; + } let Some(value) = entry.value else { continue; }; @@ -169,7 +183,7 @@ mod tests { } #[test] - fn test_add_extension_with_empty_values() -> Result<(), Box> { + fn test_default_values_are_not_sent() -> Result<(), Box> { let mut config = SessionConfig::new(); let opt = CustomExtension::default(); @@ -179,14 +193,7 @@ mod tests { assert!(flight_metadata.is_some()); let headers = get_config_extension_propagation_headers(&config)?; - assert!(headers.contains_key("x-datafusion-distributed-config-custom.foo")); - assert!(headers.contains_key("x-datafusion-distributed-config-custom.bar")); - assert!(headers.contains_key("x-datafusion-distributed-config-custom.baz")); - - let get = |key: &str| headers.get(key).unwrap().to_str().unwrap(); - assert_eq!(get("x-datafusion-distributed-config-custom.foo"), ""); - assert_eq!(get("x-datafusion-distributed-config-custom.bar"), "0"); - assert_eq!(get("x-datafusion-distributed-config-custom.baz"), "false"); + assert!(headers.is_empty()); Ok(()) } @@ -209,10 +216,10 @@ mod tests { let headers = get_config_extension_propagation_headers(&config)?; + assert!(!headers.contains_key("x-datafusion-distributed-config-custom.foo")); + assert!(!headers.contains_key("x-datafusion-distributed-config-custom.baz")); let get = |key: &str| headers.get(key).unwrap().to_str().unwrap(); - assert_eq!(get("x-datafusion-distributed-config-custom.foo"), ""); assert_eq!(get("x-datafusion-distributed-config-custom.bar"), "42"); - assert_eq!(get("x-datafusion-distributed-config-custom.baz"), "false"); Ok(()) } @@ -375,7 +382,9 @@ mod tests { #[test] fn test_invalid_header_name() { let mut config = SessionConfig::new(); - let extension = InvalidExtension::default(); + let extension = InvalidExtension { + key_with_spaces: "non-default".to_string(), + }; set_distributed_option_extension(&mut config, extension); let result = get_config_extension_propagation_headers(&config); @@ -385,7 +394,9 @@ mod tests { #[test] fn test_invalid_header_value() { let mut config = SessionConfig::new(); - let extension = InvalidValueExtension::default(); + let extension = InvalidValueExtension { + key: "\u{0000}non-default\u{0001}".to_string(), + }; set_distributed_option_extension(&mut config, extension); let result = get_config_extension_propagation_headers(&config); @@ -417,7 +428,7 @@ mod tests { extensions_options! { pub struct InvalidExtension { - pub key_with_spaces: String, default = "value".to_string() + pub key_with_spaces: String, default = "".to_string() } } @@ -427,7 +438,7 @@ mod tests { extensions_options! { pub struct InvalidValueExtension { - pub key: String, default = "\u{0000}invalid\u{0001}".to_string() + pub key: String, default = "".to_string() } } From ebc22d048388095e3fc9a43098feea231519bfcc Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 19 Nov 2026 18:31:03 +0100 Subject: [PATCH 2/2] Stress config_propagation bench with larger headers Raise the localhost workers' HTTP/2 header list limit so the bench can send ~140 KB of config headers per task, and add an all_overrides scenario. --- .../benches/config_propagation.rs | 62 ++++++++++++++----- .../distributed/src/test_utils/localhost.rs | 5 ++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/datafusion/distributed-benchmarks/benches/config_propagation.rs b/datafusion/distributed-benchmarks/benches/config_propagation.rs index 7359e6060..a56df9efd 100644 --- a/datafusion/distributed-benchmarks/benches/config_propagation.rs +++ b/datafusion/distributed-benchmarks/benches/config_propagation.rs @@ -11,7 +11,8 @@ //! dispatching tasks. To stress the headers, the session carries a user config extension next to //! `DistributedConfig` with large string options, integer, float and boolean options, and options //! with long names. `defaults` leaves every option at its default; `few_overrides` changes a -//! handful of options of each kind, as a typical session would. +//! handful of options of each kind, as a typical session would; `all_overrides` changes every +//! option of the user extension. //! An untimed preflight reports the config headers a worker received per task. use criterion::{Criterion, criterion_group, criterion_main}; @@ -35,15 +36,15 @@ use tokio::runtime::Builder as RuntimeBuilder; const NUM_WORKERS: usize = 32; const QUERY: &str = "SELECT * FROM url_emitter(128, 128, 'config_propagation')"; const CONFIG_HEADER_PREFIX: &str = "x-datafusion-distributed-config-"; -/// Size of each [LargeOptions] string value, both default and overridden. With every option -/// sent, the config headers stay just under the 16 KiB HTTP/2 header list limit that hyper -/// applies by default; beyond it the worker rejects the task. -const STRING_BYTES: usize = 1024; -/// Shared name of the [LargeOptions] options with long keys, followed by their index. -const LONG_KEY: &str = "option_with_a_deliberately_long_name_to_stress_the_header_keys"; +/// Size of each [LargeOptions] string value, both default and overridden. +const STRING_BYTES: usize = 4 * 1024; /// Scenario name and how many [LargeOptions] options of each kind are changed from their default. -const SCENARIOS: [(&str, usize); 2] = [("defaults", 0), ("few_overrides", 4)]; +const SCENARIOS: [(&str, usize); 3] = [ + ("defaults", 0), + ("few_overrides", 4), + ("all_overrides", usize::MAX), +]; fn config_propagation(c: &mut Criterion) { let rt = RuntimeBuilder::new_multi_thread() @@ -69,13 +70,18 @@ fn config_propagation(c: &mut Criterion) { ctx.set_distributed_user_codec(URLEmitterExtensionCodec); ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); let mut options = LargeOptions::default(); - for i in 0..overridden { - let string = "y".repeat(STRING_BYTES); - options.set(&format!("string_{i:02}"), &string).unwrap(); - options.set(&format!("integer_{i:02}"), "1").unwrap(); - options.set(&format!("float_{i:02}"), "1.5").unwrap(); - options.set(&format!("boolean_{i:02}"), "true").unwrap(); - options.set(&format!("{LONG_KEY}_{i:02}"), "1").unwrap(); + for entry in LargeOptions::default().entries() { + let (kind, index) = entry.key.rsplit_once('_').unwrap(); + if index.parse::().unwrap() >= overridden { + continue; + } + let value = match kind { + "string" => "y".repeat(STRING_BYTES), + "float" => "1.5".to_string(), + "boolean" => "true".to_string(), + _ => "1".to_string(), + }; + options.set(&entry.key, &value).unwrap(); } ctx.set_distributed_option_extension(options); @@ -123,6 +129,32 @@ extensions_options! { pub string_03: String, default = "x".repeat(STRING_BYTES) pub string_04: String, default = "x".repeat(STRING_BYTES) pub string_05: String, default = "x".repeat(STRING_BYTES) + pub string_06: String, default = "x".repeat(STRING_BYTES) + pub string_07: String, default = "x".repeat(STRING_BYTES) + pub string_08: String, default = "x".repeat(STRING_BYTES) + pub string_09: String, default = "x".repeat(STRING_BYTES) + pub string_10: String, default = "x".repeat(STRING_BYTES) + pub string_11: String, default = "x".repeat(STRING_BYTES) + pub string_12: String, default = "x".repeat(STRING_BYTES) + pub string_13: String, default = "x".repeat(STRING_BYTES) + pub string_14: String, default = "x".repeat(STRING_BYTES) + pub string_15: String, default = "x".repeat(STRING_BYTES) + pub string_16: String, default = "x".repeat(STRING_BYTES) + pub string_17: String, default = "x".repeat(STRING_BYTES) + pub string_18: String, default = "x".repeat(STRING_BYTES) + pub string_19: String, default = "x".repeat(STRING_BYTES) + pub string_20: String, default = "x".repeat(STRING_BYTES) + pub string_21: String, default = "x".repeat(STRING_BYTES) + pub string_22: String, default = "x".repeat(STRING_BYTES) + pub string_23: String, default = "x".repeat(STRING_BYTES) + pub string_24: String, default = "x".repeat(STRING_BYTES) + pub string_25: String, default = "x".repeat(STRING_BYTES) + pub string_26: String, default = "x".repeat(STRING_BYTES) + pub string_27: String, default = "x".repeat(STRING_BYTES) + pub string_28: String, default = "x".repeat(STRING_BYTES) + pub string_29: String, default = "x".repeat(STRING_BYTES) + pub string_30: String, default = "x".repeat(STRING_BYTES) + pub string_31: String, default = "x".repeat(STRING_BYTES) pub integer_00: usize, default = 0 pub integer_01: usize, default = 0 pub integer_02: usize, default = 0 diff --git a/datafusion/distributed/src/test_utils/localhost.rs b/datafusion/distributed/src/test_utils/localhost.rs index 50bce5f07..ca33888a6 100644 --- a/datafusion/distributed/src/test_utils/localhost.rs +++ b/datafusion/distributed/src/test_utils/localhost.rs @@ -10,6 +10,10 @@ use tokio::net::TcpListener; use tonic::transport::Server; use url::Url; +/// HTTP/2 header list limit of the localhost workers, raised from hyper's default 16 KiB so that +/// sessions carrying large config extensions can still dispatch tasks. +const MAX_HEADER_LIST_BYTES: u32 = 1024 * 1024; + /// Create workers and context on localhost with a fixed number of target partitions. /// /// Creates `num_workers` listeners, all bound to a random OS decided port on `127.0.0.1`, then @@ -54,6 +58,7 @@ where join_set.spawn(async move { Server::builder() + .http2_max_header_list_size(MAX_HEADER_LIST_BYTES) .add_service(worker.into_worker_server()) .serve_with_incoming(incoming) .await