Repository navigation
perf: benchmark task startup overlap #873
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
gabotechs
merged 2 commits into
datafusion-contrib:main
from
joybestourous:joy/codex/startup-benchmark
Nov 10, 2026
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
229 changes: 229 additions & 0 deletions
229
datafusion/distributed-benchmarks/benches/startup_overlap.rs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,229 @@ | ||
| //! # Task startup overlap | ||
| //! | ||
| //! ```shell | ||
| //! cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --save-baseline main | ||
| //! # Run the same benchmark on the candidate revision: | ||
| //! cargo bench -p datafusion-distributed-benchmarks --bench startup_overlap -- --baseline main | ||
| //! ``` | ||
| //! | ||
| //! This controlled benchmark runs two tasks over localhost gRPC in static and AQE | ||
| //! mode. Task 0 performs 100 million dependent integer operations; task 1 returns | ||
| //! one row. The existing localhost fixture starts three endpoints so both active | ||
| //! tasks can route away from the coordinator's local endpoint. Each query must | ||
| //! return task indices `[0, 1]`, and the CPU function must execute remotely once. | ||
| //! | ||
| //! `delay_light_task` adds 250 ms to task 1's routing setup, allowing task 0's CPU | ||
| //! work to overlap that wait. `delay_heavy_task` delays task 0 instead, keeping the | ||
| //! wait on the critical path; `no_delay` provides the other control. The delay is | ||
| //! only in routing setup, not in the CPU workload. Criterion measures execution | ||
| //! through complete output collection; worker startup and query planning are | ||
| //! excluded. An untimed preflight also reports first-batch time and whether remote | ||
| //! CPU work started and finished before the delayed task returned a ready | ||
| //! destination. The witnesses are reported without requiring overlap, so the | ||
| //! benchmark also runs before the optimization. | ||
| //! | ||
| //! Run both revisions on a quiet host and alternate comparisons. Verify a positive | ||
| //! overlap witness in the candidate's `delay_light_task` preflight and its absence | ||
| //! in the baseline before interpreting timings. This isolates scheduling value | ||
| //! under controlled setup skew; it does not replace representative suite results. | ||
|
|
||
| use arrow::array::Int64Array; | ||
| use arrow::datatypes::DataType; | ||
| use async_trait::async_trait; | ||
| use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; | ||
| use datafusion::common::{Result, exec_err}; | ||
| use datafusion::execution::SessionState; | ||
| use datafusion::logical_expr::{ScalarUDF, Volatility, create_udf}; | ||
| use datafusion::physical_plan::execute_stream; | ||
| use datafusion::prelude::SessionContext; | ||
| use datafusion_distributed::test_utils::localhost::start_localhost_context; | ||
| use datafusion_distributed::test_utils::routing::{ | ||
| URLEmitterExtensionCodec, URLEmitterFunction, UrlEmitterRouteTaskHandler, | ||
| url_emitter_desired_task_count, url_emitter_scale_up_leaf_node, | ||
| }; | ||
| use datafusion_distributed::{ | ||
| DistributedExt, MappedWorkerSessionBuilderExt, RouteTaskEvent, RouteTaskEventResponse, | ||
| RouteTaskHandler, WorkerQueryContext, | ||
| }; | ||
| use futures::TryStreamExt; | ||
| use std::hint::black_box; | ||
| use std::sync::Arc; | ||
| use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; | ||
| use std::time::{Duration, Instant}; | ||
| use tokio::runtime::Builder as RuntimeBuilder; | ||
| use tokio::time::timeout; | ||
|
|
||
| const QUERY: &str = "SELECT cpu_work(task_index) FROM url_emitter(2, 2, 'startup')"; | ||
| const SETUP_DELAY: Duration = Duration::from_millis(250); | ||
| const WAIT: Duration = Duration::from_secs(10); | ||
|
|
||
| fn startup_overlap(c: &mut Criterion) { | ||
| let rt = RuntimeBuilder::new_multi_thread() | ||
| .worker_threads(4) | ||
| .enable_all() | ||
| .build() | ||
| .unwrap(); | ||
| let mut group = c.benchmark_group("startup_overlap"); | ||
| group.sample_size(10); | ||
| group.warm_up_time(Duration::from_secs(1)); | ||
| group.measurement_time(Duration::from_secs(5)); | ||
|
|
||
| for (mode, adaptive) in [("static", false), ("aqe", true)] { | ||
| for (scenario, delayed_task) in [ | ||
| ("no_delay", None), | ||
| ("delay_light_task", Some(1)), | ||
| ("delay_heavy_task", Some(0)), | ||
| ] { | ||
| let probe = Arc::new(Probe::default()); | ||
| let udf = Arc::new(cpu_work(Some(Arc::clone(&probe)))); | ||
| let worker = build_state.map(move |builder| { | ||
| Ok(builder | ||
| .with_scalar_functions(vec![Arc::clone(&udf)]) | ||
| .build()) | ||
| }); | ||
| // Routing uses URLs 2 and 1, avoiding URL 0's local-worker shortcut. | ||
| let (mut ctx, _workers, _) = rt.block_on(start_localhost_context(3, worker)); | ||
| ctx.set_distributed_desired_task_count_handler(url_emitter_desired_task_count); | ||
| ctx.set_distributed_scale_up_leaf_node_handler(url_emitter_scale_up_leaf_node); | ||
| ctx.set_distributed_user_codec(URLEmitterExtensionCodec); | ||
| ctx.register_udtf("url_emitter", Arc::new(URLEmitterFunction)); | ||
| ctx.register_udf(cpu_work(None)); | ||
| let ctx = ctx | ||
| .with_distributed_dynamic_task_count(adaptive) | ||
| .unwrap() | ||
| .with_distributed_route_task_handler(SetupDelay { | ||
| delayed_task, | ||
| probe: Arc::clone(&probe), | ||
| }); | ||
| rt.block_on(ctx.sql("SET datafusion.execution.target_partitions = 2")) | ||
| .unwrap(); | ||
|
|
||
| let (first, total) = rt.block_on(run_query(&ctx, &probe)).unwrap(); | ||
| println!( | ||
| "preflight {mode}/{scenario}: first={first:?} total={total:?} \ | ||
| heavy_started_before_ready={} heavy_finished_before_ready={}", | ||
| probe.started_before_ready.load(Ordering::SeqCst), | ||
| probe.finished_before_ready.load(Ordering::SeqCst), | ||
| ); | ||
| group.bench_function(BenchmarkId::new(mode, scenario), |b| { | ||
| b.iter_custom(|iterations| { | ||
| rt.block_on(async { | ||
| let mut elapsed = Duration::ZERO; | ||
| for _ in 0..iterations { | ||
| elapsed += run_query(&ctx, &probe).await.unwrap().1; | ||
| } | ||
| elapsed | ||
| }) | ||
| }) | ||
| }); | ||
| } | ||
| } | ||
| group.finish(); | ||
| } | ||
|
|
||
| async fn run_query(ctx: &SessionContext, probe: &Probe) -> Result<(Duration, Duration)> { | ||
| let plan = ctx.sql(QUERY).await?.create_physical_plan().await?; | ||
| probe.delayed_ready.store(false, Ordering::SeqCst); | ||
| probe.started_before_ready.store(false, Ordering::SeqCst); | ||
| probe.finished_before_ready.store(false, Ordering::SeqCst); | ||
| probe.heavy_executions.store(0, Ordering::SeqCst); | ||
| let start = Instant::now(); | ||
| let mut output = execute_stream(plan, ctx.task_ctx())?; | ||
| let first = timeout(WAIT, output.try_next()).await.unwrap()?.unwrap(); | ||
| let first_elapsed = start.elapsed(); | ||
| let mut batches = vec![first]; | ||
| batches.extend( | ||
| timeout(WAIT, output.try_collect::<Vec<_>>()) | ||
| .await | ||
| .unwrap()?, | ||
| ); | ||
| let total_elapsed = start.elapsed(); | ||
| let mut values: Vec<_> = batches | ||
| .iter() | ||
| .flat_map(|batch| { | ||
| batch | ||
| .column(0) | ||
| .as_any() | ||
| .downcast_ref::<Int64Array>() | ||
| .unwrap() | ||
| .values() | ||
| .to_vec() | ||
| }) | ||
| .collect(); | ||
| values.sort_unstable(); | ||
| assert_eq!(values, vec![0, 1]); | ||
| assert_eq!(probe.heavy_executions.load(Ordering::SeqCst), 1); | ||
| Ok((first_elapsed, total_elapsed)) | ||
| } | ||
|
|
||
| async fn build_state(ctx: WorkerQueryContext) -> Result<SessionState> { | ||
| Ok(ctx | ||
| .builder | ||
| .with_distributed_user_codec(URLEmitterExtensionCodec) | ||
| .build()) | ||
| } | ||
|
|
||
| #[derive(Default)] | ||
| struct Probe { | ||
| delayed_ready: AtomicBool, | ||
| started_before_ready: AtomicBool, | ||
| finished_before_ready: AtomicBool, | ||
| heavy_executions: AtomicUsize, | ||
| } | ||
|
|
||
| struct SetupDelay { | ||
| delayed_task: Option<usize>, | ||
| probe: Arc<Probe>, | ||
| } | ||
|
|
||
| #[async_trait] | ||
| impl RouteTaskHandler for SetupDelay { | ||
| async fn handle(&self, event: RouteTaskEvent<'_>) -> Option<Result<RouteTaskEventResponse>> { | ||
| let delayed = self.delayed_task == Some(event.task_key.task_number); | ||
| if delayed { | ||
| tokio::time::sleep(SETUP_DELAY).await; | ||
| } | ||
| let response = UrlEmitterRouteTaskHandler.handle(event).await?; | ||
| if delayed || self.delayed_task.is_none() { | ||
| self.probe.delayed_ready.store(true, Ordering::SeqCst); | ||
| } | ||
| Some(response) | ||
| } | ||
| } | ||
|
|
||
| fn cpu_work(probe: Option<Arc<Probe>>) -> ScalarUDF { | ||
| create_udf( | ||
| "cpu_work", | ||
| vec![DataType::Int64], | ||
| DataType::Int64, | ||
| Volatility::Volatile, | ||
| Arc::new(move |args| { | ||
| let Some(probe) = &probe else { | ||
| return exec_err!("cpu_work must execute on a remote worker"); | ||
| }; | ||
| let input = args[0].clone().into_array(1)?; | ||
| let input = input.as_any().downcast_ref::<Int64Array>().unwrap(); | ||
| for task in input.values() { | ||
| if *task == 0 { | ||
| probe.heavy_executions.fetch_add(1, Ordering::SeqCst); | ||
| probe.started_before_ready.store( | ||
| !probe.delayed_ready.load(Ordering::SeqCst), | ||
| Ordering::SeqCst, | ||
| ); | ||
| let mut value = 1u64; | ||
| for _ in 0..100_000_000 { | ||
| value = black_box(value.wrapping_mul(17).wrapping_add(1)); | ||
| } | ||
| probe.finished_before_ready.store( | ||
| !probe.delayed_ready.load(Ordering::SeqCst), | ||
| Ordering::SeqCst, | ||
| ); | ||
| } | ||
| } | ||
| Ok(args[0].clone()) | ||
| }), | ||
| ) | ||
| } | ||
|
|
||
| criterion_group!(benches, startup_overlap); | ||
| criterion_main!(benches); | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.