From cd5f547eb6ff9ec16b08274c73ea9964a770abd9 Mon Sep 17 00:00:00 2001 From: Raghav Yadav Date: Mon, 16 Nov 2026 20:15:34 +0000 Subject: [PATCH] test: cover remote dynamic filters with a work unit feed A partitioned join with a work unit feed on the build side and a parquet scan on the probe side, in different stages. The join dynamic filter has to reach the scan through the coordinator while the feed streams its work units over the same channel. Signed-off-by: Raghav Yadav --- .../distributed/tests/work_unit_feed.rs | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/datafusion/distributed/tests/work_unit_feed.rs b/datafusion/distributed/tests/work_unit_feed.rs index ee0afef35..91f474913 100644 --- a/datafusion/distributed/tests/work_unit_feed.rs +++ b/datafusion/distributed/tests/work_unit_feed.rs @@ -5,12 +5,14 @@ mod tests { use datafusion::execution::SessionState; use datafusion::physical_plan::execute_stream; use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::test_utils::test_work_unit_feed::{ RowGeneratorExec, TestWorkUnitFeedExecCodec, TestWorkUnitFeedFunction, row_generator_desired_task_count_handler, row_generator_scale_up_leaf_node_handler, }; use datafusion_distributed::{ DistributedExt, WorkerQueryContext, assert_snapshot, display_plan_ascii, + rewrite_distributed_plan_with_dynamic_filters, }; use futures::TryStreamExt; use std::sync::Arc; @@ -673,6 +675,62 @@ mod tests { Ok(()) } + /// A feed on the build side of a partitioned join, with the probe side scanning parquet in + /// another stage. The join's dynamic filter reaches the probe scan through the coordinator + /// while the coordinator is also streaming the feed's work units. + #[tokio::test] + async fn remote_dynamic_filters_with_feed() -> Result<(), Box> { + let query = r#" + SET datafusion.optimizer.hash_join_single_partition_threshold=0; + SET datafusion.optimizer.hash_join_single_partition_threshold_rows=0; + SELECT count(*) + FROM test_work_unit('build', 2, 'rows(1)', 'rows(2)', 'rows(1)', 'rows(3)') b + INNER JOIN weather w ON w."Cloud9am" = b.partition + "#; + let (plan, results, filter_updates) = run_query_with_weather(query).await?; + let (_, results_without_filters, _) = run_query_with_weather(&format!( + "SET datafusion.optimizer.enable_dynamic_filter_pushdown=false; {query}" + )) + .await?; + + assert_eq!(results, results_without_filters); + assert!( + filter_updates > 0, + "no dynamic filter update reached the coordinator" + ); + assert_snapshot!(plan + &results, @" + ┌───── DistributedExec + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── tasks=3, partitions=3 + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(partition@0, Cloud9am@0)], projection=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── tasks=2, partitions=9 + │ RepartitionExec: partitioning=Hash([partition@0], 9), input_partitions=2 + │ RowGeneratorExec: tag=build, tasks=2, partition_ops=[[rows(1)], [rows(2)], [rows(1)], [rows(3)]] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── tasks=3, partitions=9 + │ RepartitionExec: partitioning=Hash([Cloud9am@0], 9), input_partitions=3 + │ DistributedLeafExec: + │ t0: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000000.parquet:.., /testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Cloud9am], file_type=parquet, predicate=DynamicFilter [ false OR Cloud9am@15 >= 0 AND Cloud9am@15 <= 0 AND Cloud9am@15 IN (SET) ([]) OR Cloud9am@15 >= 1 AND Cloud9am@15 <= 1 AND Cloud9am@15 IN (SET) ([1, 1, 1, 1, 1]) ], dynamic_rg_pruning=eligible, pruning_predicate=Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 0 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0) OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 1 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0), required_guarantees=[] + │ t1: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Cloud9am], file_type=parquet, predicate=DynamicFilter [ false OR Cloud9am@15 >= 0 AND Cloud9am@15 <= 0 AND Cloud9am@15 IN (SET) ([]) OR Cloud9am@15 >= 1 AND Cloud9am@15 <= 1 AND Cloud9am@15 IN (SET) ([1, 1, 1, 1, 1]) ], dynamic_rg_pruning=eligible, pruning_predicate=Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 0 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0) OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 1 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0), required_guarantees=[] + │ t2: DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet:..], [/testdata/weather/result-000001.parquet:.., /testdata/weather/result-000002.parquet:..], [/testdata/weather/result-000002.parquet:..]]}, projection=[Cloud9am], file_type=parquet, predicate=DynamicFilter [ false OR Cloud9am@15 >= 0 AND Cloud9am@15 <= 0 AND Cloud9am@15 IN (SET) ([]) OR Cloud9am@15 >= 1 AND Cloud9am@15 <= 1 AND Cloud9am@15 IN (SET) ([1, 1, 1, 1, 1]) ], dynamic_rg_pruning=eligible, pruning_predicate=Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 0 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 0 AND 0 <= Cloud9am_max@0) OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_max@0 >= 1 AND Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND (Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0 OR Cloud9am_null_count@1 != row_count@2 AND Cloud9am_min@3 <= 1 AND 1 <= Cloud9am_max@0), required_guarantees=[] + └────────────────────────────────────────────────── + +----------+ + | count(*) | + +----------+ + | 646 | + +----------+ + "); + Ok(()) + } + /// `wait()` ops in a feed must actually delay the producing stream. /// Verifies the [`crate::test_utils::test_work_unit_feed::WorkUnitOp::Wait`] /// op is wired into the producer's stream. @@ -890,6 +948,50 @@ mod tests { Ok((plan_display, formatted.to_string())) } + /// Like [run_query], but also registers the parquet tables. The plan is displayed with the + /// dynamic filters each scan ended up with, and the number of dynamic filter updates the + /// coordinator received is returned. + async fn run_query_with_weather(sql: &str) -> Result<(String, String, usize), DataFusionError> { + let (ctx, _guard, _) = start_localhost_context(3, build_state).await; + let mut ctx = ctx.with_distributed_dynamic_filter_collection(true)?; + ctx.set_distributed_work_unit_feed(|p: &RowGeneratorExec| Some(&p.feed)); + ctx.set_distributed_user_codec(TestWorkUnitFeedExecCodec); + ctx.set_distributed_desired_task_count_handler(row_generator_desired_task_count_handler); + ctx.set_distributed_scale_up_leaf_node_handler(row_generator_scale_up_leaf_node_handler); + ctx.register_udtf("test_work_unit", Arc::new(TestWorkUnitFeedFunction)); + register_parquet_tables(&ctx).await?; + + let mut df_opt = None; + for sql in sql.split(";") { + if sql.trim().is_empty() { + continue; + } + df_opt = Some(ctx.sql(sql).await?); + } + let Some(df) = df_opt else { + return Err(DataFusionError::Plan("Empty 'sql' parameter".to_string())); + }; + let plan = df.create_physical_plan().await?; + let batches = execute_stream(Arc::clone(&plan), ctx.task_ctx())? + .try_collect::>() + .await?; + // Waits for the workers to report their final dynamic filters. + let plan_with_filters = + rewrite_distributed_plan_with_dynamic_filters(Arc::clone(&plan), &ctx.task_ctx()) + .await?; + let filter_updates = plan + .metrics() + .and_then(|m| m.sum(|m| m.value().name() == "dynamic_filter_updates_received")) + .map_or(0, |m| m.as_usize()); + let plan = plan_with_filters; + + Ok(( + display_plan_ascii(plan.as_ref(), false), + pretty_format_batches(&batches)?.to_string(), + filter_updates, + )) + } + async fn build_state(ctx: WorkerQueryContext) -> Result { Ok(ctx .builder