From 86e00d45412e5e7745584c08e19a6ea6d2c22546 Mon Sep 17 00:00:00 2001 From: ewoodbury Date: Thu, 19 Nov 2026 00:04:10 -0800 Subject: [PATCH 1/3] fix(scheduler): keep a null-aware anti join's probe coalesce in the static planner The static planner left the coalesce over a null-aware join's probe side inside the join's stage. An aggregate above the join reran EnsureRequirements over that stage and removed the coalesce, so the join ran as several tasks and emitted unmatched build rows once per task. Close the stage at the join so no later pass sees the coalesce. Closes #2849. --- ballista/client/tests/null_aware.rs | 33 +++++++++++---- ballista/scheduler/src/planner.rs | 62 ++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 12 deletions(-) diff --git a/ballista/client/tests/null_aware.rs b/ballista/client/tests/null_aware.rs index fe82645741..ba14af59ca 100644 --- a/ballista/client/tests/null_aware.rs +++ b/ballista/client/tests/null_aware.rs @@ -61,7 +61,12 @@ mod null_aware { fs::write(t2.join("p3.csv"), "b,tag\n4,x\n").unwrap(); } - const QUERY: &str = "select a from t1 where a not in (select b from t2) order by a"; + const QUERIES: &[&str] = &[ + "select a from t1 where a not in (select b from t2) order by a", + // An aggregate above the join, so `EnsureRequirements` reruns over its stage. + "select a % 3, count(*) from t1 where a not in (select b from t2) \ + group by a % 3 order by 1", + ]; async fn run_case(case: &str, t2_has_null: bool) { let tmp = TempDir::new().unwrap(); @@ -70,7 +75,6 @@ mod null_aware { let df_ctx = datafusion_context(); register_csv_tables(&df_ctx, dir, &["t1", "t2"]).await; - let expected = collect_text(&df_ctx, QUERY).await; let mut mismatches = vec![]; // "default" exercises the NOT IN logical rewrite under Ballista's @@ -79,8 +83,16 @@ mod null_aware { // all. "prefer_hash_join" and "aqe" exercise the rewrite under the // hash-join planning paths. "rewrite_disabled" turns the rewrite off // with hash joins preferred, covering the fallback path where the - // null-aware join survives and is lowered to a single task. - for variant in ["default", "prefer_hash_join", "aqe", "rewrite_disabled"] { + // null-aware join survives and is lowered to a single task; + // "rewrite_disabled_static" does the same under the static planner. + // Both run one partition per task, as on a cluster. + for variant in [ + "default", + "prefer_hash_join", + "aqe", + "rewrite_disabled", + "rewrite_disabled_static", + ] { let ctx = SessionContext::standalone().await.unwrap(); if variant == "aqe" { set_option(&ctx, "ballista.planner.adaptive.enabled", "true").await; @@ -88,14 +100,21 @@ mod null_aware { if variant != "default" { set_option(&ctx, "datafusion.optimizer.prefer_hash_join", "true").await; } - if variant == "rewrite_disabled" { + if variant.starts_with("rewrite_disabled") { set_option(&ctx, "ballista.optimizer.not_in_subquery_rewrite", "false") .await; + set_option(&ctx, "ballista.scheduler.max_partitions_per_task", "1").await; + } + if variant == "rewrite_disabled_static" { + set_option(&ctx, "ballista.planner.adaptive.enabled", "false").await; } register_csv_tables(&ctx, dir, &["t1", "t2"]).await; - let label = format!("{case}/{variant}"); - check_matches(&ctx, &label, QUERY, &expected, &mut mismatches).await; + for (i, sql) in QUERIES.iter().enumerate() { + let expected = collect_text(&df_ctx, sql).await; + let label = format!("{case}/{variant}/{i}"); + check_matches(&ctx, &label, sql, &expected, &mut mismatches).await; + } } assert!(mismatches.is_empty(), "{}", mismatches.join("\n\n")); diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 84d7409086..8b3695b8f2 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -186,10 +186,26 @@ impl DefaultDistributedPlanner { )?; stages.append(&mut probe_stages); + let null_aware = hash_join.null_aware; let new_join = execution_plan.replace_children( vec![broadcast_left, probe], ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), )?; + // A null-aware join must keep the coalesce over its probe side, but + // `EnsureRequirements` removes it if it reruns over this stage at a + // boundary above. Close the stage at the join so it cannot. + if null_aware { + let writer = create_shuffle_writer_with_config( + job_id, + self.next_stage_id(), + new_join, + None, + config, + )?; + let reader = create_unresolved_shuffle(writer.as_ref()); + stages.push(writer); + return Ok((reader, stages)); + } return Ok((new_join, stages)); } @@ -2213,22 +2229,25 @@ order by Ok(()) } - #[tokio::test] - async fn nested_loop_join_probe_gets_one_partition_stage() -> Result<(), BallistaError> - { + /// A one-partition table `small` and a two-partition table `big`, both with + /// a single nullable Int32 column `k`. + fn small_big_ctx() -> Result { use datafusion::arrow::array::Int32Array; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::MemTable; use datafusion::prelude::{SessionConfig, SessionContext}; - let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int32, false)])); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int32, true)])); let batch = |keys: Vec| { RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(keys))]) .map_err(|e| BallistaError::General(e.to_string())) }; + // The scheduler enforces round-robin repartitioning off. let ctx = SessionContext::new_with_config( - SessionConfig::new().with_target_partitions(2), + SessionConfig::new() + .with_target_partitions(2) + .with_round_robin_repartition(false), ); ctx.register_batch("small", batch(vec![1, 2])?)?; // Two partitions, so the probe side has something to coalesce. @@ -2240,6 +2259,13 @@ order by ], )?; ctx.register_table("big", Arc::new(big))?; + Ok(ctx) + } + + #[tokio::test] + async fn nested_loop_join_probe_gets_one_partition_stage() -> Result<(), BallistaError> + { + let ctx = small_big_ctx()?; let options = ctx.state().config().options().clone(); let stages = stages_for_sql( @@ -2265,6 +2291,32 @@ order by Ok(()) } + #[tokio::test] + async fn null_aware_join_keeps_its_probe_coalesce() -> Result<(), BallistaError> { + let ctx = small_big_ctx()?; + let options = ctx.state().config().options().clone(); + + // The aggregate cuts a stage above the join, which reruns + // `EnsureRequirements` over it. + let stages = stages_for_sql( + &ctx, + &options, + "select count(*) from small where k not in (select k from big)", + ) + .await?; + + assert_eq!(stages.len(), 5); + assert_plan!(stages[2].as_ref(), @r" + ShuffleWriterExec: partitioning: UnknownPartitioning(1) + HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(k@0, k@0)], projection=[], null_aware + UnresolvedShuffleExec: stage=1, broadcast=true, upstream_partitions: 1 + CoalescePartitionsExec + UnresolvedShuffleExec: stage=2, partitioning: UnknownPartitioning(2) + "); + + Ok(()) + } + /// Plans `sql` on `ctx` and splits the physical plan into query stages. async fn stages_for_sql( ctx: &datafusion::prelude::SessionContext, From c46da951755f12109a995cd5489d041edc7543c3 Mon Sep 17 00:00:00 2001 From: ewoodbury Date: Thu, 19 Nov 2026 00:24:44 -0800 Subject: [PATCH 2/3] test(client): name the null-aware queries and compute their expected results once --- ballista/client/tests/null_aware.rs | 32 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/ballista/client/tests/null_aware.rs b/ballista/client/tests/null_aware.rs index ba14af59ca..083ad979fe 100644 --- a/ballista/client/tests/null_aware.rs +++ b/ballista/client/tests/null_aware.rs @@ -61,11 +61,17 @@ mod null_aware { fs::write(t2.join("p3.csv"), "b,tag\n4,x\n").unwrap(); } - const QUERIES: &[&str] = &[ - "select a from t1 where a not in (select b from t2) order by a", + const QUERIES: &[(&str, &str)] = &[ + ( + "order_by", + "select a from t1 where a not in (select b from t2) order by a", + ), // An aggregate above the join, so `EnsureRequirements` reruns over its stage. - "select a % 3, count(*) from t1 where a not in (select b from t2) \ - group by a % 3 order by 1", + ( + "group_by", + "select a % 3, count(*) from t1 where a not in (select b from t2) \ + group by a % 3 order by 1", + ), ]; async fn run_case(case: &str, t2_has_null: bool) { @@ -75,6 +81,10 @@ mod null_aware { let df_ctx = datafusion_context(); register_csv_tables(&df_ctx, dir, &["t1", "t2"]).await; + let mut expected = vec![]; + for (_, sql) in QUERIES { + expected.push(collect_text(&df_ctx, sql).await); + } let mut mismatches = vec![]; // "default" exercises the NOT IN logical rewrite under Ballista's @@ -83,9 +93,10 @@ mod null_aware { // all. "prefer_hash_join" and "aqe" exercise the rewrite under the // hash-join planning paths. "rewrite_disabled" turns the rewrite off // with hash joins preferred, covering the fallback path where the - // null-aware join survives and is lowered to a single task; - // "rewrite_disabled_static" does the same under the static planner. - // Both run one partition per task, as on a cluster. + // null-aware join survives and is lowered to a single task, under the + // default adaptive planner; "rewrite_disabled_static" does the same + // under the static planner. Both run one partition per task, as on a + // cluster. for variant in [ "default", "prefer_hash_join", @@ -110,10 +121,9 @@ mod null_aware { } register_csv_tables(&ctx, dir, &["t1", "t2"]).await; - for (i, sql) in QUERIES.iter().enumerate() { - let expected = collect_text(&df_ctx, sql).await; - let label = format!("{case}/{variant}/{i}"); - check_matches(&ctx, &label, sql, &expected, &mut mismatches).await; + for ((name, sql), expected) in QUERIES.iter().zip(&expected) { + let label = format!("{case}/{variant}/{name}"); + check_matches(&ctx, &label, sql, expected, &mut mismatches).await; } } From 5e8fcd840bece88eae4684bd422e613e557a313f Mon Sep 17 00:00:00 2001 From: ewoodbury Date: Thu, 19 Nov 2026 13:06:47 -0800 Subject: [PATCH 3/3] test(client): cover the null-aware fix in pipelined mode --- ballista/client/tests/null_aware.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/ballista/client/tests/null_aware.rs b/ballista/client/tests/null_aware.rs index 083ad979fe..a41dbd7111 100644 --- a/ballista/client/tests/null_aware.rs +++ b/ballista/client/tests/null_aware.rs @@ -24,9 +24,11 @@ mod null_aware { use std::path::Path; use crate::common::{ - check_matches, collect_text, datafusion_context, register_csv_tables, set_option, + ballista_state, check_matches, collect_text, datafusion_context, + register_csv_tables, set_option, }; use ballista::prelude::SessionContextExt; + use ballista_core::config::BALLISTA_EXECUTION_MODE; use datafusion::prelude::*; use tempfile::TempDir; @@ -94,17 +96,29 @@ mod null_aware { // hash-join planning paths. "rewrite_disabled" turns the rewrite off // with hash joins preferred, covering the fallback path where the // null-aware join survives and is lowered to a single task, under the - // default adaptive planner; "rewrite_disabled_static" does the same - // under the static planner. Both run one partition per task, as on a - // cluster. + // default adaptive planner; "rewrite_disabled_static" and + // "rewrite_disabled_pipelined" do the same under the static planner, + // which pipelined mode always uses. All three run one partition per + // task, as on a cluster. for variant in [ "default", "prefer_hash_join", "aqe", "rewrite_disabled", "rewrite_disabled_static", + "rewrite_disabled_pipelined", ] { - let ctx = SessionContext::standalone().await.unwrap(); + // A standalone cluster started pipelined runs push-staged. Its + // placement doesn't pick up a later `SET` of the task cap. + let ctx = if variant == "rewrite_disabled_pipelined" { + let state = ballista_state(&[ + (BALLISTA_EXECUTION_MODE, "pipelined"), + ("ballista.scheduler.max_partitions_per_task", "1"), + ]); + SessionContext::standalone_with_state(state).await.unwrap() + } else { + SessionContext::standalone().await.unwrap() + }; if variant == "aqe" { set_option(&ctx, "ballista.planner.adaptive.enabled", "true").await; }