diff --git a/ballista/client/tests/null_aware.rs b/ballista/client/tests/null_aware.rs index fe82645741..a41dbd7111 100644 --- a/ballista/client/tests/null_aware.rs +++ b/ballista/client/tests/null_aware.rs @@ -24,9 +24,11 @@ mod null_aware { use std::path::Path; use crate::common::{ - check_matches, collect_text, datafusion_context, register_csv_tables, set_option, + ballista_state, check_matches, collect_text, datafusion_context, + register_csv_tables, set_option, }; use ballista::prelude::SessionContextExt; + use ballista_core::config::BALLISTA_EXECUTION_MODE; use datafusion::prelude::*; use tempfile::TempDir; @@ -61,7 +63,18 @@ mod null_aware { fs::write(t2.join("p3.csv"), "b,tag\n4,x\n").unwrap(); } - const QUERY: &str = "select a from t1 where a not in (select b from t2) order by a"; + const QUERIES: &[(&str, &str)] = &[ + ( + "order_by", + "select a from t1 where a not in (select b from t2) order by a", + ), + // An aggregate above the join, so `EnsureRequirements` reruns over its stage. + ( + "group_by", + "select a % 3, count(*) from t1 where a not in (select b from t2) \ + group by a % 3 order by 1", + ), + ]; async fn run_case(case: &str, t2_has_null: bool) { let tmp = TempDir::new().unwrap(); @@ -70,7 +83,10 @@ mod null_aware { let df_ctx = datafusion_context(); register_csv_tables(&df_ctx, dir, &["t1", "t2"]).await; - let expected = collect_text(&df_ctx, QUERY).await; + let mut expected = vec![]; + for (_, sql) in QUERIES { + expected.push(collect_text(&df_ctx, sql).await); + } let mut mismatches = vec![]; // "default" exercises the NOT IN logical rewrite under Ballista's @@ -79,23 +95,50 @@ mod null_aware { // all. "prefer_hash_join" and "aqe" exercise the rewrite under the // hash-join planning paths. "rewrite_disabled" turns the rewrite off // with hash joins preferred, covering the fallback path where the - // null-aware join survives and is lowered to a single task. - for variant in ["default", "prefer_hash_join", "aqe", "rewrite_disabled"] { - let ctx = SessionContext::standalone().await.unwrap(); + // null-aware join survives and is lowered to a single task, under the + // default adaptive planner; "rewrite_disabled_static" and + // "rewrite_disabled_pipelined" do the same under the static planner, + // which pipelined mode always uses. All three run one partition per + // task, as on a cluster. + for variant in [ + "default", + "prefer_hash_join", + "aqe", + "rewrite_disabled", + "rewrite_disabled_static", + "rewrite_disabled_pipelined", + ] { + // A standalone cluster started pipelined runs push-staged. Its + // placement doesn't pick up a later `SET` of the task cap. + let ctx = if variant == "rewrite_disabled_pipelined" { + let state = ballista_state(&[ + (BALLISTA_EXECUTION_MODE, "pipelined"), + ("ballista.scheduler.max_partitions_per_task", "1"), + ]); + SessionContext::standalone_with_state(state).await.unwrap() + } else { + SessionContext::standalone().await.unwrap() + }; if variant == "aqe" { set_option(&ctx, "ballista.planner.adaptive.enabled", "true").await; } if variant != "default" { set_option(&ctx, "datafusion.optimizer.prefer_hash_join", "true").await; } - if variant == "rewrite_disabled" { + if variant.starts_with("rewrite_disabled") { set_option(&ctx, "ballista.optimizer.not_in_subquery_rewrite", "false") .await; + set_option(&ctx, "ballista.scheduler.max_partitions_per_task", "1").await; + } + if variant == "rewrite_disabled_static" { + set_option(&ctx, "ballista.planner.adaptive.enabled", "false").await; } register_csv_tables(&ctx, dir, &["t1", "t2"]).await; - let label = format!("{case}/{variant}"); - check_matches(&ctx, &label, QUERY, &expected, &mut mismatches).await; + for ((name, sql), expected) in QUERIES.iter().zip(&expected) { + let label = format!("{case}/{variant}/{name}"); + check_matches(&ctx, &label, sql, expected, &mut mismatches).await; + } } assert!(mismatches.is_empty(), "{}", mismatches.join("\n\n")); diff --git a/ballista/scheduler/src/planner.rs b/ballista/scheduler/src/planner.rs index 84d7409086..8b3695b8f2 100644 --- a/ballista/scheduler/src/planner.rs +++ b/ballista/scheduler/src/planner.rs @@ -186,10 +186,26 @@ impl DefaultDistributedPlanner { )?; stages.append(&mut probe_stages); + let null_aware = hash_join.null_aware; let new_join = execution_plan.replace_children( vec![broadcast_left, probe], ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), )?; + // A null-aware join must keep the coalesce over its probe side, but + // `EnsureRequirements` removes it if it reruns over this stage at a + // boundary above. Close the stage at the join so it cannot. + if null_aware { + let writer = create_shuffle_writer_with_config( + job_id, + self.next_stage_id(), + new_join, + None, + config, + )?; + let reader = create_unresolved_shuffle(writer.as_ref()); + stages.push(writer); + return Ok((reader, stages)); + } return Ok((new_join, stages)); } @@ -2213,22 +2229,25 @@ order by Ok(()) } - #[tokio::test] - async fn nested_loop_join_probe_gets_one_partition_stage() -> Result<(), BallistaError> - { + /// A one-partition table `small` and a two-partition table `big`, both with + /// a single nullable Int32 column `k`. + fn small_big_ctx() -> Result { use datafusion::arrow::array::Int32Array; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::MemTable; use datafusion::prelude::{SessionConfig, SessionContext}; - let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int32, false)])); + let schema = Arc::new(Schema::new(vec![Field::new("k", DataType::Int32, true)])); let batch = |keys: Vec| { RecordBatch::try_new(schema.clone(), vec![Arc::new(Int32Array::from(keys))]) .map_err(|e| BallistaError::General(e.to_string())) }; + // The scheduler enforces round-robin repartitioning off. let ctx = SessionContext::new_with_config( - SessionConfig::new().with_target_partitions(2), + SessionConfig::new() + .with_target_partitions(2) + .with_round_robin_repartition(false), ); ctx.register_batch("small", batch(vec![1, 2])?)?; // Two partitions, so the probe side has something to coalesce. @@ -2240,6 +2259,13 @@ order by ], )?; ctx.register_table("big", Arc::new(big))?; + Ok(ctx) + } + + #[tokio::test] + async fn nested_loop_join_probe_gets_one_partition_stage() -> Result<(), BallistaError> + { + let ctx = small_big_ctx()?; let options = ctx.state().config().options().clone(); let stages = stages_for_sql( @@ -2265,6 +2291,32 @@ order by Ok(()) } + #[tokio::test] + async fn null_aware_join_keeps_its_probe_coalesce() -> Result<(), BallistaError> { + let ctx = small_big_ctx()?; + let options = ctx.state().config().options().clone(); + + // The aggregate cuts a stage above the join, which reruns + // `EnsureRequirements` over it. + let stages = stages_for_sql( + &ctx, + &options, + "select count(*) from small where k not in (select k from big)", + ) + .await?; + + assert_eq!(stages.len(), 5); + assert_plan!(stages[2].as_ref(), @r" + ShuffleWriterExec: partitioning: UnknownPartitioning(1) + HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(k@0, k@0)], projection=[], null_aware + UnresolvedShuffleExec: stage=1, broadcast=true, upstream_partitions: 1 + CoalescePartitionsExec + UnresolvedShuffleExec: stage=2, partitioning: UnknownPartitioning(2) + "); + + Ok(()) + } + /// Plans `sql` on `ctx` and splits the physical plan into query stages. async fn stages_for_sql( ctx: &datafusion::prelude::SessionContext,