From ad8cef828c94ed02c0fd7cc9c7d4972bc26e14af Mon Sep 17 00:00:00 2001 From: Jayant Shrivastava Date: Tue, 29 Sep 2026 12:29:30 +0000 Subject: [PATCH 1/2] datafusion-physical-plan: handles nulls correctly in aggregate dynamic filters (#25579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fix prevents one aggregate’s filter from discarding rows that another aggregate still needs in dynamic filters. ### Example: Query: `MIN(a), MAX(a), MIN(b), MAX(b)` ``` First batch: (1, NULL), (8, NULL) | v a has bounds [1, 8] b has no bounds yet | +---------+----------+ | | Before After | | a < 1 OR a > 8 filter = true | | v v Next batch: (2, 5), (3, 7) | | Both rows rejected Both rows accepted | | v v Result: Now b has bounds [5, 7] 1 8 NULL NULL | v Filtering can start: a < 1 OR a > 8 OR b < 5 OR b > 7 | v Correct result: 1 8 5 7 ``` The latter is correct 1. According to [Postgres](https://www.postgresql.org/docs/current/functions-aggregate.html): > - MIN: “Computes the minimum of the non-null input values.” > - MAX: “Computes the maximum of the non-null input values.” 2. According to [DataFusion](https://github.com/apache/datafusion/blob/6ff484a96d7eca4b1bf4a263d7190ada5af2343c/datafusion/sqllogictest/test_files/aggregate.slt#L5490-L5500) Executing `1, NULL, 3, 4, 5` expects `MIN = 1, MAX = 5` --------- Co-authored-by: Gabriel <45515538+gabotechs@users.noreply.github.com> Backport provenance: - Upstream PR: https://github.com/apache/datafusion/pull/25579 - Upstream commit: 0af68188912349e9c865548458b37b0e66b5c31d - Conflict resolution only supplies the missing test module/import context; the upstream logic, native test body and SQL assertions are preserved. - Original upstream tests are retained; fork-only adaptation, if any, follows separately. Signed-off-by: discord9 (cherry picked from commit 390e174d1b1bbd42749a603dee6f2702985d71a1) --- .../src/aggregates/aggregate_stream.rs | 87 +++++++++++++++++-- .../push_down_filter_regression.slt | 49 +++++++++++ 2 files changed, 130 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs index e1e869105d8db..946cb70cdb1f9 100644 --- a/datafusion/physical-plan/src/aggregates/aggregate_stream.rs +++ b/datafusion/physical-plan/src/aggregates/aggregate_stream.rs @@ -77,7 +77,6 @@ struct AggregateStreamInner { } impl AggregateStreamInner { - // TODO: check if we get Null handling correct /// # Examples /// - Example 1 /// Accumulators: min(c1) @@ -105,11 +104,12 @@ impl AggregateStreamInner { Vec::with_capacity(filter_state.accumulator_dyn_filter_info.len()); for acc_info in &filter_state.accumulator_dyn_filter_info { - // Skip if we don't yet have a meaningful bound + // Every aggregate needs a bound before we can prune rows. Otherwise, + // another aggregate's predicate could discard its first non-NULL value. let bound = { let guard = acc_info.shared_bound.lock(); if (*guard).is_null() { - continue; + return Ok(lit(true)); } guard.clone() }; @@ -255,9 +255,9 @@ fn scalar_cmp_null_short_circuit( v1: &ScalarValue, v2: &ScalarValue, ) -> Option { - match (v1, v2) { - (ScalarValue::Null, ScalarValue::Null) => Some(ScalarValue::Null), - (ScalarValue::Null, other) | (other, ScalarValue::Null) => Some(other.clone()), + match (v1.is_null(), v2.is_null()) { + (true, _) => Some(v2.clone()), + (_, true) => Some(v1.clone()), _ => None, } } @@ -476,3 +476,78 @@ fn aggregate_batch( Ok(allocated) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ExecutionPlan; + use crate::aggregates::PhysicalGroupBy; + use crate::common::collect as collect_stream; + use crate::expressions::col; + use crate::test::TestMemoryExec; + use arrow::array::Int32Array; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; + use datafusion_physical_expr::aggregate::AggregateExprBuilder; + + #[tokio::test] + async fn aggregate_dynamic_filter_recovers_from_null_bounds() -> Result<()> { + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, true), + ])); + let first = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![1, 8])), + Arc::new(Int32Array::from(vec![None, None])), + ], + )?; + let second = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![2, 3])), + Arc::new(Int32Array::from(vec![5, 7])), + ], + )?; + let input = TestMemoryExec::try_new_exec( + &[vec![first], vec![second]], + Arc::clone(&schema), + None, + )?; + let mut aggregates = Vec::new(); + for column in ["a", "b"] { + for function in [min_udaf(), max_udaf()] { + let alias = format!("{}({column})", function.name()); + aggregates.push(Arc::new( + AggregateExprBuilder::new(function, vec![col(column, &schema)?]) + .schema(Arc::clone(&schema)) + .alias(alias) + .build()?, + )); + } + } + let aggregate = AggregateExec::try_new( + AggregateMode::Partial, + PhysicalGroupBy::default(), + aggregates, + vec![None; 4], + input, + schema, + )?; + let filter = &aggregate.dynamic_filter.as_ref().unwrap().filter; + let context = Arc::new(TaskContext::default()); + + // Execute partitions sequentially to make filter publication deterministic. + collect_stream(aggregate.execute(0, Arc::clone(&context))?).await?; + assert_eq!(filter.current()?.to_string(), "true"); + + // Typed NULL bounds must recover so filtering resumes for every aggregate. + collect_stream(aggregate.execute(1, context)?).await?; + assert_eq!( + filter.current()?.to_string(), + "a@0 < 1 OR a@0 > 8 OR b@1 < 5 OR b@1 > 7" + ); + Ok(()) + } +} diff --git a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt index 5a038427d0c65..525999fa411db 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_regression.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_regression.slt @@ -443,6 +443,55 @@ drop table agg_dyn_mixed; statement ok reset datafusion.execution.batch_size; +# --- one aggregate has a bound before another sees its first non-NULL value --- +# Both files establish MIN(a)=1 and MAX(a)=8 in their first row group. Until b +# has a non-NULL bound, filtering only on a would prune every remaining row and +# incorrectly leave MIN(b) and MAX(b) NULL, regardless of partition order. + +statement ok +set datafusion.execution.batch_size = 2; + +statement ok +COPY ( + SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b) +) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_0.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '2'); + +statement ok +COPY ( + SELECT * FROM (VALUES (1, NULL), (8, NULL), (2, 5), (3, 7)) AS v(a, b) +) TO 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/file_1.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '2'); + +statement ok +CREATE EXTERNAL TABLE agg_dyn_late_bound (a INT, b INT) +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_regression/agg_dyn_late_bound/'; + +statement ok +set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = false; + +query IIII +SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound; +---- +1 8 5 7 + +statement ok +set datafusion.optimizer.enable_aggregate_dynamic_filter_pushdown = true; + +query IIII +SELECT MIN(a), MAX(a), MIN(b), MAX(b) FROM agg_dyn_late_bound; +---- +1 8 5 7 + +statement ok +drop table agg_dyn_late_bound; + +statement ok +reset datafusion.execution.batch_size; + # --- all-NULLs input: filter should stay `true` (no meaningful bound) --- statement ok From bed91dcd5aa550a1fb60c1da7c99709a9e9044cc Mon Sep 17 00:00:00 2001 From: discord9 Date: Sun, 8 Nov 2026 22:11:59 +0800 Subject: [PATCH 2/2] test: retain aggregate producer NULL-bound regressions Move the existing native real-producer regressions from the former combined backport into the independent aggregate NULL-bound repair. No new test or product behavior is introduced by this history split. Signed-off-by: discord9 --- .../physical-plan/src/aggregates/mod.rs | 216 +++++++++++++++++- 1 file changed, 214 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index e0d3c2488b8c3..b9e5cc665922e 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -3152,7 +3152,7 @@ mod tests { use crate::empty::EmptyExec; use crate::execution_plan::Boundedness; use crate::expressions::col; - use crate::filter::FilterExecBuilder; + use crate::filter::{FilterExec, FilterExecBuilder, batch_filter}; use crate::metrics::MetricValue; use crate::statistics::{StatisticsArgs, StatisticsContext}; use crate::test::TestMemoryExec; @@ -3183,7 +3183,7 @@ mod tests { use datafusion_functions_aggregate::count::count_udaf; use datafusion_functions_aggregate::first_last::{first_value_udaf, last_value_udaf}; use datafusion_functions_aggregate::median::median_udaf; - use datafusion_functions_aggregate::min_max::min_udaf; + use datafusion_functions_aggregate::min_max::{max_udaf, min_udaf}; use datafusion_functions_aggregate::sum::sum_udaf; use datafusion_physical_expr::Partitioning; use datafusion_physical_expr::PhysicalSortExpr; @@ -7837,6 +7837,218 @@ mod tests { } } + // The FilterExec uses a batch size of one so that the aggregate updates the + // shared predicate before the filter reads the next input batch. + async fn run_aggregate_dynamic_filter_null_case( + rows: &[(Option, Option, Option)], + max_a: bool, + ) -> Result<( + Vec, + Arc, + Arc, + )> { + let schema = Arc::new(Schema::new(vec![ + Field::new("ts", DataType::Int64, true), + Field::new("a", DataType::Int64, true), + Field::new("b", DataType::Int64, true), + ])); + let batches = rows + .iter() + .map(|(ts, a, b)| { + RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int64Array::from(vec![*ts])), + Arc::new(Int64Array::from(vec![*a])), + Arc::new(Int64Array::from(vec![*b])), + ], + ) + }) + .collect::, _>>()?; + let input = TestMemoryExec::try_new_exec(&[batches], Arc::clone(&schema), None)?; + let aggregates = [ + ("ts", min_udaf()), + ("a", if max_a { max_udaf() } else { min_udaf() }), + ("b", min_udaf()), + ] + .into_iter() + .map(|(name, function)| { + Ok(Arc::new( + AggregateExprBuilder::new(function, vec![col(name, &schema)?]) + .schema(Arc::clone(&schema)) + .alias(name) + .build()?, + )) + }) + .collect::>>()?; + let aggregate = Arc::new(AggregateExec::try_new( + AggregateMode::Partial, + PhysicalGroupBy::default(), + aggregates, + vec![None; 3], + Arc::new(EmptyExec::new(Arc::clone(&schema))), + Arc::clone(&schema), + )?); + let mut produced = aggregate.dynamic_expressions_produced(); + assert_eq!(produced.len(), 1); + let dynamic_filter = (produced.pop().unwrap() + as Arc) + .downcast::() + .expect("aggregate should produce a dynamic filter"); + let filter = Arc::new( + FilterExecBuilder::new( + Arc::::clone(&dynamic_filter), + input, + ) + .with_batch_size(1) + .build()?, + ); + let aggregate = aggregate.replace_children( + vec![Arc::::clone(&filter)], + ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute), + )?; + let results = + collect(aggregate.execute(0, Arc::new(TaskContext::default()))?).await?; + Ok((results, filter, dynamic_filter)) + } + + fn assert_aggregate_dynamic_filter_null_result( + batches: &[RecordBatch], + expected: [Option; 3], + ) -> Result<()> { + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + for (index, value) in expected.into_iter().enumerate() { + assert_eq!( + ScalarValue::try_from_array(batches[0].column(index), 0)?, + ScalarValue::Int64(value), + "aggregate {index}" + ); + } + Ok(()) + } + + #[tokio::test] + async fn aggregate_dynamic_filter_null_keeps_unknown_columns() -> Result<()> { + let (results, filter, dynamic_filter) = run_aggregate_dynamic_filter_null_case( + &[(Some(100), None, None), (Some(101), None, Some(2))], + false, + ) + .await?; + assert_aggregate_dynamic_filter_null_result( + &results, + [Some(100), None, Some(2)], + )?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(2)); + assert_eq!( + format!("{}", dynamic_filter.current()?), + format!("{}", lit(true)) + ); + Ok(()) + } + + #[tokio::test] + async fn aggregate_dynamic_filter_null_to_value_becomes_selective() -> Result<()> { + let (results, filter, dynamic_filter) = run_aggregate_dynamic_filter_null_case( + &[ + (Some(100), None, Some(1)), + (Some(99), Some(2), None), + // No aggregate can improve: the second batch must have tightened + // the filter before this batch is read. + (Some(101), Some(3), Some(2)), + // Only min(a) improves; it must be present in the predicate. + (Some(101), Some(1), Some(2)), + ], + false, + ) + .await?; + assert_aggregate_dynamic_filter_null_result( + &results, + [Some(99), Some(1), Some(1)], + )?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(3)); + let predicate = dynamic_filter.current()?; + assert_ne!(format!("{predicate}"), format!("{}", lit(true))); + let probe = RecordBatch::try_new( + filter.schema(), + vec![ + Arc::new(Int64Array::from(vec![101, 101, 101, 98])), + Arc::new(Int64Array::from(vec![0, 2, 2, 2])), + Arc::new(Int64Array::from(vec![2, 0, 2, 2])), + ], + )?; + assert_eq!(batch_filter(&probe, &predicate)?.num_rows(), 3); + Ok(()) + } + + #[tokio::test] + async fn aggregate_dynamic_filter_null_mixed_min_max() -> Result<()> { + let (results, filter, dynamic_filter) = run_aggregate_dynamic_filter_null_case( + &[(None, None, None), (None, None, None)], + true, + ) + .await?; + assert_aggregate_dynamic_filter_null_result(&results, [None, None, None])?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(2)); + assert_eq!( + format!("{}", dynamic_filter.current()?), + format!("{}", lit(true)) + ); + + let (results, filter, dynamic_filter) = run_aggregate_dynamic_filter_null_case( + &[(Some(100), None, None), (Some(101), Some(2), None)], + true, + ) + .await?; + assert_aggregate_dynamic_filter_null_result( + &results, + [Some(100), Some(2), None], + )?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(2)); + assert_eq!( + format!("{}", dynamic_filter.current()?), + format!("{}", lit(true)) + ); + + let (results, filter, dynamic_filter) = run_aggregate_dynamic_filter_null_case( + &[ + (Some(100), None, None), + (Some(101), Some(2), None), + (Some(101), Some(3), Some(2)), + (Some(101), Some(4), Some(3)), + (Some(101), Some(2), Some(3)), + ], + true, + ) + .await?; + assert_aggregate_dynamic_filter_null_result( + &results, + [Some(100), Some(4), Some(2)], + )?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(4)); + assert_ne!( + format!("{}", dynamic_filter.current()?), + format!("{}", lit(true)) + ); + Ok(()) + } + + #[tokio::test] + async fn aggregate_dynamic_filter_null_consumer_uses_updated_predicate() -> Result<()> + { + let (results, filter, _) = run_aggregate_dynamic_filter_null_case( + &[(Some(100), Some(2), Some(1)), (Some(101), Some(3), Some(2))], + false, + ) + .await?; + assert_aggregate_dynamic_filter_null_result( + &results, + [Some(100), Some(2), Some(1)], + )?; + assert_eq!(filter.metrics().unwrap().output_rows(), Some(1)); + Ok(()) + } + /// Test that [`AggregateExec::with_dynamic_filter_expr`] overrides the existing dynamic filter #[test] fn test_with_dynamic_filter() -> Result<()> {