diff --git a/docs/source/user-guide/latest/tuning/operators.md b/docs/source/user-guide/latest/tuning/operators.md index 058a6a00cd7..5c0fb41c0c3 100644 --- a/docs/source/user-guide/latest/tuning/operators.md +++ b/docs/source/user-guide/latest/tuning/operators.md @@ -172,9 +172,10 @@ pruning does not require that setting. Set `spark.comet.exec.aggregate.skipPartial.enabled=true` to let Comet bypass partial hash aggregation for high-cardinality grouping when it is not reducing the number of rows enough. This experimental optimization is disabled by default. It currently applies only to fused native -shuffle-writer plans whose partial aggregates are grouping-only or single-argument `COUNT`. -Low-cardinality inputs continue to aggregate normally. The SQL metric -`rows bypassing partial aggregation` shows whether skipping occurred. +shuffle-writer plans whose partial aggregates are grouping-only, single-argument `COUNT`, or `AVG` +over any input except a decimal with precision above 11. Low-cardinality inputs continue to +aggregate normally. The SQL metric `rows bypassing partial aggregation` shows whether skipping +occurred. DataFusion makes the decision separately in each task. It starts checking after the first 100,000 input rows, and as soon as the number of groups divided by the number of input rows exceeds `0.8`, @@ -186,8 +187,10 @@ enabling it for a workload. Eligibility is conservative for the whole fused native plan: any unsupported partial accumulator, Spark `PartialMerge`, or mixed-mode aggregate disables skipping in that plan. Multi-argument -`COUNT` and other accumulators are not admitted. Distribution-required grouping-only stages -still fully deduplicate, and non-native-shuffle plans retain ordinary aggregation. +`COUNT`, `AVG` over a decimal with precision above 11, and other accumulators are not admitted. +Spark rewrites `AVG` over a decimal with precision 11 or lower into an `AVG` with a `DOUBLE` +result, which is admitted. Distribution-required grouping-only stages still fully deduplicate, and +non-native-shuffle plans retain ordinary aggregation. To experiment with the thresholds, also enable `spark.comet.exec.respectDataFusionConfigs`, a development and testing option that defaults to `false`. For example, the following diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index a01789dead3..2a7689f1dcf 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -61,6 +61,7 @@ use datafusion::{ }; use datafusion_comet_common::decode_string_arrays; use datafusion_comet_proto::spark_expression::agg_expr::ExprStruct as AggExprStruct; +use datafusion_comet_proto::spark_expression::data_type::DataTypeId; use datafusion_comet_proto::spark_operator::{AggregateMode, Operator, ShuffleScan}; use datafusion_comet_shuffle::decoded_batch_coalescer::DecodedBatchCoalescer; use datafusion_comet_spark_expr::url_funcs::{CometParseUrl, CometTryParseUrl}; @@ -1035,6 +1036,14 @@ fn configure_skip_partial_aggregation(config: &mut SessionConfig, plan: &Operato && agg.agg_exprs.iter().all(|expr| { matches!(&expr.expr_struct, Some(AggExprStruct::Count(count)) if count.children.len() == 1) + // Only a DOUBLE result, which the planner builds as Comet's + // Float64 AVG, can convert rows to states. A decimal result + // is an `AvgDecimal`, which cannot. + || matches!(&expr.expr_struct, Some(AggExprStruct::Avg(avg)) + if avg.child.is_some() + && avg.datatype.as_ref().is_some_and(|datatype| { + datatype.type_id == DataTypeId::Double as i32 + })) }) } // PartialMerge consumes states, not rows. @@ -2713,7 +2722,12 @@ mod tests { use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::ExecutionPlan; use datafusion_comet_proto::spark_expression; - use datafusion_comet_proto::spark_expression::{AggExpr, Count, Expr, Sum}; + use datafusion_comet_proto::spark_expression::data_type::{ + data_type_info::DatatypeStruct, DataTypeInfo, DecimalInfo, + }; + use datafusion_comet_proto::spark_expression::{ + AggExpr, Avg, Count, DataType as ProtoDataType, Expr, Sum, + }; use datafusion_comet_proto::spark_operator::{HashAggregate, ShuffleWriter}; use std::cell::Cell; use std::future::Future; @@ -2730,6 +2744,27 @@ mod tests { expr_struct: Some(AggExprStruct::Sum(Sum::default())), ..Default::default() }; + let proto_type = |type_id: DataTypeId| ProtoDataType { + type_id: type_id as i32, + ..Default::default() + }; + let decimal = ProtoDataType { + type_info: Some(Box::new(DataTypeInfo { + datatype_struct: Some(DatatypeStruct::Decimal(DecimalInfo { + precision: 38, + scale: 10, + })), + })), + ..proto_type(DataTypeId::Decimal) + }; + let avg = |child: Option, datatype: Option| AggExpr { + expr_struct: Some(AggExprStruct::Avg(Avg { + child, + datatype, + ..Default::default() + })), + ..Default::default() + }; let partial = HashAggregate { grouping_exprs: vec![Expr::default()], agg_exprs: vec![count.clone()], @@ -2775,7 +2810,14 @@ mod tests { ..partial.clone() }, HashAggregate { - agg_exprs: vec![count.clone(), count], + agg_exprs: vec![ + count.clone(), + avg(Some(Expr::default()), Some(proto_type(DataTypeId::Double))), + ], + ..partial.clone() + }, + HashAggregate { + agg_exprs: vec![count.clone(), count.clone()], ..partial.clone() }, ] { @@ -2798,6 +2840,32 @@ mod tests { agg_exprs: vec![AggExpr::default()], ..partial.clone() }, + // AVG without an input, without a result type, or with a result type other than + // DOUBLE is not admitted. A decimal result is an `AvgDecimal`, which cannot convert + // rows to partial states. + HashAggregate { + agg_exprs: vec![avg(None, Some(proto_type(DataTypeId::Double)))], + ..partial.clone() + }, + HashAggregate { + agg_exprs: vec![avg(Some(Expr::default()), None)], + ..partial.clone() + }, + HashAggregate { + agg_exprs: vec![avg(Some(Expr::default()), Some(decimal.clone()))], + ..partial.clone() + }, + HashAggregate { + agg_exprs: vec![count.clone(), avg(Some(Expr::default()), Some(decimal))], + ..partial.clone() + }, + HashAggregate { + agg_exprs: vec![avg( + Some(Expr::default()), + Some(proto_type(DataTypeId::Float)), + )], + ..partial.clone() + }, HashAggregate { agg_exprs: vec![AggExpr { expr_struct: Some(AggExprStruct::Count(Count { diff --git a/native/spark-expr/src/agg_funcs/avg.rs b/native/spark-expr/src/agg_funcs/avg.rs index 2b2422ce775..3e0f79dad42 100644 --- a/native/spark-expr/src/agg_funcs/avg.rs +++ b/native/spark-expr/src/agg_funcs/avg.rs @@ -19,8 +19,9 @@ use arrow::array::{ builder::PrimitiveBuilder, cast::AsArray, types::{Float64Type, Int64Type}, - Array, ArrayRef, ArrowNumericType, Int64Array, PrimitiveArray, + Array, ArrayRef, ArrowNumericType, BooleanArray, Int64Array, PrimitiveArray, }; +use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::compute::sum; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion::common::{not_impl_err, Result, ScalarValue}; @@ -30,6 +31,8 @@ use datafusion::logical_expr::{ use datafusion::physical_expr::expressions::format_state_name; use std::sync::Arc; +use super::collect::filter_to_nulls; + use arrow::array::ArrowNativeTypeOp; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion::logical_expr::Volatility::Immutable; @@ -235,7 +238,7 @@ where &mut self, values: &[ArrayRef], group_indices: &[usize], - opt_filter: Option<&arrow::array::BooleanArray>, + opt_filter: Option<&BooleanArray>, total_num_groups: usize, ) -> Result<()> { assert_eq!(values.len(), 1, "single argument to update_batch"); @@ -334,12 +337,43 @@ where ]) } + /// Each row's own partial state, as `state` emits it: the value and a count of one, or a + /// zero sum and count for a null or filtered-out row. fn convert_to_state( &self, - _values: &[ArrayRef], - _opt_filter: Option<&arrow::array::BooleanArray>, + values: &[ArrayRef], + opt_filter: Option<&BooleanArray>, ) -> Result> { - not_impl_err!("Input batch conversion to state not implemented") + let values = values[0].as_primitive::(); + let filter_nulls = opt_filter.map(filter_to_nulls); + let mask = NullBuffer::union(filter_nulls.as_ref(), values.nulls()); + let (sums, counts): (ScalarBuffer, ScalarBuffer) = match mask { + // Every row is included, so the input values are the sums as they are. + None => (values.values().clone(), vec![1; values.len()].into()), + // `merge_batch` reads the sums' values without their nulls, so an excluded row + // needs a zero sum as well as a zero count. + Some(mask) => { + let (sums, counts): (Vec<_>, Vec<_>) = values + .values() + .iter() + .zip(mask.iter()) + .map(|(&value, included)| { + if included { + (value, 1) + } else { + (T::default_value(), 0) + } + }) + .unzip(); + (sums.into(), counts.into()) + } + }; + let sums = + PrimitiveArray::::new(sums, None).with_data_type(self.return_data_type.clone()); + Ok(vec![ + Arc::new(sums) as ArrayRef, + Arc::new(Int64Array::new(counts, None)) as ArrayRef, + ]) } fn size(&self) -> usize { @@ -365,4 +399,56 @@ mod tests { assert_eq!(partial.evaluate()?, ScalarValue::Float64(None)); Ok(()) } + + /// Converting rows straight to partial states, as a partial aggregate that stops aggregating + /// does, gives each row the state that aggregating it alone into a group would. + #[test] + fn convert_to_state_matches_one_row_groups() -> Result<()> { + let new_accumulator = || { + AvgGroupsAccumulator::::new(&Float64, |sum: f64, count: i64| { + Ok(sum / count as f64) + }) + }; + let check = |values: ArrayRef, filter: Option<&BooleanArray>| -> Result<()> { + let converted = new_accumulator().convert_to_state(&[Arc::clone(&values)], filter)?; + let groups: Vec = (0..values.len()).collect(); + let mut grouped = new_accumulator(); + grouped.update_batch(&[values], &groups, filter, groups.len())?; + assert_eq!(converted, grouped.state(EmitTo::All)?); + Ok(()) + }; + + let values = Float64Array::from(vec![ + Some(1.5), + None, + Some(-2.0), + Some(4.0), + Some(8.0), + None, + ]); + let no_nulls = Float64Array::from(vec![1.5, -2.0, 4.0, 8.0]); + // A null predicate, as `FILTER (WHERE d > 1)` gives where `d` is null, excludes its row. + let filter = BooleanArray::from(vec![ + Some(true), + Some(true), + None, + Some(false), + Some(true), + Some(true), + ]); + + for values in [values.clone(), no_nulls.clone()] { + let filter = filter.slice(0, values.len()); + check(Arc::new(values.clone()), None)?; + check(Arc::new(values), Some(&filter))?; + } + // Sliced inputs and filters, whose buffers start at an offset. + let sliced = values.slice(1, 3); + let sliced_filter = filter.slice(1, 3); + check(Arc::new(sliced.clone()), None)?; + check(Arc::new(sliced), Some(&sliced_filter))?; + check(Arc::new(no_nulls.slice(1, 3)), None)?; + check(Arc::new(no_nulls.slice(1, 3)), Some(&filter.slice(2, 3)))?; + Ok(()) + } } diff --git a/native/spark-expr/src/agg_funcs/collect.rs b/native/spark-expr/src/agg_funcs/collect.rs index 00b2759c259..14831085f26 100644 --- a/native/spark-expr/src/agg_funcs/collect.rs +++ b/native/spark-expr/src/agg_funcs/collect.rs @@ -473,10 +473,10 @@ fn prefix_sum(counts: &[u32]) -> Vec { cursors } -/// A `NullBuffer` that is null wherever `filter` does not select the row, so that a filtered row -/// can be represented as a null list entry. Mirrors DataFusion's `filter_to_nulls`, which lives +/// A `NullBuffer` that is null wherever `filter` does not select the row, including where the +/// predicate itself is null, so that a filtered row can be masked like a null input. Mirrors DataFusion's `filter_to_nulls`, which lives /// in `datafusion-functions-aggregate-common` and is not re-exported by the `datafusion` facade. -fn filter_to_nulls(filter: &BooleanArray) -> NullBuffer { +pub(super) fn filter_to_nulls(filter: &BooleanArray) -> NullBuffer { let selected = match filter.nulls() { Some(nulls) => filter.values() & nulls.inner(), None => filter.values().clone(), diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index a80cdd76f1b..151318b07ca 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -718,9 +718,10 @@ object CometConf extends ShimCometConf { "Experimental opt-in: let a native partial aggregate stop aggregating once its input " + "looks mostly distinct, and send the rest of the task's rows to the shuffle " + "unaggregated. Only applies to partial aggregates that feed Comet native shuffle " + - "and whose aggregate functions, if any, are all single-argument COUNT. The check " + - "starts after the first 100,000 input rows of a task, and once aggregation stops " + - "it does not resume, so a task whose keys repeat after a mostly distinct start can " + + "and whose aggregate functions, if any, are all single-argument COUNT or AVG over " + + "any input except a decimal with precision above 11. The check starts after the " + + "first 100,000 input rows of a task, and once aggregation stops it does not " + + "resume, so a task whose keys repeat after a mostly distinct start can " + s"shuffle many times more rows than it would with this disabled. $TUNING_GUIDE.") .booleanConf .createWithDefault(false) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala index a5696212145..1b4b99de0d1 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala @@ -3431,6 +3431,65 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("skip partial aggregation converts AVG rows to partial states") { + withTempDir { dir => + val path = new Path(dir.toURI.toString, "input").toUri.toString + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + // Mostly distinct keys within each task, with some keys repeating across tasks so the + // final aggregate merges converted states with each other. Values include nulls. + spark + .range(0L, 16384L, 1L, 8) + .selectExpr( + "id % 12000 AS k", + "CASE WHEN id % 5 = 0 THEN NULL ELSE CAST(id % 17 AS INT) END AS i", + "CASE WHEN id % 7 = 0 THEN NULL ELSE CAST(id % 11 AS DOUBLE) / 3 END AS d", + "CASE WHEN id % 3 = 0 THEN NULL ELSE CAST(id % 13 AS DECIMAL(10, 2)) END AS small", + "CASE WHEN id % 4 = 0 THEN NULL ELSE CAST(id % 19 AS DECIMAL(38, 10)) / 7 END AS big") + .write + .parquet(path) + } + withParquetTable(path, "skip_partial_avg") { + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.SHUFFLE_PARTITIONS.key -> "8", + CometConf.COMET_BATCH_SIZE.key -> "128", + CometConf.COMET_SHUFFLE_ENABLED.key -> "true", + CometConf.COMET_SHUFFLE_MODE.key -> "native", + CometConf.COMET_EXEC_AGGREGATE_SKIP_PARTIAL_ENABLED.key -> "true", + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.skip_partial_aggregation_probe_rows_threshold" -> "100", + "spark.comet.datafusion.execution.skip_partial_aggregation_probe_ratio_threshold" -> "0.8") { + def skipped(query: String): Long = { + val (_, plan) = checkSparkAnswerAndOperator(query) + stripAQEPlan(plan).collect { case aggregate: CometHashAggregateExec => + aggregate.metrics.get("skipped_aggregation_rows").map(_.value).getOrElse(0L) + }.sum + } + def avgQuery(expressions: String): String = + s"SELECT k, $expressions FROM skip_partial_avg GROUP BY k" + + // AVG over integral and floating-point inputs stops aggregating, as does AVG over a + // decimal with precision 11 or lower, which Spark rewrites to a DOUBLE AVG over the + // unscaled value. A FILTER whose predicate is null wherever `d` is null excludes + // those rows. + for (expressions <- Seq( + "avg(i)", + "avg(d)", + "avg(small)", + "avg(i), avg(d), count(i)", + "avg(i) FILTER (WHERE d > 1)")) { + assert(skipped(avgQuery(expressions)) > 0L, s"Expected skipping for $expressions") + } + // AVG over a wider decimal, which Comet aggregates as a decimal, cannot convert rows + // to partial states, so it never stops aggregating. + for (expressions <- Seq("avg(big)", "avg(i), avg(big)")) { + assert(skipped(avgQuery(expressions)) == 0L, s"Unexpected skipping for $expressions") + } + } + } + } + } + test("first/last") { withSQLConf( SQLConf.COALESCE_PARTITIONS_ENABLED.key -> "true",