From 4fa3d5f79746767765c2647a7495a497f56c4baf Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Mon, 16 Nov 2026 21:28:06 +0000 Subject: [PATCH 1/4] feat: integrate MIN/MAX reader filtering with Spark --- .github/workflows/pr_build_linux.yml | 1 + .github/workflows/pr_build_macos.yml | 1 + .../operators/dynamic_filter/min_max.rs | 2 - .../execution/operators/dynamic_filter/mod.rs | 1 + native/core/src/execution/operators/mod.rs | 4 +- native/core/src/execution/planner.rs | 2 + .../core/src/execution/planner/aggregate.rs | 59 +++- .../planner/tests/min_max_reader_filter.rs | 246 +++++++++++++++++ native/proto/src/proto/operator.proto | 2 + .../scala/org/apache/comet/CometConf.scala | 14 + .../apache/spark/sql/comet/operators.scala | 18 +- .../apache/comet/exec/CometMinMaxSuite.scala | 252 ++++++++++++++++++ 12 files changed, 584 insertions(+), 18 deletions(-) create mode 100644 native/core/src/execution/planner/tests/min_max_reader_filter.rs create mode 100644 spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 3a1c2380aa7..61d054353a0 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -586,6 +586,7 @@ jobs: org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometMergeRowsSuite org.apache.comet.exec.CometTopKSuite + org.apache.comet.exec.CometMinMaxSuite org.apache.comet.exec.CometEmptyRelationExecSuite org.apache.comet.exec.CometRangeExecSuite org.apache.comet.exec.CometInMemoryCacheSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index 48375ba31fd..41d194c7468 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -224,6 +224,7 @@ jobs: org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometMergeRowsSuite org.apache.comet.exec.CometTopKSuite + org.apache.comet.exec.CometMinMaxSuite org.apache.comet.exec.CometEmptyRelationExecSuite org.apache.comet.exec.CometRangeExecSuite org.apache.comet.exec.CometInMemoryCacheSuite diff --git a/native/core/src/execution/operators/dynamic_filter/min_max.rs b/native/core/src/execution/operators/dynamic_filter/min_max.rs index 461bc378490..674c7d7d7d0 100644 --- a/native/core/src/execution/operators/dynamic_filter/min_max.rs +++ b/native/core/src/execution/operators/dynamic_filter/min_max.rs @@ -61,8 +61,6 @@ struct RuntimeAggregate { } impl MinMaxReaderFilterExec { - // Spark planner integration is a separate step of apache/datafusion-comet#5774. - #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn try_new( aggregate: &AggregateExec, config: &ConfigOptions, diff --git a/native/core/src/execution/operators/dynamic_filter/mod.rs b/native/core/src/execution/operators/dynamic_filter/mod.rs index 0018750aa1d..68160f3a81d 100644 --- a/native/core/src/execution/operators/dynamic_filter/mod.rs +++ b/native/core/src/execution/operators/dynamic_filter/mod.rs @@ -27,6 +27,7 @@ mod topk; mod test_utils; pub(crate) use join::DynamicFilterJoinExec; +pub(crate) use min_max::MinMaxReaderFilterExec; pub(crate) use topk::TopKReaderFilterExec; use std::fmt::Formatter; diff --git a/native/core/src/execution/operators/mod.rs b/native/core/src/execution/operators/mod.rs index ac88b787426..83e9089f7c6 100644 --- a/native/core/src/execution/operators/mod.rs +++ b/native/core/src/execution/operators/mod.rs @@ -27,7 +27,9 @@ mod arrow_python_udf; #[cfg(feature = "python-udf")] pub use arrow_python_udf::{ArrowPythonUdfExec, ArrowPythonUdfSpec}; mod dynamic_filter; -pub(crate) use dynamic_filter::{DynamicFilterJoinExec, TopKReaderFilterExec}; +pub(crate) use dynamic_filter::{ + DynamicFilterJoinExec, MinMaxReaderFilterExec, TopKReaderFilterExec, +}; pub(crate) mod iceberg_common; pub use iceberg_common::clear_file_io_cache; mod iceberg_dictionary; diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 404a394e58b..c140a96b6b8 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -3171,6 +3171,7 @@ fn needs_fields_coercion(sig: &TypeSignature) -> bool { #[cfg(test)] mod tests { mod empty_native_scan; + mod min_max_reader_filter; mod multi_store_scan; mod semi_join_reduction; @@ -4647,6 +4648,7 @@ mod tests { expr_modes: vec![], initial_input_buffer_offset: 0, ordered_by_grouping_keys: false, + dynamic_filter_enabled: false, })), }; let projection = Operator { diff --git a/native/core/src/execution/planner/aggregate.rs b/native/core/src/execution/planner/aggregate.rs index 1c03dd0d7a6..af0970cb951 100644 --- a/native/core/src/execution/planner/aggregate.rs +++ b/native/core/src/execution/planner/aggregate.rs @@ -64,7 +64,11 @@ use super::{ PlanCreationResult, }; use crate::{ - execution::{operators::ExecutionError, serde::to_arrow_datatype, spark_plan::SparkPlan}, + execution::{ + operators::{ExecutionError, MinMaxReaderFilterExec}, + serde::to_arrow_datatype, + spark_plan::SparkPlan, + }, extract_op, }; @@ -175,7 +179,9 @@ impl OperatorBuilder for HashAggBuilder { let agg_exprs: PhyAggResult = agg .agg_exprs .iter() - .map(|expr| planner.create_agg_expr(expr, Arc::clone(&schema))) + .map(|expr| { + planner.create_agg_expr(expr, Arc::clone(&schema), agg.dynamic_filter_enabled) + }) .collect(); let aggr_expr: Vec> = if has_partial_merge { @@ -276,16 +282,25 @@ impl OperatorBuilder for HashAggBuilder { mode }; - let aggregate: Arc = Arc::new( - datafusion::physical_plan::aggregates::AggregateExec::try_new( - mode, - group_by, - aggr_expr, - filter_exprs?, - Arc::clone(&child.native_plan), - Arc::clone(&schema), - )?, - ); + let aggregate = datafusion::physical_plan::aggregates::AggregateExec::try_new( + mode, + group_by, + aggr_expr, + filter_exprs?, + Arc::clone(&child.native_plan), + Arc::clone(&schema), + )?; + let aggregate: Arc = if agg.dynamic_filter_enabled { + match MinMaxReaderFilterExec::try_new( + &aggregate, + planner.session_ctx.copied_config().options(), + )? { + Some(wrapper) => Arc::new(wrapper), + None => Arc::new(aggregate), + } + } else { + Arc::new(aggregate) + }; let mut output = aggregate; let mut additional_native_plans = vec![]; @@ -375,6 +390,7 @@ impl PhysicalPlanner { &self, spark_expr: &AggExpr, schema: SchemaRef, + dynamic_filter_enabled: bool, ) -> Result { self.register_query_context(spark_expr.expr_id, spark_expr.query_context.as_ref()); @@ -399,7 +415,8 @@ impl PhysicalPlanner { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap()); let func = min_max_udaf(&datatype, false); - let child = Self::cast_min_max_child(child, &datatype, &schema)?; + let child = + Self::cast_min_max_child(child, &datatype, &schema, dynamic_filter_enabled)?; AggregateExprBuilder::new(func, vec![child]) .schema(schema) @@ -413,7 +430,8 @@ impl PhysicalPlanner { let child = self.create_expr(expr.child.as_ref().unwrap(), Arc::clone(&schema))?; let datatype = to_arrow_datatype(expr.datatype.as_ref().unwrap()); let func = min_max_udaf(&datatype, true); - let child = Self::cast_min_max_child(child, &datatype, &schema)?; + let child = + Self::cast_min_max_child(child, &datatype, &schema, dynamic_filter_enabled)?; AggregateExprBuilder::new(func, vec![child]) .schema(schema) @@ -877,7 +895,20 @@ impl PhysicalPlanner { child: Arc, datatype: &DataType, schema: &SchemaRef, + dynamic_filter_enabled: bool, ) -> Result, ExecutionError> { + // Keep the identity cast when disabled: exposing the column makes DataFusion + // maintain per-batch bounds even without an attached reader filter. + if dynamic_filter_enabled + && child.is::() + && matches!( + datatype, + DataType::Int8 | DataType::Int16 | DataType::Int32 | DataType::Int64 + ) + && child.data_type(schema.as_ref())? == *datatype + { + return Ok(child); + } if !matches!(datatype, DataType::Struct(_)) { return Ok(Arc::new(CastExpr::new(child, datatype.clone(), None))); } diff --git a/native/core/src/execution/planner/tests/min_max_reader_filter.rs b/native/core/src/execution/planner/tests/min_max_reader_filter.rs new file mode 100644 index 00000000000..e0848c0cac7 --- /dev/null +++ b/native/core/src/execution/planner/tests/min_max_reader_filter.rs @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::execution::metrics::utils::to_native_metric_node; +use datafusion::physical_plan::{aggregates::AggregateExec, collect}; +use parquet::arrow::ArrowWriter; +use parquet::file::properties::{EnabledStatistics, WriterProperties}; +use spark_expression::agg_expr::ExprStruct as AggExprStruct; +use spark_operator::{AggregateMode, HashAggregate}; + +fn native_scan(is_max: bool) -> (tempfile::NamedTempFile, Operator) { + let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, true)])); + let values = if is_max { + vec![20, 21, 10, 11, 0, 1] + } else { + vec![0, 1, 10, 11, 20, 21] + }; + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(values))], + ) + .unwrap(); + let file = tempfile::NamedTempFile::new().unwrap(); + let properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(2)) + .set_statistics_enabled(EnabledStatistics::Chunk) + .set_dictionary_enabled(false) + .build(); + let mut writer = + ArrowWriter::try_new(file.reopen().unwrap(), schema, Some(properties)).unwrap(); + writer.write(&batch).unwrap(); + assert_eq!(writer.close().unwrap().num_row_groups(), 3); + + let field = spark_operator::SparkStructField { + name: "key".to_owned(), + data_type: Some(create_proto_datatype()), + nullable: true, + ..Default::default() + }; + let file_size = file.as_file().metadata().unwrap().len() as i64; + let scan = Operator { + op_struct: Some(OpStruct::NativeScan(spark_operator::NativeScan { + common: Some(spark_operator::NativeScanCommon { + data_schema: vec![field.clone()], + required_schema: vec![field], + projection_vector: vec![0], + session_timezone: "UTC".to_owned(), + ..Default::default() + }), + file_partition: Some(spark_operator::SparkFilePartition { + partitioned_file: vec![spark_operator::SparkPartitionedFile { + file_path: url::Url::from_file_path(file.path()).unwrap().to_string(), + length: file_size, + file_size, + ..Default::default() + }], + }), + ..Default::default() + })), + ..Default::default() + }; + (file, scan) +} + +fn min_max(is_max: bool) -> spark_expression::AggExpr { + let child = Some(create_bound_reference(0)); + let datatype = Some(create_proto_datatype()); + spark_expression::AggExpr { + expr_struct: Some(if is_max { + AggExprStruct::Max(spark_expression::Max { child, datatype }) + } else { + AggExprStruct::Min(spark_expression::Min { child, datatype }) + }), + ..Default::default() + } +} + +fn aggregate(input: Operator, aggregate: HashAggregate) -> Operator { + Operator { + children: vec![input], + op_struct: Some(OpStruct::HashAgg(aggregate)), + ..Default::default() + } +} + +#[tokio::test] +async fn serialized_min_max_filter_controls_pruning_and_metric_ownership() { + for is_max in [false, true] { + let (_file, scan) = native_scan(is_max); + for enabled in [false, true] { + let mut config = SessionConfig::new() + .with_batch_size(2) + .with_target_partitions(1) + .with_parquet_page_index_pruning(false); + config.options_mut().execution.parquet.pushdown_filters = false; + let session = Arc::new(SessionContext::new_with_config(config)); + let planner = PhysicalPlanner::new(Arc::clone(&session), 0); + let operator = aggregate( + scan.clone(), + HashAggregate { + agg_exprs: vec![min_max(is_max)], + mode: AggregateMode::Partial as i32, + dynamic_filter_enabled: enabled, + ..Default::default() + }, + ); + // Spark can have several partitions, while each native task reads its + // own one-partition scan and owns an independent aggregate bound. + let (scans, _, planned) = planner.create_plan(&operator, &mut vec![], 4).unwrap(); + assert!(scans.is_empty()); + assert_eq!( + planned.native_plan.name(), + if enabled { + "CometMinMaxReaderFilterExec" + } else { + "AggregateExec" + } + ); + if !enabled { + assert!(planned + .native_plan + .dynamic_expressions_produced() + .is_empty()); + } + let batches = collect(Arc::clone(&planned.native_plan), session.task_ctx()) + .await + .unwrap(); + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + if is_max { 21 } else { 0 } + ); + let metrics = to_native_metric_node(&planned).unwrap(); + assert_eq!(metrics.metrics["output_rows"], 1); + assert!(!metrics.metrics.contains_key("bytes_scanned")); + assert_eq!(metrics.children.len(), 1); + let scan_metrics = &metrics.children[0].metrics; + assert_eq!(scan_metrics["output_rows"], if enabled { 2 } else { 6 }); + assert_eq!( + scan_metrics["row_groups_pruned_dynamic_filter"], + if enabled { 2 } else { 0 } + ); + if enabled { + assert_eq!(metrics.metrics["dynamic_filter_minmax_filters_attached"], 1); + assert_eq!( + metrics + .metrics + .get("dynamic_filter_minmax_filters_skipped") + .copied() + .unwrap_or_default(), + 0 + ); + } else { + assert!(!metrics + .metrics + .contains_key("dynamic_filter_minmax_filters_attached")); + } + } + } +} + +#[test] +fn state_merging_and_jvm_input_keep_ordinary_aggregation() { + let (_file, scan) = native_scan(false); + let planner = PhysicalPlanner::default(); + for (input, mode) in [ + (scan.clone(), AggregateMode::Final), + (scan, AggregateMode::PartialMerge), + (create_scan(), AggregateMode::Partial), + ] { + let operator = aggregate( + input, + HashAggregate { + agg_exprs: vec![min_max(false)], + mode: mode as i32, + dynamic_filter_enabled: true, + ..Default::default() + }, + ); + let (_, _, planned) = planner.create_plan(&operator, &mut vec![], 1).unwrap(); + let aggregate = if mode == AggregateMode::PartialMerge { + &planned.additional_native_plans[0] + } else { + &planned.native_plan + }; + assert_eq!(aggregate.name(), "AggregateExec"); + } +} + +#[test] +fn min_max_keeps_real_casts_and_computed_arguments() { + let (_file, scan) = native_scan(false); + let planner = PhysicalPlanner::default(); + let literal = Expr { + expr_struct: Some(ExprStruct::Literal(spark_expression::Literal { + datatype: Some(create_proto_datatype()), + value: Some(literal::Value::IntVal(7)), + is_null: false, + })), + ..Default::default() + }; + for (child, datatype) in [ + (create_bound_reference(0), create_int64_datatype()), + (literal, create_proto_datatype()), + ] { + let operator = aggregate( + scan.clone(), + HashAggregate { + agg_exprs: vec![spark_expression::AggExpr { + expr_struct: Some(AggExprStruct::Min(spark_expression::Min { + child: Some(child), + datatype: Some(datatype), + })), + ..Default::default() + }], + mode: AggregateMode::Partial as i32, + dynamic_filter_enabled: true, + ..Default::default() + }, + ); + let (_, _, planned) = planner.create_plan(&operator, &mut vec![], 1).unwrap(); + let aggregate = planned.native_plan.downcast_ref::().unwrap(); + assert!(aggregate.aggr_expr()[0].expressions()[0] + .is::()); + } +} diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto index 82699bfa737..30a18d100e3 100644 --- a/native/proto/src/proto/operator.proto +++ b/native/proto/src/proto/operator.proto @@ -953,6 +953,8 @@ message HashAggregate { // keys (ascending, nulls first). The native planner sorts the aggregate output on the // grouping columns unless the aggregate already emits its groups in that order. bool ordered_by_grouping_keys = 9; + // Attach this partial MIN/MAX execution's live bound to an eligible native reader. + bool dynamic_filter_enabled = 10; } message Limit { diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 04fb3e537f7..c3f0f72f044 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -752,6 +752,20 @@ object CometConf extends ShimCometConf { .booleanConf .createWithDefault(false) + val COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED: ConfigEntry[Boolean] = + conf(s"$COMET_EXEC_CONFIG_PREFIX.aggregate.dynamicFilter.enabled") + .category(CATEGORY_EXEC) + .doc( + "Experimental opt-in: use a partial MIN or MAX's improving bound to prune native " + + "Parquet row groups. Supports one unfiltered, non-distinct aggregate over a direct " + + "signed integer column, without grouping. Bounds remain local to each task. " + + "Expressions, filters, limits, Spark exchanges and JVM/Arrow boundaries stop reader " + + "attachment. Unsupported plans and encrypted scans retain ordinary aggregation. " + + "Inspects footers before execution and skips pruning when any file has more than " + + "64 row groups or when retained footer metadata exceeds 8 MiB.") + .booleanConf + .createWithDefault(false) + val COMET_EXEC_JOIN_DYNAMIC_FILTER_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.join.dynamicFilter.enabled") .category(CATEGORY_EXEC) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index f0f7e900aa8..bfa21038612 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -2335,6 +2335,8 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] { // handles that shape. if (aggregateExpressions.isEmpty) { val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() + hashAggBuilder.setDynamicFilterEnabled( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get(aggregate.conf)) hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) hashAggBuilder.setOrderedByGroupingKeys(orderedByGroupingKeys) // Spark has no expression mode to serialize here. An empty aggregate with a required child @@ -2398,6 +2400,8 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] { if (childOp.nonEmpty && groupingExprs.forall(_.isDefined) && aggExprs.forall(_.isDefined)) { val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() + hashAggBuilder.setDynamicFilterEnabled( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get(aggregate.conf)) hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) hashAggBuilder.addAllAggExprs(aggExprs.map(_.get).asJava) hashAggBuilder.setModeValue(mode.getNumber) @@ -2736,6 +2740,12 @@ abstract class CometBaseAggregateExec def resultExpressions: Seq[NamedExpression] def input: Seq[Attribute] + def dynamicFilterEnabled: Boolean = { + // Result expressions can wrap the serialized aggregate in a Projection. + val aggregateOp = if (nativeOp.hasProjection) nativeOp.getChildren(0) else nativeOp + aggregateOp.getHashAgg.getDynamicFilterEnabled + } + // The aggExprs could be empty. For example, if the aggregate functions only have // distinct aggregate functions or only have group by, the aggExprs is empty and // modes is empty too. @@ -2775,6 +2785,7 @@ abstract class CometBaseAggregateExec this.resultExpressions == other.resultExpressions && this.input == other.input && this.modes == other.modes && + this.dynamicFilterEnabled == other.dynamicFilterEnabled && this.child == other.child && this.serializedPlanOpt == other.serializedPlanOpt case _ => @@ -2791,10 +2802,15 @@ abstract class CometBaseAggregateExec resultExpressions, input, modes, + Boolean.box(dynamicFilterEnabled), child) override lazy val metrics: Map[String, SQLMetric] = { - val baseline = CometMetricNode.baselineMetrics(sparkContext) + val baseline = CometMetricNode.baselineMetrics(sparkContext) ++ Map( + "dynamic_filter_minmax_filters_attached" -> + SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters attached"), + "dynamic_filter_minmax_filters_skipped" -> + SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters skipped")) if (groupingExpressions.nonEmpty) { baseline ++ CometMetricNode.aggregateMetrics(sparkContext) } else { diff --git a/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala new file mode 100644 index 00000000000..c8bf8a7c2e0 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.exec + +import scala.jdk.CollectionConverters._ + +import org.apache.spark.sql.CometTestBase +import org.apache.spark.sql.catalyst.expressions.aggregate.{Final, Partial} +import org.apache.spark.sql.comet.{CometBaseAggregateExec, CometHashAggregateExec, CometNativeScanExec, CometSparkToColumnarExec} +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{ByteType, IntegerType, LongType, ShortType} + +import org.apache.comet.CometConf + +class CometMinMaxSuite extends CometTestBase { + + private val attached = "dynamic_filter_minmax_filters_attached" + private val skipped = "dynamic_filter_minmax_filters_skipped" + + private def aggregates(plan: SparkPlan): Seq[CometBaseAggregateExec] = + collect(plan) { case aggregate: CometBaseAggregateExec => aggregate } + + private def assertAttachments(plan: SparkPlan, expected: Long): Unit = { + val nodes = aggregates(plan) + assert(nodes.nonEmpty, s"Expected native aggregation:\n$plan") + assert(nodes.filter(_.modes == Seq(Partial)).map(_.metrics(attached).value).sum == expected) + assert(nodes.filterNot(_.modes == Seq(Partial)).forall(_.metrics(attached).value == 0L)) + assert(nodes.forall(_.metrics(skipped).value == 0L)) + } + + test("MIN/MAX filtering is opt-in and captured in the native aggregate plan") { + assert(!CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get()) + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + withParquetTable(Seq(Tuple1(1), Tuple1(-2)), "minmax_input") { + for (enabled <- Seq(false, true)) { + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { + val (_, plan) = checkSparkAnswerAndOperator( + sql("SELECT MIN(_1) + 1 FROM minmax_input"), + Seq(classOf[CometHashAggregateExec])) + assert(aggregates(plan).forall(_.dynamicFilterEnabled == enabled)) + assert(aggregates(plan).find(_.modes == Seq(Final)).get.nativeOp.hasProjection) + val partial = collect(plan) { + case aggregate: CometHashAggregateExec if aggregate.modes == Seq(Partial) => + aggregate + }.head + assert(partial.nativeOp.getHashAgg.getDynamicFilterEnabled == enabled) + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> (!enabled).toString) { + assert(aggregates(plan).forall(_.dynamicFilterEnabled == enabled)) + } + val changed = partial.nativeOp.toBuilder + .setHashAgg(partial.nativeOp.getHashAgg.toBuilder.setDynamicFilterEnabled(!enabled)) + .build() + assert(partial.canonicalized != partial.copy(nativeOp = changed).canonicalized) + } + } + } + } + } + + for ((keyType, parquetKey) <- Seq( + (ByteType, "int32 k (INT_8)"), + (ShortType, "int32 k (INT_16)"), + (IntegerType, "int32 k"), + (LongType, "int64 k"))) { + test(s"MIN/MAX prune their Spark Parquet reader for ${keyType.sql}") { + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "1", + CometConf.COMET_BATCH_SIZE.key -> "100", + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.parquet.enable_page_index" -> "false", + CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> "false") { + withTempDir { dir => + val path = new org.apache.hadoop.fs.Path(dir.toURI.toString, "minmax-groups.parquet") + val schema = org.apache.parquet.schema.MessageTypeParser.parseMessageType(s""" + |message root { + | required int64 payload; + | required $parquetKey; + |} + |""".stripMargin) + val writer = createParquetWriter(schema, path, rowGroupSize = 1L) + try { + // MIN improves 28 -> -20 -> -128; MAX reaches 127 in the first group. + // The second physical column also exercises the reader's column mapping. + Seq( + 28 until 128, + -20 until 80, + 0 until 100, + -128 until -28, + -80 until 20, + 20 until 120).flatten.foreach { key => + val row = new org.apache.parquet.example.data.simple.SimpleGroup(schema) + row.add(0, key.toLong) + if (keyType == LongType) row.add(1, key.toLong) else row.add(1, key) + writer.write(row) + } + } finally { + writer.close() + } + val footer = org.apache.parquet.hadoop.ParquetFileReader.open( + org.apache.parquet.hadoop.util.HadoopInputFile + .fromPath(path, spark.sessionState.newHadoopConf())) + try { + assert( + footer.getFooter.getBlocks.asScala.map(_.getRowCount).toSeq == Seq.fill(6)(100L)) + } finally { + footer.close() + } + withParquetTable(path.toString, "minmax_groups") { + assert(spark.table("minmax_groups").schema("k").dataType == keyType) + for (function <- Seq("MIN", "MAX")) { + val scans = Seq.newBuilder[Map[String, Long]] + for (enabled <- Seq(false, true)) { + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { + val (_, plan) = checkSparkAnswerAndOperator( + sql(s"SELECT $function(k) FROM minmax_groups"), + Seq(classOf[CometHashAggregateExec], classOf[CometNativeScanExec])) + assertAttachments(plan, if (enabled) 1L else 0L) + assert(aggregates(plan).forall(aggregate => + !aggregate.metrics.contains("bytes_scanned"))) + val scan = collect(plan) { case scan: CometNativeScanExec => scan }.head + scans += scan.metrics.map { case (name, metric) => name -> metric.value } + } + } + val Seq(disabled, enabled) = scans.result() + withClue(s"$function(${keyType.sql}): ") { + assert(disabled("output_rows") == 600L) + assert(disabled("row_groups_pruned_dynamic_filter") == 0L) + assert(enabled("row_groups_pruned_dynamic_filter") >= 3L) + assert(enabled("output_rows") <= 300L) + assert(enabled("bytes_scanned") < disabled("bytes_scanned")) + } + } + } + } + } + } + } + + for (adaptive <- Seq(false, true)) { + test(s"MIN/MAX filters stay within Spark partitions with AQE=$adaptive") { + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> adaptive.toString, + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "3", + SQLConf.FILES_MAX_PARTITION_BYTES.key -> "4194304") { + withTempPath { path => + spark + .range(0, 120, 1, 3) + .selectExpr("CAST(119 - id AS INT) AS k") + .write + .parquet(path.getCanonicalPath) + withParquetTable(path.getCanonicalPath, "minmax_partitions") { + for (enabled <- Seq(false, true); function <- Seq("MIN", "MAX")) { + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { + val (_, plan) = checkSparkAnswerAndOperator( + sql(s"SELECT $function(k) FROM minmax_partitions"), + Seq(classOf[CometHashAggregateExec], classOf[CometNativeScanExec])) + val scan = collect(plan) { case scan: CometNativeScanExec => scan }.head + assert(scan.executeColumnar().getNumPartitions == 3) + assertAttachments(plan, if (enabled) 3L else 0L) + } + } + } + } + } + } + } + + test("unsupported MIN/MAX plans retain ordinary aggregation") { + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + withParquetTable((0 until 100).map(i => (i - 50, i % 2)), "minmax_input") { + val queries = Seq( + "SELECT MIN(_1), MAX(_1) FROM minmax_input", + "SELECT _2, MIN(_1) FROM minmax_input GROUP BY _2", + "SELECT MIN(_1 + 1) FROM minmax_input", + "SELECT MIN(CAST(_1 AS BIGINT)) FROM minmax_input", + "SELECT MIN(_1) FILTER (WHERE _2 = 0) FROM minmax_input", + "SELECT MIN(_1) FROM minmax_input WHERE _2 = 0", + "SELECT MIN(_1) FROM (SELECT _1 FROM minmax_input ORDER BY _1 LIMIT 10)") + for (enabled <- Seq(false, true); query <- queries) { + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { + withClue(s"enabled=$enabled: $query") { + val (_, plan) = + checkSparkAnswerAndOperator(sql(query), Seq(classOf[CometHashAggregateExec])) + assertAttachments(plan, 0L) + } + } + } + } + } + } + + test("MIN/MAX filtering does not cross a JVM reader boundary") { + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false", + CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true", + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> "true") { + withParquetTable(Seq(Tuple1(-2), Tuple1(1)), "minmax_input") { + val (_, plan) = checkSparkAnswerAndOperator( + sql("SELECT MIN(_1) FROM minmax_input"), + Seq(classOf[CometHashAggregateExec], classOf[CometSparkToColumnarExec])) + val partial = aggregates(plan).find(_.modes == Seq(Partial)).get + assert(collect(partial.child) { case bridge: CometSparkToColumnarExec => + bridge + }.nonEmpty) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.isEmpty) + assertAttachments(plan, 0L) + } + } + } + + test("MIN/MAX preserve empty, null, and duplicate integer results") { + val inputs: Seq[Seq[Tuple1[Option[Int]]]] = Seq( + Seq.empty[Tuple1[Option[Int]]], + Seq(Tuple1(None), Tuple1(None)), + Seq(Tuple1(Some(-2)), Tuple1(Some(-2)), Tuple1(None), Tuple1(Some(3)))) + for (input <- inputs) { + withParquetTable(input, "minmax_input") { + for (enabled <- Seq(false, true); function <- Seq("MIN", "MAX")) { + withSQLConf( + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { + checkSparkAnswerAndOperator(s"SELECT $function(_1) FROM minmax_input") + } + } + } + } + } +} From 0267e52ab0d1ab52622294b4a2a0d173ce4b55de Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Mon, 16 Nov 2026 22:24:06 +0000 Subject: [PATCH 2/4] fix: exempt raw text fixture from archive license check --- dev/release/rat_exclude_files.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 8ffd442126b..2a90106ec5b 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -31,6 +31,7 @@ spark/src/test/resources/tpch-query-results/*.out spark/src/test/resources/tpch-extended/q*.sql spark/src/test/resources/test-data/*.csv spark/src/test/resources/test-data/*.ndjson +spark/src/test/resources/test-data/text-test-1.txt spark/src/test/resources/abfs-auth-parity/*.json spark/inspections/CometTPC*results.txt benchmarks/tpc/queries/**/*.sql From 92dbff1ae1dceee1277456fc4787d24626c7936c Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Wed, 18 Nov 2026 17:40:50 +0000 Subject: [PATCH 3/4] fix: scope MIN/MAX filter flags and metrics to partial aggregates --- .../apache/spark/sql/comet/operators.scala | 28 +++++++++---------- .../apache/comet/exec/CometMinMaxSuite.scala | 23 +++++++++++---- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala index bfa21038612..50ca5573d6a 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala @@ -2335,8 +2335,6 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] { // handles that shape. if (aggregateExpressions.isEmpty) { val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() - hashAggBuilder.setDynamicFilterEnabled( - CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get(aggregate.conf)) hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) hashAggBuilder.setOrderedByGroupingKeys(orderedByGroupingKeys) // Spark has no expression mode to serialize here. An empty aggregate with a required child @@ -2401,7 +2399,8 @@ trait CometBaseAggregate[T <: BaseAggregateExec] extends CometOperatorSerde[T] { aggExprs.forall(_.isDefined)) { val hashAggBuilder = OperatorOuterClass.HashAggregate.newBuilder() hashAggBuilder.setDynamicFilterEnabled( - CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get(aggregate.conf)) + groupingExpressions.isEmpty && modeSet == Set(Partial) && + CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.get(aggregate.conf)) hashAggBuilder.addAllGroupingExprs(groupingExprs.map(_.get).asJava) hashAggBuilder.addAllAggExprs(aggExprs.map(_.get).asJava) hashAggBuilder.setModeValue(mode.getNumber) @@ -2740,11 +2739,7 @@ abstract class CometBaseAggregateExec def resultExpressions: Seq[NamedExpression] def input: Seq[Attribute] - def dynamicFilterEnabled: Boolean = { - // Result expressions can wrap the serialized aggregate in a Projection. - val aggregateOp = if (nativeOp.hasProjection) nativeOp.getChildren(0) else nativeOp - aggregateOp.getHashAgg.getDynamicFilterEnabled - } + def dynamicFilterEnabled: Boolean = nativeOp.getHashAgg.getDynamicFilterEnabled // The aggExprs could be empty. For example, if the aggregate functions only have // distinct aggregate functions or only have group by, the aggExprs is empty and @@ -2806,16 +2801,21 @@ abstract class CometBaseAggregateExec child) override lazy val metrics: Map[String, SQLMetric] = { - val baseline = CometMetricNode.baselineMetrics(sparkContext) ++ Map( - "dynamic_filter_minmax_filters_attached" -> - SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters attached"), - "dynamic_filter_minmax_filters_skipped" -> - SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters skipped")) - if (groupingExpressions.nonEmpty) { + val baseline = CometMetricNode.baselineMetrics(sparkContext) + val aggregateMetrics = if (groupingExpressions.nonEmpty) { baseline ++ CometMetricNode.aggregateMetrics(sparkContext) } else { baseline } + if (dynamicFilterEnabled) { + aggregateMetrics ++ Map( + "dynamic_filter_minmax_filters_attached" -> + SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters attached"), + "dynamic_filter_minmax_filters_skipped" -> + SQLMetrics.createMetric(sparkContext, "MIN/MAX reader filters skipped")) + } else { + aggregateMetrics + } } override protected def outputExpressions: Seq[NamedExpression] = resultExpressions diff --git a/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala index c8bf8a7c2e0..b01db8af3ba 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala @@ -41,9 +41,12 @@ class CometMinMaxSuite extends CometTestBase { private def assertAttachments(plan: SparkPlan, expected: Long): Unit = { val nodes = aggregates(plan) assert(nodes.nonEmpty, s"Expected native aggregation:\n$plan") - assert(nodes.filter(_.modes == Seq(Partial)).map(_.metrics(attached).value).sum == expected) - assert(nodes.filterNot(_.modes == Seq(Partial)).forall(_.metrics(attached).value == 0L)) - assert(nodes.forall(_.metrics(skipped).value == 0L)) + val (enabled, disabled) = nodes.partition(_.dynamicFilterEnabled) + assert(enabled.forall(node => node.groupingExpressions.isEmpty && node.modes == Seq(Partial))) + assert(enabled.map(_.metrics(attached).value).sum == expected) + assert(enabled.forall(_.metrics(skipped).value == 0L)) + assert(disabled.forall(node => + !node.metrics.contains(attached) && !node.metrics.contains(skipped))) } test("MIN/MAX filtering is opt-in and captured in the native aggregate plan") { @@ -56,16 +59,22 @@ class CometMinMaxSuite extends CometTestBase { val (_, plan) = checkSparkAnswerAndOperator( sql("SELECT MIN(_1) + 1 FROM minmax_input"), Seq(classOf[CometHashAggregateExec])) - assert(aggregates(plan).forall(_.dynamicFilterEnabled == enabled)) - assert(aggregates(plan).find(_.modes == Seq(Final)).get.nativeOp.hasProjection) + val finalAggregate = aggregates(plan).find(_.modes == Seq(Final)).get + assert(finalAggregate.nativeOp.hasProjection) + assert(!finalAggregate.dynamicFilterEnabled) + assert(!finalAggregate.metrics.contains(attached)) + assert(!finalAggregate.metrics.contains(skipped)) val partial = collect(plan) { case aggregate: CometHashAggregateExec if aggregate.modes == Seq(Partial) => aggregate }.head assert(partial.nativeOp.getHashAgg.getDynamicFilterEnabled == enabled) + assert(partial.metrics.contains(attached) == enabled) + assert(partial.metrics.contains(skipped) == enabled) withSQLConf( CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> (!enabled).toString) { - assert(aggregates(plan).forall(_.dynamicFilterEnabled == enabled)) + assert(partial.dynamicFilterEnabled == enabled) + assert(!finalAggregate.dynamicFilterEnabled) } val changed = partial.nativeOp.toBuilder .setHashAgg(partial.nativeOp.getHashAgg.toBuilder.setDynamicFilterEnabled(!enabled)) @@ -192,7 +201,9 @@ class CometMinMaxSuite extends CometTestBase { withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { withParquetTable((0 until 100).map(i => (i - 50, i % 2)), "minmax_input") { val queries = Seq( + "SELECT DISTINCT _2 FROM minmax_input", "SELECT MIN(_1), MAX(_1) FROM minmax_input", + "SELECT MIN(_1), COUNT(DISTINCT _2) FROM minmax_input", "SELECT _2, MIN(_1) FROM minmax_input GROUP BY _2", "SELECT MIN(_1 + 1) FROM minmax_input", "SELECT MIN(CAST(_1 AS BIGINT)) FROM minmax_input", From 081687d1f45e4d5d9b9bbc0e3a29a7d70dd6fa05 Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Wed, 18 Nov 2026 17:46:42 +0000 Subject: [PATCH 4/4] test: cover MIN/MAX pruning with nulls and integer limits --- .../aggregate/min_max_dynamic_filter.sql | 45 +++++++++++++ .../apache/comet/exec/CometMinMaxSuite.scala | 63 ++++++++----------- 2 files changed, 72 insertions(+), 36 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/aggregate/min_max_dynamic_filter.sql diff --git a/spark/src/test/resources/sql-tests/expressions/aggregate/min_max_dynamic_filter.sql b/spark/src/test/resources/sql-tests/expressions/aggregate/min_max_dynamic_filter.sql new file mode 100644 index 00000000000..23967301049 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/aggregate/min_max_dynamic_filter.sql @@ -0,0 +1,45 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- ConfigMatrix: spark.comet.exec.aggregate.dynamicFilter.enabled=false,true + +statement +CREATE TABLE min_max_dynamic_filter(k int) USING parquet + +query +SELECT MIN(k) FROM min_max_dynamic_filter + +query +SELECT MAX(k) FROM min_max_dynamic_filter + +statement +INSERT INTO min_max_dynamic_filter VALUES (NULL), (NULL) + +query +SELECT MIN(k) FROM min_max_dynamic_filter + +query +SELECT MAX(k) FROM min_max_dynamic_filter + +statement +INSERT INTO min_max_dynamic_filter VALUES (-2), (-2), (3) + +query +SELECT MIN(k) FROM min_max_dynamic_filter + +query +SELECT MAX(k) FROM min_max_dynamic_filter diff --git a/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala index b01db8af3ba..1f58252e1e1 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometMinMaxSuite.scala @@ -86,11 +86,11 @@ class CometMinMaxSuite extends CometTestBase { } } - for ((keyType, parquetKey) <- Seq( - (ByteType, "int32 k (INT_8)"), - (ShortType, "int32 k (INT_16)"), - (IntegerType, "int32 k"), - (LongType, "int64 k"))) { + for ((keyType, parquetKey, minValue, maxValue) <- Seq( + (ByteType, "int32 k (INT_8)", Byte.MinValue.toLong, Byte.MaxValue.toLong), + (ShortType, "int32 k (INT_16)", Short.MinValue.toLong, Short.MaxValue.toLong), + (IntegerType, "int32 k", Int.MinValue.toLong, Int.MaxValue.toLong), + (LongType, "int64 k", Long.MinValue, Long.MaxValue))) { test(s"MIN/MAX prune their Spark Parquet reader for ${keyType.sql}") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", @@ -104,23 +104,29 @@ class CometMinMaxSuite extends CometTestBase { val schema = org.apache.parquet.schema.MessageTypeParser.parseMessageType(s""" |message root { | required int64 payload; - | required $parquetKey; + | optional $parquetKey; |} |""".stripMargin) val writer = createParquetWriter(schema, path, rowGroupSize = 1L) try { - // MIN improves 28 -> -20 -> -128; MAX reaches 127 in the first group. + // Both bounds improve before the NULL and type-extreme groups are reached. // The second physical column also exercises the reader's column mapping. - Seq( - 28 until 128, - -20 until 80, - 0 until 100, - -128 until -28, - -80 until 20, - 20 until 120).flatten.foreach { key => + def nonNullGroup(values: Seq[Long]): Seq[Option[Long]] = values.map(Some(_)) + val rowGroups = Seq( + nonNullGroup(-50L until 50L), + nonNullGroup((-100L until -50L) ++ (50L until 100L)), + Seq.fill(100)(None), + nonNullGroup(-50L until 50L), + nonNullGroup(Seq.fill(50)(Seq(minValue, maxValue)).flatten), + nonNullGroup(-80L until 20L), + nonNullGroup(20L until 120L), + nonNullGroup(-10L until 90L)) + rowGroups.flatten.zipWithIndex.foreach { case (key, index) => val row = new org.apache.parquet.example.data.simple.SimpleGroup(schema) - row.add(0, key.toLong) - if (keyType == LongType) row.add(1, key.toLong) else row.add(1, key) + row.add(0, index.toLong) + key.foreach { value => + if (keyType == LongType) row.add(1, value) else row.add(1, value.toInt) + } writer.write(row) } } finally { @@ -130,8 +136,9 @@ class CometMinMaxSuite extends CometTestBase { org.apache.parquet.hadoop.util.HadoopInputFile .fromPath(path, spark.sessionState.newHadoopConf())) try { - assert( - footer.getFooter.getBlocks.asScala.map(_.getRowCount).toSeq == Seq.fill(6)(100L)) + val rowGroups = footer.getFooter.getBlocks.asScala + assert(rowGroups.map(_.getRowCount).toSeq == Seq.fill(8)(100L)) + assert(rowGroups(2).getColumns.get(1).getStatistics.getNumNulls == 100L) } finally { footer.close() } @@ -154,10 +161,10 @@ class CometMinMaxSuite extends CometTestBase { } val Seq(disabled, enabled) = scans.result() withClue(s"$function(${keyType.sql}): ") { - assert(disabled("output_rows") == 600L) + assert(disabled("output_rows") == 800L) assert(disabled("row_groups_pruned_dynamic_filter") == 0L) assert(enabled("row_groups_pruned_dynamic_filter") >= 3L) - assert(enabled("output_rows") <= 300L) + assert(enabled("output_rows") <= 400L) assert(enabled("bytes_scanned") < disabled("bytes_scanned")) } } @@ -244,20 +251,4 @@ class CometMinMaxSuite extends CometTestBase { } } - test("MIN/MAX preserve empty, null, and duplicate integer results") { - val inputs: Seq[Seq[Tuple1[Option[Int]]]] = Seq( - Seq.empty[Tuple1[Option[Int]]], - Seq(Tuple1(None), Tuple1(None)), - Seq(Tuple1(Some(-2)), Tuple1(Some(-2)), Tuple1(None), Tuple1(Some(3)))) - for (input <- inputs) { - withParquetTable(input, "minmax_input") { - for (enabled <- Seq(false, true); function <- Seq("MIN", "MAX")) { - withSQLConf( - CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key -> enabled.toString) { - checkSparkAnswerAndOperator(s"SELECT $function(_1) FROM minmax_input") - } - } - } - } - } }