From d2f9fdfc6061c81f6fe4f4d17495d438963a6aeb Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Thu, 19 Nov 2026 05:25:15 +0000 Subject: [PATCH 1/2] perf: add MIN/MAX reader-filter benchmark --- .../sql/benchmark/CometMinMaxBenchmark.scala | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometMinMaxBenchmark.scala diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometMinMaxBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometMinMaxBenchmark.scala new file mode 100644 index 00000000000..1f8842aa5fa --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometMinMaxBenchmark.scala @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.benchmark + +import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.Path +import org.apache.parquet.hadoop.ParquetFileReader +import org.apache.parquet.hadoop.util.HadoopInputFile +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.expressions.aggregate.Partial +import org.apache.spark.sql.comet.{CometBaseAggregateExec, CometNativeScanExec} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.comet.CometConf + +/** + * Compare MIN/MAX with reader filtering disabled and enabled. Use a native release build: + * + * {{{ + * make benchmark-org.apache.spark.sql.benchmark.CometMinMaxBenchmark -- \ + * 1048576 1,4 ascending,descending,random,constant 5 off,on + * }}} + * + * Arguments are rows, scan partitions, layouts, minimum measured iterations, and mode order. + * Defaults match the example. Repeat with on,off to reverse the case order. A smoke run can use + * 16384 rows and three iterations. Timings include planning and collection; fixture creation, + * Spark result comparisons, plan checks, and metric collection run outside timings. Metrics come + * from an untimed execution. Page indexes and decoder row filters are disabled to isolate + * row-group pruning. Each file has multiple row groups and runs in a separate scan task. + */ +object CometMinMaxBenchmark extends CometBenchmarkBase { + + private val modes = Map("off" -> false, "on" -> true) + private val filterConfig = CometConf.COMET_EXEC_AGGREGATE_DYNAMIC_FILTER_ENABLED.key + + private val nativeConfigs = Seq( + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "true", + CometConf.COMET_BATCH_SIZE.key -> "4096", + CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> "false", + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.parquet.enable_page_index" -> "false", + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.LEAF_NODE_DEFAULT_PARALLELISM.key -> "1", + SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "4194304") + + private def writeInput( + directory: String, + rows: Int, + partitions: Int, + layout: String): (Long, Seq[Int]) = { + val key = layout match { + case "ascending" => s"id - ${rows / 2}" + case "descending" => s"${rows / 2} - id - 1" + case "random" => s"pmod(xxhash64(id), $rows) - ${rows / 2}" + case "constant" => "0" + } + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(0, rows.toLong, 1, partitions) + .selectExpr(s"CAST($key AS INT) AS k") + .write + // Fixed-width, uncompressed pages keep constant values from collapsing into one group. + .option("compression", "uncompressed") + .option("parquet.writer.version", "v1") + .option("parquet.enable.dictionary", "false") + // Target eight row groups per file, including for constant data. + .option("parquet.block.size", (rows.toLong / partitions * 4 / 8).toString) + .parquet(directory) + } + val path = new Path(directory) + val files = path + .getFileSystem(spark.sessionState.newHadoopConf()) + .listStatus(path) + .filter(_.getPath.getName.endsWith(".parquet")) + require(files.length == partitions, s"Expected $partitions files, found ${files.length}") + val rowGroups = files.toSeq.map { file => + val reader = ParquetFileReader.open( + HadoopInputFile.fromPath(file.getPath, spark.sessionState.newHadoopConf())) + try reader.getFooter.getBlocks.asScala.size + finally reader.close() + } + require(rowGroups.forall(_ > 1), s"Expected multiple row groups per file: $rowGroups") + (files.map(_.getLen).max, rowGroups) + } + + private def verifyAndReport( + benchmark: Benchmark, + query: String, + expected: Seq[Row], + mode: String, + partitions: Int, + rows: Int): Unit = { + val enabled = modes(mode) + val df = spark.sql(query) + val plan = stripAQEPlan(df.queryExecution.executedPlan) + val partials = collect(plan) { + case node: CometBaseAggregateExec if node.modes == Seq(Partial) => node + } + val scans = collect(plan) { case node: CometNativeScanExec => node } + require(partials.size == 1 && scans.size == 1, s"Expected MIN/MAX and native scan:\n$plan") + val partial = partials.head + require(partial.child.isInstanceOf[CometNativeScanExec], s"Expected a direct scan:\n$plan") + require(partial.dynamicFilterEnabled == enabled) + require(scans.head.executeColumnar().getNumPartitions == partitions) + require(df.collect().toSeq == expected, s"Incorrect results in mode $mode") + val metrics = scans.head.metrics + if (!enabled) { + require(metrics("output_rows").value == rows.toLong) + require(metrics("row_groups_pruned_dynamic_filter").value == 0L) + } + require(metrics("row_groups_pruned_statistics").value == 0L) + require(metrics("page_index_rows_pruned").value == 0L) + require(metrics("pushdown_rows_pruned").value == 0L) + val readerWork = Seq( + "output_rows", + "bytes_scanned", + "scan_io_data_bytes", + "scan_io_metadata_bytes", + "row_groups_pruned_dynamic_filter", + "row_groups_pruned_statistics", + "page_index_rows_pruned", + "pushdown_rows_pruned") + .map(name => s"$name=${metrics(name).value}") + val attachment = Seq("attached", "skipped").map { suffix => + val name = s"dynamic_filter_minmax_filters_$suffix" + val value = if (enabled) partial.metrics(name).value else 0L + require(value == (if (enabled && suffix == "attached") partitions.toLong else 0L)) + s"$name=$value" + } + benchmark.out.println(s"mode=$mode, ${(readerWork ++ attachment).mkString(", ")}") + } + + override def runCometBenchmark(mainArgs: Array[String]): Unit = { + require( + mainArgs.length <= 5, + "Usage: CometMinMaxBenchmark [rows] [partitions] [layouts] [iterations] [off,on or on,off]") + val rows = mainArgs.headOption.map(_.toInt).getOrElse(1 << 20) + val partitions = mainArgs.lift(1).getOrElse("1,4").split(",").map(_.toInt).toSeq + val layouts = + mainArgs.lift(2).getOrElse("ascending,descending,random,constant").split(",").toSeq + val iterations = mainArgs.lift(3).map(_.toInt).getOrElse(5) + val modeOrder = mainArgs.lift(4).getOrElse("off,on").split(",").toSeq + require(rows >= 16384, "rows must be at least 16384") + require( + partitions.forall(p => p > 0 && rows % p == 0 && rows / p >= 4096), + "partitions must divide rows and leave at least 4096 rows per file") + require( + layouts.forall(Set("ascending", "descending", "random", "constant")), + "Unknown layout") + require(iterations > 0, "iterations must be positive") + require( + modeOrder.size == modes.size && modeOrder.toSet == modes.keySet, + "mode order must contain off and on exactly once") + + withTempPath { root => + withTempTable("minmax_input") { + for (p <- partitions; layout <- layouts) { + val directory = s"${root.getCanonicalPath}/$layout-p$p" + val (maxFileBytes, rowGroups) = writeInput(directory, rows, p, layout) + // Keep one file per scan task so its bound can improve across row groups. + val scanConfigs = nativeConfigs :+ + (SQLConf.FILES_MAX_PARTITION_BYTES.key -> (maxFileBytes + 4194304L).toString) + spark.read.parquet(directory).createOrReplaceTempView("minmax_input") + for (function <- Seq("MIN", "MAX")) { + val query = s"SELECT $function(k) FROM minmax_input" + var expected = Seq.empty[Row] + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + expected = spark.sql(query).collect().toSeq + } + val name = s"$function $layout, partitions=$p" + runBenchmark(name) { + val benchmark = new Benchmark( + name, + rows.toLong, + minNumIters = iterations, + warmupTime = 500.millis, + minTime = 500.millis, + output = output) + withSQLConf(scanConfigs: _*) { + benchmark.out.println(s"row groups per file: ${rowGroups.mkString(",")}") + modeOrder.foreach { mode => + withSQLConf(filterConfig -> modes(mode).toString) { + verifyAndReport(benchmark, query, expected, mode, p, rows) + } + benchmark.addCase(mode) { _ => + withSQLConf(filterConfig -> modes(mode).toString) { + spark.sql(query).collect() + } + } + } + benchmark.run() + } + } + } + } + } + } + } +} From 2d6b1842f1485857aa118e2f8b8b11eeab7f440f Mon Sep 17 00:00:00 2001 From: Ping Zhang Date: Thu, 19 Nov 2026 05:25:17 +0000 Subject: [PATCH 2/2] docs: describe MIN/MAX reader filtering and benchmarks --- docs/source/contributor-guide/benchmarking.md | 36 ++++++++++++++++ docs/source/user-guide/latest/metrics.md | 26 +++++++++++- .../user-guide/latest/tuning/operators.md | 42 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) diff --git a/docs/source/contributor-guide/benchmarking.md b/docs/source/contributor-guide/benchmarking.md index 1fe96170253..19a98af3f87 100644 --- a/docs/source/contributor-guide/benchmarking.md +++ b/docs/source/contributor-guide/benchmarking.md @@ -43,6 +43,42 @@ These can also be run as a suite on a dedicated machine, see [Micro Benchmarking on AWS EC2](benchmarking_micro_ec2.md). Published results are in [benchmarks/results/micro](https://github.com/apache/datafusion-comet/tree/main/benchmarks/results/micro). +## MIN/MAX Reader Filtering + +`CometMinMaxBenchmark` compares one `MIN(k)` or `MAX(k)` over an `INT` Parquet column with +`spark.comet.exec.aggregate.dynamicFilter.enabled` disabled (`off`) and enabled (`on`). Run it +with a release build; the make target builds and packages the native library: + +```shell +make benchmark-org.apache.spark.sql.benchmark.CometMinMaxBenchmark -- \ + 1048576 1,4 ascending,descending,random,constant 5 off,on +``` + +The arguments are total rows, scan partition counts, layouts, minimum measured iterations, and +mode order. The example uses one million rows, one or four files/tasks, and at least five measured iterations per +case. Use `16384 1,4 ascending,descending,random,constant 3 off,on` for a smoke run. The Spark master +is `local[1]` by default; set `COMET_BENCHMARK_MASTER` to change task concurrency and record it +alongside results. + +Files use uncompressed Parquet v1 pages with dictionary encoding disabled, so even constant +values reach the row-group size target. Each file has multiple row groups. Ascending input favors +`MIN` and descending input favors `MAX`; the opposite combinations measure overhead when the bound keeps improving. Random input +uses deterministic hashed keys, while constant input measures repeated values. Page-index pruning +and decoder row filtering are disabled to isolate row-group pruning. Only the key is scanned. + +Before timing, the suite checks results against Spark, verifies the direct scan/partial aggregate +plan and task count, and requires attachment in every enabled task. It prints scan output rows, +requested data bytes, reader data and metadata bytes, pruned row groups, and attachment counters +from this untimed execution. The timed cases include query planning and result collection, but +exclude data generation and validation. + +Repeat with `on,off` to check case-order effects, including JVM warmup and metadata-cache warmth. +Compare elapsed time as well as scan metrics: fewer rows or data bytes do not guarantee a faster +query, especially for small files or when footer inspection and scheduling dominate. Record the +Comet revision, build flags, Spark/JDK versions, machine, master, arguments, and both mode orders +with any reported results. See [MIN/MAX reader filtering](../user-guide/latest/tuning/operators.md#minmax-reader-filtering) +for supported plans and fallback limits. + ```{toctree} :hidden: diff --git a/docs/source/user-guide/latest/metrics.md b/docs/source/user-guide/latest/metrics.md index 0fcacd0ba8f..684d0d82425 100644 --- a/docs/source/user-guide/latest/metrics.md +++ b/docs/source/user-guide/latest/metrics.md @@ -87,6 +87,30 @@ Native aggregates with grouping keys report these additional metrics: Spill bytes from native sorts, aggregates, and sort-merge joins are also added to Spark's task-level `diskBytesSpilled` metric in every stage, not only in shuffle stages. +#### MIN/MAX Reader Filtering + +When `spark.comet.exec.aggregate.dynamicFilter.enabled=true` is captured in an ungrouped partial +aggregate's plan, its metric map includes these counters. See +[MIN/MAX reader filtering](tuning/operators.md#minmax-reader-filtering) for eligibility and limits. + +| Metric | Description | +| ------ | ----------- | +| `dynamic_filter_minmax_filters_attached` | Executions that attach their improving MIN/MAX bound to a native Parquet reader. | +| `dynamic_filter_minmax_filters_skipped` | Executions of an eligible MIN/MAX aggregate that skip reader attachment, including those exceeding the footer limits or using encryption. | + +Plans rejected before reader attachment is attempted do not increment either counter. An attached +filter can still prune nothing, including when a file's schema adaptation disables pruning. + +Reader work remains on the scan. Compare `output_rows`, `bytes_scanned`, +`row_groups_pruned_dynamic_filter`, and `row_groups_pruned_statistics` with filtering disabled. +`bytes_scanned` measures requested data/Bloom-filter ranges, excluding footer and page-index reads; +use `scan_io_metadata_bytes` to include the metadata work of inspecting footers. Row groups pruned +while reading a file increment `row_groups_pruned_dynamic_filter`; pruning when later files open +can instead increment `row_groups_pruned_statistics`. The statistics counter also includes other +predicates, so neither counter alone measures MIN/MAX savings. Page-index and decoder filtering +retain their existing `page_index_rows_pruned` and `pushdown_rows_pruned` counters. There are no +separate evaluated/pruned row counters on the aggregate. + ### Hash Joins With `spark.comet.exec.join.dynamicFilter.enabled=true`, native broadcast and shuffled hash joins @@ -231,7 +255,7 @@ execution metrics. Counters accumulate per scan operator; they do not instrument | `scan_io_metadata_cache_hits` | Successful, cache-eligible metadata opens requiring no storage reads. | | `scan_io_metadata_cache_misses` | Successful, cache-eligible metadata opens requiring storage reads. Failed opens and encrypted opens, which bypass this shared cache, increment neither cache counter. | | `scan_io_metadata_prefetches` | Opens that used a load which read storage ahead of them, started when the previous file opened (`spark.comet.parquet.prefetchNextFileMetadata.enabled`). Each also counts as a cache miss. | -| `row_groups_pruned_dynamic_filter` | Row groups skipped by a runtime predicate whose value changes during execution, such as a local TopK threshold. | +| `row_groups_pruned_dynamic_filter` | Row groups skipped by a runtime predicate whose value changes during execution, such as a local TopK threshold or MIN/MAX bound. | Reader-level and object-store bytes are two views of the same reads; do not add them together. Likewise, footer bytes are a subset of metadata bytes, not a third reader-level category. A warm diff --git a/docs/source/user-guide/latest/tuning/operators.md b/docs/source/user-guide/latest/tuning/operators.md index 058a6a00cd7..80bf016d7bd 100644 --- a/docs/source/user-guide/latest/tuning/operators.md +++ b/docs/source/user-guide/latest/tuning/operators.md @@ -205,6 +205,48 @@ skipping more likely. These settings only tune eligible plans. They cannot enabl while `spark.comet.exec.aggregate.skipPartial.enabled` is `false`, or for unsupported accumulators and modes. +## MIN/MAX Reader Filtering + +For a Parquet table `events` with a `BIGINT` column `event_id`, enable reader filtering before +planning the query: + +```sql +SET spark.comet.exec.aggregate.dynamicFilter.enabled=true; +SELECT MIN(event_id) FROM events; +``` + +This experimental option is disabled by default. Once a task has observed a non-null minimum, +its native Parquet reader can skip later row groups whose statistics prove that no value is +smaller. `MAX` works the same way with an improving maximum. Each task starts with a fresh bound; +bounds are not shared across Spark partitions or exchanges, or retained for later executions. +The aggregate still computes the result from the rows the reader emits. + +Reader filtering applies only to these plans: + +| Requirement | Supported scope | +| ----------- | --------------- | +| Aggregate | One unfiltered, non-distinct `MIN` or `MAX`, without grouping. Multiple aggregates, including `MIN` and `MAX` together, are excluded. | +| Argument | A direct `TINYINT`, `SMALLINT`, `INT`, or `BIGINT` column. | +| Stage | A partial aggregate consuming input rows. `Final` and `PartialMerge` stages do not produce reader filters. | +| Input | A direct native Parquet scan in the same Comet pipeline. Expressions, filters, limits, standalone projections, Spark exchanges, and JVM/Arrow boundaries between the aggregate and scan stop attachment. Column projection within the reader remains supported. | + +Unsupported plans retain ordinary aggregation. Scans with supplied file statistics or encryption +also skip attachment. Before reading rows, Comet inspects the task's file footers. If any file has +more than 64 row groups, or retained decoded footer metadata exceeds 8 MiB, the whole task uses +ordinary aggregation. This inspection adds startup work even when filtering saves no data reads. +For each file, schema adaptation disables pruning if it could hide a conversion error. Missing +statistics, including unknown null counts, can prevent pruning. + +Ascending data can establish a useful `MIN` bound early; descending data can do the same for `MAX`. +The opposite ordering can keep improving the bound until the end, leaving few or no row groups to +skip. Random data and constant values exercise different pruning opportunities. Compare filtering +enabled and disabled for your layout and partition count, including query elapsed time, emitted +scan rows, bytes read, and row groups pruned. An attached filter alone does not demonstrate a saving. +See [MIN/MAX metrics](../metrics.md#minmax-reader-filtering) for how to interpret these counters. + +Existing Parquet page-index and decoder-filter options can also use the bound. This feature adds +no separate filter over decoded scan batches. + ## Local TopK Fusion Set `spark.comet.exec.topK.fusion.enabled=true` to run an eligible local TopK in the same native