From 8230783b54e559ca5e032c586c6e66c894906a9d Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 16 Nov 2026 15:06:58 -0800 Subject: [PATCH 1/2] fix: keep joins native when AQE splits a skewed partition Spark's OptimizeSkewedJoin runs in AQE's query stage preparation rules, before CometRule, and puts an AQEShuffleReadExec between each Comet shuffle stage and the join it splits. CometExecRule converted the stage into a native input but left the read as it was, so the join, the sort under a sort-merge join and everything above the join in its stage ran in Spark, with no fallback reason. Convert such a read into a native shuffle input while AQE prepares a plan, the way the stage itself is converted, and let CometExchangeSink read it through ShuffleScan. The shuffle input then reads only the partitions the read specifies, as it already does for the reads that AQE adds after conversion. --- .../apache/comet/rules/CometExecRule.scala | 11 ++++ .../comet/serde/operator/CometSink.scala | 5 +- .../comet/exec/CometNativeShuffleSuite.scala | 54 ++++++++++++++++++- 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index 12b73d768aa..d67a8099f5e 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -673,6 +673,17 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false) case s @ ShuffleQueryStageExec(_, ReusedExchangeExec(_, _: CometShuffleExchangeExec), _) => convertToComet(s, CometExchangeSink).getOrElse(s) + // For an AQE shuffle read over a Comet shuffle stage, while AQE prepares a plan. Spark's + // OptimizeSkewedJoin runs before this rule in the same pass and splits a skewed partition + // by putting such a read between the stage and the join, so the join and everything above + // it can only be converted if the read is a native input too (issue #6530). The read then + // feeds the native plan in place of the stage, and reads only the partitions it specifies. + // AQE adds its other reads after the operators above them have been converted. + case read @ AQEShuffleReadExec(CometSinkPlaceHolder(_, _, stage: ShuffleQueryStageExec), _) + if queryStagePrep && stage.shuffle.isInstanceOf[CometShuffleExchangeExec] => + val stageRead = read.withNewChildren(Seq(stage)) + convertToComet(stageRead, CometExchangeSink).getOrElse(stageRead) + case s: ShuffleExchangeExec if shouldSkipCometShuffle(s) => preserveSparkAggregateBuffers(s) diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometSink.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometSink.scala index c5fc0e4858a..6f9378ed132 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometSink.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometSink.scala @@ -24,7 +24,7 @@ import scala.jdk.CollectionConverters._ import org.apache.spark.sql.comet.{CometNativeExec, CometSinkPlaceHolder} import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec +import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.exchange.ReusedExchangeExec import org.apache.spark.sql.types.DataType @@ -114,6 +114,9 @@ object CometExchangeSink extends CometSink[SparkPlan] { case ShuffleQueryStageExec(_, ReusedExchangeExec(_, s: CometShuffleExchangeExec), _) => Some(s) case s: CometShuffleExchangeExec => Some(s) + // An AQE read over a Comet shuffle stage, which CometExecRule converts for a skew join + case AQEShuffleReadExec(stage: ShuffleQueryStageExec, _) => + Some(stage.shuffle).collect { case s: CometShuffleExchangeExec => s } case _ => None } diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index bb06a09fbf3..f7e979898b5 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -43,14 +43,14 @@ import org.apache.spark.sql.catalyst.expressions.{Ascending, AttributeReference, import org.apache.spark.sql.catalyst.expressions.aggregate.Final import org.apache.spark.sql.catalyst.plans.logical.LocalRelation import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning} -import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometCollectLimitExec, CometExec, CometHashAggregateExec, CometLocalTableScanExec, CometMetricNode, CometNativeExec, CometNativeScanExec, CometScanWrapper, CometSparkToColumnarExec, CometTakeOrderedAndProjectExec} +import org.apache.spark.sql.comet.{CometBroadcastExchangeExec, CometCollectLimitExec, CometExec, CometHashAggregateExec, CometHashJoinExec, CometLocalTableScanExec, CometMetricNode, CometNativeExec, CometNativeScanExec, CometScanWrapper, CometSortMergeJoinExec, CometSparkToColumnarExec, CometTakeOrderedAndProjectExec} import org.apache.spark.sql.comet.execution.arrow.{CometArrowStream, CometNativeArrowSource} import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, CometNativeShuffle, CometShuffleExchangeExec} import org.apache.spark.sql.execution.{FileSourceScanExec, LocalTableScanExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec} -import org.apache.spark.sql.functions.{broadcast, col, count, countDistinct, spark_partition_id, sum} +import org.apache.spark.sql.functions.{broadcast, col, count, countDistinct, lit, spark_partition_id, sum, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, MapType, StringType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} @@ -2015,6 +2015,56 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper } } + // https://github.com/apache/datafusion-comet/issues/6530 + for { + (hint, joinClass) <- Seq( + "MERGE" -> classOf[CometSortMergeJoinExec], + "SHUFFLE_HASH" -> classOf[CometHashJoinExec]) + coalesce <- Seq(false, true) + } { + test(s"AQE skew join split stays native: $hint, coalescing=$coalesce") { + // Nine in ten left rows have key 0, so its shuffle partition is skewed, and AQE splits it + // along the left side's map tasks. The partial aggregate above the join shares its stage. + def skewedJoin: DataFrame = { + val left = spark + .range(0, 100000, 1, 20) + .select( + when(col("id") % 10 < 9, lit(0L)).otherwise(col("id") % 20000).as("k"), + col("id").as("v")) + val right = spark.range(0, 20000, 1, 4).select(col("id").as("k"), (col("id") * 3).as("w")) + left.join(right.hint(hint), "k").agg(count("*"), sum(col("v") + col("w"))) + } + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.SKEW_JOIN_ENABLED.key -> "true", + SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "1KB", + SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "1KB", + SQLConf.COALESCE_PARTITIONS_ENABLED.key -> coalesce.toString, + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.SHUFFLE_PARTITIONS.key -> "10") { + Seq(true, false).foreach { directRead => + withSQLConf(CometConf.COMET_SHUFFLE_DIRECT_READ_ENABLED.key -> directRead.toString) { + val (_, plan) = checkSparkAnswerAndOperator(skewedJoin) + withClue(s"direct read $directRead:\n$plan\n") { + assert( + collect(plan) { + case r: AQEShuffleReadExec if r.hasSkewedPartition => r + }.nonEmpty, + "AQE did not split the skewed partition") + assert(collect(plan) { case j if joinClass.isInstance(j) => j }.nonEmpty) + checkCometOperatorsInFinalPlan(plan) + // Both sides of the join read their split partitions as a native shuffle input. + val leaf = if (directRead) "SHUFFLE_SCAN" else "SCAN" + val joinBlock = Seq.fill(2)("AQEShuffleReadExec" -> leaf) + assert(shuffleReadingBlocks(plan).contains(joinBlock)) + } + } + } + } + } + } + // Regression test for https://github.com/apache/datafusion-comet/issues/3846 test("repartition count") { withTempPath { dir => From 6ef02ec95439d3c44d5c3c830cc8e5e8478af552 Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 18 Nov 2026 12:59:35 -0800 Subject: [PATCH 2/2] fix: leave a forced skew join split on Spark With spark.sql.adaptive.forceOptimizeSkewedJoin, AQE keeps a skew split that adds a shuffle only because SimpleCostEvaluator counts the split join as a skew join, and it counts only Spark's ShuffledJoin. Converting the split read made the join native, so AQE dropped the split, and SPARK-33832 in AdaptiveQueryExecSuite found 2 shuffles instead of 3. While that setting is on, the read and the join now stay on Spark. --- .../apache/comet/rules/CometExecRule.scala | 6 +++- .../comet/exec/CometNativeShuffleSuite.scala | 28 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala index d67a8099f5e..4297fa8e65c 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala @@ -679,8 +679,12 @@ case class CometExecRule(session: SparkSession, queryStagePrep: Boolean = false) // it can only be converted if the read is a native input too (issue #6530). The read then // feeds the native plan in place of the stage, and reads only the partitions it specifies. // AQE adds its other reads after the operators above them have been converted. + // With forceOptimizeSkewedJoin, AQE keeps a split that adds a shuffle only because its cost + // evaluator counts the split join, and it counts only Spark's own ShuffledJoin. A native + // join there would make AQE drop the split, so the read and the join stay on Spark. case read @ AQEShuffleReadExec(CometSinkPlaceHolder(_, _, stage: ShuffleQueryStageExec), _) - if queryStagePrep && stage.shuffle.isInstanceOf[CometShuffleExchangeExec] => + if queryStagePrep && stage.shuffle.isInstanceOf[CometShuffleExchangeExec] && + !conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) => val stageRead = read.withNewChildren(Seq(stage)) convertToComet(stageRead, CometExchangeSink).getOrElse(stageRead) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala index f7e979898b5..d6b01feafcc 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeShuffleSuite.scala @@ -49,7 +49,8 @@ import org.apache.spark.sql.comet.execution.shuffle.{CometColumnarShuffle, Comet import org.apache.spark.sql.execution.{FileSourceScanExec, LocalTableScanExec, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AdaptiveSparkPlanHelper, AQEShuffleReadExec, ShuffleQueryStageExec} import org.apache.spark.sql.execution.aggregate.ObjectHashAggregateExec -import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec} +import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeExec, ShuffleExchangeLike} +import org.apache.spark.sql.execution.joins.SortMergeJoinExec import org.apache.spark.sql.functions.{broadcast, col, count, countDistinct, lit, spark_partition_id, sum, when} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, LongType, MapType, StringType, StructField, StructType} @@ -2065,6 +2066,31 @@ class CometNativeShuffleSuite extends CometTestBase with AdaptiveSparkPlanHelper } } + test("AQE forced skew join split that adds a shuffle stays a skew join") { + // Every right row has key 0, so its partition is skewed. Splitting it breaks the + // partitioning the GROUP BY needs, so the split adds a shuffle, which AQE accepts only + // when forceOptimizeSkewedJoin is on and its cost evaluator counts the split join. + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN.key -> "true", + SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "false", + SQLConf.SKEW_JOIN_SKEWED_PARTITION_THRESHOLD.key -> "100", + SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "100", + SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.SHUFFLE_PARTITIONS.key -> "10") { + val left = spark.range(0, 1000, 1, 10).select((col("id") % 3).as("k1"), col("id")) + val right = spark.range(0, 1000, 1, 10).select((col("id") % 1).as("k2")) + val (_, plan) = + checkSparkAnswer(left.join(right, col("k1") === col("k2")).groupBy("k1").count()) + withClue(plan) { + assert(collect(plan) { case j: SortMergeJoinExec if j.isSkewJoin => j }.size == 1) + assert(collect(plan) { case s: ShuffleExchangeLike => s }.size == 3) + } + } + } + // Regression test for https://github.com/apache/datafusion-comet/issues/3846 test("repartition count") { withTempPath { dir =>