Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, SortOrder}
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, SinglePartition, UnknownPartitioning}
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
import org.apache.spark.sql.execution.{SparkPlan, SQLExecution}
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
Expand Down Expand Up @@ -136,16 +136,23 @@ case class CometIcebergNativeScanExec(
// tasks here would run a placeholder subquery. At execution, CometNativeExec sizes the native
// RDD from perPartitionData instead.
//
// Spark 3.4's BatchScanExec reports SinglePartition for one input partition, which it counts
// before any DPP filtering, and Spark plans no exchange above it on that basis. The scan reports
// the same. Otherwise EnsureRequirements, which AQE runs again over a DPP subquery's converted
// plan, inserts an exchange inside a native block that is already serialized (#7422).
// serializePartitions keeps that one partition when DPP prunes it, as Spark does.
// Spark 3.4's BatchScanExec reports SinglePartition for one input partition, and every supported
// version reports a key-grouped partitioning (KeyGroupedPartitioning, called KeyedPartitioning
// from Spark 4.2) for a storage-partitioned scan. Spark takes both from the partitions it plans,
// before any DPP filtering, and plans the operators above the scan on that basis, with no
// exchange for a join or aggregate that the partitioning already satisfies. The scan reports the
// same, so reading it resolves no DPP subquery. Otherwise EnsureRequirements, which AQE runs
// again over a DPP subquery's converted plan, inserts exchanges inside a native block that is
// already serialized (#7422, #7441). serializePartitions keeps one partition for each of Spark's
// partitions when DPP prunes some, as Spark does.
override lazy val outputPartitioning: Partitioning =
if (originalPlan != null && originalPlan.outputPartitioning == SinglePartition) {
SinglePartition
} else {
if (originalPlan == null) {
UnknownPartitioning(0)
} else {
originalPlan.outputPartitioning match {
case _: UnknownPartitioning => UnknownPartitioning(0)
case reported => reported
}
Comment on lines +152 to +155

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When BatchScanExec reports no partitioning, it returns UnknownPartitioning(0) from SparkPlan.outputPartitioning on every version I checked (v3.4.3, v3.5.8, v4.0.1, v4.1.1, v4.2.1). So this match always returns its input. CometNativeScanExec.outputPartitioning returns originalPlan.outputPartitioning directly for a bucketed scan, and this could do the same:

Suggested change
originalPlan.outputPartitioning match {
case _: UnknownPartitioning => UnknownPartitioning(0)
case reported => reported
}
originalPlan.outputPartitioning

}

override lazy val outputOrdering: Seq[SortOrder] = Nil
Expand Down
44 changes: 40 additions & 4 deletions spark/src/main/scala/org/apache/spark/sql/comet/operators.scala
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffl
import org.apache.spark.sql.execution.aggregate.{BaseAggregateExec, HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
import org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase
import org.apache.spark.sql.execution.exchange.ReusedExchangeExec
import org.apache.spark.sql.execution.exchange.{ReusedExchangeExec, ShuffleExchangeLike}
import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, BroadcastNestedLoopJoinExec, CartesianProductExec, HashJoin, ShuffledHashJoinExec, SortMergeJoinExec}
import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
import org.apache.spark.sql.internal.SQLConf
Expand Down Expand Up @@ -741,17 +741,53 @@ abstract class CometExec extends CometPlan {
* Declares the distribution of the Spark operator that a Comet operator replaces. Without it
* AQE's OptimizeSkewInRebalancePartitions can split a skewed partition under an operator that
* needs its input clustered, because ValidateRequirements has nothing to reject, and one key then
* yields several rows. Mix it into operators that replace a clustering aggregate, window or join.
* yields several rows. Mix it into operators that replace a clustering aggregate or window. A
* join mixes in [[CometDelegatesJoinChildDistribution]] instead.
*/
trait CometDelegatesRequiredChildDistribution extends CometExec {
override def requiredChildDistribution: Seq[Distribution] =
originalPlan.requiredChildDistribution
}

/**
* [[CometDelegatesRequiredChildDistribution]] for a join. A storage-partitioned join, whose sides
* are key-grouped scans that Spark joins with no exchange, declares no requirement. No shuffle
* sits below it for AQE to split. EnsureRequirements only accepts such a join when it is Spark's
* own join operator, and in Spark 4.1 only over Spark's own scans. So when AQE plans a DPP
* subquery's converted plan again, it would put an exchange under each side that it has no
* logical link for (#7441). The sides are the join's current children, which are Comet operators,
* because conversion works bottom-up and AQE can later swap in query stages.
*/
trait CometDelegatesJoinChildDistribution extends CometDelegatesRequiredChildDistribution {
override def requiredChildDistribution: Seq[Distribution] =
if (children.forall(CometExec.isStoragePartitioned)) {
children.map(_ => UnspecifiedDistribution)
} else {
super.requiredChildDistribution
}
}

object CometExec {
final class InvalidSparkFallbackException(message: String)
extends IllegalArgumentException(message)

/**
* Whether `plan` reads only key-grouped scans, with no shuffle above them. This is how Spark
* plans each side of a storage-partitioned join. A scan counts only when the scan it replaces
* or wraps was planned key-grouped, so a table read without grouping does not.
*/
private[comet] def isStoragePartitioned(plan: SparkPlan): Boolean =
!plan.exists(_.isInstanceOf[ShuffleExchangeLike]) &&
plan.collectLeaves().forall {
// Spark's own scans. Comet's non-native V2 scan, CometBatchScanExec, is a leaf under a
// CometScanWrapper that this does not unwrap, so a join over it keeps its requirement.
Comment on lines +782 to +783

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This comment describes a path that no scan takes today. CometBatchScanExec is only created for CSV, Text, and Iceberg (CometScanRule.scala L415, L574, L1345), and CometExecRule handles each of those in its own case before the CometScanWrapper one (CometExecRule.scala L470-L483). An Iceberg scan that the native conversion declines stays a bare CometBatchScanExec. That class extends DataSourceV2ScanExecBase, so it matches the case below this comment and isn't excluded.

Could we reword the comment to match? The paragraph in the PR description about Comet's non-native Iceberg reader has the same problem, so it could go too.

Suggested change
// Spark's own scans. Comet's non-native V2 scan, CometBatchScanExec, is a leaf under a
// CometScanWrapper that this does not unwrap, so a join over it keeps its requirement.
// Spark's own scans, and CometBatchScanExec, which extends DataSourceV2ScanExecBase.

case scan: DataSourceV2ScanExecBase => scan.keyGroupedPartitioning.isDefined
// originalPlan is null on a canonicalized copy
case scan: CometIcebergNativeScanExec =>
scan.originalPlan != null && scan.originalPlan.keyGroupedPartitioning.isDefined
case _ => false
}

// An unique id for each CometExecIterator, used to identify the native query execution.
private val curId = new java.util.concurrent.atomic.AtomicLong()

Expand Down Expand Up @@ -3434,7 +3470,7 @@ case class CometHashJoinExec(
override val right: SparkPlan,
override val serializedPlanOpt: SerializedPlan)
extends CometBinaryExec
with CometDelegatesRequiredChildDistribution {
with CometDelegatesJoinChildDistribution {

override def outputPartitioning: Partitioning = joinType match {
case _: InnerLike =>
Expand Down Expand Up @@ -3824,7 +3860,7 @@ case class CometSortMergeJoinExec(
override val right: SparkPlan,
override val serializedPlanOpt: SerializedPlan)
extends CometBinaryExec
with CometDelegatesRequiredChildDistribution {
with CometDelegatesJoinChildDistribution {

override def outputPartitioning: Partitioning = joinType match {
case _: InnerLike =>
Expand Down
127 changes: 127 additions & 0 deletions spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -7033,6 +7033,133 @@ class CometIcebergNativeSuite
}
}

// A storage-partitioned join is a sort-merge join that Spark plans without a shuffle, because
// both scans report a KeyGroupedPartitioning. On the build side of a DPP broadcast, AQE plans
// the converted join again, and EnsureRequirements has to find the join's required distribution
// already met rather than put an exchange under each side, which AQE has no logical link for
// (#7441).
test("AQE DPP - storage-partitioned join on the DPP build side (#7441)") {
assume(icebergAvailable, "Iceberg not available")
withTempIcebergDir { warehouseDir =>
withSQLConf(
"spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog",
"spark.sql.catalog.aqe_cat.type" -> "hadoop",
"spark.sql.catalog.aqe_cat.warehouse" -> warehouseDir.getAbsolutePath,
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
SQLConf.V2_BUCKETING_ENABLED.key -> "true",
SQLConf.V2_BUCKETING_PUSH_PART_VALUES_ENABLED.key -> "true",
"spark.sql.iceberg.planning.preserve-data-grouping" -> "true",
CometConf.COMET_ENABLED.key -> "true",
CometConf.COMET_EXEC_ENABLED.key -> "true",
CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") {

spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT)
USING iceberg PARTITIONED BY (store_id)
""")
spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING)
USING iceberg PARTITIONED BY (id)
""")
spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING)
USING iceberg PARTITIONED BY (id)
""")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " +
"FROM range(0, 100)")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " +
"IF(id < 3, 'X', 'Y') FROM range(0, 10)")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " +
"FROM range(0, 10)")

// The merge join of the two dimensions is the build side of the broadcast that prunes the
// fact scan. It needs no shuffle because both of its scans are key-grouped on id.
val query =
"""SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.name
|FROM aqe_cat.db.spj_dpp_fact f JOIN (
| SELECT /*+ MERGE(a, b) */ a.id, b.name

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

CometHashJoinExec gets the new trait too, but both tests use MERGE. Could we also run them with SHUFFLE_HASH(a, b) and assert that the plan has a CometHashJoinExec? I tried that variant of the join test locally on Spark 4.1. It fails with the same AssertionError in setLogicalLinkForNewQueryStage when operators.scala is reverted to main, and passes with this PR.

The two tests also share all of their setup, including the table names. When I ran them without the operators.scala change, the aggregate test failed with TABLE_OR_VIEW_ALREADY_EXISTS on spj_dpp_fact, because the join test failed before its DROP TABLE lines ran. So a failure in one test hides the result of the other. Could we use one setup with the drops in a finally, and generate the cases with a for over the two queries and the two hints, like the test at L6170-L6174? That would also give the aggregate query the no-exchange assertion that only the join query has now.

| FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id
| WHERE a.country = 'X'
|) d ON f.store_id = d.id""".stripMargin
val (_, cometPlan) = checkSparkAnswer(query)
assertIcebergNativeScanPresent(cometPlan)
assertNoLeftoverCSAB(cometPlan)
assert(
collectWithSubqueries(cometPlan) { case e: ShuffleExchangeLike => e }.isEmpty,
s"Expected no exchange in the plan or in the DPP subquery's plan:\n$cometPlan")

spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact")
spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a")
spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b")
}
}
}

// The aggregate above a storage-partitioned join on a DPP build side needs its input clustered
// as well. The join's partitioning comes from its scans, so they have to report the key-grouped
// partitioning that Spark planned the aggregate on, or EnsureRequirements puts an exchange
// between the two when AQE plans the converted subquery again (#7441).
test("AQE DPP - aggregate over a storage-partitioned join on the DPP build side (#7441)") {
assume(icebergAvailable, "Iceberg not available")
withTempIcebergDir { warehouseDir =>
withSQLConf(
"spark.sql.catalog.aqe_cat" -> "org.apache.iceberg.spark.SparkCatalog",
"spark.sql.catalog.aqe_cat.type" -> "hadoop",
"spark.sql.catalog.aqe_cat.warehouse" -> warehouseDir.getAbsolutePath,
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true",
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true",
SQLConf.V2_BUCKETING_ENABLED.key -> "true",
SQLConf.V2_BUCKETING_PUSH_PART_VALUES_ENABLED.key -> "true",
"spark.sql.iceberg.planning.preserve-data-grouping" -> "true",
CometConf.COMET_ENABLED.key -> "true",
CometConf.COMET_EXEC_ENABLED.key -> "true",
CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") {

spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_fact (amount INT, store_id INT)
USING iceberg PARTITIONED BY (store_id)
""")
spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_dim_a (id INT, country STRING)
USING iceberg PARTITIONED BY (id)
""")
spark.sql("""
CREATE TABLE aqe_cat.db.spj_dpp_dim_b (id INT, name STRING)
USING iceberg PARTITIONED BY (id)
""")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_fact SELECT CAST(id AS INT), CAST(id % 10 AS INT) " +
"FROM range(0, 100)")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_dim_a SELECT CAST(id AS INT), " +
"IF(id < 3, 'X', 'Y') FROM range(0, 10)")
spark.sql(
"INSERT INTO aqe_cat.db.spj_dpp_dim_b SELECT CAST(id AS INT), concat('n', id) " +
"FROM range(0, 10)")

val query =
"""SELECT /*+ BROADCAST(d) */ f.store_id, f.amount, d.cnt
|FROM aqe_cat.db.spj_dpp_fact f JOIN (
| SELECT /*+ MERGE(a, b) */ a.id, count(*) AS cnt
| FROM aqe_cat.db.spj_dpp_dim_a a JOIN aqe_cat.db.spj_dpp_dim_b b ON a.id = b.id
| WHERE a.country = 'X'
| GROUP BY a.id
|) d ON f.store_id = d.id""".stripMargin
val (_, cometPlan) = checkSparkAnswer(query)
assertIcebergNativeScanPresent(cometPlan)
assertNoLeftoverCSAB(cometPlan)

spark.sql("DROP TABLE aqe_cat.db.spj_dpp_fact")
spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_a")
spark.sql("DROP TABLE aqe_cat.db.spj_dpp_dim_b")
}
}
}

// TPC-DS q64 shape (#6264): a broadcast of (fact SMJ other) per UNION ALL branch, with the
// dimension join above it. The two broadcasts differ only in the DPP filter of the fact scan
// inside a child shuffle stage, so they must not be reused for each other.
Expand Down