Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,10 @@ private[rules] object MakeDateEvaluation {

val original = node match {
case scan: CometScanExec =>
scan.wrapped.copy(
partitionFilters = scan.partitionFilters,
dataFilters = scan.dataFilters)
CometScanUtils.copyScanWithFilters(
scan.wrapped,
scan.partitionFilters,
scan.dataFilters)
case scan: CometNativeScanExec => scan.sparkFallback(Nil)
case native: CometExec => native.originalPlan
case shuffle: CometShuffleExchangeExec => shuffle.originalPlan
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,7 @@ case class CometNativeScanExec(
override def sparkFallback(newChildren: Seq[SparkPlan]): SparkPlan = {
// AQE rewrites DPP placeholders in the live scan before transition reversion.
// The frozen originalPlan can still contain unexecutable SubqueryAdaptiveBroadcastExecs.
val restoredScan =
originalPlan.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
originalPlan.logicalLink.foreach(restoredScan.setLogicalLink)
restoredScan
CometScanUtils.copyScanWithFilters(originalPlan, partitionFilters, dataFilters)
}

override lazy val metadata: Map[String, String] =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,21 @@ import org.apache.comet.objectstore.NativeConfig

object CometScanUtils {

private[apache] def copyScanWithFilters(
scan: FileSourceScanExec,
partitionFilters: Seq[Expression],
dataFilters: Seq[Expression]): FileSourceScanExec = {
// Keep the runtime's full constructor, including fields added by Spark distributions.
val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray
// Spark 4 adds a stream argument before the schema; the filters keep the same offsets
// after it. Locate the existing schema rather than assuming a Spark-specific arity.
Comment on lines +42 to +43

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably have handled this in the shim layer

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will create follow on PR

val schemaIndex = args.indexWhere(_ eq scan.requiredSchema)
require(schemaIndex >= 0, "FileSourceScanExec is missing its required schema")
args(schemaIndex + 1) = partitionFilters
args(schemaIndex + 4) = dataFilters
scan.makeCopy(args).asInstanceOf[FileSourceScanExec]
}

/**
* Filters unused DynamicPruningExpression expressions - one which has been replaced with
* DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning
Expand All @@ -55,9 +70,10 @@ object CometScanUtils {
if (originalPlan == null) {
null
} else {
originalPlan
.copy(partitionFilters =
originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]))
copyScanWithFilters(
originalPlan,
originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]),
originalPlan.dataFilters)
.doCanonicalize()
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,41 @@ class CometExecRuleSuite extends CometTestBase {
}.sum
}

test("make_date scan fallback preserves additional runtime constructor fields") {
withSQLConf(CometConf.COMET_EXEC_MAKE_DATE_OPERATOR_FALLBACK_ENABLED.key -> "true") {
val output = Seq(AttributeReference("day", DataTypes.IntegerType)())
val partitionFilters = Seq(
GreaterThan(
MakeDate(Literal(2024), Literal(1), output.head),
Literal.create(java.sql.Date.valueOf("2024-01-01"), DataTypes.DateType)))
val dataFilters = Seq(GreaterThan(output.head, Literal(1)))
val wrapped = ExtendedFileSourceScanExec(output)
val scan = CometScanExec(
relation = wrapped.relation,
output = wrapped.output,
requiredSchema = wrapped.requiredSchema,
partitionFilters = partitionFilters,
optionalBucketSet = wrapped.optionalBucketSet,
optionalNumCoalescedBuckets = wrapped.optionalNumCoalescedBuckets,
dataFilters = dataFilters,
tableIdentifier = wrapped.tableIdentifier,
disableBucketedScan = wrapped.disableBucketedScan,
wrapped = wrapped)

val restored = MakeDateEvaluation
.preserve(CollectLimitExec(1, scan))
.asInstanceOf[CollectLimitExec]
.child
.asInstanceOf[FileSourceScanExec]

assert(restored.getClass == wrapped.getClass)
assert(restored.partitionFilters == partitionFilters)
assert(restored.dataFilters == dataFilters)
assert(restored.productIterator.toSeq.takeRight(2) == Seq(true, output))
assert(wrapped.partitionFilters.isEmpty && wrapped.dataFilters.isEmpty)
}
}

test("make_date preserves row demand below lazy consumers, including reused native plans") {
withSQLConf(
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import scala.reflect.runtime.universe.runtimeMirror

import org.scalatest.funsuite.AnyFunSuite

import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.types.IntegerType
import org.apache.xbean.asm9.{ClassReader, Opcodes}
import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode}

Expand All @@ -36,6 +39,61 @@ import org.apache.comet.util.ClassLoaders

class CometPlanEqualitySuite extends AnyFunSuite {

test("native scan fallback preserves additional runtime constructor fields") {
val output = Seq(AttributeReference("id", IntegerType)())
val partitionFilters = Seq(GreaterThan(output.head, Literal(1)))
val dataFilters = Seq(LessThan(output.head, Literal(10)))
val scan = ExtendedFileSourceScanExec(output)

val nativeScan = CometNativeScanExec(
nativeOp = null,
relation = scan.relation,
output = scan.output,
requiredSchema = scan.requiredSchema,
partitionFilters = partitionFilters,
optionalBucketSet = scan.optionalBucketSet,
optionalNumCoalescedBuckets = scan.optionalNumCoalescedBuckets,
dataFilters = dataFilters,
tableIdentifier = scan.tableIdentifier,
disableBucketedScan = scan.disableBucketedScan,
originalPlan = scan,
serializedPlanOpt = SerializedPlan(None),
scan = null,
sourceKey = "")
val copied = nativeScan
.sparkFallback(Nil)
.asInstanceOf[FileSourceScanExec]

assert(copied.getClass == scan.getClass)
assert(copied.partitionFilters == partitionFilters)
assert(copied.dataFilters == dataFilters)
assert(copied.output == output)
assert(copied.tableIdentifier == scan.tableIdentifier)
assert(copied.disableBucketedScan)
assert(copied.productIterator.toSeq.takeRight(2) == Seq(true, output))
assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty)
}

test("scan canonicalization preserves runtime fields and Spark normalization") {
val output = Seq(AttributeReference("id", IntegerType)())
val staticFilter = GreaterThan(output.head, Literal(1))
val scan = ExtendedFileSourceScanExec(
output,
partitionFilters =
Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))),
dataFilters = Seq(LessThan(output.head, Literal(10))))

val canonicalized = CometScanUtils.canonicalizeOriginalPlan(scan)
val expectedSparkScan = scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize()

assert(canonicalized.getClass == scan.getClass)
assert(canonicalized.productIterator.toSeq.takeRight(2) == Seq(true, output))
// Check Spark's normalized fields separately from the distribution's extra fields.
assert(
canonicalized.productIterator.toSeq.dropRight(2) == expectedSparkScan.productIterator.toSeq)
assert(CometScanUtils.canonicalizeOriginalPlan(null) == null)
}

// Native serialization state and the source plan are deliberately not the identity of a
// converted operator. Some operators do compare these; exclusion does not forbid comparison.
private val commonExclusions = Set("nativeOp", "originalPlan")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.spark.sql.comet

import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
import org.apache.spark.sql.types.{IntegerType, StructType}
import org.apache.spark.util.collection.BitSet

private[apache] object ExtendedFileSourceScanExec {
def apply(
output: Seq[Attribute],
partitionFilters: Seq[Expression] = Nil,
dataFilters: Seq[Expression] = Nil): FileSourceScanExec = {
new ExtendedFileSourceScanExec(
null,
output,
new StructType().add("id", IntegerType),
partitionFilters,
None,
None,
dataFilters,
Some(TableIdentifier("test")),
true,
true,
output)
}
}

// Model a distribution that adds scan fields. An inherited case-class copy would discard them.
private[apache] class ExtendedFileSourceScanExec(
relation: HadoopFsRelation,
output: Seq[Attribute],
requiredSchema: StructType,
partitionFilters: Seq[Expression],
optionalBucketSet: Option[BitSet],
optionalNumCoalescedBuckets: Option[Int],
dataFilters: Seq[Expression],
tableIdentifier: Option[TableIdentifier],
disableBucketedScan: Boolean,
val vendorFlag: Boolean,
val vendorOutput: Seq[Attribute])
extends FileSourceScanExec(
relation,
output,
requiredSchema,
partitionFilters,
optionalBucketSet,
optionalNumCoalescedBuckets,
dataFilters,
tableIdentifier,
disableBucketedScan) {
override def doCanonicalize(): FileSourceScanExec = {
// Keep the simulated vendor fields observable after Spark normalizes its own fields.
val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray
makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput))
.asInstanceOf[FileSourceScanExec]
}

override def productArity: Int = 11

override def productElement(index: Int): Any = index match {
case 9 => vendorFlag
case 10 => vendorOutput
case _ => super.productElement(index)
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.spark.sql.comet

import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.connector.read.streaming.SparkDataStream
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
import org.apache.spark.sql.types.{IntegerType, StructType}
import org.apache.spark.util.collection.BitSet

private[apache] object ExtendedFileSourceScanExec {
def apply(
output: Seq[Attribute],
partitionFilters: Seq[Expression] = Nil,
dataFilters: Seq[Expression] = Nil): FileSourceScanExec = {
new ExtendedFileSourceScanExec(
null,
None,
output,
new StructType().add("id", IntegerType),
partitionFilters,
None,
None,
dataFilters,
Some(TableIdentifier("test")),
true,
true,
output)
}
}

// Model a distribution that adds scan fields. An inherited case-class copy would discard them.
private[apache] class ExtendedFileSourceScanExec(
relation: HadoopFsRelation,
stream: Option[SparkDataStream],
output: Seq[Attribute],
requiredSchema: StructType,
partitionFilters: Seq[Expression],
optionalBucketSet: Option[BitSet],
optionalNumCoalescedBuckets: Option[Int],
dataFilters: Seq[Expression],
tableIdentifier: Option[TableIdentifier],
disableBucketedScan: Boolean,
val vendorFlag: Boolean,
val vendorOutput: Seq[Attribute])
extends FileSourceScanExec(
relation,
stream,
output,
requiredSchema,
partitionFilters,
optionalBucketSet,
optionalNumCoalescedBuckets,
dataFilters,
tableIdentifier,
disableBucketedScan) {
override def doCanonicalize(): FileSourceScanExec = {
// Keep the simulated vendor fields observable after Spark normalizes its own fields.
val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray
makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput))
.asInstanceOf[FileSourceScanExec]
}

override def productArity: Int = 12

override def productElement(index: Int): Any = index match {
case 10 => vendorFlag
case 11 => vendorOutput
case _ => super.productElement(index)
}
}