Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -231,9 +231,10 @@ private[rules] object MakeDateEvaluation {

val original = node match {
case scan: CometScanExec =>
scan.wrapped.copy(
partitionFilters = scan.partitionFilters,
dataFilters = scan.dataFilters)
CometScanUtils.copyScanWithFilters(
scan.wrapped,
scan.partitionFilters,
scan.dataFilters)
case scan: CometNativeScanExec => scan.sparkFallback(Nil)
case native: CometExec => native.originalPlan
case shuffle: CometShuffleExchangeExec => shuffle.originalPlan
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ case class CometNativeScanExec(
// AQE rewrites DPP placeholders in the live scan before transition reversion.
// The frozen originalPlan can still contain unexecutable SubqueryAdaptiveBroadcastExecs.
val restoredScan =
originalPlan.copy(partitionFilters = partitionFilters, dataFilters = dataFilters)
CometScanUtils.copyScanWithFilters(originalPlan, partitionFilters, dataFilters)
originalPlan.logicalLink.foreach(restoredScan.setLogicalLink)
restoredScan
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import java.util.Locale
import scala.collection.mutable

import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression, Literal}
import org.apache.spark.sql.comet.shims.ShimFileSourceScanExec
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile}
import org.apache.spark.sql.types.StructType
Expand All @@ -33,6 +34,37 @@ import org.apache.comet.objectstore.NativeConfig

object CometScanUtils {

private[apache] def copyScanWithFilters(
scan: FileSourceScanExec,
partitionFilters: Seq[Expression],
dataFilters: Seq[Expression]): FileSourceScanExec = {
// Rebuild through makeCopy rather than the compiled case-class copy, so that a runtime
// whose FileSourceScanExec appends constructor fields, such as Amazon EMR, keeps them.
val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray
replaceArg(
args,
ShimFileSourceScanExec.partitionFiltersIndex,
scan.partitionFilters,
partitionFilters)
replaceArg(args, ShimFileSourceScanExec.dataFiltersIndex, scan.dataFilters, dataFilters)
scan.makeCopy(args).asInstanceOf[FileSourceScanExec]
}

// Fail loudly if the runtime constructor does not have the expected value at `index`,
// rather than rebuilding the scan with an argument in the wrong position.
private def replaceArg(
args: Array[AnyRef],
index: Int,
expected: AnyRef,
replacement: AnyRef): Unit = {
if (index >= args.length || args(index) != expected) {
throw new IllegalStateException(
s"Unexpected FileSourceScanExec constructor layout: argument $index is not the " +
"expected filter list")
}
args(index) = replacement
}

/**
* Filters unused DynamicPruningExpression expressions - one which has been replaced with
* DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning
Expand All @@ -55,9 +87,10 @@ object CometScanUtils {
if (originalPlan == null) {
null
} else {
originalPlan
.copy(partitionFilters =
originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]))
copyScanWithFilters(
originalPlan,
originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]),
originalPlan.dataFilters)
.doCanonicalize()
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.spark.sql.comet.shims

/**
* Positions of the filter arguments in the `FileSourceScanExec` constructor, used to rebuild a
* scan through `makeCopy` with new filters. The constructor is `FileSourceScanExec(relation,
* output, requiredSchema, partitionFilters, ...)`.
*/
object ShimFileSourceScanExec {
val partitionFiltersIndex: Int = 3
val dataFiltersIndex: Int = 6
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.spark.sql.comet.shims

/**
* Positions of the filter arguments in the `FileSourceScanExec` constructor, used to rebuild a
* scan through `makeCopy` with new filters. The constructor is `FileSourceScanExec(relation,
* stream, output, requiredSchema, partitionFilters, ...)`.
*/
object ShimFileSourceScanExec {
val partitionFiltersIndex: Int = 4
val dataFiltersIndex: Int = 7
}
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,41 @@ class CometExecRuleSuite extends CometTestBase {
}.sum
}

test("make_date scan fallback preserves additional runtime constructor fields") {
withSQLConf(CometConf.COMET_EXEC_MAKE_DATE_OPERATOR_FALLBACK_ENABLED.key -> "true") {
val output = Seq(AttributeReference("day", DataTypes.IntegerType)())
val partitionFilters = Seq(
GreaterThan(
MakeDate(Literal(2024), Literal(1), output.head),
Literal.create(java.sql.Date.valueOf("2024-01-01"), DataTypes.DateType)))
val dataFilters = Seq(GreaterThan(output.head, Literal(1)))
val wrapped = ExtendedFileSourceScanExec(output)
val scan = CometScanExec(
relation = wrapped.relation,
output = wrapped.output,
requiredSchema = wrapped.requiredSchema,
partitionFilters = partitionFilters,
optionalBucketSet = wrapped.optionalBucketSet,
optionalNumCoalescedBuckets = wrapped.optionalNumCoalescedBuckets,
dataFilters = dataFilters,
tableIdentifier = wrapped.tableIdentifier,
disableBucketedScan = wrapped.disableBucketedScan,
wrapped = wrapped)

val restored = MakeDateEvaluation
.preserve(CollectLimitExec(1, scan))
.asInstanceOf[CollectLimitExec]
.child
.asInstanceOf[FileSourceScanExec]

assert(restored.getClass == wrapped.getClass)
assert(restored.partitionFilters == partitionFilters)
assert(restored.dataFilters == dataFilters)
assert(restored.productIterator.toSeq.takeRight(2) == Seq(true, output))
assert(wrapped.partitionFilters.isEmpty && wrapped.dataFilters.isEmpty)
}
}

test("make_date preserves row demand below lazy consumers, including reused native plans") {
withSQLConf(
SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ import scala.reflect.runtime.universe.runtimeMirror

import org.scalatest.funsuite.AnyFunSuite

import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.comet.shims.ShimFileSourceScanExec
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.types.IntegerType
import org.apache.xbean.asm9.{ClassReader, Opcodes}
import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode}

Expand All @@ -36,6 +40,73 @@ import org.apache.comet.util.ClassLoaders

class CometPlanEqualitySuite extends AnyFunSuite {

test("FileSourceScanExec shim filter indices match the constructor") {
val output = Seq(AttributeReference("id", IntegerType)())
val scan = ExtendedFileSourceScanExec(
output,
partitionFilters = Seq(GreaterThan(output.head, Literal(1))),
dataFilters = Seq(LessThan(output.head, Literal(10))))

assert(
scan.productElement(ShimFileSourceScanExec.partitionFiltersIndex) == scan.partitionFilters)
assert(scan.productElement(ShimFileSourceScanExec.dataFiltersIndex) == scan.dataFilters)
}
Comment on lines +43 to +53

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test pins what scan canonicalization preserves runtime fields and Spark normalization already pins. That test builds ExtendedFileSourceScanExec with non-empty partition and data filters and goes through canonicalizeOriginalPlan, which calls copyScanWithFilters. A wrong shim index makes replaceArg throw IllegalStateException there. Could this one be removed?


test("native scan fallback preserves additional runtime constructor fields") {
val output = Seq(AttributeReference("id", IntegerType)())
val partitionFilters = Seq(GreaterThan(output.head, Literal(1)))
val dataFilters = Seq(LessThan(output.head, Literal(10)))
val scan = ExtendedFileSourceScanExec(output)

val nativeScan = CometNativeScanExec(
nativeOp = null,
relation = scan.relation,
output = scan.output,
requiredSchema = scan.requiredSchema,
partitionFilters = partitionFilters,
optionalBucketSet = scan.optionalBucketSet,
optionalNumCoalescedBuckets = scan.optionalNumCoalescedBuckets,
dataFilters = dataFilters,
tableIdentifier = scan.tableIdentifier,
disableBucketedScan = scan.disableBucketedScan,
originalPlan = scan,
serializedPlanOpt = SerializedPlan(None),
scan = null,
sourceKey = "")
val copied = nativeScan
.sparkFallback(Nil)
.asInstanceOf[FileSourceScanExec]

assert(copied.getClass == scan.getClass)
assert(copied.partitionFilters == partitionFilters)
assert(copied.dataFilters == dataFilters)
assert(copied.output == output)
assert(copied.tableIdentifier == scan.tableIdentifier)
assert(copied.disableBucketedScan)
assert(copied.productIterator.toSeq.takeRight(2) == Seq(true, output))
assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty)
}

test("scan canonicalization preserves runtime fields and Spark normalization") {
val output = Seq(AttributeReference("id", IntegerType)())
val staticFilter = GreaterThan(output.head, Literal(1))
val scan = ExtendedFileSourceScanExec(
output,
partitionFilters =
Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))),
dataFilters = Seq(LessThan(output.head, Literal(10))))

val canonicalized = CometScanUtils.canonicalizeOriginalPlan(scan)
val expectedSparkScan = scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize()

assert(canonicalized.getClass == scan.getClass)
assert(canonicalized.productIterator.toSeq.takeRight(2) == Seq(true, output))
// Check Spark's normalized fields separately from the distribution's extra fields.
assert(
canonicalized.productIterator.toSeq.dropRight(2) == expectedSparkScan.productIterator.toSeq)
assert(CometScanUtils.canonicalizeOriginalPlan(null) == null)
}

// Native serialization state and the source plan are deliberately not the identity of a
// converted operator. Some operators do compare these; exclusion does not forbid comparison.
private val commonExclusions = Set("nativeOp", "originalPlan")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.spark.sql.comet

import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
import org.apache.spark.sql.execution.FileSourceScanExec
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
import org.apache.spark.sql.types.{IntegerType, StructType}
import org.apache.spark.util.collection.BitSet

private[apache] object ExtendedFileSourceScanExec {
def apply(
output: Seq[Attribute],
partitionFilters: Seq[Expression] = Nil,
dataFilters: Seq[Expression] = Nil): FileSourceScanExec = {
new ExtendedFileSourceScanExec(
null,
output,
new StructType().add("id", IntegerType),
partitionFilters,
None,
None,
dataFilters,
Some(TableIdentifier("test")),
true,
true,
output)
}
}

// Model a distribution that adds scan fields. An inherited case-class copy would discard them.
private[apache] class ExtendedFileSourceScanExec(
relation: HadoopFsRelation,
output: Seq[Attribute],
requiredSchema: StructType,
partitionFilters: Seq[Expression],
optionalBucketSet: Option[BitSet],
optionalNumCoalescedBuckets: Option[Int],
dataFilters: Seq[Expression],
tableIdentifier: Option[TableIdentifier],
disableBucketedScan: Boolean,
val vendorFlag: Boolean,
val vendorOutput: Seq[Attribute])
extends FileSourceScanExec(
relation,
output,
requiredSchema,
partitionFilters,
optionalBucketSet,
optionalNumCoalescedBuckets,
dataFilters,
tableIdentifier,
disableBucketedScan) {
override def doCanonicalize(): FileSourceScanExec = {
// Keep the simulated vendor fields observable after Spark normalizes its own fields.
val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray
makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput))
.asInstanceOf[FileSourceScanExec]
}

override def productArity: Int = 11

override def productElement(index: Int): Any = index match {
case 9 => vendorFlag
case 10 => vendorOutput
case _ => super.productElement(index)
}
}
Loading