diff --git a/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala b/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala index 7811b893b2f..6a6242c5303 100644 --- a/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala +++ b/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala @@ -231,9 +231,10 @@ private[rules] object MakeDateEvaluation { val original = node match { case scan: CometScanExec => - scan.wrapped.copy( - partitionFilters = scan.partitionFilters, - dataFilters = scan.dataFilters) + CometScanUtils.copyScanWithFilters( + scan.wrapped, + scan.partitionFilters, + scan.dataFilters) case scan: CometNativeScanExec => scan.sparkFallback(Nil) case native: CometExec => native.originalPlan case shuffle: CometShuffleExchangeExec => shuffle.originalPlan diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala index 8c8456b074a..3fe7db06716 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala @@ -78,10 +78,7 @@ case class CometNativeScanExec( override def sparkFallback(newChildren: Seq[SparkPlan]): SparkPlan = { // AQE rewrites DPP placeholders in the live scan before transition reversion. // The frozen originalPlan can still contain unexecutable SubqueryAdaptiveBroadcastExecs. - val restoredScan = - originalPlan.copy(partitionFilters = partitionFilters, dataFilters = dataFilters) - originalPlan.logicalLink.foreach(restoredScan.setLogicalLink) - restoredScan + CometScanUtils.copyScanWithFilters(originalPlan, partitionFilters, dataFilters) } override lazy val metadata: Map[String, String] = diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala index 6eb1f9937ee..fbe97e5b088 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala @@ -33,6 +33,21 @@ import org.apache.comet.objectstore.NativeConfig object CometScanUtils { + private[apache] def copyScanWithFilters( + scan: FileSourceScanExec, + partitionFilters: Seq[Expression], + dataFilters: Seq[Expression]): FileSourceScanExec = { + // Keep the runtime's full constructor, including fields added by Spark distributions. + val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray + // Spark 4 adds a stream argument before the schema; the filters keep the same offsets + // after it. Locate the existing schema rather than assuming a Spark-specific arity. + val schemaIndex = args.indexWhere(_ eq scan.requiredSchema) + require(schemaIndex >= 0, "FileSourceScanExec is missing its required schema") + args(schemaIndex + 1) = partitionFilters + args(schemaIndex + 4) = dataFilters + scan.makeCopy(args).asInstanceOf[FileSourceScanExec] + } + /** * Filters unused DynamicPruningExpression expressions - one which has been replaced with * DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning @@ -55,9 +70,10 @@ object CometScanUtils { if (originalPlan == null) { null } else { - originalPlan - .copy(partitionFilters = - originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression])) + copyScanWithFilters( + originalPlan, + originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]), + originalPlan.dataFilters) .doCanonicalize() } } diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index ffbe1d0e1b0..3ba1a8bc925 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -155,6 +155,41 @@ class CometExecRuleSuite extends CometTestBase { }.sum } + test("make_date scan fallback preserves additional runtime constructor fields") { + withSQLConf(CometConf.COMET_EXEC_MAKE_DATE_OPERATOR_FALLBACK_ENABLED.key -> "true") { + val output = Seq(AttributeReference("day", DataTypes.IntegerType)()) + val partitionFilters = Seq( + GreaterThan( + MakeDate(Literal(2024), Literal(1), output.head), + Literal.create(java.sql.Date.valueOf("2024-01-01"), DataTypes.DateType))) + val dataFilters = Seq(GreaterThan(output.head, Literal(1))) + val wrapped = ExtendedFileSourceScanExec(output) + val scan = CometScanExec( + relation = wrapped.relation, + output = wrapped.output, + requiredSchema = wrapped.requiredSchema, + partitionFilters = partitionFilters, + optionalBucketSet = wrapped.optionalBucketSet, + optionalNumCoalescedBuckets = wrapped.optionalNumCoalescedBuckets, + dataFilters = dataFilters, + tableIdentifier = wrapped.tableIdentifier, + disableBucketedScan = wrapped.disableBucketedScan, + wrapped = wrapped) + + val restored = MakeDateEvaluation + .preserve(CollectLimitExec(1, scan)) + .asInstanceOf[CollectLimitExec] + .child + .asInstanceOf[FileSourceScanExec] + + assert(restored.getClass == wrapped.getClass) + assert(restored.partitionFilters == partitionFilters) + assert(restored.dataFilters == dataFilters) + assert(restored.productIterator.toSeq.takeRight(2) == Seq(true, output)) + assert(wrapped.partitionFilters.isEmpty && wrapped.dataFilters.isEmpty) + } + } + test("make_date preserves row demand below lazy consumers, including reused native plans") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 52a5e1fe953..765a37879e0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -26,6 +26,9 @@ import scala.reflect.runtime.universe.runtimeMirror import org.scalatest.funsuite.AnyFunSuite +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.types.IntegerType import org.apache.xbean.asm9.{ClassReader, Opcodes} import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} @@ -36,6 +39,61 @@ import org.apache.comet.util.ClassLoaders class CometPlanEqualitySuite extends AnyFunSuite { + test("native scan fallback preserves additional runtime constructor fields") { + val output = Seq(AttributeReference("id", IntegerType)()) + val partitionFilters = Seq(GreaterThan(output.head, Literal(1))) + val dataFilters = Seq(LessThan(output.head, Literal(10))) + val scan = ExtendedFileSourceScanExec(output) + + val nativeScan = CometNativeScanExec( + nativeOp = null, + relation = scan.relation, + output = scan.output, + requiredSchema = scan.requiredSchema, + partitionFilters = partitionFilters, + optionalBucketSet = scan.optionalBucketSet, + optionalNumCoalescedBuckets = scan.optionalNumCoalescedBuckets, + dataFilters = dataFilters, + tableIdentifier = scan.tableIdentifier, + disableBucketedScan = scan.disableBucketedScan, + originalPlan = scan, + serializedPlanOpt = SerializedPlan(None), + scan = null, + sourceKey = "") + val copied = nativeScan + .sparkFallback(Nil) + .asInstanceOf[FileSourceScanExec] + + assert(copied.getClass == scan.getClass) + assert(copied.partitionFilters == partitionFilters) + assert(copied.dataFilters == dataFilters) + assert(copied.output == output) + assert(copied.tableIdentifier == scan.tableIdentifier) + assert(copied.disableBucketedScan) + assert(copied.productIterator.toSeq.takeRight(2) == Seq(true, output)) + assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty) + } + + test("scan canonicalization preserves runtime fields and Spark normalization") { + val output = Seq(AttributeReference("id", IntegerType)()) + val staticFilter = GreaterThan(output.head, Literal(1)) + val scan = ExtendedFileSourceScanExec( + output, + partitionFilters = + Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), + dataFilters = Seq(LessThan(output.head, Literal(10)))) + + val canonicalized = CometScanUtils.canonicalizeOriginalPlan(scan) + val expectedSparkScan = scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize() + + assert(canonicalized.getClass == scan.getClass) + assert(canonicalized.productIterator.toSeq.takeRight(2) == Seq(true, output)) + // Check Spark's normalized fields separately from the distribution's extra fields. + assert( + canonicalized.productIterator.toSeq.dropRight(2) == expectedSparkScan.productIterator.toSeq) + assert(CometScanUtils.canonicalizeOriginalPlan(null) == null) + } + // Native serialization state and the source plan are deliberately not the identity of a // converted operator. Some operators do compare these; exclusion does not forbid comparison. private val commonExclusions = Set("nativeOp", "originalPlan") diff --git a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala new file mode 100644 index 00000000000..a3e1159f50a --- /dev/null +++ b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.HadoopFsRelation +import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.util.collection.BitSet + +private[apache] object ExtendedFileSourceScanExec { + def apply( + output: Seq[Attribute], + partitionFilters: Seq[Expression] = Nil, + dataFilters: Seq[Expression] = Nil): FileSourceScanExec = { + new ExtendedFileSourceScanExec( + null, + output, + new StructType().add("id", IntegerType), + partitionFilters, + None, + None, + dataFilters, + Some(TableIdentifier("test")), + true, + true, + output) + } +} + +// Model a distribution that adds scan fields. An inherited case-class copy would discard them. +private[apache] class ExtendedFileSourceScanExec( + relation: HadoopFsRelation, + output: Seq[Attribute], + requiredSchema: StructType, + partitionFilters: Seq[Expression], + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + dataFilters: Seq[Expression], + tableIdentifier: Option[TableIdentifier], + disableBucketedScan: Boolean, + val vendorFlag: Boolean, + val vendorOutput: Seq[Attribute]) + extends FileSourceScanExec( + relation, + output, + requiredSchema, + partitionFilters, + optionalBucketSet, + optionalNumCoalescedBuckets, + dataFilters, + tableIdentifier, + disableBucketedScan) { + override def doCanonicalize(): FileSourceScanExec = { + // Keep the simulated vendor fields observable after Spark normalizes its own fields. + val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray + makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput)) + .asInstanceOf[FileSourceScanExec] + } + + override def productArity: Int = 11 + + override def productElement(index: Int): Any = index match { + case 9 => vendorFlag + case 10 => vendorOutput + case _ => super.productElement(index) + } +} diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala new file mode 100644 index 00000000000..48ad52e270b --- /dev/null +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.connector.read.streaming.SparkDataStream +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.HadoopFsRelation +import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.util.collection.BitSet + +private[apache] object ExtendedFileSourceScanExec { + def apply( + output: Seq[Attribute], + partitionFilters: Seq[Expression] = Nil, + dataFilters: Seq[Expression] = Nil): FileSourceScanExec = { + new ExtendedFileSourceScanExec( + null, + None, + output, + new StructType().add("id", IntegerType), + partitionFilters, + None, + None, + dataFilters, + Some(TableIdentifier("test")), + true, + true, + output) + } +} + +// Model a distribution that adds scan fields. An inherited case-class copy would discard them. +private[apache] class ExtendedFileSourceScanExec( + relation: HadoopFsRelation, + stream: Option[SparkDataStream], + output: Seq[Attribute], + requiredSchema: StructType, + partitionFilters: Seq[Expression], + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + dataFilters: Seq[Expression], + tableIdentifier: Option[TableIdentifier], + disableBucketedScan: Boolean, + val vendorFlag: Boolean, + val vendorOutput: Seq[Attribute]) + extends FileSourceScanExec( + relation, + stream, + output, + requiredSchema, + partitionFilters, + optionalBucketSet, + optionalNumCoalescedBuckets, + dataFilters, + tableIdentifier, + disableBucketedScan) { + override def doCanonicalize(): FileSourceScanExec = { + // Keep the simulated vendor fields observable after Spark normalizes its own fields. + val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray + makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput)) + .asInstanceOf[FileSourceScanExec] + } + + override def productArity: Int = 12 + + override def productElement(index: Int): Any = index match { + case 10 => vendorFlag + case 11 => vendorOutput + case _ => super.productElement(index) + } +}