From 2d0c863ae9c09c2d46be44e05d655c9559e18d87 Mon Sep 17 00:00:00 2001 From: Daniel Wei Date: Thu, 5 Nov 2026 00:26:29 -0800 Subject: [PATCH 1/6] fix: preserve runtime fields when copying file scan filters --- .../spark/sql/comet/CometNativeScanExec.scala | 2 +- .../spark/sql/comet/CometScanUtils.scala | 19 ++- .../sql/comet/CometPlanEqualitySuite.scala | 109 ++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala index 8c8456b074a..bb71045f90b 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala @@ -79,7 +79,7 @@ case class CometNativeScanExec( // AQE rewrites DPP placeholders in the live scan before transition reversion. // The frozen originalPlan can still contain unexecutable SubqueryAdaptiveBroadcastExecs. val restoredScan = - originalPlan.copy(partitionFilters = partitionFilters, dataFilters = dataFilters) + CometScanUtils.copyScanWithFilters(originalPlan, partitionFilters, dataFilters) originalPlan.logicalLink.foreach(restoredScan.setLogicalLink) restoredScan } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala index 6eb1f9937ee..0a06a82e7bd 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala @@ -33,6 +33,18 @@ import org.apache.comet.objectstore.NativeConfig object CometScanUtils { + private[comet] def copyScanWithFilters( + scan: FileSourceScanExec, + partitionFilters: Seq[Expression], + dataFilters: Seq[Expression]): FileSourceScanExec = { + // Keep the runtime's full constructor, including fields added by Spark distributions. + // The supported Spark versions share these positions for partition and data filters. + val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray + args(4) = partitionFilters + args(7) = dataFilters + scan.makeCopy(args).asInstanceOf[FileSourceScanExec] + } + /** * Filters unused DynamicPruningExpression expressions - one which has been replaced with * DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning @@ -55,9 +67,10 @@ object CometScanUtils { if (originalPlan == null) { null } else { - originalPlan - .copy(partitionFilters = - originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression])) + copyScanWithFilters( + originalPlan, + originalPlan.partitionFilters.filterNot(_.isInstanceOf[DynamicPruningExpression]), + originalPlan.dataFilters) .doCanonicalize() } } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 52a5e1fe953..8466c2e49df 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -26,6 +26,13 @@ import scala.reflect.runtime.universe.runtimeMirror import org.scalatest.funsuite.AnyFunSuite +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.connector.read.streaming.SparkDataStream +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.HadoopFsRelation +import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.util.collection.BitSet import org.apache.xbean.asm9.{ClassReader, Opcodes} import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} @@ -36,6 +43,73 @@ import org.apache.comet.util.ClassLoaders class CometPlanEqualitySuite extends AnyFunSuite { + test("native scan fallback preserves additional runtime constructor fields") { + val output = Seq(AttributeReference("id", IntegerType)()) + val partitionFilters = Seq(GreaterThan(output.head, Literal(1))) + val dataFilters = Seq(LessThan(output.head, Literal(10))) + val scan = new CometPlanEqualitySuite.ExtendedFileSourceScanExec( + null, + None, + output, + new StructType().add("id", IntegerType), + Nil, + None, + None, + Nil, + Some(TableIdentifier("test")), + true, + true, + output) + + val nativeScan = CometNativeScanExec( + nativeOp = null, + relation = scan.relation, + output = scan.output, + requiredSchema = scan.requiredSchema, + partitionFilters = partitionFilters, + optionalBucketSet = scan.optionalBucketSet, + optionalNumCoalescedBuckets = scan.optionalNumCoalescedBuckets, + dataFilters = dataFilters, + tableIdentifier = scan.tableIdentifier, + disableBucketedScan = scan.disableBucketedScan, + originalPlan = scan, + serializedPlanOpt = SerializedPlan(None), + scan = null, + sourceKey = "") + val copied = nativeScan + .sparkFallback(Nil) + .asInstanceOf[CometPlanEqualitySuite.ExtendedFileSourceScanExec] + + assert(copied.partitionFilters == partitionFilters) + assert(copied.dataFilters == dataFilters) + assert(copied.output == output) + assert(copied.tableIdentifier == scan.tableIdentifier) + assert(copied.disableBucketedScan) + assert(copied.vendorFlag) + assert(copied.vendorOutput == output) + assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty) + } + + test("scan canonicalization retains Spark normalization and removes original DPP filters") { + val output = Seq(AttributeReference("id", IntegerType)()) + val staticFilter = GreaterThan(output.head, Literal(1)) + val scan = FileSourceScanExec( + null, + None, + output, + new StructType().add("id", IntegerType), + Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), + None, + None, + Seq(LessThan(output.head, Literal(10))), + Some(TableIdentifier("test"))) + + assert( + CometScanUtils.canonicalizeOriginalPlan(scan) == + scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize()) + assert(CometScanUtils.canonicalizeOriginalPlan(null) == null) + } + // Native serialization state and the source plan are deliberately not the identity of a // converted operator. Some operators do compare these; exclusion does not forbid comparison. private val commonExclusions = Set("nativeOp", "originalPlan") @@ -173,6 +247,41 @@ class CometPlanEqualitySuite extends AnyFunSuite { } object CometPlanEqualitySuite { + // Model a Spark distribution that appends fields to FileSourceScanExec. An inherited + // case-class copy would return the base class and discard these fields. + private class ExtendedFileSourceScanExec( + relation: HadoopFsRelation, + stream: Option[SparkDataStream], + output: Seq[Attribute], + requiredSchema: StructType, + partitionFilters: Seq[Expression], + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + dataFilters: Seq[Expression], + tableIdentifier: Option[TableIdentifier], + disableBucketedScan: Boolean, + val vendorFlag: Boolean, + val vendorOutput: Seq[Attribute]) + extends FileSourceScanExec( + relation, + stream, + output, + requiredSchema, + partitionFilters, + optionalBucketSet, + optionalNumCoalescedBuckets, + dataFilters, + tableIdentifier, + disableBucketedScan) { + override def productArity: Int = 12 + + override def productElement(index: Int): Any = index match { + case 10 => vendorFlag + case 11 => vendorOutput + case _ => super.productElement(index) + } + } + private class IncompleteEquality(val compared: Int, val omitted: Boolean) { override def equals(other: Any): Boolean = other match { case that: IncompleteEquality => compared == that.compared From 2986a9f2c04cb822b6aac9728b40d5dac4e53507 Mon Sep 17 00:00:00 2001 From: Daniel Wei Date: Thu, 5 Nov 2026 00:34:17 -0800 Subject: [PATCH 2/6] fix: handle Spark 3 and 4 scan constructor layouts --- .../spark/sql/comet/CometScanUtils.scala | 9 ++- .../sql/comet/CometPlanEqualitySuite.scala | 75 +++-------------- .../comet/ExtendedFileSourceScanExec.scala | 76 ++++++++++++++++++ .../comet/ExtendedFileSourceScanExec.scala | 80 +++++++++++++++++++ 4 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala create mode 100644 spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala index 0a06a82e7bd..88f02afd2aa 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala @@ -38,10 +38,13 @@ object CometScanUtils { partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileSourceScanExec = { // Keep the runtime's full constructor, including fields added by Spark distributions. - // The supported Spark versions share these positions for partition and data filters. val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray - args(4) = partitionFilters - args(7) = dataFilters + // Spark 4 adds a stream argument before the schema; the filters keep the same offsets + // after it. Locate the existing schema rather than assuming a Spark-specific arity. + val schemaIndex = args.indexWhere(_ eq scan.requiredSchema) + require(schemaIndex >= 0, "FileSourceScanExec is missing its required schema") + args(schemaIndex + 1) = partitionFilters + args(schemaIndex + 4) = dataFilters scan.makeCopy(args).asInstanceOf[FileSourceScanExec] } diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 8466c2e49df..4d4c449ac10 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -26,13 +26,9 @@ import scala.reflect.runtime.universe.runtimeMirror import org.scalatest.funsuite.AnyFunSuite -import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.connector.read.streaming.SparkDataStream import org.apache.spark.sql.execution.FileSourceScanExec -import org.apache.spark.sql.execution.datasources.HadoopFsRelation -import org.apache.spark.sql.types.{IntegerType, StructType} -import org.apache.spark.util.collection.BitSet +import org.apache.spark.sql.types.IntegerType import org.apache.xbean.asm9.{ClassReader, Opcodes} import org.apache.xbean.asm9.tree.{ClassNode, FieldInsnNode, MethodInsnNode} @@ -47,19 +43,7 @@ class CometPlanEqualitySuite extends AnyFunSuite { val output = Seq(AttributeReference("id", IntegerType)()) val partitionFilters = Seq(GreaterThan(output.head, Literal(1))) val dataFilters = Seq(LessThan(output.head, Literal(10))) - val scan = new CometPlanEqualitySuite.ExtendedFileSourceScanExec( - null, - None, - output, - new StructType().add("id", IntegerType), - Nil, - None, - None, - Nil, - Some(TableIdentifier("test")), - true, - true, - output) + val scan = ExtendedFileSourceScanExec(output) val nativeScan = CometNativeScanExec( nativeOp = null, @@ -78,31 +62,25 @@ class CometPlanEqualitySuite extends AnyFunSuite { sourceKey = "") val copied = nativeScan .sparkFallback(Nil) - .asInstanceOf[CometPlanEqualitySuite.ExtendedFileSourceScanExec] + .asInstanceOf[FileSourceScanExec] + assert(copied.getClass == scan.getClass) assert(copied.partitionFilters == partitionFilters) assert(copied.dataFilters == dataFilters) assert(copied.output == output) assert(copied.tableIdentifier == scan.tableIdentifier) assert(copied.disableBucketedScan) - assert(copied.vendorFlag) - assert(copied.vendorOutput == output) + assert(copied.productIterator.toSeq.takeRight(2) == Seq(true, output)) assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty) } test("scan canonicalization retains Spark normalization and removes original DPP filters") { val output = Seq(AttributeReference("id", IntegerType)()) val staticFilter = GreaterThan(output.head, Literal(1)) - val scan = FileSourceScanExec( - null, - None, - output, - new StructType().add("id", IntegerType), - Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), - None, - None, - Seq(LessThan(output.head, Literal(10))), - Some(TableIdentifier("test"))) + val scan = ExtendedFileSourceScanExec(output).copy( + partitionFilters = + Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), + dataFilters = Seq(LessThan(output.head, Literal(10)))) assert( CometScanUtils.canonicalizeOriginalPlan(scan) == @@ -247,41 +225,6 @@ class CometPlanEqualitySuite extends AnyFunSuite { } object CometPlanEqualitySuite { - // Model a Spark distribution that appends fields to FileSourceScanExec. An inherited - // case-class copy would return the base class and discard these fields. - private class ExtendedFileSourceScanExec( - relation: HadoopFsRelation, - stream: Option[SparkDataStream], - output: Seq[Attribute], - requiredSchema: StructType, - partitionFilters: Seq[Expression], - optionalBucketSet: Option[BitSet], - optionalNumCoalescedBuckets: Option[Int], - dataFilters: Seq[Expression], - tableIdentifier: Option[TableIdentifier], - disableBucketedScan: Boolean, - val vendorFlag: Boolean, - val vendorOutput: Seq[Attribute]) - extends FileSourceScanExec( - relation, - stream, - output, - requiredSchema, - partitionFilters, - optionalBucketSet, - optionalNumCoalescedBuckets, - dataFilters, - tableIdentifier, - disableBucketedScan) { - override def productArity: Int = 12 - - override def productElement(index: Int): Any = index match { - case 10 => vendorFlag - case 11 => vendorOutput - case _ => super.productElement(index) - } - } - private class IncompleteEquality(val compared: Int, val omitted: Boolean) { override def equals(other: Any): Boolean = other match { case that: IncompleteEquality => compared == that.compared diff --git a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala new file mode 100644 index 00000000000..2a9d158bd7f --- /dev/null +++ b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.HadoopFsRelation +import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.util.collection.BitSet + +private[comet] object ExtendedFileSourceScanExec { + def apply(output: Seq[Attribute]): FileSourceScanExec = { + new ExtendedFileSourceScanExec( + null, + output, + new StructType().add("id", IntegerType), + Nil, + None, + None, + Nil, + Some(TableIdentifier("test")), + true, + true, + output) + } +} + +// Model a distribution that adds scan fields. An inherited case-class copy would discard them. +private[comet] class ExtendedFileSourceScanExec( + relation: HadoopFsRelation, + output: Seq[Attribute], + requiredSchema: StructType, + partitionFilters: Seq[Expression], + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + dataFilters: Seq[Expression], + tableIdentifier: Option[TableIdentifier], + disableBucketedScan: Boolean, + val vendorFlag: Boolean, + val vendorOutput: Seq[Attribute]) + extends FileSourceScanExec( + relation, + output, + requiredSchema, + partitionFilters, + optionalBucketSet, + optionalNumCoalescedBuckets, + dataFilters, + tableIdentifier, + disableBucketedScan) { + override def productArity: Int = 11 + + override def productElement(index: Int): Any = index match { + case 9 => vendorFlag + case 10 => vendorOutput + case _ => super.productElement(index) + } +} diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala new file mode 100644 index 00000000000..d561fa973a7 --- /dev/null +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} +import org.apache.spark.sql.connector.read.streaming.SparkDataStream +import org.apache.spark.sql.execution.FileSourceScanExec +import org.apache.spark.sql.execution.datasources.HadoopFsRelation +import org.apache.spark.sql.types.{IntegerType, StructType} +import org.apache.spark.util.collection.BitSet + +private[comet] object ExtendedFileSourceScanExec { + def apply(output: Seq[Attribute]): FileSourceScanExec = { + new ExtendedFileSourceScanExec( + null, + None, + output, + new StructType().add("id", IntegerType), + Nil, + None, + None, + Nil, + Some(TableIdentifier("test")), + true, + true, + output) + } +} + +// Model a distribution that adds scan fields. An inherited case-class copy would discard them. +private[comet] class ExtendedFileSourceScanExec( + relation: HadoopFsRelation, + stream: Option[SparkDataStream], + output: Seq[Attribute], + requiredSchema: StructType, + partitionFilters: Seq[Expression], + optionalBucketSet: Option[BitSet], + optionalNumCoalescedBuckets: Option[Int], + dataFilters: Seq[Expression], + tableIdentifier: Option[TableIdentifier], + disableBucketedScan: Boolean, + val vendorFlag: Boolean, + val vendorOutput: Seq[Attribute]) + extends FileSourceScanExec( + relation, + stream, + output, + requiredSchema, + partitionFilters, + optionalBucketSet, + optionalNumCoalescedBuckets, + dataFilters, + tableIdentifier, + disableBucketedScan) { + override def productArity: Int = 12 + + override def productElement(index: Int): Any = index match { + case 10 => vendorFlag + case 11 => vendorOutput + case _ => super.productElement(index) + } +} From c05f349b40f433500d5ae32d1a72bdd9860a4ec5 Mon Sep 17 00:00:00 2001 From: Daniel Wei Date: Thu, 5 Nov 2026 13:21:47 -0800 Subject: [PATCH 3/6] fix: preserve runtime scan fields during make_date fallback --- .../comet/rules/MakeDateEvaluation.scala | 7 ++-- .../spark/sql/comet/CometScanUtils.scala | 2 +- .../comet/rules/CometExecRuleSuite.scala | 35 +++++++++++++++++++ .../sql/comet/CometPlanEqualitySuite.scala | 3 +- .../comet/ExtendedFileSourceScanExec.scala | 13 ++++--- .../comet/ExtendedFileSourceScanExec.scala | 13 ++++--- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala b/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala index 7811b893b2f..6a6242c5303 100644 --- a/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala +++ b/spark/src/main/scala/org/apache/comet/rules/MakeDateEvaluation.scala @@ -231,9 +231,10 @@ private[rules] object MakeDateEvaluation { val original = node match { case scan: CometScanExec => - scan.wrapped.copy( - partitionFilters = scan.partitionFilters, - dataFilters = scan.dataFilters) + CometScanUtils.copyScanWithFilters( + scan.wrapped, + scan.partitionFilters, + scan.dataFilters) case scan: CometNativeScanExec => scan.sparkFallback(Nil) case native: CometExec => native.originalPlan case shuffle: CometShuffleExchangeExec => shuffle.originalPlan diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala index 88f02afd2aa..fbe97e5b088 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala @@ -33,7 +33,7 @@ import org.apache.comet.objectstore.NativeConfig object CometScanUtils { - private[comet] def copyScanWithFilters( + private[apache] def copyScanWithFilters( scan: FileSourceScanExec, partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileSourceScanExec = { diff --git a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala index ffbe1d0e1b0..3ba1a8bc925 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometExecRuleSuite.scala @@ -155,6 +155,41 @@ class CometExecRuleSuite extends CometTestBase { }.sum } + test("make_date scan fallback preserves additional runtime constructor fields") { + withSQLConf(CometConf.COMET_EXEC_MAKE_DATE_OPERATOR_FALLBACK_ENABLED.key -> "true") { + val output = Seq(AttributeReference("day", DataTypes.IntegerType)()) + val partitionFilters = Seq( + GreaterThan( + MakeDate(Literal(2024), Literal(1), output.head), + Literal.create(java.sql.Date.valueOf("2024-01-01"), DataTypes.DateType))) + val dataFilters = Seq(GreaterThan(output.head, Literal(1))) + val wrapped = ExtendedFileSourceScanExec(output) + val scan = CometScanExec( + relation = wrapped.relation, + output = wrapped.output, + requiredSchema = wrapped.requiredSchema, + partitionFilters = partitionFilters, + optionalBucketSet = wrapped.optionalBucketSet, + optionalNumCoalescedBuckets = wrapped.optionalNumCoalescedBuckets, + dataFilters = dataFilters, + tableIdentifier = wrapped.tableIdentifier, + disableBucketedScan = wrapped.disableBucketedScan, + wrapped = wrapped) + + val restored = MakeDateEvaluation + .preserve(CollectLimitExec(1, scan)) + .asInstanceOf[CollectLimitExec] + .child + .asInstanceOf[FileSourceScanExec] + + assert(restored.getClass == wrapped.getClass) + assert(restored.partitionFilters == partitionFilters) + assert(restored.dataFilters == dataFilters) + assert(restored.productIterator.toSeq.takeRight(2) == Seq(true, output)) + assert(wrapped.partitionFilters.isEmpty && wrapped.dataFilters.isEmpty) + } + } + test("make_date preserves row demand below lazy consumers, including reused native plans") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 4d4c449ac10..cc687318864 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -77,7 +77,8 @@ class CometPlanEqualitySuite extends AnyFunSuite { test("scan canonicalization retains Spark normalization and removes original DPP filters") { val output = Seq(AttributeReference("id", IntegerType)()) val staticFilter = GreaterThan(output.head, Literal(1)) - val scan = ExtendedFileSourceScanExec(output).copy( + val scan = ExtendedFileSourceScanExec( + output, partitionFilters = Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), dataFilters = Seq(LessThan(output.head, Literal(10)))) diff --git a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala index 2a9d158bd7f..ede67648426 100644 --- a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala +++ b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -26,16 +26,19 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.spark.util.collection.BitSet -private[comet] object ExtendedFileSourceScanExec { - def apply(output: Seq[Attribute]): FileSourceScanExec = { +private[apache] object ExtendedFileSourceScanExec { + def apply( + output: Seq[Attribute], + partitionFilters: Seq[Expression] = Nil, + dataFilters: Seq[Expression] = Nil): FileSourceScanExec = { new ExtendedFileSourceScanExec( null, output, new StructType().add("id", IntegerType), - Nil, + partitionFilters, None, None, - Nil, + dataFilters, Some(TableIdentifier("test")), true, true, @@ -44,7 +47,7 @@ private[comet] object ExtendedFileSourceScanExec { } // Model a distribution that adds scan fields. An inherited case-class copy would discard them. -private[comet] class ExtendedFileSourceScanExec( +private[apache] class ExtendedFileSourceScanExec( relation: HadoopFsRelation, output: Seq[Attribute], requiredSchema: StructType, diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala index d561fa973a7..7259f3da53b 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -27,17 +27,20 @@ import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.types.{IntegerType, StructType} import org.apache.spark.util.collection.BitSet -private[comet] object ExtendedFileSourceScanExec { - def apply(output: Seq[Attribute]): FileSourceScanExec = { +private[apache] object ExtendedFileSourceScanExec { + def apply( + output: Seq[Attribute], + partitionFilters: Seq[Expression] = Nil, + dataFilters: Seq[Expression] = Nil): FileSourceScanExec = { new ExtendedFileSourceScanExec( null, None, output, new StructType().add("id", IntegerType), - Nil, + partitionFilters, None, None, - Nil, + dataFilters, Some(TableIdentifier("test")), true, true, @@ -46,7 +49,7 @@ private[comet] object ExtendedFileSourceScanExec { } // Model a distribution that adds scan fields. An inherited case-class copy would discard them. -private[comet] class ExtendedFileSourceScanExec( +private[apache] class ExtendedFileSourceScanExec( relation: HadoopFsRelation, stream: Option[SparkDataStream], output: Seq[Attribute], From 759231a764310b3d9f24d5a4a5a07f9a2a605e7c Mon Sep 17 00:00:00 2001 From: Daniel Wei Date: Sun, 15 Nov 2026 16:00:15 -0800 Subject: [PATCH 4/6] test: verify runtime scan fields survive canonicalization --- .../spark/sql/comet/CometPlanEqualitySuite.scala | 11 ++++++++--- .../spark/sql/comet/ExtendedFileSourceScanExec.scala | 7 +++++++ .../spark/sql/comet/ExtendedFileSourceScanExec.scala | 7 +++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index cc687318864..765a37879e0 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -74,7 +74,7 @@ class CometPlanEqualitySuite extends AnyFunSuite { assert(scan.partitionFilters.isEmpty && scan.dataFilters.isEmpty) } - test("scan canonicalization retains Spark normalization and removes original DPP filters") { + test("scan canonicalization preserves runtime fields and Spark normalization") { val output = Seq(AttributeReference("id", IntegerType)()) val staticFilter = GreaterThan(output.head, Literal(1)) val scan = ExtendedFileSourceScanExec( @@ -83,9 +83,14 @@ class CometPlanEqualitySuite extends AnyFunSuite { Seq(staticFilter, DynamicPruningExpression(EqualTo(output.head, Literal(2)))), dataFilters = Seq(LessThan(output.head, Literal(10)))) + val canonicalized = CometScanUtils.canonicalizeOriginalPlan(scan) + val expectedSparkScan = scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize() + + assert(canonicalized.getClass == scan.getClass) + assert(canonicalized.productIterator.toSeq.takeRight(2) == Seq(true, output)) + // Check Spark's normalized fields separately from the distribution's extra fields. assert( - CometScanUtils.canonicalizeOriginalPlan(scan) == - scan.copy(partitionFilters = Seq(staticFilter)).doCanonicalize()) + canonicalized.productIterator.toSeq.dropRight(2) == expectedSparkScan.productIterator.toSeq) assert(CometScanUtils.canonicalizeOriginalPlan(null) == null) } diff --git a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala index ede67648426..a3e1159f50a 100644 --- a/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala +++ b/spark/src/test/spark-3.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -69,6 +69,13 @@ private[apache] class ExtendedFileSourceScanExec( dataFilters, tableIdentifier, disableBucketedScan) { + override def doCanonicalize(): FileSourceScanExec = { + // Keep the simulated vendor fields observable after Spark normalizes its own fields. + val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray + makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput)) + .asInstanceOf[FileSourceScanExec] + } + override def productArity: Int = 11 override def productElement(index: Int): Any = index match { diff --git a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala index 7259f3da53b..48ad52e270b 100644 --- a/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala +++ b/spark/src/test/spark-4.x/org/apache/spark/sql/comet/ExtendedFileSourceScanExec.scala @@ -73,6 +73,13 @@ private[apache] class ExtendedFileSourceScanExec( dataFilters, tableIdentifier, disableBucketedScan) { + override def doCanonicalize(): FileSourceScanExec = { + // Keep the simulated vendor fields observable after Spark normalizes its own fields. + val args = super.doCanonicalize().productIterator.map(_.asInstanceOf[AnyRef]).toArray + makeCopy(args ++ Array[AnyRef](Boolean.box(vendorFlag), vendorOutput)) + .asInstanceOf[FileSourceScanExec] + } + override def productArity: Int = 12 override def productElement(index: Int): Any = index match { From ea4a9d80c69a5a9bba4c1b585da74e2ea43cf326 Mon Sep 17 00:00:00 2001 From: Daniel Wei Date: Mon, 16 Nov 2026 11:22:58 -0800 Subject: [PATCH 5/6] ci: trigger fork checks From 8dbae851b91faa2ebb05d23655fe7655dfc7aec1 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 15:07:34 -0700 Subject: [PATCH 6/6] refactor: locate FileSourceScanExec filter arguments through a shim copyScanWithFilters found the filters by searching the constructor arguments for requiredSchema and writing at fixed offsets from it. Move the Spark 3 versus Spark 4 layout difference into a ShimFileSourceScanExec in the spark-3.x and spark-4.x source sets, and check that each slot holds the scan's current filter list before replacing it, so an unexpected layout fails with a clear error. Co-Authored-By: Claude Opus 5.5 --- .../spark/sql/comet/CometScanUtils.scala | 31 ++++++++++++++----- .../comet/shims/ShimFileSourceScanExec.scala | 30 ++++++++++++++++++ .../comet/shims/ShimFileSourceScanExec.scala | 30 ++++++++++++++++++ .../sql/comet/CometPlanEqualitySuite.scala | 13 ++++++++ 4 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 spark/src/main/spark-3.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala create mode 100644 spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala index fbe97e5b088..79366bc5804 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometScanUtils.scala @@ -25,6 +25,7 @@ import java.util.Locale import scala.collection.mutable import org.apache.spark.sql.catalyst.expressions.{DynamicPruningExpression, Expression, Literal} +import org.apache.spark.sql.comet.shims.ShimFileSourceScanExec import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile} import org.apache.spark.sql.types.StructType @@ -37,17 +38,33 @@ object CometScanUtils { scan: FileSourceScanExec, partitionFilters: Seq[Expression], dataFilters: Seq[Expression]): FileSourceScanExec = { - // Keep the runtime's full constructor, including fields added by Spark distributions. + // Rebuild through makeCopy rather than the compiled case-class copy, so that a runtime + // whose FileSourceScanExec appends constructor fields, such as Amazon EMR, keeps them. val args = scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray - // Spark 4 adds a stream argument before the schema; the filters keep the same offsets - // after it. Locate the existing schema rather than assuming a Spark-specific arity. - val schemaIndex = args.indexWhere(_ eq scan.requiredSchema) - require(schemaIndex >= 0, "FileSourceScanExec is missing its required schema") - args(schemaIndex + 1) = partitionFilters - args(schemaIndex + 4) = dataFilters + replaceArg( + args, + ShimFileSourceScanExec.partitionFiltersIndex, + scan.partitionFilters, + partitionFilters) + replaceArg(args, ShimFileSourceScanExec.dataFiltersIndex, scan.dataFilters, dataFilters) scan.makeCopy(args).asInstanceOf[FileSourceScanExec] } + // Fail loudly if the runtime constructor does not have the expected value at `index`, + // rather than rebuilding the scan with an argument in the wrong position. + private def replaceArg( + args: Array[AnyRef], + index: Int, + expected: AnyRef, + replacement: AnyRef): Unit = { + if (index >= args.length || args(index) != expected) { + throw new IllegalStateException( + s"Unexpected FileSourceScanExec constructor layout: argument $index is not the " + + "expected filter list") + } + args(index) = replacement + } + /** * Filters unused DynamicPruningExpression expressions - one which has been replaced with * DynamicPruningExpression(Literal.TrueLiteral) during Physical Planning diff --git a/spark/src/main/spark-3.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala b/spark/src/main/spark-3.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala new file mode 100644 index 00000000000..a2611372afd --- /dev/null +++ b/spark/src/main/spark-3.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet.shims + +/** + * Positions of the filter arguments in the `FileSourceScanExec` constructor, used to rebuild a + * scan through `makeCopy` with new filters. The constructor is `FileSourceScanExec(relation, + * output, requiredSchema, partitionFilters, ...)`. + */ +object ShimFileSourceScanExec { + val partitionFiltersIndex: Int = 3 + val dataFiltersIndex: Int = 6 +} diff --git a/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala new file mode 100644 index 00000000000..a0422f85187 --- /dev/null +++ b/spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/ShimFileSourceScanExec.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet.shims + +/** + * Positions of the filter arguments in the `FileSourceScanExec` constructor, used to rebuild a + * scan through `makeCopy` with new filters. The constructor is `FileSourceScanExec(relation, + * stream, output, requiredSchema, partitionFilters, ...)`. + */ +object ShimFileSourceScanExec { + val partitionFiltersIndex: Int = 4 + val dataFiltersIndex: Int = 7 +} diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala index 765a37879e0..3e5ff0d4326 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometPlanEqualitySuite.scala @@ -27,6 +27,7 @@ import scala.reflect.runtime.universe.runtimeMirror import org.scalatest.funsuite.AnyFunSuite import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.comet.shims.ShimFileSourceScanExec import org.apache.spark.sql.execution.FileSourceScanExec import org.apache.spark.sql.types.IntegerType import org.apache.xbean.asm9.{ClassReader, Opcodes} @@ -39,6 +40,18 @@ import org.apache.comet.util.ClassLoaders class CometPlanEqualitySuite extends AnyFunSuite { + test("FileSourceScanExec shim filter indices match the constructor") { + val output = Seq(AttributeReference("id", IntegerType)()) + val scan = ExtendedFileSourceScanExec( + output, + partitionFilters = Seq(GreaterThan(output.head, Literal(1))), + dataFilters = Seq(LessThan(output.head, Literal(10)))) + + assert( + scan.productElement(ShimFileSourceScanExec.partitionFiltersIndex) == scan.partitionFilters) + assert(scan.productElement(ShimFileSourceScanExec.dataFiltersIndex) == scan.dataFilters) + } + test("native scan fallback preserves additional runtime constructor fields") { val output = Seq(AttributeReference("id", IntegerType)()) val partitionFilters = Seq(GreaterThan(output.head, Literal(1)))