From ee4b08b4fa3936bbc59003c5ce2fe4e1f989811d Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 15 Nov 2026 00:54:06 +0800 Subject: [PATCH 1/4] chore: remove remaining Spark 3.4 compatibility code Co-authored-by: Codex --- .github/workflows/pr_build_linux.yml | 2 +- .github/workflows/pr_build_macos.yml | 2 +- benchmarks/README.md | 2 +- .../contrib/delta/DeltaScanSupport.scala | 7 +- .../delta/CometDeltaNativeScanSuite.scala | 3 +- dev/diffs/3.4.3.diff | 3448 ---------------- dev/diffs/iceberg/1.8.1.diff | 3509 ----------------- dev/regenerate-golden-files.sh | 5 +- .../latest/s3-credential-providers.md | 2 +- .../operators/iceberg_partition_path.rs | 8 +- native/core/src/execution/planner.rs | 2 +- .../src/execution/spark_aggregate_state.rs | 2 +- native/core/src/parquet/parquet_support.rs | 2 +- native/proto/src/proto/expr.proto | 11 +- native/spark-expr/src/agg_funcs/max_min_by.rs | 2 +- native/spark-expr/src/agg_funcs/mode.rs | 4 +- .../src/agg_funcs/quantile_summaries.rs | 2 +- native/spark-expr/src/agg_funcs/regr.rs | 14 +- native/spark-expr/src/math_funcs/negative.rs | 2 +- .../src/nondetermenistic_funcs/uuid.rs | 2 +- native/spark-expr/src/string_funcs/base64.rs | 4 +- .../scala/org/apache/comet/CometConf.scala | 10 +- .../comet/CometSparkSessionExtensions.scala | 17 +- .../comet/iceberg/IcebergReflection.scala | 89 +- .../iceberg/IcebergWriteReportListener.scala | 4 - .../comet/iceberg/IcebergWriteStrategy.scala | 11 - .../iceberg/ReplaceDataDispatchInfo.scala | 6 +- .../apache/comet/rules/CometExecRule.scala | 41 +- .../apache/comet/rules/CometScanRule.scala | 44 +- .../CometSpark34AqeDppFallbackRule.scala | 389 -- .../rules/EliminateRedundantTransitions.scala | 2 +- .../comet/serde/CometScalarFunction.scala | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 3 +- .../org/apache/comet/serde/SparkFixes.scala | 6 +- .../org/apache/comet/serde/aggregates.scala | 6 +- .../scala/org/apache/comet/serde/arrays.scala | 2 +- .../operator/CometIcebergNativeScan.scala | 27 +- .../operator/CometIcebergNativeWrite.scala | 4 +- .../serde/operator/CometNativeScan.scala | 47 +- .../org/apache/comet/serde/predicates.scala | 16 +- .../org/apache/comet/serde/statics.scala | 10 +- .../org/apache/comet/serde/strings.scala | 24 +- .../main/scala/org/apache/spark/Plugins.scala | 2 +- .../comet/CometBroadcastExchangeExec.scala | 2 +- .../comet/CometIcebergNativeScanExec.scala | 22 +- .../comet/CometInMemoryTableScanExec.scala | 4 +- .../CometSubqueryAdaptiveBroadcastExec.scala | 4 +- .../sql/comet/CometWindowGroupLimitExec.scala | 13 +- .../spark/sql/comet/DecimalPrecision.scala | 2 +- .../arrow/ArrowCachedBatchSerializer.scala | 2 +- .../shuffle/CometCelebornShuffleManager.scala | 2 +- .../shuffle/CometShuffleExchangeExec.scala | 6 +- .../shuffle/CometShuffleManager.scala | 5 +- .../apache/spark/sql/comet/operators.scala | 8 +- .../iceberg/IcebergTableAsSelectShim.scala | 34 - .../ShimCometSparkSessionExtensions.scala | 6 - .../shims/ShimCometWindowGroupLimit.scala | 3 +- .../cloud/s3/SdkCredentialExtraction.java | 4 +- .../apache/comet/shims/CometCollectShim.scala | 2 +- .../apache/comet/shims/CometSampleShim.scala | 2 +- .../shims/ShimCometShuffleExchangeExec.scala | 7 +- .../comet/shims/ShimCometBatchScanExec.scala | 2 +- .../shims/ShimDataSourceRDDPartition.scala | 2 +- .../apache/comet/shims/CometCollectShim.scala | 2 +- .../apache/comet/shims/CometSampleShim.scala | 2 +- .../apache/comet/shims/CometCollectShim.scala | 2 +- .../apache/comet/shims/CometSampleShim.scala | 2 +- .../iceberg/IcebergTableAsSelectShim.scala | 34 - .../ShimCometSparkSessionExtensions.scala | 6 - .../org/apache/comet/shims/ShimSQLConf.scala | 7 +- .../expressions/aggregate/collect_list.sql | 2 +- .../sql-tests/expressions/aggregate/mode.sql | 2 +- .../array/array_append_eager_eval.sql | 2 +- .../sql-tests/expressions/array/explode.sql | 2 +- .../expressions/array/sort_array.sql | 2 +- .../array/sort_array_min_spark_4_0.sql | 2 +- .../expressions/bitwise/bit_count_types.sql | 5 - .../expressions/cast/cast_string_trim.sql | 2 +- .../sql-tests/expressions/csv/to_csv.sql | 4 +- .../expressions/csv/to_csv_nested.sql | 2 +- .../datetime/make_timestamp_ansi.sql | 2 +- .../datetime/timestamp_seconds.sql | 2 +- .../to_unix_timestamp_ansi_spark34.sql | 39 - .../try_to_timestamp_time_parser_policy.sql | 1 - ...timestamp_time_parser_policy_corrected.sql | 1 - ...timestamp_time_parser_policy_exception.sql | 1 - ...to_timestamp_time_parser_policy_legacy.sql | 1 - .../map_builders_float_key_gate_legacy.sql | 2 +- .../sql-tests/expressions/misc/equal_null.sql | 2 - .../expressions/misc/uuid_with_seed.sql | 2 +- .../expressions/string/base64_unchunked.sql | 2 - .../string/decode_invalid_utf8.sql | 2 +- .../sql-tests/expressions/string/encode.sql | 2 +- .../string/encode_invalid_charset.sql | 4 +- .../string/encode_invalid_charset_strict.sql | 2 +- .../expressions/string/encode_unmappable.sql | 4 +- .../string/encode_unmappable_strict.sql | 2 +- .../q33/extended.txt | 96 - .../q44/extended.txt | 59 - .../q58/extended.txt | 111 - .../q67/extended.txt | 40 - .../q70/extended.txt | 56 - .../q83/extended.txt | 104 - .../approved-plans-v1_4/q33/extended.txt | 2 +- .../approved-plans-v1_4/q44/extended.txt | 69 +- .../approved-plans-v1_4/q58/extended.txt | 26 +- .../approved-plans-v1_4/q67/extended.txt | 65 +- .../approved-plans-v1_4/q70/extended.txt | 41 +- .../approved-plans-v1_4/q83/extended.txt | 17 +- .../q67a/extended.txt | 288 -- .../q70a/extended.txt | 159 - .../approved-plans-v2_7/q67a/extended.txt | 555 +-- .../approved-plans-v2_7/q70a/extended.txt | 119 +- .../comet/CometArrayExpressionSuite.scala | 6 +- .../comet/CometBroadcastBuildSideSuite.scala | 57 +- .../apache/comet/CometExpressionSuite.scala | 18 +- .../comet/CometIcebergNativeSuite.scala | 417 +- .../CometIcebergRewriteActionSuite.scala | 2 - .../comet/CometIcebergWriteActionSuite.scala | 111 +- .../CometIcebergWriteDetectionSuite.scala | 9 +- .../comet/CometMathExpressionSuite.scala | 5 - .../apache/comet/CometNativeCastSuite.scala | 2 +- .../comet/CometTemporalExpressionSuite.scala | 2 +- .../comet/CometUuidExpressionSuite.scala | 9 +- .../comet/SparkErrorConverterSuite.scala | 2 +- .../apache/comet/csv/CsvReadFromS3Suite.scala | 45 +- .../comet/exec/CometAggregateSuite.scala | 14 +- ...cala => CometExecCompatibilitySuite.scala} | 8 +- .../apache/comet/exec/CometExecSuite.scala | 613 ++- .../comet/exec/CometInMemoryCacheSuite.scala | 17 +- .../apache/comet/exec/CometJoinSuite.scala | 38 +- .../exec/CometMergeRowsNativeSuiteBase.scala | 11 - .../comet/exec/CometNativeShuffleSuite.scala | 3 +- .../comet/exec/CometWindowExecSuite.scala | 8 +- .../parquet/CometParquetWriterSuite.scala | 4 +- .../parquet/ParquetReadFromS3Suite.scala | 2 - .../comet/parquet/ParquetReadSuite.scala | 12 +- .../comet/rules/CometExecRuleSuite.scala | 19 +- .../rules/CometMultiStoreScanSuite.scala | 21 +- ...tNativeForTransitionHeavyStagesSuite.scala | 3 +- .../apache/comet/serde/SparkFixesSuite.scala | 5 - .../org/apache/spark/CometPluginsSuite.scala | 4 +- .../org/apache/spark/sql/CometTestBase.scala | 4 +- .../CometCartesianProductBenchmark.scala | 2 +- .../CometCodegenDispatchBenchmark.scala | 2 +- .../CometIcebergSystemFunctionBenchmark.scala | 2 +- .../CometStringExpressionBenchmark.scala | 2 +- .../sql/comet/CometCollectMetricsSuite.scala | 8 +- .../sql/comet/CometPlanStabilitySuite.scala | 5 +- .../arrow/CometCachedBatchHelper.scala | 4 +- .../CometCelebornShuffleManagerSuite.scala | 2 +- 151 files changed, 1150 insertions(+), 10235 deletions(-) delete mode 100644 dev/diffs/3.4.3.diff delete mode 100644 dev/diffs/iceberg/1.8.1.diff delete mode 100644 spark/src/main/scala/org/apache/comet/rules/CometSpark34AqeDppFallbackRule.scala delete mode 100644 spark/src/main/spark-3.5/org/apache/comet/iceberg/IcebergTableAsSelectShim.scala delete mode 100644 spark/src/main/spark-4.x/org/apache/comet/iceberg/IcebergTableAsSelectShim.scala delete mode 100644 spark/src/test/resources/sql-tests/expressions/datetime/to_unix_timestamp_ansi_spark34.sql delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/extended.txt delete mode 100644 spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/extended.txt rename spark/src/test/scala/org/apache/comet/exec/{CometExec3_4PlusSuite.scala => CometExecCompatibilitySuite.scala} (98%) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index f13101deeeb..3c48f07ae74 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -582,7 +582,7 @@ jobs: - name: "exec" value: | org.apache.comet.exec.CometAggregateSuite - org.apache.comet.exec.CometExec3_4PlusSuite + org.apache.comet.exec.CometExecCompatibilitySuite org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometMergeRowsSuite org.apache.comet.exec.CometTopKSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index ee13480830e..eae6aef64ad 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -220,7 +220,7 @@ jobs: - name: "exec" value: | org.apache.comet.exec.CometAggregateSuite - org.apache.comet.exec.CometExec3_4PlusSuite + org.apache.comet.exec.CometExecCompatibilitySuite org.apache.comet.exec.CometExecSuite org.apache.comet.exec.CometMergeRowsSuite org.apache.comet.exec.CometTopKSuite diff --git a/benchmarks/README.md b/benchmarks/README.md index 0b055900946..9cbd64e59d4 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -62,7 +62,7 @@ docker push localhost:32000/apache/datafusion-comet-tpcbench:latest export SPARK_MASTER=k8s://https://127.0.0.1:16443 export COMET_DOCKER_IMAGE=localhost:32000/apache/datafusion-comet-tpcbench:latest # Location of Comet JAR within the Docker image -export COMET_JAR=/opt/spark/jars/comet-spark-spark3.4_2.12-0.5.0-SNAPSHOT.jar +export COMET_JAR=/opt/spark/jars/comet-spark-spark3.5_2.12-0.5.0-SNAPSHOT.jar $SPARK_HOME/bin/spark-submit \ --master $SPARK_MASTER \ diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala index 96c18a85de0..15684b47c06 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala @@ -435,10 +435,9 @@ object DeltaScanSupport { return providerReason } - // Reuse core's generic native-scan gates (ignoreCorruptFiles/ignoreMissingFiles, AQE DPP on - // Spark 3.4, exec enabled, existence default values, the Variant read confs, and a proto - // representation for every serialized data and partition type); tags its own fallback - // reasons. + // Reuse core's generic native-scan gates (ignoreCorruptFiles/ignoreMissingFiles, exec enabled, + // existence default values, the Variant read confs, and a proto representation for every + // serialized data and partition type); tags its own fallback reasons. if (!CometNativeScan.isSupported(scanExec)) { return Some("Core native scan gates rejected the scan (see reasons above)") } diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala index 766fc4a3c8f..99bc8b3a5cd 100644 --- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala +++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala @@ -38,7 +38,7 @@ import org.apache.spark.sql.types.{ByteType, LongType, StringType, StructField, import org.apache.spark.sql.util.QueryExecutionListener import org.apache.comet.CometConf -import org.apache.comet.CometSparkSessionExtensions.{isSpark35Plus, isSpark40Plus} +import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus} import org.apache.comet.ExtendedExplainInfo import org.apache.comet.serde.OperatorOuterClass import org.apache.comet.serde.operator.CometNativeScan @@ -1751,7 +1751,6 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase { test( "DPP stays executable when transition reversion restores the delta scan: " + s"AQE=$adaptive") { - assume(isSpark35Plus, "Comet AQE DPP query-stage optimizer rules require Spark 3.5+") withTempPath { factDir => withTempPath { dimDir => val factPath = factDir.getAbsolutePath diff --git a/dev/diffs/3.4.3.diff b/dev/diffs/3.4.3.diff deleted file mode 100644 index faf550c0809..00000000000 --- a/dev/diffs/3.4.3.diff +++ /dev/null @@ -1,3448 +0,0 @@ -diff --git a/pom.xml b/pom.xml -index d3544881af1..f079228da3d 100644 ---- a/pom.xml -+++ b/pom.xml -@@ -148,6 +148,8 @@ - 0.10.0 - 2.5.1 - 2.0.8 -+ 3.4 -+ 1.3.0-SNAPSHOT - - -Wconf:cat=unused-nowarn&site=org\.apache\.spark\.sql\.comet\.execution\.shuffle\.Comet(Celeborn)?ShuffleManager\.get(Reader|Writer):s + + -Wconf:cat=unused-nowarn&src=.*/org/apache/spark/sql/comet/execution/shuffle/Comet(Celeborn)?ShuffleManager\.scala:s -Wconf:cat=unused-params&site=org\.apache\.spark\.sql\.comet\.CometIcebergNativeScanExec\.apply\.session:s -Wconf:cat=unused-params&site=org\.apache\.spark\.sql\.comet\.CometNativeScanExec\.apply\.session:s @@ -1062,8 +1063,9 @@ under the License. -Wconf:cat=unused-params&site=org\.apache\.comet\.udf\.CometUDF\.evaluate\.(partitionIndex|planId):s -Wconf:cat=unused-params&site=org\.apache\.comet\.udf\.CometUDF\.releasePlan\.planId:s -Wconf:cat=unused-params&site=org\.apache\.spark\.sql\.comet\.CometBaseAggregate\.operatorSupportLevel\.op:s - - -Wconf:cat=unused-nowarn&site=org\.apache\.spark\.sql\.comet\.execution\.shuffle\.Comet(Celeborn)?ShuffleManager\.get(Reader|Writer):s + + -Wconf:cat=unused-nowarn&src=.*/org/apache/spark/sql/comet/execution/shuffle/Comet(Celeborn)?ShuffleManager\.scala:s -Wconf:cat=unused-params&site=org\.apache\.spark\.sql\.comet\.CometIcebergNativeScanExec\.apply\.session:s -Wconf:cat=unused-params&site=org\.apache\.spark\.sql\.comet\.CometNativeScanExec\.apply\.session:s diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index c4c61ce550b..722ec1a4402 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -32,7 +32,7 @@ import scala.jdk.CollectionConverters._ import org.apache.hadoop.conf.Configuration import org.apache.spark.internal.Logging import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, Expression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} +import org.apache.spark.sql.catalyst.expressions.{Attribute, DynamicPruningExpression, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.util.{sideBySide, ArrayBasedMapData, DateTimeUtils, GenericArrayData, MetadataColumnHelper} import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala index a2bcde5d3c8..fba592852f6 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometNativeScanSuite.scala @@ -35,22 +35,20 @@ class CometNativeScanSuite extends AnyFunSuite { import CometNativeScan.{constantMetadataFieldPrefix, uniqueConstantMetadataFields} - test("native data schema preserves required root field ids and table projection indexes") { + test("native data schema preserves table root field ids and projection indexes") { val nestedType = StructType(Seq(StructField("a", IntegerType), StructField("b", StringType))) - val root = StructField("renamed", nestedType, nullable = false) - val dataSchema = - StructType(Seq(StructField("unrequested", LongType), root, StructField("id", IntegerType))) val partitionSchema = StructType(Seq(StructField("part", StringType))) - Seq(true, false).foreach { hasRequiredId => + Seq(true, false).foreach { hasFieldId => val metadata = new MetadataBuilder() - if (hasRequiredId) { + if (hasFieldId) { metadata.putLong(ParquetUtils.FIELD_ID_METADATA_KEY, 2L) } - val requiredRoot = root.copy( - dataType = StructType(Seq(nestedType("a"))), - nullable = true, - metadata = metadata.build()) + val root = StructField("renamed", nestedType, nullable = false, metadata = metadata.build()) + val dataSchema = + StructType( + Seq(StructField("unrequested", LongType), root, StructField("id", IntegerType))) + val requiredRoot = root.copy(dataType = StructType(Seq(nestedType("a"))), nullable = true) val requiredSchema = StructType(Seq(dataSchema("id"), requiredRoot)) val output = (requiredSchema.fields ++ partitionSchema.fields).map { field => AttributeReference(field.name, field.dataType, field.nullable, field.metadata)() @@ -74,8 +72,8 @@ class CometNativeScanSuite extends AnyFunSuite { val serializedRoot = common.getDataSchema(1) val fieldIdKey = CometParquetUtils.PARQUET_FIELD_ID_META_KEY - assert(serializedRoot.getMetadataMap.containsKey(fieldIdKey) == hasRequiredId) - if (hasRequiredId) { + assert(serializedRoot.getMetadataMap.containsKey(fieldIdKey) == hasFieldId) + if (hasFieldId) { assert(serializedRoot.getMetadataMap.get(fieldIdKey) == "2") } // Replacing the whole field with requiredRoot would lose the table's type and nullability. From 535cff32d81d63b7b907d66aa2c0b33b35a4bb67 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Mon, 16 Nov 2026 23:49:13 +0800 Subject: [PATCH 3/4] docs: align version guidance after Spark 3.4 removal Match Iceberg reflection documentation to the tested version matrix and drop Spark 3.4.3 from expression audit skills and their callers. Co-authored-by: Codex --- .ai/skills/audit-comet-expression/SKILL.md | 17 ++++++++--------- .ai/skills/implement-comet-expression/SKILL.md | 2 +- .ai/skills/wire-datafusion-function/SKILL.md | 2 +- .../comet/iceberg/IcebergReflection.scala | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.ai/skills/audit-comet-expression/SKILL.md b/.ai/skills/audit-comet-expression/SKILL.md index e864d23fa88..f0435c0593f 100644 --- a/.ai/skills/audit-comet-expression/SKILL.md +++ b/.ai/skills/audit-comet-expression/SKILL.md @@ -1,6 +1,6 @@ --- name: audit-comet-expression -description: Audit an existing Comet expression for correctness and test coverage. Studies the Spark implementation across versions 3.4.3, 3.5.8, 4.0.1, and 4.1.1, reviews the Comet and DataFusion implementations, identifies missing test coverage, and offers to implement additional tests. +description: Audit an existing Comet expression for correctness and test coverage. Studies the Spark implementation across versions 3.5.8, 4.0.1, and 4.1.1, reviews the Comet and DataFusion implementations, identifies missing test coverage, and offers to implement additional tests. argument-hint: --- @@ -29,7 +29,7 @@ Audit the Comet implementation of the `$ARGUMENTS` expression for correctness an This audit covers: -1. Spark implementation across versions 3.4.3, 3.5.8, 4.0.1, and 4.1.1 +1. Spark implementation across versions 3.5.8, 4.0.1, and 4.1.1 2. Comet Scala serde implementation 3. Comet Rust / DataFusion implementation 4. Existing test coverage (Comet SQL Tests and Comet Scala Tests) @@ -43,7 +43,7 @@ Clone specific Spark version tags (use shallow clones to avoid polluting the wor ```bash set -eu -o pipefail -for tag in v3.4.3 v3.5.8 v4.0.1 v4.1.1; do +for tag in v3.5.8 v4.0.1 v4.1.1; do dir="/tmp/spark-${tag}" if [ ! -d "$dir" ]; then git clone --depth 1 --branch "$tag" https://github.com/apache/spark.git "$dir" @@ -56,7 +56,7 @@ done Search the Catalyst SQL expressions source: ```bash -for tag in v3.4.3 v3.5.8 v4.0.1 v4.1.1; do +for tag in v3.5.8 v4.0.1 v4.1.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql/catalyst/src/main/scala" -name "*.scala" | \ @@ -67,7 +67,7 @@ done If the expression is not found in catalyst, also check core: ```bash -for tag in v3.4.3 v3.5.8 v4.0.1 v4.1.1; do +for tag in v3.5.8 v4.0.1 v4.1.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql" -name "*.scala" | \ @@ -90,7 +90,6 @@ For each Spark version, read the expression file and note: Produce a concise diff summary of what changed between: -- 3.4.3 → 3.5.8 - 3.5.8 → 4.0.1 - 4.0.1 → 4.1.1 @@ -107,7 +106,7 @@ Pay attention to: ## Step 2: Locate the Spark Tests ```bash -for tag in v3.4.3 v3.5.8 v4.0.1 v4.1.1; do +for tag in v3.5.8 v4.0.1 v4.1.1; do dir="/tmp/spark-${tag}" echo "=== $tag ===" find "$dir/sql" -name "*.scala" -path "*/test/*" | \ @@ -732,7 +731,7 @@ used for the expression in `docs/source/user-guide/latest/expressions.md`. other pages. - Add (or update) a `## ` section, keeping sections alphabetically ordered. - Under that heading, add one bullet per Spark version checked, each including: - - Spark version (e.g. 3.4.3, 3.5.8, 4.0.1, 4.1.1) + - Spark version (e.g. 3.5.8, 4.0.1, 4.1.1) - Today's date - A brief note for any version-specific finding (behavioral difference, known incompatibility); omit the note if nothing notable. @@ -744,7 +743,7 @@ used for the expression in `docs/source/user-guide/latest/expressions.md`. Present the audit as: 1. **Expression Summary** - Brief description of what `$ARGUMENTS` does, its input/output types, and null behavior -2. **Spark Version Differences** - Summary of any behavioral or API differences across Spark 3.4.3, 3.5.8, 4.0.1, and 4.1.1 +2. **Spark Version Differences** - Summary of any behavioral or API differences across Spark 3.5.8, 4.0.1, and 4.1.1 3. **Comet Implementation Notes** - Summary of how Comet implements this expression and any concerns 4. **Coverage Gap Analysis** - The gap table from Step 5, plus implementation gaps 5. **Recommendations** - Prioritized list from Step 6 diff --git a/.ai/skills/implement-comet-expression/SKILL.md b/.ai/skills/implement-comet-expression/SKILL.md index 1ee9eff091f..85daf081176 100644 --- a/.ai/skills/implement-comet-expression/SKILL.md +++ b/.ai/skills/implement-comet-expression/SKILL.md @@ -89,7 +89,7 @@ make ### 4. Run the audit skill -Once the initial implementation passes its smoke test, run the `audit-comet-expression` skill on `$ARGUMENTS`. It compares the implementation and tests against Spark 3.4.3, 3.5.8, and 4.0.1 and produces a prioritized list of gaps. +Once the initial implementation passes its smoke test, run the `audit-comet-expression` skill on `$ARGUMENTS`. It compares the implementation and tests against Spark 3.5.8, 4.0.1, and 4.1.1 and produces a prioritized list of gaps. ### 5. Implement audit-recommended tests and iterate diff --git a/.ai/skills/wire-datafusion-function/SKILL.md b/.ai/skills/wire-datafusion-function/SKILL.md index 4688f076c15..0cfc6e0e313 100644 --- a/.ai/skills/wire-datafusion-function/SKILL.md +++ b/.ai/skills/wire-datafusion-function/SKILL.md @@ -141,7 +141,7 @@ make cd native && cargo clippy --all-targets --workspace -- -D warnings ``` -Then run `audit-comet-expression` on `$ARGUMENTS` to compare against Spark 3.4.3 / 3.5.8 / 4.0.1 and surface coverage gaps; iterate on tests. +Then run `audit-comet-expression` on `$ARGUMENTS` to compare against Spark 3.5.8 / 4.0.1 / 4.1.1 and surface coverage gaps; iterate on tests. The user generally runs tests themselves. If asked for a smoke test: diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index c9d4b797451..4f252558377 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -1418,7 +1418,7 @@ object IcebergReflection extends Logging { /** * Reads a `private final` field from a `SparkWrite` instance via reflection. The field names * referenced by callers (`queryId`, `targetFileSize`, `useFanoutWriter`, `outputSpecId`, - * `writeSchema`) are present on `SparkWrite` across the Iceberg 1.8.1 / 1.10.0 / 1.11.0 + * `writeSchema`) are present on `SparkWrite` across the Iceberg 1.9.1 / 1.10.0 / 1.11.0 * versions Comet supports. */ private def getSparkWriteField(sparkWrite: Any, fieldName: String): Option[Any] = From edd2ac5b3a68b5530419a34223ec1d233bc0f635 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Wed, 18 Nov 2026 11:11:03 +0800 Subject: [PATCH 4/4] fix: finish Spark 3.4 cleanup after review Remove the unreachable JSON compatibility function and redundant version selector while retaining Jackson limits for supported Spark versions. Drop the redundant expected error-class alias and stale compatibility references. Co-authored-by: Codex --- .ai/skills/review-comet-pr/SKILL.md | 4 +- native/spark-expr/src/comet_scalar_funcs.rs | 4 -- .../src/string_funcs/get_json_object.rs | 57 +++---------------- native/spark-expr/src/string_funcs/mod.rs | 2 +- .../org/apache/comet/serde/strings.scala | 2 +- .../apache/comet/shims/CometExprShim.scala | 2 - .../apache/comet/shims/CometExprShim.scala | 2 - .../apache/comet/shims/CometExprShim.scala | 2 - .../expressions/string/get_json_object.sql | 11 ++-- .../apache/comet/CometExpressionSuite.scala | 3 +- 10 files changed, 16 insertions(+), 73 deletions(-) diff --git a/.ai/skills/review-comet-pr/SKILL.md b/.ai/skills/review-comet-pr/SKILL.md index 988ba875f17..2c6bb6f0db3 100644 --- a/.ai/skills/review-comet-pr/SKILL.md +++ b/.ai/skills/review-comet-pr/SKILL.md @@ -183,8 +183,8 @@ such as `wrap_second_millisecond_overflow`, never the version, such as `spark_42 fork that backports a Spark fix then changes the one Scala check that sets the flag and leaves the protobuf and Rust alone. Flag a version-named field, argument, or function, and comments that say "Spark 4.2" for behavior later releases inherit or that leave out the Spark JIRA. -`Cast.is_spark4_plus` and `get_json_object_spark34` predate the rule and #6891 tracks renaming them, -so flag them only when a PR adds behavior behind them. +`Cast.is_spark4_plus` predates the rule and #6891 tracks renaming it, +so flag it only when a PR adds behavior behind it. When Spark changed the behavior in a patch release, such as SPARK-55969 or SPARK-54918, a check on the minor version is wrong for every earlier patch. CI builds only the newest patch of each line, so diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 0086538b276..28661938b49 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -278,10 +278,6 @@ pub fn create_comet_physical_fun_with_eval_mode( let func = Arc::new(crate::string_funcs::spark_get_json_object); make_comet_scalar_udf!("get_json_object", func, without data_type) } - "get_json_object_spark34" => { - let func = Arc::new(crate::string_funcs::spark_get_json_object_spark34); - make_comet_scalar_udf!("get_json_object_spark34", func, without data_type) - } "map" => Ok(Arc::new(ScalarUDF::new_from_impl( SparkMapFromArrays::default(), ))), diff --git a/native/spark-expr/src/string_funcs/get_json_object.rs b/native/spark-expr/src/string_funcs/get_json_object.rs index 45f86e8baff..fc17ff6808a 100644 --- a/native/spark-expr/src/string_funcs/get_json_object.rs +++ b/native/spark-expr/src/string_funcs/get_json_object.rs @@ -55,18 +55,6 @@ fn scalar_to_str(scalar: &ScalarValue, arg_name: &str) -> DataFusionResult DataFusionResult { - spark_get_json_object_impl(args, true) -} - -/// Spark 3.4's Jackson 2.14 does not impose a default number-length limit. -pub fn spark_get_json_object_spark34(args: &[ColumnarValue]) -> DataFusionResult { - spark_get_json_object_impl(args, false) -} - -fn spark_get_json_object_impl( - args: &[ColumnarValue], - check_jackson_limits: bool, -) -> DataFusionResult { if args.len() != 2 { return exec_err!( "get_json_object expects 2 arguments (json, path), got {}", @@ -101,11 +89,7 @@ fn spark_get_json_object_impl( builder.append_null(); } else { let json_str = json_strings.value(i); - match evaluate_path_with_jackson_limits( - json_str, - &parsed_path, - check_jackson_limits, - ) { + match evaluate_path(json_str, &parsed_path) { Some(result) => builder.append_value(&result), None => builder.append_null(), } @@ -130,8 +114,7 @@ fn spark_get_json_object_impl( None => return Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None))), }; - let result = - evaluate_path_with_jackson_limits(&json_str, &parsed_path, check_jackson_limits); + let result = evaluate_path(&json_str, &parsed_path); Ok(ColumnarValue::Scalar(ScalarValue::Utf8(result))) } // Column json, column path @@ -147,11 +130,7 @@ fn spark_get_json_object_impl( let json_str = json_strings.value(i); let path_str = path_strings.value(i); match parse_json_path(path_str) { - Some(parsed_path) => match evaluate_path_with_jackson_limits( - json_str, - &parsed_path, - check_jackson_limits, - ) { + Some(parsed_path) => match evaluate_path(json_str, &parsed_path) { Some(result) => builder.append_value(&result), None => builder.append_null(), }, @@ -341,10 +320,9 @@ fn skip_string_body(bytes: &[u8], mut i: usize) -> Option { } } -/// Spark 3.5+ rejects overlong number tokens and documents nested beyond 1000 +/// Spark rejects overlong number tokens and documents nested beyond 1000 /// containers, including values this evaluation skips. serde_json's /// `IgnoredAny` enforces neither constraint, so inspect them before parsing. -/// Spark 3.4's Jackson has no default limits and bypasses this scan. fn violates_jackson_limits(json: &str) -> bool { let bytes = json.as_bytes(); let mut i = 0; @@ -424,22 +402,10 @@ fn violates_jackson_limits(json: &str) -> bool { /// Evaluate a parsed JSONPath against a JSON string. /// Returns the result as a string, or None if no match. -#[cfg(test)] fn evaluate_path(json_str: &str, path: &ParsedPath) -> Option { - evaluate_path_with_jackson_limits(json_str, path, true) -} - -fn evaluate_path_with_jackson_limits( - json_str: &str, - path: &ParsedPath, - check_jackson_limits: bool, -) -> Option { // A number longer than Jackson's limit needs at least 1,001 ASCII bytes. // Most JSON rows are shorter, so avoid a second pass over them entirely. - if check_jackson_limits - && json_str.len() > MAX_NUMBER_DIGITS - && violates_jackson_limits(json_str) - { + if json_str.len() > MAX_NUMBER_DIGITS && violates_jackson_limits(json_str) { return None; } @@ -1373,15 +1339,10 @@ mod tests { } #[test] - fn test_number_limit_depends_on_spark_version() { + fn test_number_limit_in_skipped_object_field() { let path = parse_json_path("$.a").unwrap(); let json = format!(r#"{{"a":1,"ignored":{}}}"#, "9".repeat(1001)); - // Spark 3.4 uses Jackson 2.14, which has no default numeric length limit. - assert_eq!( - evaluate_path_with_jackson_limits(&json, &path, false), - Some("1".to_string()) - ); - assert_eq!(evaluate_path_with_jackson_limits(&json, &path, true), None); + assert_eq!(evaluate_path(&json, &path), None); } #[test] @@ -1423,10 +1384,6 @@ mod tests { format!(r#"[{{"skip":{nested},"a":1}}]"#), ] { assert_eq!(evaluate_path(&json, &path), expected); - assert_eq!( - evaluate_path_with_jackson_limits(&json, &path, false), - Some("1".to_string()) - ); } } diff --git a/native/spark-expr/src/string_funcs/mod.rs b/native/spark-expr/src/string_funcs/mod.rs index b5cf38c2bea..d406c620a19 100644 --- a/native/spark-expr/src/string_funcs/mod.rs +++ b/native/spark-expr/src/string_funcs/mod.rs @@ -30,7 +30,7 @@ mod unbase64; pub use base64::spark_base64; pub use concat_ws::CometConcatWs; pub use contains::SparkContains; -pub use get_json_object::{spark_get_json_object, spark_get_json_object_spark34}; +pub use get_json_object::spark_get_json_object; pub use levenshtein::spark_levenshtein; pub use pattern_cache::PatternCache; pub use regexp_extract::spark_regexp_extract; diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala index cda3afd182b..5700564f574 100644 --- a/spark/src/main/scala/org/apache/comet/serde/strings.scala +++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala @@ -836,7 +836,7 @@ object CometGetJsonObject val jsonExpr = exprToProtoInternal(expr.json, inputs, binding) val pathExpr = exprToProtoInternal(expr.path, inputs, binding) val optExpr = scalarFunctionExprToProtoWithReturnType( - getJsonObjectNativeFunctionName, + "get_json_object", expr.dataType, false, jsonExpr, diff --git a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala index 997321c0152..e9383ae280c 100644 --- a/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-3.5/org/apache/comet/shims/CometExprShim.scala @@ -35,8 +35,6 @@ trait CometExprShim { def binaryOutputStyle: BinaryOutputStyle = BinaryOutputStyle.HEX_DISCRETE - def getJsonObjectNativeFunctionName: String = "get_json_object" - def sparkVersionSpecificStringExpressions : Map[Class[_ <: Expression], CometExpressionSerde[_]] = Map(classOf[StringDecode] -> CometStringDecode, classOf[Encode] -> CometEncode) diff --git a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala index f0a4c66fe7f..ca3c79930f3 100644 --- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala @@ -31,8 +31,6 @@ import org.apache.comet.serde.ExprOuterClass.BinaryOutputStyle */ trait CometExprShim extends Spark4xCometExprShim { - def getJsonObjectNativeFunctionName: String = "get_json_object" - def binaryOutputStyle: BinaryOutputStyle = { SQLConf.get .getConf(SQLConf.BINARY_OUTPUT_STYLE) diff --git a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala index fee16b9f076..5573d045832 100644 --- a/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala +++ b/spark/src/main/spark-4.1+/org/apache/comet/shims/CometExprShim.scala @@ -35,8 +35,6 @@ import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, scalarFunctio */ trait CometExprShim extends Spark4xCometExprShim { - def getJsonObjectNativeFunctionName: String = "get_json_object" - def binaryOutputStyle: BinaryOutputStyle = { // Since Spark 4.1, BINARY_OUTPUT_STYLE is an enumConf so getConf already returns the enum value. SQLConf.get.getConf(SQLConf.BINARY_OUTPUT_STYLE) match { diff --git a/spark/src/test/resources/sql-tests/expressions/string/get_json_object.sql b/spark/src/test/resources/sql-tests/expressions/string/get_json_object.sql index 4e2fa2164c0..da5a7368498 100644 --- a/spark/src/test/resources/sql-tests/expressions/string/get_json_object.sql +++ b/spark/src/test/resources/sql-tests/expressions/string/get_json_object.sql @@ -243,10 +243,8 @@ SELECT get_json_object('{"a":[{"b":[1,2]},{"b":[3]}]}', '$.a[*].b[*]') query SELECT get_json_object('[[[1,2],[]]]', '$[*][*][*]'), get_json_object('[[[1,2]]]', '$[*][*][*]') --- Spark 3.5+ rejects numbers whose digit count exceeds Jackson's default --- 1000-digit limit, including values the path never selects. Spark 3.4's --- default Jackson version (2.14) does not impose this limit, but Comet follows --- whichever Jackson version is on the classpath. +-- Spark rejects numbers whose digit count exceeds Jackson's default +-- 1000-digit limit, including values the path never selects. query SELECT get_json_object(concat('[{"a":1,"b":', repeat('9', 1000), '}]'), '$[*].a'), get_json_object(concat('[{"a":1,"b":', repeat('9', 1001), '}]'), '$[*].a'), @@ -262,9 +260,8 @@ SELECT get_json_object(concat('[{"a":1,"b":', repeat('9', 1000), '}]'), '$[*].a' -- Spark's result can change when a previous parse leaves a larger buffer in -- Jackson's thread-local recycler. The opt-in native path uses a fixed rule. --- Spark 3.5+ rejects a document deeper than Jackson's default 1000-level --- nesting limit, including a subtree skipped by the wildcard path. Spark 3.4 --- has no default limit. +-- Spark rejects a document deeper than Jackson's default 1000-level +-- nesting limit, including a subtree skipped by the wildcard path. query SELECT get_json_object(concat('[{"a":1,"skip":', repeat('[', 998), '0', repeat(']', 998), '}]'), '$[*].a'), get_json_object(concat('[{"a":1,"skip":', repeat('[', 999), '0', repeat(']', 999), '}]'), '$[*].a'), diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala index a74fb37c1ef..4abd96100b4 100644 --- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala @@ -3636,7 +3636,6 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { ("INT", Int.MinValue.toLong, Int.MaxValue.toLong, "ARITHMETIC_OVERFLOW"), ("BIGINT", Long.MinValue, Long.MaxValue, "ARITHMETIC_OVERFLOW"))) { test(s"ANSI integral overflow fidelity - $sqlType") { - val expectedErrorClass = errorClass val cases = Seq( ("+", "try_add", max, 1L), ("+", "try_add", min, -1L), @@ -3672,7 +3671,7 @@ class CometExpressionSuite extends CometTestBase with AdaptiveSparkPlanHelper { } val expected = structured(sparkError) val actual = structured(cometError) - assert(expected.getErrorClass == expectedErrorClass, query) + assert(expected.getErrorClass == errorClass, query) assert(actual.getClass == expected.getClass, query) assert(actual.getErrorClass == expected.getErrorClass, query) assert(actual.getSqlState == expected.getSqlState, query)