From c3d9bbdfb8843128b3c48a0d3b8fc7514dfacfe7 Mon Sep 17 00:00:00 2001 From: Samarth Mukhija <139998526+Samearth17@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:27:09 +0530 Subject: [PATCH 1/3] fix(cast): fall back for duplicate struct fields Signed-off-by: Samarth Mukhija <139998526+Samearth17@users.noreply.github.com> --- .../apache/comet/expressions/CometCast.scala | 8 ++++- .../apache/comet/CometNativeCastSuite.scala | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index e14fb086b12..5ab89802930 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} -import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.DataTypeSupport.{hasDuplicateFieldNames, isComplexType} import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde, CometTimeZone, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported} import org.apache.comet.serde.ExprOuterClass.Expr import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProtoInternal, serializeDataType} @@ -37,6 +37,9 @@ object CometCast with CometTypeShim with CodegenDispatchFallback { + private[comet] val duplicateStructFieldNamesReason: String = + "Cast target contains a struct with duplicate field names" + // Shared with CometNativeCastSuite so the asserted reason cannot drift from production. private[comet] val negativeScaleDecimalToStringReason: String = "Negative-scale decimal requires spark.sql.legacy.allowNegativeScaleOfDecimal=true" @@ -270,6 +273,9 @@ object CometCast case (DataTypes.DoubleType, _) => canCastFromDouble(toType) case (from_struct: StructType, to_struct: StructType) => + if (hasDuplicateFieldNames(to_struct.fields)) { + return Unsupported(Some(duplicateStructFieldNamesReason)) + } from_struct.fields.zip(to_struct.fields).foreach { case (a, b) => // `convert` replaces a top-level cast that is always null (DATE to a numeric or boolean // type in LEGACY mode) with a null literal, so the native cast never sees one. A struct diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 4db950e73c9..c7b62cbe6fc 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2032,6 +2032,35 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("cast to a struct with duplicate field names falls back") { + val sourceType = StructType(Seq(StructField("p", IntegerType), StructField("q", IntegerType))) + val duplicateTarget = + StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType))) + val uniqueTarget = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + val reason = CometCast.duplicateStructFieldNamesReason + + Seq( + sourceType -> duplicateTarget, + ArrayType(sourceType) -> ArrayType(duplicateTarget), + MapType(StringType, sourceType) -> MapType(StringType, duplicateTarget)).foreach { + case (fromType, toType) => + assert( + CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY) == + Unsupported(Some(reason))) + } + assert( + CometCast + .isSupported(sourceType, uniqueTarget, None, CometEvalMode.LEGACY) + .isInstanceOf[Compatible]) + + val input = spark + .range(3) + .selectExpr("named_struct('p', CAST(id AS INT), 'q', CAST(id + 10 AS INT)) AS s") + + checkSparkAnswerAndFallbackReason(input.select(col("s").cast(duplicateTarget)), reason) + } + test("cast between decimals with different precision and scale") { val rowData = Seq( Row(BigDecimal("12345.6789")), From e112b0f045815bf0f8e4103a0b4db9c68655518c Mon Sep 17 00:00:00 2001 From: samearth17 Date: Sat, 10 Oct 2026 10:04:37 +0530 Subject: [PATCH 2/3] test(cast): cover duplicate fields in SQL fixture --- .../cast/cast_duplicate_struct_fields.sql | 39 +++++++++++++++++++ .../apache/comet/CometNativeCastSuite.scala | 7 +--- 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql new file mode 100644 index 00000000000..f4b0931073b --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql @@ -0,0 +1,39 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +statement +CREATE TABLE test_cast_duplicate_struct_fields(s STRUCT) USING parquet + +statement +INSERT INTO test_cast_duplicate_struct_fields VALUES + (named_struct('p', 1, 'q', 11)), + (named_struct('p', 2, 'q', 12)) + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(s AS STRUCT) FROM test_cast_duplicate_struct_fields + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(array(s) AS ARRAY>) +FROM test_cast_duplicate_struct_fields + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(map('key', s) AS MAP>) +FROM test_cast_duplicate_struct_fields + +-- Distinct field names remain on the native path. +query +SELECT CAST(s AS STRUCT) FROM test_cast_duplicate_struct_fields diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index c7b62cbe6fc..e13ee4e70b5 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2032,7 +2032,7 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } - test("cast to a struct with duplicate field names falls back") { + test("cast to a struct with duplicate field names is unsupported") { val sourceType = StructType(Seq(StructField("p", IntegerType), StructField("q", IntegerType))) val duplicateTarget = StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType))) @@ -2054,11 +2054,6 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { .isSupported(sourceType, uniqueTarget, None, CometEvalMode.LEGACY) .isInstanceOf[Compatible]) - val input = spark - .range(3) - .selectExpr("named_struct('p', CAST(id AS INT), 'q', CAST(id + 10 AS INT)) AS s") - - checkSparkAnswerAndFallbackReason(input.select(col("s").cast(duplicateTarget)), reason) } test("cast between decimals with different precision and scale") { From 4d67e509c4a4748e6bbd6610a31eb969b36515c2 Mon Sep 17 00:00:00 2001 From: samearth17 Date: Fri, 16 Oct 2026 02:37:48 +0530 Subject: [PATCH 3/3] fix(cast): reject duplicate targets through null paths --- .../org/apache/comet/DataTypeSupport.scala | 11 +++++++++++ .../apache/comet/expressions/CometCast.scala | 12 ++++++++---- .../scala/org/apache/comet/serde/literals.scala | 6 +++++- .../cast/cast_duplicate_struct_fields.sql | 17 +++++++++++++++++ .../org/apache/comet/CometNativeCastSuite.scala | 2 ++ .../apache/comet/serde/CometLiteralSuite.scala | 12 ++++++++++++ 6 files changed, 55 insertions(+), 5 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala index 4d482c156ea..16a3e5dac5d 100644 --- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala +++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala @@ -93,6 +93,17 @@ object DataTypeSupport { def hasDuplicateFieldNames(fields: Array[StructField]): Boolean = fields.map(_.name).distinct.length != fields.length + /** True when any struct nested in `dt` has byte-identical sibling field names. */ + def containsDuplicateFieldNames(dt: DataType): Boolean = dt match { + case StructType(fields) => + hasDuplicateFieldNames(fields) || fields.exists(f => + containsDuplicateFieldNames(f.dataType)) + case ArrayType(elementType, _) => containsDuplicateFieldNames(elementType) + case MapType(keyType, valueType, _) => + containsDuplicateFieldNames(keyType) || containsDuplicateFieldNames(valueType) + case _ => false + } + /** * True when two sibling struct fields anywhere in `dt` fold to one name under * `toLowerCase(Locale.ROOT)`, the fold Spark's Parquet reader matches requested fields with diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 5ab89802930..ea1d882f004 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, import org.apache.comet.CometConf import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} -import org.apache.comet.DataTypeSupport.{hasDuplicateFieldNames, isComplexType} +import org.apache.comet.DataTypeSupport.{containsDuplicateFieldNames, isComplexType} import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde, CometTimeZone, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported} import org.apache.comet.serde.ExprOuterClass.Expr import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProtoInternal, serializeDataType} @@ -204,6 +204,13 @@ object CometCast return unsupported(fromType, toType) } + // Check the entire target before the NullType and identity shortcuts below. Arrow Java can + // collapse duplicate struct children even when the source reaches the target through a null, + // array, map, or nested struct path. + if (containsDuplicateFieldNames(toType)) { + return Unsupported(Some(duplicateStructFieldNamesReason)) + } + // Spark 4.0's collation metadata rides on `StringType`, but `serializeDataType` maps every // `StringType` to the same proto id, so a non-default collation is dropped on the way into // the native plan with no warning. Reject the cast outright rather than relying on the @@ -273,9 +280,6 @@ object CometCast case (DataTypes.DoubleType, _) => canCastFromDouble(toType) case (from_struct: StructType, to_struct: StructType) => - if (hasDuplicateFieldNames(to_struct.fields)) { - return Unsupported(Some(duplicateStructFieldNamesReason)) - } from_struct.fields.zip(to_struct.fields).foreach { case (a, b) => // `convert` replaces a top-level cast that is always null (DATE to a numeric or boolean // type in LEGACY mode) with a null literal, so the native cast never sees one. A struct diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index 621dd99ebe8..a6dc9ecf9e7 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -44,7 +44,11 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit override def getSupportLevel(expr: Literal): SupportLevel = { val serializable = expr.dataType match { // A null literal carries only its type on the wire, so any type serializeDataType handles. - case _ if expr.value == null => supportedDataType(expr.dataType, allowComplex = true) + case _ if expr.value == null => + supportedDataType( + expr.dataType, + allowComplex = true, + allowDuplicateStructFieldNames = false) case ArrayType(elementType, _) => listLiteralElementSupported(elementType) case dt => supportedDataType(dt) } diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql index f4b0931073b..e7040ac7f05 100644 --- a/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql @@ -34,6 +34,23 @@ query expect_fallback(Cast target contains a struct with duplicate field names) SELECT CAST(map('key', s) AS MAP>) FROM test_cast_duplicate_struct_fields +-- A null source must not bypass duplicate-name validation in a nested struct cast. +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(named_struct('a', NULL, 'b', s.p) + AS STRUCT, b: INT>) +FROM test_cast_duplicate_struct_fields + +-- The ARRAY shortcut must not bypass validation of the target element type. +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT s.p, CAST(e AS STRUCT) +FROM test_cast_duplicate_struct_fields +LATERAL VIEW explode(array(NULL, NULL)) v AS e + +-- Constant folding produces a typed null literal, whose own serde must reject the target shape. +query expect_fallback(Unsupported data type) +SELECT s, CAST(NULL AS STRUCT) +FROM test_cast_duplicate_struct_fields + -- Distinct field names remain on the native path. query SELECT CAST(s AS STRUCT) FROM test_cast_duplicate_struct_fields diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index e13ee4e70b5..c87578c3521 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2042,6 +2042,8 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { Seq( sourceType -> duplicateTarget, + DataTypes.NullType -> duplicateTarget, + ArrayType(DataTypes.NullType) -> ArrayType(duplicateTarget), ArrayType(sourceType) -> ArrayType(duplicateTarget), MapType(StringType, sourceType) -> MapType(StringType, duplicateTarget)).foreach { case (fromType, toType) => diff --git a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala index fd2f3aca61d..44c8691ed47 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala @@ -42,6 +42,18 @@ import org.apache.comet.shims.CometTypeShim */ class CometLiteralSuite extends CometTestBase with CometTypeShim { + test("a null literal with duplicate struct field names is unsupported") { + val duplicateType = + StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType))) + val distinctType = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + + assert( + CometLiteral.getSupportLevel(Literal.create(null, duplicateType)).isInstanceOf[Unsupported]) + assert( + CometLiteral.getSupportLevel(Literal.create(null, distinctType)).isInstanceOf[Compatible]) + } + /** * Element types the two sides are compared over. Both halves of the list matter. A type the * encoder handles today that drops out of the gate silently costs a native projection; a type