diff --git a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala index 4d482c156ea..16a3e5dac5d 100644 --- a/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala +++ b/spark/src/main/scala/org/apache/comet/DataTypeSupport.scala @@ -93,6 +93,17 @@ object DataTypeSupport { def hasDuplicateFieldNames(fields: Array[StructField]): Boolean = fields.map(_.name).distinct.length != fields.length + /** True when any struct nested in `dt` has byte-identical sibling field names. */ + def containsDuplicateFieldNames(dt: DataType): Boolean = dt match { + case StructType(fields) => + hasDuplicateFieldNames(fields) || fields.exists(f => + containsDuplicateFieldNames(f.dataType)) + case ArrayType(elementType, _) => containsDuplicateFieldNames(elementType) + case MapType(keyType, valueType, _) => + containsDuplicateFieldNames(keyType) || containsDuplicateFieldNames(valueType) + case _ => false + } + /** * True when two sibling struct fields anywhere in `dt` fold to one name under * `toLowerCase(Locale.ROOT)`, the fold Spark's Parquet reader matches requested fields with diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index d37c621f18e..ac98445a108 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, StructType, TimestampNTZType, TimestampType} import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} -import org.apache.comet.DataTypeSupport.isComplexType +import org.apache.comet.DataTypeSupport.{containsDuplicateFieldNames, isComplexType} import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde, CometTimeZone, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported} import org.apache.comet.serde.ExprOuterClass.Expr import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProtoInternal, serializeDataType} @@ -36,6 +36,9 @@ object CometCast with CometTypeShim with CodegenDispatchFallback { + private[comet] val duplicateStructFieldNamesReason: String = + "Cast target contains a struct with duplicate field names" + // Shared with CometNativeCastSuite so the asserted reason cannot drift from production. private[comet] val negativeScaleDecimalToStringReason: String = "Negative-scale decimal requires spark.sql.legacy.allowNegativeScaleOfDecimal=true" @@ -196,6 +199,13 @@ object CometCast return unsupported(fromType, toType) } + // Check the entire target before the NullType and identity shortcuts below. Arrow Java can + // collapse duplicate struct children even when the source reaches the target through a null, + // array, map, or nested struct path. + if (containsDuplicateFieldNames(toType)) { + return Unsupported(Some(duplicateStructFieldNamesReason)) + } + // Spark 4.0's collation metadata rides on `StringType`, but `serializeDataType` maps every // `StringType` to the same proto id, so a non-default collation is dropped on the way into // the native plan with no warning. Reject the cast outright rather than relying on the diff --git a/spark/src/main/scala/org/apache/comet/serde/literals.scala b/spark/src/main/scala/org/apache/comet/serde/literals.scala index ef6fa1fb466..d299e984e4d 100644 --- a/spark/src/main/scala/org/apache/comet/serde/literals.scala +++ b/spark/src/main/scala/org/apache/comet/serde/literals.scala @@ -44,7 +44,11 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit override def getSupportLevel(expr: Literal): SupportLevel = { val serializable = expr.dataType match { // A null literal carries only its type on the wire, so any type serializeDataType handles. - case _ if expr.value == null => supportedDataType(expr.dataType, allowComplex = true) + case _ if expr.value == null => + supportedDataType( + expr.dataType, + allowComplex = true, + allowDuplicateStructFieldNames = false) case ArrayType(elementType, _) => listLiteralElementSupported(elementType) case dt => supportedDataType(dt) } diff --git a/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql new file mode 100644 index 00000000000..e7040ac7f05 --- /dev/null +++ b/spark/src/test/resources/sql-tests/expressions/cast/cast_duplicate_struct_fields.sql @@ -0,0 +1,56 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +statement +CREATE TABLE test_cast_duplicate_struct_fields(s STRUCT) USING parquet + +statement +INSERT INTO test_cast_duplicate_struct_fields VALUES + (named_struct('p', 1, 'q', 11)), + (named_struct('p', 2, 'q', 12)) + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(s AS STRUCT) FROM test_cast_duplicate_struct_fields + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(array(s) AS ARRAY>) +FROM test_cast_duplicate_struct_fields + +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(map('key', s) AS MAP>) +FROM test_cast_duplicate_struct_fields + +-- A null source must not bypass duplicate-name validation in a nested struct cast. +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT CAST(named_struct('a', NULL, 'b', s.p) + AS STRUCT, b: INT>) +FROM test_cast_duplicate_struct_fields + +-- The ARRAY shortcut must not bypass validation of the target element type. +query expect_fallback(Cast target contains a struct with duplicate field names) +SELECT s.p, CAST(e AS STRUCT) +FROM test_cast_duplicate_struct_fields +LATERAL VIEW explode(array(NULL, NULL)) v AS e + +-- Constant folding produces a typed null literal, whose own serde must reject the target shape. +query expect_fallback(Unsupported data type) +SELECT s, CAST(NULL AS STRUCT) +FROM test_cast_duplicate_struct_fields + +-- Distinct field names remain on the native path. +query +SELECT CAST(s AS STRUCT) FROM test_cast_duplicate_struct_fields diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 4db950e73c9..c87578c3521 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -2032,6 +2032,32 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("cast to a struct with duplicate field names is unsupported") { + val sourceType = StructType(Seq(StructField("p", IntegerType), StructField("q", IntegerType))) + val duplicateTarget = + StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType))) + val uniqueTarget = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + val reason = CometCast.duplicateStructFieldNamesReason + + Seq( + sourceType -> duplicateTarget, + DataTypes.NullType -> duplicateTarget, + ArrayType(DataTypes.NullType) -> ArrayType(duplicateTarget), + ArrayType(sourceType) -> ArrayType(duplicateTarget), + MapType(StringType, sourceType) -> MapType(StringType, duplicateTarget)).foreach { + case (fromType, toType) => + assert( + CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY) == + Unsupported(Some(reason))) + } + assert( + CometCast + .isSupported(sourceType, uniqueTarget, None, CometEvalMode.LEGACY) + .isInstanceOf[Compatible]) + + } + test("cast between decimals with different precision and scale") { val rowData = Seq( Row(BigDecimal("12345.6789")), diff --git a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala index fd2f3aca61d..44c8691ed47 100644 --- a/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala @@ -42,6 +42,18 @@ import org.apache.comet.shims.CometTypeShim */ class CometLiteralSuite extends CometTestBase with CometTypeShim { + test("a null literal with duplicate struct field names is unsupported") { + val duplicateType = + StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType))) + val distinctType = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + + assert( + CometLiteral.getSupportLevel(Literal.create(null, duplicateType)).isInstanceOf[Unsupported]) + assert( + CometLiteral.getSupportLevel(Literal.create(null, distinctType)).isInstanceOf[Compatible]) + } + /** * Element types the two sides are compared over. Both halves of the list matter. A type the * encoder handles today that drops out of the gate silently costs a native projection; a type