Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions spark/src/main/scala/org/apache/comet/DataTypeSupport.scala
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,17 @@ object DataTypeSupport {
def hasDuplicateFieldNames(fields: Array[StructField]): Boolean =
fields.map(_.name).distinct.length != fields.length

/** True when any struct nested in `dt` has byte-identical sibling field names. */
def containsDuplicateFieldNames(dt: DataType): Boolean = dt match {
case StructType(fields) =>
hasDuplicateFieldNames(fields) || fields.exists(f =>
containsDuplicateFieldNames(f.dataType))
case ArrayType(elementType, _) => containsDuplicateFieldNames(elementType)
case MapType(keyType, valueType, _) =>
containsDuplicateFieldNames(keyType) || containsDuplicateFieldNames(valueType)
case _ => false
}

/**
* True when two sibling struct fields anywhere in `dt` fold to one name under
* `toLowerCase(Locale.ROOT)`, the fold Spark's Parquet reader matches requested fields with
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, StructType, TimestampNTZType, TimestampType}

import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason}
import org.apache.comet.DataTypeSupport.isComplexType
import org.apache.comet.DataTypeSupport.{containsDuplicateFieldNames, isComplexType}
import org.apache.comet.serde.{CodegenDispatchFallback, CometExpressionSerde, CometTimeZone, Compatible, ExprOuterClass, Incompatible, SupportLevel, Unsupported}
import org.apache.comet.serde.ExprOuterClass.Expr
import org.apache.comet.serde.QueryPlanSerde.{evalModeToProto, exprToProtoInternal, serializeDataType}
Expand All @@ -36,6 +36,9 @@ object CometCast
with CometTypeShim
with CodegenDispatchFallback {

private[comet] val duplicateStructFieldNamesReason: String =
"Cast target contains a struct with duplicate field names"

// Shared with CometNativeCastSuite so the asserted reason cannot drift from production.
private[comet] val negativeScaleDecimalToStringReason: String =
"Negative-scale decimal requires spark.sql.legacy.allowNegativeScaleOfDecimal=true"
Expand Down Expand Up @@ -196,6 +199,13 @@ object CometCast
return unsupported(fromType, toType)
}

// Check the entire target before the NullType and identity shortcuts below. Arrow Java can
// collapse duplicate struct children even when the source reaches the target through a null,
// array, map, or nested struct path.
if (containsDuplicateFieldNames(toType)) {
return Unsupported(Some(duplicateStructFieldNamesReason))
}

// Spark 4.0's collation metadata rides on `StringType`, but `serializeDataType` maps every
// `StringType` to the same proto id, so a non-default collation is dropped on the way into
// the native plan with no warning. Reject the cast outright rather than relying on the
Expand Down
6 changes: 5 additions & 1 deletion spark/src/main/scala/org/apache/comet/serde/literals.scala
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,11 @@ object CometLiteral extends CometExpressionSerde[Literal] with CometTypeShim wit
override def getSupportLevel(expr: Literal): SupportLevel = {
val serializable = expr.dataType match {
// A null literal carries only its type on the wire, so any type serializeDataType handles.
case _ if expr.value == null => supportedDataType(expr.dataType, allowComplex = true)
case _ if expr.value == null =>
supportedDataType(
expr.dataType,
allowComplex = true,
allowDuplicateStructFieldNames = false)
case ArrayType(elementType, _) => listLiteralElementSupported(elementType)
case dt => supportedDataType(dt)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
-- Licensed to the Apache Software Foundation (ASF) under one
-- or more contributor license agreements. See the NOTICE file
-- distributed with this work for additional information
-- regarding copyright ownership. The ASF licenses this file
-- to you under the Apache License, Version 2.0 (the
-- "License"); you may not use this file except in compliance
-- with the License. You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing,
-- software distributed under the License is distributed on an
-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-- KIND, either express or implied. See the License for the
-- specific language governing permissions and limitations
-- under the License.

statement
CREATE TABLE test_cast_duplicate_struct_fields(s STRUCT<p: INT, q: INT>) USING parquet

statement
INSERT INTO test_cast_duplicate_struct_fields VALUES
(named_struct('p', 1, 'q', 11)),
(named_struct('p', 2, 'q', 12))

query expect_fallback(Cast target contains a struct with duplicate field names)
SELECT CAST(s AS STRUCT<x: INT, x: INT>) FROM test_cast_duplicate_struct_fields

query expect_fallback(Cast target contains a struct with duplicate field names)
SELECT CAST(array(s) AS ARRAY<STRUCT<x: INT, x: INT>>)
FROM test_cast_duplicate_struct_fields

query expect_fallback(Cast target contains a struct with duplicate field names)
SELECT CAST(map('key', s) AS MAP<STRING, STRUCT<x: INT, x: INT>>)
FROM test_cast_duplicate_struct_fields

-- A null source must not bypass duplicate-name validation in a nested struct cast.
query expect_fallback(Cast target contains a struct with duplicate field names)
SELECT CAST(named_struct('a', NULL, 'b', s.p)
AS STRUCT<a: STRUCT<x: INT, x: INT>, b: INT>)
FROM test_cast_duplicate_struct_fields

-- The ARRAY<NULL> shortcut must not bypass validation of the target element type.
query expect_fallback(Cast target contains a struct with duplicate field names)
SELECT s.p, CAST(e AS STRUCT<x: INT, x: INT>)
FROM test_cast_duplicate_struct_fields
LATERAL VIEW explode(array(NULL, NULL)) v AS e

-- Constant folding produces a typed null literal, whose own serde must reject the target shape.
query expect_fallback(Unsupported data type)
SELECT s, CAST(NULL AS STRUCT<x: INT, x: INT>)
FROM test_cast_duplicate_struct_fields

-- Distinct field names remain on the native path.
query
SELECT CAST(s AS STRUCT<x: INT, y: INT>) FROM test_cast_duplicate_struct_fields
26 changes: 26 additions & 0 deletions spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2032,6 +2032,32 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper {
}
}

test("cast to a struct with duplicate field names is unsupported") {
val sourceType = StructType(Seq(StructField("p", IntegerType), StructField("q", IntegerType)))
val duplicateTarget =
StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType)))
val uniqueTarget =
StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType)))
val reason = CometCast.duplicateStructFieldNamesReason

Seq(
sourceType -> duplicateTarget,
DataTypes.NullType -> duplicateTarget,
ArrayType(DataTypes.NullType) -> ArrayType(duplicateTarget),
ArrayType(sourceType) -> ArrayType(duplicateTarget),
MapType(StringType, sourceType) -> MapType(StringType, duplicateTarget)).foreach {
case (fromType, toType) =>
assert(
CometCast.isSupported(fromType, toType, None, CometEvalMode.LEGACY) ==
Unsupported(Some(reason)))
}
assert(
CometCast
.isSupported(sourceType, uniqueTarget, None, CometEvalMode.LEGACY)
.isInstanceOf[Compatible])

}

test("cast between decimals with different precision and scale") {
val rowData = Seq(
Row(BigDecimal("12345.6789")),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ import org.apache.comet.shims.CometTypeShim
*/
class CometLiteralSuite extends CometTestBase with CometTypeShim {

test("a null literal with duplicate struct field names is unsupported") {
val duplicateType =
StructType(Seq(StructField("x", IntegerType), StructField("x", IntegerType)))
val distinctType =
StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType)))

assert(
CometLiteral.getSupportLevel(Literal.create(null, duplicateType)).isInstanceOf[Unsupported])
assert(
CometLiteral.getSupportLevel(Literal.create(null, distinctType)).isInstanceOf[Compatible])
}

/**
* Element types the two sides are compared over. Both halves of the list matter. A type the
* encoder handles today that drops out of the gate silently costs a native projection; a type
Expand Down
Loading