From 474cd12f1e10f4aa77fc20cb3675a31ea003a7f2 Mon Sep 17 00:00:00 2001 From: Artem Kupchinskiy Date: Sat, 14 Nov 2026 20:10:07 +0000 Subject: [PATCH] fix: cast a non-INT `GetArrayItem` index to INT like Spark (#7552) * fix: cast a non-INT `GetArrayItem` index to INT like Spark * test: literal array with a non-INT column index in GetArrayItem (cherry picked from commit c40c8c2822d2859a6377b8c7a616b211272d9226) --- .../scala/org/apache/comet/serde/arrays.scala | 10 +++++- .../expressions/array/get_array_item.sql | 34 +++++++++++++++++++ .../expressions/array/get_array_item_ansi.sql | 30 ++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala index 07edb45b741..3df09ddf874 100644 --- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala +++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.types._ import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.DataTypeSupport.{deepNullable, isComplexType} +import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.serde.QueryPlanSerde._ import org.apache.comet.shims.{CometExprShim, CometTypeShim} @@ -840,7 +841,14 @@ object CometGetArrayItem extends CometExpressionSerde[GetArrayItem] { inputs: Seq[Attribute], binding: Boolean): Option[ExprOuterClass.Expr] = { val childExpr = exprToProtoInternal(expr.child, inputs, binding) - val ordinalExpr = exprToProtoInternal(expr.ordinal, inputs, binding) + // Spark narrows any integral ordinal to an int; ListExtract reads an Int32 ordinal. + val ordinalExpr = exprToProtoInternal(expr.ordinal, inputs, binding).flatMap { ordinal => + if (expr.ordinal.dataType == IntegerType) { + Some(ordinal) + } else { + CometCast.castToProto(expr, None, IntegerType, ordinal, CometEvalMode.LEGACY) + } + } if (childExpr.isDefined && ordinalExpr.isDefined) { val listExtractBuilder = ExprOuterClass.ListExtract diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql index 1fcedd6e85c..72108d72433 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item.sql @@ -45,3 +45,37 @@ SELECT nested[idx] FROM test_get_array_item_complex query SELECT structs[idx] FROM test_get_array_item_complex + +-- Non-INT ordinals: Spark keeps the low 32 bits of a BIGINT +statement +CREATE TABLE test_get_array_item_ordinals(arr array, b tinyint, s smallint, l bigint) USING parquet + +statement +INSERT INTO test_get_array_item_ordinals VALUES + (array(10, 20, 30), 0, 1, 2), + (array(10, 20, 30), 2, 0, 1), + (array(10, 20, 30), 1, 2, 4294967296), + (array(10, 20, 30), 0, 1, 4294967298), + (array(10, 20, 30), 0, 0, -4294967296), + (array(10, 20, 30), 0, 0, -4294967294), + (array(10, 20, 30), -1, -1, 2147483648), + (array(10, 20, 30), 3, 3, 4294967295), + (array(10, 20, 30), 0, 0, 9223372036854775807), + (array(10, 20, 30), 0, 0, -9223372036854775808), + (NULL, 0, 0, 0), + (array(10, 20), NULL, NULL, NULL) + +query +SELECT arr[b], arr[s], arr[l] FROM test_get_array_item_ordinals + +query +SELECT arr[CAST(1 AS BIGINT)], arr[l - 1], arr[floor(l / 2)] FROM test_get_array_item_ordinals WHERE l BETWEEN 0 AND 2 + +-- A literal array with a non-INT column index +query +SELECT array(10, 20, 30)[b], array(10, 20, 30)[s], array(10, 20, 30)[l] FROM test_get_array_item_ordinals + +-- A median: floor(DOUBLE) returns a BIGINT ordinal +query +SELECT g, sort_array(collect_list(v))[floor(count(*) / 2)] FROM + (SELECT CAST(id % 3 AS INT) g, CAST(id AS INT) v FROM range(10)) GROUP BY g ORDER BY g diff --git a/spark/src/test/resources/sql-tests/expressions/array/get_array_item_ansi.sql b/spark/src/test/resources/sql-tests/expressions/array/get_array_item_ansi.sql index c21d27426f6..424ad5b00f8 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/get_array_item_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/get_array_item_ansi.sql @@ -61,3 +61,33 @@ SELECT arr[-1] FROM ansi_array_oob -- literal array with a negative column index query expect_error([INVALID_ARRAY_INDEX]) SELECT array(1, 2, 3)[negative_idx] FROM ansi_array_oob + +-- ============================================================================ +-- Non-INT ordinals +-- ============================================================================ + +statement +CREATE TABLE ansi_array_ordinals(arr array, b tinyint, s smallint, l bigint, l_oob bigint) USING parquet + +statement +INSERT INTO ansi_array_ordinals VALUES (array(1, 2, 3), 2, 1, 4294967296, 4294967295) + +query +SELECT arr[b], arr[s], arr[l], arr[l + 2], arr[-l] FROM ansi_array_ordinals + +-- A literal array with a non-INT column index +query +SELECT array(1, 2, 3)[b], array(1, 2, 3)[s], array(1, 2, 3)[l] FROM ansi_array_ordinals + +query expect_error(The index -1 is out of bounds) +SELECT array(1, 2, 3)[l_oob] FROM ansi_array_ordinals + +query expect_error_class(INVALID_ARRAY_INDEX) +SELECT arr[l_oob] FROM ansi_array_ordinals + +-- The error reports the narrowed index +query expect_error(The index -1 is out of bounds) +SELECT arr[l_oob] FROM ansi_array_ordinals + +query expect_error_class(INVALID_ARRAY_INDEX) +SELECT arr[CAST(b - 3 AS TINYINT)] FROM ansi_array_ordinals