Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion spark/src/main/scala/org/apache/comet/serde/arrays.scala
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import org.apache.spark.sql.types._

import org.apache.comet.CometSparkSessionExtensions.withFallbackReason
import org.apache.comet.DataTypeSupport.{deepNullable, isComplexType}
import org.apache.comet.expressions.{CometCast, CometEvalMode}
import org.apache.comet.serde.QueryPlanSerde._
import org.apache.comet.shims.{CometExprShim, CometTypeShim}

Expand Down Expand Up @@ -840,7 +841,14 @@ object CometGetArrayItem extends CometExpressionSerde[GetArrayItem] {
inputs: Seq[Attribute],
binding: Boolean): Option[ExprOuterClass.Expr] = {
val childExpr = exprToProtoInternal(expr.child, inputs, binding)
val ordinalExpr = exprToProtoInternal(expr.ordinal, inputs, binding)
// Spark narrows any integral ordinal to an int; ListExtract reads an Int32 ordinal.
val ordinalExpr = exprToProtoInternal(expr.ordinal, inputs, binding).flatMap { ordinal =>
if (expr.ordinal.dataType == IntegerType) {
Some(ordinal)
} else {
CometCast.castToProto(expr, None, IntegerType, ordinal, CometEvalMode.LEGACY)
}
}

if (childExpr.isDefined && ordinalExpr.isDefined) {
val listExtractBuilder = ExprOuterClass.ListExtract
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,37 @@ SELECT nested[idx] FROM test_get_array_item_complex

query
SELECT structs[idx] FROM test_get_array_item_complex

-- Non-INT ordinals: Spark keeps the low 32 bits of a BIGINT
statement
CREATE TABLE test_get_array_item_ordinals(arr array<int>, b tinyint, s smallint, l bigint) USING parquet

statement
INSERT INTO test_get_array_item_ordinals VALUES
(array(10, 20, 30), 0, 1, 2),
(array(10, 20, 30), 2, 0, 1),
(array(10, 20, 30), 1, 2, 4294967296),
(array(10, 20, 30), 0, 1, 4294967298),
(array(10, 20, 30), 0, 0, -4294967296),
(array(10, 20, 30), 0, 0, -4294967294),
(array(10, 20, 30), -1, -1, 2147483648),
(array(10, 20, 30), 3, 3, 4294967295),
(array(10, 20, 30), 0, 0, 9223372036854775807),
(array(10, 20, 30), 0, 0, -9223372036854775808),
(NULL, 0, 0, 0),
(array(10, 20), NULL, NULL, NULL)

query
SELECT arr[b], arr[s], arr[l] FROM test_get_array_item_ordinals

query
SELECT arr[CAST(1 AS BIGINT)], arr[l - 1], arr[floor(l / 2)] FROM test_get_array_item_ordinals WHERE l BETWEEN 0 AND 2

-- A literal array with a non-INT column index
query
SELECT array(10, 20, 30)[b], array(10, 20, 30)[s], array(10, 20, 30)[l] FROM test_get_array_item_ordinals

-- A median: floor(DOUBLE) returns a BIGINT ordinal
query
SELECT g, sort_array(collect_list(v))[floor(count(*) / 2)] FROM
(SELECT CAST(id % 3 AS INT) g, CAST(id AS INT) v FROM range(10)) GROUP BY g ORDER BY g
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,33 @@ SELECT arr[-1] FROM ansi_array_oob
-- literal array with a negative column index
query expect_error([INVALID_ARRAY_INDEX])
SELECT array(1, 2, 3)[negative_idx] FROM ansi_array_oob

-- ============================================================================
-- Non-INT ordinals
-- ============================================================================

statement
CREATE TABLE ansi_array_ordinals(arr array<int>, b tinyint, s smallint, l bigint, l_oob bigint) USING parquet

statement
INSERT INTO ansi_array_ordinals VALUES (array(1, 2, 3), 2, 1, 4294967296, 4294967295)

query
SELECT arr[b], arr[s], arr[l], arr[l + 2], arr[-l] FROM ansi_array_ordinals

-- A literal array with a non-INT column index
query
SELECT array(1, 2, 3)[b], array(1, 2, 3)[s], array(1, 2, 3)[l] FROM ansi_array_ordinals

query expect_error(The index -1 is out of bounds)
SELECT array(1, 2, 3)[l_oob] FROM ansi_array_ordinals

query expect_error_class(INVALID_ARRAY_INDEX)
SELECT arr[l_oob] FROM ansi_array_ordinals

-- The error reports the narrowed index
query expect_error(The index -1 is out of bounds)
SELECT arr[l_oob] FROM ansi_array_ordinals

query expect_error_class(INVALID_ARRAY_INDEX)
SELECT arr[CAST(b - 3 AS TINYINT)] FROM ansi_array_ordinals
Loading