From 18a78d1795ce1f553b8f39f53da8ee6ef5dee30d Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Tue, 3 Nov 2026 00:23:34 -0800 Subject: [PATCH 01/11] fix: fall back TopK whose later sort key can fail under ANSI Spark's TakeOrderedAndProject ordering evaluates a sort key after the first only for rows whose earlier keys tie, while Comet's native TopK evaluates every key over the whole batch. Under ANSI mode a later key such as `b + 1` could raise ARITHMETIC_OVERFLOW on a row Spark never evaluates it for. Mark such a TopK Incompatible so it runs on Spark. Part of https://github.com/apache/datafusion-comet/issues/7246 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../latest/compatibility/operators.md | 8 ++ .../apache/comet/expressions/CometCast.scala | 24 +++++- .../apache/comet/serde/QueryPlanSerde.scala | 32 ++++++++ .../CometTakeOrderedAndProjectExec.scala | 11 ++- .../operators/topk_later_key_ansi.sql | 76 +++++++++++++++++++ 5 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql diff --git a/docs/source/user-guide/latest/compatibility/operators.md b/docs/source/user-guide/latest/compatibility/operators.md index 43f118942d1..3cd85d313c6 100644 --- a/docs/source/user-guide/latest/compatibility/operators.md +++ b/docs/source/user-guide/latest/compatibility/operators.md @@ -94,6 +94,14 @@ TopK, and ranking window order keys run natively under both default and nondefau searches boundaries over a nullable nested key still runs on Spark ([#6477](https://github.com/apache/datafusion-comet/issues/6477)). +Spark's TopK (`TakeOrderedAndProject`, an `ORDER BY` with a `LIMIT`) evaluates a sort key after +the first only for the rows whose earlier keys tie. Comet evaluates every key over the whole batch, +so under `spark.sql.ansi.enabled=true` a later key such as `b + 1` could raise +`ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. A TopK whose later sort key can +fail therefore runs on Spark, unless +`spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true` +([#7246](https://github.com/apache/datafusion-comet/issues/7246)). + ## Hash Aggregation Ungrouped `AVG` and `TRY_AVG` on `DECIMAL(p, s)` fall back to Spark when the intermediate sum diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index f57b6074a81..0b2d2904c79 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -24,7 +24,7 @@ import scala.util.Try import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, NumericType, StructType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, NumericType, StringType, StructType, TimestampNTZType, TimestampType} import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} import org.apache.comet.DataTypeSupport.{containsDuplicateFieldNames, isComplexType} @@ -572,6 +572,28 @@ object CometCast private[comet] def canProduceNullMapKey(cast: Cast): Boolean = evalMode(cast) == CometEvalMode.TRY && mayNullMapKey(cast.child.dataType, cast.dataType) + /** + * Whether `cast` cannot raise an error for any value. A TRY cast returns NULL for a value it + * cannot convert, and so does a LEGACY cast between the atomic types listed here. An ANSI cast + * cannot fail only where the conversion is always exact, such as a widening one. + */ + private[comet] def cannotFail(cast: Cast): Boolean = { + val from = cast.child.dataType + val to = cast.dataType + evalMode(cast) match { + case CometEvalMode.TRY => true + case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true + case _ => keyCastCannotFail(from, to) + } + } + + private def isAtomicCastType(dataType: DataType): Boolean = dataType match { + case _: NumericType | _: StringType | DataTypes.BooleanType | DataTypes.BinaryType | + DataTypes.DateType | DataTypes.TimestampType | _: TimestampNTZType => + true + case _ => false + } + private def mayNullMapKey(fromType: DataType, toType: DataType): Boolean = (fromType, toType) match { case (MapType(fromKey, fromValue, _), MapType(toKey, toValue, _)) => diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8b89b0742c..36944f2e256 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke} import org.apache.spark.sql.catalyst.expressions.xml.{XPathBoolean, XPathDouble, XPathFloat, XPathInt, XPathList, XPathLong, XPathShort, XPathString} +import org.apache.spark.sql.catalyst.optimizer.NormalizeNaNAndZero import org.apache.spark.sql.comet.DecimalPrecision import org.apache.spark.sql.execution.{ScalarSubquery, SparkPlan} import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils @@ -1394,6 +1395,37 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { } } + /** + * Whether `expr` cannot raise an error for any row, so native execution may evaluate it for + * rows that Spark skips. Like the native `cannot_fail`, it is conservative: columns, literals, + * and comparisons, boolean logic, null checks, conditionals, numeric arithmetic, negation and + * absolute value outside ANSI mode and casts that cannot fail over them. Anything else is + * treated as able to fail. + */ + def cannotFail(expr: Expression): Boolean = { + def numeric: Boolean = expr.dataType.isInstanceOf[NumericType] + def nonAnsiNumeric(evalMode: EvalMode.Value): Boolean = evalMode != EvalMode.ANSI && numeric + val nodeCannotFail = expr match { + case _: Attribute | _: BoundReference | _: Literal => true + case _: BinaryComparison | _: And | _: Or | _: Not | _: IsNull | _: IsNotNull => true + case _: If | _: CaseWhen | _: Coalesce => true + case _: KnownFloatingPointNormalized | _: NormalizeNaNAndZero => true + case c: Cast => CometCast.cannotFail(c) + case _: UnaryPositive => true + case u: UnaryMinus => !u.failOnError && numeric + case a: Abs => !a.failOnError && numeric + case a: Add => nonAnsiNumeric(a.evalMode) + case s: Subtract => nonAnsiNumeric(s.evalMode) + case m: Multiply => nonAnsiNumeric(m.evalMode) + case d: Divide => nonAnsiNumeric(d.evalMode) + case d: IntegralDivide => nonAnsiNumeric(d.evalMode) + case r: Remainder => nonAnsiNumeric(r.evalMode) + case p: Pmod => nonAnsiNumeric(p.evalMode) + case _ => false + } + nodeCannotFail && expr.children.forall(cannotFail) + } + /** * Has native execution evaluate the arguments of `scalarFunc`, the serialized `expr`, the way * Spark evaluates those of a null-intolerant `BinaryExpression` or `TernaryExpression`: each diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala index f9597ca046b..1ff1af4ffb3 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala @@ -33,8 +33,8 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.isCometShuffleEnabled -import org.apache.comet.serde.{Compatible, OperatorOuterClass, SupportLevel, Unsupported} -import org.apache.comet.serde.QueryPlanSerde.{exprToProto, supportedSortType, withSharedSubexpressions} +import org.apache.comet.serde.{Compatible, Incompatible, OperatorOuterClass, SupportLevel, Unsupported} +import org.apache.comet.serde.QueryPlanSerde.{cannotFail, exprToProto, supportedSortType, withSharedSubexpressions} import org.apache.comet.serde.operator.CometSink object CometTakeOrderedAndProjectExec extends CometSink[TakeOrderedAndProjectExec] { @@ -63,6 +63,13 @@ object CometTakeOrderedAndProjectExec extends CometSink[TakeOrderedAndProjectExe if (!supportedSortType(op, op.sortOrder)) { return Unsupported(Some("unsupported data type in sort order")) } + // Spark's ordering evaluates a sort key only for the rows whose earlier keys tie, but native + // execution evaluates every key over the whole batch, so a later key that can fail may raise + // on a row that Spark skips (#7246). + op.sortOrder.drop(1).find(s => !cannotFail(s.child)).foreach { s => + return Incompatible( + Some(s"sort key ${s.child.sql} can fail on rows that Spark's ordering skips")) + } Compatible() } diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql new file mode 100644 index 00000000000..8288fad7ecb --- /dev/null +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -0,0 +1,76 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- Spark's top-K ordering evaluates a later sort key only for rows whose earlier keys tie, so a +-- later key that fails under ANSI mode does not fail on a row Spark skips. Comet evaluates every +-- key over the whole batch, so a top-K whose later key can fail runs on Spark (#7246). +-- Each table is one batch. + +-- Config: spark.sql.ansi.enabled=true +-- Config: spark.sql.adaptive.enabled=false + +statement +CREATE TABLE topk(a int, b int) USING parquet + +statement +INSERT INTO topk SELECT /*+ REPARTITION(1) */ * FROM VALUES + (1, 2147483647), (2, 5), (3, 2147483647), (4, 7), (5, 9) AS t(a, b) + +-- No two rows tie on a, so Spark never evaluates b + 1 +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a, b + 1 LIMIT 2 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a DESC, CAST(CAST(b AS STRING) AS INT) * 2 LIMIT 2 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a, abs(b) LIMIT 2 + +-- Later keys that cannot fail stay native +query +SELECT a, b FROM topk ORDER BY a, b LIMIT 2 + +query +SELECT a, b FROM topk ORDER BY a, b IS NULL, CAST(b AS BIGINT), try_add(b, 1) LIMIT 2 + +-- The first key is evaluated for every row in Spark too +query +SELECT a, b FROM topk ORDER BY a + 1, b LIMIT 2 + +statement +CREATE TABLE topk_ties(a int, b int) USING parquet + +statement +INSERT INTO topk_ties SELECT /*+ REPARTITION(1) */ * FROM VALUES + (1, 2147483647), (1, 5), (2, 7) AS t(a, b) + +-- Rows tie on a, so Spark evaluates b + 1 and fails +query expect_error(ARITHMETIC_OVERFLOW) +SELECT a, b FROM topk_ties ORDER BY a, b + 1 LIMIT 2 + +-- Without ANSI mode the key cannot fail and stays native +statement +SET spark.sql.ansi.enabled=false + +query +SELECT a, b FROM topk ORDER BY a, b + 1 LIMIT 2 + +query +SELECT a, b FROM topk_ties ORDER BY a, b + 1 LIMIT 2 + +query +SELECT a, b FROM topk_ties ORDER BY a, -b, abs(b) LIMIT 2 From 307fb82f6ac05b383e86bffc95f71ad7cfe7166c Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Tue, 3 Nov 2026 01:31:20 -0800 Subject: [PATCH 02/11] fix: keep TopK native for string keys that cannot fail Whitelist Substring, Upper, Lower and Length, which never raise, so a TopK such as TPC-DS q79 stays native. Allow the incompatible TopK in array_null_short_circuit.sql, whose later key holds an ANSI cast that can fail, to keep testing that TopK keys skip a shared cast. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../scala/org/apache/comet/serde/QueryPlanSerde.scala | 7 ++++--- .../expressions/array/array_null_short_circuit.sql | 9 ++++++++- .../sql-tests/operators/topk_later_key_ansi.sql | 4 ++++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 36944f2e256..c8fa0da3b41 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1398,9 +1398,9 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { /** * Whether `expr` cannot raise an error for any row, so native execution may evaluate it for * rows that Spark skips. Like the native `cannot_fail`, it is conservative: columns, literals, - * and comparisons, boolean logic, null checks, conditionals, numeric arithmetic, negation and - * absolute value outside ANSI mode and casts that cannot fail over them. Anything else is - * treated as able to fail. + * and comparisons, boolean logic, null checks, conditionals, substrings, case conversion and + * length, numeric arithmetic, negation and absolute value outside ANSI mode and casts that + * cannot fail over them. Anything else is treated as able to fail. */ def cannotFail(expr: Expression): Boolean = { def numeric: Boolean = expr.dataType.isInstanceOf[NumericType] @@ -1410,6 +1410,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { case _: BinaryComparison | _: And | _: Or | _: Not | _: IsNull | _: IsNotNull => true case _: If | _: CaseWhen | _: Coalesce => true case _: KnownFloatingPointNormalized | _: NormalizeNaNAndZero => true + case _: Substring | _: Upper | _: Lower | _: Length => true case c: Cast => CometCast.cannotFail(c) case _: UnaryPositive => true case u: UnaryMinus => !u.failOnError && numeric diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql b/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql index 559574fc38f..9eec890d230 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql @@ -108,11 +108,18 @@ SELECT array_contains(ai, CAST(s AS INT)) FROM (SELECT * FROM array_null_short_circuit ORDER BY i LIMIT 10) -- A TakeOrderedAndProject compares rows one sort key at a time and evaluates its projection apart --- from the keys, so a cast in two sort keys, or in a sort key and the projection, is skipped too +-- from the keys, so a cast in two sort keys, or in a sort key and the projection, is skipped too. +-- A later sort key that can fail runs the top-K on Spark (#7246), so allow it here. +statement +SET spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true + query expect_native(array_contains, array_position) SELECT i FROM (SELECT * FROM array_null_short_circuit ORDER BY array_contains(ai, CAST(s AS INT)), array_position(ai, CAST(s AS INT)), i LIMIT 10) +statement +RESET spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible + query expect_native(array_contains, array_position) SELECT array_contains(ai, CAST(s AS INT)) FROM (SELECT * FROM array_null_short_circuit ORDER BY array_position(ai, CAST(s AS INT)), i LIMIT 10) diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 8288fad7ecb..ca8317ea174 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -47,6 +47,10 @@ SELECT a, b FROM topk ORDER BY a, b LIMIT 2 query SELECT a, b FROM topk ORDER BY a, b IS NULL, CAST(b AS BIGINT), try_add(b, 1) LIMIT 2 +query +SELECT a, b FROM topk ORDER BY a, substr(CAST(b AS STRING), 1, 3), upper(CAST(b AS STRING)), + length(lower(CAST(b AS STRING))) LIMIT 2 + -- The first key is evaluated for every row in Spark too query SELECT a, b FROM topk ORDER BY a + 1, b LIMIT 2 From fa0d55349b8073391f85be93b9142a9842ad08ce Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Tue, 3 Nov 2026 15:27:22 -0800 Subject: [PATCH 03/11] fix: widen TopK later-key whitelist and address review Extract laterSortKeyThatCanFail helper, add string, datetime, math and hash functions that cannot fail in Spark, treat string-to-binary casts as infallible, extend tests and reword the Sort compatibility notes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../latest/compatibility/operators.md | 10 +++- .../apache/comet/expressions/CometCast.scala | 4 +- .../apache/comet/serde/QueryPlanSerde.scala | 43 +++++++++++++-- .../CometTakeOrderedAndProjectExec.scala | 7 +-- .../operators/topk_later_key_ansi.sql | 55 +++++++++++++++++++ 5 files changed, 104 insertions(+), 15 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/operators.md b/docs/source/user-guide/latest/compatibility/operators.md index 3cd85d313c6..18b8aec6c9b 100644 --- a/docs/source/user-guide/latest/compatibility/operators.md +++ b/docs/source/user-guide/latest/compatibility/operators.md @@ -97,10 +97,14 @@ searches boundaries over a nullable nested key still runs on Spark Spark's TopK (`TakeOrderedAndProject`, an `ORDER BY` with a `LIMIT`) evaluates a sort key after the first only for the rows whose earlier keys tie. Comet evaluates every key over the whole batch, so under `spark.sql.ansi.enabled=true` a later key such as `b + 1` could raise -`ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. A TopK whose later sort key can -fail therefore runs on Spark, unless +`ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. A TopK therefore runs on Spark +when Comet cannot show that a later sort key never fails, unless `spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true` -([#7246](https://github.com/apache/datafusion-comet/issues/7246)). +([#7246](https://github.com/apache/datafusion-comet/issues/7246)). The check does not depend on +`spark.sql.ansi.enabled`: a later key built from an expression outside the set Comet recognizes as +unable to fail runs on Spark in either mode. A sort without a `LIMIT` that has no exchange beneath +it, such as `SORT BY`, still runs natively and can raise such an error +([#7470](https://github.com/apache/datafusion-comet/issues/7470)). ## Hash Aggregation diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 0b2d2904c79..4975fda2fb4 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -575,7 +575,8 @@ object CometCast /** * Whether `cast` cannot raise an error for any value. A TRY cast returns NULL for a value it * cannot convert, and so does a LEGACY cast between the atomic types listed here. An ANSI cast - * cannot fail only where the conversion is always exact, such as a widening one. + * cannot fail only where the conversion is always exact, such as a widening one or a string to + * its bytes. */ private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType @@ -583,6 +584,7 @@ object CometCast evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true + case _ if from.isInstanceOf[StringType] && to == DataTypes.BinaryType => true case _ => keyCastCannotFail(from, to) } } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8fa0da3b41..dc750cdd3e1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1397,10 +1397,10 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { /** * Whether `expr` cannot raise an error for any row, so native execution may evaluate it for - * rows that Spark skips. Like the native `cannot_fail`, it is conservative: columns, literals, - * and comparisons, boolean logic, null checks, conditionals, substrings, case conversion and - * length, numeric arithmetic, negation and absolute value outside ANSI mode and casts that - * cannot fail over them. Anything else is treated as able to fail. + * rows that Spark skips. It answers from Spark's semantics, independently of the native + * `cannot_fail` in `case_when.rs`, which judges the physical expressions it is given. It is + * conservative: an allowed expression whose children also cannot fail. Anything not listed is + * treated as able to fail. */ def cannotFail(expr: Expression): Boolean = { def numeric: Boolean = expr.dataType.isInstanceOf[NumericType] @@ -1408,9 +1408,25 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { val nodeCannotFail = expr match { case _: Attribute | _: BoundReference | _: Literal => true case _: BinaryComparison | _: And | _: Or | _: Not | _: IsNull | _: IsNotNull => true - case _: If | _: CaseWhen | _: Coalesce => true + case _: In | _: InSet | _: Greatest | _: Least => true + case _: If | _: CaseWhen | _: Coalesce | _: IsNaN | _: NaNvl => true case _: KnownFloatingPointNormalized | _: NormalizeNaNAndZero => true - case _: Substring | _: Upper | _: Lower | _: Length => true + case _: Substring | _: Upper | _: Lower | _: Length | _: OctetLength | _: BitLength => true + case _: StringTrim | _: StringTrimLeft | _: StringTrimRight => true + case _: StringReplace | _: InitCap => true + // These fail only when the result would exceed 2GB, more than a row can hold + case _: ConcatWs => true + case c: Concat => c.children.nonEmpty && c.dataType == StringType + case _: Reverse => expr.dataType == StringType + case _: Year | _: Quarter | _: Month | _: DayOfMonth | _: DayOfWeek | _: WeekDay | + _: DayOfYear | _: Hour | _: Minute | _: Second => + true + case _: Murmur3Hash | _: XxHash64 | _: Md5 | _: Sha1 | _: Sha2 | _: Crc32 => true + // Invalid input gives NaN or NULL, and a floating-point floor or ceil saturates to a long + case _: Floor | _: Ceil | _: Sqrt | _: Exp | _: Log | _: Signum => true + // Rounding an integer fails on overflow in ANSI mode; a decimal is never treated as safe + case r: Round => roundCannotFail(r.dataType, r.ansiEnabled) + case r: BRound => roundCannotFail(r.dataType, r.ansiEnabled) case c: Cast => CometCast.cannotFail(c) case _: UnaryPositive => true case u: UnaryMinus => !u.failOnError && numeric @@ -1427,6 +1443,21 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { nodeCannotFail && expr.children.forall(cannotFail) } + private def roundCannotFail(dataType: DataType, ansiEnabled: Boolean): Boolean = + dataType match { + case FloatType | DoubleType => true + case ByteType | ShortType | IntegerType | LongType => !ansiEnabled + case _ => false + } + + /** + * The first sort key after the first that can fail. Spark's ordering evaluates a sort key only + * for the rows whose earlier keys tie, but native execution evaluates every key over the whole + * batch, so such a key may raise on a row that Spark skips (#7246). + */ + def laterSortKeyThatCanFail(sortOrder: Seq[SortOrder]): Option[SortOrder] = + sortOrder.drop(1).find(s => !cannotFail(s.child)) + /** * Has native execution evaluate the arguments of `scalarFunc`, the serialized `expr`, the way * Spark evaluates those of a null-intolerant `BinaryExpression` or `TernaryExpression`: each diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala index 1ff1af4ffb3..a68b4b9acc1 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTakeOrderedAndProjectExec.scala @@ -34,7 +34,7 @@ import org.apache.spark.sql.vectorized.ColumnarBatch import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.isCometShuffleEnabled import org.apache.comet.serde.{Compatible, Incompatible, OperatorOuterClass, SupportLevel, Unsupported} -import org.apache.comet.serde.QueryPlanSerde.{cannotFail, exprToProto, supportedSortType, withSharedSubexpressions} +import org.apache.comet.serde.QueryPlanSerde.{exprToProto, laterSortKeyThatCanFail, supportedSortType, withSharedSubexpressions} import org.apache.comet.serde.operator.CometSink object CometTakeOrderedAndProjectExec extends CometSink[TakeOrderedAndProjectExec] { @@ -63,10 +63,7 @@ object CometTakeOrderedAndProjectExec extends CometSink[TakeOrderedAndProjectExe if (!supportedSortType(op, op.sortOrder)) { return Unsupported(Some("unsupported data type in sort order")) } - // Spark's ordering evaluates a sort key only for the rows whose earlier keys tie, but native - // execution evaluates every key over the whole batch, so a later key that can fail may raise - // on a row that Spark skips (#7246). - op.sortOrder.drop(1).find(s => !cannotFail(s.child)).foreach { s => + laterSortKeyThatCanFail(op.sortOrder).foreach { s => return Incompatible( Some(s"sort key ${s.child.sql} can fail on rows that Spark's ordering skips")) } diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index ca8317ea174..5060421820f 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -51,6 +51,55 @@ query SELECT a, b FROM topk ORDER BY a, substr(CAST(b AS STRING), 1, 3), upper(CAST(b AS STRING)), length(lower(CAST(b AS STRING))) LIMIT 2 +-- A later key that can fail falls back even when an allowed expression wraps it +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a, coalesce(b + 1, 0) LIMIT 2 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a, IF(a > 3, b + 1, 0) LIMIT 2 + +-- Rounding an integer overflows under ANSI mode +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk ORDER BY a, round(b, -1) LIMIT 2 + +query +SELECT a, b FROM topk ORDER BY a, try_cast(CAST(b AS STRING) AS INT) LIMIT 2 + +statement +CREATE TABLE topk_types(a int, b int, l bigint, d double, s string, dt date, ts timestamp) +USING parquet + +statement +INSERT INTO topk_types SELECT /*+ REPARTITION(1) */ * FROM VALUES + (1, 2147483647, 9223372036854775807, double('NaN'), ' x ', date'2024-01-31', + timestamp'2024-01-31 23:59:59'), + (2, NULL, NULL, -0.0D, NULL, NULL, NULL), + (3, -2147483648, -9223372036854775808, 0.0D, 'abc', date'1970-01-01', + timestamp'1970-01-01 00:00:00'), + (4, 7, 7, 1.5D, 'Spark', date'2000-02-29', timestamp'2000-02-29 12:30:45'), + (5, 0, 0, -1.0E308D, '', date'9999-12-31', timestamp'9999-12-31 23:59:59') + AS t(a, b, l, d, s, dt, ts) + +-- Later keys built from functions that cannot fail stay native +query +SELECT a, s FROM topk_types ORDER BY a, trim(s), ltrim(s), rtrim(s), replace(s, 'a', 'b'), + reverse(s), octet_length(s), bit_length(s), concat(s, 'x'), concat_ws('-', s, s) LIMIT 3 + +query +SELECT a, dt FROM topk_types ORDER BY a, year(dt), quarter(dt), month(dt), dayofmonth(dt), + dayofweek(dt), weekday(dt), dayofyear(dt), hour(ts), minute(ts), second(ts) LIMIT 3 + +query +SELECT a, d FROM topk_types ORDER BY a, floor(d), ceil(d), sqrt(d), exp(d), ln(d), signum(d), + isnan(d), nanvl(d, 0D), round(d, 1), greatest(b, 0), least(l, 0), b IN (1, 2) LIMIT 3 + +query +SELECT a, s FROM topk_types ORDER BY a, hash(b, s), xxhash64(s), md5(s), sha1(s), sha2(s, 256), + crc32(s) LIMIT 3 + +query +SELECT a, s FROM topk_types ORDER BY a, initcap(s) LIMIT 3 + -- The first key is evaluated for every row in Spark too query SELECT a, b FROM topk ORDER BY a + 1, b LIMIT 2 @@ -78,3 +127,9 @@ SELECT a, b FROM topk_ties ORDER BY a, b + 1 LIMIT 2 query SELECT a, b FROM topk_ties ORDER BY a, -b, abs(b) LIMIT 2 + +query +SELECT a, b FROM topk ORDER BY a, round(b, -1), CAST(CAST(b AS STRING) AS INT) LIMIT 2 + +query +SELECT a, b FROM topk_types ORDER BY a, b / 0, b % 0, b DIV 0, l + 1, l * 2 LIMIT 3 From f073526c7bcd57908b63aee95157301371edb636 Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Tue, 3 Nov 2026 15:55:21 -0800 Subject: [PATCH 04/11] fix: treat timestamp casts as fallible TopK later keys Native timestamp cast kernels go through chrono, which rejects years Spark accepts, and raise even for TRY casts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../scala/org/apache/comet/expressions/CometCast.scala | 8 +++++--- .../sql-tests/operators/topk_later_key_ansi.sql | 10 ++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 4975fda2fb4..cb4016ea719 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -576,17 +576,19 @@ object CometCast * Whether `cast` cannot raise an error for any value. A TRY cast returns NULL for a value it * cannot convert, and so does a LEGACY cast between the atomic types listed here. An ANSI cast * cannot fail only where the conversion is always exact, such as a widening one or a string to - * its bytes. + * its bytes. A cast to or from a timestamp is excluded in every mode: the native kernels go + * through chrono, which rejects years Spark accepts, and raise even for TRY. */ private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType - evalMode(cast) match { + def isTimestamp(t: DataType) = t == TimestampType || t == TimestampNTZType + !isTimestamp(from) && !isTimestamp(to) && (evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true case _ if from.isInstanceOf[StringType] && to == DataTypes.BinaryType => true case _ => keyCastCannotFail(from, to) - } + }) } private def isAtomicCastType(dataType: DataType): Boolean = dataType match { diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 5060421820f..468d16856d0 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -100,6 +100,16 @@ SELECT a, s FROM topk_types ORDER BY a, hash(b, s), xxhash64(s), md5(s), sha1(s) query SELECT a, s FROM topk_types ORDER BY a, initcap(s) LIMIT 3 +-- Native timestamp casts reject years that Spark accepts, even for TRY +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, ts FROM topk_types ORDER BY a, CAST(ts AS STRING) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, ts FROM topk_types ORDER BY a, try_cast(ts AS DATE) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, dt FROM topk_types ORDER BY a, try_cast(dt AS TIMESTAMP) LIMIT 3 + -- The first key is evaluated for every row in Spark too query SELECT a, b FROM topk ORDER BY a + 1, b LIMIT 2 From 12509a2f5235f55a2c33ed587ff5fcf478d5d73f Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Thu, 5 Nov 2026 17:41:39 -0800 Subject: [PATCH 05/11] Address review: allow float arithmetic and struct fields, exclude date to string Floating-point add, subtract, multiply, negation and abs cannot fail in any mode, and reading a struct field cannot fail, so a later TopK key built from them stays native. A cast from a date to a string raises natively for years beyond chrono's range in every mode, so it is treated as able to fail. The test comment about the first key now notes it holds for at least two rows (#7527). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../apache/comet/expressions/CometCast.scala | 6 +++-- .../apache/comet/serde/QueryPlanSerde.scala | 13 ++++++---- .../operators/topk_later_key_ansi.sql | 25 ++++++++++++++++++- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index cb4016ea719..7cfdda3e9ec 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -577,13 +577,15 @@ object CometCast * cannot convert, and so does a LEGACY cast between the atomic types listed here. An ANSI cast * cannot fail only where the conversion is always exact, such as a widening one or a string to * its bytes. A cast to or from a timestamp is excluded in every mode: the native kernels go - * through chrono, which rejects years Spark accepts, and raise even for TRY. + * through chrono, which rejects years Spark accepts, and raise even for TRY. So is a cast from + * a date to a string, whose native formatting raises in every mode for such a year. */ private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType def isTimestamp(t: DataType) = t == TimestampType || t == TimestampNTZType - !isTimestamp(from) && !isTimestamp(to) && (evalMode(cast) match { + val dateToString = from == DataTypes.DateType && to.isInstanceOf[StringType] + !isTimestamp(from) && !isTimestamp(to) && !dateToString && (evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true case _ if from.isInstanceOf[StringType] && to == DataTypes.BinaryType => true diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index dc750cdd3e1..552f72bbfd6 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1405,8 +1405,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { def cannotFail(expr: Expression): Boolean = { def numeric: Boolean = expr.dataType.isInstanceOf[NumericType] def nonAnsiNumeric(evalMode: EvalMode.Value): Boolean = evalMode != EvalMode.ANSI && numeric + // Floating-point arithmetic never overflows into an error, even in ANSI mode + def floating: Boolean = expr.dataType == FloatType || expr.dataType == DoubleType val nodeCannotFail = expr match { case _: Attribute | _: BoundReference | _: Literal => true + case _: GetStructField => true case _: BinaryComparison | _: And | _: Or | _: Not | _: IsNull | _: IsNotNull => true case _: In | _: InSet | _: Greatest | _: Least => true case _: If | _: CaseWhen | _: Coalesce | _: IsNaN | _: NaNvl => true @@ -1429,11 +1432,11 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { case r: BRound => roundCannotFail(r.dataType, r.ansiEnabled) case c: Cast => CometCast.cannotFail(c) case _: UnaryPositive => true - case u: UnaryMinus => !u.failOnError && numeric - case a: Abs => !a.failOnError && numeric - case a: Add => nonAnsiNumeric(a.evalMode) - case s: Subtract => nonAnsiNumeric(s.evalMode) - case m: Multiply => nonAnsiNumeric(m.evalMode) + case u: UnaryMinus => floating || (!u.failOnError && numeric) + case a: Abs => floating || (!a.failOnError && numeric) + case a: Add => floating || nonAnsiNumeric(a.evalMode) + case s: Subtract => floating || nonAnsiNumeric(s.evalMode) + case m: Multiply => floating || nonAnsiNumeric(m.evalMode) case d: Divide => nonAnsiNumeric(d.evalMode) case d: IntegralDivide => nonAnsiNumeric(d.evalMode) case r: Remainder => nonAnsiNumeric(r.evalMode) diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 468d16856d0..8ffcc12a4d7 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -110,7 +110,27 @@ SELECT a, ts FROM topk_types ORDER BY a, try_cast(ts AS DATE) LIMIT 3 query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, dt FROM topk_types ORDER BY a, try_cast(dt AS TIMESTAMP) LIMIT 3 --- The first key is evaluated for every row in Spark too +-- Native date formatting rejects years that Spark accepts, in every mode +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, dt FROM topk_types ORDER BY a, CAST(dt AS STRING) LIMIT 3 + +-- Floating-point arithmetic cannot fail even under ANSI mode +query +SELECT a, d FROM topk_types ORDER BY a, d * 1.1D, d + 1, d - 1, -d, abs(d) LIMIT 3 + +statement +CREATE TABLE topk_struct(a int, s struct) USING parquet + +statement +INSERT INTO topk_struct SELECT /*+ REPARTITION(1) */ a, IF(x IS NULL, NULL, named_struct('x', x, + 'y', y)) FROM VALUES (1, 2147483647, 'p'), (1, 5, 'q'), (2, NULL, NULL), (3, -1, '') AS t(a, x, y) + +-- Reading a struct field cannot fail +query +SELECT a, s FROM topk_struct ORDER BY a, s.x, s.y LIMIT 3 + +-- The first key is evaluated for every row in Spark too when the input has at least two rows +-- (https://github.com/apache/datafusion-comet/issues/7527) query SELECT a, b FROM topk ORDER BY a + 1, b LIMIT 2 @@ -143,3 +163,6 @@ SELECT a, b FROM topk ORDER BY a, round(b, -1), CAST(CAST(b AS STRING) AS INT) L query SELECT a, b FROM topk_types ORDER BY a, b / 0, b % 0, b DIV 0, l + 1, l * 2 LIMIT 3 + +query +SELECT a, s FROM topk_struct ORDER BY a, s.x, s.y LIMIT 3 From c6f1dcece1d4a7f223d5b5f8c6478674457c209b Mon Sep 17 00:00:00 2001 From: Tien-Lung Fu <0050211@gmail.com> Date: Thu, 5 Nov 2026 21:09:45 -0800 Subject: [PATCH 06/11] Treat nested timestamp and date-to-string casts as able to fail in TopK A cast over an array, struct or map applies the same native kernels to each element, field, key and value, so a nested timestamp, or a nested date cast to a string, raises for years Spark accepts, even under TRY. Check them at any depth and add TopK regressions for each nested shape. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../apache/comet/expressions/CometCast.scala | 19 +++++-- .../operators/topk_later_key_ansi.sql | 51 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 7cfdda3e9ec..dfe3ce2adbe 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -578,14 +578,18 @@ object CometCast * cannot fail only where the conversion is always exact, such as a widening one or a string to * its bytes. A cast to or from a timestamp is excluded in every mode: the native kernels go * through chrono, which rejects years Spark accepts, and raise even for TRY. So is a cast from - * a date to a string, whose native formatting raises in every mode for such a year. + * a date to a string, whose native formatting raises in every mode for such a year. Both are + * checked at any depth, since a nested cast applies the same kernels to each element, field, + * key and value (https://github.com/apache/datafusion-comet/issues/7471). */ private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType def isTimestamp(t: DataType) = t == TimestampType || t == TimestampNTZType - val dateToString = from == DataTypes.DateType && to.isInstanceOf[StringType] - !isTimestamp(from) && !isTimestamp(to) && !dateToString && (evalMode(cast) match { + val hasTimestamp = existsNested(from)(isTimestamp) || existsNested(to)(isTimestamp) + val dateToString = existsNested(from)(_ == DataTypes.DateType) && + existsNested(to)(_.isInstanceOf[StringType]) + !hasTimestamp && !dateToString && (evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true case _ if from.isInstanceOf[StringType] && to == DataTypes.BinaryType => true @@ -593,6 +597,15 @@ object CometCast }) } + private def existsNested(dataType: DataType)(p: DataType => Boolean): Boolean = + p(dataType) || (dataType match { + case ArrayType(elementType, _) => existsNested(elementType)(p) + case MapType(keyType, valueType, _) => + existsNested(keyType)(p) || existsNested(valueType)(p) + case StructType(fields) => fields.exists(f => existsNested(f.dataType)(p)) + case _ => false + }) + private def isAtomicCastType(dataType: DataType): Boolean = dataType match { case _: NumericType | _: StringType | DataTypes.BooleanType | DataTypes.BinaryType | DataTypes.DateType | DataTypes.TimestampType | _: TimestampNTZType => diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 8ffcc12a4d7..33e34807e52 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -22,6 +22,7 @@ -- Config: spark.sql.ansi.enabled=true -- Config: spark.sql.adaptive.enabled=false +-- Config: spark.sql.parquet.outputTimestampType=TIMESTAMP_MICROS statement CREATE TABLE topk(a int, b int) USING parquet @@ -114,6 +115,56 @@ SELECT a, dt FROM topk_types ORDER BY a, try_cast(dt AS TIMESTAMP) LIMIT 3 query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, dt FROM topk_types ORDER BY a, CAST(dt AS STRING) LIMIT 3 +statement +CREATE TABLE topk_nested(a int, tsa array, dta array, sts struct, + std struct, md map, mt map, ast array>) +USING parquet + +statement +INSERT INTO topk_nested SELECT /*+ REPARTITION(1) */ a, array(ts), array(dt), named_struct('f', ts), + named_struct('f', dt), map(1, dt), map(ts, 1), array(named_struct('f', ts)) FROM VALUES + (1, timestamp_micros(9000000000000000000), date_from_unix_date(100000000)), + (1, timestamp'2024-01-31 23:59:59', date'2024-01-31'), + (2, timestamp'1970-01-01 00:00:00', date'1970-01-01') AS t(a, ts, dt) + +-- The same native kernels cast each element, field, key and value of a nested type +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(tsa AS STRING) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(dta AS STRING) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(tsa AS array) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(dta AS array) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(sts AS STRING) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(std AS STRING) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(std AS struct) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(sts AS struct) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(std AS struct) LIMIT 3 + +-- A map cannot be a sort key, so test it under IS NULL, which cannot fail by itself +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(md AS map) IS NULL LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(mt AS map) IS NULL LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_nested ORDER BY a, try_cast(ast AS STRING) LIMIT 3 + -- Floating-point arithmetic cannot fail even under ANSI mode query SELECT a, d FROM topk_types ORDER BY a, d * 1.1D, d + 1, d - 1, -d, abs(d) LIMIT 3 From 37be546ae3b04a281169322e4d7045171dfda9cc Mon Sep 17 00:00:00 2001 From: tlfua <0050211@gmail.com> Date: Mon, 9 Nov 2026 16:40:35 -0800 Subject: [PATCH 07/11] Reuse SupportLevel.containsType in CometCast.cannotFail Replace the private existsNested walker with SupportLevel.containsType, and drop the timestamp arms of isAtomicCastType, which cannotFail never reaches because it rejects any cast with a timestamp at any depth first. No change in behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../apache/comet/expressions/CometCast.scala | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index dfe3ce2adbe..47350bcb348 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -24,7 +24,7 @@ import scala.util.Try import org.apache.spark.SPARK_VERSION import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, NumericType, StringType, StructType, TimestampNTZType, TimestampType} +import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DateType, DecimalType, MapType, NullType, NumericType, StringType, StructType, TimestampNTZType, TimestampType} import org.apache.comet.CometSparkSessionExtensions.{isSpark40Plus, withFallbackReason} import org.apache.comet.DataTypeSupport.{containsDuplicateFieldNames, isComplexType} @@ -585,10 +585,10 @@ object CometCast private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType - def isTimestamp(t: DataType) = t == TimestampType || t == TimestampNTZType - val hasTimestamp = existsNested(from)(isTimestamp) || existsNested(to)(isTimestamp) - val dateToString = existsNested(from)(_ == DataTypes.DateType) && - existsNested(to)(_.isInstanceOf[StringType]) + val hasTimestamp = Seq(from, to).exists( + SupportLevel.containsType(_, classOf[TimestampType], classOf[TimestampNTZType])) + val dateToString = SupportLevel.containsType(from, classOf[DateType]) && + SupportLevel.containsType(to, classOf[StringType]) !hasTimestamp && !dateToString && (evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true @@ -597,18 +597,10 @@ object CometCast }) } - private def existsNested(dataType: DataType)(p: DataType => Boolean): Boolean = - p(dataType) || (dataType match { - case ArrayType(elementType, _) => existsNested(elementType)(p) - case MapType(keyType, valueType, _) => - existsNested(keyType)(p) || existsNested(valueType)(p) - case StructType(fields) => fields.exists(f => existsNested(f.dataType)(p)) - case _ => false - }) - + // Timestamps are left out because `cannotFail` rejects them before reaching this check. private def isAtomicCastType(dataType: DataType): Boolean = dataType match { case _: NumericType | _: StringType | DataTypes.BooleanType | DataTypes.BinaryType | - DataTypes.DateType | DataTypes.TimestampType | _: TimestampNTZType => + DataTypes.DateType => true case _ => false } From 201edc6cccfb38f056f57aafade386813bb4dcf2 Mon Sep 17 00:00:00 2001 From: tlfua <0050211@gmail.com> Date: Mon, 16 Nov 2026 21:35:42 -0800 Subject: [PATCH 08/11] fix: only fall back a TopK under ANSI off for keys that raise in any mode With spark.sql.ansi.enabled=false most invalid inputs give NULL instead of an error, so falling back every TopK with an unlisted later key lost native execution that Comet 1.2 had. Keep the allow-list under ANSI, and outside it fall back only for a later key that can raise in any mode: element_at on an array with an index that may be 0, rlike with a non-constant pattern, a user-defined function, raise_error, or a cast to or from a timestamp or from a date to a string (CometCast.failsInAnyMode). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../latest/compatibility/operators.md | 15 ++++++---- .../apache/comet/expressions/CometCast.scala | 20 +++++++++---- .../apache/comet/serde/QueryPlanSerde.scala | 28 +++++++++++++++++-- .../operators/topk_later_key_ansi.sql | 18 ++++++++++++ 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/operators.md b/docs/source/user-guide/latest/compatibility/operators.md index 6c0726257e2..e5035d1da9c 100644 --- a/docs/source/user-guide/latest/compatibility/operators.md +++ b/docs/source/user-guide/latest/compatibility/operators.md @@ -97,13 +97,16 @@ searches boundaries over a nullable nested key still runs on Spark Spark's TopK (`TakeOrderedAndProject`, an `ORDER BY` with a `LIMIT`) evaluates a sort key after the first only for the rows whose earlier keys tie. Comet evaluates every key over the whole batch, so under `spark.sql.ansi.enabled=true` a later key such as `b + 1` could raise -`ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. A TopK therefore runs on Spark -when Comet cannot show that a later sort key never fails, unless +`ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. Under ANSI a TopK therefore runs +on Spark when Comet cannot show that a later sort key never fails, unless `spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true` -([#7246](https://github.com/apache/datafusion-comet/issues/7246)). The check does not depend on -`spark.sql.ansi.enabled`: a later key built from an expression outside the set Comet recognizes as -unable to fail runs on Spark in either mode. A sort without a `LIMIT` that has no exchange beneath -it, such as `SORT BY`, still runs natively and can raise such an error +([#7246](https://github.com/apache/datafusion-comet/issues/7246)). With +`spark.sql.ansi.enabled=false`, where most invalid inputs give NULL instead of an error, a TopK +runs on Spark only when a later key uses an expression that can raise in any mode, such as +`element_at` on an array with an index that may be 0, `rlike` with a pattern that is not a +constant, a user-defined function, or a cast to or from a timestamp or from a date to a string. A +sort without a `LIMIT` that has no exchange beneath it, such as `SORT BY`, still runs natively +and can raise such an error ([#7470](https://github.com/apache/datafusion-comet/issues/7470)). ## Hash Aggregation diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index c2724811f0d..4a2a2c2ea7e 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -592,11 +592,7 @@ object CometCast private[comet] def cannotFail(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType - val hasTimestamp = Seq(from, to).exists( - SupportLevel.containsType(_, classOf[TimestampType], classOf[TimestampNTZType])) - val dateToString = SupportLevel.containsType(from, classOf[DateType]) && - SupportLevel.containsType(to, classOf[StringType]) - !hasTimestamp && !dateToString && (evalMode(cast) match { + !failsInAnyMode(cast) && (evalMode(cast) match { case CometEvalMode.TRY => true case CometEvalMode.LEGACY if isAtomicCastType(from) && isAtomicCastType(to) => true case _ if from.isInstanceOf[StringType] && to == DataTypes.BinaryType => true @@ -604,6 +600,20 @@ object CometCast }) } + /** + * Whether `cast` can raise whatever its eval mode: a cast to or from a timestamp, or from a + * date to a string, at any depth, as `cannotFail` describes. + */ + private[comet] def failsInAnyMode(cast: Cast): Boolean = { + val from = cast.child.dataType + val to = cast.dataType + val hasTimestamp = Seq(from, to).exists( + SupportLevel.containsType(_, classOf[TimestampType], classOf[TimestampNTZType])) + val dateToString = SupportLevel.containsType(from, classOf[DateType]) && + SupportLevel.containsType(to, classOf[StringType]) + hasTimestamp || dateToString + } + // Timestamps are left out because `cannotFail` rejects them before reaching this check. private def isAtomicCastType(dataType: DataType): Boolean = dataType match { case _: NumericType | _: StringType | DataTypes.BooleanType | DataTypes.BinaryType | diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 552f72bbfd6..6e6ac7109d0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1456,10 +1456,32 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { /** * The first sort key after the first that can fail. Spark's ordering evaluates a sort key only * for the rows whose earlier keys tie, but native execution evaluates every key over the whole - * batch, so such a key may raise on a row that Spark skips (#7246). + * batch, so such a key may raise on a row that Spark skips (#7246). Under ANSI any key that + * `cannotFail` does not accept counts; otherwise only one that `raisesInAnyMode` flags. */ - def laterSortKeyThatCanFail(sortOrder: Seq[SortOrder]): Option[SortOrder] = - sortOrder.drop(1).find(s => !cannotFail(s.child)) + def laterSortKeyThatCanFail(sortOrder: Seq[SortOrder]): Option[SortOrder] = { + val canFail: Expression => Boolean = + if (SQLConf.get.ansiEnabled) e => !cannotFail(e) else raisesInAnyMode + sortOrder.drop(1).find(s => canFail(s.child)) + } + + /** + * With ANSI off, whether `expr` can still raise on a row that Spark's ordering skips. Comet 1.2 + * ran every top-K natively, so this only adds safety over that release even though the list is + * best-effort (https://github.com/apache/datafusion-comet/issues/7585). + */ + private def raisesInAnyMode(expr: Expression): Boolean = expr.find { + // assert_true is replaced by If(_, null, RaiseError) before planning + case _: RaiseError | _: UserDefinedExpression => true + // element_at(array, 0) raises INVALID_INDEX_OF_ZERO in every mode + case e: ElementAt => + e.left.dataType.isInstanceOf[ArrayType] && !(e.right.foldable && e.right.eval() != 0) + // an invalid pattern raises in every mode + case r: RLike => !r.right.foldable + // the native timestamp and date to string kernels raise even for TRY (#7471) + case c: Cast => CometCast.failsInAnyMode(c) + case _ => false + }.isDefined /** * Has native execution evaluate the arguments of `scalarFunc`, the serialized `expr`, the way diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 33e34807e52..0ae807776e9 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -217,3 +217,21 @@ SELECT a, b FROM topk_types ORDER BY a, b / 0, b % 0, b DIV 0, l + 1, l * 2 LIMI query SELECT a, s FROM topk_struct ORDER BY a, s.x, s.y LIMIT 3 + +-- Without ANSI mode a later key fails over to Spark only when it can raise in any mode +query +SELECT a, dt FROM topk_types ORDER BY a, date_add(dt, 1) LIMIT 3 + +query +SELECT a, s FROM topk_types ORDER BY a, lpad(s, 5) LIMIT 3 + +statement +CREATE TABLE topk_arrays(a int, arr array) USING parquet + +statement +INSERT INTO topk_arrays SELECT /*+ REPARTITION(1) */ * FROM VALUES + (1, array(1, 2)), (2, array(3)), (3, NULL), (4, array()) AS t(a, arr) + +-- element_at(array, 0) raises INVALID_INDEX_OF_ZERO in every mode +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_arrays ORDER BY a, element_at(arr, 0) LIMIT 3 From ade66196f102207257bd53fa3bae3e6134cf207e Mon Sep 17 00:00:00 2001 From: tlfua <0050211@gmail.com> Date: Tue, 17 Nov 2026 09:59:49 -0800 Subject: [PATCH 09/11] Narrow the ANSI-off TopK check per review: drop timestamp casts, add views, slice, array_insert and sequence - With ANSI off, a cast to or from a timestamp no longer makes a TopK fall back. Spark does not raise there; the Comet kernel failure on extreme years also hits projections and is tracked in https://github.com/apache/datafusion-comet/issues/7471. - A later key that still carries ANSI eval mode, such as one from a view created with ANSI on, falls back even when the session has ANSI off. - slice with a start of 0 or a negative length, array_insert at position 0, and sequence with non-constant bounds raise in every mode, so they fall back with ANSI off. - Tests for timestamp-cast keys staying in Comet, element_at(arr, 1), the new array functions, a view, and a ScalaUDF later key. - operators.md: use "runs in Comet" (style guide N1) and say allowIncompatible applies in either mode. Replace "native" wording in comments with "in Comet" or "Rust". Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../latest/compatibility/operators.md | 19 +++-- .../apache/comet/expressions/CometCast.scala | 6 +- .../apache/comet/serde/QueryPlanSerde.scala | 80 ++++++++++++++----- .../array/array_null_short_circuit.sql | 3 +- .../operators/topk_later_key_ansi.sql | 62 +++++++++++--- .../org/apache/comet/CometCodegenSuite.scala | 14 ++++ 6 files changed, 143 insertions(+), 41 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/operators.md b/docs/source/user-guide/latest/compatibility/operators.md index e5035d1da9c..1f68be99dc9 100644 --- a/docs/source/user-guide/latest/compatibility/operators.md +++ b/docs/source/user-guide/latest/compatibility/operators.md @@ -98,15 +98,20 @@ Spark's TopK (`TakeOrderedAndProject`, an `ORDER BY` with a `LIMIT`) evaluates a the first only for the rows whose earlier keys tie. Comet evaluates every key over the whole batch, so under `spark.sql.ansi.enabled=true` a later key such as `b + 1` could raise `ARITHMETIC_OVERFLOW` on a row that Spark never evaluates it for. Under ANSI a TopK therefore runs -on Spark when Comet cannot show that a later sort key never fails, unless -`spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true` +on Spark when Comet cannot show that a later sort key never fails ([#7246](https://github.com/apache/datafusion-comet/issues/7246)). With `spark.sql.ansi.enabled=false`, where most invalid inputs give NULL instead of an error, a TopK -runs on Spark only when a later key uses an expression that can raise in any mode, such as -`element_at` on an array with an index that may be 0, `rlike` with a pattern that is not a -constant, a user-defined function, or a cast to or from a timestamp or from a date to a string. A -sort without a `LIMIT` that has no exchange beneath it, such as `SORT BY`, still runs natively -and can raise such an error +runs on Spark only when a later key uses an expression that Spark can raise for in any mode, such +as `element_at` on an array with an index that may be 0, `slice` with a start that may be 0 or a +length that may be negative, `array_insert` with a position that may be 0, `sequence` with bounds +that are not constants, `rlike` with a pattern that is not a constant, or a user-defined function. +It also runs on Spark when a later key still evaluates in ANSI mode, such as `b + 1` in the +ordering of a view created with ANSI on. In either mode, +`spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true` runs such a TopK in +Comet. With ANSI off, a later key that casts to or from a timestamp runs in Comet, and like any +other expression it can raise for a year beyond what Comet's timestamp casts support +([#7471](https://github.com/apache/datafusion-comet/issues/7471)). A sort without a `LIMIT` that +has no exchange beneath it, such as `SORT BY`, still runs in Comet and can raise such an error ([#7470](https://github.com/apache/datafusion-comet/issues/7470)). ## Hash Aggregation diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index 4a2a2c2ea7e..ca24f6cb653 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -583,9 +583,9 @@ object CometCast * Whether `cast` cannot raise an error for any value. A TRY cast returns NULL for a value it * cannot convert, and so does a LEGACY cast between the atomic types listed here. An ANSI cast * cannot fail only where the conversion is always exact, such as a widening one or a string to - * its bytes. A cast to or from a timestamp is excluded in every mode: the native kernels go + * its bytes. A cast to or from a timestamp is excluded in every mode: the Rust kernels go * through chrono, which rejects years Spark accepts, and raise even for TRY. So is a cast from - * a date to a string, whose native formatting raises in every mode for such a year. Both are + * a date to a string, whose Rust formatting raises in every mode for such a year. Both are * checked at any depth, since a nested cast applies the same kernels to each element, field, * key and value (https://github.com/apache/datafusion-comet/issues/7471). */ @@ -604,7 +604,7 @@ object CometCast * Whether `cast` can raise whatever its eval mode: a cast to or from a timestamp, or from a * date to a string, at any depth, as `cannotFail` describes. */ - private[comet] def failsInAnyMode(cast: Cast): Boolean = { + private def failsInAnyMode(cast: Cast): Boolean = { val from = cast.child.dataType val to = cast.dataType val hasTimestamp = Seq(from, to).exists( diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 6e6ac7109d0..44665695ac9 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1396,8 +1396,8 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { } /** - * Whether `expr` cannot raise an error for any row, so native execution may evaluate it for - * rows that Spark skips. It answers from Spark's semantics, independently of the native + * Whether `expr` cannot raise an error for any row, so Comet may evaluate it for rows that + * Spark skips. It answers from Spark's semantics, independently of the Rust function * `cannot_fail` in `case_when.rs`, which judges the physical expressions it is given. It is * conservative: an allowed expression whose children also cannot fail. Anything not listed is * treated as able to fail. @@ -1455,33 +1455,71 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { /** * The first sort key after the first that can fail. Spark's ordering evaluates a sort key only - * for the rows whose earlier keys tie, but native execution evaluates every key over the whole - * batch, so such a key may raise on a row that Spark skips (#7246). Under ANSI any key that - * `cannotFail` does not accept counts; otherwise only one that `raisesInAnyMode` flags. + * for the rows whose earlier keys tie, but Comet evaluates every key over the whole batch, so + * such a key may raise on a row that Spark skips + * (https://github.com/apache/datafusion-comet/issues/7246). Under ANSI any key that + * `cannotFail` does not accept counts. Otherwise a key counts when `raisesInAnyMode` flags it, + * or when it holds a node still in ANSI mode that `cannotFail` does not accept, such as `b + 1` + * in the ordering of a view created with ANSI on. */ def laterSortKeyThatCanFail(sortOrder: Seq[SortOrder]): Option[SortOrder] = { val canFail: Expression => Boolean = - if (SQLConf.get.ansiEnabled) e => !cannotFail(e) else raisesInAnyMode + if (SQLConf.get.ansiEnabled) e => !cannotFail(e) + else e => raisesInAnyMode(e) || e.exists(n => hasAnsiEvalMode(n) && !cannotFail(n)) sortOrder.drop(1).find(s => canFail(s.child)) } /** - * With ANSI off, whether `expr` can still raise on a row that Spark's ordering skips. Comet 1.2 - * ran every top-K natively, so this only adds safety over that release even though the list is - * best-effort (https://github.com/apache/datafusion-comet/issues/7585). + * Whether `expr` was resolved in ANSI mode, whatever the session's setting is now. A view keeps + * the configs it was created with, so its expressions keep their ANSI eval mode when it is + * queried with ANSI off. Like `raisesInAnyMode`, it is best-effort: it covers arithmetic, casts + * and the expressions listed here (https://github.com/apache/datafusion-comet/issues/7585). */ - private def raisesInAnyMode(expr: Expression): Boolean = expr.find { - // assert_true is replaced by If(_, null, RaiseError) before planning - case _: RaiseError | _: UserDefinedExpression => true - // element_at(array, 0) raises INVALID_INDEX_OF_ZERO in every mode - case e: ElementAt => - e.left.dataType.isInstanceOf[ArrayType] && !(e.right.foldable && e.right.eval() != 0) - // an invalid pattern raises in every mode - case r: RLike => !r.right.foldable - // the native timestamp and date to string kernels raise even for TRY (#7471) - case c: Cast => CometCast.failsInAnyMode(c) - case _ => false - }.isDefined + private def hasAnsiEvalMode(expr: Expression): Boolean = expr match { + case u: UnaryMinus => u.failOnError + case a: Abs => a.failOnError + case r: Round => r.ansiEnabled + case r: BRound => r.ansiEnabled + case e: ElementAt => e.failOnError + case g: GetArrayItem => g.failOnError + case e: Elt => e.failOnError + // arithmetic and casts carry an `EvalMode` + case _ => expr.productIterator.contains(EvalMode.ANSI) + } + + /** + * With ANSI off, whether `expr` can still raise on a row that Spark's ordering skips, because + * Spark raises for it in every mode. Comet 1.2 ran every TopK in Comet, so this only adds + * safety over that release even though the list is best-effort + * (https://github.com/apache/datafusion-comet/issues/7585). A Rust kernel that raises where + * Spark does not, such as a timestamp cast of a year beyond chrono's range, raises in a + * projection too and is left to its own issue + * (https://github.com/apache/datafusion-comet/issues/7471). + */ + private def raisesInAnyMode(expr: Expression): Boolean = { + def constantWhere(e: Expression)(ok: Int => Boolean): Boolean = + e.foldable && (e.eval() match { + case null => true + case v: Int => ok(v) + case _ => false + }) + expr.find { + // assert_true is replaced by If(_, null, RaiseError) before planning + case _: RaiseError | _: UserDefinedExpression => true + // element_at(array, 0) raises INVALID_INDEX_OF_ZERO in every mode + case e: ElementAt => + e.left.dataType.isInstanceOf[ArrayType] && !(e.right.foldable && e.right.eval() != 0) + // an invalid pattern raises in every mode + case r: RLike => !r.right.foldable + // slice raises for a start of 0 or a negative length in every mode + case s: Slice => !(constantWhere(s.start)(_ != 0) && constantWhere(s.length)(_ >= 0)) + // array_insert raises INVALID_INDEX_OF_ZERO in every mode + case a: ArrayInsert => !constantWhere(a.posExpr)(_ != 0) + // sequence raises for bounds that do not match the step in every mode + case s: Sequence => !s.children.forall(_.foldable) + case _ => false + }.isDefined + } /** * Has native execution evaluate the arguments of `scalarFunc`, the serialized `expr`, the way diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql b/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql index 9eec890d230..5e64fcdf15f 100644 --- a/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql +++ b/spark/src/test/resources/sql-tests/expressions/array/array_null_short_circuit.sql @@ -109,7 +109,8 @@ FROM (SELECT * FROM array_null_short_circuit ORDER BY i LIMIT 10) -- A TakeOrderedAndProject compares rows one sort key at a time and evaluates its projection apart -- from the keys, so a cast in two sort keys, or in a sort key and the projection, is skipped too. --- A later sort key that can fail runs the top-K on Spark (#7246), so allow it here. +-- A later sort key that can fail runs the TopK on Spark +-- (https://github.com/apache/datafusion-comet/issues/7246), so allow it here. statement SET spark.comet.operator.TakeOrderedAndProjectExec.allowIncompatible=true diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 0ae807776e9..9d309e97f03 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -15,9 +15,10 @@ -- specific language governing permissions and limitations -- under the License. --- Spark's top-K ordering evaluates a later sort key only for rows whose earlier keys tie, so a --- later key that fails under ANSI mode does not fail on a row Spark skips. Comet evaluates every --- key over the whole batch, so a top-K whose later key can fail runs on Spark (#7246). +-- Spark's TopK ordering evaluates a later sort key only for rows whose earlier keys tie, so a +-- later key that can fail does not fail on a row Spark skips. Comet evaluates every key over the +-- whole batch, so a TopK whose later key can fail runs on Spark +-- (https://github.com/apache/datafusion-comet/issues/7246). -- Each table is one batch. -- Config: spark.sql.ansi.enabled=true @@ -41,7 +42,7 @@ SELECT a, b FROM topk ORDER BY a DESC, CAST(CAST(b AS STRING) AS INT) * 2 LIMIT query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, b FROM topk ORDER BY a, abs(b) LIMIT 2 --- Later keys that cannot fail stay native +-- Later keys that cannot fail stay in Comet query SELECT a, b FROM topk ORDER BY a, b LIMIT 2 @@ -81,7 +82,7 @@ INSERT INTO topk_types SELECT /*+ REPARTITION(1) */ * FROM VALUES (5, 0, 0, -1.0E308D, '', date'9999-12-31', timestamp'9999-12-31 23:59:59') AS t(a, b, l, d, s, dt, ts) --- Later keys built from functions that cannot fail stay native +-- Later keys built from functions that cannot fail stay in Comet query SELECT a, s FROM topk_types ORDER BY a, trim(s), ltrim(s), rtrim(s), replace(s, 'a', 'b'), reverse(s), octet_length(s), bit_length(s), concat(s, 'x'), concat_ws('-', s, s) LIMIT 3 @@ -101,7 +102,7 @@ SELECT a, s FROM topk_types ORDER BY a, hash(b, s), xxhash64(s), md5(s), sha1(s) query SELECT a, s FROM topk_types ORDER BY a, initcap(s) LIMIT 3 --- Native timestamp casts reject years that Spark accepts, even for TRY +-- Comet's Rust timestamp casts reject years that Spark accepts, even for TRY query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, ts FROM topk_types ORDER BY a, CAST(ts AS STRING) LIMIT 3 @@ -111,7 +112,7 @@ SELECT a, ts FROM topk_types ORDER BY a, try_cast(ts AS DATE) LIMIT 3 query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, dt FROM topk_types ORDER BY a, try_cast(dt AS TIMESTAMP) LIMIT 3 --- Native date formatting rejects years that Spark accepts, in every mode +-- Comet's Rust date formatting rejects years that Spark accepts, in every mode query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a, dt FROM topk_types ORDER BY a, CAST(dt AS STRING) LIMIT 3 @@ -127,7 +128,7 @@ INSERT INTO topk_nested SELECT /*+ REPARTITION(1) */ a, array(ts), array(dt), na (1, timestamp'2024-01-31 23:59:59', date'2024-01-31'), (2, timestamp'1970-01-01 00:00:00', date'1970-01-01') AS t(a, ts, dt) --- The same native kernels cast each element, field, key and value of a nested type +-- The same Rust kernels cast each element, field, key and value of a nested type query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a FROM topk_nested ORDER BY a, try_cast(tsa AS STRING) LIMIT 3 @@ -196,7 +197,7 @@ INSERT INTO topk_ties SELECT /*+ REPARTITION(1) */ * FROM VALUES query expect_error(ARITHMETIC_OVERFLOW) SELECT a, b FROM topk_ties ORDER BY a, b + 1 LIMIT 2 --- Without ANSI mode the key cannot fail and stays native +-- Without ANSI mode the key cannot fail and stays in Comet statement SET spark.sql.ansi.enabled=false @@ -225,6 +226,14 @@ SELECT a, dt FROM topk_types ORDER BY a, date_add(dt, 1) LIMIT 3 query SELECT a, s FROM topk_types ORDER BY a, lpad(s, 5) LIMIT 3 +-- Spark adds casts to and from a timestamp quietly, and with ANSI off they give NULL rather +-- than raise, so these keys stay in Comet +query +SELECT a, ts FROM topk_types ORDER BY a, date_add(ts, 1) LIMIT 3 + +query +SELECT a, ts FROM topk_types ORDER BY a, concat(s, ts) LIMIT 3 + statement CREATE TABLE topk_arrays(a int, arr array) USING parquet @@ -235,3 +244,38 @@ INSERT INTO topk_arrays SELECT /*+ REPARTITION(1) */ * FROM VALUES -- element_at(array, 0) raises INVALID_INDEX_OF_ZERO in every mode query expect_fallback(can fail on rows that Spark's ordering skips) SELECT a FROM topk_arrays ORDER BY a, element_at(arr, 0) LIMIT 3 + +-- A constant index other than 0 gives NULL when out of bounds +query +SELECT a FROM topk_arrays ORDER BY a, element_at(arr, 1) LIMIT 3 + +-- slice raises for a start of 0, array_insert for a position of 0, and sequence for bounds that +-- do not match the step, in every mode. No two rows tie on a, so Spark never evaluates them. +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_arrays ORDER BY a, size(slice(arr, a - 1, 1)) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_arrays ORDER BY a, size(array_insert(arr, a - 1, 0)) LIMIT 3 + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_arrays ORDER BY a, size(sequence(a, 1, 1)) LIMIT 3 + +query +SELECT a FROM topk_arrays ORDER BY a, size(slice(arr, 1, 1)), size(array_insert(arr, 1, 0)) +LIMIT 3 + +-- A view keeps the configs it was created with, so the b + 1 in its ordering stays in ANSI mode +statement +SET spark.sql.ansi.enabled=true + +statement +CREATE TEMPORARY VIEW topk_ansi_view AS SELECT a, b FROM topk ORDER BY a, b + 1 LIMIT 2 + +statement +SET spark.sql.ansi.enabled=false + +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a, b FROM topk_ansi_view + +statement +DROP VIEW topk_ansi_view diff --git a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala index 6d7a1624a6f..5996627205b 100644 --- a/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometCodegenSuite.scala @@ -315,6 +315,20 @@ class CometCodegenSuite } } + test("a ScalaUDF later sort key runs a TopK on Spark with ANSI off") { + // Spark's TopK ordering evaluates a later sort key only for the rows whose earlier keys + // tie, and a user-defined function can raise in any mode, so Comet must not evaluate it for + // every row (https://github.com/apache/datafusion-comet/issues/7246). + spark.udf.register("strictLen", (s: String) => s.length) + withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") { + withSubjects("a", "bb", "ccc") { + checkSparkAnswerAndFallbackReason( + sql("SELECT s FROM t ORDER BY s, strictLen(s) LIMIT 2"), + "can fail on rows that Spark's ordering skips") + } + } + } + test("disabled mode bypasses the dispatcher") { // When the per-feature config is off, `CometScalaUDF.convert` returns None and the enclosing // operator falls back to Spark. The dispatcher's counters must not move. From d4f087ea5f49567e9dbaa32f2eee4dad46c2295a Mon Sep 17 00:00:00 2001 From: tlfua <0050211@gmail.com> Date: Tue, 17 Nov 2026 11:10:45 -0800 Subject: [PATCH 10/11] Read the arithmetic eval mode by name for Spark 4.1 Spark 4.1 keeps the eval mode of Add and the other arithmetic expressions inside a NumericEvalContext, so scanning the product fields for EvalMode.ANSI missed an ANSI b + 1 from a view. Match the arithmetic and cast classes and read their evalMode instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../scala/org/apache/comet/serde/QueryPlanSerde.scala | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index 44665695ac9..b38150fcb79 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -1483,7 +1483,15 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { case e: ElementAt => e.failOnError case g: GetArrayItem => g.failOnError case e: Elt => e.failOnError - // arithmetic and casts carry an `EvalMode` + // Spark 4.1 keeps the arithmetic eval mode inside a `NumericEvalContext`, so read it by name + case a: Add => a.evalMode == EvalMode.ANSI + case s: Subtract => s.evalMode == EvalMode.ANSI + case m: Multiply => m.evalMode == EvalMode.ANSI + case d: Divide => d.evalMode == EvalMode.ANSI + case d: IntegralDivide => d.evalMode == EvalMode.ANSI + case r: Remainder => r.evalMode == EvalMode.ANSI + case p: Pmod => p.evalMode == EvalMode.ANSI + case c: Cast => c.evalMode == EvalMode.ANSI case _ => expr.productIterator.contains(EvalMode.ANSI) } From 1650696bfcf8b08fd7eaec28ed967eae60bb8428 Mon Sep 17 00:00:00 2001 From: tlfua <0050211@gmail.com> Date: Wed, 18 Nov 2026 22:28:25 -0800 Subject: [PATCH 11/11] Test that a TopK with a column rlike pattern falls back Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3ec1c07f-32b3-4cb1-a6ad-09293da13ab9 --- .../sql-tests/operators/topk_later_key_ansi.sql | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql index 9d309e97f03..ac0b1084cb4 100644 --- a/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql +++ b/spark/src/test/resources/sql-tests/operators/topk_later_key_ansi.sql @@ -264,6 +264,21 @@ query SELECT a FROM topk_arrays ORDER BY a, size(slice(arr, 1, 1)), size(array_insert(arr, 1, 0)) LIMIT 3 +statement +CREATE TABLE topk_regex(a int, s string, p string) USING parquet + +statement +INSERT INTO topk_regex SELECT /*+ REPARTITION(1) */ * FROM VALUES + (1, 'abc', 'a.*'), (2, 'abc', '('), (3, 'xyz', 'x'), (4, NULL, NULL) AS t(a, s, p) + +-- rlike raises for an invalid pattern in every mode. No two rows tie on a, so Spark never +-- evaluates the pattern '(' on row 2. +query expect_fallback(can fail on rows that Spark's ordering skips) +SELECT a FROM topk_regex ORDER BY a, s RLIKE p LIMIT 3 + +query +SELECT a FROM topk_regex ORDER BY a, s RLIKE '^a' LIMIT 3 + -- A view keeps the configs it was created with, so the b + 1 in its ordering stays in ANSI mode statement SET spark.sql.ansi.enabled=true