From 7cdd147fd1c3a8a396c579a510ed929a31c390cd Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 24 Aug 2026 11:53:32 -0700 Subject: [PATCH 01/13] fix: make wide date-to-timestamp casts safe --- native/common/src/error.rs | 24 +++ .../src/conversion_funcs/temporal.rs | 188 ++++++++++++----- .../apache/comet/SparkErrorConverter.scala | 9 +- .../codegen/CometBatchKernelCodegen.scala | 4 + .../apache/comet/expressions/CometCast.scala | 46 +++- .../apache/comet/CometNativeCastSuite.scala | 197 +++++++++++++++++- .../comet/SparkErrorConverterSuite.scala | 8 + 7 files changed, 411 insertions(+), 65 deletions(-) diff --git a/native/common/src/error.rs b/native/common/src/error.rs index 8e6385065a3..47c715e6fc9 100644 --- a/native/common/src/error.rs +++ b/native/common/src/error.rs @@ -74,6 +74,10 @@ pub enum SparkError { #[error("[ARITHMETIC_OVERFLOW] {from_type} overflow. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] ArithmeticOverflow { from_type: String }, + // Spark's checked date/timestamp conversions throw this even with ANSI disabled. + #[error("long overflow")] + LongOverflow, + #[error("[ARITHMETIC_OVERFLOW] Overflow in integral divide. Use 'try_divide' to tolerate overflow and return NULL instead. If necessary set \"spark.sql.ansi.enabled\" to \"false\" to bypass this error.")] IntegralDivideOverflow, @@ -315,6 +319,7 @@ impl SparkError { SparkError::CastOverFlow { .. } => "CastOverFlow", SparkError::CannotParseDecimal => "CannotParseDecimal", SparkError::ArithmeticOverflow { .. } => "ArithmeticOverflow", + SparkError::LongOverflow => "LongOverflow", SparkError::IntegralDivideOverflow => "IntegralDivideOverflow", SparkError::DecimalSumOverflow { .. } => "DecimalSumOverflow", SparkError::DivideByZero => "DivideByZero", @@ -647,6 +652,8 @@ impl SparkError { /// Returns the appropriate Spark exception class for this error pub fn exception_class(&self) -> &'static str { match self { + SparkError::LongOverflow => "java/lang/ArithmeticException", + // ArithmeticException SparkError::DivideByZero | SparkError::RemainderByZero @@ -756,6 +763,7 @@ impl SparkError { SparkError::RemainderByZero => Some("REMAINDER_BY_ZERO"), SparkError::IntervalDividedByZero => Some("INTERVAL_DIVIDED_BY_ZERO"), SparkError::ArithmeticOverflow { .. } => Some("ARITHMETIC_OVERFLOW"), + SparkError::LongOverflow => None, SparkError::IntegralDivideOverflow => Some("ARITHMETIC_OVERFLOW"), SparkError::DecimalSumOverflow { .. } => Some("ARITHMETIC_OVERFLOW"), SparkError::BinaryArithmeticOverflow { .. } => Some("BINARY_ARITHMETIC_OVERFLOW"), @@ -975,6 +983,22 @@ mod tests { assert!(json.contains("\"errorClass\":\"REMAINDER_BY_ZERO\"")); } + #[test] + fn test_long_overflow_json() { + let error = SparkError::LongOverflow; + let parsed: serde_json::Value = serde_json::from_str(&error.to_json()).unwrap(); + assert_eq!( + parsed, + serde_json::json!({ + "errorType": "LongOverflow", + "errorClass": "", + "params": {}, + }) + ); + assert_eq!(error.exception_class(), "java/lang/ArithmeticException"); + assert_eq!(error.to_string(), "long overflow"); + } + #[test] fn test_binary_overflow_json() { let error = SparkError::BinaryArithmeticOverflow { diff --git a/native/spark-expr/src/conversion_funcs/temporal.rs b/native/spark-expr/src/conversion_funcs/temporal.rs index 31e63644fd5..fe0961fabc5 100644 --- a/native/spark-expr/src/conversion_funcs/temporal.rs +++ b/native/spark-expr/src/conversion_funcs/temporal.rs @@ -15,15 +15,11 @@ // specific language governing permissions and limitations // under the License. -use crate::utils::resolve_local_datetime; -use crate::{EvalMode, SparkCastOptions, SparkResult}; -use arrow::array::timezone::Tz; +use crate::{EvalMode, SparkCastOptions, SparkError, SparkResult}; use arrow::array::{ArrayRef, AsArray, TimestampMicrosecondBuilder}; -use arrow::compute::{cast_with_options, CastOptions}; -use arrow::datatypes::{DataType, Date32Type, TimeUnit}; -use chrono::NaiveDate; -use datafusion::common::format::DEFAULT_CAST_OPTIONS; -use std::str::FromStr; +use arrow::datatypes::{DataType, Date32Type}; +use arrow::error::ArrowError; +use chrono::FixedOffset; use std::sync::Arc; pub(crate) fn is_df_cast_from_date_spark_compatible(to_type: &DataType) -> bool { @@ -42,50 +38,44 @@ pub(crate) fn cast_date_to_timestamp( cast_options: &SparkCastOptions, target_tz: &Option>, ) -> SparkResult { - if target_tz.is_none() { - // TIMESTAMP_NTZ ignores session TZ. Only TRY_CAST turns overflow into null. - return Ok(cast_with_options( - array_ref.as_ref(), - &DataType::Timestamp(TimeUnit::Microsecond, None), - &CastOptions { - safe: cast_options.eval_mode == EvalMode::Try, - ..DEFAULT_CAST_OPTIONS - }, - )?); - } - let date_array = array_ref.as_primitive::(); let mut builder = TimestampMicrosecondBuilder::with_capacity(date_array.len()); - // TIMESTAMP: midnight in session TZ → UTC epoch μs - let tz_str = if cast_options.timezone.is_empty() { - "UTC" + + // Date32 has a wider range than both chrono's calendar and an i64 microsecond timestamp. + // Use day arithmetic for NTZ and fixed-offset zones. Region-zone casts run through Spark's + // JVM codegen dispatcher (see CometCast.canCastFromDate), which owns their full-range rules. + let offset_seconds = if target_tz.is_none() + || cast_options.timezone.is_empty() + || cast_options.timezone == "UTC" + { + 0 } else { - cast_options.timezone.as_str() + // Reject any unexpected region-zone call safely instead of constructing a chrono date. + cast_options + .timezone + .parse::() + .map_err(|_| { + ArrowError::ParseError(format!( + "Invalid timezone \"{}\": expected a fixed offset", + cast_options.timezone + )) + })? + .local_minus_utc() }; - // safe to unwrap since we are falling back to UTC above - let tz = Tz::from_str(tz_str)?; - let epoch = NaiveDate::from_ymd_opt(1970, 1, 1).unwrap(); for date in date_array.iter() { match date { Some(d) => { - // safe to unwrap since chrono's range ( 262,143 yrs) is higher than - // number of years possible with days as i32 (~ 6 mil yrs) - // convert date in session timezone to timestamp in UTC - let naive_date = epoch + chrono::Duration::days(d as i64); - let local_midnight = naive_date.and_hms_opt(0, 0, 0).unwrap(); - // Use resolve_local_datetime to correctly handle DST transitions: - // - Single: normal case, uses the given offset - // - Ambiguous (fall back): uses the earlier/DST occurrence, matching Spark - // - None (spring forward gap at midnight, e.g. America/Sao_Paulo): uses the - // pre-transition offset to compute the correct UTC time, matching Spark's - // LocalDate.atStartOfDay(zoneId) behaviour. - let local_midnight_in_microsec = - resolve_local_datetime(&tz, local_midnight).timestamp_micros(); - builder.append_value(local_midnight_in_microsec); - } - None => { - builder.append_null(); + // Date32 days fit i64 seconds. Apply the offset before checking the microsecond + // range, since the offset can move midnight across a timestamp range boundary. + let seconds = i64::from(d) * 86_400 - i64::from(offset_seconds); + match seconds.checked_mul(1_000_000) { + Some(micros) => builder.append_value(micros), + None if cast_options.eval_mode == EvalMode::Try => builder.append_null(), + // Spark's daysToMicros uses Math.multiplyExact even in non-ANSI mode. + None => return Err(SparkError::LongOverflow), + } } + None => builder.append_null(), } } Ok(Arc::new( @@ -104,7 +94,7 @@ mod tests { use arrow::array::{Array, ArrayRef}; use arrow::datatypes::TimestampMicrosecondType; - // verifying epoch , DST change dates (US) and a null value (comprehensive tests on spark side) + // Region-zone and DST behavior is covered by Spark's codegen fallback tests. let dates: ArrayRef = Arc::new(Date32Array::from(vec![ Some(0), Some(19723), @@ -131,24 +121,23 @@ mod tests { assert_eq!(ts.value(2), dst_date); assert!(ts.is_null(3)); - // validate LA timezone (follows Daylight savings) + // Negative fixed offset let result = cast_date_to_timestamp( &dates, - &SparkCastOptions::new(EvalMode::Legacy, "America/Los_Angeles", false), + &SparkCastOptions::new(EvalMode::Legacy, "-08:00", false), &target_tz, ) .unwrap(); let ts = result.as_primitive::(); assert_eq!(ts.value(0), eight_hours_ts); assert_eq!(ts.value(1), non_dst_date + eight_hours_ts); - // should adjust for DST - assert_eq!(ts.value(2), dst_date + seven_hours_ts); + assert_eq!(ts.value(2), dst_date + eight_hours_ts); assert!(ts.is_null(3)); - // Phoenix timezone (does not follow Daylight savings) + // A different fixed offset let result = cast_date_to_timestamp( &dates, - &SparkCastOptions::new(EvalMode::Legacy, "America/Phoenix", false), + &SparkCastOptions::new(EvalMode::Legacy, "-07:00", false), &target_tz, ) .unwrap(); @@ -159,6 +148,103 @@ mod tests { assert!(ts.is_null(3)); } + #[test] + fn test_cast_wide_dates_to_timestamp() { + use arrow::array::{Array, Date32Array}; + use arrow::datatypes::TimestampMicrosecondType; + + // +262143-01-01, both valid UTC-midnight timestamp boundaries, and null. + let dates: ArrayRef = Arc::new(Date32Array::from(vec![ + Some(95_026_237), + Some(106_751_991), + Some(-106_751_991), + None, + ])); + for target_tz in [None, Some("UTC".into())] { + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(EvalMode::Legacy, "UTC", false), + &target_tz, + ) + .unwrap(); + let ts = result.as_primitive::(); + assert_eq!(ts.value(0), 8_210_266_876_800_000_000); + assert_eq!(ts.value(1), 9_223_372_022_400_000_000); + assert_eq!(ts.value(2), -9_223_372_022_400_000_000); + assert!(ts.is_null(3)); + assert_eq!(ts.timezone(), target_tz.as_deref()); + } + + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(EvalMode::Legacy, "+01:00", false), + &Some("UTC".into()), + ) + .unwrap(); + let ts = result.as_primitive::(); + assert_eq!(ts.value(0), 8_210_266_873_200_000_000); + assert_eq!(ts.value(1), 9_223_372_018_800_000_000); + assert_eq!(ts.value(2), -9_223_372_026_000_000_000); + assert!(ts.is_null(3)); + } + + #[test] + fn test_cast_date_to_timestamp_overflow() { + use arrow::array::{Array, Date32Array}; + use arrow::datatypes::TimestampMicrosecondType; + + for (timezone, days) in [ + ("UTC", 108_853_388), // +300000-06-15 + ("UTC", 106_751_992), + ("UTC", -106_751_992), + ("UTC", i32::MAX), + ("UTC", i32::MIN), + ("-18:00", 106_751_991), + ("+18:00", -106_751_991), + ] { + for target_tz in [None, Some("UTC".into())] { + // NTZ ignores the session offset: these midnight boundaries remain valid. + if target_tz.is_none() && timezone != "UTC" { + continue; + } + let dates: ArrayRef = + Arc::new(Date32Array::from(vec![Some(0), Some(days), None, Some(1)])); + for mode in [EvalMode::Legacy, EvalMode::Ansi] { + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(mode, timezone, false), + &target_tz, + ); + assert!(matches!(result, Err(SparkError::LongOverflow))); + } + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(EvalMode::Try, timezone, false), + &target_tz, + ) + .unwrap(); + let ts = result.as_primitive::(); + assert!(!ts.is_null(0)); + assert!(ts.is_null(1)); + assert!(ts.is_null(2)); + assert!(!ts.is_null(3)); + } + } + } + + #[test] + fn test_cast_date_to_timestamp_rejects_region_timezone() { + use arrow::array::Date32Array; + + let dates: ArrayRef = Arc::new(Date32Array::from(vec![95_026_237])); + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(EvalMode::Legacy, "America/Los_Angeles", false), + &Some("UTC".into()), + ); + assert!(result.is_err()); + } + #[test] fn test_cast_date_to_timestamp_ntz() { use crate::EvalMode; diff --git a/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala b/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala index a6bc21aca71..853b0fa5094 100644 --- a/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala +++ b/spark/src/main/scala/org/apache/comet/SparkErrorConverter.scala @@ -117,8 +117,13 @@ object SparkErrorConverter extends ShimSparkErrorConverter { val summary: String = errorJson.summary.getOrElse("") - // Delegate to version-specific shim - let conversion exceptions propagate - val optEx = convertErrorType(errorJson.errorType, errorClass, params, sparkContext, summary) + // Math.multiplyExact throws a plain JVM exception in every Spark version, without an + // ANSI error class or configuration advice. Delegate other errors to the version-specific shim. + val optEx = if (errorJson.errorType == "LongOverflow") { + Some(new ArithmeticException("long overflow")) + } else { + convertErrorType(errorJson.errorType, errorClass, params, sparkContext, summary) + } optEx match { case Some(exception) => // successfully converted - return the proper typed exception diff --git a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala index d762e66540e..7dd41355941 100644 --- a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala +++ b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.CometArrowAllocator +import org.apache.comet.expressions.CometCast import org.apache.comet.shims.{CometExprTraitShim, CometTypeShim} /** @@ -144,6 +145,9 @@ object CometBatchKernelCodegen extends Logging with CometExprTraitShim with Come * nested-field count on `spark.sql.codegen.maxFields`. */ def canHandle(boundExpr: Expression): Option[String] = { + if (boundExpr.exists(CometCast.requiresSparkDateTimestampTryCast)) { + return Some("TRY_CAST date-to-timestamp nullability requires Spark row execution") + } if (!isSupportedDataType(boundExpr.dataType)) { return Some(s"codegen dispatch: unsupported output type ${boundExpr.dataType}") } diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index e14fb086b12..fd4251ce90b 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -97,7 +97,9 @@ object CometCast // would only duplicate the matrix and risk drifting from it. override def getSupportLevel(cast: Cast): SupportLevel = { - if (cast.child.isInstanceOf[Literal]) { + if (requiresSparkDateTimestampTryCast(cast)) { + Unsupported(Some("TRY_CAST date-to-timestamp nullability requires Spark row execution")) + } else if (cast.child.isInstanceOf[Literal]) { // A cast whose child is a literal is folded by Spark at planning time via `cast.eval()` // (see `convert`), so the cast never executes natively and the result matches Spark by // definition. `CometLiteral` then validates the resulting literal's data type, except @@ -114,6 +116,31 @@ object CometCast } } + /** + * Spark's cast nullability inference can miss date-to-timestamp overflow. A TRY cast can + * therefore produce a null with non-nullable metadata, including map keys and struct fields. + * Keep those casts out of both native Arrow arrays and the Arrow-based codegen dispatcher. + */ + private[comet] def requiresSparkDateTimestampTryCast(expr: Expression): Boolean = expr match { + case cast: Cast if evalMode(cast) == CometEvalMode.TRY => + (!cast.nullable || isComplexType(cast.dataType)) && + containsDateTimestampCast(cast.child.dataType, cast.dataType) + case _ => false + } + + private def containsDateTimestampCast(fromType: DataType, toType: DataType): Boolean = + (fromType, toType) match { + case (DataTypes.DateType, TimestampType | TimestampNTZType) => true + case (ArrayType(from, _), ArrayType(to, _)) => containsDateTimestampCast(from, to) + case (MapType(fromKey, fromValue, _), MapType(toKey, toValue, _)) => + containsDateTimestampCast(fromKey, toKey) || containsDateTimestampCast(fromValue, toValue) + case (from: StructType, to: StructType) => + from.fields.zip(to.fields).exists { case (a, b) => + containsDateTimestampCast(a.dataType, b.dataType) + } + case _ => false + } + override def convert( cast: Cast, inputs: Seq[Attribute], @@ -301,7 +328,7 @@ object CometCast case other => other } } - case (DataTypes.DateType, toType) => canCastFromDate(toType, evalMode) + case (DataTypes.DateType, toType) => canCastFromDate(toType, timeZoneId, evalMode) case _ => unsupported(fromType, toType) } } @@ -507,10 +534,21 @@ object CometCast case _ => Unsupported(Some(s"Cast from DecimalType to $toType is not supported")) } - private def canCastFromDate(toType: DataType, evalMode: CometEvalMode.Value): SupportLevel = + private def canCastFromDate( + toType: DataType, + timeZoneId: Option[String], + evalMode: CometEvalMode.Value): SupportLevel = toType match { case DataTypes.TimestampType => - Compatible() + val timezone = timeZoneId.getOrElse("UTC") + if (timezone == "UTC" || timezone.matches("[+-][0-9]{2}:[0-9]{2}")) { + Compatible() + } else { + // DateType extends beyond chrono's calendar. Let Spark handle region-zone rules + // across the entire date range, including future DST and historical offsets. + Unsupported( + Some("Date-to-timestamp casts are native only in UTC or +/-HH:MM timezones")) + } case _: TimestampNTZType => Compatible() case DataTypes.BooleanType | DataTypes.ByteType | DataTypes.ShortType | diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 4db950e73c9..163adee0b62 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -31,6 +31,7 @@ import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, DataFrame, Row, SaveMode} import org.apache.spark.sql.catalyst.expressions.Cast import org.apache.spark.sql.catalyst.parser.ParseException +import org.apache.spark.sql.comet.CometProjectExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper import org.apache.spark.sql.functions.{col, monotonically_increasing_id} import org.apache.spark.sql.internal.SQLConf @@ -39,6 +40,7 @@ import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, ByteType, import org.apache.comet.expressions.{CometCast, CometEvalMode} import org.apache.comet.rules.CometScanTypeChecker import org.apache.comet.serde.{Compatible, Incompatible, Unsupported} +import org.apache.comet.udf.codegen.CometScalaUDFCodegen /** * Spark-parity coverage for Comet's **native** `Cast` implementation, and for the @@ -50,18 +52,21 @@ import org.apache.comet.serde.{Compatible, Incompatible, Unsupported} * decisions themselves (`Compatible` / `Incompatible` / `Unsupported`, and the reason strings) * are asserted directly against `CometCast.isSupported`. * - * Out of scope: the JVM codegen dispatch path. `CometCast` mixes in `CodegenDispatchFallback`, so - * a cast that `isSupported` rejects still runs inside the Comet operator by way of the - * Arrow-direct codegen dispatcher, evaluating Spark's own `Cast` against Arrow vectors. That path - * is covered by `CometCodegenSuite` and friends. Where this suite touches it at all, it does so - * only to pin that an `Unsupported` cast keeps the enclosing operator native rather than falling - * back to Spark; it does not attempt to cover dispatch semantics. + * `CometCast` mixes in `CodegenDispatchFallback`, so a cast that `isSupported` rejects still runs + * inside the Comet operator by way of the Arrow-direct codegen dispatcher, evaluating Spark's own + * `Cast` against Arrow vectors. That path is primarily covered by `CometCodegenSuite` and + * friends. Targeted routing regressions here also pin that an `Unsupported` cast uses the + * dispatcher, retaining Spark parity and keeping the enclosing operator native rather than + * falling back to Spark. * * Because of that split, a cast marked `Unsupported` here is not untested overall — it is tested * through the dispatcher instead. Adding a native cast implementation therefore means moving a * pair out of the `Unsupported` assertions and into the parity matrix below. */ -class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { +class CometNativeCastSuite + extends CometTestBase + with AdaptiveSparkPlanHelper + with CometCodegenAssertions { import testImplicits._ @@ -1859,6 +1864,152 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { } } + test("cast DateType to timestamps preserves wide make_date results") { + // Store the components, not dates: this both avoids Parquet rebasing and prevents Spark + // from constant-folding make_date before it reaches the native cast. + val input: Seq[(Option[Int], Int, Int)] = Seq( + (Some(262143), 1, 1), + (Some(-262144), 1, 1), + (Some(1970), 1, 1), + (Some(2024), 11, 3), + (None, 1, 1)) + withParquetTable(input, "wide_dates") { + for { + (target, tz) <- Seq( + ("TIMESTAMP_NTZ", "America/Los_Angeles"), + ("TIMESTAMP", "UTC"), + ("TIMESTAMP", "+05:30"), + ("TIMESTAMP", "-08:00")) + ansi <- Seq("false", "true") + } { + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, + SQLConf.ANSI_ENABLED.key -> ansi, + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + assertNoCodegenRan { + checkSparkAnswerAndOperator( + s"SELECT CAST(make_date(_1, _2, _3) AS $target) FROM wide_dates") + } + } + } + + // Named regions and offsets with seconds must use Spark's timezone rules instead of + // constructing chrono dates. Neither the wide positive nor negative year fits chrono. + Seq("America/Los_Angeles", "GMT", "+05:30:15").foreach { tz => + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + assertCodegenRan { + checkSparkAnswerAndOperator( + "SELECT CAST(make_date(_1, _2, _3) AS TIMESTAMP) FROM wide_dates") + } + } + } + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Los_Angeles", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false", + CometConf.COMET_STRICT_TESTING.key -> "true") { + checkSparkAnswer("SELECT CAST(make_date(_1, _2, _3) AS TIMESTAMP) FROM wide_dates") + } + } + } + + test("cast DateType to timestamps checks microsecond overflow") { + // +/-106751991 are the last whole days representable in microseconds at UTC. The next + // days overflow; a fixed offset can also push an otherwise valid boundary day out of range. + val input = withNulls(Seq(-106751992, -106751991, 0, 106751991, 106751992)) + .map(Tuple1(_)) + val wideDate = + "make_date(CASE WHEN _1 = 0 THEN 300000 WHEN _1 > 0 THEN 2024 END, 6, 15)" + withParquetTable(input, "date_boundaries") { + for { + (target, tz, overflowDays) <- Seq( + ("TIMESTAMP_NTZ", "America/Los_Angeles", Seq(-106751992, 106751992)), + ("TIMESTAMP", "UTC", Seq(-106751992, 106751992)), + ("TIMESTAMP", "+18:00", Seq(-106751991)), + ("TIMESTAMP", "-18:00", Seq(106751991))) + ansi <- Seq("false", "true") + } { + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, + SQLConf.ANSI_ENABLED.key -> ansi, + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { + val overflowQueries = overflowDays.map { day => + s"SELECT CAST(date_from_unix_date(_1) AS $target) " + + s"FROM date_boundaries WHERE _1 = $day" + } ++ (if (target == "TIMESTAMP_NTZ") { + Seq(s"SELECT CAST($wideDate AS $target) FROM date_boundaries") + } else Seq.empty) + overflowQueries.foreach { query => + assertNoCodegenRan { + val df = sql(query) + checkCometOperators(stripAQEPlan(df.queryExecution.executedPlan)) + val (sparkError, cometError) = checkSparkAnswerMaybeThrows(df) + assert(sparkError.isDefined, s"Spark should overflow for $query in $tz") + assert(cometError.isDefined, s"Comet should overflow for $query in $tz") + Seq(sparkError.get, cometError.get).foreach { error => + assert( + causeChain(error).exists { cause => + cause.getClass == classOf[ArithmeticException] && + cause.getMessage == "long overflow" + }, + s"Expected ArithmeticException: long overflow, got $error") + } + } + } + // A mixed batch must retain valid values and return NULL only for overflowing rows + // and input NULLs. TRY_CAST does this regardless of the session ANSI setting. + // Compare LTZ as epoch seconds (these date casts are second-aligned): Spark's + // java.sql.Timestamp row conversion overflows when rebasing the extreme boundaries. + val casts = Seq( + s"TRY_CAST(date_from_unix_date(_1) AS $target)", + s"TRY_CAST($wideDate AS $target)").map { expr => + if (target == "TIMESTAMP") s"CAST($expr AS BIGINT)" else expr + } + assertNoCodegenRan { + checkSparkAnswerAndOperator( + s"SELECT _1, ${casts.mkString(", ")} " + + "FROM date_boundaries ORDER BY _1") + } + } + } + } + } + + test("cast DateType to timestamps falls back for unsafe TRY_CAST nullability") { + withParquetTable(Seq(Tuple1(300000), Tuple1(2024)), "try_wide_dates") { + val date = "make_date(_1, 6, 15)" + for { + target <- Seq("TIMESTAMP", "TIMESTAMP_NTZ") + codegen <- Seq("true", "false") + } { + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC", + SQLConf.ANSI_ENABLED.key -> "true", + "spark.sql.legacy.castComplexTypesToString.enabled" -> "true", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> codegen, + CometConf.COMET_STRICT_TESTING.key -> "true") { + val expressions = Seq( + s"TRY_CAST(make_date(coalesce(_1, 300000), 6, 15) AS $target)", + s"TRY_CAST(map(1, $date) AS MAP)", + s"TRY_CAST(named_struct('d', $date) AS STRUCT)", + // The outer cast would otherwise dispatch. Its entire tree must be checked. + s"CAST(array(TRY_CAST(make_date(coalesce(_1, 300000), 6, 15) AS $target)) AS STRING)") ++ + (if (target == "TIMESTAMP") Seq(s"TRY_CAST(map($date, 1) AS MAP<$target, INT>)") + else Seq.empty) + expressions.foreach { expr => + assertNoCodegenRan { + val query = s"SELECT $expr FROM try_wide_dates" + val plan = stripAQEPlan(sql(query).queryExecution.executedPlan) + assert(!plan.exists(_.isInstanceOf[CometProjectExec])) + checkSparkAnswer(query) + } + } + } + } + } + } + // CAST from TimestampType ignore("cast TimestampType to BooleanType") { @@ -2773,7 +2924,37 @@ class CometNativeCastSuite extends CometTestBase with AdaptiveSparkPlanHelper { values.map(v => Some(v)) ++ Seq(None) } - private def castTimestampTest(input: DataFrame, toType: DataType, assertNative: Boolean) = { + private def assertNoCodegenRan(f: => Unit): Unit = { + CometScalaUDFCodegen.resetStats() + f + val stats = CometScalaUDFCodegen.stats() + assert( + stats.compileCount + stats.cacheHitCount == 0, + s"Expected a native cast, but the JVM codegen dispatcher ran: $stats") + } + + private def castFallbackTest( + input: DataFrame, + toType: DataType, + expectedMessage: String): Unit = { + withTempPath { dir => + val data = roundtripParquet(input, dir).coalesce(1) + data.createOrReplaceTempView("t") + + withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { + val df = data.withColumn("converted", col("a").cast(toType)) + df.collect() + val str = + new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) + assert(str.contains(expectedMessage)) + } + } + } + + private def castTimestampTest( + input: DataFrame, + toType: DataType, + assertNative: Boolean = false) = { withTempPath { dir => val data = roundtripParquet(input, dir).coalesce(1) data.createOrReplaceTempView("t") diff --git a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala index 3b81a76ead5..7da4779b531 100644 --- a/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala +++ b/spark/src/test/scala/org/apache/comet/SparkErrorConverterSuite.scala @@ -23,6 +23,14 @@ import org.scalatest.funsuite.AnyFunSuite class SparkErrorConverterSuite extends AnyFunSuite { + test("LongOverflow converts to a plain ArithmeticException") { + val json = """{"errorType":"LongOverflow","errorClass":"","params":{}}""" + val ex = SparkErrorConverter.convertToSparkException( + new org.apache.comet.exceptions.CometQueryExecutionException(json)) + assert(ex.getClass == classOf[ArithmeticException]) + assert(ex.getMessage == "long overflow") + } + test("CannotReadFile converts to a FAILED_READ_FILE SparkException naming the file") { val ex = SparkErrorConverter .convertErrorType( From 73f298e62a07829ba60d03545bd40a2c80efff1b Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 24 Aug 2026 13:29:50 -0700 Subject: [PATCH 02/13] fix: skip hidden child values in nested casts --- .../spark-expr/src/conversion_funcs/cast.rs | 193 +++++++++++++++++- .../apache/comet/CometNativeCastSuite.scala | 50 +++++ 2 files changed, 238 insertions(+), 5 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 26659f7489f..a1ba91e5a2b 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -50,9 +50,9 @@ use arrow::error::ArrowError; use arrow::{ array::{ cast::AsArray, types::Decimal128Type, Array, ArrayRef, Int16Array, Int32Array, Int64Array, - Int8Array, OffsetSizeTrait, + Int8Array, OffsetSizeTrait, UInt64Array, }, - compute::{cast_with_options, rescale_decimal, CastOptions}, + compute::{cast_with_options, rescale_decimal, take, CastOptions}, record_batch::RecordBatch, util::display::FormatOptions, }; @@ -250,6 +250,7 @@ pub(crate) fn cast_array( return Ok(Arc::new(array)); } + let array = prepare_nested_cast_input(array)?; let array = array_with_timezone(array, cast_options.timezone.clone(), Some(to_type))?; let eval_mode = cast_options.eval_mode; @@ -475,6 +476,36 @@ pub(crate) fn cast_array( Ok(spark_cast_postprocess(cast_result?, &from_type, to_type)) } +/// Recursive casts must not evaluate child values hidden by a null parent or outside a slice. +fn prepare_nested_cast_input(array: ArrayRef) -> DataFusionResult { + let has_unused_values = match array.data_type() { + DataType::Struct(_) => false, + DataType::List(_) => { + let list = array.as_list::(); + list.value_offsets()[0] != 0 + || list.value_offsets()[list.len()] as usize != list.values().len() + } + DataType::Map(_, _) => { + let map = array.as_map(); + map.value_offsets()[0] != 0 + || map.value_offsets()[map.len()] as usize != map.entries().len() + } + _ => return Ok(array), + }; + if array.null_count() == 0 && !has_unused_values { + return Ok(array); + } + + // Nullable identity indices preserve the rows and their validity. Arrow's take propagates + // struct nulls to children and compacts list/map entries, without introducing null map keys + // or changing field nullability. Contiguous, non-null containers need no normalization. + let indices = UInt64Array::new( + (0..array.len() as u64).collect::>().into(), + array.nulls().cloned(), + ); + Ok(take(array.as_ref(), &indices, None)?) +} + /// Determines if DataFusion supports the given cast in a way that is /// compatible with Spark fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> bool { @@ -931,10 +962,10 @@ fn cast_binary_to_string( mod tests { use super::*; use arrow::array::{ - BinaryArray, Decimal128Array, ListArray, NullArray, PrimitiveArray, StringArray, + BinaryArray, Date32Array, Decimal128Array, ListArray, NullArray, PrimitiveArray, StringArray, }; - use arrow::buffer::OffsetBuffer; - use arrow::datatypes::{Field, Fields, Int32Type, TimestampMicrosecondType}; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{Field, Fields, Int32Type, TimeUnit, TimestampMicrosecondType}; #[test] fn test_cast_to_dictionary_is_rejected() { @@ -1259,6 +1290,158 @@ mod tests { assert!(result.is_err()); } + fn nested_date_cast_inputs( + days: Vec, + nulls: Option, + ) -> Vec<(ArrayRef, DataType)> { + let offsets = OffsetBuffer::new((0..=days.len() as i32).collect::>().into()); + let dates: ArrayRef = Arc::new(Date32Array::from(days)); + let timestamp = DataType::Timestamp(TimeUnit::Microsecond, None); + let fields = Fields::from(vec![Field::new("d", DataType::Date32, false)]); + let entries_fields = Fields::from(vec![ + Field::new("key", DataType::Date32, false), + Field::new("value", DataType::Date32, false), + ]); + vec![ + ( + Arc::new(StructArray::new( + fields, + vec![Arc::clone(&dates)], + nulls.clone(), + )), + DataType::Struct(Fields::from(vec![Field::new( + "d", + timestamp.clone(), + false, + )])), + ), + ( + Arc::new(ListArray::new( + Arc::new(Field::new("element", DataType::Date32, false)), + offsets.clone(), + Arc::clone(&dates), + nulls.clone(), + )), + DataType::List(Arc::new(Field::new("element", timestamp.clone(), false))), + ), + ( + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(entries_fields.clone()), + false, + )), + offsets, + StructArray::new(entries_fields, vec![Arc::clone(&dates), dates], None), + nulls, + false, + )), + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new( + "key", + DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())), + false, + ), + Field::new("value", timestamp, false), + ])), + false, + )), + false, + ), + ), + ] + } + + fn assert_nested_timestamp_values(array: &ArrayRef, expected: &[i64]) { + array.to_data().validate_full().unwrap(); + let values = match array.data_type() { + DataType::Struct(_) => array.as_struct().column(0), + DataType::List(_) => array.as_list::().values(), + DataType::Map(_, _) => { + let keys = array + .as_map() + .keys() + .as_primitive::(); + assert_eq!(keys.null_count(), 0); + assert_eq!(keys.values().as_ref(), expected); + array.as_map().values() + } + _ => unreachable!(), + }; + assert_eq!( + values + .as_primitive::() + .iter() + .flatten() + .collect::>(), + expected, + ); + } + + #[test] + fn test_cast_nested_dates_ignores_null_parent_values() { + // Non-nullable children contain overflowing dates beneath null parent rows, as IF can + // produce without changing its child buffers. Map keys must remain non-nullable. + for (array, to_type) in nested_date_cast_inputs( + vec![i32::MAX, 0, i32::MIN, 1], + Some(NullBuffer::from(vec![false, true, false, true])), + ) { + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(mode, "UTC", false); + let casted = cast_array(Arc::clone(&array), &to_type, &options).unwrap(); + assert_eq!(casted.data_type(), &to_type); + assert_eq!(casted.len(), 4); + assert!(casted.is_null(0) && casted.is_null(2)); + assert!(!casted.is_null(1) && !casted.is_null(3)); + assert_nested_timestamp_values(&casted, &[0, 86_400_000_000]); + + let all_null = cast_array(array.slice(0, 1), &to_type, &options).unwrap(); + assert_eq!(all_null.len(), 1); + assert!(all_null.is_null(0)); + assert_nested_timestamp_values(&all_null, &[]); + } + } + } + + #[test] + fn test_cast_nested_dates_ignores_values_outside_slice() { + for (array, to_type) in nested_date_cast_inputs(vec![i32::MAX, 0, 1, i32::MIN], None) { + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(mode, "UTC", false); + // Lists/maps keep their original child buffer even when the parent is sliced. + let casted = cast_array(array.slice(1, 2), &to_type, &options).unwrap(); + assert_eq!(casted.len(), 2); + assert_eq!(casted.null_count(), 0); + assert_nested_timestamp_values(&casted, &[0, 86_400_000_000]); + + let empty = cast_array(array.slice(1, 0), &to_type, &options).unwrap(); + assert_eq!(empty.len(), 0); + assert_nested_timestamp_values(&empty, &[]); + } + } + } + + #[test] + fn test_cast_nested_dates_still_checks_visible_overflow() { + for (array, to_type) in nested_date_cast_inputs( + vec![i32::MAX, i32::MAX, 0], + Some(NullBuffer::from(vec![false, true, true])), + ) { + for mode in [EvalMode::Legacy, EvalMode::Ansi] { + let error = cast_array( + Arc::clone(&array), + &to_type, + &SparkCastOptions::new(mode, "UTC", false), + ) + .unwrap_err(); + assert!(error.to_string().contains("long overflow")); + } + } + } + #[test] fn test_cast_struct_to_struct_drop_column() { let a: ArrayRef = Arc::new(Int32Array::from(vec![ diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 163adee0b62..9e05e96ca32 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1976,6 +1976,56 @@ class CometNativeCastSuite } } + test("cast DateType to timestamps ignores values in null containers") { + withTempPath { path => + // Materialize the containers to prevent constant folding. The native IF can null a parent + // while retaining its children; keep hidden overflow and visible valid dates in one batch. + withSQLConf( + CometConf.COMET_ENABLED.key -> "false", + "spark.sql.parquet.datetimeRebaseModeInWrite" -> "CORRECTED") { + val date = "make_date(year, 6, 15)" + Seq((false, 300000), (true, 2024), (false, -300000), (true, 1970)) + .toDF("flag", "year") + .selectExpr( + "flag", + s"named_struct('d', $date) AS s", + s"map(1, $date) AS m", + s"map($date, 1) AS k", + s"array(named_struct('d', $date)) AS a") + .coalesce(1) + .write + .parquet(path.toString) + } + withParquetTable(path.toString, "masked_dates") { + for { + target <- Seq("TIMESTAMP", "TIMESTAMP_NTZ") + ansi <- Seq("false", "true") + } { + withSQLConf( + SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC", + SQLConf.ANSI_ENABLED.key -> ansi, + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "false") { + val casts = Seq( + ("s", s"STRUCT"), + ("m", s"MAP"), + ("a", s"ARRAY>")) ++ + // Spark rejects DATE -> TIMESTAMP_NTZ map keys in legacy mode. + (if (target == "TIMESTAMP") Seq(("k", s"MAP<$target, INT>")) else Seq.empty) + casts.foreach { case (column, targetType) => + // Referencing the alias twice keeps Spark from pushing CAST inside IF. CASE uses + // a different native path that already masks the children before casting them. + val query = s"SELECT masked, CAST(masked AS $targetType) FROM " + + s"(SELECT IF(flag, $column, NULL) AS masked FROM masked_dates)" + assertNoCodegenRan { + checkSparkAnswerAndOperator(sql(query), Seq(classOf[CometProjectExec])) + } + } + } + } + } + } + } + test("cast DateType to timestamps falls back for unsafe TRY_CAST nullability") { withParquetTable(Seq(Tuple1(300000), Tuple1(2024)), "try_wide_dates") { val date = "make_date(_1, 6, 15)" From e76cd0b25e70debeff49779e77e6e9239e0a29f4 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 27 Aug 2026 17:22:41 +0000 Subject: [PATCH 03/13] fix: refine date cast routing and nested input preparation --- native/spark-expr/Cargo.toml | 4 + native/spark-expr/benches/cast_nested.rs | 238 ++++++++++++++++++ .../spark-expr/src/conversion_funcs/cast.rs | 226 ++++++++++++++++- .../src/conversion_funcs/temporal.rs | 26 ++ .../apache/comet/expressions/CometCast.scala | 28 ++- .../apache/comet/CometNativeCastSuite.scala | 50 +++- .../CometCastDateToTimestampBenchmark.scala | 107 ++++++++ 7 files changed, 669 insertions(+), 10 deletions(-) create mode 100644 native/spark-expr/benches/cast_nested.rs create mode 100644 spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala diff --git a/native/spark-expr/Cargo.toml b/native/spark-expr/Cargo.toml index b52732e70b6..18af2a9722e 100644 --- a/native/spark-expr/Cargo.toml +++ b/native/spark-expr/Cargo.toml @@ -94,6 +94,10 @@ harness = false name = "cast_numeric" harness = false +[[bench]] +name = "cast_nested" +harness = false + [[bench]] name = "conditional" harness = false diff --git a/native/spark-expr/benches/cast_nested.rs b/native/spark-expr/benches/cast_nested.rs new file mode 100644 index 00000000000..7aded8776fa --- /dev/null +++ b/native/spark-expr/benches/cast_nested.rs @@ -0,0 +1,238 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hint::black_box; +use std::sync::Arc; +use std::time::Duration; + +use arrow::array::{Array, ArrayRef, Date32Array, Int32Array, ListArray, MapArray, StructArray}; +use arrow::buffer::{NullBuffer, OffsetBuffer}; +use arrow::datatypes::{DataType, Field, TimeUnit}; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use datafusion::physical_plan::ColumnarValue; +use datafusion_comet_spark_expr::{spark_cast, EvalMode, SparkCastOptions}; + +const ROWS: usize = 8192; + +fn validity(null_percent: usize) -> Option { + (null_percent != 0).then(|| { + NullBuffer::from( + (0..ROWS) + .map(|row| match null_percent { + // Exact ratios exercise the sparse-compaction boundary without rounding. + 25 => row % 4 != 0, + 50 => row % 2 != 0, + _ => row % 100 >= null_percent, + }) + .collect::>(), + ) + }) +} + +fn list_input(width: usize, null_percent: usize, fallible: bool) -> (ArrayRef, DataType) { + let offsets = OffsetBuffer::from_lengths(std::iter::repeat_n(width, ROWS)); + let values = (0..ROWS * width) + .map(|index| (index % 31 != 0).then_some((index % 20000) as i32)) + .collect::>(); + let (values, target): (ArrayRef, DataType) = if fallible { + ( + Arc::new(Date32Array::from(values)), + DataType::Timestamp(TimeUnit::Microsecond, None), + ) + } else { + (Arc::new(Int32Array::from(values)), DataType::Int64) + }; + let from_field = Arc::new(Field::new("item", values.data_type().clone(), true)); + ( + Arc::new(ListArray::new( + from_field, + offsets, + values, + validity(null_percent), + )), + DataType::List(Arc::new(Field::new("item", target, true))), + ) +} + +fn struct_input(null_percent: usize) -> (ArrayRef, DataType) { + let fields = vec![Arc::new(Field::new("value", DataType::Int32, true))].into(); + ( + Arc::new(StructArray::new( + fields, + vec![Arc::new(Int32Array::from_iter_values(0..ROWS as i32))], + validity(null_percent), + )), + DataType::Struct(vec![Field::new("value", DataType::Int64, true)].into()), + ) +} + +fn map_input(width: usize, null_percent: usize) -> (ArrayRef, DataType) { + let fields = vec![ + Arc::new(Field::new("key", DataType::Int32, false)), + Arc::new(Field::new("value", DataType::Int32, true)), + ] + .into(); + let entries = StructArray::new( + fields, + vec![ + Arc::new(Int32Array::from_iter_values(0..(ROWS * width) as i32)), + Arc::new(Int32Array::from_iter( + (0..ROWS * width).map(|index| (index % 31 != 0).then_some(index as i32)), + )), + ], + None, + ); + let target_entries = DataType::Struct( + vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", DataType::Int64, true), + ] + .into(), + ); + ( + Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + OffsetBuffer::from_lengths(std::iter::repeat_n(width, ROWS)), + entries, + validity(null_percent), + false, + )), + DataType::Map( + Arc::new(Field::new("entries", target_entries, false)), + false, + ), + ) +} + +fn criterion_benchmark(c: &mut Criterion) { + let options = SparkCastOptions::new(EvalMode::Ansi, "UTC", false); + let mut group = c.benchmark_group("cast_nested"); + group.sample_size(30); + group.warm_up_time(Duration::from_secs(1)); + group.measurement_time(Duration::from_secs(2)); + let mut bench = |name: &str, width: usize, null_percent: usize, input: (ArrayRef, DataType)| { + let (array, target) = input; + group.throughput(Throughput::Elements(array.len() as u64)); + let id = BenchmarkId::new(name, format!("width={width}/nulls={null_percent}%")); + group.bench_function(id, |b| { + b.iter(|| { + black_box( + spark_cast( + ColumnarValue::Array(Arc::clone(black_box(&array))), + black_box(&target), + black_box(&options), + ) + .unwrap(), + ) + }); + }); + }; + + for null_percent in [0, 10, 25, 50, 75, 99] { + for width in [4, 64] { + bench( + "list_int_to_long", + width, + null_percent, + list_input(width, null_percent, false), + ); + bench( + "map_int_to_long", + width, + null_percent, + map_input(width, null_percent), + ); + // Checked DATE conversion must retain normalization, even in legacy mode. + bench( + "list_date_to_timestamp", + width, + null_percent, + list_input(width, null_percent, true), + ); + } + bench( + "struct_int_to_long", + 1, + null_percent, + struct_input(null_percent), + ); + } + + let (array, target) = list_input(4, 10, false); + bench( + "sliced_list_int_to_long", + 4, + 10, + (array.slice(1, ROWS - 2), target), + ); + let (array, target) = list_input(64, 0, false); + bench( + "small_slice_int_to_long", + 64, + 0, + (array.slice(ROWS / 2, 32), target), + ); + + // One null parent can hide most values even when the parent null count is tiny. + let lengths = (0..ROWS).map(|row| if row == ROWS - 1 { 1024 * 1024 } else { 4 }); + let offsets = OffsetBuffer::from_lengths(lengths); + let child_len = offsets[ROWS] as usize; + let array: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", DataType::Int32, false)), + offsets, + Arc::new(Int32Array::from_iter_values(0..child_len as i32)), + Some(NullBuffer::from( + (0..ROWS).map(|row| row != ROWS - 1).collect::>(), + )), + )); + let target = DataType::List(Arc::new(Field::new("item", DataType::Int64, false))); + group.throughput(Throughput::Elements(ROWS as u64)); + group.bench_function("list_int_to_long/one_large_null_parent", |b| { + b.iter(|| { + black_box( + spark_cast(ColumnarValue::Array(Arc::clone(&array)), &target, &options).unwrap(), + ) + }); + }); + + // Invalid visible values exercise the error path instead of measuring a successful cast. + let (array, target) = list_input(4, 10, true); + let list = array.as_any().downcast_ref::().unwrap(); + let invalid: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", DataType::Date32, true)), + list.offsets().clone(), + Arc::new(Date32Array::from(vec![i32::MAX; list.values().len()])), + list.nulls().cloned(), + )); + group.throughput(Throughput::Elements(ROWS as u64)); + group.bench_function("list_date_to_timestamp/visible_overflow", |b| { + b.iter(|| { + black_box( + spark_cast( + ColumnarValue::Array(Arc::clone(&invalid)), + &target, + &options, + ) + .unwrap_err(), + ) + }); + }); + group.finish(); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index a1ba91e5a2b..37fc5cf9793 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -250,7 +250,7 @@ pub(crate) fn cast_array( return Ok(Arc::new(array)); } - let array = prepare_nested_cast_input(array)?; + let array = prepare_nested_cast_input(array, to_type)?; let array = array_with_timezone(array, cast_options.timezone.clone(), Some(to_type))?; let eval_mode = cast_options.eval_mode; @@ -476,8 +476,33 @@ pub(crate) fn cast_array( Ok(spark_cast_postprocess(cast_result?, &from_type, to_type)) } -/// Recursive casts must not evaluate child values hidden by a null parent or outside a slice. -fn prepare_nested_cast_input(array: ArrayRef) -> DataFusionResult { +/// True only for casts whose child values cannot fail in any evaluation mode. +/// This is deliberately a whitelist, not a replacement for the cast support matrix. +fn is_infallible_cast(from_type: &DataType, to_type: &DataType) -> bool { + use DataType::*; + if from_type == to_type { + return true; + } + match (from_type, to_type) { + (Int8, Int16 | Int32 | Int64) | (Int16, Int32 | Int64) | (Int32, Int64) => true, + (Struct(from), Struct(to)) => { + from.len() == to.len() + && from + .iter() + .zip(to) + .all(|(from, to)| is_infallible_cast(from.data_type(), to.data_type())) + } + (List(from), List(to)) | (Map(from, _), Map(to, _)) => { + is_infallible_cast(from.data_type(), to.data_type()) + } + // In particular, DATE to TIMESTAMP can overflow even in legacy mode. Retain + // normalization for every conversion not proven safe above, including TRY casts. + _ => false, + } +} + +/// Fallible recursive casts must ignore child values hidden by a null parent or outside a slice. +fn prepare_nested_cast_input(array: ArrayRef, to_type: &DataType) -> DataFusionResult { let has_unused_values = match array.data_type() { DataType::Struct(_) => false, DataType::List(_) => { @@ -492,7 +517,18 @@ fn prepare_nested_cast_input(array: ArrayRef) -> DataFusionResult { } _ => return Ok(array), }; - if array.null_count() == 0 && !has_unused_values { + let null_count = array.null_count(); + if null_count == 0 && !has_unused_values { + return Ok(array); + } + + // Widening integers does not need a copy to mask hidden children. Still compact slices + // and dense-null lists/maps: retaining most of their unused children can make even an + // infallible cast slower. Struct children always have one slot per parent row. + if !has_unused_values + && is_infallible_cast(array.data_type(), to_type) + && !should_compact_infallible_values(&array) + { return Ok(array); } @@ -506,6 +542,27 @@ fn prepare_nested_cast_input(array: ArrayRef) -> DataFusionResult { Ok(take(array.as_ref(), &indices, None)?) } +fn should_compact_infallible_values(array: &ArrayRef) -> bool { + let offsets = match array.data_type() { + DataType::List(_) => array.as_list::().value_offsets(), + DataType::Map(_, _) => array.as_map().value_offsets(), + _ => return false, + }; + // Keep the existing path for dense parents without scanning their offsets. In the sparse + // case, count child values too: a single null list/map can contain most of the batch. + if array.null_count() > array.len() / 4 { + return true; + } + let visible_values: usize = array + .nulls() + .unwrap() + .valid_slices() + .map(|(start, end)| (offsets[end] - offsets[start]) as usize) + .sum(); + let total_values = offsets[array.len()] as usize; + visible_values < total_values - total_values / 4 +} + /// Determines if DataFusion supports the given cast in a way that is /// compatible with Spark fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> bool { @@ -1442,6 +1499,167 @@ mod tests { } } + #[test] + fn test_infallible_nested_casts_reuse_input_buffers() { + let values: ArrayRef = Arc::new(Int32Array::from(vec![i32::MIN, 42, i32::MAX, 0])); + let nulls = Some(NullBuffer::from(vec![true, false, true, true])); + let offsets = OffsetBuffer::new(vec![0, 1, 2, 3, 4].into()); + let field = Arc::new(Field::new("value", DataType::Int32, false)); + let wide_field = Arc::new(Field::new("value", DataType::Int64, false)); + let entry_fields: Fields = vec![ + Arc::new(Field::new("key", DataType::Int32, false)), + Arc::clone(&field), + ] + .into(); + let wide_entry_fields: Fields = + vec![Arc::clone(&entry_fields[0]), Arc::clone(&wide_field)].into(); + let struct_values: ArrayRef = Arc::new(StructArray::new( + vec![Arc::clone(&field)].into(), + vec![Arc::clone(&values)], + None, + )); + let inputs: Vec<(ArrayRef, DataType)> = vec![ + ( + Arc::new(StructArray::new( + vec![Arc::clone(&field)].into(), + vec![Arc::clone(&values)], + nulls.clone(), + )), + DataType::Struct(vec![Arc::clone(&wide_field)].into()), + ), + ( + Arc::new(ListArray::new( + Arc::clone(&field), + offsets.clone(), + Arc::clone(&values), + nulls.clone(), + )), + DataType::List(Arc::clone(&wide_field)), + ), + ( + Arc::new(MapArray::new( + Arc::new(Field::new( + "entries", + DataType::Struct(entry_fields.clone()), + false, + )), + offsets.clone(), + StructArray::new(entry_fields, vec![Arc::clone(&values), values], None), + nulls.clone(), + false, + )), + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(wide_entry_fields), + false, + )), + false, + ), + ), + ( + Arc::new(ListArray::new( + Arc::new(Field::new("item", struct_values.data_type().clone(), false)), + offsets, + struct_values, + nulls, + )), + DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(vec![wide_field].into()), + false, + ))), + ), + ]; + + for (array, to_type) in inputs { + let prepared = prepare_nested_cast_input(Arc::clone(&array), &to_type).unwrap(); + assert!( + Arc::ptr_eq(&array, &prepared), + "unexpected copy for {to_type}" + ); + let indices = UInt64Array::new(vec![0, 1, 2, 3].into(), array.nulls().cloned()); + let normalized = take(array.as_ref(), &indices, None).unwrap(); + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(mode, "UTC", false); + let actual = cast_array(Arc::clone(&array), &to_type, &options).unwrap(); + let expected = cast_array(Arc::clone(&normalized), &to_type, &options).unwrap(); + // Arrow equality compares visible values and validity, allowing unused child + // slots under a null parent to retain their original (legal) contents. + assert_eq!(actual.to_data(), expected.to_data()); + assert_eq!(actual.nulls(), array.nulls()); + actual.to_data().validate_full().unwrap(); + if matches!(actual.data_type(), DataType::Map(_, _)) { + assert!(Arc::ptr_eq(array.as_map().keys(), actual.as_map().keys())); + assert_eq!(actual.as_map().keys().null_count(), 0); + } + for input in [array.slice(1, 2), array.slice(1, 0)] { + let casted = cast_array(Arc::clone(&input), &to_type, &options).unwrap(); + casted.to_data().validate_full().unwrap(); + assert_eq!(casted.to_data(), actual.slice(1, input.len()).to_data()); + assert_eq!(casted.nulls(), input.nulls()); + } + } + } + } + + #[test] + fn test_infallible_cast_whitelist_keeps_checked_conversions() { + for from in [DataType::Int8, DataType::Int16, DataType::Int32] { + assert!(is_infallible_cast(&from, &DataType::Int64)); + } + for (from, to) in [ + (DataType::Int64, DataType::Int32), + (DataType::Utf8, DataType::Int32), + ( + DataType::Date32, + DataType::Timestamp(TimeUnit::Microsecond, None), + ), + ] { + assert!(!is_infallible_cast(&from, &to)); + let from = DataType::Struct( + vec![ + Field::new("widen", DataType::Int32, false), + Field::new("checked", from, false), + ] + .into(), + ); + let to = DataType::Struct( + vec![ + Field::new("widen", DataType::Int64, false), + Field::new("checked", to, false), + ] + .into(), + ); + assert!(!is_infallible_cast(&from, &to)); + } + } + + #[test] + fn test_infallible_cast_still_compacts_a_large_hidden_list() { + let array: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", DataType::Int32, false)), + OffsetBuffer::new(vec![0, 1, 98, 99, 100].into()), + Arc::new(Int32Array::from_iter_values(0..100)), + Some(NullBuffer::from(vec![true, false, true, true])), + )); + let target = DataType::List(Arc::new(Field::new("item", DataType::Int64, false))); + let prepared = prepare_nested_cast_input(Arc::clone(&array), &target).unwrap(); + assert!(!Arc::ptr_eq(&array, &prepared)); + assert_eq!(prepared.as_list::().values().len(), 3); + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let result = cast_array( + Arc::clone(&array), + &target, + &SparkCastOptions::new(mode, "UTC", false), + ) + .unwrap(); + assert_eq!(result.as_list::().values().len(), 3); + assert_eq!(result.nulls(), array.nulls()); + result.to_data().validate_full().unwrap(); + } + } + #[test] fn test_cast_struct_to_struct_drop_column() { let a: ArrayRef = Arc::new(Int32Array::from(vec![ diff --git a/native/spark-expr/src/conversion_funcs/temporal.rs b/native/spark-expr/src/conversion_funcs/temporal.rs index fe0961fabc5..61f12fb893a 100644 --- a/native/spark-expr/src/conversion_funcs/temporal.rs +++ b/native/spark-expr/src/conversion_funcs/temporal.rs @@ -51,6 +51,7 @@ pub(crate) fn cast_date_to_timestamp( 0 } else { // Reject any unexpected region-zone call safely instead of constructing a chrono date. + // CometCast canonicalizes fixed-zone aliases to UTC or +/-HH:MM before serialization. cast_options .timezone .parse::() @@ -87,6 +88,31 @@ pub(crate) fn cast_date_to_timestamp( mod tests { use super::*; use std::sync::Arc; + + #[test] + fn test_cast_date_to_timestamp_accepts_canonical_minute_offsets() { + use arrow::array::Date32Array; + use arrow::datatypes::TimestampMicrosecondType; + + let dates: ArrayRef = Arc::new(Date32Array::from(vec![Some(0), None])); + for minutes in -18_i32 * 60..=18 * 60 { + let sign = if minutes < 0 { '-' } else { '+' }; + let timezone = format!("{sign}{:02}:{:02}", minutes.abs() / 60, minutes.abs() % 60); + let result = cast_date_to_timestamp( + &dates, + &SparkCastOptions::new(EvalMode::Ansi, &timezone, false), + &Some("UTC".into()), + ) + .unwrap(); + assert_eq!( + result.as_primitive::().value(0), + -i64::from(minutes) * 60_000_000, + "{timezone}" + ); + assert_eq!(result.nulls(), dates.nulls()); + } + } + #[test] fn test_cast_date_to_timestamp() { use crate::EvalMode; diff --git a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala index fd4251ce90b..753066adaaf 100644 --- a/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala +++ b/spark/src/main/scala/org/apache/comet/expressions/CometCast.scala @@ -19,7 +19,10 @@ package org.apache.comet.expressions +import java.time.ZoneOffset + import org.apache.spark.sql.catalyst.expressions.{Attribute, Cast, Expression, Literal} +import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{ArrayType, DataType, DataTypes, DecimalType, MapType, NullType, StructType, TimestampNTZType, TimestampType} @@ -155,7 +158,14 @@ object CometCast } else { val childExpr = exprToProtoInternal(cast.child, inputs, binding) if (childExpr.isDefined) { - castToProto(cast, cast.timeZoneId, cast.dataType, childExpr.get, cometEvalMode) + val timeZoneId = cast.timeZoneId.map { timezone => + if (containsDateTimestampCast(cast.child.dataType, cast.dataType)) { + nativeDateTimestampTimezone(timezone).getOrElse(timezone) + } else { + timezone + } + } + castToProto(cast, timeZoneId, cast.dataType, childExpr.get, cometEvalMode) } else { None } @@ -541,13 +551,13 @@ object CometCast toType match { case DataTypes.TimestampType => val timezone = timeZoneId.getOrElse("UTC") - if (timezone == "UTC" || timezone.matches("[+-][0-9]{2}:[0-9]{2}")) { + if (nativeDateTimestampTimezone(timezone).isDefined) { Compatible() } else { // DateType extends beyond chrono's calendar. Let Spark handle region-zone rules // across the entire date range, including future DST and historical offsets. Unsupported( - Some("Date-to-timestamp casts are native only in UTC or +/-HH:MM timezones")) + Some("Native date-to-timestamp casts require a fixed whole-minute timezone offset")) } case _: TimestampNTZType => Compatible() @@ -558,6 +568,18 @@ object CometCast case _ => Unsupported(Some(s"Cast from DateType to $toType is not supported")) } + /** + * Resolve fixed-zone aliases with Spark's rules, then serialize only the subset understood by + * the native fixed-offset parser. Checking `isFixedOffset` alone would admit UTC aliases that + * chrono rejects and offsets with seconds that chrono can silently truncate to whole minutes. + */ + private[comet] def nativeDateTimestampTimezone(timezone: String): Option[String] = + DateTimeUtils.getZoneId(timezone).normalized() match { + case offset: ZoneOffset if offset.getTotalSeconds % 60 == 0 => + Some(if (offset.getTotalSeconds == 0) "UTC" else offset.getId) + case _ => None + } + private def unsupported(fromType: DataType, toType: DataType): Unsupported = { Unsupported(Some(s"Cast from $fromType to $toType is not supported")) } diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 9e05e96ca32..eba25d10db6 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -21,6 +21,7 @@ package org.apache.comet import java.io.File import java.nio.charset.StandardCharsets +import java.time.ZoneOffset import scala.collection.mutable.ListBuffer import scala.jdk.CollectionConverters._ @@ -29,7 +30,7 @@ import scala.util.Random import org.apache.hadoop.fs.Path import org.apache.spark.SparkConf import org.apache.spark.sql.{CometTestBase, DataFrame, Row, SaveMode} -import org.apache.spark.sql.catalyst.expressions.Cast +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Cast} import org.apache.spark.sql.catalyst.parser.ParseException import org.apache.spark.sql.comet.CometProjectExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -1864,6 +1865,37 @@ class CometNativeCastSuite } } + test("cast DateType to TimestampType canonicalizes fixed timezone aliases") { + val date = AttributeReference("d", DateType, nullable = true)() + val aliases = Seq( + "GMT" -> "UTC", + "Etc/UTC" -> "UTC", + "Z" -> "UTC", + "UTC+00:00" -> "UTC", + "+00:00" -> "UTC", + "-00:00" -> "UTC", + "Etc/GMT+8" -> "-08:00", + "GMT+05:30" -> "+05:30") + val minuteOffsets = (-18 * 60 to 18 * 60).map { minutes => + val offset = ZoneOffset.ofTotalSeconds(minutes * 60).getId + offset -> (if (minutes == 0) "UTC" else offset) + } + (aliases ++ minuteOffsets).foreach { case (timezone, expected) => + val cast = Cast(date, TimestampType, Some(timezone)) + assert(CometCast.getSupportLevel(cast) == Compatible(), timezone) + val serialized = CometCast.convert(cast, Seq(date), binding = true).get.getCast + assert(serialized.getTimezone == expected, timezone) + } + Seq("America/Los_Angeles", "Asia/Kolkata", "+05:30:15", "UTC-00:00:01").foreach { timezone => + assert(CometCast.nativeDateTimestampTimezone(timezone).isEmpty, timezone) + assert( + CometCast + .getSupportLevel(Cast(date, TimestampType, Some(timezone))) + .isInstanceOf[Unsupported], + timezone) + } + } + test("cast DateType to timestamps preserves wide make_date results") { // Store the components, not dates: this both avoids Parquet rebasing and prevents Spark // from constant-folding make_date before it reaches the native cast. @@ -1872,12 +1904,20 @@ class CometNativeCastSuite (Some(-262144), 1, 1), (Some(1970), 1, 1), (Some(2024), 11, 3), + // Inside chrono's calendar, but chrono-tz 0.10.4 uses the wrong Los Angeles DST offset. + (Some(2101), 7, 4), (None, 1, 1)) withParquetTable(input, "wide_dates") { for { (target, tz) <- Seq( ("TIMESTAMP_NTZ", "America/Los_Angeles"), ("TIMESTAMP", "UTC"), + ("TIMESTAMP", "GMT"), + ("TIMESTAMP", "Etc/UTC"), + ("TIMESTAMP", "Z"), + ("TIMESTAMP", "UTC+00:00"), + ("TIMESTAMP", "GMT+05:30"), + ("TIMESTAMP", "Etc/GMT+8"), ("TIMESTAMP", "+05:30"), ("TIMESTAMP", "-08:00")) ansi <- Seq("false", "true") @@ -1888,14 +1928,15 @@ class CometNativeCastSuite CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { assertNoCodegenRan { checkSparkAnswerAndOperator( - s"SELECT CAST(make_date(_1, _2, _3) AS $target) FROM wide_dates") + s"SELECT CAST(make_date(_1, _2, _3) AS $target), " + + s"CAST(array(make_date(_1, _2, _3)) AS ARRAY<$target>) FROM wide_dates") } } } // Named regions and offsets with seconds must use Spark's timezone rules instead of // constructing chrono dates. Neither the wide positive nor negative year fits chrono. - Seq("America/Los_Angeles", "GMT", "+05:30:15").foreach { tz => + Seq("America/Los_Angeles", "+05:30:15", "UTC-00:00:01").foreach { tz => withSQLConf( SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz, CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { @@ -1926,7 +1967,10 @@ class CometNativeCastSuite (target, tz, overflowDays) <- Seq( ("TIMESTAMP_NTZ", "America/Los_Angeles", Seq(-106751992, 106751992)), ("TIMESTAMP", "UTC", Seq(-106751992, 106751992)), + ("TIMESTAMP", "Etc/UTC", Seq(-106751992, 106751992)), ("TIMESTAMP", "+18:00", Seq(-106751991)), + ("TIMESTAMP", "GMT+18:00", Seq(-106751991)), + ("TIMESTAMP", "UTC-18:00", Seq(106751991)), ("TIMESTAMP", "-18:00", Seq(106751991))) ansi <- Seq("false", "true") } { diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala new file mode 100644 index 00000000000..67025bb8ab3 --- /dev/null +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.benchmark + +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.comet.CometProjectExec +import org.apache.spark.sql.internal.SQLConf + +import org.apache.comet.CometConf +import org.apache.comet.udf.codegen.CometScalaUDFCodegen + +/** + * Compare native, dispatcher, and Spark row projection costs for column DATE to TIMESTAMP. Run + * the same benchmark on the PR base and head with a release native library: + * {{{ + * make release + * make benchmark-org.apache.spark.sql.benchmark.CometCastDateToTimestampBenchmark + * }}} + * The optional first argument sets the row count (default 1,048,576). + */ +object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { + + override def runCometBenchmark(mainArgs: Array[String]): Unit = { + val rows = mainArgs.headOption.map(_.toInt).getOrElse(1024 * 1024) + withTempPath { dir => + withTempTable("date_cast_input") { + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + // Ordinary dates from 1960 through 2029, with 1% nulls. Materialize the DATE column + // so neither engine can fold the cast or include date construction in the measurement. + spark + .range(rows) + .selectExpr("CASE WHEN id % 100 = 0 THEN NULL " + + "ELSE date_add(DATE '1960-01-01', CAST(id % 25567 AS INT)) END AS d") + .coalesce(1) + .write + .parquet(dir.getCanonicalPath) + } + spark.read.parquet(dir.getCanonicalPath).createOrReplaceTempView("date_cast_input") + val query = "SELECT CAST(d AS TIMESTAMP) FROM date_cast_input" + + for (timezone <- Seq("UTC", "America/Los_Angeles")) { + val benchmark = new Benchmark( + s"DATE to TIMESTAMP in $timezone", + rows, + minNumIters = 10, + output = output) + for (dispatch <- Seq("true", "false")) { + val configs = Seq( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", + SQLConf.SESSION_LOCAL_TIMEZONE.key -> timezone, + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_BATCH_SIZE.key -> "8192", + CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> dispatch) + val route = withSQLConf(configs: _*) { + CometScalaUDFCodegen.resetStats() + val df = spark.sql(query) + df.noop() + val plan = stripAQEPlan(df.queryExecution.executedPlan) + val nativeProject = plan.exists(_.isInstanceOf[CometProjectExec]) + val stats = CometScalaUDFCodegen.stats() + val dispatched = stats.compileCount + stats.cacheHitCount > 0 + val route = if (dispatched) { + require(nativeProject, s"Dispatcher must remain in a Comet projection: $plan") + "JVM dispatcher" + } else if (nativeProject) { + "native" + } else { + require(dispatch == "false", s"Unexpected row fallback: $plan") + "Spark row projection" + } + if (timezone == "UTC") { + require(route == "native", s"UTC control must stay native: $plan") + } + benchmark.out.println(s"timezone=$timezone dispatcher=$dispatch route=$route") + benchmark.out.println(plan.treeString) + route + } + benchmark.addCase(s"$route, dispatcher=$dispatch") { _ => + withSQLConf(configs: _*) { + spark.sql(query).noop() + } + } + } + benchmark.run() + } + } + } + } +} From 1c66e750d499486b03f69f49c6d67b2d9f24c1e8 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 27 Aug 2026 17:28:31 +0000 Subject: [PATCH 04/13] test: compare logical validity for empty cast slices --- native/spark-expr/src/conversion_funcs/cast.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 37fc5cf9793..9794c10540b 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -1597,7 +1597,11 @@ mod tests { let casted = cast_array(Arc::clone(&input), &to_type, &options).unwrap(); casted.to_data().validate_full().unwrap(); assert_eq!(casted.to_data(), actual.slice(1, input.len()).to_data()); - assert_eq!(casted.nulls(), input.nulls()); + assert_eq!(casted.null_count(), input.null_count()); + // Arrow may omit the bitmap for an empty or entirely valid slice. + if input.null_count() != 0 { + assert_eq!(casted.nulls(), input.nulls()); + } } } } From 9d5283b5e5d4049d4cf9a6b3fa73e53ba8079182 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 27 Aug 2026 18:05:29 +0000 Subject: [PATCH 05/13] test: refine nested date cast and benchmark checks --- .../test/scala/org/apache/comet/CometNativeCastSuite.scala | 5 ++++- .../sql/benchmark/CometCastDateToTimestampBenchmark.scala | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index eba25d10db6..fb544c3f22a 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1927,9 +1927,12 @@ class CometNativeCastSuite SQLConf.ANSI_ENABLED.key -> ansi, CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> "true") { assertNoCodegenRan { + // Direct Array to Array has a separate support restriction; + // Array exercises recursive serialization on an admitted native path. checkSparkAnswerAndOperator( s"SELECT CAST(make_date(_1, _2, _3) AS $target), " + - s"CAST(array(make_date(_1, _2, _3)) AS ARRAY<$target>) FROM wide_dates") + s"CAST(array(named_struct('d', make_date(_1, _2, _3))) " + + s"AS ARRAY>) FROM wide_dates") } } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala index 67025bb8ab3..3b3dc53de1d 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala @@ -37,7 +37,7 @@ import org.apache.comet.udf.codegen.CometScalaUDFCodegen */ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { - override def runCometBenchmark(mainArgs: Array[String]): Unit = { + override def runCometBenchmark(mainArgs: Array[String]): Unit = try { val rows = mainArgs.headOption.map(_.toInt).getOrElse(1024 * 1024) withTempPath { dir => withTempTable("date_cast_input") { @@ -103,5 +103,7 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { } } } + } finally { + spark.stop() } } From 2e14d1d7651e1e441fb98b32dc117b4dea0e5614 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Thu, 27 Aug 2026 18:31:46 +0000 Subject: [PATCH 06/13] fix: bound nested cast reuse to fixed-width children --- .../spark-expr/src/conversion_funcs/cast.rs | 219 ++++++++++++++++-- 1 file changed, 200 insertions(+), 19 deletions(-) diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 9794c10540b..13ca0b135aa 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -476,27 +476,36 @@ pub(crate) fn cast_array( Ok(spark_cast_postprocess(cast_result?, &from_type, to_type)) } -/// True only for casts whose child values cannot fail in any evaluation mode. -/// This is deliberately a whitelist, not a replacement for the cast support matrix. -fn is_infallible_cast(from_type: &DataType, to_type: &DataType) -> bool { +/// True only for infallible casts whose leaves all have fixed width. Even an identity cast on +/// a variable-width child could retain a large payload hidden by a null ancestor. Structs keep +/// one slot per parent; lists/maps are only considered at the current preparation boundary, +/// where their offsets let us bound the hidden values. This is not a cast support matrix. +fn is_infallible_fixed_width_cast(from_type: &DataType, to_type: &DataType) -> bool { use DataType::*; - if from_type == to_type { - return true; - } match (from_type, to_type) { (Int8, Int16 | Int32 | Int64) | (Int16, Int32 | Int64) | (Int32, Int64) => true, (Struct(from), Struct(to)) => { from.len() == to.len() - && from - .iter() - .zip(to) - .all(|(from, to)| is_infallible_cast(from.data_type(), to.data_type())) + && from.iter().zip(to).all(|(from, to)| { + is_infallible_fixed_width_cast(from.data_type(), to.data_type()) + }) + } + // In particular, DATE to TIMESTAMP can overflow even in legacy mode. Retain + // normalization for every changed leaf not proven safe above, including TRY casts. + _ => { + (from_type.is_primitive() || matches!(from_type, Boolean | Null)) + && from_type == to_type } + } +} + +fn has_infallible_fixed_width_children(from_type: &DataType, to_type: &DataType) -> bool { + use DataType::*; + match (from_type, to_type) { + (Struct(_), Struct(_)) => is_infallible_fixed_width_cast(from_type, to_type), (List(from), List(to)) | (Map(from, _), Map(to, _)) => { - is_infallible_cast(from.data_type(), to.data_type()) + is_infallible_fixed_width_cast(from.data_type(), to.data_type()) } - // In particular, DATE to TIMESTAMP can overflow even in legacy mode. Retain - // normalization for every conversion not proven safe above, including TRY casts. _ => false, } } @@ -524,9 +533,10 @@ fn prepare_nested_cast_input(array: ArrayRef, to_type: &DataType) -> DataFusionR // Widening integers does not need a copy to mask hidden children. Still compact slices // and dense-null lists/maps: retaining most of their unused children can make even an - // infallible cast slower. Struct children always have one slot per parent row. + // infallible cast slower. Reuse only fixed-width leaves (possibly inside structs), so a + // hidden nested list/map or unchanged string cannot escape this container's density check. if !has_unused_values - && is_infallible_cast(array.data_type(), to_type) + && has_infallible_fixed_width_children(array.data_type(), to_type) && !should_compact_infallible_values(&array) { return Ok(array); @@ -1022,7 +1032,9 @@ mod tests { BinaryArray, Date32Array, Decimal128Array, ListArray, NullArray, PrimitiveArray, StringArray, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; - use arrow::datatypes::{Field, Fields, Int32Type, TimeUnit, TimestampMicrosecondType}; + use arrow::datatypes::{ + Field, Fields, Int32Type, Int64Type, TimeUnit, TimestampMicrosecondType, + }; #[test] fn test_cast_to_dictionary_is_rejected() { @@ -1610,7 +1622,23 @@ mod tests { #[test] fn test_infallible_cast_whitelist_keeps_checked_conversions() { for from in [DataType::Int8, DataType::Int16, DataType::Int32] { - assert!(is_infallible_cast(&from, &DataType::Int64)); + assert!(is_infallible_fixed_width_cast(&from, &DataType::Int64)); + } + for fixed in [ + DataType::Int32, + DataType::Date32, + DataType::Boolean, + DataType::Null, + ] { + assert!(is_infallible_fixed_width_cast(&fixed, &fixed)); + } + for variable in [ + DataType::Utf8, + DataType::Binary, + DataType::List(Arc::new(Field::new("item", DataType::Int32, false))), + DataType::Struct(vec![Field::new("text", DataType::Utf8, false)].into()), + ] { + assert!(!is_infallible_fixed_width_cast(&variable, &variable)); } for (from, to) in [ (DataType::Int64, DataType::Int32), @@ -1620,7 +1648,7 @@ mod tests { DataType::Timestamp(TimeUnit::Microsecond, None), ), ] { - assert!(!is_infallible_cast(&from, &to)); + assert!(!is_infallible_fixed_width_cast(&from, &to)); let from = DataType::Struct( vec![ Field::new("widen", DataType::Int32, false), @@ -1635,7 +1663,7 @@ mod tests { ] .into(), ); - assert!(!is_infallible_cast(&from, &to)); + assert!(!is_infallible_fixed_width_cast(&from, &to)); } } @@ -1664,6 +1692,159 @@ mod tests { } } + #[test] + fn test_infallible_cast_preserves_compaction_through_variable_width_children() { + // Each immediate child has one slot, but a null outer parent hides most leaf values. + let hidden_values = 65_536; + let values: ArrayRef = Arc::new(ListArray::new( + Arc::new(Field::new("item", DataType::Int32, false)), + OffsetBuffer::new( + vec![ + 0, + 1, + hidden_values + 1, + hidden_values + 2, + hidden_values + 3, + ] + .into(), + ), + Arc::new(Int32Array::from_iter_values(0..hidden_values + 3)), + None, + )); + let target_values = DataType::List(Arc::new(Field::new("item", DataType::Int64, false))); + let nulls = Some(NullBuffer::from(vec![true, false, true, true])); + let offsets = OffsetBuffer::new(vec![0, 1, 2, 3, 4].into()); + let entries = StructArray::new( + vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", values.data_type().clone(), false), + ] + .into(), + vec![ + Arc::new(Int32Array::from_iter_values(0..4)), + Arc::clone(&values), + ], + None, + ); + let inputs: Vec<(ArrayRef, DataType)> = vec![ + ( + Arc::new(StructArray::new( + vec![Field::new("value", values.data_type().clone(), false)].into(), + vec![Arc::clone(&values)], + nulls.clone(), + )), + DataType::Struct(vec![Field::new("value", target_values.clone(), false)].into()), + ), + ( + Arc::new(ListArray::new( + Arc::new(Field::new("item", values.data_type().clone(), false)), + offsets.clone(), + values, + nulls.clone(), + )), + DataType::List(Arc::new(Field::new("item", target_values.clone(), false))), + ), + ( + Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets, + entries, + nulls, + false, + )), + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Int32, false), + Field::new("value", target_values, false), + ] + .into(), + ), + false, + )), + false, + ), + ), + ]; + + fn leaf_values(array: &ArrayRef) -> &ArrayRef { + let lists = match array.data_type() { + DataType::Struct(_) => array.as_struct().column(0), + DataType::List(_) => array.as_list::().values(), + DataType::Map(_, _) => array.as_map().values(), + _ => unreachable!(), + }; + lists.as_list::().values() + } + + for (array, target) in inputs { + let prepared = prepare_nested_cast_input(Arc::clone(&array), &target).unwrap(); + assert!(!Arc::ptr_eq(&array, &prepared), "{target}"); + assert_eq!(leaf_values(&prepared).len(), 3, "{target}"); + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(mode, "UTC", false); + let result = cast_array(Arc::clone(&array), &target, &options).unwrap(); + let expected = cast_array(Arc::clone(&prepared), &target, &options).unwrap(); + assert_eq!(result.to_data(), expected.to_data()); + assert_eq!(result.nulls(), array.nulls()); + assert_eq!( + leaf_values(&result) + .as_primitive::() + .values() + .as_ref(), + &[ + 0, + i64::from(hidden_values + 1), + i64::from(hidden_values + 2) + ] + ); + result.to_data().validate_full().unwrap(); + } + } + } + + #[test] + fn test_infallible_cast_preserves_compaction_of_variable_width_identity_values() { + // An unchanged string child needs no conversion, but reusing it would retain the + // large hidden payload in the output of the changed struct cast. + let hidden = "x".repeat(65_536); + let array: ArrayRef = Arc::new(StructArray::new( + vec![ + Field::new("widen", DataType::Int32, false), + Field::new("unchanged", DataType::Utf8, false), + ] + .into(), + vec![ + Arc::new(Int32Array::from_iter_values(0..4)), + Arc::new(StringArray::from(vec!["a", hidden.as_str(), "b", "c"])), + ], + Some(NullBuffer::from(vec![true, false, true, true])), + )); + let target = DataType::Struct( + vec![ + Field::new("widen", DataType::Int64, false), + Field::new("unchanged", DataType::Utf8, false), + ] + .into(), + ); + let prepared = prepare_nested_cast_input(Arc::clone(&array), &target).unwrap(); + assert!(!Arc::ptr_eq(&array, &prepared)); + for mode in [EvalMode::Legacy, EvalMode::Ansi, EvalMode::Try] { + let options = SparkCastOptions::new(mode, "UTC", false); + let result = cast_array(Arc::clone(&array), &target, &options).unwrap(); + let expected = cast_array(Arc::clone(&prepared), &target, &options).unwrap(); + assert_eq!(result.to_data(), expected.to_data()); + assert_eq!(result.nulls(), array.nulls()); + assert_eq!( + result.as_struct().column(1).as_string::().value_data(), + b"abc" + ); + result.to_data().validate_full().unwrap(); + } + } + #[test] fn test_cast_struct_to_struct_drop_column() { let a: ArrayRef = Arc::new(Int32Array::from(vec![ From f4bc7c2defe957fc9e57403fe3ffe1b053886651 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 28 Aug 2026 01:23:50 +0000 Subject: [PATCH 07/13] fix: remove redundant interpolation in date cast tests --- .../src/test/scala/org/apache/comet/CometNativeCastSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index fb544c3f22a..52f0839935e 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -1931,7 +1931,7 @@ class CometNativeCastSuite // Array exercises recursive serialization on an admitted native path. checkSparkAnswerAndOperator( s"SELECT CAST(make_date(_1, _2, _3) AS $target), " + - s"CAST(array(named_struct('d', make_date(_1, _2, _3))) " + + "CAST(array(named_struct('d', make_date(_1, _2, _3))) " + s"AS ARRAY>) FROM wide_dates") } } From 8c6257530890fde0e6d7dcb1cba93d0b17141d8a Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sun, 4 Oct 2026 18:01:32 +0000 Subject: [PATCH 08/13] fix: check Parquet date-to-NTZ conversion without blanket fallback --- .../expressions/_category_template/cast.md | 12 ++ native/core/src/parquet/parquet_support.rs | 108 ++++++++++++++++-- native/core/src/parquet/schema_adapter.rs | 10 +- .../spark-expr/src/conversion_funcs/cast.rs | 3 +- .../src/conversion_funcs/temporal.rs | 3 + .../comet/rules/CometScanRuleSuite.scala | 58 ++++++++++ 6 files changed, 180 insertions(+), 14 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index c023539f36e..82d16449bb0 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -33,6 +33,18 @@ Cast operations in Comet fall into three levels of support: Incompatible and unsupported casts fall back to Spark only when Comet's codegen dispatcher cannot handle them (for example, casts involving `VariantType`) or when `spark.comet.exec.scalaUDF.codegen.enabled=false`. +## Date-to-timestamp casts + +`DATE` to `TIMESTAMP_NTZ` uses checked day arithmetic. `DATE` to `TIMESTAMP` runs natively for +fixed whole-minute timezone offsets, including aliases such as `Etc/UTC` and `GMT+05:30`. +Region zones and offsets containing seconds use Spark's codegen dispatcher, or Spark operators +when dispatch is disabled, to preserve Spark's calendar and timezone rules across the full date range. + +Overflow raises `ArithmeticException("long overflow")` in both ANSI and legacy modes. A nullable +scalar `TRY_CAST` returns null instead. Hidden children beneath null containers and values outside +sliced list/map ranges are not evaluated. These casts retain Comet's existing batch evaluation; +this does not add row-by-row short-circuit guarantees beneath limits or joins. + ## ANSI Mode Support Enabling ANSI mode does not by itself make a cast fall back to Spark or require diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index e8987a79da6..dee91306c76 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -112,9 +112,9 @@ pub struct SparkParquetOptions { /// (Spark 3.x, SPARK-36182). Mirrors Comet's per-Spark-version constant /// in ShimCometConf. pub allow_timestamp_ltz_to_ntz: bool, - /// When true (the default), a TIMESTAMP_MILLIS field that overflows during - /// the millis->micros upscale raises an error, matching Spark's checked - /// `millisToMicros`. Filtered scans set this to false and retain the safe cast + /// When true (the default), TIMESTAMP_MILLIS or DATE fields that overflow during + /// conversion to microsecond timestamps raise an error, matching Spark's checked + /// `millisToMicros` / `daysToMicros`. Filtered scans retain the safe cast /// (overflow -> NULL), because Spark may discard values through pruning paths that /// DataFusion cannot fully mirror before conversion. pub checked_timestamp_overflow: bool, @@ -198,11 +198,17 @@ fn parquet_convert_array_impl( ) -> DataFusionResult { use DataType::*; let from_type = array.data_type(); - // Only checked millis-to-micros casts consume ancestor visibility. In particular, + // Only checked timestamp conversions consume ancestor visibility. In particular, // unchanged array/map siblings must not expand a mask over their backing values. let checked_timestamp_overflow = parquet_options.checked_timestamp_overflow - && has_timestamp_unit(from_type, TimeUnit::Millisecond) - && has_timestamp_unit(to_type, TimeUnit::Microsecond); + && ((has_type(from_type, |dt| { + matches!(dt, Timestamp(TimeUnit::Millisecond, _)) + }) && has_type(to_type, |dt| { + matches!(dt, Timestamp(TimeUnit::Microsecond, _)) + })) || (has_type(from_type, |dt| matches!(dt, Date32)) + && has_type(to_type, |dt| { + matches!(dt, Timestamp(TimeUnit::Microsecond, None)) + }))); let visible = if checked_timestamp_overflow { NullBuffer::union(array.nulls(), parent_nulls) } else { @@ -279,6 +285,19 @@ fn parquet_convert_array_impl( .with_timezone_opt(target_tz.clone()); Ok(Arc::new(micros)) } + (Date32, Timestamp(TimeUnit::Microsecond, None)) if checked_timestamp_overflow => { + // Spark's Parquet DATE -> NTZ updater calls checked daysToMicros. Reuse the + // same visibility and filtered-scan policy as the checked millisecond updater. + let dates = array.as_primitive::(); + let micros = arrow::array::Date32Array::new(dates.values().clone(), visible) + .try_unary::<_, TimestampMicrosecondType, _>(|value| { + i64::from(value).mul_checked(86_400_000_000) + }) + .map_err(|_| SparkError::LongOverflow)?; + // Required nested children retain their original validity, even below null parents. + Ok(Arc::new(arrow::array::TimestampMicrosecondArray::new( + micros.values().clone(), dates.nulls().cloned()))) + } (Timestamp(TimeUnit::Microsecond, None), Timestamp(TimeUnit::Microsecond, Some(tz))) => { Ok(Arc::new( array @@ -328,18 +347,20 @@ fn parquet_convert_array_impl( // Struct fields are matched by name/field ID later. This type-only check is conservative // until that matching occurs; each selected child is checked again before conversion. -fn has_timestamp_unit(data_type: &DataType, unit: TimeUnit) -> bool { +fn has_type(data_type: &DataType, predicate: fn(&DataType) -> bool) -> bool { + if predicate(data_type) { + return true; + } match data_type { - DataType::Timestamp(timestamp_unit, _) => *timestamp_unit == unit, DataType::Struct(fields) => fields .iter() - .any(|field| has_timestamp_unit(field.data_type(), unit)), + .any(|field| has_type(field.data_type(), predicate)), DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) | DataType::ListView(field) | DataType::LargeListView(field) - | DataType::Map(field, _) => has_timestamp_unit(field.data_type(), unit), + | DataType::Map(field, _) => has_type(field.data_type(), predicate), _ => false, } } @@ -1505,6 +1526,73 @@ mod tests { } } + #[test] + fn test_date_to_ntz_checked_overflow_and_hidden_values() { + use super::{parquet_convert_array, SparkParquetOptions}; + use arrow::array::{Array, ArrayRef, AsArray, Date32Array, ListArray, StructArray}; + use arrow::buffer::{NullBuffer, OffsetBuffer}; + use arrow::datatypes::{DataType, Field, TimeUnit, TimestampMicrosecondType}; + use datafusion_comet_spark_expr::EvalMode; + use std::sync::Arc; + + let target = DataType::Timestamp(TimeUnit::Microsecond, None); + let options = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false); + for day in [106_751_992, -106_751_992, i32::MIN, i32::MAX] { + let values: ArrayRef = Arc::new(Date32Array::from(vec![Some(day), None, Some(1)])); + let error = parquet_convert_array(Arc::clone(&values), &target, &options).unwrap_err(); + assert!(error.to_string().contains("long overflow"), "{error}"); + let mut filtered = options.clone(); + filtered.checked_timestamp_overflow = false; + let result = parquet_convert_array(values, &target, &filtered).unwrap(); + assert!(result.is_null(0)); + assert!(result.is_null(1)); + assert_eq!( + result.as_primitive::().value(2), + 86_400_000_000 + ); + } + let values: ArrayRef = Arc::new(Date32Array::from(vec![i32::MAX, 1, i32::MIN])); + let input = StructArray::new( + vec![Arc::new(Field::new("d", DataType::Date32, false))].into(), + vec![Arc::clone(&values)], + Some(NullBuffer::from(vec![false, true, false])), + ); + let struct_type = + DataType::Struct(vec![Arc::new(Field::new("d", target.clone(), false))].into()); + let output = parquet_convert_array(Arc::new(input), &struct_type, &options).unwrap(); + assert!(output.is_null(0)); + assert!(output.is_null(2)); + assert_eq!(output.as_struct().column(0).null_count(), 0); + assert_eq!( + output + .as_struct() + .column(0) + .as_primitive::() + .value(1), + 86_400_000_000 + ); + + // A sliced list retains overflowing backing values outside its visible offsets. + let list = ListArray::new( + Arc::new(Field::new("item", DataType::Date32, false)), + OffsetBuffer::new(vec![0, 1, 2, 3].into()), + values, + None, + ); + let target_list = DataType::List(Arc::new(Field::new("item", target, false))); + assert!(parquet_convert_array(Arc::new(list.clone()), &target_list, &options).is_err()); + let result = + parquet_convert_array(Arc::new(list.slice(1, 1)), &target_list, &options).unwrap(); + assert_eq!( + result + .as_list::() + .value(0) + .as_primitive::() + .value(0), + 86_400_000_000 + ); + } + #[test] fn test_millis_to_micros_overflow_checked_in_nested_fields() { use crate::parquet::parquet_support::{parquet_convert_array, SparkParquetOptions}; diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 6bcc8d3518b..63e40c975f4 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -23,7 +23,7 @@ use crate::parquet::parquet_support::{ }; use arrow::array::new_empty_array; use arrow::compute::can_cast_types; -use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::common::{DataFusionError, Result as DataFusionResult}; @@ -1435,8 +1435,8 @@ impl SparkPhysicalExprAdapter { // Complex casts (including changes in list representation), timestamp tz relabel // (e.g. Timestamp(us, None) -> Timestamp(us, Some("UTC")) for INT96 reads), and - // Timestamp -> Int64 - // (Spark's `nanosAsLong`) need spark_parquet_convert: it handles nested + // Timestamp -> Int64 (Spark's `nanosAsLong`), and DATE -> NTZ need + // spark_parquet_convert: it handles checked reader conversions, nested // field selection, metadata-only tz changes, and raw-value reinterpretation // that Spark's Cast would otherwise convert incorrectly. if matches!( @@ -1457,6 +1457,10 @@ impl SparkPhysicalExprAdapter { | (DataType::Map(_, _), DataType::Map(_, _)) | (DataType::Timestamp(_, _), DataType::Timestamp(_, _)) | (DataType::Timestamp(_, _), DataType::Int64) + | ( + DataType::Date32, + DataType::Timestamp(TimeUnit::Microsecond, None) + ) ) { let comet_cast: Arc = Arc::new( CometCastColumnExpr::try_new( diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 13ca0b135aa..e9d0aeaf0f6 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -1029,7 +1029,8 @@ fn cast_binary_to_string( mod tests { use super::*; use arrow::array::{ - BinaryArray, Date32Array, Decimal128Array, ListArray, NullArray, PrimitiveArray, StringArray, + BinaryArray, Date32Array, Decimal128Array, ListArray, NullArray, PrimitiveArray, + StringArray, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; use arrow::datatypes::{ diff --git a/native/spark-expr/src/conversion_funcs/temporal.rs b/native/spark-expr/src/conversion_funcs/temporal.rs index 61f12fb893a..9340ab421c6 100644 --- a/native/spark-expr/src/conversion_funcs/temporal.rs +++ b/native/spark-expr/src/conversion_funcs/temporal.rs @@ -38,6 +38,8 @@ pub(crate) fn cast_date_to_timestamp( cast_options: &SparkCastOptions, target_tz: &Option>, ) -> SparkResult { + // Retain checked arithmetic instead of Arrow's generic NTZ cast so failures preserve + // Spark's plain ArithmeticException("long overflow") in both ANSI and legacy modes. let date_array = array_ref.as_primitive::(); let mut builder = TimestampMicrosecondBuilder::with_capacity(date_array.len()); @@ -87,6 +89,7 @@ pub(crate) fn cast_date_to_timestamp( #[cfg(test)] mod tests { use super::*; + use arrow::datatypes::TimeUnit; use std::sync::Arc; #[test] diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 52c6159d11e..71941dd7216 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -100,6 +100,64 @@ class CometScanRuleSuite extends CometTestBase { } } + test("Parquet DATE to NTZ keeps native scans and checks visible overflow") { + assume(org.apache.comet.CometSparkSessionExtensions.isSpark40Plus) + val schema = StructType(Seq(StructField("id", LongType), StructField("d", TimestampNTZType))) + withSQLConf( + "spark.sql.parquet.datetimeRebaseModeInWrite" -> "CORRECTED", + "spark.sql.parquet.datetimeRebaseModeInRead" -> "CORRECTED") { + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(3) + .selectExpr("id", "date_from_unix_date(CAST(id AS INT)) AS d") + .coalesce(1) + .write + .parquet(path.toString) + } + val (_, plan) = + checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) + } + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(3) + .selectExpr("id", "TIMESTAMP_NTZ'2000-01-01 00:00:00' AS d") + .coalesce(1) + .write + .parquet(path.toString) + } + val (_, plan) = + checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) + } + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(6000) + .selectExpr("id", "date_from_unix_date(IF(id = 5000, 106751992, 0)) AS d") + .coalesce(1) + .write + .parquet(path.toString) + } + val read = () => spark.read.schema(schema).parquet(path.toString) + val (sparkError, cometError) = checkSparkAnswerMaybeThrows(read()) + for (error <- Seq(sparkError, cometError)) { + assert(error.nonEmpty) + assert(causeChain(error.get).exists { + case e: ArithmeticException => e.getMessage == "long overflow" + case _ => false + }) + } + // Retain the existing filtered-scan policy: a later physical DATE overflow must not + // prevent returning the first valid row when Spark prunes or stops before that batch. + val (_, plan) = checkSparkAnswerAndOperator(read().filter("id = 0").limit(1)) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) + } + } + } + test("CometScanRule should fallback to Spark for ShortType when safety check enabled") { withTempPath { path => // Create test data with ShortType which may be from unsigned UINT_8 From 9a634af421b27f277d4ba2d66a1e939ec9128729 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Sun, 4 Oct 2026 18:16:34 +0000 Subject: [PATCH 09/13] test: satisfy strict Scala checks in date cast coverage --- .../apache/comet/CometNativeCastSuite.scala | 23 +------------------ .../CometCastDateToTimestampBenchmark.scala | 10 ++++---- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala index 52f0839935e..53d74998b23 100644 --- a/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometNativeCastSuite.scala @@ -3030,28 +3030,7 @@ class CometNativeCastSuite s"Expected a native cast, but the JVM codegen dispatcher ran: $stats") } - private def castFallbackTest( - input: DataFrame, - toType: DataType, - expectedMessage: String): Unit = { - withTempPath { dir => - val data = roundtripParquet(input, dir).coalesce(1) - data.createOrReplaceTempView("t") - - withSQLConf((SQLConf.ANSI_ENABLED.key, "false")) { - val df = data.withColumn("converted", col("a").cast(toType)) - df.collect() - val str = - new ExtendedExplainInfo().generateExtendedInfo(df.queryExecution.executedPlan) - assert(str.contains(expectedMessage)) - } - } - } - - private def castTimestampTest( - input: DataFrame, - toType: DataType, - assertNative: Boolean = false) = { + private def castTimestampTest(input: DataFrame, toType: DataType, assertNative: Boolean) = { withTempPath { dir => val data = roundtripParquet(input, dir).coalesce(1) data.createOrReplaceTempView("t") diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala index 3b3dc53de1d..b8dc48a675e 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastDateToTimestampBenchmark.scala @@ -45,7 +45,7 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { // Ordinary dates from 1960 through 2029, with 1% nulls. Materialize the DATE column // so neither engine can fold the cast or include date construction in the measurement. spark - .range(rows) + .range(rows.toLong) .selectExpr("CASE WHEN id % 100 = 0 THEN NULL " + "ELSE date_add(DATE '1960-01-01', CAST(id % 25567 AS INT)) END AS d") .coalesce(1) @@ -58,7 +58,7 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { for (timezone <- Seq("UTC", "America/Los_Angeles")) { val benchmark = new Benchmark( s"DATE to TIMESTAMP in $timezone", - rows, + rows.toLong, minNumIters = 10, output = output) for (dispatch <- Seq("true", "false")) { @@ -69,7 +69,8 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_BATCH_SIZE.key -> "8192", CometConf.COMET_SCALA_UDF_CODEGEN_ENABLED.key -> dispatch) - val route = withSQLConf(configs: _*) { + var route = "" + withSQLConf(configs: _*) { CometScalaUDFCodegen.resetStats() val df = spark.sql(query) df.noop() @@ -77,7 +78,7 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { val nativeProject = plan.exists(_.isInstanceOf[CometProjectExec]) val stats = CometScalaUDFCodegen.stats() val dispatched = stats.compileCount + stats.cacheHitCount > 0 - val route = if (dispatched) { + route = if (dispatched) { require(nativeProject, s"Dispatcher must remain in a Comet projection: $plan") "JVM dispatcher" } else if (nativeProject) { @@ -91,7 +92,6 @@ object CometCastDateToTimestampBenchmark extends CometBenchmarkBase { } benchmark.out.println(s"timezone=$timezone dispatcher=$dispatch route=$route") benchmark.out.println(plan.treeString) - route } benchmark.addCase(s"$route, dispatcher=$dispatch") { _ => withSQLConf(configs: _*) { From 633a8058896abe41ed018f4b3a9fa126ef781956 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Mon, 5 Oct 2026 21:28:00 +0000 Subject: [PATCH 10/13] fix: preserve filtered Parquet DATE to NTZ overflow errors --- .../expressions/_category_template/cast.md | 7 + .../serde/operator/CometNativeScan.scala | 23 ++- .../comet/rules/CometScanRuleSuite.scala | 135 +++++++++++------- 3 files changed, 115 insertions(+), 50 deletions(-) diff --git a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md index 82d16449bb0..36e5f9c81ae 100644 --- a/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md +++ b/docs/source/user-guide/latest/compatibility/expressions/_category_template/cast.md @@ -111,6 +111,13 @@ timezone-independent: each date is converted to midnight as pure arithmetic (`days * 86,400,000,000` microseconds) with no session timezone offset applied. The result is the same regardless of the session timezone setting. +Parquet scans with data filters and requested `TIMESTAMP_NTZ` fields use Spark's reader, +including nested fields. A physical `DATE` column can overflow while adapting to the +requested type, and Spark may skip its conversion through pruning or an early limit. +The physical column type is unavailable during planning, so this fallback also applies +to filtered scans of genuine NTZ columns. Unfiltered scans and projections that do not +read NTZ fields remain eligible for native execution. + ## Date to Numeric Types In Legacy mode, `CAST(date AS INT)`, `CAST(date AS LONG)`, and casts to all other numeric diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala index 286937635b7..3d3a297d804 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometNativeScan.scala @@ -30,7 +30,7 @@ import org.apache.spark.sql.comet.{CometNativeExec, CometNativeScanExec, CometSc import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, SubqueryAdaptiveBroadcastExec} import org.apache.spark.sql.execution.datasources.parquet.ParquetUtils import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{StructField, StructType} +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType, TimestampNTZType} import org.apache.comet.{CometConf, ConfigEntry, DataTypeSupport} import org.apache.comet.CometConf.COMET_EXEC_ENABLED @@ -121,6 +121,18 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS } }) + private def containsTimestampNTZ(dataType: DataType): Boolean = dataType match { + case TimestampNTZType => true + case struct: StructType => struct.fields.exists(field => containsTimestampNTZ(field.dataType)) + case ArrayType(elementType, _) => containsTimestampNTZ(elementType) + case MapType(keyType, valueType, _) => + containsTimestampNTZ(keyType) || containsTimestampNTZ(valueType) + case _ => false + } + + private[comet] val filteredTimestampNTZReason = + "Native Parquet scan cannot preserve DATE to TIMESTAMP_NTZ overflow semantics with data filters" + /** Determine whether the scan is supported and tag the Spark plan with any fallback reasons */ def isSupported(scanExec: FileSourceScanExec): Boolean = { @@ -163,6 +175,15 @@ object CometNativeScan extends CometOperatorSerde[CometScanExec] with CometTypeS withFallbackReason(scanExec, "Full native scan disabled because ignoreMissingFiles enabled") } + // A requested NTZ field can be backed by a physical DATE column. Filtered native scans + // currently use a safe cast that turns DATE overflow into NULL. Checking eagerly instead + // would fail on rows Spark can skip through pruning or an early LIMIT. The physical schema + // is only available when each file opens, so conservatively retain Spark's reader for this + // projected type, including genuine NTZ files, until conversion follows Spark's row demand. + if (scanExec.dataFilters.nonEmpty && containsTimestampNTZ(scanExec.requiredSchema)) { + withFallbackReason(scanExec, filteredTimestampNTZReason) + } + if (serializeExistenceDefaultValues(scanExec.requiredSchema, scanExec.output).isEmpty) { withFallbackReason(scanExec, unsupportedDefaultReason) } diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala index 71941dd7216..eba841d6257 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanRuleSuite.scala @@ -30,6 +30,7 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types._ import org.apache.comet.{CometConf, DataTypeSupport} +import org.apache.comet.serde.operator.CometNativeScan import org.apache.comet.testing.{DataGenOptions, FuzzDataGenerator} /** @@ -100,60 +101,96 @@ class CometScanRuleSuite extends CometTestBase { } } - test("Parquet DATE to NTZ keeps native scans and checks visible overflow") { + test("Parquet DATE to NTZ checks overflow and preserves filtered row demand") { assume(org.apache.comet.CometSparkSessionExtensions.isSpark40Plus) val schema = StructType(Seq(StructField("id", LongType), StructField("d", TimestampNTZType))) - withSQLConf( - "spark.sql.parquet.datetimeRebaseModeInWrite" -> "CORRECTED", - "spark.sql.parquet.datetimeRebaseModeInRead" -> "CORRECTED") { - withTempPath { path => - withSQLConf(CometConf.COMET_ENABLED.key -> "false") { - spark - .range(3) - .selectExpr("id", "date_from_unix_date(CAST(id AS INT)) AS d") - .coalesce(1) - .write - .parquet(path.toString) - } - val (_, plan) = - checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) - assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) - } - withTempPath { path => - withSQLConf(CometConf.COMET_ENABLED.key -> "false") { - spark - .range(3) - .selectExpr("id", "TIMESTAMP_NTZ'2000-01-01 00:00:00' AS d") - .coalesce(1) - .write - .parquet(path.toString) + for (pushdownEnabled <- Seq("true", "false")) { + withSQLConf( + "spark.sql.parquet.filterPushdown" -> pushdownEnabled, + "spark.sql.parquet.datetimeRebaseModeInWrite" -> "CORRECTED", + "spark.sql.parquet.datetimeRebaseModeInRead" -> "CORRECTED") { + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(3) + .selectExpr("id", "date_from_unix_date(CAST(id AS INT)) AS d") + .coalesce(1) + .write + .parquet(path.toString) + } + val (_, plan) = + checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) } - val (_, plan) = - checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) - assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) - } - withTempPath { path => - withSQLConf(CometConf.COMET_ENABLED.key -> "false") { - spark - .range(6000) - .selectExpr("id", "date_from_unix_date(IF(id = 5000, 106751992, 0)) AS d") - .coalesce(1) - .write - .parquet(path.toString) + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(3) + .selectExpr("id", "TIMESTAMP_NTZ'2000-01-01 00:00:00' AS d") + .selectExpr( + "id", + "d", + "struct(d) AS s", + "array(d) AS a", + "map(d, id) AS mk", + "map(id, d) AS mv") + .coalesce(1) + .write + .parquet(path.toString) + } + val (_, plan) = + checkSparkAnswerAndOperator(spark.read.schema(schema).parquet(path.toString)) + assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) + // Physical file types are unavailable to the planner. Genuine NTZ columns, including + // nested ones, therefore share the conservative filtered-scan fallback. + for (column <- Seq("d", "s", "a", "mk", "mv")) { + checkSparkAnswerAndFallbackReason( + spark.read.parquet(path.toString).filter("id = 1").select(column), + CometNativeScan.filteredTimestampNTZReason) + } + val (_, prunedPlan) = checkSparkAnswerAndOperator( + spark.read.parquet(path.toString).filter("id = 1").select("id")) + assert(collect(prunedPlan) { case scan: CometNativeScanExec => scan }.nonEmpty) } - val read = () => spark.read.schema(schema).parquet(path.toString) - val (sparkError, cometError) = checkSparkAnswerMaybeThrows(read()) - for (error <- Seq(sparkError, cometError)) { - assert(error.nonEmpty) - assert(causeChain(error.get).exists { - case e: ArithmeticException => e.getMessage == "long overflow" - case _ => false - }) + withTempPath { path => + withSQLConf(CometConf.COMET_ENABLED.key -> "false") { + spark + .range(6000) + .selectExpr( + "id", + "date_from_unix_date(CASE id WHEN 5000 THEN 106751992 " + + "WHEN 5001 THEN -106751992 ELSE 0 END) AS d") + .coalesce(1) + .write + .parquet(path.toString) + } + val read = () => spark.read.schema(schema).parquet(path.toString) + val (sparkError, cometError) = checkSparkAnswerMaybeThrows(read()) + for (error <- Seq(sparkError, cometError)) { + assert(error.nonEmpty) + assert(causeChain(error.get).exists { + case e: ArithmeticException => e.getMessage == "long overflow" + case _ => false + }) + } + for (id <- Seq(5000, 5001)) { + val (sparkError, cometError) = checkSparkAnswerMaybeThrows(read().filter(s"id = $id")) + for ((engine, error) <- Seq("Spark" -> sparkError, "Comet" -> cometError)) { + withClue(s"$engine must reject selected overflowing row $id: ") { + assert(error.nonEmpty) + assert(causeChain(error.get).exists { + case e: ArithmeticException => e.getMessage == "long overflow" + case _ => false + }) + } + } + } + // A later physical DATE overflow must not prevent returning the first valid row when + // Spark prunes or stops before that batch. Eager checked native conversion is unsafe. + checkSparkAnswerAndFallbackReason( + read().filter("id = 0").limit(1), + CometNativeScan.filteredTimestampNTZReason) } - // Retain the existing filtered-scan policy: a later physical DATE overflow must not - // prevent returning the first valid row when Spark prunes or stops before that batch. - val (_, plan) = checkSparkAnswerAndOperator(read().filter("id = 0").limit(1)) - assert(collect(plan) { case scan: CometNativeScanExec => scan }.nonEmpty) } } } From 56f0dfe6cba1ce90389312c24998987bca3a99f4 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 20 Nov 2026 23:29:29 +0000 Subject: [PATCH 11/13] Check hinted Parquet dates and preserve pruning and nested buffers --- native/common/src/struct_nulls.rs | 40 +++- native/core/src/parquet/parquet_exec.rs | 5 +- native/core/src/parquet/parquet_support.rs | 210 ++++++++++-------- native/core/src/parquet/schema_adapter.rs | 80 ++++++- .../spark-expr/src/conversion_funcs/cast.rs | 198 +++++++++-------- .../apache/comet/serde/CometTimeZone.scala | 27 +-- .../comet/parquet/ParquetReadSuite.scala | 21 +- 7 files changed, 363 insertions(+), 218 deletions(-) diff --git a/native/common/src/struct_nulls.rs b/native/common/src/struct_nulls.rs index 64d8c4ece88..c061dc1209d 100644 --- a/native/common/src/struct_nulls.rs +++ b/native/common/src/struct_nulls.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Pushing a struct's null mask down into its children. +//! Propagating parent visibility to nested children. //! //! Arrow stores a `StructArray`'s children with their own validity, independent of the parent's //! null buffer, so at a row where the struct itself is null a child buffer can still hold a value @@ -28,10 +28,11 @@ //! whatever sits in the child slot //! (). -use arrow::array::{make_array, Array, ArrayRef, StructArray}; +use arrow::array::{make_array, Array, ArrayRef, NullBufferBuilder, OffsetSizeTrait, StructArray}; use arrow::buffer::NullBuffer; use arrow::datatypes::DataType; use arrow::error::ArrowError; +use std::cell::LazyCell; use std::sync::Arc; /// The parent's nulls unioned into one child, or the child unchanged when there is nothing to push. @@ -76,6 +77,41 @@ pub fn children_with_parent_nulls(struct_array: &StructArray) -> Result = LazyCell, &'a dyn Fn() -> Option>; + +/// List/map values can include entries outside a slice or beneath a null parent. +pub fn repeated_visibility( + offsets: &[O], + len: usize, + nulls: Option<&NullBuffer>, +) -> Option { + let start = offsets[0].as_usize(); + let end = offsets.last().unwrap().as_usize(); + let mut visible = NullBufferBuilder::new(len); + if start > 0 { + visible.append_n_nulls(start); + } + if let Some(nulls) = nulls.filter(|n| n.null_count() > 0) { + for (valid, range) in nulls.iter().zip(offsets.windows(2)) { + let count = range[1].as_usize() - range[0].as_usize(); + if valid { + visible.append_n_non_nulls(count); + } else if count > 0 { + visible.append_n_nulls(count); + } + } + } else { + visible.append_n_non_nulls(end - start); + } + if end < len { + visible.append_n_nulls(len - end); + } + visible.finish() +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index 8e6aa8b8d73..93f43d5f029 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -324,7 +324,10 @@ pub(crate) fn init_datasource_exec( file_source }; let expr_adapter_factory: Arc = Arc::new( - SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values), + SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values) + .with_pruning_only_filters( + !session_config.options().execution.parquet.pushdown_filters, + ), ); let file_groups = file_groups diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index e7dd162ab21..635340d068c 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -24,7 +24,7 @@ use arrow::array::{ }; use arrow::buffer::NullBuffer; use arrow::compute::{can_cast_types, take}; -use arrow::datatypes::{ArrowPrimitiveType, Date32Type, FieldRef, Fields}; +use arrow::datatypes::{ArrowPrimitiveType, Date32Type, Date64Type, FieldRef, Fields}; use arrow::{ array::{ cast::AsArray, new_null_array, types::TimestampMicrosecondType, @@ -41,6 +41,7 @@ use datafusion::execution::object_store::{ }; use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::physical_plan::ColumnarValue; +use datafusion_comet_common::struct_nulls::{repeated_visibility, Visibility}; use datafusion_comet_common::SparkError; use datafusion_comet_spark_expr::{spark_cast, EvalMode, SparkCastOptions}; use log::debug; @@ -210,7 +211,14 @@ fn parquet_convert_array( to_type: &DataType, parquet_options: &SparkParquetOptions, ) -> DataFusionResult { - parquet_convert_array_impl(array, to_type, parquet_options, None, false).map(|(array, _)| array) + parquet_convert_array_impl( + array, + to_type, + parquet_options, + &Visibility::new(&|| None), + false, + ) + .map(|(array, _)| array) } // The boolean records whether any requested leaf exists in the file schema, independent @@ -221,52 +229,39 @@ fn parquet_convert_array_impl( array: ArrayRef, to_type: &DataType, parquet_options: &SparkParquetOptions, - parent_nulls: Option<&NullBuffer>, + parent_nulls: &Visibility<'_>, in_map_key: bool, ) -> DataFusionResult<(ArrayRef, bool)> { use DataType::*; let from_type = array.data_type(); - // Only checked conversions need a mask for values hidden below a null parent or slice. - let checked_timestamp_overflow = may_read_date_as_ntz(from_type, to_type) - || (parquet_options.checked_timestamp_overflow - && contains_type(from_type, |dt| { - matches!(dt, Timestamp(TimeUnit::Millisecond, _)) - }) - && contains_type(to_type, |dt| { - matches!(dt, Timestamp(TimeUnit::Microsecond, _)) - })); - let visible = if checked_timestamp_overflow { - NullBuffer::union(array.nulls(), parent_nulls) - } else { - None - }; + let visible = || NullBuffer::union(array.nulls(), parent_nulls.as_ref()); // Try Comet specific handlers first, then arrow-rs cast if supported, and fail otherwise. let converted: DataFusionResult = match (from_type, to_type) { - (Struct(_), Struct(_)) => return parquet_convert_struct_to_struct( - array.as_struct(), - from_type, - to_type, - parquet_options, - visible.as_ref(), - in_map_key, - ), + (Struct(_), Struct(_)) => { + return parquet_convert_struct_to_struct( + array.as_struct(), + from_type, + to_type, + parquet_options, + &Visibility::new(&visible), + in_map_key, + ) + } ( List(_) | LargeList(_) | FixedSizeList(_, _) | ListView(_) | LargeListView(_), - List(to_inner_type) | LargeList(to_inner_type) | FixedSizeList(to_inner_type, _) - | ListView(to_inner_type) | LargeListView(to_inner_type), + List(to_inner_type) + | LargeList(to_inner_type) + | FixedSizeList(to_inner_type, _) + | ListView(to_inner_type) + | LargeListView(to_inner_type), ) => { let data = array.to_data(); - let child_visibility = if checked_timestamp_overflow { - list_child_visibility(array.as_ref(), visible.as_ref()) - } else { - None - }; let (cast_field, field_overlap) = parquet_convert_array_impl( make_array(data.child_data()[0].clone()), to_inner_type.data_type(), parquet_options, - child_visibility.as_ref(), + &Visibility::new(&|| list_child_visibility(array.as_ref(), visible().as_ref())), in_map_key, )?; if !field_overlap && parquet_options.return_null_struct_if_all_fields_missing { @@ -283,10 +278,12 @@ fn parquet_convert_array_impl( _ => unreachable!(), }; // Retain the source offsets, sizes and null buffer while replacing its values. - let resolved = make_array(data.into_builder() - .data_type(resolved_type) - .child_data(vec![cast_field.to_data()]) - .build()?); + let resolved = make_array( + data.into_builder() + .data_type(resolved_type) + .child_data(vec![cast_field.to_data()]) + .build()?, + ); let output = if resolved.data_type() == to_type { resolved } else { @@ -294,10 +291,9 @@ fn parquet_convert_array_impl( }; return Ok((output, field_overlap)); } - ( - Timestamp(TimeUnit::Millisecond, _), - Timestamp(TimeUnit::Microsecond, target_tz), - ) if checked_timestamp_overflow => { + (Timestamp(TimeUnit::Millisecond, _), Timestamp(TimeUnit::Microsecond, target_tz)) + if parquet_options.checked_timestamp_overflow => + { // Spark's Parquet reader calls the checked `millisToMicros` conversion for both // direct and dictionary values, independent of CAST evaluation mode: // https://github.com/apache/spark/blob/v4.2.0/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorUpdaterFactory.java#L817-L833 @@ -308,14 +304,23 @@ fn parquet_convert_array_impl( // pruning paths, including nested predicates. checked_micros( array.as_primitive::(), - visible, + visible(), 1_000, target_tz.clone(), ) } (Date32, Timestamp(TimeUnit::Microsecond, None)) => { // Spark 4.x's DateToTimestampNTZUpdater calls daysToMicros at UTC. - checked_micros(array.as_primitive::(), visible, 86_400_000_000, None) + checked_micros( + array.as_primitive::(), + visible(), + 86_400_000_000, + None, + ) + } + (Date64, Timestamp(TimeUnit::Microsecond, None)) => { + // ARROW:schema can restore a physical INT32 DATE as milliseconds. + checked_micros(array.as_primitive::(), visible(), 1_000, None) } (Dictionary(_, values), _) if may_read_date_as_ntz(values, to_type) => { // An ARROW:schema dictionary can include days no row references. @@ -337,9 +342,16 @@ fn parquet_convert_array_impl( .with_timezone(Arc::clone(tz)), )) } - (Map(_, ordered_from), Map(_, ordered_to)) if ordered_from == ordered_to => - return parquet_convert_map_to_map(array.as_map(), to_type, parquet_options, *ordered_to, visible.as_ref(), checked_timestamp_overflow, in_map_key) - , + (Map(_, ordered_from), Map(_, ordered_to)) if ordered_from == ordered_to => { + return parquet_convert_map_to_map( + array.as_map(), + to_type, + parquet_options, + *ordered_to, + &Visibility::new(&visible), + in_map_key, + ); + } // Iceberg stores UUIDs as 16-byte fixed binary but Spark expects string representation. // Arrow doesn't support casting FixedSizeBinary to Utf8, so we handle it manually. (FixedSizeBinary(16), Utf8) => { @@ -352,9 +364,8 @@ fn parquet_convert_array_impl( .iter() .map(|opt_bytes| { opt_bytes.map(|bytes| { - let uuid = uuid::Uuid::from_bytes( - bytes.try_into().expect("Expected 16 bytes") - ); + let uuid = + uuid::Uuid::from_bytes(bytes.try_into().expect("Expected 16 bytes")); uuid.to_string() }) }) @@ -393,7 +404,9 @@ fn parquet_convert_array_impl( // the batch into one string array with 32-bit offsets, so a batch whose rows reference // wide entries many times can decode past `i32::MAX` bytes and fail where Spark, which // keeps the dictionary, succeeds (#7254). - (Dictionary(_, value_type), Utf8) if matches!(value_type.as_ref(), Binary | LargeBinary) => { + (Dictionary(_, value_type), Utf8) + if matches!(value_type.as_ref(), Binary | LargeBinary) => + { let dictionary = array.as_any_dictionary(); let keys = dictionary.normalized_keys(); let key_nulls = dictionary.keys().nulls(); @@ -401,7 +414,7 @@ fn parquet_convert_array_impl( referenced_dictionary_values(dictionary.values(), &keys, key_nulls)?, to_type, parquet_options, - None, + &Visibility::new(&|| None), in_map_key, )?; let decoded_bytes = @@ -453,50 +466,29 @@ where Ok(Arc::new(micros)) } -pub(super) fn contains_ntz(data_type: &DataType) -> bool { - contains_type(data_type, |dt| { - matches!(dt, DataType::Timestamp(TimeUnit::Microsecond, None)) - }) -} - -// Struct fields are matched by name/field ID later. This type-only check is conservative -// until that matching occurs; each selected child is checked again before conversion. pub(super) fn may_read_date_as_ntz(physical_type: &DataType, target_type: &DataType) -> bool { - contains_ntz(target_type) && contains_type(physical_type, |dt| matches!(dt, DataType::Date32)) + match (physical_type, target_type) { + (DataType::Date32 | DataType::Date64, DataType::Timestamp(TimeUnit::Microsecond, None)) => { + true + } + (DataType::Dictionary(_, value), _) => may_read_date_as_ntz(value, target_type), + _ => false, + } } -fn contains_type(data_type: &DataType, matches: fn(&DataType) -> bool) -> bool { +pub(super) fn contains_ntz(data_type: &DataType) -> bool { match data_type { - DataType::Struct(fields) => fields - .iter() - .any(|field| contains_type(field.data_type(), matches)), + DataType::Timestamp(TimeUnit::Microsecond, None) => true, + DataType::Struct(fields) => fields.iter().any(|field| contains_ntz(field.data_type())), DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) | DataType::ListView(field) | DataType::LargeListView(field) - | DataType::Map(field, _) => contains_type(field.data_type(), matches), - DataType::Dictionary(_, values) => contains_type(values, matches), - _ => matches(data_type), - } -} - -// List/map values can include entries outside a slice or beneath a null parent. -fn repeated_visibility( - offsets: &[O], - len: usize, - nulls: Option<&NullBuffer>, -) -> Option { - if nulls.is_none() && offsets[0].as_usize() == 0 && offsets.last().unwrap().as_usize() == len { - return None; - } - let mut valid = vec![false; len]; - for (row, range) in offsets.windows(2).enumerate() { - if nulls.is_none_or(|nulls| nulls.is_valid(row)) { - valid[range[0].as_usize()..range[1].as_usize()].fill(true); - } + | DataType::Map(field, _) => contains_ntz(field.data_type()), + DataType::Dictionary(_, values) => contains_ntz(values), + _ => false, } - Some(NullBuffer::from(valid)) } fn list_child_visibility(array: &dyn Array, nulls: Option<&NullBuffer>) -> Option { @@ -592,7 +584,7 @@ fn parquet_convert_struct_to_struct( from_type: &DataType, to_type: &DataType, parquet_options: &SparkParquetOptions, - parent_nulls: Option<&NullBuffer>, + parent_nulls: &Visibility<'_>, in_map_key: bool, ) -> DataFusionResult<(ArrayRef, bool)> { match (from_type, to_type) { @@ -647,8 +639,7 @@ fn parquet_convert_map_to_map( to_data_type: &DataType, parquet_options: &SparkParquetOptions, to_ordered: bool, - parent_nulls: Option<&NullBuffer>, - checked_timestamp_overflow: bool, + parent_nulls: &Visibility<'_>, in_map_key: bool, ) -> Result<(ArrayRef, bool), DataFusionError> { match to_data_type { @@ -660,23 +651,26 @@ fn parquet_convert_map_to_map( "map is missing value field".to_string(), ))?; - let child_visibility = if checked_timestamp_overflow { - repeated_visibility(from.value_offsets(), from.keys().len(), parent_nulls) - } else { - None + let mask = || { + repeated_visibility( + from.value_offsets(), + from.keys().len(), + parent_nulls.as_ref(), + ) }; + let child_visibility = Visibility::new(&mask); let (key_array, key_overlap) = parquet_convert_array_impl( Arc::clone(from.keys()), key_field.data_type(), parquet_options, - child_visibility.as_ref(), + &child_visibility, true, )?; let (value_array, value_overlap) = parquet_convert_array_impl( Arc::clone(from.values()), value_field.data_type(), parquet_options, - child_visibility.as_ref(), + &child_visibility, in_map_key, )?; let field_overlap = key_overlap || value_overlap; @@ -2580,6 +2574,34 @@ mod tests { } } + #[test] + fn widening_sibling_of_date_and_ntz_skips_parent_visibility() { + use super::{parquet_convert_array_impl, SparkParquetOptions, Visibility}; + use arrow::array::new_null_array; + use arrow::datatypes::{DataType, Field, TimeUnit}; + use datafusion_comet_spark_expr::EvalMode; + + let list = |int| { + let fields = vec![ + Field::new("d", DataType::Date32, true), + Field::new("t", DataType::Timestamp(TimeUnit::Microsecond, None), true), + Field::new("i", int, true), + ]; + DataType::new_list(DataType::Struct(fields.into()), true) + }; + let array = new_null_array(&list(DataType::Int32), 1); + let target = list(DataType::Int64); + let opts = SparkParquetOptions::new(EvalMode::Legacy, "UTC"); + parquet_convert_array_impl( + array, + &target, + &opts, + &Visibility::new(&|| panic!("no leaf needs the parent mask")), + false, + ) + .unwrap(); + } + #[test] fn test_date_to_ntz_checked_overflow_and_hidden_values() { use super::{parquet_convert_array, SparkParquetOptions}; diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index a260c8b7f58..595c748c08a 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -32,7 +32,7 @@ use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion}; use datafusion::common::{DataFusionError, Result as DataFusionResult}; -use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr::expressions::{Column, IsNotNullExpr, IsNullExpr}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; use datafusion::scalar::ScalarValue; @@ -55,6 +55,7 @@ use std::sync::Arc; /// to inject Spark-compatible casts where needed. #[derive(Clone, Debug)] pub struct SparkPhysicalExprAdapterFactory { + pruning_only_filters: bool, /// Spark-specific parquet options for type conversions parquet_options: SparkParquetOptions, /// Default values for columns that may be missing from the physical schema. @@ -69,10 +70,17 @@ impl SparkPhysicalExprAdapterFactory { default_values: Option>, ) -> Self { Self { + pruning_only_filters: false, parquet_options, default_values, } } + + /// Enable only when the scan projects columns and uses predicates solely for pruning. + pub(super) fn with_pruning_only_filters(mut self, enabled: bool) -> Self { + self.pruning_only_filters = enabled; + self + } } /// Returns true when casting `physical_type` to `target_type` is a *pure* structural @@ -388,7 +396,7 @@ fn parquet_primitive_name(dt: &DataType) -> &'static str { DataType::Float64 => "DOUBLE", DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary => "BINARY", // Spark stores DATE as INT32 with a DATE logical-type annotation. - DataType::Date32 => "INT32", + DataType::Date32 | DataType::Date64 => "INT32", // Spark stores TIMESTAMP as INT64 with a timestamp annotation, or as // INT96 (legacy nanos). arrow-rs surfaces both as `Timestamp`; without // the original physical name we report INT64, which matches the @@ -620,7 +628,10 @@ fn spark_has_updater( (DataType::Int32, DataType::Int64) | (DataType::Float32, DataType::Float64) | (DataType::Int32, DataType::Float64) - | (DataType::Date32, DataType::Timestamp(_, None)) + | ( + DataType::Date32 | DataType::Date64, + DataType::Timestamp(_, None) + ) ); if is_disallowed_promotion { return false; @@ -1092,6 +1103,7 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { ); Ok(Arc::new(SparkPhysicalExprAdapter { + pruning_only_filters: self.pruning_only_filters, logical_file_schema, physical_file_schema: adapted_physical_schema, parquet_options: self.parquet_options.clone(), @@ -1116,6 +1128,7 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { /// 3. Handle case-insensitive column matching #[derive(Debug)] struct SparkPhysicalExprAdapter { + pruning_only_filters: bool, /// The logical schema expected by the query logical_file_schema: SchemaRef, /// The physical schema of the actual file being read @@ -1405,11 +1418,34 @@ impl SparkPhysicalExprAdapter { } } } - Ok(Transformed::no(e)) + self.adapt_date_null_check(e) }) .data() } + /// DataFusion requires a bare column to prune by null count. DATE -> NTZ preserves + /// validity, so use the physical column for these statistics. The scan separately + /// adapts the projected column and still checks every decoded day for overflow. + fn adapt_date_null_check( + &self, + expr: Arc, + ) -> DataFusionResult>> { + if self.pruning_only_filters && (expr.is::() || expr.is::()) { + let cast = expr.children()[0]; + if cast.is::() { + let source = cast.children()[0]; + if may_read_date_as_ntz( + &source.data_type(&self.physical_file_schema)?, + &cast.data_type(&self.physical_file_schema)?, + ) { + let source = Arc::clone(source); + return Ok(Transformed::yes(expr.with_new_children(vec![source])?)); + } + } + } + Ok(Transformed::no(expr)) + } + /// Replace CastExpr (DataFusion's cast) with Spark's Cast expression. /// /// Only the casts that reconcile a file column with its table-schema type are Parquet read @@ -1601,7 +1637,7 @@ impl SparkPhysicalExprAdapter { )?)); } - Ok(Transformed::no(expr)) + self.adapt_date_null_check(expr) } /// Replace missing columns with defaults cast to the logical schema, propagating conversion @@ -2706,6 +2742,40 @@ pub(crate) mod test { Ok(()) } + #[tokio::test] + async fn parquet_date_with_date64_hint_read_as_ntz() -> Result<(), DataFusionError> { + use arrow::array::Date64Array; + use parquet::file::properties::WriterProperties; + + // Coercion writes INT32 DATE while the Arrow footer hint retains Date64. + let days = i64::MAX / 86_400_000_000 + 1; + let values: ArrayRef = Arc::new(Date64Array::from(vec![days * 86_400_000])); + let batch = RecordBatch::try_from_iter([("d", values)])?; + let props = WriterProperties::builder().set_coerce_types(true).build(); + let filename = get_temp_filename().to_str().unwrap().to_string(); + let mut writer = + ArrowWriter::try_new(File::create(&filename)?, batch.schema(), Some(props))?; + writer.write(&batch)?; + writer.close()?; + + let ntz = DataType::Timestamp(TimeUnit::Microsecond, None); + let required = Arc::new(Schema::new(vec![Field::new("d", ntz, false)])); + for (promote, expected) in [(true, "long overflow"), (false, "Found: INT32")] { + let mut options = default_options(); + options.allow_type_promotion = promote; + let mut stream = scan_parquet_file( + filename.clone(), + Arc::clone(&required), + options, + None, + false, + )?; + let error = stream.next().await.unwrap().unwrap_err().to_string(); + assert!(error.contains(expected), "unexpected error: {error}"); + } + Ok(()) + } + /// `Date32 -> Timestamp(_, Some(_))` (LTZ). Spark's vectorized reader /// allows `Date -> TimestampNTZ` but not `Date -> Timestamp(LTZ)`. #[tokio::test] diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 2a858334953..a7aa33cdf2c 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -44,8 +44,8 @@ use crate::{cast_whole_num_to_binary, BinaryOutputStyle}; use crate::{EvalMode, SparkError}; use arrow::array::builder::{GenericStringBuilder, StringBuilder}; use arrow::array::{ - new_null_array, BinaryBuilder, GenericByteArray, GenericStringArray, ListArray, MapArray, - StringArray, StructArray, + new_null_array, BinaryBuilder, BooleanArray, Date32Array, GenericByteArray, GenericStringArray, + ListArray, MapArray, StringArray, StructArray, }; use arrow::buffer::{BooleanBuffer, Buffer, NullBuffer, OffsetBuffer}; use arrow::datatypes::{ @@ -56,9 +56,9 @@ use arrow::error::ArrowError; use arrow::{ array::{ cast::AsArray, types::Decimal128Type, Array, ArrayRef, Int16Array, Int32Array, Int64Array, - Int8Array, OffsetSizeTrait, UInt64Array, + Int8Array, OffsetSizeTrait, }, - compute::{cast_with_options, rescale_decimal, take, CastOptions}, + compute::{cast_with_options, kernels::zip::zip, rescale_decimal, CastOptions}, record_batch::RecordBatch, util::display::FormatOptions, }; @@ -67,7 +67,8 @@ use base64::Engine; use datafusion::common::{internal_err, DataFusionError, Result as DataFusionResult, ScalarValue}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; -use datafusion_comet_common::{child_with_parent_nulls, decode_utf8_spark_lossy}; +use datafusion_comet_common::decode_utf8_spark_lossy; +use datafusion_comet_common::struct_nulls::{repeated_visibility, Visibility}; use std::{ fmt::{Debug, Display, Formatter, Write}, hash::Hash, @@ -265,6 +266,15 @@ pub(crate) fn cast_array( array: ArrayRef, to_type: &DataType, cast_options: &SparkCastOptions, +) -> DataFusionResult { + cast_array_impl(array, to_type, cast_options, &Visibility::new(&|| None)) +} + +fn cast_array_impl( + array: ArrayRef, + to_type: &DataType, + cast_options: &SparkCastOptions, + parent_visibility: &Visibility<'_>, ) -> DataFusionResult { use DataType::*; let from_type = array.data_type().clone(); @@ -302,6 +312,20 @@ pub(crate) fn cast_array( } } + let child_visibility = || { + let visible = NullBuffer::union(array.nulls(), parent_visibility.as_ref()); + match &from_type { + List(_) => { + let list = array.as_list::(); + repeated_visibility(list.value_offsets(), list.values().len(), visible.as_ref()) + } + Map(_, _) => { + let map = array.as_map(); + repeated_visibility(map.value_offsets(), map.entries().len(), visible.as_ref()) + } + _ => visible, + } + }; let cast_result = match (&from_type, to_type) { // Null arrays carry no concrete values, so Arrow's native cast can change only the // logical type while preserving length and nullness. @@ -440,20 +464,21 @@ pub(crate) fn cast_array( &from_type, to_type, cast_options, + &Visibility::new(&child_visibility), )?), (List(_), Utf8) => Ok(cast_array_to_string(array.as_list(), cast_options)?), (List(_), List(to)) => { - let array = prepare_repeated_date_cast(array, to_type)?; // Cast list elements recursively so nested array casts follow Spark semantics // instead of relying on Arrow's top-level cast support. let list_array = array.as_list::(); let casted_values = match (list_array.values().data_type(), to.data_type()) { // Spark legacy array casts produce null elements for array -> array. (Date32, Int32) => new_null_array(to.data_type(), list_array.values().len()), - _ => cast_array( + _ => cast_array_impl( Arc::clone(list_array.values()), to.data_type(), cast_options, + &Visibility::new(&child_visibility), )?, }; Ok(Arc::new(ListArray::new( @@ -463,17 +488,31 @@ pub(crate) fn cast_array( list_array.nulls().cloned(), )) as ArrayRef) } - (Map(_, _), Map(_, _)) => { - let array = prepare_repeated_date_cast(array, to_type)?; - Ok(cast_map_to_map(&array, &from_type, to_type, cast_options)?) - } + (Map(_, _), Map(_, _)) => Ok(cast_map_to_map( + &array, + &from_type, + to_type, + cast_options, + &Visibility::new(&child_visibility), + )?), (UInt8 | UInt16 | UInt32 | UInt64, Int8 | Int16 | Int32 | Int64) if cast_options.allow_cast_unsigned_ints => { Ok(cast_with_options(&array, to_type, &CAST_OPTIONS)?) } (Binary, Utf8) => Ok(cast_binary_to_string::(&array, cast_options)?), - (Date32, Timestamp(_, tz)) => Ok(cast_date_to_timestamp(&array, cast_options, tz)?), + (Date32, Timestamp(_, tz)) => { + // An invisible date can still be required, e.g. a map key. Use the epoch as its + // placeholder so both checked and TRY casts preserve required child validity. + let dates = + if let Some(visible) = parent_visibility.as_ref().filter(|n| n.null_count() != 0) { + let visible = BooleanArray::new(visible.inner().clone(), None); + zip(&visible, &array, &Date32Array::new_scalar(0))? + } else { + array + }; + Ok(cast_date_to_timestamp(&dates, cast_options, tz)?) + } (Int8, Binary) if (eval_mode == Legacy) => cast_whole_num_to_binary!(&array, Int8Array, 1), (Int16, Binary) if (eval_mode == Legacy) => { cast_whole_num_to_binary!(&array, Int16Array, 2) @@ -559,52 +598,6 @@ fn is_datafusion_spark_compatible(from_type: &DataType, to_type: &DataType) -> b } } -fn casts_date_to_timestamp(from: &DataType, to: &DataType) -> bool { - use DataType::*; - match (from, to) { - (Date32, Timestamp(_, _)) => true, - (List(from), List(to)) | (Map(from, _), Map(to, _)) => { - casts_date_to_timestamp(from.data_type(), to.data_type()) - } - (Struct(from), Struct(to)) => from - .iter() - .zip(to) - .any(|(from, to)| casts_date_to_timestamp(from.data_type(), to.data_type())), - _ => false, - } -} - -/// List and map casts visit the whole child array, including entries outside a slice and -/// entries owned by null rows. Drop those entries before checking for date overflow. -fn prepare_repeated_date_cast(array: ArrayRef, to_type: &DataType) -> DataFusionResult { - let (offsets, len) = match array.data_type() { - DataType::List(_) => { - let list = array.as_list::(); - (list.value_offsets(), list.values().len()) - } - DataType::Map(_, _) => { - let map = array.as_map(); - (map.value_offsets(), map.entries().len()) - } - _ => unreachable!("expected a list or map"), - }; - let sliced = offsets[0] != 0 || offsets[array.len()] as usize != len; - if (!sliced && array.null_count() == 0) || !casts_date_to_timestamp(array.data_type(), to_type) - { - return Ok(array); - } - if !sliced && (0..array.len()).all(|i| array.is_valid(i) || offsets[i] == offsets[i + 1]) { - return Ok(array); - } - - // Null indices omit their entries and preserve non-nullable map keys. - let indices = UInt64Array::new( - (0..array.len() as u64).collect::>().into(), - array.nulls().cloned(), - ); - Ok(take(array.as_ref(), &indices, None)?) -} - /// Cast between struct types based on logic in /// `org.apache.spark.sql.catalyst.expressions.Cast#castStruct`. fn cast_struct_to_struct( @@ -612,28 +605,21 @@ fn cast_struct_to_struct( from_type: &DataType, to_type: &DataType, cast_options: &SparkCastOptions, + child_visibility: &Visibility<'_>, ) -> DataFusionResult { match (from_type, to_type) { - (DataType::Struct(from_fields), DataType::Struct(to_fields)) => { - let cast_fields: Vec = from_fields + (DataType::Struct(_), DataType::Struct(to_fields)) => { + let cast_fields = array + .columns() .iter() - .enumerate() - .zip(to_fields.iter()) - .map(|((idx, from), to)| { - let from_field = if array.null_count() != 0 - && casts_date_to_timestamp(from.data_type(), to.data_type()) - { - child_with_parent_nulls(array, idx)? - } else { - Arc::clone(array.column(idx)) - }; - let array_length = from_field.len(); - let cast_result = spark_cast( - ColumnarValue::from(from_field), - to.data_type(), + .zip(to_fields) + .map(|(child, field)| { + cast_array_impl( + Arc::clone(child), + field.data_type(), cast_options, - )?; - cast_result.to_array(array_length) + child_visibility, + ) }) .collect::>>()?; @@ -667,6 +653,7 @@ fn cast_map_to_map( from_type: &DataType, to_type: &DataType, cast_options: &SparkCastOptions, + child_visibility: &Visibility<'_>, ) -> DataFusionResult { let map_array = array .as_any() @@ -729,13 +716,23 @@ fn cast_map_to_map( let cast_keys = if key_type_unchanged { Arc::clone(keys) } else { - cast_array(Arc::clone(keys), to_fields[0].data_type(), cast_options)? + cast_array_impl( + Arc::clone(keys), + to_fields[0].data_type(), + cast_options, + child_visibility, + )? }; let values = map_array.values(); let cast_values = if value_type_unchanged { Arc::clone(values) } else { - cast_array(Arc::clone(values), to_fields[1].data_type(), cast_options)? + cast_array_impl( + Arc::clone(values), + to_fields[1].data_type(), + cast_options, + child_visibility, + )? }; // `None` rather than the source entries null buffer, which is always absent. @@ -2095,35 +2092,34 @@ mod tests { use arrow::array::{Date32Array, TimestampMicrosecondArray}; // One child per row, even when the struct/list/map is null. - fn containers(values: ArrayRef) -> [ArrayRef; 3] { - let keys = Arc::new(Field::new("key", values.data_type().clone(), false)); + fn containers(values: ArrayRef) -> [ArrayRef; 4] { let field = Arc::new(Field::new("value", values.data_type().clone(), false)); let offsets = OffsetBuffer::new(vec![0, 1, 2, 3, 4, 5].into()); let nulls = Some(NullBuffer::from(vec![true, true, false, true, true])); - let entries = StructArray::new( - vec![keys, Arc::clone(&field)].into(), - vec![Arc::clone(&values), Arc::clone(&values)], - None, - ); + let map = |keys: ArrayRef| -> ArrayRef { + let key_field = Arc::new(Field::new("key", keys.data_type().clone(), false)); + let entries = StructArray::new( + vec![key_field, Arc::clone(&field)].into(), + vec![keys, Arc::clone(&values)], + None, + ); + Arc::new(MapArray::new( + Arc::new(Field::new("entries", entries.data_type().clone(), false)), + offsets.clone(), + entries, + nulls.clone(), + false, + )) + }; [ + map(Arc::clone(&values)), + map(Arc::new(StringArray::from(vec!["a", "b", "c", "d", "e"]))), Arc::new(StructArray::new( vec![Arc::clone(&field)].into(), vec![Arc::clone(&values)], nulls.clone(), )), - Arc::new(ListArray::new( - field, - offsets.clone(), - values, - nulls.clone(), - )), - Arc::new(MapArray::new( - Arc::new(Field::new("entries", entries.data_type().clone(), false)), - offsets, - entries, - nulls, - false, - )), + Arc::new(ListArray::new(field, offsets, values, nulls)), ] } @@ -2149,6 +2145,12 @@ mod tests { let actual = cast(dates.slice(start, len)).unwrap(); actual.to_data().validate_full().unwrap(); assert_eq!(actual.to_data(), expected.slice(start, len).to_data()); + if let Some(map) = dates + .as_map_opt() + .filter(|m| m.keys().data_type() == &DataType::Utf8) + { + assert!(Arc::ptr_eq(actual.as_map().keys(), map.keys())); + } } if mode != EvalMode::Try { let error = cast(dates.slice(1, 4)).unwrap_err(); diff --git a/spark/src/main/scala/org/apache/comet/serde/CometTimeZone.scala b/spark/src/main/scala/org/apache/comet/serde/CometTimeZone.scala index 3aa0908eca1..64f7a763065 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometTimeZone.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometTimeZone.scala @@ -40,24 +40,19 @@ object CometTimeZone { * offset becomes `UTC`, and a short ID becomes its region. An expression with no timezone gets * `UTC`: Spark leaves it unset only on casts that do not use it. */ - def nativeId(timeZoneId: Option[String]): Option[String] = - nativeId(timeZoneId, allowRegion = true) + def nativeId(timeZoneId: Option[String]): Option[String] = timeZoneId match { + case None => Some("UTC") + case Some(id) => + Try(DateTimeUtils.getZoneId(id).normalized()).toOption.flatMap { + case offset: ZoneOffset if offset.getTotalSeconds == 0 => Some("UTC") + case offset: ZoneOffset if offset.getTotalSeconds % 60 == 0 => Some(offset.getId) + case _: ZoneOffset => None + case region => Some(region.getId) + } + } private[comet] def hasNativeFixedOffset(timeZoneId: Option[String]): Boolean = - nativeId(timeZoneId, allowRegion = false).isDefined - - private def nativeId(timeZoneId: Option[String], allowRegion: Boolean): Option[String] = - timeZoneId match { - case None => Some("UTC") - case Some(id) => - Try(DateTimeUtils.getZoneId(id).normalized()).toOption.flatMap { - case offset: ZoneOffset if offset.getTotalSeconds == 0 => Some("UTC") - case offset: ZoneOffset if offset.getTotalSeconds % 60 == 0 => Some(offset.getId) - case _: ZoneOffset => None - case region if allowRegion => Some(region.getId) - case _ => None - } - } + nativeId(timeZoneId).exists(id => id == "UTC" || id.startsWith("+") || id.startsWith("-")) /** Whether `timeZoneId` has a zero offset at every instant. */ def isUtc(timeZoneId: Option[String]): Boolean = nativeId(timeZoneId).contains("UTC") diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 67f0fcd7094..8cc48e35d40 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -367,9 +367,12 @@ abstract class ParquetReadSuite extends CometTestBase { | optional int32 d(DATE); | } | optional int64 stored(TIMESTAMP(MICROS,false)); + | optional int32 probe(DATE); |} |""".stripMargin) - val readSchema = "id INT, d TIMESTAMP_NTZ, s STRUCT, stored TIMESTAMP_NTZ" + val readSchema = + "id INT, d TIMESTAMP_NTZ, s STRUCT, " + + "stored TIMESTAMP_NTZ, probe TIMESTAMP_NTZ" withTempDir { dir => val path = new Path(dir.toURI.toString, "days.parquet") // Keep the positive and negative overflows in distinct row groups, away from id = 1. @@ -383,9 +386,10 @@ abstract class ParquetReadSuite extends CometTestBase { } val record = new SimpleGroup(schema) record.add(0, id) - record.add(1, day) + if (id % 3 != 0) record.add(1, day) record.addGroup(2).add(0, day) record.add(3, id.toLong) + if (id < 1500) record.add(4, id) writer.write(record) } } finally { @@ -400,6 +404,19 @@ abstract class ParquetReadSuite extends CometTestBase { checkSparkAnswerAndOperator(df.where("id = 1000").select("stored")) if (isSpark40Plus) { checkSparkAnswerAndOperator(df.where("id = 1")) + if (!pushdown) { + // Null counts on probe must exclude the group at 1000 for IS NULL and the + // group at 2000 for IS NOT NULL before the reader converts the nested date. + checkSparkAnswerAndOperator(df.where("id <= 1500 AND probe IS NULL").select("s")) + checkSparkAnswerAndOperator( + df.where("id >= 1499 AND probe IS NOT NULL").select("s")) + // Both overflow groups contain nulls in d. The reader must convert d in + // those groups before the residual filter selects its nulls. + val error = + checkSparkError(df.where("d IS NULL").select("id"), "FAILED_READ_FILE.NO_HINT") + assert(error.getCause.getClass == classOf[ArithmeticException]) + assert(error.getCause.getMessage == "long overflow") + } for (id <- Seq(1000, 2000); column <- Seq("d", "s")) { val selected = df.where(s"id = $id").select(column) val error = checkSparkError(selected, "FAILED_READ_FILE.NO_HINT") From a27c2ee7a6f89448196889ff31f65053965cf512 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 20 Nov 2026 23:48:32 +0000 Subject: [PATCH 12/13] Distinguish physical dates from hinted integers and prune before updater selection --- native/common/src/struct_nulls.rs | 57 +++++- .../eager_page_index_reader_factory.rs | 62 +++++- native/core/src/parquet/parquet_exec.rs | 7 +- native/core/src/parquet/parquet_support.rs | 193 ++++++------------ native/core/src/parquet/schema_adapter.rs | 71 ++++--- .../spark-expr/src/conversion_funcs/cast.rs | 24 +-- .../comet/parquet/ParquetReadSuite.scala | 4 + 7 files changed, 216 insertions(+), 202 deletions(-) diff --git a/native/common/src/struct_nulls.rs b/native/common/src/struct_nulls.rs index c061dc1209d..16be6a99769 100644 --- a/native/common/src/struct_nulls.rs +++ b/native/common/src/struct_nulls.rs @@ -28,7 +28,10 @@ //! whatever sits in the child slot //! (). -use arrow::array::{make_array, Array, ArrayRef, NullBufferBuilder, OffsetSizeTrait, StructArray}; +use arrow::array::{ + make_array, Array, ArrayRef, AsArray, GenericListViewArray, NullBufferBuilder, OffsetSizeTrait, + StructArray, +}; use arrow::buffer::NullBuffer; use arrow::datatypes::DataType; use arrow::error::ArrowError; @@ -82,8 +85,56 @@ pub fn children_with_parent_nulls(struct_array: &StructArray) -> Result = LazyCell, &'a dyn Fn() -> Option>; -/// List/map values can include entries outside a slice or beneath a null parent. -pub fn repeated_visibility( +/// Map the visible slots of a container to the indices of its children. +pub fn child_visibility(array: &dyn Array, parent: &Visibility<'_>) -> Option { + let visible = NullBuffer::union(array.nulls(), parent.as_ref()); + let nulls = visible.as_ref(); + match array.data_type() { + DataType::List(_) => { + let list = array.as_list::(); + repeated_visibility(list.value_offsets(), list.values().len(), nulls) + } + DataType::LargeList(_) => { + let list = array.as_list::(); + repeated_visibility(list.value_offsets(), list.values().len(), nulls) + } + DataType::FixedSizeList(_, size) => { + // Arrow slices the child values along with a fixed-size list. Expand in those + // child coordinates; applying a parent slice offset again would shift the mask. + nulls.map(|nulls| nulls.expand(*size as usize)) + } + DataType::ListView(_) => list_view_visibility(array.as_list_view::(), nulls), + DataType::LargeListView(_) => list_view_visibility(array.as_list_view::(), nulls), + DataType::Map(_, _) => { + let map = array.as_map(); + repeated_visibility(map.value_offsets(), map.entries().len(), nulls) + } + _ => visible, + } +} + +fn list_view_visibility( + list: &GenericListViewArray, + nulls: Option<&NullBuffer>, +) -> Option { + let mut valid = vec![false; list.values().len()]; + for (row, (offset, size)) in list + .value_offsets() + .iter() + .zip(list.value_sizes()) + .enumerate() + { + if nulls.is_none_or(|nulls| nulls.is_valid(row)) { + let start = offset.as_usize(); + // Views can overlap or be out of order. Accumulate the union of visible ranges; + // a null parent must never clear values another parent can see. + valid[start..start + size.as_usize()].fill(true); + } + } + Some(NullBuffer::from(valid)) +} + +fn repeated_visibility( offsets: &[O], len: usize, nulls: Option<&NullBuffer>, diff --git a/native/core/src/parquet/eager_page_index_reader_factory.rs b/native/core/src/parquet/eager_page_index_reader_factory.rs index 00220e26666..7561b05ed2c 100644 --- a/native/core/src/parquet/eager_page_index_reader_factory.rs +++ b/native/core/src/parquet/eager_page_index_reader_factory.rs @@ -94,7 +94,7 @@ //! bring it back. use crate::parquet::datetime_rebase::stamp_int96_leaves; -use crate::parquet::parquet_support::SparkParquetOptions; +use crate::parquet::parquet_support::{contains_ntz, contains_type, SparkParquetOptions}; use crate::parquet::schema_adapter::{ check_file_conversions, RejectOnNonEmpty, NANOS_TIMESTAMP_KEY, REPEATED_PRIMITIVE_KEY, }; @@ -482,19 +482,21 @@ fn spark_enum_schema(schema: &SchemaDescriptor) -> ParquetResult> ))) } +fn has_arrow_schema(file: &FileMetaData) -> bool { + file.key_value_metadata() + .into_iter() + .flatten() + .any(|kv| kv.key == ARROW_SCHEMA_META_KEY) +} + /// Arrow restores advisory `ARROW:schema` types that can differ from Spark's physical Parquet /// interpretation. Replace that hint with physical inference and the ENUM string mapping. /// Rebuild only the returned metadata; the shared cache retains the original footer. /// https://github.com/apache/datafusion-comet/issues/5477 fn with_spark_arrow_schema(metadata: Arc) -> ParquetResult> { let file = metadata.file_metadata(); - let has_arrow_schema = file.key_value_metadata().is_some_and(|key_values| { - key_values - .iter() - .any(|key_value| key_value.key == ARROW_SCHEMA_META_KEY) - }); let enum_schema = spark_enum_schema(file.schema_descr())?; - if !has_arrow_schema && enum_schema.is_none() { + if !has_arrow_schema(file) && enum_schema.is_none() { return Ok(metadata); } @@ -540,6 +542,32 @@ fn is_nanos_timestamp(column: &ColumnDescPtr) -> bool { ) } +fn is_plain_int64(column: &ColumnDescPtr) -> bool { + column.physical_type() == parquet::basic::Type::INT64 + && matches!( + column.converted_type(), + ConvertedType::NONE | ConvertedType::INT_64 + ) + && !matches!(column.logical_type_ref(), Some(LogicalType::Timestamp(_))) +} + +// Only parse the hint when a requested NTZ could read a plain INT64. The adapter +// resolves exact nested paths, field IDs and Unicode names after the hint check. +fn may_read_hinted_int64(file: &FileMetaData, required: &Schema, use_field_id: bool) -> bool { + has_arrow_schema(file) + && required.fields().iter().any(|field| { + contains_ntz(field.data_type()) + && file.schema_descr().columns().iter().any(|column| { + let root = &column.path().parts()[0]; + is_plain_int64(column) + && (use_field_id + || root.eq_ignore_ascii_case(field.name()) + || !field.name().is_ascii() + || !root.is_ascii()) + }) + }) +} + /// Arrow normalizes both standard and legacy LISTs to the same type, and gives INT96 and /// `TIMESTAMP(NANOS)` leaves the same nanosecond timestamp type. Carry the repeated primitive /// and `TIMESTAMP(NANOS)` distinctions in an ephemeral schema used by the conversion check, @@ -571,6 +599,12 @@ fn mark_parquet_leaves(schema: &Schema, parquet: &SchemaDescriptor) -> Schema { }); let nanos = column.is_some_and(is_nanos_timestamp); let mut field = field.as_ref().clone(); + // A Date64 hint can describe INT32 DATE or an unannotated INT64. + if column.is_some_and(is_plain_int64) + && contains_type(field.data_type(), &DataType::Date64) + { + field = field.with_data_type(DataType::Int64); + } if repeated || nanos { let mut metadata = field.metadata().clone(); if repeated { @@ -783,11 +817,21 @@ impl AsyncFileReader for EagerPageIndexReader { column.self_type().get_basic_info().repetition() == parquet::basic::Repetition::REPEATED }); - if !needs_check { + let check_hint = + may_read_hinted_int64(file, required, conversion_options.use_field_id); + if !needs_check && !check_hint { return Ok(metadata); } let schema = parquet_to_arrow_schema(file.schema_descr(), file.key_value_metadata())?; + let has_date_hint = check_hint + && schema + .fields() + .iter() + .any(|f| contains_type(f.data_type(), &DataType::Date64)); + if !needs_check && !has_date_hint { + return Ok(metadata); + } let schema = mark_parquet_leaves(&schema, file.schema_descr()); let rejection = check_file_conversions( Arc::clone(required), @@ -800,7 +844,7 @@ impl AsyncFileReader for EagerPageIndexReader { DataFusionError::External(source) => ParquetError::External(source), other => ParquetError::External(Box::new(other)), })?; - if *row_filters || has_nanos { + if *row_filters || has_nanos || has_date_hint { if let Some(rejection) = &rejection { self.deferred_rejections.lock().unwrap().insert( object_meta.location.clone(), diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index 93f43d5f029..1e83383cd88 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -323,11 +323,10 @@ pub(crate) fn init_datasource_exec( } else { file_source }; + spark_parquet_options.pruning_only_filters = + !session_config.options().execution.parquet.pushdown_filters; let expr_adapter_factory: Arc = Arc::new( - SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values) - .with_pruning_only_filters( - !session_config.options().execution.parquet.pushdown_filters, - ), + SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values), ); let file_groups = file_groups diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index 635340d068c..ad74a7ac98f 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -19,8 +19,8 @@ use crate::execution::operators::ExecutionError; use crate::parquet::field_resolver::FieldResolver; use crate::parquet::name_fold::fold_names; use arrow::array::{ - make_array, BooleanBufferBuilder, FixedSizeBinaryArray, GenericListViewArray, MapArray, - OffsetSizeTrait, PrimitiveArray, StringArray, TimestampMicrosecondArray, + make_array, BooleanBufferBuilder, FixedSizeBinaryArray, MapArray, PrimitiveArray, StringArray, + TimestampMicrosecondArray, }; use arrow::buffer::NullBuffer; use arrow::compute::{can_cast_types, take}; @@ -41,7 +41,7 @@ use datafusion::execution::object_store::{ }; use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::physical_plan::ColumnarValue; -use datafusion_comet_common::struct_nulls::{repeated_visibility, Visibility}; +use datafusion_comet_common::struct_nulls::{child_visibility, Visibility}; use datafusion_comet_common::SparkError; use datafusion_comet_spark_expr::{spark_cast, EvalMode, SparkCastOptions}; use log::debug; @@ -122,6 +122,8 @@ pub struct SparkParquetOptions { /// (overflow -> NULL), because Spark may discard values through pruning paths that /// DataFusion cannot fully mirror before conversion. pub checked_timestamp_overflow: bool, + /// The scan projects columns and uses predicates solely for statistics pruning. + pub pruning_only_filters: bool, /// When true, resolve each file's datetime calendar-rebase policy from its parquet footer /// metadata (`org.apache.spark.legacyDateTime` and friends) and rebase -- or refuse -- /// affected values, mirroring Spark's per-file `DataSourceUtils.datetimeRebaseSpec` @@ -153,6 +155,7 @@ impl SparkParquetOptions { allow_type_promotion: false, allow_timestamp_ltz_to_ntz: false, checked_timestamp_overflow: true, + pruning_only_filters: false, rebase_from_file_metadata: false, datetime_rebase_mode_in_read: String::new(), int96_rebase_mode_in_read: String::new(), @@ -171,6 +174,7 @@ impl SparkParquetOptions { allow_type_promotion: false, allow_timestamp_ltz_to_ntz: false, checked_timestamp_overflow: true, + pruning_only_filters: false, rebase_from_file_metadata: false, datetime_rebase_mode_in_read: String::new(), int96_rebase_mode_in_read: String::new(), @@ -234,7 +238,8 @@ fn parquet_convert_array_impl( ) -> DataFusionResult<(ArrayRef, bool)> { use DataType::*; let from_type = array.data_type(); - let visible = || NullBuffer::union(array.nulls(), parent_nulls.as_ref()); + let child_mask = || child_visibility(array.as_ref(), parent_nulls); + let children = Visibility::new(&child_mask); // Try Comet specific handlers first, then arrow-rs cast if supported, and fail otherwise. let converted: DataFusionResult = match (from_type, to_type) { @@ -244,7 +249,7 @@ fn parquet_convert_array_impl( from_type, to_type, parquet_options, - &Visibility::new(&visible), + &children, in_map_key, ) } @@ -261,7 +266,7 @@ fn parquet_convert_array_impl( make_array(data.child_data()[0].clone()), to_inner_type.data_type(), parquet_options, - &Visibility::new(&|| list_child_visibility(array.as_ref(), visible().as_ref())), + &children, in_map_key, )?; if !field_overlap && parquet_options.return_null_struct_if_all_fields_missing { @@ -304,7 +309,7 @@ fn parquet_convert_array_impl( // pruning paths, including nested predicates. checked_micros( array.as_primitive::(), - visible(), + parent_nulls.as_ref(), 1_000, target_tz.clone(), ) @@ -313,14 +318,19 @@ fn parquet_convert_array_impl( // Spark 4.x's DateToTimestampNTZUpdater calls daysToMicros at UTC. checked_micros( array.as_primitive::(), - visible(), + parent_nulls.as_ref(), 86_400_000_000, None, ) } (Date64, Timestamp(TimeUnit::Microsecond, None)) => { // ARROW:schema can restore a physical INT32 DATE as milliseconds. - checked_micros(array.as_primitive::(), visible(), 1_000, None) + checked_micros( + array.as_primitive::(), + parent_nulls.as_ref(), + 1_000, + None, + ) } (Dictionary(_, values), _) if may_read_date_as_ntz(values, to_type) => { // An ARROW:schema dictionary can include days no row references. @@ -348,7 +358,7 @@ fn parquet_convert_array_impl( to_type, parquet_options, *ordered_to, - &Visibility::new(&visible), + &children, in_map_key, ); } @@ -446,13 +456,14 @@ fn parquet_convert_array_impl( fn checked_micros( input: &PrimitiveArray, - visible: Option, + parent_nulls: Option<&NullBuffer>, micros_per_unit: i64, timezone: Option>, ) -> DataFusionResult where i64: From, { + let visible = NullBuffer::union(input.nulls(), parent_nulls); let micros = PrimitiveArray::::new(input.values().clone(), visible) .try_unary::<_, TimestampMicrosecondType, _>(|value| { i64::from(value).checked_mul(micros_per_unit).ok_or(()) @@ -477,62 +488,25 @@ pub(super) fn may_read_date_as_ntz(physical_type: &DataType, target_type: &DataT } pub(super) fn contains_ntz(data_type: &DataType) -> bool { + contains_type(data_type, &DataType::Timestamp(TimeUnit::Microsecond, None)) +} + +pub(super) fn contains_type(data_type: &DataType, leaf: &DataType) -> bool { match data_type { - DataType::Timestamp(TimeUnit::Microsecond, None) => true, - DataType::Struct(fields) => fields.iter().any(|field| contains_ntz(field.data_type())), + DataType::Struct(fields) => fields + .iter() + .any(|field| contains_type(field.data_type(), leaf)), DataType::List(field) | DataType::LargeList(field) | DataType::FixedSizeList(field, _) | DataType::ListView(field) | DataType::LargeListView(field) - | DataType::Map(field, _) => contains_ntz(field.data_type()), - DataType::Dictionary(_, values) => contains_ntz(values), - _ => false, - } -} - -fn list_child_visibility(array: &dyn Array, nulls: Option<&NullBuffer>) -> Option { - match array.data_type() { - DataType::List(_) => { - let list = array.as_list::(); - repeated_visibility(list.value_offsets(), list.values().len(), nulls) - } - DataType::LargeList(_) => { - let list = array.as_list::(); - repeated_visibility(list.value_offsets(), list.values().len(), nulls) - } - DataType::FixedSizeList(_, size) => { - // Arrow slices the child values along with a fixed-size list. Expand in those - // child coordinates; applying a parent slice offset again would shift the mask. - nulls.map(|nulls| nulls.expand(*size as usize)) - } - DataType::ListView(_) => list_view_visibility(array.as_list_view::(), nulls), - DataType::LargeListView(_) => list_view_visibility(array.as_list_view::(), nulls), - _ => unreachable!("only called for list arrays"), + | DataType::Map(field, _) => contains_type(field.data_type(), leaf), + DataType::Dictionary(_, values) => contains_type(values, leaf), + _ => data_type == leaf, } } -fn list_view_visibility( - list: &GenericListViewArray, - nulls: Option<&NullBuffer>, -) -> Option { - let mut valid = vec![false; list.values().len()]; - for (row, (offset, size)) in list - .value_offsets() - .iter() - .zip(list.value_sizes()) - .enumerate() - { - if nulls.is_none_or(|nulls| nulls.is_valid(row)) { - let start = offset.as_usize(); - // Views can overlap or be out of order. Accumulate the union of visible ranges; - // a null parent must never clear values another parent can see. - valid[start..start + size.as_usize()].fill(true); - } - } - Some(NullBuffer::from(valid)) -} - /// Read the Parquet field id stored under arrow-rs's `PARQUET_FIELD_ID_META_KEY`. pub(crate) fn field_id(field: &arrow::datatypes::Field) -> Option { field @@ -651,26 +625,18 @@ fn parquet_convert_map_to_map( "map is missing value field".to_string(), ))?; - let mask = || { - repeated_visibility( - from.value_offsets(), - from.keys().len(), - parent_nulls.as_ref(), - ) - }; - let child_visibility = Visibility::new(&mask); let (key_array, key_overlap) = parquet_convert_array_impl( Arc::clone(from.keys()), key_field.data_type(), parquet_options, - &child_visibility, + parent_nulls, true, )?; let (value_array, value_overlap) = parquet_convert_array_impl( Arc::clone(from.values()), value_field.data_type(), parquet_options, - &child_visibility, + parent_nulls, in_map_key, )?; let field_overlap = key_overlap || value_overlap; @@ -2603,14 +2569,10 @@ mod tests { } #[test] - fn test_date_to_ntz_checked_overflow_and_hidden_values() { + fn date_to_ntz_ignores_unreferenced_dictionary_values() { use super::{parquet_convert_array, SparkParquetOptions}; - use arrow::array::{ - Array, ArrayRef, AsArray, Date32Array, DictionaryArray, Int32Array, ListArray, - StructArray, - }; - use arrow::buffer::{NullBuffer, OffsetBuffer}; - use arrow::datatypes::{DataType, Field, Int32Type, TimeUnit, TimestampMicrosecondType}; + use arrow::array::{ArrayRef, AsArray, Date32Array, DictionaryArray, Int32Array}; + use arrow::datatypes::{DataType, Int32Type, TimeUnit, TimestampMicrosecondType}; use datafusion_comet_spark_expr::EvalMode; use std::sync::Arc; @@ -2619,56 +2581,14 @@ mod tests { // Filtering only relaxes the overflow check for TIMESTAMP_MILLIS. options.checked_timestamp_overflow = false; let values: ArrayRef = Arc::new(Date32Array::from(vec![i32::MAX, 1, i32::MIN])); - let dictionary = DictionaryArray::::new( - Int32Array::from(vec![Some(1), None]), - Arc::clone(&values), - ); + let dictionary = + DictionaryArray::::new(Int32Array::from(vec![Some(1), None]), values); let decoded = parquet_convert_array(Arc::new(dictionary), &target, &options).unwrap(); assert_eq!( decoded.as_primitive::().value(0), 86_400_000_000 ); assert!(decoded.is_null(1)); - - let input = StructArray::new( - vec![Arc::new(Field::new("d", DataType::Date32, false))].into(), - vec![Arc::clone(&values)], - Some(NullBuffer::from(vec![false, true, false])), - ); - let struct_type = - DataType::Struct(vec![Arc::new(Field::new("d", target.clone(), false))].into()); - let output = parquet_convert_array(Arc::new(input), &struct_type, &options).unwrap(); - assert!(output.is_null(0)); - assert!(output.is_null(2)); - assert_eq!(output.as_struct().column(0).null_count(), 0); - assert_eq!( - output - .as_struct() - .column(0) - .as_primitive::() - .value(1), - 86_400_000_000 - ); - - // A sliced list retains overflowing backing values outside its visible offsets. - let list = ListArray::new( - Arc::new(Field::new("item", DataType::Date32, false)), - OffsetBuffer::new(vec![0, 1, 2, 3].into()), - values, - None, - ); - let target_list = DataType::List(Arc::new(Field::new("item", target, false))); - assert!(parquet_convert_array(Arc::new(list.clone()), &target_list, &options).is_err()); - let result = - parquet_convert_array(Arc::new(list.slice(1, 1)), &target_list, &options).unwrap(); - assert_eq!( - result - .as_list::() - .value(0) - .as_primitive::() - .value(0), - 86_400_000_000 - ); } #[test] @@ -2838,11 +2758,11 @@ mod tests { } #[test] - fn test_millis_to_micros_nested_visibility() { + fn test_checked_micros_nested_visibility() { use super::{parquet_convert_array, SparkParquetOptions}; use arrow::array::{ - Array, ArrayRef, ListArray, MapArray, StructArray, TimestampMicrosecondArray, - TimestampMillisecondArray, + Array, ArrayRef, Date32Array, ListArray, MapArray, StructArray, + TimestampMicrosecondArray, TimestampMillisecondArray, }; use arrow::buffer::{NullBuffer, OffsetBuffer}; use arrow::datatypes::{DataType, Field, TimeUnit}; @@ -2850,13 +2770,19 @@ mod tests { use std::sync::Arc; for timezone in [None::>, Some(Arc::from("UTC"))] { + let mut cases: Vec<(ArrayRef, i64)> = Vec::new(); for overflow in [i64::MAX, i64::MIN] { + let millis = TimestampMillisecondArray::from(vec![overflow, 7, overflow]) + .with_timezone_opt(timezone.clone()); + cases.push((Arc::new(millis), 7000)); + } + if timezone.is_none() { + let dates = Date32Array::from(vec![i32::MIN, 1, i32::MAX]); + cases.push((Arc::new(dates), 86_400_000_000)); + } + for (input, expected) in cases { let options = SparkParquetOptions::new(EvalMode::Legacy, "UTC"); - let millis: ArrayRef = Arc::new( - TimestampMillisecondArray::from(vec![overflow, 7, overflow]) - .with_timezone_opt(timezone.clone()), - ); - let field = Arc::new(Field::new("ts", millis.data_type().clone(), false)); + let field = Arc::new(Field::new("ts", input.data_type().clone(), false)); let target_field = Arc::new(Field::new( "ts", DataType::Timestamp(TimeUnit::Microsecond, timezone.clone()), @@ -2865,22 +2791,22 @@ mod tests { let validity = Some(NullBuffer::from(vec![false, true, false])); let strukt: ArrayRef = Arc::new(StructArray::new( vec![Arc::clone(&field)].into(), - vec![Arc::clone(&millis)], + vec![Arc::clone(&input)], validity.clone(), )); let list: ArrayRef = Arc::new(ListArray::new( Arc::clone(&field), OffsetBuffer::new(vec![0, 1, 2, 3].into()), - Arc::clone(&millis), + Arc::clone(&input), validity.clone(), )); let entries = StructArray::new( vec![ Arc::clone(&field), - Arc::new(Field::new("value", millis.data_type().clone(), false)), + Arc::new(Field::new("value", input.data_type().clone(), false)), ] .into(), - vec![Arc::clone(&millis), Arc::clone(&millis)], + vec![Arc::clone(&input), Arc::clone(&input)], None, ); let map: ArrayRef = Arc::new(MapArray::new( @@ -2947,7 +2873,10 @@ mod tests { .as_any() .downcast_ref::() .unwrap(); - assert_eq!(values.value(if values.len() == 3 { 1 } else { 0 }), 7000); + assert_eq!( + values.value(if values.len() == 3 { 1 } else { 0 }), + expected + ); assert_eq!(values.null_count(), 0); } // Removing parent nulls makes the overflow visible and must fail. diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 595c748c08a..108e5641947 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -55,7 +55,6 @@ use std::sync::Arc; /// to inject Spark-compatible casts where needed. #[derive(Clone, Debug)] pub struct SparkPhysicalExprAdapterFactory { - pruning_only_filters: bool, /// Spark-specific parquet options for type conversions parquet_options: SparkParquetOptions, /// Default values for columns that may be missing from the physical schema. @@ -70,17 +69,10 @@ impl SparkPhysicalExprAdapterFactory { default_values: Option>, ) -> Self { Self { - pruning_only_filters: false, parquet_options, default_values, } } - - /// Enable only when the scan projects columns and uses predicates solely for pruning. - pub(super) fn with_pruning_only_filters(mut self, enabled: bool) -> Self { - self.pruning_only_filters = enabled; - self - } } /// Returns true when casting `physical_type` to `target_type` is a *pure* structural @@ -1103,7 +1095,6 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { ); Ok(Arc::new(SparkPhysicalExprAdapter { - pruning_only_filters: self.pruning_only_filters, logical_file_schema, physical_file_schema: adapted_physical_schema, parquet_options: self.parquet_options.clone(), @@ -1128,7 +1119,6 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { /// 3. Handle case-insensitive column matching #[derive(Debug)] struct SparkPhysicalExprAdapter { - pruning_only_filters: bool, /// The logical schema expected by the query logical_file_schema: SchemaRef, /// The physical schema of the actual file being read @@ -1423,16 +1413,17 @@ impl SparkPhysicalExprAdapter { .data() } - /// DataFusion requires a bare column to prune by null count. DATE -> NTZ preserves - /// validity, so use the physical column for these statistics. The scan separately - /// adapts the projected column and still checks every decoded day for overflow. + /// DataFusion prunes by null count only on a bare column. Spark prunes before it + /// selects a DATE -> NTZ updater. The projected column still checks decoded values. fn adapt_date_null_check( &self, expr: Arc, ) -> DataFusionResult>> { - if self.pruning_only_filters && (expr.is::() || expr.is::()) { + if self.parquet_options.pruning_only_filters + && (expr.is::() || expr.is::()) + { let cast = expr.children()[0]; - if cast.is::() { + if cast.is::() || cast.is::() { let source = cast.children()[0]; if may_read_date_as_ntz( &source.data_type(&self.physical_file_schema)?, @@ -2743,35 +2734,43 @@ pub(crate) mod test { } #[tokio::test] - async fn parquet_date_with_date64_hint_read_as_ntz() -> Result<(), DataFusionError> { + async fn parquet_date64_hint_read_as_ntz_checks_physical_type() -> Result<(), DataFusionError> { use arrow::array::Date64Array; use parquet::file::properties::WriterProperties; - // Coercion writes INT32 DATE while the Arrow footer hint retains Date64. let days = i64::MAX / 86_400_000_000 + 1; let values: ArrayRef = Arc::new(Date64Array::from(vec![days * 86_400_000])); let batch = RecordBatch::try_from_iter([("d", values)])?; - let props = WriterProperties::builder().set_coerce_types(true).build(); - let filename = get_temp_filename().to_str().unwrap().to_string(); - let mut writer = - ArrowWriter::try_new(File::create(&filename)?, batch.schema(), Some(props))?; - writer.write(&batch)?; - writer.close()?; - let ntz = DataType::Timestamp(TimeUnit::Microsecond, None); let required = Arc::new(Schema::new(vec![Field::new("d", ntz, false)])); - for (promote, expected) in [(true, "long overflow"), (false, "Found: INT32")] { - let mut options = default_options(); - options.allow_type_promotion = promote; - let mut stream = scan_parquet_file( - filename.clone(), - Arc::clone(&required), - options, - None, - false, - )?; - let error = stream.next().await.unwrap().unwrap_err().to_string(); - assert!(error.contains(expected), "unexpected error: {error}"); + // Coercion writes INT32 DATE, otherwise the writer stores INT64. + // The Arrow footer hint says Date64 in both files. + for coerce in [true, false] { + let props = WriterProperties::builder().set_coerce_types(coerce).build(); + let filename = get_temp_filename().to_str().unwrap().to_string(); + let mut writer = + ArrowWriter::try_new(File::create(&filename)?, batch.schema(), Some(props))?; + writer.write(&batch)?; + writer.close()?; + + for promote in [true, false] { + let mut options = default_options(); + options.allow_type_promotion = promote; + let mut stream = scan_parquet_file( + filename.clone(), + Arc::clone(&required), + options, + None, + false, + )?; + let expected = match (coerce, promote) { + (true, true) => "long overflow", + (true, false) => "Found: INT32", + (false, _) => "Found: INT64", + }; + let error = stream.next().await.unwrap().unwrap_err().to_string(); + assert!(error.contains(expected), "unexpected error: {error}"); + } } Ok(()) } diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index a7aa33cdf2c..2e7161e2960 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -68,7 +68,7 @@ use datafusion::common::{internal_err, DataFusionError, Result as DataFusionResu use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; use datafusion_comet_common::decode_utf8_spark_lossy; -use datafusion_comet_common::struct_nulls::{repeated_visibility, Visibility}; +use datafusion_comet_common::struct_nulls::{child_visibility, Visibility}; use std::{ fmt::{Debug, Display, Formatter, Write}, hash::Hash, @@ -312,20 +312,8 @@ fn cast_array_impl( } } - let child_visibility = || { - let visible = NullBuffer::union(array.nulls(), parent_visibility.as_ref()); - match &from_type { - List(_) => { - let list = array.as_list::(); - repeated_visibility(list.value_offsets(), list.values().len(), visible.as_ref()) - } - Map(_, _) => { - let map = array.as_map(); - repeated_visibility(map.value_offsets(), map.entries().len(), visible.as_ref()) - } - _ => visible, - } - }; + let child_mask = || child_visibility(array.as_ref(), parent_visibility); + let children = Visibility::new(&child_mask); let cast_result = match (&from_type, to_type) { // Null arrays carry no concrete values, so Arrow's native cast can change only the // logical type while preserving length and nullness. @@ -464,7 +452,7 @@ fn cast_array_impl( &from_type, to_type, cast_options, - &Visibility::new(&child_visibility), + &children, )?), (List(_), Utf8) => Ok(cast_array_to_string(array.as_list(), cast_options)?), (List(_), List(to)) => { @@ -478,7 +466,7 @@ fn cast_array_impl( Arc::clone(list_array.values()), to.data_type(), cast_options, - &Visibility::new(&child_visibility), + &children, )?, }; Ok(Arc::new(ListArray::new( @@ -493,7 +481,7 @@ fn cast_array_impl( &from_type, to_type, cast_options, - &Visibility::new(&child_visibility), + &children, )?), (UInt8 | UInt16 | UInt32 | UInt64, Int8 | Int16 | Int32 | Int64) if cast_options.allow_cast_unsigned_ints => diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 8cc48e35d40..f028eb92d1a 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -402,6 +402,10 @@ abstract class ParquetReadSuite extends CometTestBase { CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> pushdown.toString) { val df = spark.read.schema(readSchema).parquet(path.toString) checkSparkAnswerAndOperator(df.where("id = 1000").select("stored")) + if (!pushdown) { + // Even Spark 3.5 skips the updater when statistics exclude every row group. + checkSparkAnswerAndOperator(df.where("id = 1 AND probe IS NULL").select("id")) + } if (isSpark40Plus) { checkSparkAnswerAndOperator(df.where("id = 1")) if (!pushdown) { From 8708f93bd9d13789d3aff80690aaac9e1a872028 Mon Sep 17 00:00:00 2001 From: Chao Sun Date: Fri, 20 Nov 2026 23:56:29 +0000 Subject: [PATCH 13/13] Index requested NTZ fields once and retain pruning with deferred schema errors --- .../eager_page_index_reader_factory.rs | 80 ++++++++++++------- native/core/src/parquet/parquet_exec.rs | 6 +- native/core/src/parquet/parquet_support.rs | 8 +- native/core/src/parquet/schema_adapter.rs | 2 +- .../comet/parquet/ParquetReadSuite.scala | 6 +- 5 files changed, 61 insertions(+), 41 deletions(-) diff --git a/native/core/src/parquet/eager_page_index_reader_factory.rs b/native/core/src/parquet/eager_page_index_reader_factory.rs index 7561b05ed2c..72f62a6dd65 100644 --- a/native/core/src/parquet/eager_page_index_reader_factory.rs +++ b/native/core/src/parquet/eager_page_index_reader_factory.rs @@ -135,7 +135,7 @@ use parquet::file::metadata::{ FooterTail, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, }; use parquet::schema::types::{ColumnDescPtr, SchemaDescriptor, Type as ParquetType}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt::{Debug, Display, Formatter}; use std::ops::Range; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -233,7 +233,7 @@ pub struct EagerPageIndexReaderFactory { // does when the requested schema carries one and `ignoreMissing` is not set. require_field_ids: bool, stamp_int96_leaves: bool, - conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>, + conversion_check: Option>, deferred_rejections: Arc>>, next_file_prefetch: Option>, } @@ -272,16 +272,28 @@ impl EagerPageIndexReaderFactory { } } - /// Retain raw LIST encoding and `TIMESTAMP(NANOS)` leaves for open-time validation. When row - /// filters are enabled, or the file has a `TIMESTAMP(NANOS)` leaf, enforce deferred failures - /// at the first data-page read, after format pruning. + /// Check physical encodings that Arrow normalizes. Enforce deferred schema errors on + /// the first data-page read, after pruning and before row filtering. pub(crate) fn with_conversion_check( mut self, required: SchemaRef, options: SparkParquetOptions, row_filters: bool, ) -> Self { - self.conversion_check = Some((required, options, row_filters)); + let ntz_roots = required + .fields() + .iter() + .filter(|f| contains_ntz(f.data_type())) + .map(|f| f.name().to_ascii_lowercase()) + .collect::>(); + let match_any_root = options.use_field_id || ntz_roots.iter().any(|name| !name.is_ascii()); + self.conversion_check = Some(Arc::new(FileConversionCheck { + required, + options, + row_filters, + ntz_roots, + match_any_root, + })); self } @@ -393,7 +405,7 @@ struct EagerPageIndexReader { spark_variant_schema: bool, require_field_ids: bool, stamp_int96_leaves: bool, - conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>, + conversion_check: Option>, deferred_rejections: Arc>>, deferred_rejection: Option, next_file_prefetch: Option>, @@ -551,21 +563,32 @@ fn is_plain_int64(column: &ColumnDescPtr) -> bool { && !matches!(column.logical_type_ref(), Some(LogicalType::Timestamp(_))) } -// Only parse the hint when a requested NTZ could read a plain INT64. The adapter -// resolves exact nested paths, field IDs and Unicode names after the hint check. -fn may_read_hinted_int64(file: &FileMetaData, required: &Schema, use_field_id: bool) -> bool { - has_arrow_schema(file) - && required.fields().iter().any(|field| { - contains_ntz(field.data_type()) - && file.schema_descr().columns().iter().any(|column| { - let root = &column.path().parts()[0]; - is_plain_int64(column) - && (use_field_id - || root.eq_ignore_ascii_case(field.name()) - || !field.name().is_ascii() - || !root.is_ascii()) +#[derive(Debug)] +struct FileConversionCheck { + required: SchemaRef, + options: SparkParquetOptions, + row_filters: bool, + ntz_roots: HashSet, + match_any_root: bool, +} + +impl FileConversionCheck { + // The adapter resolves exact nested paths, field IDs and Unicode names after this check. + fn may_read_hinted_int64(&self, file: &FileMetaData) -> bool { + !self.ntz_roots.is_empty() + && has_arrow_schema(file) + && file + .schema_descr() + .columns() + .iter() + .filter(|c| is_plain_int64(c)) + .any(|c| { + let root = &c.path().parts()[0]; + self.match_any_root + || !root.is_ascii() + || self.ntz_roots.contains(&root.to_ascii_lowercase()) }) - }) + } } /// Arrow normalizes both standard and legacy LISTs to the same type, and gives INT96 and @@ -803,22 +826,19 @@ impl AsyncFileReader for EagerPageIndexReader { } else { metadata }; - if let Some((required, conversion_options, row_filters)) = - self.conversion_check.as_ref() - { + if let Some(check) = &self.conversion_check { let file = metadata.file_metadata(); let columns = file.schema_descr().columns(); // Only this check sees the Parquet type of a nanosecond timestamp leaf, so a // `TIMESTAMP(NANOS)` rejection is deferred from here even without row filters. let has_nanos = columns.iter().any(is_nanos_timestamp); - let needs_check = *row_filters + let needs_check = check.row_filters || has_nanos || columns.iter().any(|column| { column.self_type().get_basic_info().repetition() == parquet::basic::Repetition::REPEATED }); - let check_hint = - may_read_hinted_int64(file, required, conversion_options.use_field_id); + let check_hint = check.may_read_hinted_int64(file); if !needs_check && !check_hint { return Ok(metadata); } @@ -834,9 +854,9 @@ impl AsyncFileReader for EagerPageIndexReader { } let schema = mark_parquet_leaves(&schema, file.schema_descr()); let rejection = check_file_conversions( - Arc::clone(required), + Arc::clone(&check.required), Arc::new(schema), - conversion_options.clone(), + check.options.clone(), ) .map_err(|e| match e { // Keep structured Spark errors directly inside the Parquet wrapper so @@ -844,7 +864,7 @@ impl AsyncFileReader for EagerPageIndexReader { DataFusionError::External(source) => ParquetError::External(source), other => ParquetError::External(Box::new(other)), })?; - if *row_filters || has_nanos || has_date_hint { + if check.row_filters || has_nanos || has_date_hint { if let Some(rejection) = &rejection { self.deferred_rejections.lock().unwrap().insert( object_meta.location.clone(), diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index 1e83383cd88..d76f4851f6c 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -323,8 +323,10 @@ pub(crate) fn init_datasource_exec( } else { file_source }; - spark_parquet_options.pruning_only_filters = - !session_config.options().execution.parquet.pushdown_filters; + // Row filters need the checked cast when promotion is supported. On Spark 3.5 + // the footer check above rejects promotion at the first data-page read. + spark_parquet_options.prune_date_nulls = + !allow_type_promotion || !session_config.options().execution.parquet.pushdown_filters; let expr_adapter_factory: Arc = Arc::new( SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values), ); diff --git a/native/core/src/parquet/parquet_support.rs b/native/core/src/parquet/parquet_support.rs index ad74a7ac98f..59436fd1b61 100644 --- a/native/core/src/parquet/parquet_support.rs +++ b/native/core/src/parquet/parquet_support.rs @@ -122,8 +122,8 @@ pub struct SparkParquetOptions { /// (overflow -> NULL), because Spark may discard values through pruning paths that /// DataFusion cannot fully mirror before conversion. pub checked_timestamp_overflow: bool, - /// The scan projects columns and uses predicates solely for statistics pruning. - pub pruning_only_filters: bool, + /// Whether null predicates can use the physical date during NTZ promotion. + pub prune_date_nulls: bool, /// When true, resolve each file's datetime calendar-rebase policy from its parquet footer /// metadata (`org.apache.spark.legacyDateTime` and friends) and rebase -- or refuse -- /// affected values, mirroring Spark's per-file `DataSourceUtils.datetimeRebaseSpec` @@ -155,7 +155,7 @@ impl SparkParquetOptions { allow_type_promotion: false, allow_timestamp_ltz_to_ntz: false, checked_timestamp_overflow: true, - pruning_only_filters: false, + prune_date_nulls: false, rebase_from_file_metadata: false, datetime_rebase_mode_in_read: String::new(), int96_rebase_mode_in_read: String::new(), @@ -174,7 +174,7 @@ impl SparkParquetOptions { allow_type_promotion: false, allow_timestamp_ltz_to_ntz: false, checked_timestamp_overflow: true, - pruning_only_filters: false, + prune_date_nulls: false, rebase_from_file_metadata: false, datetime_rebase_mode_in_read: String::new(), int96_rebase_mode_in_read: String::new(), diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 108e5641947..c3357babee6 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -1419,7 +1419,7 @@ impl SparkPhysicalExprAdapter { &self, expr: Arc, ) -> DataFusionResult>> { - if self.parquet_options.pruning_only_filters + if self.parquet_options.prune_date_nulls && (expr.is::() || expr.is::()) { let cast = expr.children()[0]; diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index f028eb92d1a..d9c96864ec9 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -402,10 +402,8 @@ abstract class ParquetReadSuite extends CometTestBase { CometConf.COMET_PARQUET_ROW_FILTER_PUSHDOWN_ENABLED.key -> pushdown.toString) { val df = spark.read.schema(readSchema).parquet(path.toString) checkSparkAnswerAndOperator(df.where("id = 1000").select("stored")) - if (!pushdown) { - // Even Spark 3.5 skips the updater when statistics exclude every row group. - checkSparkAnswerAndOperator(df.where("id = 1 AND probe IS NULL").select("id")) - } + // Even Spark 3.5 skips the updater when statistics exclude every row group. + checkSparkAnswerAndOperator(df.where("id = 1 AND probe IS NULL").select("id")) if (isSpark40Plus) { checkSparkAnswerAndOperator(df.where("id = 1")) if (!pushdown) {