From fc2555f135a8cb8a1e08ef5bd342f30d5285af54 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Sun, 1 Nov 2026 10:22:34 +0800 Subject: [PATCH 1/8] fix: recognize casts that keep order and check the rule against the cast kernel --- .../physical-expr/src/expressions/cast.rs | 277 +++++++++++++++++- .../physical-expr/src/expressions/mod.rs | 2 +- datafusion/sqllogictest/test_files/cast.slt | 16 + 3 files changed, 283 insertions(+), 12 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index e56bf4dcbc8b9..38904bae767ef 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -379,14 +379,39 @@ fn is_fixed_offset(tz: &str) -> bool { tz == "UTC" || tz.starts_with(['+', '-']) } -/// Whether successful casts preserve order when conversion failures return errors. -/// Unlike `check_bigger_cast`, this allows precision loss and is not sufficient -/// for propagating distinct counts or the ordering of subsequent sort keys. -fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool { +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + Binary | LargeBinary | BinaryView | FixedSizeBinary(_) + ) +} + +/// Temporal types stored as a count of units since the epoch (or midnight), +/// which a cast from or to a number reads or writes as that count. +fn is_temporal_count(data_type: &DataType) -> bool { + matches!( + data_type, + Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_) + ) +} + +/// Whether the cast never reverses the order of two values that both convert: +/// `a <= b` implies `cast(a) <= cast(b)`. Precision loss may merge values, so +/// this is not sufficient for propagating distinct counts or the ordering of +/// subsequent sort keys (see [`CastExpr::check_bigger_cast`] for that). +/// +/// Values that fail to convert are not covered: callers must account for the +/// error or NULL a failure produces. +pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool { use arrow::datatypes::TimeUnit::*; if source_type == target_type || (source_type.is_numeric() || *source_type == Boolean) && target_type.is_numeric() + // Changing the string or binary encoding keeps the byte order. + || source_type.is_string() && target_type.is_string() + || is_binary(source_type) && is_binary(target_type) + || source_type.is_numeric() && is_temporal_count(target_type) + || is_temporal_count(source_type) && target_type.is_numeric() { return true; } @@ -394,7 +419,7 @@ fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> b // at midnight, and timezone transitions can reverse the local date. match (source_type, target_type) { (Date32 | Date64, Date32 | Date64) - | (Date32 | Date64, Timestamp(_, None)) + | (Date32 | Date64, Timestamp(_, _)) | (Timestamp(_, None), Date32) => true, (Timestamp(_, Some(tz)), Date32) => is_fixed_offset(tz), // Arrow converts timestamps to Date64 by scaling the epoch value, @@ -406,11 +431,13 @@ fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> b | (Time32(Second | Millisecond), Time64(Microsecond | Nanosecond)) | (Time64(Nanosecond), Time64(Microsecond)) | (Duration(_), Duration(_)) => true, - (Timestamp(_, from_tz), Timestamp(_, to_tz)) => { - // Adding a timezone to naive timestamps interprets local times; - // other timezone changes only change metadata on the epoch value. - from_tz.is_some() || to_tz.as_deref().is_none_or(is_fixed_offset) - } + // Changing the timezone only changes metadata on the epoch value. + // Adding a timezone to a naive timestamp (or a date) reads it as a + // local time, and Arrow rejects local times that a transition skips + // or repeats. A transition skips or repeats exactly the local times + // its offset change covers, so the local times that remain keep + // their order in UTC. + (Timestamp(_, _), Timestamp(_, _)) => true, _ => false, } } @@ -2283,6 +2310,7 @@ mod tests { (Date64, Date32, true), (Date32, Timestamp(Nanosecond, None), true), (Date64, Timestamp(Second, None), true), + (Date32, Timestamp(Second, timezone.clone()), true), (Timestamp(Second, None), Date32, true), (Timestamp(Second, Some("UTC".into())), Date32, true), (Timestamp(Second, Some("+08:00".into())), Date32, true), @@ -2290,10 +2318,12 @@ mod tests { (Timestamp(Second, timezone.clone()), Date64, true), (Timestamp(Second, None), Timestamp(Nanosecond, None), true), (Timestamp(Nanosecond, None), Timestamp(Second, None), true), + // Local times that a transition skips or repeats fail to convert, + // and the remaining ones keep their order. ( Timestamp(Second, None), Timestamp(Second, timezone.clone()), - false, + true, ), ( Timestamp(Second, None), @@ -2360,6 +2390,231 @@ mod tests { } } + /// Sorted, distinct, non-null sample values of `data_type`, obtained by + /// casting a fixed set of seeds and keeping the values that convert. + fn order_samples(data_type: &DataType) -> Option { + use arrow::array::{ + BooleanArray, FixedSizeBinaryArray, Float64Array, TimestampSecondArray, + UInt32Array, make_comparator, + }; + use arrow::compute::{ + SortOptions, concat, filter, is_not_null, sort_to_indices, take, + }; + + let seeds: Vec = vec![ + Arc::new(Int64Array::from(vec![ + i64::MIN, + -2_147_483_649, + -129, + -1, + 0, + 1, + 9, + 10, + 127, + 128, + 255, + 256, + 86_399, + // America/Goose_Bay fell back from 00:01 to 23:01 the previous + // day at this instant, so the local date goes backwards. + 562_129_259, + 562_129_260, + 2_147_483_647, + i64::MAX, + ])), + Arc::new(Float64Array::from(vec![ + f64::NEG_INFINITY, + -1.5, + -0.5, + 0.0, + 0.5, + 1.7, + 1e300, + f64::INFINITY, + ])), + Arc::new(StringArray::from(vec!["", "-1", "0", "1", "10", "9", "a"])), + Arc::new(BooleanArray::from(vec![false, true])), + Arc::new( + FixedSizeBinaryArray::try_from_iter( + ["abcd", "abce", "zzzz"].into_iter().map(str::as_bytes), + ) + .expect("valid fixed size binary"), + ), + Arc::new(TimestampSecondArray::from(vec![ + // 23:00, 00:00 and 01:00 around midnight + 82_800, + 86_400, + 90_000, + // Local times around the America/New_York transitions of + // 2024: before, inside and after the skipped hour in March and + // the repeated hour in November. + 1_710_035_999, + 1_710_037_800, + 1_710_039_600, + 1_730_595_599, + 1_730_597_400, + 1_730_599_200, + ])), + ]; + // Cast each seed on its own, so one value that fails to convert does + // not discard the others. + let parts = seeds + .iter() + .flat_map(|seed| (0..seed.len()).map(|i| seed.slice(i, 1))) + .filter_map(|seed| arrow::compute::cast(&seed, data_type).ok()) + .map(|array| filter(&array, &is_not_null(&array).unwrap()).unwrap()) + .filter(|array| !array.is_empty()) + .collect::>(); + if parts.is_empty() { + return None; + } + let parts = parts.iter().map(|a| a.as_ref()).collect::>(); + let all = concat(&parts).unwrap(); + let sorted = + take(&all, &sort_to_indices(&all, None, None).unwrap(), None).unwrap(); + let cmp = make_comparator(&sorted, &sorted, SortOptions::default()).unwrap(); + let distinct = (0..sorted.len()) + .filter(|&i| i == 0 || cmp(i - 1, i).is_ne()) + .map(|i| i as u32) + .collect::(); + Some(take(&sorted, &distinct, None).unwrap()) + } + + /// Casts the sorted `samples` to `target` and returns the first pair of + /// converted values whose order the cast reverses. Each value is cast on + /// its own, and values that fail to convert are skipped: + /// `is_order_preserving_cast` does not cover them. + /// + /// Returns `None` when the cast keeps the order, and `Some(Err(_))` when + /// the cast kernel panics. + fn order_violation( + samples: &ArrayRef, + target: &DataType, + ) -> Option> { + use arrow::array::make_comparator; + use arrow::compute::SortOptions; + + let mut converted: Vec<(usize, ArrayRef)> = vec![]; + for i in 0..samples.len() { + let value = samples.slice(i, 1); + let output = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + arrow::compute::cast_with_options( + &value, + target, + &DEFAULT_SAFE_CAST_OPTIONS, + ) + })); + match output { + Ok(Ok(output)) if output.is_valid(0) => converted.push((i, output)), + Ok(_) => {} + Err(_) => return Some(Err("cast kernel panicked".to_string())), + } + } + converted.windows(2).find_map(|w| { + let ((i, left), (j, right)) = (&w[0], &w[1]); + let cmp = make_comparator(left, right, SortOptions::default()).unwrap(); + cmp(0, 0).is_gt().then(|| { + let value = |array: &ArrayRef, i| { + ScalarValue::try_from_array(array, i).unwrap().to_string() + }; + Ok(format!( + "{} -> {}, but {} -> {}", + value(samples, *i), + value(left, 0), + value(samples, *j), + value(right, 0), + )) + }) + }) + } + + /// Checks `is_order_preserving_cast` against the Arrow cast kernel: every + /// pair it accepts must keep the order of the sample values that convert. + /// Pruning and sort elimination both rely on this rule, so a wrong entry + /// drops matching rows or returns unsorted output. + #[test] + fn test_order_preserving_cast_matches_cast_kernel() { + use TimeUnit::*; + + let tz = |unit, tz: &str| Timestamp(unit, Some(tz.into())); + let types = [ + Boolean, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + Decimal128(10, 2), + Decimal256(40, 2), + Utf8, + LargeUtf8, + Utf8View, + Binary, + LargeBinary, + BinaryView, + FixedSizeBinary(4), + Date32, + Date64, + Timestamp(Second, None), + Timestamp(Nanosecond, None), + tz(Millisecond, "UTC"), + tz(Second, "+05:00"), + tz(Second, "America/New_York"), + tz(Second, "America/Goose_Bay"), + Time32(Second), + Time32(Millisecond), + Time64(Microsecond), + Time64(Nanosecond), + Duration(Second), + Duration(Nanosecond), + ]; + let mut violations = vec![]; + for source in &types { + let Some(samples) = order_samples(source) else { + continue; + }; + for target in &types { + if !can_cast_types(source, target) + || !is_order_preserving_cast(source, target) + { + continue; + } + match order_violation(&samples, target) { + None => {} + // A panic is an unchecked overflow in a debug build, which + // wraps in a release build, so it is not evidence of order + // either. These pairs are listed separately in the output. + Some(Err(e)) => println!("{source} -> {target}: {e}"), + Some(Ok(e)) => violations.push(format!("{source} -> {target}: {e}")), + } + } + } + assert!(violations.is_empty(), "{}", violations.join("\n")); + + // The samples are able to expose the casts the rule rejects. + for (source, target) in [ + (Utf8, Int32), + (Int32, Boolean), + (Int16, Binary), + (Timestamp(Second, None), Time32(Second)), + (tz(Second, "+05:00"), Time64(Nanosecond)), + (tz(Second, "America/Goose_Bay"), Date32), + ] { + assert!(!is_order_preserving_cast(&source, &target)); + let samples = order_samples(&source).unwrap(); + assert!( + matches!(order_violation(&samples, &target), Some(Ok(_))), + "{source} -> {target}: no sample exposes the reversed order" + ); + } + } + #[test] fn test_cast_ordering_with_null_on_failure() { use arrow::array::TimestampSecondArray; diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 767c0b4ce40f3..bb5c818ece6a6 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -30,7 +30,7 @@ pub use crate::aggregate::stats::StatsType; pub use binary::{BinaryExpr, binary, similar_to}; pub use case::{CaseExpr, case}; -pub use cast::{CastExpr, cast}; +pub use cast::{CastExpr, cast, is_order_preserving_cast}; pub use column::{Column, col, with_new_schema}; pub use datafusion_expr::utils::format_state_name; pub use dynamic_filters::{ diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt index 3d11899c9304a..583131f4a4815 100644 --- a/datafusion/sqllogictest/test_files/cast.slt +++ b/datafusion/sqllogictest/test_files/cast.slt @@ -164,6 +164,22 @@ physical_plan 02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] 03)----DataSourceExec: partitions=1, partition_sizes=[1] +# An integer read as a count of seconds keeps its order, so the outer sort is +# removed. +query TT +EXPLAIN SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts +FROM ( + SELECT column1 AS i + FROM (VALUES (3::bigint), (1::bigint), (2::bigint)) + ORDER BY i LIMIT 3 +) +ORDER BY ts; +---- +physical_plan +01)ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts] +02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + statement ok SET datafusion.explain.physical_plan_only = false; From 2785372bbe94434a546f4b38e3d92e9fa6de5a22 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Sun, 1 Nov 2026 10:22:35 +0800 Subject: [PATCH 2/8] fix: prune statistics only through casts that keep order --- datafusion/pruning/src/pruning_predicate.rs | 99 +++++++++++++++---- .../test_files/push_down_filter_parquet.slt | 87 ++++++++++++++++ 2 files changed, 168 insertions(+), 18 deletions(-) diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 2e3c84673f90c..6ab0ca903928c 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -1331,10 +1331,10 @@ fn is_compare_op(op: Operator) -> bool { ) } -// The pruning logic is based on the comparing the min/max bounds. -// Must make sure the two type has order. -// For example, casts from string to numbers is not correct. -// Because the "13" is less than "3" with UTF8 comparison order. +// Pruning casts the container's min/max statistics and compares them with the +// literal, which is only sound when the cast keeps the order of the values, +// e.g. not for strings to numbers ("13" < "3") or timestamps to time of day +// (wraps at midnight). Ask the same rule that sort elimination uses. fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Result<()> { // Dictionary casts are always supported as long as the value types are supported let from_type = match from_type { @@ -1349,20 +1349,7 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re } _ => to_type, }; - // Casting a timestamp to a time-of-day wraps at midnight and does not - // preserve ordering across day boundaries. Casting timestamp min/max - // statistics could therefore produce invalid bounds and prune matches. - if matches!(from_type, DataType::Timestamp(_, _)) - && matches!(to_type, DataType::Time32(_) | DataType::Time64(_)) - { - return plan_err!( - "Try Cast/Cast with from type {from_type} to type {to_type} is not supported" - ); - } - // If both types are strings or both are not strings (number, timestamp, etc) - // then we can compare them. - // PruningPredicate does not support casting of strings to numbers and such. - if from_type.is_string() == to_type.is_string() { + if phys_expr::is_order_preserving_cast(from_type, to_type) { Ok(()) } else { plan_err!( @@ -5095,6 +5082,82 @@ mod tests { Ok(()) } + /// Pruning casts the min/max statistics, so it may only use casts that + /// keep the order of the values. + #[test] + fn row_group_predicate_cast_must_preserve_order() { + use TimeUnit::*; + + let new_york = Some(Arc::from("America/New_York")); + let dictionary = |value: DataType| { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(value)) + }; + let cases = [ + // 0 is false while -1 and 1 are true + (DataType::Int32, ScalarValue::Boolean(Some(false)), false), + (DataType::Float64, ScalarValue::Boolean(Some(false)), false), + ( + dictionary(DataType::Int64), + ScalarValue::Boolean(Some(false)), + false, + ), + // little-endian bytes: 256 is [0, 1, ..] and 255 is [255, 0, ..] + (DataType::Int64, ScalarValue::Binary(Some(vec![5])), false), + // a named timezone can move the local date backwards + ( + DataType::Timestamp(Nanosecond, new_york.clone()), + ScalarValue::Date32(Some(0)), + false, + ), + ( + DataType::Int64, + ScalarValue::TimestampNanosecond(Some(0), None), + true, + ), + (DataType::Int32, ScalarValue::Date32(Some(0)), true), + ( + DataType::Utf8, + ScalarValue::Utf8View(Some("a".into())), + true, + ), + (dictionary(DataType::Utf8), ScalarValue::from("a"), true), + ( + DataType::FixedSizeBinary(1), + ScalarValue::Binary(Some(vec![5])), + true, + ), + ( + DataType::Timestamp(Nanosecond, None), + ScalarValue::TimestampNanosecond(Some(0), new_york.clone()), + true, + ), + ( + DataType::Date32, + ScalarValue::TimestampSecond(Some(0), new_york), + true, + ), + ]; + for (source, literal, prunable) in cases { + let target = literal.data_type(); + let schema = Schema::new(vec![Field::new("c1", source.clone(), false)]); + for expr in [ + cast(col("c1"), target.clone()).eq(lit(literal.clone())), + try_cast(col("c1"), target.clone()).eq(lit(literal.clone())), + ] { + let predicate_expr = test_build_predicate_expression( + &expr, + &schema, + &mut RequiredColumns::new(), + ); + assert_eq!( + predicate_expr.to_string() != "true", + prunable, + "{source} -> {target}: {predicate_expr}" + ); + } + } + } + #[test] fn row_group_predicate_date_date() -> Result<()> { let schema = Schema::new(vec![Field::new("c1", DataType::Date32, false)]); diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index ea61f0be6c6a4..2745a29c70534 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1602,3 +1602,90 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok RESET datafusion.explain.physical_plan_only; + +########## +# Pruning casts the min/max statistics of each row group, so it may only use a +# cast that keeps the order of the values. Each predicate below runs against +# an in-memory table, which is never pruned, and the same rows in one Parquet +# row group. The two must agree. +########## + +statement ok +CREATE TABLE cast_order_src(i INT, ts TIMESTAMP) AS VALUES + (-1, TIMESTAMP '2024-03-10 23:00:00'), + (0, TIMESTAMP '2024-03-11 00:00:00'), + (1, TIMESTAMP '2024-03-11 01:00:00'), + (255, TIMESTAMP '2024-03-11 02:00:00'), + (256, TIMESTAMP '2024-03-11 03:00:00'); + +statement ok +COPY cast_order_src +TO 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet' +STORED AS PARQUET; + +statement ok +CREATE EXTERNAL TABLE cast_order +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet'; + +# -1 and 1 are true and 0 is false, so the bounds -1 and 256 both cast to true. +query I +SELECT i FROM cast_order_src WHERE CAST(i AS BOOLEAN) <= false; +---- +0 + +query I +SELECT i FROM cast_order WHERE CAST(i AS BOOLEAN) <= false; +---- +0 + +# An integer casts to its little-endian bytes: -1 is X'FFFFFFFF' and 256 is +# X'00010000', so the bounds no longer bracket the values in between. +query I +SELECT i FROM cast_order_src WHERE CAST(i AS BYTEA) < X'05' ORDER BY i; +---- +0 +1 +256 + +query I +SELECT i FROM cast_order WHERE CAST(i AS BYTEA) < X'05' ORDER BY i; +---- +0 +1 +256 + +# Comparing a timestamp without a timezone to one with a named timezone casts +# the column to that timezone, reading the values as New York local times +# (05:30 UTC is 01:30 local). That cast keeps the order, so it still prunes. +statement ok +SET datafusion.execution.time_zone = 'America/New_York'; + +statement ok +SET datafusion.explain.physical_plan_only = true; + +query TT +EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-11 05:30:00+00:00'; +---- +physical_plan +01)FilterExec: CAST(ts@1 AS Timestamp(ns, "America/New_York")) > 1710135000000000000, projection=[i@0] +02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "America/New_York")) > 1710135000000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "America/New_York");N) > 1710135000000000000, required_guarantees=[] + +query I +SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-11 05:30:00+00:00' ORDER BY i; +---- +255 +256 + +statement ok +RESET datafusion.explain.physical_plan_only; + +statement ok +RESET datafusion.execution.time_zone; + +statement ok +DROP TABLE cast_order; + +statement ok +DROP TABLE cast_order_src; From e0f5d51195be5146c277408a0def9949327b707f Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 3 Nov 2026 20:13:55 +0800 Subject: [PATCH 3/8] test: trigger the Bloom filter error path with a literal that fails to cast --- .../datasource-parquet/src/row_group_filter.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index d605f0cf49afd..23291099ecac7 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -1663,22 +1663,25 @@ mod tests { fn bloom_filter_pruning_error_retains_row_group_without_match() { let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); - let cast_expr: Arc = Arc::new(CastExpr::new( - Arc::new(PhysicalColumn::new("c1", 0)), - DataType::FixedSizeBinary(4), + // c1 = CAST('abc' AS Int32): constant folding fails, so the + // simplifier keeps the cast, and the cast fails at evaluation. + let value: Arc = Arc::new(CastExpr::new( + Arc::new(Literal::new(ScalarValue::from("abc"))), + DataType::Int32, None, )); - let literal: Arc = Arc::new(Literal::new( - ScalarValue::FixedSizeBinary(4, Some(vec![0, 0, 0, 1])), + let expr = Arc::new(BinaryExpr::new( + Arc::new(PhysicalColumn::new("c1", 0)), + Operator::Eq, + value, )); - let expr = Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal)); let pruning_predicate = PruningPredicateBuilder::new() .with_file_schema(schema) .try_build(expr) .unwrap(); // A nonempty map bypasses the no-filter path. Evaluating the predicate - // then fails because Arrow cannot cast Int32 statistics to FixedSizeBinary. + // then fails because 'abc' is not an Int32. let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); sbbf.insert(&1_i32); let mut bloom_statistics = BloomFilterStatistics::new(); From 382192637425f5bae30e8f4b767c6d2a803cf4e7 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 3 Nov 2026 20:13:55 +0800 Subject: [PATCH 4/8] test: check cast order through the strict cast_to path --- .../physical-expr/src/expressions/cast.rs | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 38904bae767ef..bdd3e9c8817d4 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -2481,36 +2481,26 @@ mod tests { Some(take(&sorted, &distinct, None).unwrap()) } - /// Casts the sorted `samples` to `target` and returns the first pair of - /// converted values whose order the cast reverses. Each value is cast on - /// its own, and values that fail to convert are skipped: - /// `is_order_preserving_cast` does not cover them. - /// - /// Returns `None` when the cast keeps the order, and `Some(Err(_))` when - /// the cast kernel panics. - fn order_violation( - samples: &ArrayRef, - target: &DataType, - ) -> Option> { + /// Casts the sorted `samples` to `target` with the strict cast that + /// `CastExpr` runs by default, and returns the first pair of converted + /// values whose order the cast reverses. Each value is cast on its own, + /// and values that fail to convert are skipped: `is_order_preserving_cast` + /// does not cover them. + fn order_violation(samples: &ArrayRef, target: &DataType) -> Option { use arrow::array::make_comparator; use arrow::compute::SortOptions; - let mut converted: Vec<(usize, ArrayRef)> = vec![]; - for i in 0..samples.len() { - let value = samples.slice(i, 1); - let output = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - arrow::compute::cast_with_options( - &value, - target, - &DEFAULT_SAFE_CAST_OPTIONS, - ) - })); - match output { - Ok(Ok(output)) if output.is_valid(0) => converted.push((i, output)), - Ok(_) => {} - Err(_) => return Some(Err("cast kernel panicked".to_string())), - } - } + let converted = (0..samples.len()) + .filter_map(|i| { + let value = ColumnarValue::Array(samples.slice(i, 1)); + match value.cast_to(target, None) { + Ok(ColumnarValue::Array(output)) if output.is_valid(0) => { + Some((i, output)) + } + _ => None, + } + }) + .collect::>(); converted.windows(2).find_map(|w| { let ((i, left), (j, right)) = (&w[0], &w[1]); let cmp = make_comparator(left, right, SortOptions::default()).unwrap(); @@ -2518,19 +2508,20 @@ mod tests { let value = |array: &ArrayRef, i| { ScalarValue::try_from_array(array, i).unwrap().to_string() }; - Ok(format!( + format!( "{} -> {}, but {} -> {}", value(samples, *i), value(left, 0), value(samples, *j), value(right, 0), - )) + ) }) }) } - /// Checks `is_order_preserving_cast` against the Arrow cast kernel: every - /// pair it accepts must keep the order of the sample values that convert. + /// Checks `is_order_preserving_cast` against the strict cast that + /// `CastExpr` runs: every pair it accepts must keep the order of the sample + /// values that convert. /// Pruning and sort elimination both rely on this rule, so a wrong entry /// drops matching rows or returns unsorted output. #[test] @@ -2585,13 +2576,8 @@ mod tests { { continue; } - match order_violation(&samples, target) { - None => {} - // A panic is an unchecked overflow in a debug build, which - // wraps in a release build, so it is not evidence of order - // either. These pairs are listed separately in the output. - Some(Err(e)) => println!("{source} -> {target}: {e}"), - Some(Ok(e)) => violations.push(format!("{source} -> {target}: {e}")), + if let Some(e) = order_violation(&samples, target) { + violations.push(format!("{source} -> {target}: {e}")); } } } @@ -2609,7 +2595,7 @@ mod tests { assert!(!is_order_preserving_cast(&source, &target)); let samples = order_samples(&source).unwrap(); assert!( - matches!(order_violation(&samples, &target), Some(Ok(_))), + order_violation(&samples, &target).is_some(), "{source} -> {target}: no sample exposes the reversed order" ); } From 4c76cce4ea6151b39b8cf2155d0ad3121d767caa Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 3 Nov 2026 20:13:56 +0800 Subject: [PATCH 5/8] test: check the row order of the timestamp sort elimination case --- datafusion/sqllogictest/test_files/cast.slt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt index 583131f4a4815..c83568310f83d 100644 --- a/datafusion/sqllogictest/test_files/cast.slt +++ b/datafusion/sqllogictest/test_files/cast.slt @@ -180,6 +180,19 @@ physical_plan 02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] 03)----DataSourceExec: partitions=1, partition_sizes=[1] +query P +SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts +FROM ( + SELECT column1 AS i + FROM (VALUES (3::bigint), (1::bigint), (2::bigint)) + ORDER BY i LIMIT 3 +) +ORDER BY ts; +---- +1970-01-01T00:00:01 +1970-01-01T00:00:02 +1970-01-01T00:00:03 + statement ok SET datafusion.explain.physical_plan_only = false; From 87c244725a69253bb2f83ae9f53e4ca504f9c395 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Tue, 3 Nov 2026 20:13:56 +0800 Subject: [PATCH 6/8] fix: keep the cast order rule in line with arrow 60.1.0 timezone casts --- .../physical-expr/src/expressions/cast.rs | 60 +++++++++++++------ datafusion/pruning/src/pruning_predicate.rs | 13 +++- datafusion/sqllogictest/test_files/cast.slt | 13 ++++ .../test_files/push_down_filter_parquet.slt | 51 ++++++++++++---- 4 files changed, 107 insertions(+), 30 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index bdd3e9c8817d4..a9ec39f7ab52b 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -418,13 +418,11 @@ pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) // Temporal casts are not generally monotonic: extracting time-of-day wraps // at midnight, and timezone transitions can reverse the local date. match (source_type, target_type) { - (Date32 | Date64, Date32 | Date64) - | (Date32 | Date64, Timestamp(_, _)) - | (Timestamp(_, None), Date32) => true, - (Timestamp(_, Some(tz)), Date32) => is_fixed_offset(tz), - // Arrow converts timestamps to Date64 by scaling the epoch value, - // whereas Date32 extracts the date in the timestamp's timezone. - (Timestamp(_, _), Date64) => true, + (Date32 | Date64, Date32 | Date64) | (Timestamp(_, None), Date32 | Date64) => { + true + } + // Both date types take the date in the timestamp's timezone. + (Timestamp(_, Some(tz)), Date32 | Date64) => is_fixed_offset(tz), // Only admit scaling that cannot wrap on overflow. Time64 widening // and narrowing to Time32 do not currently check overflow in Arrow. (Time32(Second | Millisecond), Time32(Second | Millisecond)) @@ -432,12 +430,18 @@ pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) | (Time64(Nanosecond), Time64(Microsecond)) | (Duration(_), Duration(_)) => true, // Changing the timezone only changes metadata on the epoch value. - // Adding a timezone to a naive timestamp (or a date) reads it as a - // local time, and Arrow rejects local times that a transition skips - // or repeats. A transition skips or repeats exactly the local times - // its offset change covers, so the local times that remain keep - // their order in UTC. - (Timestamp(_, _), Timestamp(_, _)) => true, + (Timestamp(_, Some(_)), Timestamp(_, _)) => true, + // Adding a timezone reads a naive timestamp (or a Date64, which may + // hold any time of day) as a local time. Arrow moves a local time that + // a transition skips forward by the length of the gap, past the local + // times just after the gap, so only a fixed offset keeps the order. + (Timestamp(_, None) | Date64, Timestamp(_, tz)) => { + tz.as_deref().is_none_or(is_fixed_offset) + } + // A Date32 reads as local midnight. A skipped midnight moves forward + // by the gap, and no gap is longer than a day, so it never passes the + // next midnight. + (Date32, Timestamp(_, _)) => true, _ => false, } } @@ -2311,19 +2315,21 @@ mod tests { (Date32, Timestamp(Nanosecond, None), true), (Date64, Timestamp(Second, None), true), (Date32, Timestamp(Second, timezone.clone()), true), + (Date64, Timestamp(Second, Some("+08:00".into())), true), + (Date64, Timestamp(Second, timezone.clone()), false), (Timestamp(Second, None), Date32, true), + (Timestamp(Second, None), Date64, true), (Timestamp(Second, Some("UTC".into())), Date32, true), (Timestamp(Second, Some("+08:00".into())), Date32, true), + (Timestamp(Second, Some("+08:00".into())), Date64, true), (Timestamp(Second, timezone.clone()), Date32, false), - (Timestamp(Second, timezone.clone()), Date64, true), + (Timestamp(Second, timezone.clone()), Date64, false), (Timestamp(Second, None), Timestamp(Nanosecond, None), true), (Timestamp(Nanosecond, None), Timestamp(Second, None), true), - // Local times that a transition skips or repeats fail to convert, - // and the remaining ones keep their order. ( Timestamp(Second, None), Timestamp(Second, timezone.clone()), - true, + false, ), ( Timestamp(Second, None), @@ -2415,6 +2421,11 @@ mod tests { 128, 255, 256, + // As dates, 2011-12-29 to 2011-12-31: Pacific/Apia skipped + // all of 2011-12-30. + 15_337, + 15_338, + 15_339, 86_399, // America/Goose_Bay fell back from 00:01 to 23:01 the previous // day at this instant, so the local date goes backwards. @@ -2527,6 +2538,7 @@ mod tests { #[test] fn test_order_preserving_cast_matches_cast_kernel() { use TimeUnit::*; + use arrow::array::{AsArray, Date32Array}; let tz = |unit, tz: &str| Timestamp(unit, Some(tz.into())); let types = [ @@ -2558,6 +2570,7 @@ mod tests { tz(Second, "+05:00"), tz(Second, "America/New_York"), tz(Second, "America/Goose_Bay"), + tz(Second, "Pacific/Apia"), Time32(Second), Time32(Millisecond), Time64(Microsecond), @@ -2591,6 +2604,8 @@ mod tests { (Timestamp(Second, None), Time32(Second)), (tz(Second, "+05:00"), Time64(Nanosecond)), (tz(Second, "America/Goose_Bay"), Date32), + (tz(Second, "America/Goose_Bay"), Date64), + (Timestamp(Second, None), tz(Second, "America/New_York")), ] { assert!(!is_order_preserving_cast(&source, &target)); let samples = order_samples(&source).unwrap(); @@ -2599,6 +2614,17 @@ mod tests { "{source} -> {target}: no sample exposes the reversed order" ); } + + // The samples reach the edge of the Date32 entry: a skipped day starts + // at the same instant as the day after it. + let skipped = + ColumnarValue::Array(Arc::new(Date32Array::from(vec![15_338, 15_339]))) + .cast_to(&tz(Second, "Pacific/Apia"), None) + .unwrap() + .into_array(2) + .unwrap(); + let skipped = skipped.as_primitive::(); + assert_eq!(skipped.value(0), skipped.value(1)); } #[test] diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 6ab0ca903928c..64d675ff2ffdd 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -5109,6 +5109,17 @@ mod tests { ScalarValue::Date32(Some(0)), false, ), + ( + DataType::Timestamp(Nanosecond, new_york.clone()), + ScalarValue::Date64(Some(0)), + false, + ), + // a local time that a transition skips moves past later ones + ( + DataType::Timestamp(Nanosecond, None), + ScalarValue::TimestampNanosecond(Some(0), new_york.clone()), + false, + ), ( DataType::Int64, ScalarValue::TimestampNanosecond(Some(0), None), @@ -5128,7 +5139,7 @@ mod tests { ), ( DataType::Timestamp(Nanosecond, None), - ScalarValue::TimestampNanosecond(Some(0), new_york.clone()), + ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("-05:00"))), true, ), ( diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt index c83568310f83d..6dedfa39c8c49 100644 --- a/datafusion/sqllogictest/test_files/cast.slt +++ b/datafusion/sqllogictest/test_files/cast.slt @@ -142,6 +142,19 @@ ORDER BY d; 562129260 1987-10-24 562129259 1987-10-25 +# Date64 also takes the local date, so it reverses the same way. +query ID +SELECT arrow_cast(ts, 'Int64') AS epoch, arrow_cast(ts, 'Date64') AS d +FROM ( + SELECT arrow_cast(column1, 'Timestamp(Second, Some("America/Goose_Bay"))') AS ts + FROM (VALUES (562129259::bigint), (562129260::bigint)) + ORDER BY ts LIMIT 2 +) +ORDER BY d; +---- +562129260 1987-10-24T00:00:00 +562129259 1987-10-25T00:00:00 + # Reducing timestamp precision remains order-preserving, even when values merge. statement ok SET datafusion.explain.physical_plan_only = true; diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 2745a29c70534..3e5a35e0fe6c8 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1612,11 +1612,11 @@ RESET datafusion.explain.physical_plan_only; statement ok CREATE TABLE cast_order_src(i INT, ts TIMESTAMP) AS VALUES - (-1, TIMESTAMP '2024-03-10 23:00:00'), - (0, TIMESTAMP '2024-03-11 00:00:00'), - (1, TIMESTAMP '2024-03-11 01:00:00'), - (255, TIMESTAMP '2024-03-11 02:00:00'), - (256, TIMESTAMP '2024-03-11 03:00:00'); + (-1, TIMESTAMP '2024-03-10 00:00:00'), + (0, TIMESTAMP '2024-03-10 01:00:00'), + (1, TIMESTAMP '2024-03-10 02:30:00'), + (255, TIMESTAMP '2024-03-10 02:45:00'), + (256, TIMESTAMP '2024-03-10 03:00:00'); statement ok COPY cast_order_src @@ -1655,26 +1655,53 @@ SELECT i FROM cast_order WHERE CAST(i AS BYTEA) < X'05' ORDER BY i; 1 256 -# Comparing a timestamp without a timezone to one with a named timezone casts -# the column to that timezone, reading the values as New York local times -# (05:30 UTC is 01:30 local). That cast keeps the order, so it still prunes. +# Comparing a timestamp without a timezone to one with a timezone casts the +# column to that timezone, reading the values as local times. New York skipped +# 02:00 to 03:00 on 2024-03-10, and Arrow reads 02:30 as 03:30 (07:30 UTC), +# after 03:00 (07:00 UTC). The bound 03:00 is no longer the largest value after +# the cast, so this cast must not prune. statement ok SET datafusion.execution.time_zone = 'America/New_York'; +query I +SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 + +query I +SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 + +# A fixed offset skips no local times, so that cast keeps the order and still +# prunes. +statement ok +SET datafusion.execution.time_zone = '-05:00'; + statement ok SET datafusion.explain.physical_plan_only = true; query TT -EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-11 05:30:00+00:00'; +EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00'; ---- physical_plan -01)FilterExec: CAST(ts@1 AS Timestamp(ns, "America/New_York")) > 1710135000000000000, projection=[i@0] +01)FilterExec: CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, projection=[i@0] 02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "America/New_York")) > 1710135000000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "America/New_York");N) > 1710135000000000000, required_guarantees=[] +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "-05:00");N) > 1710054900000000000, required_guarantees=[] query I -SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-11 05:30:00+00:00' ORDER BY i; +SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; ---- +1 +255 +256 + +query I +SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 255 256 From ff13f59a1d126e59b9a69b2c5e1fb28234cc11f9 Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Sat, 7 Nov 2026 11:44:08 +0800 Subject: [PATCH 7/8] fix: accept Time64 casts now that Arrow checks them for overflow --- datafusion/physical-expr/src/expressions/cast.rs | 16 +++++++--------- datafusion/pruning/src/pruning_predicate.rs | 6 ++++++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 0390d7667f445..945bfe662ae17 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -403,7 +403,6 @@ fn is_temporal_count(data_type: &DataType) -> bool { /// Values that fail to convert are not covered: callers must account for the /// error or NULL a failure produces. pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool { - use arrow::datatypes::TimeUnit::*; if source_type == target_type || (source_type.is_numeric() || *source_type == Boolean) && target_type.is_numeric() @@ -423,12 +422,11 @@ pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) } // Both date types take the date in the timestamp's timezone. (Timestamp(_, Some(tz)), Date32 | Date64) => is_fixed_offset(tz), - // Only admit scaling that cannot wrap on overflow. Time64 widening - // and narrowing to Time32 do not currently check overflow in Arrow. - (Time32(Second | Millisecond), Time32(Second | Millisecond)) - | (Time32(Second | Millisecond), Time64(Microsecond | Nanosecond)) - | (Time64(Nanosecond), Time64(Microsecond)) - | (Duration(_), Duration(_)) => true, + // Arrow checks every scaling that can overflow, so a value that does + // not fit fails to convert instead of wrapping. + (Time32(_) | Time64(_), Time32(_) | Time64(_)) | (Duration(_), Duration(_)) => { + true + } // Changing the timezone only changes metadata on the epoch value. (Timestamp(_, Some(_)), Timestamp(_, _)) => true, // Adding a timezone reads a naive timestamp (or a Date64, which may @@ -2255,8 +2253,8 @@ mod tests { (Time32(Millisecond), Time32(Second), true), (Time32(Second), Time64(Nanosecond), true), (Time64(Nanosecond), Time64(Microsecond), true), - (Time64(Microsecond), Time64(Nanosecond), false), - (Time64(Nanosecond), Time32(Second), false), + (Time64(Microsecond), Time64(Nanosecond), true), + (Time64(Nanosecond), Time32(Second), true), (Duration(Second), Duration(Nanosecond), true), (Duration(Nanosecond), Duration(Second), true), (Null, Timestamp(Second, None), false), diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 64d675ff2ffdd..b7325d90d5672 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -5126,6 +5126,12 @@ mod tests { true, ), (DataType::Int32, ScalarValue::Date32(Some(0)), true), + // Parquet TIME(MICROS) read as a TIME column + ( + DataType::Time64(Microsecond), + ScalarValue::Time64Nanosecond(Some(0)), + true, + ), ( DataType::Utf8, ScalarValue::Utf8View(Some("a".into())), From 9648551350dad1b604c8e6094ee5c4872e5de32b Mon Sep 17 00:00:00 2001 From: Jay Zhan Date: Sat, 7 Nov 2026 11:46:07 +0800 Subject: [PATCH 8/8] test: check the cast order rule against TRY_CAST as well --- .../physical-expr/src/expressions/cast.rs | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 945bfe662ae17..a832ce06629e3 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -2394,18 +2394,27 @@ mod tests { } /// Casts the sorted `samples` to `target` with the strict cast that - /// `CastExpr` runs by default, and returns the first pair of converted - /// values whose order the cast reverses. Each value is cast on its own, - /// and values that fail to convert are skipped: `is_order_preserving_cast` - /// does not cover them. - fn order_violation(samples: &ArrayRef, target: &DataType) -> Option { + /// `CastExpr` runs by default (`safe = false`) or the cast that + /// `TryCastExpr` runs (`safe = true`), and returns the first pair of + /// converted values whose order the cast reverses. Each value is cast on + /// its own, and values that fail to convert are skipped: + /// `is_order_preserving_cast` does not cover them. + fn order_violation( + samples: &ArrayRef, + target: &DataType, + safe: bool, + ) -> Option { use arrow::array::make_comparator; use arrow::compute::SortOptions; + let options = CastOptions { + safe, + format_options: DEFAULT_FORMAT_OPTIONS, + }; let converted = (0..samples.len()) .filter_map(|i| { let value = ColumnarValue::Array(samples.slice(i, 1)); - match value.cast_to(target, None) { + match value.cast_to(target, Some(&options)) { Ok(ColumnarValue::Array(output)) if output.is_valid(0) => { Some((i, output)) } @@ -2431,11 +2440,12 @@ mod tests { }) } - /// Checks `is_order_preserving_cast` against the strict cast that - /// `CastExpr` runs: every pair it accepts must keep the order of the sample - /// values that convert. - /// Pruning and sort elimination both rely on this rule, so a wrong entry - /// drops matching rows or returns unsorted output. + /// Checks `is_order_preserving_cast` against the casts that `CastExpr` and + /// `TryCastExpr` run: every pair it accepts must keep the order of the + /// sample values that convert. + /// Pruning (for both casts) and sort elimination (for the strict cast) rely + /// on this rule, so a wrong entry drops matching rows or returns unsorted + /// output. #[test] fn test_order_preserving_cast_matches_cast_kernel() { use TimeUnit::*; @@ -2490,8 +2500,11 @@ mod tests { { continue; } - if let Some(e) = order_violation(&samples, target) { - violations.push(format!("{source} -> {target}: {e}")); + for safe in [false, true] { + if let Some(e) = order_violation(&samples, target, safe) { + violations + .push(format!("{source} -> {target} (safe={safe}): {e}")); + } } } } @@ -2511,7 +2524,7 @@ mod tests { assert!(!is_order_preserving_cast(&source, &target)); let samples = order_samples(&source).unwrap(); assert!( - order_violation(&samples, &target).is_some(), + order_violation(&samples, &target, false).is_some(), "{source} -> {target}: no sample exposes the reversed order" ); }