diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index d605f0cf49afd..23291099ecac7 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -1663,22 +1663,25 @@ mod tests { fn bloom_filter_pruning_error_retains_row_group_without_match() { let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); - let cast_expr: Arc = Arc::new(CastExpr::new( - Arc::new(PhysicalColumn::new("c1", 0)), - DataType::FixedSizeBinary(4), + // c1 = CAST('abc' AS Int32): constant folding fails, so the + // simplifier keeps the cast, and the cast fails at evaluation. + let value: Arc = Arc::new(CastExpr::new( + Arc::new(Literal::new(ScalarValue::from("abc"))), + DataType::Int32, None, )); - let literal: Arc = Arc::new(Literal::new( - ScalarValue::FixedSizeBinary(4, Some(vec![0, 0, 0, 1])), + let expr = Arc::new(BinaryExpr::new( + Arc::new(PhysicalColumn::new("c1", 0)), + Operator::Eq, + value, )); - let expr = Arc::new(BinaryExpr::new(cast_expr, Operator::Eq, literal)); let pruning_predicate = PruningPredicateBuilder::new() .with_file_schema(schema) .try_build(expr) .unwrap(); // A nonempty map bypasses the no-filter path. Evaluating the predicate - // then fails because Arrow cannot cast Int32 statistics to FixedSizeBinary. + // then fails because 'abc' is not an Int32. let mut sbbf = Sbbf::new_with_ndv_fpp(10, 0.01).unwrap(); sbbf.insert(&1_i32); let mut bloom_statistics = BloomFilterStatistics::new(); diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs index 7fbed9187b971..a832ce06629e3 100644 --- a/datafusion/physical-expr/src/expressions/cast.rs +++ b/datafusion/physical-expr/src/expressions/cast.rs @@ -379,38 +379,67 @@ fn is_fixed_offset(tz: &str) -> bool { tz == "UTC" || tz.starts_with(['+', '-']) } -/// Whether successful casts preserve order when conversion failures return errors. -/// Unlike `check_bigger_cast`, this allows precision loss and is not sufficient -/// for propagating distinct counts or the ordering of subsequent sort keys. -fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool { - use arrow::datatypes::TimeUnit::*; +fn is_binary(data_type: &DataType) -> bool { + matches!( + data_type, + Binary | LargeBinary | BinaryView | FixedSizeBinary(_) + ) +} + +/// Temporal types stored as a count of units since the epoch (or midnight), +/// which a cast from or to a number reads or writes as that count. +fn is_temporal_count(data_type: &DataType) -> bool { + matches!( + data_type, + Date32 | Date64 | Timestamp(_, _) | Time32(_) | Time64(_) | Duration(_) + ) +} + +/// Whether the cast never reverses the order of two values that both convert: +/// `a <= b` implies `cast(a) <= cast(b)`. Precision loss may merge values, so +/// this is not sufficient for propagating distinct counts or the ordering of +/// subsequent sort keys (see [`CastExpr::check_bigger_cast`] for that). +/// +/// Values that fail to convert are not covered: callers must account for the +/// error or NULL a failure produces. +pub fn is_order_preserving_cast(source_type: &DataType, target_type: &DataType) -> bool { if source_type == target_type || (source_type.is_numeric() || *source_type == Boolean) && target_type.is_numeric() + // Changing the string or binary encoding keeps the byte order. + || source_type.is_string() && target_type.is_string() + || is_binary(source_type) && is_binary(target_type) + || source_type.is_numeric() && is_temporal_count(target_type) + || is_temporal_count(source_type) && target_type.is_numeric() { return true; } // Temporal casts are not generally monotonic: extracting time-of-day wraps // at midnight, and timezone transitions can reverse the local date. match (source_type, target_type) { - (Date32 | Date64, Date32 | Date64) - | (Date32 | Date64, Timestamp(_, None)) - | (Timestamp(_, None), Date32) => true, - (Timestamp(_, Some(tz)), Date32) => is_fixed_offset(tz), - // Arrow converts timestamps to Date64 by scaling the epoch value, - // whereas Date32 extracts the date in the timestamp's timezone. - (Timestamp(_, _), Date64) => true, - // Only admit scaling that cannot wrap on overflow. Time64 widening - // and narrowing to Time32 do not currently check overflow in Arrow. - (Time32(Second | Millisecond), Time32(Second | Millisecond)) - | (Time32(Second | Millisecond), Time64(Microsecond | Nanosecond)) - | (Time64(Nanosecond), Time64(Microsecond)) - | (Duration(_), Duration(_)) => true, - (Timestamp(_, from_tz), Timestamp(_, to_tz)) => { - // Adding a timezone to naive timestamps interprets local times; - // other timezone changes only change metadata on the epoch value. - from_tz.is_some() || to_tz.as_deref().is_none_or(is_fixed_offset) + (Date32 | Date64, Date32 | Date64) | (Timestamp(_, None), Date32 | Date64) => { + true + } + // Both date types take the date in the timestamp's timezone. + (Timestamp(_, Some(tz)), Date32 | Date64) => is_fixed_offset(tz), + // Arrow checks every scaling that can overflow, so a value that does + // not fit fails to convert instead of wrapping. + (Time32(_) | Time64(_), Time32(_) | Time64(_)) | (Duration(_), Duration(_)) => { + true } + // Changing the timezone only changes metadata on the epoch value. + (Timestamp(_, Some(_)), Timestamp(_, _)) => true, + // Adding a timezone reads a naive timestamp (or a Date64, which may + // hold any time of day) as a local time. Arrow moves a local time that + // a transition skips forward by the length of the gap, past the local + // times just after the gap, so only a fixed offset keeps the order. + (Timestamp(_, None) | Date64, Timestamp(_, tz)) => { + tz.as_deref().is_none_or(is_fixed_offset) + } + // A Date32 reads as local midnight. A skipped midnight moves forward + // by the gap, and no gap is longer than a day, so it never passes the + // next midnight. + (Date32, Timestamp(_, _)) => true, _ => false, } } @@ -2186,11 +2215,16 @@ mod tests { (Date64, Date32, true), (Date32, Timestamp(Nanosecond, None), true), (Date64, Timestamp(Second, None), true), + (Date32, Timestamp(Second, timezone.clone()), true), + (Date64, Timestamp(Second, Some("+08:00".into())), true), + (Date64, Timestamp(Second, timezone.clone()), false), (Timestamp(Second, None), Date32, true), + (Timestamp(Second, None), Date64, true), (Timestamp(Second, Some("UTC".into())), Date32, true), (Timestamp(Second, Some("+08:00".into())), Date32, true), + (Timestamp(Second, Some("+08:00".into())), Date64, true), (Timestamp(Second, timezone.clone()), Date32, false), - (Timestamp(Second, timezone.clone()), Date64, true), + (Timestamp(Second, timezone.clone()), Date64, false), (Timestamp(Second, None), Timestamp(Nanosecond, None), true), (Timestamp(Nanosecond, None), Timestamp(Second, None), true), ( @@ -2219,8 +2253,8 @@ mod tests { (Time32(Millisecond), Time32(Second), true), (Time32(Second), Time64(Nanosecond), true), (Time64(Nanosecond), Time64(Microsecond), true), - (Time64(Microsecond), Time64(Nanosecond), false), - (Time64(Nanosecond), Time32(Second), false), + (Time64(Microsecond), Time64(Nanosecond), true), + (Time64(Nanosecond), Time32(Second), true), (Duration(Second), Duration(Nanosecond), true), (Duration(Nanosecond), Duration(Second), true), (Null, Timestamp(Second, None), false), @@ -2263,6 +2297,250 @@ mod tests { } } + /// Sorted, distinct, non-null sample values of `data_type`, obtained by + /// casting a fixed set of seeds and keeping the values that convert. + fn order_samples(data_type: &DataType) -> Option { + use arrow::array::{ + BooleanArray, FixedSizeBinaryArray, Float64Array, TimestampSecondArray, + UInt32Array, make_comparator, + }; + use arrow::compute::{ + SortOptions, concat, filter, is_not_null, sort_to_indices, take, + }; + + let seeds: Vec = vec![ + Arc::new(Int64Array::from(vec![ + i64::MIN, + -2_147_483_649, + -129, + -1, + 0, + 1, + 9, + 10, + 127, + 128, + 255, + 256, + // As dates, 2011-12-29 to 2011-12-31: Pacific/Apia skipped + // all of 2011-12-30. + 15_337, + 15_338, + 15_339, + 86_399, + // America/Goose_Bay fell back from 00:01 to 23:01 the previous + // day at this instant, so the local date goes backwards. + 562_129_259, + 562_129_260, + 2_147_483_647, + i64::MAX, + ])), + Arc::new(Float64Array::from(vec![ + f64::NEG_INFINITY, + -1.5, + -0.5, + 0.0, + 0.5, + 1.7, + 1e300, + f64::INFINITY, + ])), + Arc::new(StringArray::from(vec!["", "-1", "0", "1", "10", "9", "a"])), + Arc::new(BooleanArray::from(vec![false, true])), + Arc::new( + FixedSizeBinaryArray::try_from_iter( + ["abcd", "abce", "zzzz"].into_iter().map(str::as_bytes), + ) + .expect("valid fixed size binary"), + ), + Arc::new(TimestampSecondArray::from(vec![ + // 23:00, 00:00 and 01:00 around midnight + 82_800, + 86_400, + 90_000, + // Local times around the America/New_York transitions of + // 2024: before, inside and after the skipped hour in March and + // the repeated hour in November. + 1_710_035_999, + 1_710_037_800, + 1_710_039_600, + 1_730_595_599, + 1_730_597_400, + 1_730_599_200, + ])), + ]; + // Cast each seed on its own, so one value that fails to convert does + // not discard the others. + let parts = seeds + .iter() + .flat_map(|seed| (0..seed.len()).map(|i| seed.slice(i, 1))) + .filter_map(|seed| arrow::compute::cast(&seed, data_type).ok()) + .map(|array| filter(&array, &is_not_null(&array).unwrap()).unwrap()) + .filter(|array| !array.is_empty()) + .collect::>(); + if parts.is_empty() { + return None; + } + let parts = parts.iter().map(|a| a.as_ref()).collect::>(); + let all = concat(&parts).unwrap(); + let sorted = + take(&all, &sort_to_indices(&all, None, None).unwrap(), None).unwrap(); + let cmp = make_comparator(&sorted, &sorted, SortOptions::default()).unwrap(); + let distinct = (0..sorted.len()) + .filter(|&i| i == 0 || cmp(i - 1, i).is_ne()) + .map(|i| i as u32) + .collect::(); + Some(take(&sorted, &distinct, None).unwrap()) + } + + /// Casts the sorted `samples` to `target` with the strict cast that + /// `CastExpr` runs by default (`safe = false`) or the cast that + /// `TryCastExpr` runs (`safe = true`), and returns the first pair of + /// converted values whose order the cast reverses. Each value is cast on + /// its own, and values that fail to convert are skipped: + /// `is_order_preserving_cast` does not cover them. + fn order_violation( + samples: &ArrayRef, + target: &DataType, + safe: bool, + ) -> Option { + use arrow::array::make_comparator; + use arrow::compute::SortOptions; + + let options = CastOptions { + safe, + format_options: DEFAULT_FORMAT_OPTIONS, + }; + let converted = (0..samples.len()) + .filter_map(|i| { + let value = ColumnarValue::Array(samples.slice(i, 1)); + match value.cast_to(target, Some(&options)) { + Ok(ColumnarValue::Array(output)) if output.is_valid(0) => { + Some((i, output)) + } + _ => None, + } + }) + .collect::>(); + converted.windows(2).find_map(|w| { + let ((i, left), (j, right)) = (&w[0], &w[1]); + let cmp = make_comparator(left, right, SortOptions::default()).unwrap(); + cmp(0, 0).is_gt().then(|| { + let value = |array: &ArrayRef, i| { + ScalarValue::try_from_array(array, i).unwrap().to_string() + }; + format!( + "{} -> {}, but {} -> {}", + value(samples, *i), + value(left, 0), + value(samples, *j), + value(right, 0), + ) + }) + }) + } + + /// Checks `is_order_preserving_cast` against the casts that `CastExpr` and + /// `TryCastExpr` run: every pair it accepts must keep the order of the + /// sample values that convert. + /// Pruning (for both casts) and sort elimination (for the strict cast) rely + /// on this rule, so a wrong entry drops matching rows or returns unsorted + /// output. + #[test] + fn test_order_preserving_cast_matches_cast_kernel() { + use TimeUnit::*; + use arrow::array::{AsArray, Date32Array}; + + let tz = |unit, tz: &str| Timestamp(unit, Some(tz.into())); + let types = [ + Boolean, + Int8, + Int16, + Int32, + Int64, + UInt8, + UInt16, + UInt32, + UInt64, + Float32, + Float64, + Decimal128(10, 2), + Decimal256(40, 2), + Utf8, + LargeUtf8, + Utf8View, + Binary, + LargeBinary, + BinaryView, + FixedSizeBinary(4), + Date32, + Date64, + Timestamp(Second, None), + Timestamp(Nanosecond, None), + tz(Millisecond, "UTC"), + tz(Second, "+05:00"), + tz(Second, "America/New_York"), + tz(Second, "America/Goose_Bay"), + tz(Second, "Pacific/Apia"), + Time32(Second), + Time32(Millisecond), + Time64(Microsecond), + Time64(Nanosecond), + Duration(Second), + Duration(Nanosecond), + ]; + let mut violations = vec![]; + for source in &types { + let Some(samples) = order_samples(source) else { + continue; + }; + for target in &types { + if !can_cast_types(source, target) + || !is_order_preserving_cast(source, target) + { + continue; + } + for safe in [false, true] { + if let Some(e) = order_violation(&samples, target, safe) { + violations + .push(format!("{source} -> {target} (safe={safe}): {e}")); + } + } + } + } + assert!(violations.is_empty(), "{}", violations.join("\n")); + + // The samples are able to expose the casts the rule rejects. + for (source, target) in [ + (Utf8, Int32), + (Int32, Boolean), + (Int16, Binary), + (Timestamp(Second, None), Time32(Second)), + (tz(Second, "+05:00"), Time64(Nanosecond)), + (tz(Second, "America/Goose_Bay"), Date32), + (tz(Second, "America/Goose_Bay"), Date64), + (Timestamp(Second, None), tz(Second, "America/New_York")), + ] { + assert!(!is_order_preserving_cast(&source, &target)); + let samples = order_samples(&source).unwrap(); + assert!( + order_violation(&samples, &target, false).is_some(), + "{source} -> {target}: no sample exposes the reversed order" + ); + } + + // The samples reach the edge of the Date32 entry: a skipped day starts + // at the same instant as the day after it. + let skipped = + ColumnarValue::Array(Arc::new(Date32Array::from(vec![15_338, 15_339]))) + .cast_to(&tz(Second, "Pacific/Apia"), None) + .unwrap() + .into_array(2) + .unwrap(); + let skipped = skipped.as_primitive::(); + assert_eq!(skipped.value(0), skipped.value(1)); + } + #[test] fn test_cast_ordering_with_null_on_failure() { use arrow::array::TimestampSecondArray; diff --git a/datafusion/physical-expr/src/expressions/mod.rs b/datafusion/physical-expr/src/expressions/mod.rs index 767c0b4ce40f3..bb5c818ece6a6 100644 --- a/datafusion/physical-expr/src/expressions/mod.rs +++ b/datafusion/physical-expr/src/expressions/mod.rs @@ -30,7 +30,7 @@ pub use crate::aggregate::stats::StatsType; pub use binary::{BinaryExpr, binary, similar_to}; pub use case::{CaseExpr, case}; -pub use cast::{CastExpr, cast}; +pub use cast::{CastExpr, cast, is_order_preserving_cast}; pub use column::{Column, col, with_new_schema}; pub use datafusion_expr::utils::format_state_name; pub use dynamic_filters::{ diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 2e3c84673f90c..b7325d90d5672 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -1331,10 +1331,10 @@ fn is_compare_op(op: Operator) -> bool { ) } -// The pruning logic is based on the comparing the min/max bounds. -// Must make sure the two type has order. -// For example, casts from string to numbers is not correct. -// Because the "13" is less than "3" with UTF8 comparison order. +// Pruning casts the container's min/max statistics and compares them with the +// literal, which is only sound when the cast keeps the order of the values, +// e.g. not for strings to numbers ("13" < "3") or timestamps to time of day +// (wraps at midnight). Ask the same rule that sort elimination uses. fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Result<()> { // Dictionary casts are always supported as long as the value types are supported let from_type = match from_type { @@ -1349,20 +1349,7 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re } _ => to_type, }; - // Casting a timestamp to a time-of-day wraps at midnight and does not - // preserve ordering across day boundaries. Casting timestamp min/max - // statistics could therefore produce invalid bounds and prune matches. - if matches!(from_type, DataType::Timestamp(_, _)) - && matches!(to_type, DataType::Time32(_) | DataType::Time64(_)) - { - return plan_err!( - "Try Cast/Cast with from type {from_type} to type {to_type} is not supported" - ); - } - // If both types are strings or both are not strings (number, timestamp, etc) - // then we can compare them. - // PruningPredicate does not support casting of strings to numbers and such. - if from_type.is_string() == to_type.is_string() { + if phys_expr::is_order_preserving_cast(from_type, to_type) { Ok(()) } else { plan_err!( @@ -5095,6 +5082,99 @@ mod tests { Ok(()) } + /// Pruning casts the min/max statistics, so it may only use casts that + /// keep the order of the values. + #[test] + fn row_group_predicate_cast_must_preserve_order() { + use TimeUnit::*; + + let new_york = Some(Arc::from("America/New_York")); + let dictionary = |value: DataType| { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(value)) + }; + let cases = [ + // 0 is false while -1 and 1 are true + (DataType::Int32, ScalarValue::Boolean(Some(false)), false), + (DataType::Float64, ScalarValue::Boolean(Some(false)), false), + ( + dictionary(DataType::Int64), + ScalarValue::Boolean(Some(false)), + false, + ), + // little-endian bytes: 256 is [0, 1, ..] and 255 is [255, 0, ..] + (DataType::Int64, ScalarValue::Binary(Some(vec![5])), false), + // a named timezone can move the local date backwards + ( + DataType::Timestamp(Nanosecond, new_york.clone()), + ScalarValue::Date32(Some(0)), + false, + ), + ( + DataType::Timestamp(Nanosecond, new_york.clone()), + ScalarValue::Date64(Some(0)), + false, + ), + // a local time that a transition skips moves past later ones + ( + DataType::Timestamp(Nanosecond, None), + ScalarValue::TimestampNanosecond(Some(0), new_york.clone()), + false, + ), + ( + DataType::Int64, + ScalarValue::TimestampNanosecond(Some(0), None), + true, + ), + (DataType::Int32, ScalarValue::Date32(Some(0)), true), + // Parquet TIME(MICROS) read as a TIME column + ( + DataType::Time64(Microsecond), + ScalarValue::Time64Nanosecond(Some(0)), + true, + ), + ( + DataType::Utf8, + ScalarValue::Utf8View(Some("a".into())), + true, + ), + (dictionary(DataType::Utf8), ScalarValue::from("a"), true), + ( + DataType::FixedSizeBinary(1), + ScalarValue::Binary(Some(vec![5])), + true, + ), + ( + DataType::Timestamp(Nanosecond, None), + ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("-05:00"))), + true, + ), + ( + DataType::Date32, + ScalarValue::TimestampSecond(Some(0), new_york), + true, + ), + ]; + for (source, literal, prunable) in cases { + let target = literal.data_type(); + let schema = Schema::new(vec![Field::new("c1", source.clone(), false)]); + for expr in [ + cast(col("c1"), target.clone()).eq(lit(literal.clone())), + try_cast(col("c1"), target.clone()).eq(lit(literal.clone())), + ] { + let predicate_expr = test_build_predicate_expression( + &expr, + &schema, + &mut RequiredColumns::new(), + ); + assert_eq!( + predicate_expr.to_string() != "true", + prunable, + "{source} -> {target}: {predicate_expr}" + ); + } + } + } + #[test] fn row_group_predicate_date_date() -> Result<()> { let schema = Schema::new(vec![Field::new("c1", DataType::Date32, false)]); diff --git a/datafusion/sqllogictest/test_files/cast.slt b/datafusion/sqllogictest/test_files/cast.slt index 3d11899c9304a..6dedfa39c8c49 100644 --- a/datafusion/sqllogictest/test_files/cast.slt +++ b/datafusion/sqllogictest/test_files/cast.slt @@ -142,6 +142,19 @@ ORDER BY d; 562129260 1987-10-24 562129259 1987-10-25 +# Date64 also takes the local date, so it reverses the same way. +query ID +SELECT arrow_cast(ts, 'Int64') AS epoch, arrow_cast(ts, 'Date64') AS d +FROM ( + SELECT arrow_cast(column1, 'Timestamp(Second, Some("America/Goose_Bay"))') AS ts + FROM (VALUES (562129259::bigint), (562129260::bigint)) + ORDER BY ts LIMIT 2 +) +ORDER BY d; +---- +562129260 1987-10-24T00:00:00 +562129259 1987-10-25T00:00:00 + # Reducing timestamp precision remains order-preserving, even when values merge. statement ok SET datafusion.explain.physical_plan_only = true; @@ -164,6 +177,35 @@ physical_plan 02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] 03)----DataSourceExec: partitions=1, partition_sizes=[1] +# An integer read as a count of seconds keeps its order, so the outer sort is +# removed. +query TT +EXPLAIN SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts +FROM ( + SELECT column1 AS i + FROM (VALUES (3::bigint), (1::bigint), (2::bigint)) + ORDER BY i LIMIT 3 +) +ORDER BY ts; +---- +physical_plan +01)ProjectionExec: expr=[CAST(column1@0 AS Timestamp(s)) as ts] +02)--SortExec: TopK(fetch=3), expr=[column1@0 ASC NULLS LAST], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +query P +SELECT arrow_cast(i, 'Timestamp(Second, None)') AS ts +FROM ( + SELECT column1 AS i + FROM (VALUES (3::bigint), (1::bigint), (2::bigint)) + ORDER BY i LIMIT 3 +) +ORDER BY ts; +---- +1970-01-01T00:00:01 +1970-01-01T00:00:02 +1970-01-01T00:00:03 + statement ok SET datafusion.explain.physical_plan_only = false; diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index ea61f0be6c6a4..3e5a35e0fe6c8 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1602,3 +1602,117 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok RESET datafusion.explain.physical_plan_only; + +########## +# Pruning casts the min/max statistics of each row group, so it may only use a +# cast that keeps the order of the values. Each predicate below runs against +# an in-memory table, which is never pruned, and the same rows in one Parquet +# row group. The two must agree. +########## + +statement ok +CREATE TABLE cast_order_src(i INT, ts TIMESTAMP) AS VALUES + (-1, TIMESTAMP '2024-03-10 00:00:00'), + (0, TIMESTAMP '2024-03-10 01:00:00'), + (1, TIMESTAMP '2024-03-10 02:30:00'), + (255, TIMESTAMP '2024-03-10 02:45:00'), + (256, TIMESTAMP '2024-03-10 03:00:00'); + +statement ok +COPY cast_order_src +TO 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet' +STORED AS PARQUET; + +statement ok +CREATE EXTERNAL TABLE cast_order +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/cast_order/data.parquet'; + +# -1 and 1 are true and 0 is false, so the bounds -1 and 256 both cast to true. +query I +SELECT i FROM cast_order_src WHERE CAST(i AS BOOLEAN) <= false; +---- +0 + +query I +SELECT i FROM cast_order WHERE CAST(i AS BOOLEAN) <= false; +---- +0 + +# An integer casts to its little-endian bytes: -1 is X'FFFFFFFF' and 256 is +# X'00010000', so the bounds no longer bracket the values in between. +query I +SELECT i FROM cast_order_src WHERE CAST(i AS BYTEA) < X'05' ORDER BY i; +---- +0 +1 +256 + +query I +SELECT i FROM cast_order WHERE CAST(i AS BYTEA) < X'05' ORDER BY i; +---- +0 +1 +256 + +# Comparing a timestamp without a timezone to one with a timezone casts the +# column to that timezone, reading the values as local times. New York skipped +# 02:00 to 03:00 on 2024-03-10, and Arrow reads 02:30 as 03:30 (07:30 UTC), +# after 03:00 (07:00 UTC). The bound 03:00 is no longer the largest value after +# the cast, so this cast must not prune. +statement ok +SET datafusion.execution.time_zone = 'America/New_York'; + +query I +SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 + +query I +SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 + +# A fixed offset skips no local times, so that cast keeps the order and still +# prunes. +statement ok +SET datafusion.execution.time_zone = '-05:00'; + +statement ok +SET datafusion.explain.physical_plan_only = true; + +query TT +EXPLAIN SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00'; +---- +physical_plan +01)FilterExec: CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, projection=[i@0] +02)--RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/push_down_filter_parquet/cast_order/data.parquet]]}, projection=[i, ts], file_type=parquet, predicate=CAST(ts@1 AS Timestamp(ns, "-05:00")) > 1710054900000000000, pruning_predicate=ts_null_count@1 != row_count@2 AND CAST(ts_max@0 AS Timestamp(ns, "-05:00");N) > 1710054900000000000, required_guarantees=[] + +query I +SELECT i FROM cast_order_src WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 +256 + +query I +SELECT i FROM cast_order WHERE ts > TIMESTAMPTZ '2024-03-10 07:15:00+00:00' ORDER BY i; +---- +1 +255 +256 + +statement ok +RESET datafusion.explain.physical_plan_only; + +statement ok +RESET datafusion.execution.time_zone; + +statement ok +DROP TABLE cast_order; + +statement ok +DROP TABLE cast_order_src;