Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 61 additions & 2 deletions native/spark-expr/src/conversion_funcs/boolean.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

use crate::conversion_funcs::utils::timestamp_micros_with_tz;
use crate::SparkResult;
use arrow::array::{ArrayRef, AsArray, PrimitiveArray};
use arrow::datatypes::{DataType, TimestampMicrosecondType};
use arrow::array::{ArrayRef, AsArray, PrimitiveArray, StringBuilder};
use arrow::datatypes::{ArrowPrimitiveType, DataType, TimestampMicrosecondType};
use std::sync::Arc;

/// Boolean -> Decimal is intentionally absent: it has no native path and is routed through the
Expand All @@ -43,6 +43,34 @@ pub(crate) fn cast_boolean_to_timestamp(
Ok(timestamp_micros_with_tz(micros, target_tz))
}

/// true is 1 and false is 0, as in Spark. `from_unary` maps every slot in one pass and keeps the
/// null buffer, where Arrow's cast checks each row for null.
pub(crate) fn cast_boolean_to_numeric<T>(array_ref: &ArrayRef) -> ArrayRef
where
T: ArrowPrimitiveType,
T::Native: From<bool>,
{
Arc::new(PrimitiveArray::<T>::from_unary(
array_ref.as_boolean(),
T::Native::from,
))
}

/// "true" or "false", as in Spark, into a builder sized for the longest value, where Arrow's cast
/// formats each row through a dynamic formatter.
pub(crate) fn cast_boolean_to_utf8(array_ref: &ArrayRef) -> ArrayRef {
let bool_array = array_ref.as_boolean();
let mut builder = StringBuilder::with_capacity(bool_array.len(), bool_array.len() * 5);
for value in bool_array.iter() {
match value {
Some(true) => builder.append_value("true"),
Some(false) => builder.append_value("false"),
None => builder.append_null(),
}
}
Arc::new(builder.finish())
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -264,4 +292,35 @@ mod tests {
}
}
}

#[test]
fn test_bool_casts_match_arrow_cast() {
// The native kernels replaced Arrow's cast for these targets; on a sliced input with nulls
// they must give the same arrays.
let input: ArrayRef = Arc::new(BooleanArray::from(vec![
Some(false),
Some(true),
None,
Some(false),
None,
Some(true),
]));
let sliced = input.slice(1, 5);
for to_type in [
DataType::Int8,
DataType::Int16,
DataType::Int32,
DataType::Int64,
DataType::Float32,
DataType::Float64,
DataType::Utf8,
] {
for array in [&input, &sliced] {
let actual =
cast_array(Arc::clone(array), &to_type, &test_input_spark_opts()).unwrap();
let expected = arrow::compute::cast(array, &to_type).unwrap();
assert_eq!(&actual, &expected, "{to_type}");
}
}
}
}
100 changes: 81 additions & 19 deletions native/spark-expr/src/conversion_funcs/cast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
// under the License.

use crate::conversion_funcs::boolean::{
cast_boolean_to_timestamp, is_df_cast_from_bool_spark_compatible,
cast_boolean_to_numeric, cast_boolean_to_timestamp, cast_boolean_to_utf8,
is_df_cast_from_bool_spark_compatible,
};
use crate::conversion_funcs::numeric::{
cast_decimal128_to_float32, cast_decimal128_to_float64, cast_decimal128_to_utf8,
Expand Down Expand Up @@ -47,7 +48,10 @@ use arrow::array::{
StringArray, StructArray,
};
use arrow::buffer::{BooleanBuffer, Buffer, NullBuffer, OffsetBuffer};
use arrow::datatypes::{format_decimal_str, DataType, GenericBinaryType, Schema};
use arrow::datatypes::{
format_decimal_str, DataType, Float32Type, Float64Type, GenericBinaryType, Int16Type,
Int32Type, Int64Type, Int8Type, Schema,
};
use arrow::error::ArrowError;
use arrow::{
array::{
Expand Down Expand Up @@ -471,6 +475,13 @@ pub(crate) fn cast_array(
(Int8 | Int16 | Int32 | Int64, Timestamp(_, tz)) => cast_int_to_timestamp(&array, tz),
(Float32 | Float64, Timestamp(_, tz)) => cast_float_to_timestamp(&array, tz, eval_mode),
(Boolean, Timestamp(_, tz)) => cast_boolean_to_timestamp(&array, tz),
(Boolean, Int8) => Ok(cast_boolean_to_numeric::<Int8Type>(&array)),
(Boolean, Int16) => Ok(cast_boolean_to_numeric::<Int16Type>(&array)),
(Boolean, Int32) => Ok(cast_boolean_to_numeric::<Int32Type>(&array)),
(Boolean, Int64) => Ok(cast_boolean_to_numeric::<Int64Type>(&array)),
(Boolean, Float32) => Ok(cast_boolean_to_numeric::<Float32Type>(&array)),
(Boolean, Float64) => Ok(cast_boolean_to_numeric::<Float64Type>(&array)),
(Boolean, Utf8) => Ok(cast_boolean_to_utf8(&array)),
(Decimal128(_, scale), Timestamp(_, tz)) => cast_decimal_to_timestamp(&array, tz, *scale),
// NTZ → TIMESTAMP: interpret NTZ local-epoch value as session-TZ local time, convert to UTC.
// Must come before the is_datafusion_spark_compatible fallthrough which would
Expand Down Expand Up @@ -689,7 +700,6 @@ fn cast_array_to_string(
spark_cast_options: &SparkCastOptions,
) -> DataFusionResult<ArrayRef> {
let mut builder = StringBuilder::with_capacity(array.len(), array.len() * 16);
let mut str = String::with_capacity(array.len() * 16);

let casted_values = cast_array(
Arc::clone(array.values()),
Expand All @@ -706,25 +716,26 @@ fn cast_array_to_string(
if array.is_null(row_index) {
builder.append_null();
} else {
str.clear();
let start = offsets[row_index] as usize;
let end = offsets[row_index + 1] as usize;

str.push('[');
let mut first = true;
for idx in start..end {
if !first {
str.push_str(", ");
}
if string_values.is_null(idx) {
str.push_str(&spark_cast_options.null_string);
} else {
str.push_str(string_values.value(idx));
// Write the row straight into the builder's value buffer, then end it with an empty
// `append_value`, instead of assembling it in a scratch `String` and copying it again.
let written = builder.write_char('[').and_then(|()| {
for idx in start..end {
if idx > start {
builder.write_str(", ")?;
}
if string_values.is_null(idx) {
builder.write_str(&spark_cast_options.null_string)?;
} else {
builder.write_str(string_values.value(idx))?;
}
}
first = false;
}
str.push(']');
builder.append_value(&str);
builder.write_char(']')
});
written.expect("writing to a string builder cannot fail");
builder.append_value("");
}
}
Ok(Arc::new(builder.finish()))
Expand Down Expand Up @@ -862,7 +873,25 @@ fn write_upper_hex<W: Write>(out: &mut W, byte: u8) -> std::fmt::Result {
/// Writes `byte` reinterpreted as a signed decimal, as Spark does when printing a byte array.
#[inline]
fn write_i8<W: Write>(out: &mut W, byte: u8) -> std::fmt::Result {
write!(out, "{}", byte as i8)
let value = byte as i8;
let mut magnitude = value.unsigned_abs();
// At most "-128": a sign and three digits, filled from the end.
let mut buf = [0u8; 4];
let mut start = buf.len();
loop {
start -= 1;
buf[start] = b'0' + magnitude % 10;
magnitude /= 10;
if magnitude == 0 {
break;
}
}
if value < 0 {
start -= 1;
buf[start] = b'-';
}
// SAFETY: `buf[start..]` holds only ASCII digits and an optional '-', so it is valid UTF-8.
out.write_str(unsafe { std::str::from_utf8_unchecked(&buf[start..]) })
}

/// Writes the bytes of `value` between square brackets, encoded with `encode` and joined by
Expand Down Expand Up @@ -1932,6 +1961,39 @@ mod tests {
assert_eq!(r#"[]"#, string_array.value(3));
}

#[test]
fn test_cast_sliced_array_with_null_rows_to_string() {
// A null row writes no bytes, so the rows after it must still start at the right offset,
// and a sliced list reads its values from a non-zero offset into the child array.
let list_array = ListArray::from_iter_primitive::<Int32Type, _, _>(vec![
Some(vec![Some(0)]),
Some(vec![Some(1), None]),
None,
Some(vec![]),
None,
Some(vec![Some(-2), Some(3)]),
]);
let sliced = list_array.slice(1, 5);
let string_array =
cast_array_to_string(&sliced, &SparkCastOptions::new(EvalMode::Legacy, "UTC")).unwrap();
let string_array = string_array.as_string::<i32>();
assert_eq!(string_array.len(), 5);
assert_eq!(string_array.value(0), "[1, null]");
assert!(string_array.is_null(1));
assert_eq!(string_array.value(2), "[]");
assert!(string_array.is_null(3));
assert_eq!(string_array.value(4), "[-2, 3]");
}

#[test]
fn test_write_i8_matches_signed_display_for_every_byte() {
for byte in 0..=u8::MAX {
let mut out = String::new();
write_i8(&mut out, byte).unwrap();
assert_eq!(out, (byte as i8).to_string(), "byte {byte}");
}
}

#[test]
fn test_cast_array_of_nulls_to_array() {
let offsets_buffer = OffsetBuffer::<i32>::new(vec![0, 2, 3, 3].into());
Expand Down
Loading