Skip to content
25 changes: 9 additions & 16 deletions crates/spark/src/function/datetime/date_from_unix_date.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,16 @@ use std::sync::Arc;
use arrow::array::{Array, Date32Array};
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::cast::as_int32_array;
use datafusion_common::types::{NativeType, logical_int32, logical_string};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, Volatility,
};
use datafusion_macros::user_doc;

use super::date_part_name::int32_coercion;

/// Spark-compatible `date_from_unix_date` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#date_from_unix_date>
Expand Down Expand Up @@ -56,20 +57,12 @@ impl Default for SparkDateFromUnixDate {

impl SparkDateFromUnixDate {
pub fn new() -> Self {
// Spark declares the argument as INT and implicitly casts other numeric
// and string types to it. DataFusion types bare integer literals as BIGINT.
let int32 = Coercion::new_implicit(
TypeSignatureClass::Native(logical_int32()),
vec![
TypeSignatureClass::Integer,
TypeSignatureClass::Float,
TypeSignatureClass::Decimal,
TypeSignatureClass::Native(logical_string()),
],
NativeType::Int32,
);
// Spark declares the argument as INT.
Self {
signature: Signature::coercible(vec![int32], Volatility::Immutable),
signature: Signature::coercible(
vec![int32_coercion()],
Volatility::Immutable,
),
}
}
}
Expand Down
25 changes: 22 additions & 3 deletions crates/spark/src/function/datetime/date_part_name.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,15 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! Shared code for `dayname` and `monthname`, which map one [`DatePart`] of a
//! date to a name.
//! Shared code for the date functions: the argument coercions, and the
//! [`DatePart`] to name mapping that `dayname` and `monthname` use.

use std::sync::Arc;

use arrow::array::{AsArray, StringArray};
use arrow::compute::{DatePart, date_part};
use arrow::datatypes::Int32Type;
use datafusion_common::types::{NativeType, logical_date, logical_string};
use datafusion_common::types::{NativeType, logical_date, logical_int32, logical_string};
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
Coercion, ColumnarValue, Signature, TypeSignatureClass, Volatility,
Expand All @@ -34,6 +34,25 @@ pub(super) fn date_signature() -> Signature {
)
}

/// One INT argument. Spark implicitly casts other numeric and string types to
/// INT, and DataFusion types bare integer literals as BIGINT. The cast is
/// DataFusion's, which errors in cases where Spark with ANSI mode off returns a
/// value or NULL (see <https://github.com/apache/datafusion-spark/issues/88>). A
/// stopgap until input types are declared and coerced in one place
/// (<https://github.com/apache/datafusion-spark/issues/346>).
pub(super) fn int32_coercion() -> Coercion {
Coercion::new_implicit(
TypeSignatureClass::Native(logical_int32()),
vec![
TypeSignatureClass::Integer,
TypeSignatureClass::Float,
TypeSignatureClass::Decimal,
TypeSignatureClass::Native(logical_string()),
],
NativeType::Int32,
)
}

/// Extracts `part` from each date in `arg` and maps it to a name with `name`.
/// NULL dates, and values `name` returns `None` for, give NULL.
pub(super) fn date_part_name(
Expand Down
Loading
Loading