Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions crates/spark/src/function/hash/crc32.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,18 @@ use datafusion_common::cast::{
as_binary_array, as_binary_view_array, as_fixed_size_binary_array,
as_large_binary_array,
};
use datafusion_common::types::{NativeType, logical_string};
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
ScalarUDFImpl, Signature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;

use crate::function::implicit_cast::{SparkInputType, SparkInputTypes};

/// Spark-compatible `crc32` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#crc32>
Expand All @@ -45,7 +47,7 @@ NULL
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkCrc32 {
signature: Signature,
input_types: SparkInputTypes,
}

impl Default for SparkCrc32 {
Expand All @@ -57,12 +59,9 @@ impl Default for SparkCrc32 {
impl SparkCrc32 {
pub fn new() -> Self {
Self {
signature: Signature::coercible(
vec![Coercion::new_implicit(
TypeSignatureClass::Binary,
vec![TypeSignatureClass::Native(logical_string())],
NativeType::Binary,
)],
// Spark's `Crc32.inputTypes`
input_types: SparkInputTypes::new(
vec![SparkInputType::Binary],
Volatility::Immutable,
),
}
Expand All @@ -75,7 +74,16 @@ impl ScalarUDFImpl for SparkCrc32 {
}

fn signature(&self) -> &Signature {
&self.signature
self.input_types.signature()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
self.input_types.coerce(self.name(), arg_types)
}

fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {
let input_types = self.input_types.with_config(config)?;
Some(ScalarUDF::from(Self { input_types }))
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Expand Down
49 changes: 49 additions & 0 deletions crates/sqllogictest/test_files/hash/crc32.slt
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,52 @@ query I
select crc32(arrow_cast(null, 'Dictionary(Int32, Utf8)'))
----
NULL

query I
SELECT crc32(arrow_cast('Spark', 'Dictionary(Int32, Utf8)'));
----
1557323817

query I
SELECT crc32(v) FROM (VALUES ('Spark'), (NULL), ('Spark')) AS t(v);
----
1557323817
NULL
1557323817

# Spark's `Crc32.inputTypes` is BINARY. Only a STRING argument is cast to it.
query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09
SELECT crc32(1::INT);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "BIGINT"\. SQLSTATE: 42K09
SELECT crc32(1);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09
SELECT crc32(1.5::DOUBLE);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09
SELECT crc32(true);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "DATE"\. SQLSTATE: 42K09
SELECT crc32(DATE '2024-01-01');

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "ARRAY<STRING>"\. SQLSTATE: 42K09
SELECT crc32(array('a'));

query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `crc32` requires 1 parameters but the actual number is 2\.
SELECT crc32('a', 'b');

# With ANSI mode on, a STRING argument is still cast to BINARY, and an INT is still rejected.
statement ok
set datafusion.execution.enable_ansi_mode = true;

query I
SELECT crc32('Spark');
----
1557323817

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09
SELECT crc32(1::INT);

statement ok
set datafusion.execution.enable_ansi_mode = false;
Loading