Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions crates/spark/src/function/hash/sha1.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,20 @@ use datafusion_common::cast::{
as_binary_array, as_binary_view_array, as_fixed_size_binary_array,
as_large_binary_array,
};
use datafusion_common::types::{NativeType, logical_string};
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::hex::{HexCase, encode_bytes_into};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF,
ScalarUDFImpl, Signature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;
use sha1::{Digest, Sha1};

use crate::function::implicit_cast::{SparkInputType, SparkInputTypes};

/// Spark-compatible `sha1` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#sha1>
Expand All @@ -44,7 +46,7 @@ NULL
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkSha1 {
signature: Signature,
input_types: SparkInputTypes,
aliases: Vec<String>,
}

Expand All @@ -57,12 +59,9 @@ impl Default for SparkSha1 {
impl SparkSha1 {
pub fn new() -> Self {
Self {
signature: Signature::coercible(
vec![Coercion::new_implicit(
TypeSignatureClass::Binary,
vec![TypeSignatureClass::Native(logical_string())],
NativeType::Binary,
)],
// Spark's `Sha1.inputTypes`
input_types: SparkInputTypes::new(
vec![SparkInputType::Binary],
Volatility::Immutable,
),
aliases: vec!["sha".to_string()],
Expand All @@ -80,7 +79,19 @@ impl ScalarUDFImpl for SparkSha1 {
}

fn signature(&self) -> &Signature {
&self.signature
self.input_types.signature()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
self.input_types.coerce(self.name(), arg_types)
}

fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {
let input_types = self.input_types.with_config(config)?;
Some(ScalarUDF::from(Self {
input_types,
aliases: self.aliases.clone(),
}))
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Expand Down
49 changes: 49 additions & 0 deletions crates/sqllogictest/test_files/hash/sha1.slt
Original file line number Diff line number Diff line change
Expand Up @@ -67,3 +67,52 @@ query T
SELECT sha1(arrow_cast('Spark', 'BinaryView'));
----
85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c

query T
SELECT sha1(arrow_cast('Spark', 'Dictionary(Int32, Utf8)'));
----
85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c

query T
SELECT sha1(v) FROM (VALUES ('Spark'), (NULL), ('Spark')) AS t(v);
----
85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c
NULL
85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c

# Spark's `Sha1.inputTypes` is BINARY. Only a STRING argument is cast to it.
query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09
SELECT sha1(1::INT);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "BIGINT"\. SQLSTATE: 42K09
SELECT sha1(1);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09
SELECT sha1(1.5::DOUBLE);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09
SELECT sha1(true);

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "DATE"\. SQLSTATE: 42K09
SELECT sha1(DATE '2024-01-01');

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "ARRAY<STRING>"\. SQLSTATE: 42K09
SELECT sha1(array('a'));

query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `sha1` requires 1 parameters but the actual number is 2\.
SELECT sha1('a', 'b');

# With ANSI mode on, a STRING argument is still cast to BINARY, and an INT is still rejected.
statement ok
set datafusion.execution.enable_ansi_mode = true;

query T
SELECT sha1('Spark');
----
85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c

query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09
SELECT sha1(1::INT);

statement ok
set datafusion.execution.enable_ansi_mode = false;
Loading