From 2c64e052fceca36fb10645375479f05e66818d1f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 09:54:00 -0700 Subject: [PATCH] feat(spark): cast sha1's argument to Spark's declared input type Spark declares sha1's input type as BINARY. Use SparkInputTypes so a STRING argument is cast to BINARY and other types fail with Spark's UNEXPECTED_INPUT_TYPE and WRONG_NUM_ARGS errors. Part of #353. --- crates/spark/src/function/hash/sha1.rs | 33 ++++++++----- crates/sqllogictest/test_files/hash/sha1.slt | 49 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/crates/spark/src/function/hash/sha1.rs b/crates/spark/src/function/hash/sha1.rs index 42f44454..650baf02 100644 --- a/crates/spark/src/function/hash/sha1.rs +++ b/crates/spark/src/function/hash/sha1.rs @@ -10,18 +10,20 @@ use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, as_large_binary_array, }; -use datafusion_common::types::{NativeType, logical_string}; +use datafusion_common::config::ConfigOptions; use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use sha1::{Digest, Sha1}; +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; + /// Spark-compatible `sha1` expression. /// /// @@ -44,7 +46,7 @@ NULL )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkSha1 { - signature: Signature, + input_types: SparkInputTypes, aliases: Vec, } @@ -57,12 +59,9 @@ impl Default for SparkSha1 { impl SparkSha1 { pub fn new() -> Self { Self { - signature: Signature::coercible( - vec![Coercion::new_implicit( - TypeSignatureClass::Binary, - vec![TypeSignatureClass::Native(logical_string())], - NativeType::Binary, - )], + // Spark's `Sha1.inputTypes` + input_types: SparkInputTypes::new( + vec![SparkInputType::Binary], Volatility::Immutable, ), aliases: vec!["sha".to_string()], @@ -80,7 +79,19 @@ impl ScalarUDFImpl for SparkSha1 { } fn signature(&self) -> &Signature { - &self.signature + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.input_types.coerce(self.name(), arg_types) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { + input_types, + aliases: self.aliases.clone(), + })) } fn return_type(&self, _arg_types: &[DataType]) -> Result { diff --git a/crates/sqllogictest/test_files/hash/sha1.slt b/crates/sqllogictest/test_files/hash/sha1.slt index 8ee8db7c..c4917774 100644 --- a/crates/sqllogictest/test_files/hash/sha1.slt +++ b/crates/sqllogictest/test_files/hash/sha1.slt @@ -67,3 +67,52 @@ query T SELECT sha1(arrow_cast('Spark', 'BinaryView')); ---- 85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + +query T +SELECT sha1(arrow_cast('Spark', 'Dictionary(Int32, Utf8)')); +---- +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + +query T +SELECT sha1(v) FROM (VALUES ('Spark'), (NULL), ('Spark')) AS t(v); +---- +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c +NULL +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + +# Spark's `Sha1.inputTypes` is BINARY. Only a STRING argument is cast to it. +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT sha1(1::INT); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "BIGINT"\. SQLSTATE: 42K09 +SELECT sha1(1); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09 +SELECT sha1(1.5::DOUBLE); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT sha1(true); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "DATE"\. SQLSTATE: 42K09 +SELECT sha1(DATE '2024-01-01'); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "ARRAY"\. SQLSTATE: 42K09 +SELECT sha1(array('a')); + +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `sha1` requires 1 parameters but the actual number is 2\. +SELECT sha1('a', 'b'); + +# With ANSI mode on, a STRING argument is still cast to BINARY, and an INT is still rejected. +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query T +SELECT sha1('Spark'); +---- +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `sha1` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT sha1(1::INT); + +statement ok +set datafusion.execution.enable_ansi_mode = false;