Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 40 additions & 21 deletions crates/spark/src/function/hash/sha2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,21 @@ use arrow::array::{
ArrayRef, AsArray, BinaryArrayType, BinaryBuilder, Int32Array, StringArray,
};
use arrow::datatypes::{DataType, Int32Type};
use datafusion_common::types::{
NativeType, logical_binary, logical_int32, logical_string,
};
use datafusion_common::config::ConfigOptions;
use datafusion_common::utils::hex::{HexCase, encode_bytes_into};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignatureClass, Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl,
Signature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;
use sha2::{self, Digest};
use std::sync::Arc;

use crate::function::implicit_cast::{SparkInputType, SparkInputTypes};

/// Spark-compatible `sha2` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#sha2>
Expand All @@ -42,12 +42,12 @@ NULL
),
argument(
name = "bit_length",
description = "Integer SHA-2 variant: 224, 256, 384 or 512. 0 is treated as 256. Any other value, or NULL, returns NULL."
description = "SHA-2 variant: 224, 256, 384 or 512, cast to INT as in Spark. 0 is treated as 256. Any other value, or NULL, returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkSha2 {
signature: Signature,
input_types: SparkInputTypes,
}

impl Default for SparkSha2 {
Expand All @@ -59,19 +59,9 @@ impl Default for SparkSha2 {
impl SparkSha2 {
pub fn new() -> Self {
Self {
signature: Signature::coercible(
vec![
Coercion::new_implicit(
TypeSignatureClass::Native(logical_binary()),
vec![TypeSignatureClass::Native(logical_string())],
NativeType::Binary,
),
Coercion::new_implicit(
TypeSignatureClass::Native(logical_int32()),
vec![TypeSignatureClass::Integer],
NativeType::Int32,
),
],
// Spark's `Sha2.inputTypes`
input_types: SparkInputTypes::new(
vec![SparkInputType::Binary, SparkInputType::Integer],
Volatility::Immutable,
),
}
Expand All @@ -84,7 +74,16 @@ impl ScalarUDFImpl for SparkSha2 {
}

fn signature(&self) -> &Signature {
&self.signature
self.input_types.signature()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
self.input_types.coerce(self.name(), arg_types)
}

fn with_updated_config(&self, config: &ConfigOptions) -> Option<ScalarUDF> {
let input_types = self.input_types.with_config(config)?;
Some(ScalarUDF::from(Self { input_types }))
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Expand Down Expand Up @@ -315,3 +314,23 @@ where
};
Arc::new(str_array)
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn ansi_mode_reaches_coercion() {
let mut config = ConfigOptions::default();
assert!(SparkSha2::new().with_updated_config(&config).is_none());

config.execution.enable_ansi_mode = true;
let ansi = SparkSha2::new().with_updated_config(&config).unwrap();
assert_eq!(
ansi.coerce_types(&[DataType::Utf8, DataType::Utf8])
.unwrap(),
vec![DataType::Binary, DataType::Int32]
);
assert!(ansi.inner().with_updated_config(&config).is_none());
}
}
Loading
Loading