From 3a983c2dc6b482634ca7606da40819b7743f7843 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 18 Nov 2026 09:54:00 -0700 Subject: [PATCH] feat(spark): cast crc32's argument to Spark's declared input type Spark declares crc32's input type as BINARY. Use SparkInputTypes so a STRING argument is cast to BINARY and other types fail with Spark's UNEXPECTED_INPUT_TYPE and WRONG_NUM_ARGS errors. Part of #353. --- crates/spark/src/function/hash/crc32.rs | 30 +++++++----- crates/sqllogictest/test_files/hash/crc32.slt | 49 +++++++++++++++++++ 2 files changed, 68 insertions(+), 11 deletions(-) diff --git a/crates/spark/src/function/hash/crc32.rs b/crates/spark/src/function/hash/crc32.rs index a6b436d6..9bc224e9 100644 --- a/crates/spark/src/function/hash/crc32.rs +++ b/crates/spark/src/function/hash/crc32.rs @@ -11,16 +11,18 @@ use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_fixed_size_binary_array, as_large_binary_array, }; -use datafusion_common::types::{NativeType, logical_string}; +use datafusion_common::config::ConfigOptions; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; + /// Spark-compatible `crc32` expression. /// /// @@ -45,7 +47,7 @@ NULL )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkCrc32 { - signature: Signature, + input_types: SparkInputTypes, } impl Default for SparkCrc32 { @@ -57,12 +59,9 @@ impl Default for SparkCrc32 { impl SparkCrc32 { pub fn new() -> Self { Self { - signature: Signature::coercible( - vec![Coercion::new_implicit( - TypeSignatureClass::Binary, - vec![TypeSignatureClass::Native(logical_string())], - NativeType::Binary, - )], + // Spark's `Crc32.inputTypes` + input_types: SparkInputTypes::new( + vec![SparkInputType::Binary], Volatility::Immutable, ), } @@ -75,7 +74,16 @@ impl ScalarUDFImpl for SparkCrc32 { } fn signature(&self) -> &Signature { - &self.signature + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + self.input_types.coerce(self.name(), arg_types) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { input_types })) } fn return_type(&self, _arg_types: &[DataType]) -> Result { diff --git a/crates/sqllogictest/test_files/hash/crc32.slt b/crates/sqllogictest/test_files/hash/crc32.slt index 3259082b..9bd094cc 100644 --- a/crates/sqllogictest/test_files/hash/crc32.slt +++ b/crates/sqllogictest/test_files/hash/crc32.slt @@ -72,3 +72,52 @@ query I select crc32(arrow_cast(null, 'Dictionary(Int32, Utf8)')) ---- NULL + +query I +SELECT crc32(arrow_cast('Spark', 'Dictionary(Int32, Utf8)')); +---- +1557323817 + +query I +SELECT crc32(v) FROM (VALUES ('Spark'), (NULL), ('Spark')) AS t(v); +---- +1557323817 +NULL +1557323817 + +# Spark's `Crc32.inputTypes` is BINARY. Only a STRING argument is cast to it. +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT crc32(1::INT); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "BIGINT"\. SQLSTATE: 42K09 +SELECT crc32(1); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "DOUBLE"\. SQLSTATE: 42K09 +SELECT crc32(1.5::DOUBLE); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "BOOLEAN"\. SQLSTATE: 42K09 +SELECT crc32(true); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "DATE"\. SQLSTATE: 42K09 +SELECT crc32(DATE '2024-01-01'); + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "ARRAY"\. SQLSTATE: 42K09 +SELECT crc32(array('a')); + +query error \[WRONG_NUM_ARGS\.WITHOUT_SUGGESTION\] The `crc32` requires 1 parameters but the actual number is 2\. +SELECT crc32('a', 'b'); + +# With ANSI mode on, a STRING argument is still cast to BINARY, and an INT is still rejected. +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query I +SELECT crc32('Spark'); +---- +1557323817 + +query error \[DATATYPE_MISMATCH\.UNEXPECTED_INPUT_TYPE\] The first parameter of `crc32` requires the "BINARY" type, however it has the type "INT"\. SQLSTATE: 42K09 +SELECT crc32(1::INT); + +statement ok +set datafusion.execution.enable_ansi_mode = false;