diff --git a/crates/spark/src/function/string/find_in_set.rs b/crates/spark/src/function/string/find_in_set.rs new file mode 100644 index 00000000..9f94c10c --- /dev/null +++ b/crates/spark/src/function/string/find_in_set.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::Result; +use datafusion_common::config::ConfigOptions; +use datafusion_common::utils::take_function_args; +use datafusion_expr::{ + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, + ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_functions::unicode::find_in_set as core_find_in_set; +use datafusion_macros::user_doc; + +use crate::function::implicit_cast::{SparkInputType, SparkInputTypes}; + +/// Spark-compatible `find_in_set` expression. +/// +#[user_doc( + doc_section(label = "String Functions"), + description = "Returns the 1-based index of the first matching element as an INT. \ + Uses the UTF8_BINARY collation: the search is case-sensitive and elements are split on commas without trimming. \ + Empty elements count, including the single empty element in an empty list. \ + Returns 0 when the string contains a comma or has no match and NULL if either argument is NULL.", + syntax_example = "find_in_set(str, strlist)", + sql_example = r#"```sql +> SELECT find_in_set('b', 'a,b,c'); +2 +> SELECT find_in_set('', 'a,,b'); +2 +```"#, + argument(name = "str", description = "String expression to match."), + argument(name = "strlist", description = "Comma-separated list of strings.") +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkFindInSet { + input_types: SparkInputTypes, +} + +impl Default for SparkFindInSet { + fn default() -> Self { + Self::new() + } +} + +impl SparkFindInSet { + pub fn new() -> Self { + let input_types = SparkInputTypes::new( + vec![SparkInputType::String, SparkInputType::String], + Volatility::Immutable, + ); + Self { input_types } + } +} + +impl ScalarUDFImpl for SparkFindInSet { + fn name(&self) -> &str { + "find_in_set" + } + + fn signature(&self) -> &Signature { + self.input_types.signature() + } + + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { + let types = self.input_types.coerce(self.name(), arg_types)?; + // The core kernel expects both string arrays to use the same layout. + let target = if types.contains(&DataType::LargeUtf8) { + DataType::LargeUtf8 + } else if types.contains(&DataType::Utf8View) { + DataType::Utf8View + } else { + DataType::Utf8 + }; + Ok(vec![target; types.len()]) + } + + fn with_updated_config(&self, config: &ConfigOptions) -> Option { + let input_types = self.input_types.with_config(config)?; + Some(ScalarUDF::from(Self { input_types })) + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Int32) + } + + fn return_field_from_args(&self, args: ReturnFieldArgs<'_>) -> Result { + let nullable = args.arg_fields.iter().any(|field| field.is_nullable()); + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, mut args: ScalarFunctionArgs) -> Result { + let core = core_find_in_set(); + let [needle, list] = take_function_args(self.name(), &args.args)?; + let return_type = core.return_type(&[needle.data_type(), list.data_type()])?; + if return_type == DataType::Int32 { + return core.invoke_with_args(args); + } + + // DataFusion returns Int64 for LargeUtf8. Spark always returns INT. + args.return_field = Arc::new( + args.return_field + .as_ref() + .clone() + .with_data_type(return_type), + ); + core.invoke_with_args(args)?.cast_to(&DataType::Int32, None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn return_field_matches_spark_nullability() -> Result<()> { + let function = SparkFindInSet::new(); + for (needle_nullable, list_nullable) in + [(false, false), (true, false), (false, true)] + { + let fields = [ + Arc::new(Field::new("str", DataType::LargeUtf8, needle_nullable)), + Arc::new(Field::new("strlist", DataType::LargeUtf8, list_nullable)), + ]; + let result = function.return_field_from_args(ReturnFieldArgs { + arg_fields: &fields, + scalar_arguments: &[None, None], + })?; + assert_eq!(result.data_type(), &DataType::Int32); + assert_eq!(result.is_nullable(), needle_nullable || list_nullable); + } + Ok(()) + } +} diff --git a/crates/spark/src/function/string/mod.rs b/crates/spark/src/function/string/mod.rs index b9729acd..810beb80 100644 --- a/crates/spark/src/function/string/mod.rs +++ b/crates/spark/src/function/string/mod.rs @@ -13,6 +13,7 @@ pub mod concat_ws; pub mod contains; pub mod elt; pub mod encode; +pub mod find_in_set; pub mod format_string; pub mod ilike; pub mod is_valid_utf8; @@ -40,6 +41,7 @@ make_udf_function!(ilike::SparkILike, ilike); make_udf_function!(length::SparkLengthFunc, length); make_udf_function!(elt::SparkElt, elt); make_udf_function!(encode::SparkEncode, encode); +make_udf_function!(find_in_set::SparkFindInSet, find_in_set); make_udf_function!(like::SparkLike, like); make_udf_function!(luhn_check::SparkLuhnCheck, luhn_check); make_udf_function!(format_string::FormatStringFunc, format_string); @@ -96,6 +98,11 @@ pub mod expr_fn { "Encodes a string or binary value into binary using the specified character encoding.", string_or_binary charset )); + export_functions!(( + find_in_set, + "Returns the 1-based position of the first match in a comma-separated list, or 0.", + str strlist + )); export_functions!(( ilike, "Returns true if str matches pattern (case insensitive).", @@ -170,6 +177,7 @@ pub fn functions() -> Vec> { contains(), elt(), encode(), + find_in_set(), ilike(), length(), like(), diff --git a/crates/sqllogictest/test_files/string/find_in_set.slt b/crates/sqllogictest/test_files/string/find_in_set.slt index bae2a4bc..1c050d64 100644 --- a/crates/sqllogictest/test_files/string/find_in_set.slt +++ b/crates/sqllogictest/test_files/string/find_in_set.slt @@ -220,3 +220,64 @@ SELECT find_in_set(arrow_cast(a, 'LargeUtf8'), arrow_cast('abc,b,ab,c,def', 'Lar 3 0 NULL + +# Spark implicitly casts both arguments to strings. +query II +SELECT find_in_set(2::INT, '1,2,3'), find_in_set('2', 2::INT); +---- +2 1 + +query I +SELECT find_in_set(a, 'false,true') +FROM (VALUES (true), (false), (NULL::BOOLEAN)) AS t(a); +---- +2 +1 +NULL + +query I +SELECT find_in_set(X'62', 'a,b'); +---- +2 + +query I +SELECT find_in_set(DATE '2020-01-01', 'x,2020-01-01'); +---- +2 + +query III +SELECT find_in_set(NULL, 'a'), find_in_set('a', NULL), find_in_set(NULL, NULL); +---- +NULL NULL NULL + +query I +SELECT find_in_set(arrow_cast(a, 'LargeUtf8'), arrow_cast(b, 'Utf8View')) +FROM (VALUES ('b', 'a,b'), ('界', 'a,界'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +2 +2 +NULL + +query I +SELECT find_in_set(arrow_cast(a, 'Dictionary(Int8, Utf8)'), b) +FROM (VALUES ('b', 'a,b'), ('a', 'a,a'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +2 +1 +NULL + +query T +SELECT arrow_typeof(find_in_set(arrow_cast('b', 'LargeUtf8'), 'a,b')); +---- +Int32 + +query T rowsort +SELECT arrow_typeof(find_in_set(arrow_cast(a, 'LargeUtf8'), b)) +FROM (VALUES ('b', 'a,b'), (NULL::STRING, 'a,b')) AS t(a, b); +---- +Int32 +Int32 + +# xfail spark: collations are not supported yet (https://github.com/apache/datafusion-spark/issues/299). +statement error Unsupported ast node in sqltorel: Collate +SELECT find_in_set('B' COLLATE UTF8_LCASE, 'a,b');