diff --git a/crates/spark/src/config.rs b/crates/spark/src/config.rs new file mode 100644 index 00000000..ecd22c10 --- /dev/null +++ b/crates/spark/src/config.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::fmt; +use std::str::FromStr; + +use datafusion_common::config::{ConfigExtension, ConfigField, ConfigOptions, Visit}; +use datafusion_common::extensions_options; +use datafusion_common::{DataFusionError, Result}; + +/// Policy for handling duplicate keys in Spark map functions. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum MapKeyDedupPolicy { + #[default] + Exception, + LastWin, +} + +impl FromStr for MapKeyDedupPolicy { + type Err = DataFusionError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_str() { + "EXCEPTION" => Ok(Self::Exception), + "LAST_WIN" => Ok(Self::LastWin), + other => Err(DataFusionError::Configuration(format!( + "Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN" + ))), + } + } +} + +impl ConfigField for MapKeyDedupPolicy { + fn visit(&self, v: &mut V, key: &str, description: &'static str) { + v.some(key, self, description) + } + + fn set(&mut self, _: &str, value: &str) -> Result<()> { + *self = value.parse()?; + Ok(()) + } +} + +impl fmt::Display for MapKeyDedupPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Exception => write!(f, "EXCEPTION"), + Self::LastWin => write!(f, "LAST_WIN"), + } + } +} + +extensions_options! { + /// Spark-specific configuration options under the `spark` namespace. + pub struct SparkOptions { + /// How to handle duplicate map keys: EXCEPTION raises an error (default), + /// while LAST_WIN keeps the last value for each key. + pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception + } +} + +impl ConfigExtension for SparkOptions { + const PREFIX: &'static str = "spark"; +} + +/// Returns whether Spark map functions should keep the last value for duplicate keys. +/// +/// The new key is `spark.map_key_dedup_policy`. For DataFusion 56 compatibility, +/// the legacy `datafusion.spark.map_key_dedup_policy` is also honored: LAST_WIN +/// on either key takes precedence over EXCEPTION on the other. +/// Remove the legacy lookup when upgrading to DataFusion 57. +pub fn map_last_value_wins(options: &ConfigOptions) -> bool { + options + .extensions + .get::() + .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin) + || options.spark.map_key_dedup_policy + == datafusion_common::config::MapKeyDedupPolicy::LastWin +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion_common::config::ConfigOptions; + + #[test] + fn spark_options_use_spark_prefix() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions::default()); + + config + .set("spark.map_key_dedup_policy", "last_win") + .unwrap(); + + assert_eq!( + config + .extensions + .get::() + .unwrap() + .map_key_dedup_policy, + MapKeyDedupPolicy::LastWin + ); + } + + #[test] + fn spark_options_reject_invalid_policy() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions::default()); + + let error = config + .set("spark.map_key_dedup_policy", "bogus") + .unwrap_err() + .to_string(); + + assert!(error.contains("Invalid MapKeyDedupPolicy"), "{error}"); + } + + #[test] + fn missing_spark_options_use_exception_policy() { + assert!(!map_last_value_wins(&ConfigOptions::default())); + } + + #[test] + fn configured_spark_options_enable_last_win() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions { + map_key_dedup_policy: MapKeyDedupPolicy::LastWin, + }); + + assert!(map_last_value_wins(&config)); + } + + #[test] + fn legacy_and_extension_policy_combinations() { + for legacy in ["EXCEPTION", "LAST_WIN"] { + for extension in [None, Some("EXCEPTION"), Some("LAST_WIN")] { + let mut config = ConfigOptions::default(); + config + .set("datafusion.spark.map_key_dedup_policy", legacy) + .unwrap(); + if let Some(policy) = extension { + config.extensions.insert(SparkOptions::default()); + config.set("spark.map_key_dedup_policy", policy).unwrap(); + } + + assert_eq!( + map_last_value_wins(&config), + legacy == "LAST_WIN" || extension == Some("LAST_WIN"), + "legacy={legacy}, extension={extension:?}" + ); + } + } + } + + #[test] + fn map_policy_has_description() { + let entries = datafusion_common::config::ExtensionOptions::entries( + &SparkOptions::default(), + ); + let entry = entries + .iter() + .find(|entry| entry.key == "map_key_dedup_policy") + .unwrap(); + assert!(entry.description.contains("EXCEPTION")); + assert!(entry.description.contains("LAST_WIN")); + } +} diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index 2233ab40..971f77f8 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::config::map_last_value_wins; use crate::function::map::utils::{ get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, @@ -9,7 +10,6 @@ use crate::function::map::utils::{ use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::config::MapKeyDedupPolicy; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ @@ -69,8 +69,7 @@ impl ScalarUDFImpl for MapFromArrays { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( move |args: &[ArrayRef]| map_from_arrays_inner(args, last_value_wins), vec![], diff --git a/crates/spark/src/function/map/map_from_entries.rs b/crates/spark/src/function/map/map_from_entries.rs index d8514ca6..85e02de8 100644 --- a/crates/spark/src/function/map/map_from_entries.rs +++ b/crates/spark/src/function/map/map_from_entries.rs @@ -4,6 +4,7 @@ use std::sync::Arc; +use crate::config::map_last_value_wins; use crate::function::map::utils::{ get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, @@ -11,7 +12,6 @@ use crate::function::map::utils::{ use arrow::array::{Array, ArrayRef, NullBufferBuilder, StructArray}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::config::MapKeyDedupPolicy; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ @@ -89,8 +89,7 @@ impl ScalarUDFImpl for MapFromEntries { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( move |args: &[ArrayRef]| map_from_entries_inner(args, last_value_wins), vec![], diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 1c9e1285..039041c4 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -19,8 +19,8 @@ use datafusion_expr::{ TypeSignature, Volatility, }; +use crate::config::map_last_value_wins; use crate::function::map::utils::map_type_from_key_value_types; -use datafusion_common::config::MapKeyDedupPolicy; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; @@ -37,7 +37,7 @@ const DEFAULT_KV_DELIM: &str = ":"; /// /// # Duplicate Key Handling /// Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4502-L4511), -/// wired through DataFusion's `datafusion.spark.map_key_dedup_policy`: +/// wired through DataFusion Spark's `spark.map_key_dedup_policy`: /// - `EXCEPTION` (default): error on duplicate keys. /// - `LAST_WIN`: keep the last occurrence of each duplicate key. #[derive(Debug, PartialEq, Eq, Hash)] @@ -89,8 +89,7 @@ impl ScalarUDFImpl for SparkStrToMap { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = map_last_value_wins(&args.config_options); let arrays: Vec = ColumnarValue::values_to_arrays(&args.args)?; let result = str_to_map_inner(&arrays, last_value_wins)?; Ok(ColumnarValue::Array(result)) @@ -275,7 +274,7 @@ fn str_to_map_impl<'a, V: StringArrayType<'a> + Copy>( "[DUPLICATED_MAP_KEY] Duplicate map key '{key}' was found, \ please check the input data. To allow duplicate keys with \ last-value-wins semantics, set \ - `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." + `spark.map_key_dedup_policy` to `LAST_WIN`." ); } diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index e1d4207a..3c8eaa85 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -111,8 +111,6 @@ pub fn map_type_from_key_value_types( /// and is driven by `last_value_wins`: /// - `false` (Spark's default `EXCEPTION`): raise `[DUPLICATED_MAP_KEY]` on any duplicate. /// - `true` (`LAST_WIN`): keep the last occurrence of each duplicate key. -/// -/// Callers wire this from `datafusion.spark.map_key_dedup_policy`. pub fn map_from_keys_values_offsets_nulls( flat_keys: &ArrayRef, flat_values: &ArrayRef, @@ -222,7 +220,7 @@ fn map_deduplicate_keys( "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ please check the input data. To allow duplicate keys with \ last-value-wins semantics, set \ - `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." + `spark.map_key_dedup_policy` to `LAST_WIN`." ); } keys_mask_builder.append_value(true); diff --git a/crates/spark/src/function/string/substring.rs b/crates/spark/src/function/string/substring.rs index 354f87c8..3fda3918 100644 --- a/crates/spark/src/function/string/substring.rs +++ b/crates/spark/src/function/string/substring.rs @@ -360,7 +360,7 @@ impl SubstringBuilder for GenericBinaryBuilder { GenericBinaryBuilder::append_null(self); } fn append_empty(&mut self) { - GenericBinaryBuilder::append_value(self, &[]); + GenericBinaryBuilder::append_value(self, []); } } diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 16fd969f..4930a9e5 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -168,10 +168,14 @@ //! [Getting started](#getting-started), then use the extension trait - see //! [`SessionStateBuilderSpark::with_spark_features`] for an example. -mod eval_mode; +/// Spark configuration extensions and map-key deduplication policy. +pub mod config; pub mod function; pub mod planner; +pub use config::{MapKeyDedupPolicy, SparkOptions}; +mod eval_mode; + pub use eval_mode::EvalMode; #[cfg(feature = "core")] diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index ce1a489e..e6c900c1 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use datafusion::execution::SessionStateBuilder; +use crate::SparkOptions; use crate::planner::SparkFunctionPlanner; use crate::{ all_default_aggregate_functions, all_default_scalar_functions, @@ -69,7 +70,23 @@ pub trait SessionStateBuilderSpark { } impl SessionStateBuilderSpark for SessionStateBuilder { + /// Registers Spark-compatible planners, functions, and a default + /// [`SparkOptions`] extension (only if one isn't already set). + /// + /// Call `with_config` **before** this method: `with_config` replaces the + /// whole config and would drop the `SparkOptions` extension. fn with_spark_features(mut self) -> Self { + // Check if SparkOptions already exists; if not, insert default. + // The immutable borrow from .get() is released before .insert(). + let extensions = &mut self + .config() + .get_or_insert_with(Default::default) + .options_mut() + .extensions; + if extensions.get::().is_none() { + extensions.insert(SparkOptions::default()); + } + self.expr_planners() .get_or_insert_with(Vec::new) // planners are evaluated in order of insertion. Push Apache Spark function planner to the front @@ -111,6 +128,15 @@ mod tests { fn test_session_state_with_spark_features() { let state = SessionStateBuilder::new().with_spark_features().build(); + assert!( + state + .config_options() + .extensions + .get::() + .is_some(), + "Spark options should be registered" + ); + assert!( state.scalar_functions().contains_key("sha2"), "Apache Spark scalar function 'sha2' should be registered" @@ -127,6 +153,26 @@ mod tests { ); } + #[test] + fn test_spark_features_preserve_configured_policy() { + let state = SessionStateBuilder::new() + .with_config(SessionConfig::new().with_option_extension(SparkOptions { + map_key_dedup_policy: crate::MapKeyDedupPolicy::LastWin, + })) + .with_spark_features() + .build(); + + assert_eq!( + state + .config_options() + .extensions + .get::() + .unwrap() + .map_key_dedup_policy, + crate::MapKeyDedupPolicy::LastWin + ); + } + #[tokio::test] async fn test_spark_dialect_with_spark_functions() { let query = "SELECT sha2('abc', 256), CAST(1 AS LONG)"; diff --git a/crates/sqllogictest/test_files/map/map_from_arrays.slt b/crates/sqllogictest/test_files/map/map_from_arrays.slt index b31ea664..ce3158c9 100644 --- a/crates/sqllogictest/test_files/map/map_from_arrays.slt +++ b/crates/sqllogictest/test_files/map/map_from_arrays.slt @@ -138,7 +138,7 @@ SELECT map_from_arrays(arrow_cast(array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)' # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c')); @@ -171,7 +171,7 @@ AS tab(a, b); NULL statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Sliced input: OFFSET makes the function see list arrays whose first offset is # non-zero, so the child arrays still contain the skipped row's elements. diff --git a/crates/sqllogictest/test_files/map/map_from_entries.slt b/crates/sqllogictest/test_files/map/map_from_entries.slt index a9905235..044955f1 100644 --- a/crates/sqllogictest/test_files/map/map_from_entries.slt +++ b/crates/sqllogictest/test_files/map/map_from_entries.slt @@ -166,7 +166,7 @@ AS tab(data); # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT map_from_entries(array( @@ -201,7 +201,7 @@ AS tab(data); {1: y} statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Sliced input: OFFSET makes the function see a list array whose first offset is # non-zero, so the child array still contains the skipped row's entries. diff --git a/crates/sqllogictest/test_files/map/str_to_map.slt b/crates/sqllogictest/test_files/map/str_to_map.slt index 55d2470e..6817c492 100644 --- a/crates/sqllogictest/test_files/map/str_to_map.slt +++ b/crates/sqllogictest/test_files/map/str_to_map.slt @@ -116,7 +116,7 @@ NULL # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT str_to_map('a:1,b:2,a:3'); @@ -143,12 +143,10 @@ SELECT str_to_map(col) FROM (VALUES ('a:1,b:2'), ('a:3,a:4')) AS t(col); {a: 4} statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Invalid policy values are rejected at SET time with a clear message. statement error -set datafusion.spark.map_key_dedup_policy = 'BOGUS'; +set spark.map_key_dedup_policy = 'BOGUS'; ---- -DataFusion error: Error setting config datafusion.spark.map_key_dedup_policy -caused by -Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN +DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN