From f67e3d58ce06a107bbd3cbf939492f817c030a26 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 20:39:40 +0530 Subject: [PATCH 01/10] SparkOptions port --- crates/spark/src/config.rs | 99 +++++++++++++++++++ .../spark/src/function/map/map_from_arrays.rs | 19 ++-- .../src/function/map/map_from_entries.rs | 51 +++++----- crates/spark/src/function/map/str_to_map.rs | 65 ++++++------ crates/spark/src/function/map/utils.rs | 97 ++++++++---------- crates/spark/src/lib.rs | 3 + crates/spark/src/session_state.rs | 20 +++- .../test_files/map/map_key_dedup_policy.slt | 16 +++ 8 files changed, 238 insertions(+), 132 deletions(-) create mode 100644 crates/spark/src/config.rs create mode 100644 crates/sqllogictest/test_files/map/map_key_dedup_policy.slt diff --git a/crates/spark/src/config.rs b/crates/spark/src/config.rs new file mode 100644 index 00000000..e3cf05ba --- /dev/null +++ b/crates/spark/src/config.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::fmt; +use std::str::FromStr; + +use datafusion_common::config::{ConfigExtension, ConfigField, Visit}; +use datafusion_common::extensions_options; +use datafusion_common::{DataFusionError, Result}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum MapKeyDedupPolicy { + #[default] + Exception, + LastWin, +} + +impl FromStr for MapKeyDedupPolicy { + type Err = DataFusionError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_str() { + "EXCEPTION" => Ok(Self::Exception), + "LAST_WIN" => Ok(Self::LastWin), + other => Err(DataFusionError::Configuration(format!( + "Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN" + ))), + } + } +} + +impl ConfigField for MapKeyDedupPolicy { + fn visit(&self, v: &mut V, key: &str, description: &'static str) { + v.some(key, self, description) + } + + fn set(&mut self, _: &str, value: &str) -> Result<()> { + *self = value.parse()?; + Ok(()) + } +} + +impl fmt::Display for MapKeyDedupPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Exception => write!(f, "EXCEPTION"), + Self::LastWin => write!(f, "LAST_WIN"), + } + } +} + +extensions_options! { + pub struct SparkOptions { + pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception + } +} + +impl ConfigExtension for SparkOptions { + const PREFIX: &'static str = "spark"; +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion_common::config::ConfigOptions; + + #[test] + fn spark_options_use_spark_prefix() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions::default()); + + config + .set("spark.map_key_dedup_policy", "last_win") + .unwrap(); + + assert_eq!( + config + .extensions + .get::() + .unwrap() + .map_key_dedup_policy, + MapKeyDedupPolicy::LastWin + ); + } + + #[test] + fn spark_options_reject_invalid_policy() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions::default()); + + let error = config + .set("spark.map_key_dedup_policy", "bogus") + .unwrap_err() + .to_string(); + + assert!(error.contains("Invalid MapKeyDedupPolicy"), "{error}"); + } +} diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index 2233ab40..c8efcf68 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -3,18 +3,17 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::map::utils::{ - get_element_type, get_list_offsets, get_list_values, - map_from_keys_values_offsets_nulls, map_type_from_key_value_types, + get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, + map_type_from_key_value_types, }; +use crate::{MapKeyDedupPolicy, SparkOptions}; use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::config::MapKeyDedupPolicy; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; use std::sync::Arc; @@ -69,8 +68,11 @@ impl ScalarUDFImpl for MapFromArrays { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = args + .config_options + .extensions + .get::() + .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); make_scalar_function( move |args: &[ArrayRef]| map_from_arrays_inner(args, last_value_wins), vec![], @@ -128,8 +130,7 @@ mod tests { }) .expect("return_field_from_args should succeed"); - let expected_type = - map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); + let expected_type = map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); assert_eq!(out.data_type(), &expected_type); assert!( !out.is_nullable(), diff --git a/crates/spark/src/function/map/map_from_entries.rs b/crates/spark/src/function/map/map_from_entries.rs index d8514ca6..28fbf2cf 100644 --- a/crates/spark/src/function/map/map_from_entries.rs +++ b/crates/spark/src/function/map/map_from_entries.rs @@ -8,15 +8,14 @@ use crate::function::map::utils::{ get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; +use crate::{MapKeyDedupPolicy, SparkOptions}; use arrow::array::{Array, ArrayRef, NullBufferBuilder, StructArray}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::config::MapKeyDedupPolicy; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions::utils::make_scalar_function; @@ -59,18 +58,15 @@ impl ScalarUDFImpl for MapFromEntries { return exec_err!("map_from_entries: expected one argument"); }; - let (entries_element_field, entries_element_type) = - match entries_field.data_type() { - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => { - Ok((field.as_ref(), field.data_type())) - } - wrong_type => exec_err!( - "map_from_entries: expected array>, got {:?}", - wrong_type - ), - }?; + let (entries_element_field, entries_element_type) = match entries_field.data_type() { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => Ok((field.as_ref(), field.data_type())), + wrong_type => exec_err!( + "map_from_entries: expected array>, got {:?}", + wrong_type + ), + }?; let (keys_type, values_type) = match entries_element_type { DataType::Struct(fields) if fields.len() == 2 => { @@ -89,8 +85,11 @@ impl ScalarUDFImpl for MapFromEntries { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = args + .config_options + .extensions + .get::() + .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); make_scalar_function( move |args: &[ArrayRef]| map_from_entries_inner(args, last_value_wins), vec![], @@ -103,14 +102,13 @@ fn map_from_entries_inner(args: &[ArrayRef], last_value_wins: bool) -> Result() { - Some(a) => Ok((a.column(0), a.column(1))), - None => exec_err!( - "map_from_entries: expected array>, got {:?}", - entries_values.data_type() - ), - }?; + let (flat_keys, flat_values) = match entries_values.as_any().downcast_ref::() { + Some(a) => Ok((a.column(0), a.column(1))), + None => exec_err!( + "map_from_entries: expected array>, got {:?}", + entries_values.data_type() + ), + }?; let entries_with_nulls = entries_values.nulls().and_then(|entries_inner_nulls| { let mut builder = NullBufferBuilder::new_with_len(0); @@ -165,8 +163,7 @@ mod tests { #[test] fn test_map_from_entries_nullability_matches_input() { let func = MapFromEntries::new(); - let expected_type = - map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); + let expected_type = map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); // Non-nullable array and elements => non-nullable result let non_nullable_field = make_entries_field(false, false); diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 1c9e1285..28962cc4 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -5,22 +5,18 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use arrow::array::{ - Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder, -}; +use arrow::array::{Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::cast::{ - as_large_string_array, as_string_array, as_string_view_array, -}; +use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array}; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - TypeSignature, Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + Volatility, }; use crate::function::map::utils::map_type_from_key_value_types; -use datafusion_common::config::MapKeyDedupPolicy; +use crate::{MapKeyDedupPolicy, SparkOptions}; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; @@ -37,7 +33,7 @@ const DEFAULT_KV_DELIM: &str = ":"; /// /// # Duplicate Key Handling /// Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4502-L4511), -/// wired through DataFusion's `datafusion.spark.map_key_dedup_policy`: +/// wired through DataFusion Spark's `spark.map_key_dedup_policy`: /// - `EXCEPTION` (default): error on duplicate keys. /// - `LAST_WIN`: keep the last occurrence of each duplicate key. #[derive(Debug, PartialEq, Eq, Hash)] @@ -89,8 +85,11 @@ impl ScalarUDFImpl for SparkStrToMap { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = - args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin; + let last_value_wins = args + .config_options + .extensions + .get::() + .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); let arrays: Vec = ColumnarValue::values_to_arrays(&args.args)?; let result = str_to_map_inner(&arrays, last_value_wins)?; Ok(ColumnarValue::Array(result)) @@ -109,12 +108,9 @@ fn str_to_map_inner(args: &[ArrayRef], last_value_wins: bool) -> Result str_to_map_impl( - as_string_view_array(&args[0])?, - None, - None, - last_value_wins, - ), + DataType::Utf8View => { + str_to_map_impl(as_string_view_array(&args[0])?, None, None, last_value_wins) + } other => exec_err!( "Unsupported data type {other:?} for str_to_map, \ expected Utf8, LargeUtf8, or Utf8View" @@ -155,22 +151,18 @@ fn str_to_map_inner(args: &[ArrayRef], last_value_wins: bool) -> Result { - str_to_map_impl( - as_large_string_array(&args[0])?, - Some(as_large_string_array(&args[1])?), - Some(as_large_string_array(&args[2])?), - last_value_wins, - ) - } - (DataType::Utf8View, DataType::Utf8View, DataType::Utf8View) => { - str_to_map_impl( - as_string_view_array(&args[0])?, - Some(as_string_view_array(&args[1])?), - Some(as_string_view_array(&args[2])?), - last_value_wins, - ) - } + (DataType::LargeUtf8, DataType::LargeUtf8, DataType::LargeUtf8) => str_to_map_impl( + as_large_string_array(&args[0])?, + Some(as_large_string_array(&args[1])?), + Some(as_large_string_array(&args[2])?), + last_value_wins, + ), + (DataType::Utf8View, DataType::Utf8View, DataType::Utf8View) => str_to_map_impl( + as_string_view_array(&args[0])?, + Some(as_string_view_array(&args[1])?), + Some(as_string_view_array(&args[2])?), + last_value_wins, + ), (t1, t2, t3) => exec_err!( "Unsupported data types ({t1:?}, {t2:?}, {t3:?}) for str_to_map, \ expected matching Utf8, LargeUtf8, or Utf8View" @@ -221,8 +213,7 @@ fn str_to_map_impl<'a, V: StringArrayType<'a> + Copy>( } // Per-row delimiter extraction - let pair_delim = - pair_delim_array.map_or(DEFAULT_PAIR_DELIM, |a| a.value(row_idx)); + let pair_delim = pair_delim_array.map_or(DEFAULT_PAIR_DELIM, |a| a.value(row_idx)); let kv_delim = kv_delim_array.map_or(DEFAULT_KV_DELIM, |a| a.value(row_idx)); let text = text_array.value(row_idx); @@ -275,7 +266,7 @@ fn str_to_map_impl<'a, V: StringArrayType<'a> + Copy>( "[DUPLICATED_MAP_KEY] Duplicate map key '{key}' was found, \ please check the input data. To allow duplicate keys with \ last-value-wins semantics, set \ - `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." + `spark.map_key_dedup_policy` to `LAST_WIN`." ); } diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index e1d4207a..491e7992 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -3,14 +3,12 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use std::borrow::Cow; -use std::collections::HashMap; +use std::collections::HashSet; use std::sync::Arc; -use arrow::array::{ - Array, ArrayRef, AsArray, BooleanBuilder, Int32Array, MapArray, StructArray, -}; +use arrow::array::{Array, ArrayRef, AsArray, BooleanBuilder, MapArray, StructArray}; use arrow::buffer::{NullBuffer, OffsetBuffer}; -use arrow::compute::{filter, take}; +use arrow::compute::filter; use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; @@ -80,10 +78,7 @@ pub fn get_list_offsets(array: &ArrayRef) -> Result> { /// - Map keys are non-nullable /// - Map entries are non-nullable /// - Map values can be null -pub fn map_type_from_key_value_types( - key_type: &DataType, - value_type: &DataType, -) -> DataType { +pub fn map_type_from_key_value_types(key_type: &DataType, value_type: &DataType) -> DataType { DataType::Map( Arc::new(Field::new( "entries", @@ -112,7 +107,6 @@ pub fn map_type_from_key_value_types( /// - `false` (Spark's default `EXCEPTION`): raise `[DUPLICATED_MAP_KEY]` on any duplicate. /// - `true` (`LAST_WIN`): keep the last occurrence of each duplicate key. /// -/// Callers wire this from `datafusion.spark.map_key_dedup_policy`. pub fn map_from_keys_values_offsets_nulls( flat_keys: &ArrayRef, flat_values: &ArrayRef, @@ -173,18 +167,8 @@ fn map_deduplicate_keys( let mut new_last_offset = 0; new_offsets.push(new_last_offset); - // Mirror Spark's `ArrayBasedMapBuilder`: the first occurrence of a key - // fixes its position in the output; under LAST_WIN a later duplicate - // overwrites that slot's value. `keys_mask` selects the first-seen keys - // and has one bit per key entry of the input rows, so bit 0 corresponds to - // `flat_keys[first_keys_offset]`. `value_indices` records the source index - // in `flat_values` to materialize for each output slot (updated in place - // on overwrite). let mut keys_mask_builder = BooleanBuilder::new(); - let mut value_indices: Vec = Vec::with_capacity( - keys_offsets.last().map_or(0, |offset| *offset as usize) - first_keys_offset, - ); - let mut key_to_output_idx: HashMap = HashMap::new(); + let mut values_mask_builder = BooleanBuilder::new(); for (row_idx, (next_keys_offset, next_values_offset)) in keys_offsets .iter() .zip(values_offsets.iter()) @@ -203,50 +187,52 @@ fn map_deduplicate_keys( "map_deduplicate_keys: keys and values lists in the same row must have equal lengths" ); } - key_to_output_idx.clear(); - for cur_entry_idx in 0..num_keys_entries { - let key = ScalarValue::try_from_array( - &flat_keys, - cur_keys_offset + cur_entry_idx, - )? - .compacted(); - let abs_value_idx = (cur_values_offset + cur_entry_idx) as i32; - - if let Some(&output_idx) = key_to_output_idx.get(&key) { - if last_value_wins { - value_indices[output_idx] = abs_value_idx; - keys_mask_builder.append_value(false); - continue; + if num_keys_entries != 0 { + let mut seen_keys = HashSet::new(); + let mut keys_mask_one = vec![false; num_keys_entries]; + let mut values_mask_one = vec![false; num_values_entries]; + for cur_entry_idx in (0..num_keys_entries).rev() { + let key = + ScalarValue::try_from_array(&flat_keys, cur_keys_offset + cur_entry_idx)? + .compacted(); + if seen_keys.contains(&key) { + if !last_value_wins { + return exec_err!( + "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ + please check the input data. To allow duplicate keys with \ + last-value-wins semantics, set \ + `spark.map_key_dedup_policy` to `LAST_WIN`." + ); + } + } else { + keys_mask_one[cur_entry_idx] = true; + values_mask_one[cur_entry_idx] = true; + seen_keys.insert(key); + new_last_offset += 1; } - return exec_err!( - "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ - please check the input data. To allow duplicate keys with \ - last-value-wins semantics, set \ - `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." - ); } - keys_mask_builder.append_value(true); - key_to_output_idx.insert(key, value_indices.len()); - value_indices.push(abs_value_idx); - new_last_offset += 1; + keys_mask_builder.append_array(&keys_mask_one.into()); + values_mask_builder.append_array(&values_mask_one.into()); } } else { - // The result entry is NULL — no keys/values emitted. Still pad the - // mask so it stays aligned with the key entries of later rows. - keys_mask_builder.append_n(num_keys_entries, false); + let keys_mask_one = vec![false; num_keys_entries]; + let values_mask_one = vec![false; num_values_entries]; + keys_mask_builder.append_array(&keys_mask_one.into()); + values_mask_builder.append_array(&values_mask_one.into()); } new_offsets.push(new_last_offset); cur_keys_offset += num_keys_entries; cur_values_offset += num_values_entries; } let keys_mask = keys_mask_builder.finish(); - // Every key entry of the input rows must have exactly one mask bit; - // otherwise the mask selects the wrong keys for all later rows. - debug_assert_eq!(keys_mask.len(), cur_keys_offset - first_keys_offset); let input_keys = flat_keys.slice(first_keys_offset, keys_mask.len()); let needed_keys = filter(&input_keys, &keys_mask)?; - let value_indices_array = Int32Array::from(value_indices); - let needed_values = take(&flat_values, &value_indices_array, None)?; + let values_mask = values_mask_builder.finish(); + let input_values = flat_values.slice( + values_offsets.first().map_or(0, |offset| *offset as usize), + values_mask.len(), + ); + let needed_values = filter(&input_values, &values_mask)?; let offsets = OffsetBuffer::new(new_offsets.into()); Ok((needed_keys, needed_values, offsets)) } @@ -257,10 +243,7 @@ mod tests { use arrow::array::{Int32Array, LargeListArray, NullArray, StringArray}; use arrow::datatypes::Int32Type; - fn int32_utf8_inputs( - keys: Vec, - values: Vec>, - ) -> (ArrayRef, ArrayRef) { + fn int32_utf8_inputs(keys: Vec, values: Vec>) -> (ArrayRef, ArrayRef) { let keys: ArrayRef = Arc::new(Int32Array::from(keys)); let values: ArrayRef = Arc::new(StringArray::from(values)); (keys, values) diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 9c5a6737..95db320e 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -115,9 +115,12 @@ //! Then use the extension trait - see [`SessionStateBuilderSpark::with_spark_features`] //! for an example. +pub mod config; pub mod function; pub mod planner; +pub use config::{MapKeyDedupPolicy, SparkOptions}; + #[cfg(feature = "core")] mod session_state; diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index 43a6610a..a73fbd89 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -7,10 +7,11 @@ use std::sync::Arc; use datafusion::execution::SessionStateBuilder; +use crate::SparkOptions; use crate::planner::SparkFunctionPlanner; use crate::{ - all_default_aggregate_functions, all_default_scalar_functions, - all_default_table_functions, all_default_window_functions, + all_default_aggregate_functions, all_default_scalar_functions, all_default_table_functions, + all_default_window_functions, }; /// Extension trait for adding Apache Spark features to [`SessionStateBuilder`]. @@ -42,6 +43,12 @@ pub trait SessionStateBuilderSpark { impl SessionStateBuilderSpark for SessionStateBuilder { fn with_spark_features(mut self) -> Self { + self.config() + .get_or_insert_with(Default::default) + .options_mut() + .extensions + .insert(SparkOptions::default()); + self.expr_planners() .get_or_insert_with(Vec::new) // planners are evaluated in order of insertion. Push Apache Spark function planner to the front @@ -83,6 +90,15 @@ mod tests { fn test_session_state_with_spark_features() { let state = SessionStateBuilder::new().with_spark_features().build(); + assert!( + state + .config_options() + .extensions + .get::() + .is_some(), + "Spark options should be registered" + ); + assert!( state.scalar_functions().contains_key("sha2"), "Apache Spark scalar function 'sha2' should be registered" diff --git a/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt b/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt new file mode 100644 index 00000000..2784f462 --- /dev/null +++ b/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt @@ -0,0 +1,16 @@ +# Spark map construction uses EXCEPTION by default. +statement ok +set spark.map_key_dedup_policy = 'LAST_WIN'; + +query ? +SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c')); +---- +{2: b, 1: c} + +statement ok +set spark.map_key_dedup_policy = 'EXCEPTION'; + +statement error +set spark.map_key_dedup_policy = 'BOGUS'; +---- +DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN From 1465181004d2777d847f768bbf01360c04fa54a8 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 21:37:54 +0530 Subject: [PATCH 02/10] mapdedp reverted back to prev impl --- crates/spark/src/function/map/utils.rs | 81 ++++++++++++++------------ 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index 491e7992..31472ad1 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -3,15 +3,15 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use std::borrow::Cow; -use std::collections::HashSet; +use std::collections::HashMap; use std::sync::Arc; +use arrow::array::Int32Array; use arrow::array::{Array, ArrayRef, AsArray, BooleanBuilder, MapArray, StructArray}; use arrow::buffer::{NullBuffer, OffsetBuffer}; -use arrow::compute::filter; +use arrow::compute::{filter, take}; use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; - /// Helper function to get element [`DataType`] /// from [`List`](DataType::List)/[`LargeList`](DataType::LargeList)/[`FixedSizeList`](DataType::FixedSizeList)
/// [`Null`](DataType::Null) can be coerced to `ListType`([`Null`](DataType::Null)), so [`Null`](DataType::Null) is returned
@@ -167,8 +167,18 @@ fn map_deduplicate_keys( let mut new_last_offset = 0; new_offsets.push(new_last_offset); + // Mirror Spark's `ArrayBasedMapBuilder`: the first occurrence of a key + // fixes its position in the output; under LAST_WIN a later duplicate + // overwrites that slot's value. `keys_mask` selects the first-seen keys + // and has one bit per key entry of the input rows, so bit 0 corresponds to + // `flat_keys[first_keys_offset]`. `value_indices` records the source index + // in `flat_values` to materialize for each output slot (updated in place + // on overwrite). let mut keys_mask_builder = BooleanBuilder::new(); - let mut values_mask_builder = BooleanBuilder::new(); + let mut value_indices: Vec = Vec::with_capacity( + keys_offsets.last().map_or(0, |offset| *offset as usize) - first_keys_offset, + ); + let mut key_to_output_idx: HashMap = HashMap::new(); for (row_idx, (next_keys_offset, next_values_offset)) in keys_offsets .iter() .zip(values_offsets.iter()) @@ -187,52 +197,47 @@ fn map_deduplicate_keys( "map_deduplicate_keys: keys and values lists in the same row must have equal lengths" ); } - if num_keys_entries != 0 { - let mut seen_keys = HashSet::new(); - let mut keys_mask_one = vec![false; num_keys_entries]; - let mut values_mask_one = vec![false; num_values_entries]; - for cur_entry_idx in (0..num_keys_entries).rev() { - let key = - ScalarValue::try_from_array(&flat_keys, cur_keys_offset + cur_entry_idx)? - .compacted(); - if seen_keys.contains(&key) { - if !last_value_wins { - return exec_err!( - "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ - please check the input data. To allow duplicate keys with \ - last-value-wins semantics, set \ - `spark.map_key_dedup_policy` to `LAST_WIN`." - ); - } - } else { - keys_mask_one[cur_entry_idx] = true; - values_mask_one[cur_entry_idx] = true; - seen_keys.insert(key); - new_last_offset += 1; + key_to_output_idx.clear(); + for cur_entry_idx in 0..num_keys_entries { + let key = ScalarValue::try_from_array(&flat_keys, cur_keys_offset + cur_entry_idx)? + .compacted(); + let abs_value_idx = (cur_values_offset + cur_entry_idx) as i32; + + if let Some(&output_idx) = key_to_output_idx.get(&key) { + if last_value_wins { + value_indices[output_idx] = abs_value_idx; + keys_mask_builder.append_value(false); + continue; } + return exec_err!( + "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ + please check the input data. To allow duplicate keys with \ + last-value-wins semantics, set \ + `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." + ); } - keys_mask_builder.append_array(&keys_mask_one.into()); - values_mask_builder.append_array(&values_mask_one.into()); + keys_mask_builder.append_value(true); + key_to_output_idx.insert(key, value_indices.len()); + value_indices.push(abs_value_idx); + new_last_offset += 1; } } else { - let keys_mask_one = vec![false; num_keys_entries]; - let values_mask_one = vec![false; num_values_entries]; - keys_mask_builder.append_array(&keys_mask_one.into()); - values_mask_builder.append_array(&values_mask_one.into()); + // The result entry is NULL — no keys/values emitted. Still pad the + // mask so it stays aligned with the key entries of later rows. + keys_mask_builder.append_n(num_keys_entries, false); } new_offsets.push(new_last_offset); cur_keys_offset += num_keys_entries; cur_values_offset += num_values_entries; } let keys_mask = keys_mask_builder.finish(); + // Every key entry of the input rows must have exactly one mask bit; + // otherwise the mask selects the wrong keys for all later rows. + debug_assert_eq!(keys_mask.len(), cur_keys_offset - first_keys_offset); let input_keys = flat_keys.slice(first_keys_offset, keys_mask.len()); let needed_keys = filter(&input_keys, &keys_mask)?; - let values_mask = values_mask_builder.finish(); - let input_values = flat_values.slice( - values_offsets.first().map_or(0, |offset| *offset as usize), - values_mask.len(), - ); - let needed_values = filter(&input_values, &values_mask)?; + let value_indices_array = Int32Array::from(value_indices); + let needed_values = take(&flat_values, &value_indices_array, None)?; let offsets = OffsetBuffer::new(new_offsets.into()); Ok((needed_keys, needed_values, offsets)) } From f41898338ec4b63114ea4f37efb143d067ff7d5a Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 21:45:38 +0530 Subject: [PATCH 03/10] mapdedp reverted back to prev impl --- crates/spark/src/function/map/utils.rs | 24 +++++++++++++------ .../test_files/map/map_from_arrays.slt | 4 ++-- .../test_files/map/map_from_entries.slt | 4 ++-- .../test_files/map/str_to_map.slt | 8 +++---- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index 31472ad1..e592e651 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -6,12 +6,14 @@ use std::borrow::Cow; use std::collections::HashMap; use std::sync::Arc; -use arrow::array::Int32Array; -use arrow::array::{Array, ArrayRef, AsArray, BooleanBuilder, MapArray, StructArray}; +use arrow::array::{ + Array, ArrayRef, AsArray, BooleanBuilder, Int32Array, MapArray, StructArray, +}; use arrow::buffer::{NullBuffer, OffsetBuffer}; use arrow::compute::{filter, take}; use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::{Result, ScalarValue, exec_datafusion_err, exec_err}; + /// Helper function to get element [`DataType`] /// from [`List`](DataType::List)/[`LargeList`](DataType::LargeList)/[`FixedSizeList`](DataType::FixedSizeList)
/// [`Null`](DataType::Null) can be coerced to `ListType`([`Null`](DataType::Null)), so [`Null`](DataType::Null) is returned
@@ -78,7 +80,10 @@ pub fn get_list_offsets(array: &ArrayRef) -> Result> { /// - Map keys are non-nullable /// - Map entries are non-nullable /// - Map values can be null -pub fn map_type_from_key_value_types(key_type: &DataType, value_type: &DataType) -> DataType { +pub fn map_type_from_key_value_types( + key_type: &DataType, + value_type: &DataType, +) -> DataType { DataType::Map( Arc::new(Field::new( "entries", @@ -137,7 +142,6 @@ pub fn map_from_keys_values_offsets_nulls( field, offsets, entries, nulls, false, )?)) } - #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key fn map_deduplicate_keys( flat_keys: &ArrayRef, @@ -199,8 +203,11 @@ fn map_deduplicate_keys( } key_to_output_idx.clear(); for cur_entry_idx in 0..num_keys_entries { - let key = ScalarValue::try_from_array(&flat_keys, cur_keys_offset + cur_entry_idx)? - .compacted(); + let key = ScalarValue::try_from_array( + &flat_keys, + cur_keys_offset + cur_entry_idx, + )? + .compacted(); let abs_value_idx = (cur_values_offset + cur_entry_idx) as i32; if let Some(&output_idx) = key_to_output_idx.get(&key) { @@ -248,7 +255,10 @@ mod tests { use arrow::array::{Int32Array, LargeListArray, NullArray, StringArray}; use arrow::datatypes::Int32Type; - fn int32_utf8_inputs(keys: Vec, values: Vec>) -> (ArrayRef, ArrayRef) { + fn int32_utf8_inputs( + keys: Vec, + values: Vec>, + ) -> (ArrayRef, ArrayRef) { let keys: ArrayRef = Arc::new(Int32Array::from(keys)); let values: ArrayRef = Arc::new(StringArray::from(values)); (keys, values) diff --git a/crates/sqllogictest/test_files/map/map_from_arrays.slt b/crates/sqllogictest/test_files/map/map_from_arrays.slt index b31ea664..ce3158c9 100644 --- a/crates/sqllogictest/test_files/map/map_from_arrays.slt +++ b/crates/sqllogictest/test_files/map/map_from_arrays.slt @@ -138,7 +138,7 @@ SELECT map_from_arrays(arrow_cast(array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)' # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c')); @@ -171,7 +171,7 @@ AS tab(a, b); NULL statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Sliced input: OFFSET makes the function see list arrays whose first offset is # non-zero, so the child arrays still contain the skipped row's elements. diff --git a/crates/sqllogictest/test_files/map/map_from_entries.slt b/crates/sqllogictest/test_files/map/map_from_entries.slt index a9905235..044955f1 100644 --- a/crates/sqllogictest/test_files/map/map_from_entries.slt +++ b/crates/sqllogictest/test_files/map/map_from_entries.slt @@ -166,7 +166,7 @@ AS tab(data); # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT map_from_entries(array( @@ -201,7 +201,7 @@ AS tab(data); {1: y} statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Sliced input: OFFSET makes the function see a list array whose first offset is # non-zero, so the child array still contains the skipped row's entries. diff --git a/crates/sqllogictest/test_files/map/str_to_map.slt b/crates/sqllogictest/test_files/map/str_to_map.slt index 55d2470e..854cd77d 100644 --- a/crates/sqllogictest/test_files/map/str_to_map.slt +++ b/crates/sqllogictest/test_files/map/str_to_map.slt @@ -116,7 +116,7 @@ NULL # LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones. statement ok -set datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT str_to_map('a:1,b:2,a:3'); @@ -143,12 +143,12 @@ SELECT str_to_map(col) FROM (VALUES ('a:1,b:2'), ('a:3,a:4')) AS t(col); {a: 4} statement ok -set datafusion.spark.map_key_dedup_policy = 'EXCEPTION'; +set spark.map_key_dedup_policy = 'EXCEPTION'; # Invalid policy values are rejected at SET time with a clear message. statement error -set datafusion.spark.map_key_dedup_policy = 'BOGUS'; +set spark.map_key_dedup_policy = 'BOGUS'; ---- -DataFusion error: Error setting config datafusion.spark.map_key_dedup_policy +DataFusion error: Error setting config spark.map_key_dedup_policy caused by Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN From 718703f05cfa810abea0941bc8238e50be77353e Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 22:14:05 +0530 Subject: [PATCH 04/10] map_key_dedup.slt test changed --- crates/sqllogictest/test_files/map/map_key_dedup_policy.slt | 2 +- crates/sqllogictest/test_files/map/str_to_map.slt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt b/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt index 2784f462..17ebf174 100644 --- a/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt +++ b/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt @@ -5,7 +5,7 @@ set spark.map_key_dedup_policy = 'LAST_WIN'; query ? SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c')); ---- -{2: b, 1: c} +{1: c, 2: b} statement ok set spark.map_key_dedup_policy = 'EXCEPTION'; diff --git a/crates/sqllogictest/test_files/map/str_to_map.slt b/crates/sqllogictest/test_files/map/str_to_map.slt index 854cd77d..f6b5a961 100644 --- a/crates/sqllogictest/test_files/map/str_to_map.slt +++ b/crates/sqllogictest/test_files/map/str_to_map.slt @@ -149,6 +149,6 @@ set spark.map_key_dedup_policy = 'EXCEPTION'; statement error set spark.map_key_dedup_policy = 'BOGUS'; ---- -DataFusion error: Error setting config spark.map_key_dedup_policy +DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN caused by Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN From d28450c5e93741d59c1c3c96cbf8a75aa22def3f Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 22:35:18 +0530 Subject: [PATCH 05/10] spark option insert check --- crates/spark/src/session_state.rs | 15 ++++++++++----- crates/sqllogictest/test_files/map/str_to_map.slt | 2 -- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index a73fbd89..cda910c3 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -10,8 +10,8 @@ use datafusion::execution::SessionStateBuilder; use crate::SparkOptions; use crate::planner::SparkFunctionPlanner; use crate::{ - all_default_aggregate_functions, all_default_scalar_functions, all_default_table_functions, - all_default_window_functions, + all_default_aggregate_functions, all_default_scalar_functions, + all_default_table_functions, all_default_window_functions, }; /// Extension trait for adding Apache Spark features to [`SessionStateBuilder`]. @@ -43,11 +43,16 @@ pub trait SessionStateBuilderSpark { impl SessionStateBuilderSpark for SessionStateBuilder { fn with_spark_features(mut self) -> Self { - self.config() + // Check if SparkOptions already exists; if not, insert default. + // The immutable borrow from .get() is released before .insert(). + let mut extensions = &mut self + .config() .get_or_insert_with(Default::default) .options_mut() - .extensions - .insert(SparkOptions::default()); + .extensions; + if extensions.get::().is_none() { + extensions.insert(SparkOptions::default()); + } self.expr_planners() .get_or_insert_with(Vec::new) diff --git a/crates/sqllogictest/test_files/map/str_to_map.slt b/crates/sqllogictest/test_files/map/str_to_map.slt index f6b5a961..6817c492 100644 --- a/crates/sqllogictest/test_files/map/str_to_map.slt +++ b/crates/sqllogictest/test_files/map/str_to_map.slt @@ -150,5 +150,3 @@ statement error set spark.map_key_dedup_policy = 'BOGUS'; ---- DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN -caused by -Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN From 0dadf0eb66db48c2e7208de14db0e69a43ca70f7 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 22:56:28 +0530 Subject: [PATCH 06/10] docs for making with_config survive --- crates/spark/src/session_state.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index cda910c3..a2e0c56a 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -42,6 +42,11 @@ pub trait SessionStateBuilderSpark { } impl SessionStateBuilderSpark for SessionStateBuilder { + /// Registers Spark-compatible planners, functions, and a default + /// [`SparkOptions`] extension (only if one isn't already set). + /// + /// Call `with_config` **before** this method: `with_config` replaces the + /// whole config and would drop the `SparkOptions` extension. fn with_spark_features(mut self) -> Self { // Check if SparkOptions already exists; if not, insert default. // The immutable borrow from .get() is released before .insert(). From aaafb9eaa930befcb72628320c72337e0a964932 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Fri, 13 Nov 2026 23:06:06 +0530 Subject: [PATCH 07/10] fix as per feedback --- crates/spark/src/config.rs | 25 +++++++- .../spark/src/function/map/map_from_arrays.rs | 18 +++--- .../src/function/map/map_from_entries.rs | 50 ++++++++-------- crates/spark/src/function/map/str_to_map.rs | 60 +++++++++++-------- crates/spark/src/session_state.rs | 2 +- 5 files changed, 93 insertions(+), 62 deletions(-) diff --git a/crates/spark/src/config.rs b/crates/spark/src/config.rs index e3cf05ba..bce60677 100644 --- a/crates/spark/src/config.rs +++ b/crates/spark/src/config.rs @@ -5,7 +5,7 @@ use std::fmt; use std::str::FromStr; -use datafusion_common::config::{ConfigExtension, ConfigField, Visit}; +use datafusion_common::config::{ConfigExtension, ConfigField, ConfigOptions, Visit}; use datafusion_common::extensions_options; use datafusion_common::{DataFusionError, Result}; @@ -60,6 +60,14 @@ impl ConfigExtension for SparkOptions { const PREFIX: &'static str = "spark"; } +/// Returns whether Spark map functions should keep the last value for duplicate keys. +pub fn map_last_value_wins(options: &ConfigOptions) -> bool { + options + .extensions + .get::() + .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin) +} + #[cfg(test)] mod tests { use super::*; @@ -96,4 +104,19 @@ mod tests { assert!(error.contains("Invalid MapKeyDedupPolicy"), "{error}"); } + + #[test] + fn missing_spark_options_use_exception_policy() { + assert!(!map_last_value_wins(&ConfigOptions::default())); + } + + #[test] + fn configured_spark_options_enable_last_win() { + let mut config = ConfigOptions::default(); + config.extensions.insert(SparkOptions { + map_key_dedup_policy: MapKeyDedupPolicy::LastWin, + }); + + assert!(map_last_value_wins(&config)); + } } diff --git a/crates/spark/src/function/map/map_from_arrays.rs b/crates/spark/src/function/map/map_from_arrays.rs index c8efcf68..971f77f8 100644 --- a/crates/spark/src/function/map/map_from_arrays.rs +++ b/crates/spark/src/function/map/map_from_arrays.rs @@ -2,18 +2,19 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +use crate::config::map_last_value_wins; use crate::function::map::utils::{ - get_element_type, get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, - map_type_from_key_value_types, + get_element_type, get_list_offsets, get_list_values, + map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; -use crate::{MapKeyDedupPolicy, SparkOptions}; use arrow::array::{Array, ArrayRef, NullArray}; use arrow::compute::kernels::cast; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, }; use datafusion_functions::utils::make_scalar_function; use std::sync::Arc; @@ -68,11 +69,7 @@ impl ScalarUDFImpl for MapFromArrays { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = args - .config_options - .extensions - .get::() - .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); + let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( move |args: &[ArrayRef]| map_from_arrays_inner(args, last_value_wins), vec![], @@ -130,7 +127,8 @@ mod tests { }) .expect("return_field_from_args should succeed"); - let expected_type = map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); + let expected_type = + map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); assert_eq!(out.data_type(), &expected_type); assert!( !out.is_nullable(), diff --git a/crates/spark/src/function/map/map_from_entries.rs b/crates/spark/src/function/map/map_from_entries.rs index 28fbf2cf..85e02de8 100644 --- a/crates/spark/src/function/map/map_from_entries.rs +++ b/crates/spark/src/function/map/map_from_entries.rs @@ -4,18 +4,19 @@ use std::sync::Arc; +use crate::config::map_last_value_wins; use crate::function::map::utils::{ get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls, map_type_from_key_value_types, }; -use crate::{MapKeyDedupPolicy, SparkOptions}; use arrow::array::{Array, ArrayRef, NullBufferBuilder, StructArray}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, }; use datafusion_functions::utils::make_scalar_function; @@ -58,15 +59,18 @@ impl ScalarUDFImpl for MapFromEntries { return exec_err!("map_from_entries: expected one argument"); }; - let (entries_element_field, entries_element_type) = match entries_field.data_type() { - DataType::List(field) - | DataType::LargeList(field) - | DataType::FixedSizeList(field, _) => Ok((field.as_ref(), field.data_type())), - wrong_type => exec_err!( - "map_from_entries: expected array>, got {:?}", - wrong_type - ), - }?; + let (entries_element_field, entries_element_type) = + match entries_field.data_type() { + DataType::List(field) + | DataType::LargeList(field) + | DataType::FixedSizeList(field, _) => { + Ok((field.as_ref(), field.data_type())) + } + wrong_type => exec_err!( + "map_from_entries: expected array>, got {:?}", + wrong_type + ), + }?; let (keys_type, values_type) = match entries_element_type { DataType::Struct(fields) if fields.len() == 2 => { @@ -85,11 +89,7 @@ impl ScalarUDFImpl for MapFromEntries { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = args - .config_options - .extensions - .get::() - .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); + let last_value_wins = map_last_value_wins(&args.config_options); make_scalar_function( move |args: &[ArrayRef]| map_from_entries_inner(args, last_value_wins), vec![], @@ -102,13 +102,14 @@ fn map_from_entries_inner(args: &[ArrayRef], last_value_wins: bool) -> Result() { - Some(a) => Ok((a.column(0), a.column(1))), - None => exec_err!( - "map_from_entries: expected array>, got {:?}", - entries_values.data_type() - ), - }?; + let (flat_keys, flat_values) = + match entries_values.as_any().downcast_ref::() { + Some(a) => Ok((a.column(0), a.column(1))), + None => exec_err!( + "map_from_entries: expected array>, got {:?}", + entries_values.data_type() + ), + }?; let entries_with_nulls = entries_values.nulls().and_then(|entries_inner_nulls| { let mut builder = NullBufferBuilder::new_with_len(0); @@ -163,7 +164,8 @@ mod tests { #[test] fn test_map_from_entries_nullability_matches_input() { let func = MapFromEntries::new(); - let expected_type = map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); + let expected_type = + map_type_from_key_value_types(&DataType::Int32, &DataType::Utf8); // Non-nullable array and elements => non-nullable result let non_nullable_field = make_entries_field(false, false); diff --git a/crates/spark/src/function/map/str_to_map.rs b/crates/spark/src/function/map/str_to_map.rs index 28962cc4..039041c4 100644 --- a/crates/spark/src/function/map/str_to_map.rs +++ b/crates/spark/src/function/map/str_to_map.rs @@ -5,18 +5,22 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use arrow::array::{Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder}; +use arrow::array::{ + Array, ArrayRef, MapBuilder, MapFieldNames, StringArrayType, StringBuilder, +}; use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::cast::{as_large_string_array, as_string_array, as_string_view_array}; +use datafusion_common::cast::{ + as_large_string_array, as_string_array, as_string_view_array, +}; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, - Volatility, + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, Volatility, }; +use crate::config::map_last_value_wins; use crate::function::map::utils::map_type_from_key_value_types; -use crate::{MapKeyDedupPolicy, SparkOptions}; const DEFAULT_PAIR_DELIM: &str = ","; const DEFAULT_KV_DELIM: &str = ":"; @@ -85,11 +89,7 @@ impl ScalarUDFImpl for SparkStrToMap { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { - let last_value_wins = args - .config_options - .extensions - .get::() - .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin); + let last_value_wins = map_last_value_wins(&args.config_options); let arrays: Vec = ColumnarValue::values_to_arrays(&args.args)?; let result = str_to_map_inner(&arrays, last_value_wins)?; Ok(ColumnarValue::Array(result)) @@ -108,9 +108,12 @@ fn str_to_map_inner(args: &[ArrayRef], last_value_wins: bool) -> Result { - str_to_map_impl(as_string_view_array(&args[0])?, None, None, last_value_wins) - } + DataType::Utf8View => str_to_map_impl( + as_string_view_array(&args[0])?, + None, + None, + last_value_wins, + ), other => exec_err!( "Unsupported data type {other:?} for str_to_map, \ expected Utf8, LargeUtf8, or Utf8View" @@ -151,18 +154,22 @@ fn str_to_map_inner(args: &[ArrayRef], last_value_wins: bool) -> Result str_to_map_impl( - as_large_string_array(&args[0])?, - Some(as_large_string_array(&args[1])?), - Some(as_large_string_array(&args[2])?), - last_value_wins, - ), - (DataType::Utf8View, DataType::Utf8View, DataType::Utf8View) => str_to_map_impl( - as_string_view_array(&args[0])?, - Some(as_string_view_array(&args[1])?), - Some(as_string_view_array(&args[2])?), - last_value_wins, - ), + (DataType::LargeUtf8, DataType::LargeUtf8, DataType::LargeUtf8) => { + str_to_map_impl( + as_large_string_array(&args[0])?, + Some(as_large_string_array(&args[1])?), + Some(as_large_string_array(&args[2])?), + last_value_wins, + ) + } + (DataType::Utf8View, DataType::Utf8View, DataType::Utf8View) => { + str_to_map_impl( + as_string_view_array(&args[0])?, + Some(as_string_view_array(&args[1])?), + Some(as_string_view_array(&args[2])?), + last_value_wins, + ) + } (t1, t2, t3) => exec_err!( "Unsupported data types ({t1:?}, {t2:?}, {t3:?}) for str_to_map, \ expected matching Utf8, LargeUtf8, or Utf8View" @@ -213,7 +220,8 @@ fn str_to_map_impl<'a, V: StringArrayType<'a> + Copy>( } // Per-row delimiter extraction - let pair_delim = pair_delim_array.map_or(DEFAULT_PAIR_DELIM, |a| a.value(row_idx)); + let pair_delim = + pair_delim_array.map_or(DEFAULT_PAIR_DELIM, |a| a.value(row_idx)); let kv_delim = kv_delim_array.map_or(DEFAULT_KV_DELIM, |a| a.value(row_idx)); let text = text_array.value(row_idx); diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index a2e0c56a..ea43d7b8 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -50,7 +50,7 @@ impl SessionStateBuilderSpark for SessionStateBuilder { fn with_spark_features(mut self) -> Self { // Check if SparkOptions already exists; if not, insert default. // The immutable borrow from .get() is released before .insert(). - let mut extensions = &mut self + let extensions = &mut self .config() .get_or_insert_with(Default::default) .options_mut() From 55b794f263a15d4ee46b9cc6c059ac42e026ce9f Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Sat, 14 Nov 2026 12:15:38 +0530 Subject: [PATCH 08/10] fix as per feedback --- crates/spark/src/function/map/utils.rs | 2 +- .../test_files/map/map_key_dedup_policy.slt | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 crates/sqllogictest/test_files/map/map_key_dedup_policy.slt diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index e592e651..f061f727 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -220,7 +220,7 @@ fn map_deduplicate_keys( "[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \ please check the input data. To allow duplicate keys with \ last-value-wins semantics, set \ - `datafusion.spark.map_key_dedup_policy` to `LAST_WIN`." + `spark.map_key_dedup_policy` to `LAST_WIN`." ); } keys_mask_builder.append_value(true); diff --git a/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt b/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt deleted file mode 100644 index 17ebf174..00000000 --- a/crates/sqllogictest/test_files/map/map_key_dedup_policy.slt +++ /dev/null @@ -1,16 +0,0 @@ -# Spark map construction uses EXCEPTION by default. -statement ok -set spark.map_key_dedup_policy = 'LAST_WIN'; - -query ? -SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c')); ----- -{1: c, 2: b} - -statement ok -set spark.map_key_dedup_policy = 'EXCEPTION'; - -statement error -set spark.map_key_dedup_policy = 'BOGUS'; ----- -DataFusion error: Invalid or Unsupported Configuration: Invalid MapKeyDedupPolicy: BOGUS. Expected one of: EXCEPTION, LAST_WIN From f3b40a4bbfb45a3f43338ece4da57ca3d72c0605 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Mon, 16 Nov 2026 19:34:25 +0530 Subject: [PATCH 09/10] fix(spark): honor legacy map policy and address config review --- crates/spark/src/config.rs | 46 +++++++++++++++++++ crates/spark/src/function/map/utils.rs | 2 +- crates/spark/src/function/string/substring.rs | 2 +- crates/spark/src/lib.rs | 1 + crates/spark/src/session_state.rs | 20 ++++++++ 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/crates/spark/src/config.rs b/crates/spark/src/config.rs index bce60677..ecd22c10 100644 --- a/crates/spark/src/config.rs +++ b/crates/spark/src/config.rs @@ -9,6 +9,7 @@ use datafusion_common::config::{ConfigExtension, ConfigField, ConfigOptions, Vis use datafusion_common::extensions_options; use datafusion_common::{DataFusionError, Result}; +/// Policy for handling duplicate keys in Spark map functions. #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum MapKeyDedupPolicy { #[default] @@ -51,7 +52,10 @@ impl fmt::Display for MapKeyDedupPolicy { } extensions_options! { + /// Spark-specific configuration options under the `spark` namespace. pub struct SparkOptions { + /// How to handle duplicate map keys: EXCEPTION raises an error (default), + /// while LAST_WIN keeps the last value for each key. pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception } } @@ -61,11 +65,18 @@ impl ConfigExtension for SparkOptions { } /// Returns whether Spark map functions should keep the last value for duplicate keys. +/// +/// The new key is `spark.map_key_dedup_policy`. For DataFusion 56 compatibility, +/// the legacy `datafusion.spark.map_key_dedup_policy` is also honored: LAST_WIN +/// on either key takes precedence over EXCEPTION on the other. +/// Remove the legacy lookup when upgrading to DataFusion 57. pub fn map_last_value_wins(options: &ConfigOptions) -> bool { options .extensions .get::() .is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin) + || options.spark.map_key_dedup_policy + == datafusion_common::config::MapKeyDedupPolicy::LastWin } #[cfg(test)] @@ -119,4 +130,39 @@ mod tests { assert!(map_last_value_wins(&config)); } + + #[test] + fn legacy_and_extension_policy_combinations() { + for legacy in ["EXCEPTION", "LAST_WIN"] { + for extension in [None, Some("EXCEPTION"), Some("LAST_WIN")] { + let mut config = ConfigOptions::default(); + config + .set("datafusion.spark.map_key_dedup_policy", legacy) + .unwrap(); + if let Some(policy) = extension { + config.extensions.insert(SparkOptions::default()); + config.set("spark.map_key_dedup_policy", policy).unwrap(); + } + + assert_eq!( + map_last_value_wins(&config), + legacy == "LAST_WIN" || extension == Some("LAST_WIN"), + "legacy={legacy}, extension={extension:?}" + ); + } + } + } + + #[test] + fn map_policy_has_description() { + let entries = datafusion_common::config::ExtensionOptions::entries( + &SparkOptions::default(), + ); + let entry = entries + .iter() + .find(|entry| entry.key == "map_key_dedup_policy") + .unwrap(); + assert!(entry.description.contains("EXCEPTION")); + assert!(entry.description.contains("LAST_WIN")); + } } diff --git a/crates/spark/src/function/map/utils.rs b/crates/spark/src/function/map/utils.rs index f061f727..3c8eaa85 100644 --- a/crates/spark/src/function/map/utils.rs +++ b/crates/spark/src/function/map/utils.rs @@ -111,7 +111,6 @@ pub fn map_type_from_key_value_types( /// and is driven by `last_value_wins`: /// - `false` (Spark's default `EXCEPTION`): raise `[DUPLICATED_MAP_KEY]` on any duplicate. /// - `true` (`LAST_WIN`): keep the last occurrence of each duplicate key. -/// pub fn map_from_keys_values_offsets_nulls( flat_keys: &ArrayRef, flat_values: &ArrayRef, @@ -142,6 +141,7 @@ pub fn map_from_keys_values_offsets_nulls( field, offsets, entries, nulls, false, )?)) } + #[allow(clippy::allow_attributes, clippy::mutable_key_type)] // ScalarValue has interior mutability but is intentionally used as hash key fn map_deduplicate_keys( flat_keys: &ArrayRef, diff --git a/crates/spark/src/function/string/substring.rs b/crates/spark/src/function/string/substring.rs index 354f87c8..3fda3918 100644 --- a/crates/spark/src/function/string/substring.rs +++ b/crates/spark/src/function/string/substring.rs @@ -360,7 +360,7 @@ impl SubstringBuilder for GenericBinaryBuilder { GenericBinaryBuilder::append_null(self); } fn append_empty(&mut self) { - GenericBinaryBuilder::append_value(self, &[]); + GenericBinaryBuilder::append_value(self, []); } } diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 95db320e..91a5a6da 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -115,6 +115,7 @@ //! Then use the extension trait - see [`SessionStateBuilderSpark::with_spark_features`] //! for an example. +/// Spark configuration extensions and map-key deduplication policy. pub mod config; pub mod function; pub mod planner; diff --git a/crates/spark/src/session_state.rs b/crates/spark/src/session_state.rs index ea43d7b8..f7628f9d 100644 --- a/crates/spark/src/session_state.rs +++ b/crates/spark/src/session_state.rs @@ -125,6 +125,26 @@ mod tests { ); } + #[test] + fn test_spark_features_preserve_configured_policy() { + let state = SessionStateBuilder::new() + .with_config(SessionConfig::new().with_option_extension(SparkOptions { + map_key_dedup_policy: crate::MapKeyDedupPolicy::LastWin, + })) + .with_spark_features() + .build(); + + assert_eq!( + state + .config_options() + .extensions + .get::() + .unwrap() + .map_key_dedup_policy, + crate::MapKeyDedupPolicy::LastWin + ); + } + #[tokio::test] async fn test_spark_dialect_with_spark_functions() { let query = "SELECT sha2('abc', 256), CAST(1 AS LONG)"; From 07d88f67d1bc45a9ae924bd5c3a8aeecec54ed65 Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Tue, 17 Nov 2026 19:04:09 +0530 Subject: [PATCH 10/10] fix(spark): remove duplicate module declarations --- crates/spark/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/spark/src/lib.rs b/crates/spark/src/lib.rs index 3b813b69..32445e27 100644 --- a/crates/spark/src/lib.rs +++ b/crates/spark/src/lib.rs @@ -168,8 +168,6 @@ pub mod planner; pub use config::{MapKeyDedupPolicy, SparkOptions}; mod eval_mode; -pub mod function; -pub mod planner; pub use eval_mode::EvalMode;