Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
f67e3d5
SparkOptions port
rizzabhdwaj Nov 13, 2026
19868f7
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 13, 2026
1465181
mapdedp reverted back to prev impl
rizzabhdwaj Nov 13, 2026
3bef740
Merge branch 'main' of https://github.com/apache/datafusion-spark int…
rizzabhdwaj Nov 13, 2026
f418983
mapdedp reverted back to prev impl
rizzabhdwaj Nov 13, 2026
91ab726
Merge branch 'port-spark-options-config-extension' of https://github.…
rizzabhdwaj Nov 13, 2026
718703f
map_key_dedup.slt test changed
rizzabhdwaj Nov 13, 2026
d28450c
spark option insert check
rizzabhdwaj Nov 13, 2026
0dadf0e
docs for making with_config survive
rizzabhdwaj Nov 13, 2026
aaafb9e
fix as per feedback
rizzabhdwaj Nov 13, 2026
789cdc1
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 13, 2026
55b794f
fix as per feedback
rizzabhdwaj Nov 14, 2026
437631a
Merge branch 'port-spark-options-config-extension' of https://github.…
rizzabhdwaj Nov 14, 2026
6abf445
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 14, 2026
6decf32
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 15, 2026
ad51835
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 16, 2026
7e23b4b
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 16, 2026
f3b40a4
fix(spark): honor legacy map policy and address config review
rizzabhdwaj Nov 16, 2026
0767ec6
Merge remote-tracking branch 'origin2/port-spark-options-config-exten…
rizzabhdwaj Nov 16, 2026
68a6786
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 16, 2026
059876e
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 17, 2026
07d88f6
fix(spark): remove duplicate module declarations
rizzabhdwaj Nov 17, 2026
b9912b2
Merge branch 'main' into port-spark-options-config-extension
rizzabhdwaj Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 168 additions & 0 deletions crates/spark/src/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use std::fmt;
use std::str::FromStr;

use datafusion_common::config::{ConfigExtension, ConfigField, ConfigOptions, Visit};
use datafusion_common::extensions_options;
use datafusion_common::{DataFusionError, Result};

/// Policy for handling duplicate keys in Spark map functions.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum MapKeyDedupPolicy {
#[default]
Exception,
LastWin,
}

impl FromStr for MapKeyDedupPolicy {
type Err = DataFusionError;

fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_ascii_uppercase().as_str() {
"EXCEPTION" => Ok(Self::Exception),
"LAST_WIN" => Ok(Self::LastWin),
other => Err(DataFusionError::Configuration(format!(
"Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN"
))),
}
}
}

impl ConfigField for MapKeyDedupPolicy {
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
v.some(key, self, description)
}

fn set(&mut self, _: &str, value: &str) -> Result<()> {
*self = value.parse()?;
Ok(())
}
}

impl fmt::Display for MapKeyDedupPolicy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Exception => write!(f, "EXCEPTION"),
Self::LastWin => write!(f, "LAST_WIN"),
}
}
}

extensions_options! {
/// Spark-specific configuration options under the `spark` namespace.
pub struct SparkOptions {
/// How to handle duplicate map keys: EXCEPTION raises an error (default),
/// while LAST_WIN keeps the last value for each key.
pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception
}
}

impl ConfigExtension for SparkOptions {
const PREFIX: &'static str = "spark";
}

/// Returns whether Spark map functions should keep the last value for duplicate keys.
///
/// The new key is `spark.map_key_dedup_policy`. For DataFusion 56 compatibility,
/// the legacy `datafusion.spark.map_key_dedup_policy` is also honored: LAST_WIN
/// on either key takes precedence over EXCEPTION on the other.
/// Remove the legacy lookup when upgrading to DataFusion 57.
pub fn map_last_value_wins(options: &ConfigOptions) -> bool {
options
.extensions
.get::<SparkOptions>()
.is_some_and(|options| options.map_key_dedup_policy == MapKeyDedupPolicy::LastWin)
|| options.spark.map_key_dedup_policy
== datafusion_common::config::MapKeyDedupPolicy::LastWin
}

#[cfg(test)]
mod tests {
use super::*;
use datafusion_common::config::ConfigOptions;

#[test]
fn spark_options_use_spark_prefix() {
let mut config = ConfigOptions::default();
config.extensions.insert(SparkOptions::default());

config
.set("spark.map_key_dedup_policy", "last_win")
.unwrap();

assert_eq!(
config
.extensions
.get::<SparkOptions>()
.unwrap()
.map_key_dedup_policy,
MapKeyDedupPolicy::LastWin
);
}

#[test]
fn spark_options_reject_invalid_policy() {
let mut config = ConfigOptions::default();
config.extensions.insert(SparkOptions::default());

let error = config
.set("spark.map_key_dedup_policy", "bogus")
.unwrap_err()
.to_string();

assert!(error.contains("Invalid MapKeyDedupPolicy"), "{error}");
}

#[test]
fn missing_spark_options_use_exception_policy() {
assert!(!map_last_value_wins(&ConfigOptions::default()));
}

#[test]
fn configured_spark_options_enable_last_win() {
let mut config = ConfigOptions::default();
config.extensions.insert(SparkOptions {
map_key_dedup_policy: MapKeyDedupPolicy::LastWin,
});

assert!(map_last_value_wins(&config));
}

#[test]
fn legacy_and_extension_policy_combinations() {
for legacy in ["EXCEPTION", "LAST_WIN"] {
for extension in [None, Some("EXCEPTION"), Some("LAST_WIN")] {
let mut config = ConfigOptions::default();
config
.set("datafusion.spark.map_key_dedup_policy", legacy)
.unwrap();
if let Some(policy) = extension {
config.extensions.insert(SparkOptions::default());
config.set("spark.map_key_dedup_policy", policy).unwrap();
}

assert_eq!(
map_last_value_wins(&config),
legacy == "LAST_WIN" || extension == Some("LAST_WIN"),
"legacy={legacy}, extension={extension:?}"
);
}
}
}

#[test]
fn map_policy_has_description() {
let entries = datafusion_common::config::ExtensionOptions::entries(
&SparkOptions::default(),
);
let entry = entries
.iter()
.find(|entry| entry.key == "map_key_dedup_policy")
.unwrap();
assert!(entry.description.contains("EXCEPTION"));
assert!(entry.description.contains("LAST_WIN"));
}
}
5 changes: 2 additions & 3 deletions crates/spark/src/function/map/map_from_arrays.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::config::map_last_value_wins;
use crate::function::map::utils::{
get_element_type, get_list_offsets, get_list_values,
map_from_keys_values_offsets_nulls, map_type_from_key_value_types,
};
use arrow::array::{Array, ArrayRef, NullArray};
use arrow::compute::kernels::cast;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::config::MapKeyDedupPolicy;
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Expand Down Expand Up @@ -69,8 +69,7 @@ impl ScalarUDFImpl for MapFromArrays {
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let last_value_wins =
args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin;
let last_value_wins = map_last_value_wins(&args.config_options);
make_scalar_function(
move |args: &[ArrayRef]| map_from_arrays_inner(args, last_value_wins),
vec![],
Expand Down
5 changes: 2 additions & 3 deletions crates/spark/src/function/map/map_from_entries.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@

use std::sync::Arc;

use crate::config::map_last_value_wins;
use crate::function::map::utils::{
get_list_offsets, get_list_values, map_from_keys_values_offsets_nulls,
map_type_from_key_value_types,
};
use arrow::array::{Array, ArrayRef, NullBufferBuilder, StructArray};
use arrow::buffer::NullBuffer;
use arrow::datatypes::{DataType, Field, FieldRef};
use datafusion_common::config::MapKeyDedupPolicy;
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, exec_err, internal_err};
use datafusion_expr::{
Expand Down Expand Up @@ -89,8 +89,7 @@ impl ScalarUDFImpl for MapFromEntries {
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let last_value_wins =
args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin;
let last_value_wins = map_last_value_wins(&args.config_options);
make_scalar_function(
move |args: &[ArrayRef]| map_from_entries_inner(args, last_value_wins),
vec![],
Expand Down
9 changes: 4 additions & 5 deletions crates/spark/src/function/map/str_to_map.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ use datafusion_expr::{
TypeSignature, Volatility,
};

use crate::config::map_last_value_wins;
use crate::function::map::utils::map_type_from_key_value_types;
use datafusion_common::config::MapKeyDedupPolicy;

const DEFAULT_PAIR_DELIM: &str = ",";
const DEFAULT_KV_DELIM: &str = ":";
Expand All @@ -37,7 +37,7 @@ const DEFAULT_KV_DELIM: &str = ":";
///
/// # Duplicate Key Handling
/// Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4502-L4511),
/// wired through DataFusion's `datafusion.spark.map_key_dedup_policy`:
/// wired through DataFusion Spark's `spark.map_key_dedup_policy`:
/// - `EXCEPTION` (default): error on duplicate keys.
/// - `LAST_WIN`: keep the last occurrence of each duplicate key.
#[derive(Debug, PartialEq, Eq, Hash)]
Expand Down Expand Up @@ -89,8 +89,7 @@ impl ScalarUDFImpl for SparkStrToMap {
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let last_value_wins =
args.config_options.spark.map_key_dedup_policy == MapKeyDedupPolicy::LastWin;
let last_value_wins = map_last_value_wins(&args.config_options);
let arrays: Vec<ArrayRef> = ColumnarValue::values_to_arrays(&args.args)?;
let result = str_to_map_inner(&arrays, last_value_wins)?;
Ok(ColumnarValue::Array(result))
Expand Down Expand Up @@ -275,7 +274,7 @@ fn str_to_map_impl<'a, V: StringArrayType<'a> + Copy>(
"[DUPLICATED_MAP_KEY] Duplicate map key '{key}' was found, \
please check the input data. To allow duplicate keys with \
last-value-wins semantics, set \
`datafusion.spark.map_key_dedup_policy` to `LAST_WIN`."
`spark.map_key_dedup_policy` to `LAST_WIN`."
Comment thread
rizzabhdwaj marked this conversation as resolved.
Resolved
);
}

Expand Down
4 changes: 1 addition & 3 deletions crates/spark/src/function/map/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,8 +111,6 @@ pub fn map_type_from_key_value_types(
/// and is driven by `last_value_wins`:
/// - `false` (Spark's default `EXCEPTION`): raise `[DUPLICATED_MAP_KEY]` on any duplicate.
/// - `true` (`LAST_WIN`): keep the last occurrence of each duplicate key.
///
/// Callers wire this from `datafusion.spark.map_key_dedup_policy`.
pub fn map_from_keys_values_offsets_nulls(
flat_keys: &ArrayRef,
flat_values: &ArrayRef,
Expand Down Expand Up @@ -222,7 +220,7 @@ fn map_deduplicate_keys(
"[DUPLICATED_MAP_KEY] Duplicate map key {key} was found, \
please check the input data. To allow duplicate keys with \
last-value-wins semantics, set \
`datafusion.spark.map_key_dedup_policy` to `LAST_WIN`."
`spark.map_key_dedup_policy` to `LAST_WIN`."
);
}
keys_mask_builder.append_value(true);
Expand Down
2 changes: 1 addition & 1 deletion crates/spark/src/function/string/substring.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ impl<O: OffsetSizeTrait> SubstringBuilder for GenericBinaryBuilder<O> {
GenericBinaryBuilder::append_null(self);
}
fn append_empty(&mut self) {
GenericBinaryBuilder::append_value(self, &[]);
GenericBinaryBuilder::append_value(self, []);
}
}

Expand Down
6 changes: 5 additions & 1 deletion crates/spark/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,14 @@
//! [Getting started](#getting-started), then use the extension trait - see
//! [`SessionStateBuilderSpark::with_spark_features`] for an example.

mod eval_mode;
/// Spark configuration extensions and map-key deduplication policy.
pub mod config;
pub mod function;
pub mod planner;

pub use config::{MapKeyDedupPolicy, SparkOptions};
mod eval_mode;

pub use eval_mode::EvalMode;

#[cfg(feature = "core")]
Expand Down
46 changes: 46 additions & 0 deletions crates/spark/src/session_state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use std::sync::Arc;

use datafusion::execution::SessionStateBuilder;

use crate::SparkOptions;
use crate::planner::SparkFunctionPlanner;
use crate::{
all_default_aggregate_functions, all_default_scalar_functions,
Expand Down Expand Up @@ -69,7 +70,23 @@ pub trait SessionStateBuilderSpark {
}

impl SessionStateBuilderSpark for SessionStateBuilder {
/// Registers Spark-compatible planners, functions, and a default
/// [`SparkOptions`] extension (only if one isn't already set).
///
/// Call `with_config` **before** this method: `with_config` replaces the
/// whole config and would drop the `SparkOptions` extension.
fn with_spark_features(mut self) -> Self {
// Check if SparkOptions already exists; if not, insert default.
// The immutable borrow from .get() is released before .insert().
let extensions = &mut self
.config()
.get_or_insert_with(Default::default)
.options_mut()
.extensions;
if extensions.get::<SparkOptions>().is_none() {
extensions.insert(SparkOptions::default());
}

self.expr_planners()
.get_or_insert_with(Vec::new)
// planners are evaluated in order of insertion. Push Apache Spark function planner to the front
Expand Down Expand Up @@ -111,6 +128,15 @@ mod tests {
fn test_session_state_with_spark_features() {
let state = SessionStateBuilder::new().with_spark_features().build();

assert!(
state
.config_options()
.extensions
.get::<SparkOptions>()
.is_some(),
"Spark options should be registered"
);

assert!(
state.scalar_functions().contains_key("sha2"),
"Apache Spark scalar function 'sha2' should be registered"
Expand All @@ -127,6 +153,26 @@ mod tests {
);
}

#[test]
fn test_spark_features_preserve_configured_policy() {
let state = SessionStateBuilder::new()
.with_config(SessionConfig::new().with_option_extension(SparkOptions {
map_key_dedup_policy: crate::MapKeyDedupPolicy::LastWin,
}))
.with_spark_features()
.build();

assert_eq!(
state
.config_options()
.extensions
.get::<SparkOptions>()
.unwrap()
.map_key_dedup_policy,
crate::MapKeyDedupPolicy::LastWin
);
}

#[tokio::test]
async fn test_spark_dialect_with_spark_functions() {
let query = "SELECT sha2('abc', 256), CAST(1 AS LONG)";
Expand Down
4 changes: 2 additions & 2 deletions crates/sqllogictest/test_files/map/map_from_arrays.slt
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ SELECT map_from_arrays(arrow_cast(array('a', 'b', 'c'), 'FixedSizeList(3, Utf8)'

# LAST_WIN policy: duplicates are allowed; later occurrences overwrite earlier ones.
statement ok
set datafusion.spark.map_key_dedup_policy = 'LAST_WIN';
set spark.map_key_dedup_policy = 'LAST_WIN';

query ?
SELECT map_from_arrays(array(1, 2, 1), array('a', 'b', 'c'));
Expand Down Expand Up @@ -171,7 +171,7 @@ AS tab(a, b);
NULL

statement ok
set datafusion.spark.map_key_dedup_policy = 'EXCEPTION';
set spark.map_key_dedup_policy = 'EXCEPTION';

# Sliced input: OFFSET makes the function see list arrays whose first offset is
# non-zero, so the child arrays still contain the skipped row's elements.
Expand Down
Loading
Loading