From 115838bc6c15aa07f207e75755edaa6b6ecc063c Mon Sep 17 00:00:00 2001 From: rishabh1244 <1244rishabh@gmail.com> Date: Thu, 12 Nov 2026 22:03:16 +0530 Subject: [PATCH] moved SparkOptions to datafusion-spark --- datafusion/common/src/config.rs | 88 ------------------- datafusion/spark/src/config.rs | 61 +++++++++++++ .../test_files/information_schema.slt | 2 - .../library-user-guide/upgrading/55.0.0.md | 5 +- docs/source/user-guide/configs.md | 1 - 5 files changed, 63 insertions(+), 94 deletions(-) create mode 100644 datafusion/spark/src/config.rs diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 6f33782ef91f2..c95c43653c9d2 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -830,53 +830,6 @@ impl Display for ConfigFilterSelectivity { } } -/// Policy for handling duplicate keys in Spark-compatible map-construction -/// functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors -/// Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961). -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub enum MapKeyDedupPolicy { - /// Raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - #[default] - Exception, - /// Keep the last occurrence of each duplicate key. - LastWin, -} - -impl FromStr for MapKeyDedupPolicy { - type Err = DataFusionError; - - fn from_str(s: &str) -> Result { - match s.to_ascii_uppercase().as_str() { - "EXCEPTION" => Ok(Self::Exception), - "LAST_WIN" => Ok(Self::LastWin), - other => Err(DataFusionError::Configuration(format!( - "Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN" - ))), - } - } -} - -impl ConfigField for MapKeyDedupPolicy { - fn visit(&self, v: &mut V, key: &str, description: &'static str) { - v.some(key, self, description) - } - - fn set(&mut self, _: &str, value: &str) -> Result<()> { - *self = MapKeyDedupPolicy::from_str(value)?; - Ok(()) - } -} - -impl Display for MapKeyDedupPolicy { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let str = match self { - Self::Exception => "EXCEPTION", - Self::LastWin => "LAST_WIN", - }; - write!(f, "{str}") - } -} - impl From for Option { fn from(c: SpillCompression) -> Self { match c { @@ -2248,24 +2201,6 @@ impl<'a> TryFrom<&'a FormatOptions> for arrow::util::display::FormatOptions<'a> } } -config_namespace! { - /// Options controlling DataFusion's Spark-compatibility layer (the - /// `datafusion-spark` crate). Keys here mirror their `spark.sql.*` - /// equivalents in Apache Spark. - pub struct SparkOptions { - /// Policy for handling duplicate keys in Spark-compatible map-construction - /// functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). - /// - /// Mirrors Spark's - /// [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - /// - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - /// - `LAST_WIN`: keep the last occurrence of each duplicate key. - /// - /// Values are case-insensitive. - pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception - } -} - /// A key value pair, with a corresponding description #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub struct ConfigEntry { @@ -2297,8 +2232,6 @@ pub struct ConfigOptions { pub extensions: Extensions, /// Formatting options when printing batches pub format: FormatOptions, - /// Spark-compatibility options (the `datafusion-spark` crate) - pub spark: SparkOptions, } impl ConfigField for ConfigOptions { @@ -2309,7 +2242,6 @@ impl ConfigField for ConfigOptions { self.explain.visit(v, "datafusion.explain", ""); self.sql_parser.visit(v, "datafusion.sql_parser", ""); self.format.visit(v, "datafusion.format", ""); - self.spark.visit(v, "datafusion.spark", ""); } fn set(&mut self, key: &str, value: &str) -> Result<()> { @@ -2322,7 +2254,6 @@ impl ConfigField for ConfigOptions { "explain" => self.explain.set(rem, value), "sql_parser" => self.sql_parser.set(rem, value), "format" => self.format.set(rem, value), - "spark" => self.spark.set(rem, value), _ => _config_err!("Config value \"{key}\" not found on ConfigOptions"), } } @@ -2362,7 +2293,6 @@ impl ConfigField for ConfigOptions { "explain" => self.explain.reset(rem), "sql_parser" => self.sql_parser.reset(rem), "format" => self.format.reset(rem), - "spark" => self.spark.reset(rem), other => _config_err!("Config value \"{other}\" not found on ConfigOptions"), } } @@ -4134,17 +4064,6 @@ config_namespace! { /// of columns, it creates a union schema containing all columns found across the files, and will /// pad any files missing columns with null values for their rows. pub truncated_rows: Option, default = None - /// Whether to map file columns to the table schema by header name instead of by position. - /// - /// Only applies when `has_header` is true. When enabled, each file's header row is read - /// and columns are matched by name: table columns missing from a file are filled with - /// nulls, and file columns not declared in the table are ignored. Names are matched exactly - /// (case-sensitive) and duplicate header names resolve to the first occurrence. Header - /// names containing quoted newlines are not supported. The header is located the same - /// way the CSV parser does: a leading UTF-8 BOM, blank lines, and comment lines are skipped. - /// - /// Defaults to `false` when `None`. - pub match_columns_by_header: Option, default = None } } @@ -4270,13 +4189,6 @@ impl CsvOptions { self } - /// Whether to map file columns to the table schema by header name instead of by position. - /// Only applies when the file has a header. - pub fn with_match_columns_by_header(mut self, match_by_header: bool) -> Self { - self.match_columns_by_header = Some(match_by_header); - self - } - /// Set the compression level for the output file. /// The valid range depends on the compression algorithm. /// If not specified, the default level for the algorithm is used. diff --git a/datafusion/spark/src/config.rs b/datafusion/spark/src/config.rs new file mode 100644 index 0000000000000..876b098e53462 --- /dev/null +++ b/datafusion/spark/src/config.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use std::fmt; +use std::str::FromStr; + +use datafusion_common::config::{ConfigExtension, ConfigField, Visit}; +use datafusion_common::extensions_options; +use datafusion_common::{DataFusionError, Result}; + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum MapKeyDedupPolicy { + #[default] + Exception, + LastWin, +} + +impl FromStr for MapKeyDedupPolicy { + type Err = DataFusionError; + + fn from_str(s: &str) -> Result { + match s.to_ascii_uppercase().as_str() { + "EXCEPTION" => Ok(Self::Exception), + "LAST_WIN" => Ok(Self::LastWin), + other => Err(DataFusionError::Configuration(format!( + "Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN" + ))), + } + } +} + +impl ConfigField for MapKeyDedupPolicy { + fn visit(&self, v: &mut V, key: &str, description: &'static str) { + v.some(key, self, description) + } + + fn set(&mut self, _: &str, value: &str) -> Result<()> { + *self = value.parse()?; + Ok(()) + } +} + +impl fmt::Display for MapKeyDedupPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Exception => write!(f, "EXCEPTION"), + Self::LastWin => write!(f, "LAST_WIN"), + } + } +} + +extensions_options! { + pub struct SparkOptions { + pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception + } +} + +impl ConfigExtension for SparkOptions { + const PREFIX: &'static str = "spark"; +} diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 407f3fdd5f6c7..47d4f663c70ca 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -350,7 +350,6 @@ datafusion.runtime.max_temp_directory_size 100G datafusion.runtime.memory_limit unlimited datafusion.runtime.metadata_cache_limit 50M datafusion.runtime.temp_directory NULL -datafusion.spark.map_key_dedup_policy EXCEPTION datafusion.sql_parser.collect_spans false datafusion.sql_parser.default_null_ordering nulls_max datafusion.sql_parser.dialect generic @@ -520,7 +519,6 @@ datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.temp_directory NULL The path to the temporary file directory. -datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. diff --git a/docs/source/library-user-guide/upgrading/55.0.0.md b/docs/source/library-user-guide/upgrading/55.0.0.md index 2e5c9a782de13..6e236b08284f5 100644 --- a/docs/source/library-user-guide/upgrading/55.0.0.md +++ b/docs/source/library-user-guide/upgrading/55.0.0.md @@ -788,8 +788,7 @@ The Spark-compatibility map-construction functions (`map_from_arrays`, when constructing a map that contains duplicate keys. This matches the default of Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4502-L4511). -A new config option, `datafusion.spark.map_key_dedup_policy`, controls the -behavior: +A config option controls the behavior: - `EXCEPTION` (default): raise on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. The key stays at @@ -807,7 +806,7 @@ behavior: To restore lenient duplicate-key handling, set the policy to `LAST_WIN`: ```sql -SET datafusion.spark.map_key_dedup_policy = 'LAST_WIN'; +SET spark.map_key_dedup_policy = 'LAST_WIN'; ``` See [PR #21720](https://github.com/apache/datafusion/pull/21720) for details. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index c7a2fabe131a2..7bf2ddf1fdcc1 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -208,7 +208,6 @@ The following configuration settings are available: | datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | | datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | | datafusion.format.types_info | false | Show types in visual representation batches | -| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | You can also reset configuration options to default settings via SQL using the `RESET` command. For example, to set and reset `datafusion.execution.batch_size`: