Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 0 additions & 88 deletions datafusion/common/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -830,53 +830,6 @@ impl Display for ConfigFilterSelectivity {
}
}

/// Policy for handling duplicate keys in Spark-compatible map-construction
/// functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors
/// Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961).
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum MapKeyDedupPolicy {
/// Raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key.
#[default]
Exception,
/// Keep the last occurrence of each duplicate key.
LastWin,
}

impl FromStr for MapKeyDedupPolicy {
type Err = DataFusionError;

fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_ascii_uppercase().as_str() {
"EXCEPTION" => Ok(Self::Exception),
"LAST_WIN" => Ok(Self::LastWin),
other => Err(DataFusionError::Configuration(format!(
"Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN"
))),
}
}
}

impl ConfigField for MapKeyDedupPolicy {
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
v.some(key, self, description)
}

fn set(&mut self, _: &str, value: &str) -> Result<()> {
*self = MapKeyDedupPolicy::from_str(value)?;
Ok(())
}
}

impl Display for MapKeyDedupPolicy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let str = match self {
Self::Exception => "EXCEPTION",
Self::LastWin => "LAST_WIN",
};
write!(f, "{str}")
}
}

impl From<SpillCompression> for Option<CompressionType> {
fn from(c: SpillCompression) -> Self {
match c {
Expand Down Expand Up @@ -2245,24 +2198,6 @@ impl<'a> TryFrom<&'a FormatOptions> for arrow::util::display::FormatOptions<'a>
}
}

config_namespace! {
/// Options controlling DataFusion's Spark-compatibility layer (the
/// `datafusion-spark` crate). Keys here mirror their `spark.sql.*`
/// equivalents in Apache Spark.
pub struct SparkOptions {
/// Policy for handling duplicate keys in Spark-compatible map-construction
/// functions (`map_from_arrays`, `map_from_entries`, `str_to_map`).
///
/// Mirrors Spark's
/// [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961):
/// - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key.
/// - `LAST_WIN`: keep the last occurrence of each duplicate key.
///
/// Values are case-insensitive.
pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception
}
}

/// A key value pair, with a corresponding description
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct ConfigEntry {
Expand Down Expand Up @@ -2294,8 +2229,6 @@ pub struct ConfigOptions {
pub extensions: Extensions,
/// Formatting options when printing batches
pub format: FormatOptions,
/// Spark-compatibility options (the `datafusion-spark` crate)
pub spark: SparkOptions,
}

impl ConfigField for ConfigOptions {
Expand All @@ -2306,7 +2239,6 @@ impl ConfigField for ConfigOptions {
self.explain.visit(v, "datafusion.explain", "");
self.sql_parser.visit(v, "datafusion.sql_parser", "");
self.format.visit(v, "datafusion.format", "");
self.spark.visit(v, "datafusion.spark", "");
}

fn set(&mut self, key: &str, value: &str) -> Result<()> {
Expand All @@ -2319,7 +2251,6 @@ impl ConfigField for ConfigOptions {
"explain" => self.explain.set(rem, value),
"sql_parser" => self.sql_parser.set(rem, value),
"format" => self.format.set(rem, value),
"spark" => self.spark.set(rem, value),
_ => _config_err!("Config value \"{key}\" not found on ConfigOptions"),
}
}
Expand Down Expand Up @@ -2359,7 +2290,6 @@ impl ConfigField for ConfigOptions {
"explain" => self.explain.reset(rem),
"sql_parser" => self.sql_parser.reset(rem),
"format" => self.format.reset(rem),
"spark" => self.spark.reset(rem),
other => _config_err!("Config value \"{other}\" not found on ConfigOptions"),
}
}
Expand Down Expand Up @@ -4131,17 +4061,6 @@ config_namespace! {
/// of columns, it creates a union schema containing all columns found across the files, and will
/// pad any files missing columns with null values for their rows.
pub truncated_rows: Option<bool>, default = None
/// Whether to map file columns to the table schema by header name instead of by position.
///
/// Only applies when `has_header` is true. When enabled, each file's header row is read
/// and columns are matched by name: table columns missing from a file are filled with
/// nulls, and file columns not declared in the table are ignored. Names are matched exactly
/// (case-sensitive) and duplicate header names resolve to the first occurrence. Header
/// names containing quoted newlines are not supported. The header is located the same
/// way the CSV parser does: a leading UTF-8 BOM, blank lines, and comment lines are skipped.
///
/// Defaults to `false` when `None`.
pub match_columns_by_header: Option<bool>, default = None
}
}

Expand Down Expand Up @@ -4267,13 +4186,6 @@ impl CsvOptions {
self
}

/// Whether to map file columns to the table schema by header name instead of by position.
/// Only applies when the file has a header.
pub fn with_match_columns_by_header(mut self, match_by_header: bool) -> Self {
self.match_columns_by_header = Some(match_by_header);
self
}

/// Set the compression level for the output file.
/// The valid range depends on the compression algorithm.
/// If not specified, the default level for the algorithm is used.
Expand Down
61 changes: 61 additions & 0 deletions datafusion/spark/src/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use std::fmt;
use std::str::FromStr;

use datafusion_common::config::{ConfigExtension, ConfigField, Visit};
use datafusion_common::extensions_options;
use datafusion_common::{DataFusionError, Result};

#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum MapKeyDedupPolicy {
#[default]
Exception,
LastWin,
}

impl FromStr for MapKeyDedupPolicy {
type Err = DataFusionError;

fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.to_ascii_uppercase().as_str() {
"EXCEPTION" => Ok(Self::Exception),
"LAST_WIN" => Ok(Self::LastWin),
other => Err(DataFusionError::Configuration(format!(
"Invalid MapKeyDedupPolicy: {other}. Expected one of: EXCEPTION, LAST_WIN"
))),
}
}
}

impl ConfigField for MapKeyDedupPolicy {
fn visit<V: Visit>(&self, v: &mut V, key: &str, description: &'static str) {
v.some(key, self, description)
}

fn set(&mut self, _: &str, value: &str) -> Result<()> {
*self = value.parse()?;
Ok(())
}
}

impl fmt::Display for MapKeyDedupPolicy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Exception => write!(f, "EXCEPTION"),
Self::LastWin => write!(f, "LAST_WIN"),
}
}
}

extensions_options! {
pub struct SparkOptions {
pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception
}
}

impl ConfigExtension for SparkOptions {
const PREFIX: &'static str = "spark";
}
2 changes: 0 additions & 2 deletions datafusion/sqllogictest/test_files/information_schema.slt
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,6 @@ datafusion.runtime.max_temp_directory_size 100G
datafusion.runtime.memory_limit unlimited
datafusion.runtime.metadata_cache_limit 50M
datafusion.runtime.temp_directory NULL
datafusion.spark.map_key_dedup_policy EXCEPTION
datafusion.sql_parser.collect_spans false
datafusion.sql_parser.default_null_ordering nulls_max
datafusion.sql_parser.dialect generic
Expand Down Expand Up @@ -520,7 +519,6 @@ datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory
datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes.
datafusion.runtime.temp_directory NULL The path to the temporary file directory.
datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive.
datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes.
datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: <https://www.postgresql.org/docs/current/queries-order.html>
datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark.
Expand Down
5 changes: 2 additions & 3 deletions docs/source/library-user-guide/upgrading/55.0.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -788,8 +788,7 @@ The Spark-compatibility map-construction functions (`map_from_arrays`,
when constructing a map that contains duplicate keys. This matches the default
of Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/v4.0.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4502-L4511).

A new config option, `datafusion.spark.map_key_dedup_policy`, controls the
behavior:
A config option controls the behavior:

- `EXCEPTION` (default): raise on any duplicate key.
- `LAST_WIN`: keep the last occurrence of each duplicate key. The key stays at
Expand All @@ -807,7 +806,7 @@ behavior:
To restore lenient duplicate-key handling, set the policy to `LAST_WIN`:

```sql
SET datafusion.spark.map_key_dedup_policy = 'LAST_WIN';
SET spark.map_key_dedup_policy = 'LAST_WIN';
```

See [PR #21720](https://github.com/apache/datafusion/pull/21720) for details.
Expand Down
1 change: 0 additions & 1 deletion docs/source/user-guide/configs.md
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,6 @@ The following configuration settings are available:
| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays |
| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` |
| datafusion.format.types_info | false | Show types in visual representation batches |
| datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. |

You can also reset configuration options to default settings via SQL using the `RESET` command. For
example, to set and reset `datafusion.execution.batch_size`:
Expand Down