diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 63a7797e396ef..f46b8023a5d01 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -2243,6 +2243,19 @@ config_namespace! { /// under `datafusion/spark`). Keys here mirror their `spark.sql.*` /// equivalents in Apache Spark. pub struct SparkOptions { + /// When true, restores legacy (pre-Spark 4.0) unmappable character handling in `encode`, + /// replacing unmappable characters with `?` rather than raising `[MALFORMED_CHARACTER_CODING]`. + /// + /// Mirrors Spark's `spark.sql.legacy.codingErrorAction`. + pub legacy_coding_error_action: bool, default = false + + /// When true, restores legacy (pre-Spark 4.0) charset handling in `encode`, + /// accepting non-canonical Java/alias charset names rather than only the seven + /// canonical Spark charsets (US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32). + /// + /// Mirrors Spark's `spark.sql.legacy.javaCharsets`. + pub legacy_java_charsets: bool, default = false + /// Policy for handling duplicate keys in Spark-compatible map-construction /// functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). /// diff --git a/datafusion/spark/src/function/string/encode.rs b/datafusion/spark/src/function/string/encode.rs index 5276ada9b658c..2d675f0531610 100644 --- a/datafusion/spark/src/function/string/encode.rs +++ b/datafusion/spark/src/function/string/encode.rs @@ -21,27 +21,22 @@ use datafusion_expr::{ /// Binary input is interpreted as UTF-8 with lossy conversion (invalid bytes become U+FFFD). /// /// # Target Spark version -/// Emulates Spark 3.5 semantics: -/// - Accepts canonical charset names and common aliases (`UTF8`, `LATIN1`, -/// `ISO88591`, `ASCII`, `UTF-32BE`, etc.). -/// - Unmappable characters (non-ASCII in `US-ASCII`, code points above -/// `U+00FF` in `ISO-8859-1`) are silently replaced with `?`. -/// - `UTF-32` is an alias for `UTF-32BE` (no BOM), matching both Spark 3.5 -/// and Spark 4.1. -/// -/// # Spark 4.0 differences (not implemented) -/// Spark 4.0 tightened `encode` in two ways, each gated by a `spark.sql.legacy.*` -/// config that can restore the 3.5 behavior: -/// -/// - Charset whitelist — rejects aliases with `INVALID_PARAMETER_VALUE.CHARSET`. -/// Controlled by `spark.sql.legacy.javaCharsets`. -/// - Unmappable characters — raises `MALFORMED_CHARACTER_CODING`. -/// Controlled by `spark.sql.legacy.codingErrorAction`. +/// By default emulates Spark 4.0+ semantics: +/// - Accepts only the 7 canonical Spark charsets (`US-ASCII`, `ISO-8859-1`, `UTF-8`, +/// `UTF-16`, `UTF-16BE`, `UTF-16LE`, `UTF-32`). Non-canonical aliases (e.g., `UTF8`, +/// `LATIN1`, `ASCII`, `UTF-32BE`, `UTF-32LE`) raise `[INVALID_PARAMETER_VALUE.CHARSET]`. +/// Controlled by `datafusion.spark.legacy_java_charsets`. +/// - Unmappable characters (non-ASCII in `US-ASCII`, code points above `U+00FF` in +/// `ISO-8859-1`) raise `[MALFORMED_CHARACTER_CODING] ... SQLSTATE: 22000`. +/// Controlled by `datafusion.spark.legacy_coding_error_action`. +/// - `UTF-32` is encoded as big-endian without BOM, matching Spark 3.5 and 4.x. /// +/// Setting `datafusion.spark.legacy_java_charsets = true` restores Spark 3.5 charset +/// alias resolution, and setting `datafusion.spark.legacy_coding_error_action = true` +/// restores Spark 3.5 silent replacement of unmappable characters with `?`. /// /// -// TODO: wire both configs so Spark 4.0 behavior can be selected at runtime. -// See: +/// #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkEncode { signature: Signature, @@ -61,20 +56,95 @@ impl SparkEncode { } } +/// The seven canonical Spark 4.0 charset names (case-insensitive). +const CANONICAL_CHARSETS: [&str; 7] = [ + "US-ASCII", + "ISO-8859-1", + "UTF-8", + "UTF-16", + "UTF-16BE", + "UTF-16LE", + "UTF-32", +]; + +fn is_canonical_charset(upper: &str) -> bool { + CANONICAL_CHARSETS.contains(&upper) +} + +fn validate_charset(raw: &str, upper: &str, legacy_java_charsets: bool) -> Result<()> { + if !legacy_java_charsets { + if !is_canonical_charset(upper) { + return exec_err!( + "[INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) 'charset' in 'encode' is invalid: expects one of the [US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32], but got '{raw}'." + ); + } + } else { + match upper { + "UTF-8" | "UTF8" | "US-ASCII" | "ASCII" | "ISO-8859-1" | "ISO88591" + | "LATIN1" | "UTF-16BE" | "UTF16BE" | "UTF-16LE" | "UTF16LE" | "UTF-16" + | "UTF16" | "UTF-32" | "UTF32" | "UTF-32BE" | "UTF32BE" | "UTF-32LE" + | "UTF32LE" => {} + _ => { + return exec_err!( + "Unsupported charset for encode: '{raw}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE" + ); + } + } + } + Ok(()) +} + /// Encodes a single string value using the specified charset into `bytes`, /// replacing its previous contents. -/// Unmappable characters are silently replaced with `?` (Spark 3.5 behavior). -fn encode_string(s: &str, charset: &str, bytes: &mut Vec) -> Result<()> { +fn encode_string( + s: &str, + raw_charset: &str, + upper_charset: &str, + legacy_coding_error_action: bool, + bytes: &mut Vec, +) -> Result<()> { bytes.clear(); - match charset { + match upper_charset { "UTF-8" | "UTF8" => bytes.extend_from_slice(s.as_bytes()), "US-ASCII" | "ASCII" => { - bytes.extend(s.chars().map(|c| if c.is_ascii() { c as u8 } else { b'?' })) + if legacy_coding_error_action { + bytes + .extend(s.chars().map(|c| if c.is_ascii() { c as u8 } else { b'?' })); + } else { + let _ = bytes.try_reserve(s.len()); + for c in s.chars() { + if c.is_ascii() { + bytes.push(c as u8); + } else { + return exec_err!( + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", + c as u32 + ); + } + } + } + } + "ISO-8859-1" | "ISO88591" | "LATIN1" => { + if legacy_coding_error_action { + bytes.extend(s.chars().map(|c| { + let cp = c as u32; + if cp <= 255 { cp as u8 } else { b'?' } + })); + } else { + let _ = bytes.try_reserve(s.len()); + for c in s.chars() { + let cp = c as u32; + if cp <= 255 { + bytes.push(cp as u8); + } else { + return exec_err!( + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", + cp + ); + } + } + } } - "ISO-8859-1" | "ISO88591" | "LATIN1" => bytes.extend(s.chars().map(|c| { - let cp = c as u32; - if cp <= 255 { cp as u8 } else { b'?' } - })), "UTF-16BE" | "UTF16BE" => { for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_be_bytes()); @@ -103,29 +173,44 @@ fn encode_string(s: &str, charset: &str, bytes: &mut Vec) -> Result<()> { bytes.extend_from_slice(&(c as u32).to_le_bytes()); } } - _ => { - return exec_err!( - "Unsupported charset for encode: '{}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE", - charset - ); - } + _ => unreachable!("charset validation must precede encode_string"), } Ok(()) } -/// The charset for each row. `Constant` is a single uppercased charset; `PerRow` -/// uppercases each row on demand, yielding `None` for null entries. +/// The charset for each row. `Constant` holds a single raw and uppercased charset; +/// `PerRow` retrieves and uppercases each row on demand, yielding `None` for null entries. enum Charsets<'a> { - Constant(&'a str), + Constant { raw: &'a str, upper: String }, PerRow(&'a StringArray), } -impl Charsets<'_> { - fn get(&self, i: usize) -> Option> { +impl<'a> Charsets<'a> { + fn constant(raw: &'a str) -> Self { + let upper = raw.to_uppercase(); + Self::Constant { raw, upper } + } + + fn per_row(array: &'a StringArray) -> Self { + Self::PerRow(array) + } + + fn is_constant(&self) -> bool { + matches!(self, Self::Constant { .. }) + } + + fn get(&self, i: usize) -> Option<(Cow<'_, str>, Cow<'_, str>)> { match self { - Charsets::Constant(charset) => Some(Cow::Borrowed(charset)), + Charsets::Constant { raw, upper } => { + Some((Cow::Borrowed(*raw), Cow::Borrowed(upper.as_str()))) + } Charsets::PerRow(array) => { - (!array.is_null(i)).then(|| Cow::Owned(array.value(i).to_uppercase())) + if array.is_null(i) { + None + } else { + let raw = array.value(i); + Some((Cow::Borrowed(raw), Cow::Owned(raw.to_uppercase()))) + } } } } @@ -140,16 +225,32 @@ fn encode_rows<'a>( is_null: impl Fn(usize) -> bool, decode: impl Fn(usize) -> Cow<'a, str>, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { let mut builder = BinaryBuilder::with_capacity(len, len * 4); let mut bytes = Vec::new(); + let is_constant = charsets.is_constant(); for i in 0..len { - match (is_null(i), charsets.get(i)) { - (false, Some(charset)) => { - encode_string(&decode(i), &charset, &mut bytes)?; - builder.append_value(&bytes); + match charsets.get(i) { + Some((raw, upper)) => { + if !is_constant { + validate_charset(&raw, &upper, legacy_java_charsets)?; + } + if is_null(i) { + builder.append_null(); + } else { + encode_string( + &decode(i), + &raw, + &upper, + legacy_coding_error_action, + &mut bytes, + )?; + builder.append_value(&bytes); + } } - _ => builder.append_null(), + None => builder.append_null(), } } Ok(Arc::new(builder.finish())) @@ -158,45 +259,88 @@ fn encode_rows<'a>( fn encode_array<'a, S: StringArrayType<'a>>( array: &S, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| Cow::Borrowed(array.value(i)), charsets, + legacy_java_charsets, + legacy_coding_error_action, ) } fn encode_binary_array<'a, B: arrow::array::BinaryArrayType<'a>>( array: &'a B, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| String::from_utf8_lossy(array.value(i)), charsets, + legacy_java_charsets, + legacy_coding_error_action, ) } -fn encode_dispatch(arr: &ArrayRef, charsets: &Charsets) -> Result { +fn encode_dispatch( + arr: &ArrayRef, + charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, +) -> Result { match arr.data_type() { - DataType::Utf8 => encode_array(&arr.as_string::(), charsets), - DataType::LargeUtf8 => encode_array(&arr.as_string::(), charsets), - DataType::Utf8View => encode_array(&arr.as_string_view(), charsets), - DataType::Binary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::LargeBinary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::BinaryView => encode_binary_array(&arr.as_binary_view(), charsets), + DataType::Utf8 => encode_array( + &arr.as_string::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::LargeUtf8 => encode_array( + &arr.as_string::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::Utf8View => encode_array( + &arr.as_string_view(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::Binary => encode_binary_array( + &arr.as_binary::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::LargeBinary => encode_binary_array( + &arr.as_binary::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::BinaryView => encode_binary_array( + &arr.as_binary_view(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), DataType::Null => Ok(Arc::new(BinaryArray::new_null(arr.len()))), dt => exec_err!("encode expects a string or binary argument, got {dt:?}"), } } -/// The uppercased charset from a scalar, or `None` if it is null. -fn scalar_charset(scalar: &ScalarValue) -> Result> { +/// The raw charset string from a scalar, or `None` if it is null. +fn scalar_charset(scalar: &ScalarValue) -> Result> { match scalar { ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s) => { - Ok(s.as_ref().map(|s| s.to_uppercase())) + Ok(s.as_deref()) } ScalarValue::Null => Ok(None), other => exec_err!("encode charset argument must be a string, got {other:?}"), @@ -265,6 +409,10 @@ impl ScalarUDFImpl for SparkEncode { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [value, charset_arg] = take_function_args(self.name(), args.args)?; + let legacy_java_charsets = args.config_options.spark.legacy_java_charsets; + let legacy_coding_error_action = + args.config_options.spark.legacy_coding_error_action; + let len = [&value, &charset_arg] .into_iter() .find_map(|arg| match arg { @@ -283,7 +431,16 @@ impl ScalarUDFImpl for SparkEncode { let result = match &charset_arg { ColumnarValue::Scalar(scalar) => match scalar_charset(scalar)? { Some(charset) => { - encode_dispatch(&value_arr, &Charsets::Constant(&charset))? + let charsets = Charsets::constant(charset); + if let Some((raw, upper)) = charsets.get(0) { + validate_charset(&raw, &upper, legacy_java_charsets)?; + } + encode_dispatch( + &value_arr, + &charsets, + legacy_java_charsets, + legacy_coding_error_action, + )? } None => { return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))); @@ -291,7 +448,9 @@ impl ScalarUDFImpl for SparkEncode { }, ColumnarValue::Array(charset_array) => encode_dispatch( &value_arr, - &Charsets::PerRow(charset_array.as_string::()), + &Charsets::per_row(charset_array.as_string::()), + legacy_java_charsets, + legacy_coding_error_action, )?, }; @@ -306,6 +465,7 @@ impl ScalarUDFImpl for SparkEncode { #[cfg(test)] mod tests { use super::*; + use datafusion_common::config::ConfigOptions; #[test] fn test_encode_return_field_nullable() { @@ -334,4 +494,312 @@ mod tests { .unwrap(); assert!(!non_nullable.is_nullable()); } + + fn invoke_encode( + input: &str, + charset: &str, + legacy_charsets: bool, + legacy_coding: bool, + ) -> Result> { + let func = SparkEncode::new(); + let mut config = ConfigOptions::default(); + config.spark.legacy_java_charsets = legacy_charsets; + config.spark.legacy_coding_error_action = legacy_coding; + + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some(input.to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(charset.to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::new(config), + }; + + match func.invoke_with_args(args)? { + ColumnarValue::Scalar(ScalarValue::Binary(Some(bytes))) => Ok(bytes), + other => panic!("expected scalar binary, got {other:?}"), + } + } + + #[test] + fn test_canonical_charsets_strict_mode() { + // UTF-8 + let res = invoke_encode("Spark SQL", "utf-8", false, false).unwrap(); + assert_eq!(res, b"Spark SQL"); + + // US-ASCII + let res = invoke_encode("Hello", "US-ASCII", false, false).unwrap(); + assert_eq!(res, b"Hello"); + + // ISO-8859-1 (naïve: ï = 0xEF) + let res = invoke_encode("naïve", "iso-8859-1", false, false).unwrap(); + assert_eq!(res, &[0x6E, 0x61, 0xEF, 0x76, 0x65]); + + // UTF-16BE + let res = invoke_encode("AB", "utf-16be", false, false).unwrap(); + assert_eq!(res, &[0x00, 0x41, 0x00, 0x42]); + + // UTF-16LE + let res = invoke_encode("AB", "utf-16le", false, false).unwrap(); + assert_eq!(res, &[0x41, 0x00, 0x42, 0x00]); + + // UTF-16 (with BOM) + let res = invoke_encode("AB", "utf-16", false, false).unwrap(); + assert_eq!(res, &[0xFE, 0xFF, 0x00, 0x41, 0x00, 0x42]); + + // UTF-32 (big-endian without BOM, matching Spark 4.0) + let res = invoke_encode("A", "utf-32", false, false).unwrap(); + assert_eq!(res, &[0x00, 0x00, 0x00, 0x41]); + + // UTF-32 with emoji (non-BMP: 😀 = U+1F600) + let res = invoke_encode("😀", "UTF-32", false, false).unwrap(); + assert_eq!(res, &[0x00, 0x01, 0xF6, 0x00]); + } + + #[test] + fn test_canonical_case_insensitivity() { + let res1 = invoke_encode("hello", "Utf-8", false, false).unwrap(); + let res2 = invoke_encode("hello", "uS-AsCiI", false, false).unwrap(); + let res3 = invoke_encode("hello", "Iso-8859-1", false, false).unwrap(); + let res4 = invoke_encode("hello", "uTf-16bE", false, false).unwrap(); + let res5 = invoke_encode("hello", "UtF-16lE", false, false).unwrap(); + let res6 = invoke_encode("hello", "UtF-16", false, false).unwrap(); + let res7 = invoke_encode("hello", "UtF-32", false, false).unwrap(); + assert_eq!(res1, b"hello"); + assert_eq!(res2, b"hello"); + assert_eq!(res3, b"hello"); + assert_eq!(res4.len(), 10); + assert_eq!(res5.len(), 10); + assert_eq!(res6.len(), 12); + assert_eq!(res7.len(), 20); + } + + #[test] + fn test_alias_rejection_strict_mode() { + // Non-canonical aliases rejected under Spark 4.0 strict mode + let aliases = [ + "utf8", "ascii", "latin1", "iso88591", "utf16", "utf16be", "utf16le", + "utf32", "utf-32be", "utf-32le", "ebcdic", + ]; + for alias in aliases { + let err = invoke_encode("A", alias, false, false).unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("[INVALID_PARAMETER_VALUE.CHARSET]"), + "expected INVALID_PARAMETER_VALUE.CHARSET for alias '{alias}', got: {msg}" + ); + assert!( + msg.contains(&format!("got '{alias}'")), + "expected error to mention raw alias '{alias}', got: {msg}" + ); + } + } + + #[test] + fn test_alias_acceptance_legacy_mode() { + // When legacy_java_charsets = true, common aliases are accepted + let res_utf8 = invoke_encode("Spark", "utf8", true, false).unwrap(); + assert_eq!(res_utf8, b"Spark"); + + let res_ascii = invoke_encode("Spark", "ascii", true, false).unwrap(); + assert_eq!(res_ascii, b"Spark"); + + let res_latin1 = invoke_encode("naïve", "latin1", true, false).unwrap(); + assert_eq!(res_latin1, &[0x6E, 0x61, 0xEF, 0x76, 0x65]); + + let res_utf32be = invoke_encode("A", "utf-32be", true, false).unwrap(); + assert_eq!(res_utf32be, &[0x00, 0x00, 0x00, 0x41]); + + let res_utf32le = invoke_encode("AB", "utf-32le", true, false).unwrap(); + assert_eq!( + res_utf32le, + &[0x41, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00] + ); + + // Truly unsupported charset still errors with "Unsupported charset for encode" + let err = invoke_encode("A", "EBCDIC", true, false).unwrap_err(); + assert!( + err.to_string() + .contains("Unsupported charset for encode: 'EBCDIC'") + ); + } + + #[test] + fn test_unmappable_characters_strict_mode() { + // US-ASCII with 'é' (non-ASCII) + let err_ascii = invoke_encode("é", "us-ascii", false, false).unwrap_err(); + let msg_ascii = err_ascii.to_string(); + assert!( + msg_ascii.contains("[MALFORMED_CHARACTER_CODING]"), + "expected MALFORMED_CHARACTER_CODING for US-ASCII, got: {msg_ascii}" + ); + assert!( + msg_ascii.contains("with us-ascii"), + "expected charset context in error, got: {msg_ascii}" + ); + assert!( + msg_ascii.contains("SQLSTATE: 22000"), + "expected SQLSTATE: 22000 in error, got: {msg_ascii}" + ); + + // ISO-8859-1 with 'Ā' (U+0100 > 0xFF) + let err_iso = invoke_encode("Ā", "iso-8859-1", false, false).unwrap_err(); + let msg_iso = err_iso.to_string(); + assert!( + msg_iso.contains("[MALFORMED_CHARACTER_CODING]"), + "expected MALFORMED_CHARACTER_CODING for ISO-8859-1, got: {msg_iso}" + ); + assert!( + msg_iso.contains("with iso-8859-1"), + "expected charset context in error, got: {msg_iso}" + ); + assert!( + msg_iso.contains("SQLSTATE: 22000"), + "expected SQLSTATE: 22000 in error, got: {msg_iso}" + ); + } + + #[test] + fn test_unmappable_characters_legacy_mode() { + // US-ASCII with 'é' replaced with '?' (b'?') + let res_ascii = invoke_encode("é", "us-ascii", false, true).unwrap(); + assert_eq!(res_ascii, &[b'?']); + + // ISO-8859-1 with 'Ā' replaced with '?' (b'?') + let res_iso = invoke_encode("Ā", "iso-8859-1", false, true).unwrap(); + assert_eq!(res_iso, &[b'?']); + + // Mixed valid and invalid + let res_mixed = invoke_encode("AéB", "us-ascii", false, true).unwrap(); + assert_eq!(res_mixed, &[b'A', b'?', b'B']); + } + + #[test] + fn test_independent_flags() { + // 1. legacy_java_charsets = true, legacy_coding_error_action = false: + // Alias 'ascii' is accepted, but 'é' triggers MALFORMED_CHARACTER_CODING + let err = invoke_encode("é", "ascii", true, false).unwrap_err(); + assert!(err.to_string().contains("[MALFORMED_CHARACTER_CODING]")); + + // 2. legacy_java_charsets = false, legacy_coding_error_action = true: + // 'us-ascii' accepted, and 'é' replaced with '?' + let res = invoke_encode("é", "us-ascii", false, true).unwrap(); + assert_eq!(res, &[b'?']); + + // But non-canonical 'ascii' is still rejected + let err2 = invoke_encode("é", "ascii", false, true).unwrap_err(); + assert!( + err2.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + } + + #[test] + fn test_empty_and_null_inputs() { + let func = SparkEncode::new(); + let config = Arc::new(ConfigOptions::default()); + + // Empty string + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::new()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("utf-8".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(Some(b))) => { + assert_eq!(b, Vec::::new()) + } + other => panic!("expected empty binary, got {other:?}"), + } + + // Null value yields null + let args_null_val = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("utf-8".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_null_val).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(None)) => {} + other => panic!("expected null binary, got {other:?}"), + } + + // Null charset yields null (NullIntolerant) + let args_null_charset = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some("hello".to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_null_charset).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(None)) => {} + other => panic!("expected null binary, got {other:?}"), + } + } + + #[test] + fn test_per_row_charsets_array() { + let func = SparkEncode::new(); + let config = Arc::new(ConfigOptions::default()); + + let values = Arc::new(StringArray::from(vec![Some("hello"), Some("world")])); + let charsets = Arc::new(StringArray::from(vec![Some("utf-8"), Some("utf-16be")])); + + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ColumnarValue::Array(values), ColumnarValue::Array(charsets)], + number_rows: 2, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + + let res = func.invoke_with_args(args).unwrap(); + match res { + ColumnarValue::Array(arr) => { + let bin = arr.as_binary::(); + assert_eq!(bin.value(0), b"hello"); + assert_eq!( + bin.value(1), + &[0x00, 0x77, 0x00, 0x6F, 0x00, 0x72, 0x00, 0x6C, 0x00, 0x64] + ); + } + other => panic!("expected array, got {other:?}"), + } + + // Array with invalid alias in strict mode should fail + let bad_charsets = Arc::new(StringArray::from(vec![Some("utf-8"), Some("utf8")])); + let bad_args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Array(Arc::new(StringArray::from(vec![ + Some("hello"), + Some("world"), + ]))), + ColumnarValue::Array(bad_charsets), + ], + number_rows: 2, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + let err = func.invoke_with_args(bad_args).unwrap_err(); + assert!( + err.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + } } diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index c1aabc2946ea6..8332c55cd15fa 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -349,6 +349,8 @@ datafusion.runtime.max_temp_directory_size 100G datafusion.runtime.memory_limit unlimited datafusion.runtime.metadata_cache_limit 50M datafusion.runtime.temp_directory NULL +datafusion.spark.legacy_coding_error_action false +datafusion.spark.legacy_java_charsets false datafusion.spark.map_key_dedup_policy EXCEPTION datafusion.sql_parser.collect_spans false datafusion.sql_parser.default_null_ordering nulls_max @@ -518,6 +520,8 @@ datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.temp_directory NULL The path to the temporary file directory. +datafusion.spark.legacy_coding_error_action false When true, restores legacy (pre-Spark 4.0) unmappable character handling in `encode`, replacing unmappable characters with `?` rather than raising `[MALFORMED_CHARACTER_CODING]`. Mirrors Spark's `spark.sql.legacy.codingErrorAction`. +datafusion.spark.legacy_java_charsets false When true, restores legacy (pre-Spark 4.0) charset handling in `encode`, accepting non-canonical Java/alias charset names rather than only the seven canonical Spark charsets (US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32). Mirrors Spark's `spark.sql.legacy.javaCharsets`. datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: diff --git a/datafusion/sqllogictest/test_files/spark/string/encode.slt b/datafusion/sqllogictest/test_files/spark/string/encode.slt index 93df7e80ec7a3..3d7c462922d54 100644 --- a/datafusion/sqllogictest/test_files/spark/string/encode.slt +++ b/datafusion/sqllogictest/test_files/spark/string/encode.slt @@ -38,17 +38,13 @@ SELECT encode('Spark SQL'::string, 'utf-16'::string); ---- feff0053007000610072006b002000530051004c -# UTF-32BE encoding -query ? +# UTF-32BE encoding (alias rejected in Spark 4 strict mode) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('A'::string, 'utf-32be'::string); ----- -00000041 -# UTF-32LE encoding -query ? +# UTF-32LE encoding (alias rejected in Spark 4 strict mode) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('AB'::string, 'utf-32le'::string); ----- -4100000042000000 # UTF-32 (Spark 3.5 / 4.1: no BOM, identical to UTF-32BE) query ? @@ -62,17 +58,13 @@ SELECT encode('hello'::string, 'Utf-8'::string); ---- 68656c6c6f -# US-ASCII: unmappable characters are silently replaced with '?' (Spark 3.5 behavior) -query ? +# US-ASCII: unmappable characters fail in Spark 4 strict mode +statement error MALFORMED_CHARACTER_CODING SELECT encode('é'::string, 'us-ascii'::string); ----- -3f -# ISO-8859-1: code points above U+00FF are replaced with '?' (Ā is U+0100) -query ? +# ISO-8859-1: code points above U+00FF fail in Spark 4 strict mode (Ā is U+0100) +statement error MALFORMED_CHARACTER_CODING SELECT encode('Ā'::string, 'iso-8859-1'::string); ----- -3f # Multi-byte input (emoji 😀 = U+1F600) across charsets query ? @@ -95,15 +87,11 @@ SELECT encode('😀'::string, 'utf-16'::string); ---- feffd83dde00 -query ? +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32be'::string); ----- -0001f600 -query ? +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32le'::string); ----- -00f60100 query ? SELECT encode('😀'::string, 'utf-32'::string); @@ -174,10 +162,76 @@ SELECT encode(a, b) FROM VALUES (X'0a', 'US-ASCII'), (X'deadbeef', 'UTF-16be') A 0a 07adfffdfffd -# Error: unsupported charset -statement error Unsupported charset for encode +# Error: unsupported charset (strict Spark 4 rejects unknown charsets with INVALID_PARAMETER_VALUE.CHARSET) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('hello'::string, 'EBCDIC'::string); # Error: no arguments statement error encode function requires 2 arguments SELECT encode(); + +# ----------------------------------------------------------------------------- +# Legacy compatibility: legacy_java_charsets and legacy_coding_error_action +# ----------------------------------------------------------------------------- + +statement ok +SET datafusion.spark.legacy_java_charsets = true; + +statement ok +SET datafusion.spark.legacy_coding_error_action = true; + +# Legacy mode: UTF-32BE alias accepted +query ? +SELECT encode('A'::string, 'utf-32be'::string); +---- +00000041 + +# Legacy mode: UTF-32LE alias accepted +query ? +SELECT encode('AB'::string, 'utf-32le'::string); +---- +4100000042000000 + +# Legacy mode: ASCII and UTF8 aliases accepted +query ? +SELECT encode('Hello'::string, 'ascii'::string); +---- +48656c6c6f + +query ? +SELECT encode('Spark'::string, 'utf8'::string); +---- +537061726b + +# Legacy mode: unmappable characters replaced with '?' (0x3F) +query ? +SELECT encode('é'::string, 'us-ascii'::string); +---- +3f + +query ? +SELECT encode('Ā'::string, 'iso-8859-1'::string); +---- +3f + +# Legacy mode: emoji with utf-32be and utf-32le +query ? +SELECT encode('😀'::string, 'utf-32be'::string); +---- +0001f600 + +query ? +SELECT encode('😀'::string, 'utf-32le'::string); +---- +00f60100 + +# Legacy mode: truly unsupported charset error message +statement error Unsupported charset for encode: 'EBCDIC' +SELECT encode('hello'::string, 'EBCDIC'::string); + +# Reset configuration to default Spark 4 strict settings +statement ok +SET datafusion.spark.legacy_java_charsets = false; + +statement ok +SET datafusion.spark.legacy_coding_error_action = false; diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 2c3ca887d4a38..a29be1dae3ba0 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -207,6 +207,8 @@ The following configuration settings are available: | datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | | datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | | datafusion.format.types_info | false | Show types in visual representation batches | +| datafusion.spark.legacy_coding_error_action | false | When true, restores legacy (pre-Spark 4.0) unmappable character handling in `encode`, replacing unmappable characters with `?` rather than raising `[MALFORMED_CHARACTER_CODING]`. Mirrors Spark's `spark.sql.legacy.codingErrorAction`. | +| datafusion.spark.legacy_java_charsets | false | When true, restores legacy (pre-Spark 4.0) charset handling in `encode`, accepting non-canonical Java/alias charset names rather than only the seven canonical Spark charsets (US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32). Mirrors Spark's `spark.sql.legacy.javaCharsets`. | | datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | You can also reset configuration options to default settings via SQL using the `RESET` command. For