From 5385530642241411ac2862abba3bb4584a29d426 Mon Sep 17 00:00:00 2001 From: Subodh Bansode Date: Sun, 15 Nov 2026 08:18:56 +0530 Subject: [PATCH 1/2] fix(spark): match Spark 4 encode charset behavior Match Spark 4.0 canonical charset whitelist (US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32), reject non-canonical aliases with [INVALID_PARAMETER_VALUE.CHARSET], and reject unmappable characters with [MALFORMED_CHARACTER_CODING] (SQLSTATE 22000). Provide with_legacy_java_charsets and with_legacy_coding_error_action builder methods for Spark 3.5 legacy compatibility. Closes #46. Signed-off-by: Subodh Bansode --- crates/spark/src/function/string/encode.rs | 587 ++++++++++++++++-- .../sqllogictest/test_files/string/encode.slt | 36 +- 2 files changed, 558 insertions(+), 65 deletions(-) diff --git a/crates/spark/src/function/string/encode.rs b/crates/spark/src/function/string/encode.rs index dad8ff26..b8dbb49d 100644 --- a/crates/spark/src/function/string/encode.rs +++ b/crates/spark/src/function/string/encode.rs @@ -21,30 +21,29 @@ use datafusion_macros::user_doc; /// /// // TODO: wire `spark.sql.legacy.javaCharsets` and `spark.sql.legacy.codingErrorAction` -// so Spark 4.0 behavior can be selected at runtime. +// via SparkOptions once the config extension is available in datafusion-spark. // See: #[user_doc( doc_section(label = "Binary String Functions"), description = "Encodes a string into binary using a character set. Unlike DataFusion's `encode`, which takes \ `base64` or `hex` and returns a string, this takes a character set and returns binary. \ Supported character sets are `US-ASCII`, `ISO-8859-1`, `UTF-8`, `UTF-16`, `UTF-16BE`, `UTF-16LE`, \ - `UTF-32`, `UTF-32BE` and `UTF-32LE`, matched case-insensitively, and the aliases `ASCII`, `LATIN1`, \ - `ISO88591` and the spellings without a hyphen (`UTF8`, `UTF16BE`, ...). Any other character set is an error. \ + and `UTF-32`, matched case-insensitively. Any other character set raises `[INVALID_PARAMETER_VALUE.CHARSET]`. \ `UTF-16` writes a big-endian byte order mark first. `UTF-32` is big-endian with no byte order mark. \ Characters the character set cannot represent (non-ASCII in `US-ASCII`, code points above U+00FF in \ - `ISO-8859-1`) are replaced with `?`. Binary input is decoded as UTF-8 first, with each invalid byte \ - replaced by U+FFFD. Returns NULL if either input is NULL. \ - This follows Spark 3.5. Spark 4.0 rejects the aliases and raises an error on characters it cannot \ - represent, unless `spark.sql.legacy.javaCharsets` and `spark.sql.legacy.codingErrorAction` are set; \ - those settings are not supported yet.", + `ISO-8859-1`) raise `[MALFORMED_CHARACTER_CODING]`. Binary input is decoded as UTF-8 first, with each \ + invalid byte replaced by U+FFFD. Returns NULL if either input is NULL. \ + This follows Spark 4.0. Pre-Spark 4.0 accepted Java aliases (`UTF8`, `ASCII`, `LATIN1`, `UTF-32BE`, ...) \ + and replaced unmappable characters with `?`; those legacy behaviors can be enabled programmatically \ + via `with_legacy_java_charsets(true)` and `with_legacy_coding_error_action(true)`.", syntax_example = "encode(str, charset)", sql_example = r#"```sql > SELECT encode('Spark SQL', 'utf-8'); 537061726b2053514c > SELECT encode('AB', 'utf-16le'); 41004200 -> SELECT encode('é', 'us-ascii'); -3f +> SELECT encode('Spark SQL', 'us-ascii'); +537061726b2053514c ```"#, argument( name = "str", @@ -58,6 +57,8 @@ use datafusion_macros::user_doc; #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkEncode { signature: Signature, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, } impl Default for SparkEncode { @@ -68,37 +69,137 @@ impl Default for SparkEncode { impl SparkEncode { pub fn new() -> Self { + Self::with_options(false, false) + } + + pub fn with_options( + legacy_java_charsets: bool, + legacy_coding_error_action: bool, + ) -> Self { Self { signature: Signature::user_defined(Volatility::Immutable), + legacy_java_charsets, + legacy_coding_error_action, } } + + pub fn with_legacy_java_charsets(mut self, legacy_java_charsets: bool) -> Self { + self.legacy_java_charsets = legacy_java_charsets; + self + } + + pub fn with_legacy_coding_error_action( + mut self, + legacy_coding_error_action: bool, + ) -> Self { + self.legacy_coding_error_action = legacy_coding_error_action; + self + } +} + +/// The seven canonical Spark 4.0 charset names (case-insensitive). +const CANONICAL_CHARSETS: [&str; 7] = [ + "US-ASCII", + "ISO-8859-1", + "UTF-8", + "UTF-16", + "UTF-16BE", + "UTF-16LE", + "UTF-32", +]; + +fn is_canonical_charset(upper: &str) -> bool { + CANONICAL_CHARSETS.contains(&upper) +} + +fn validate_charset(raw: &str, upper: &str, legacy_java_charsets: bool) -> Result<()> { + if !legacy_java_charsets { + if !is_canonical_charset(upper) { + return exec_err!( + "[INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) 'charset' in 'encode' is invalid: expects one of the [US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32], but got '{raw}'." + ); + } + } else { + match upper { + "UTF-8" | "UTF8" | "US-ASCII" | "ASCII" | "ISO-8859-1" | "ISO88591" + | "LATIN1" | "UTF-16BE" | "UTF16BE" | "UTF-16LE" | "UTF16LE" | "UTF-16" + | "UTF16" | "UTF-32" | "UTF32" | "UTF-32BE" | "UTF32BE" | "UTF-32LE" + | "UTF32LE" => {} + _ => { + return exec_err!( + "Unsupported charset for encode: '{raw}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE" + ); + } + } + } + Ok(()) } /// Encodes a single string value using the specified charset into `bytes`, /// replacing its previous contents. -/// Unmappable characters are silently replaced with `?` (Spark 3.5 behavior). -fn encode_string(s: &str, charset: &str, bytes: &mut Vec) -> Result<()> { +fn encode_string( + s: &str, + raw_charset: &str, + upper_charset: &str, + legacy_coding_error_action: bool, + bytes: &mut Vec, +) -> Result<()> { bytes.clear(); - match charset { + match upper_charset { "UTF-8" | "UTF8" => bytes.extend_from_slice(s.as_bytes()), "US-ASCII" | "ASCII" => { - bytes.extend(s.chars().map(|c| if c.is_ascii() { c as u8 } else { b'?' })) + if legacy_coding_error_action { + bytes + .extend(s.chars().map(|c| if c.is_ascii() { c as u8 } else { b'?' })); + } else { + let _ = bytes.try_reserve(s.len()); + for c in s.chars() { + if c.is_ascii() { + bytes.push(c as u8); + } else { + return exec_err!( + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", + c as u32 + ); + } + } + } + } + "ISO-8859-1" | "ISO88591" | "LATIN1" => { + if legacy_coding_error_action { + bytes.extend(s.chars().map(|c| { + let cp = c as u32; + if cp <= 255 { cp as u8 } else { b'?' } + })); + } else { + let _ = bytes.try_reserve(s.len()); + for c in s.chars() { + let cp = c as u32; + if cp <= 255 { + bytes.push(cp as u8); + } else { + return exec_err!( + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", + cp + ); + } + } + } } - "ISO-8859-1" | "ISO88591" | "LATIN1" => bytes.extend(s.chars().map(|c| { - let cp = c as u32; - if cp <= 255 { cp as u8 } else { b'?' } - })), "UTF-16BE" | "UTF16BE" => { + let _ = bytes.try_reserve(s.len() * 2); for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_be_bytes()); } } "UTF-16LE" | "UTF16LE" => { + let _ = bytes.try_reserve(s.len() * 2); for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_le_bytes()); } } "UTF-16" | "UTF16" => { + let _ = bytes.try_reserve(2 + s.len() * 2); // BOM (big-endian marker) followed by UTF-16BE encoded bytes bytes.extend_from_slice(&[0xFE, 0xFF]); for code_unit in s.encode_utf16() { @@ -107,39 +208,53 @@ fn encode_string(s: &str, charset: &str, bytes: &mut Vec) -> Result<()> { } // Spark treats UTF-32 as UTF-32BE (no BOM), matching Spark 3.5 and 4.1. "UTF-32" | "UTF32" | "UTF-32BE" | "UTF32BE" => { + let _ = bytes.try_reserve(s.len() * 4); for c in s.chars() { bytes.extend_from_slice(&(c as u32).to_be_bytes()); } } "UTF-32LE" | "UTF32LE" => { + let _ = bytes.try_reserve(s.len() * 4); for c in s.chars() { bytes.extend_from_slice(&(c as u32).to_le_bytes()); } } _ => { return exec_err!( - "Unsupported charset for encode: '{}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE", - charset + "Unsupported charset for encode: '{raw_charset}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE" ); } } Ok(()) } -/// The charset for each row. `Constant` is a single uppercased charset; `PerRow` -/// uppercases each row on demand, yielding `None` for null entries. +/// The charset for each row. +/// +/// In Spark 4 strict mode, charsets must be validated against `CANONICAL_CHARSETS`. +/// To produce accurate error messages with the exact raw parameter value provided by the user, +/// both the raw string and its uppercase representation are stored. enum Charsets<'a> { - Constant(&'a str), + Constant { raw: &'a str, upper: String }, PerRow(&'a StringArray), } -impl Charsets<'_> { - fn get(&self, i: usize) -> Option> { +impl<'a> Charsets<'a> { + fn constant(raw: &'a str) -> Self { + Self::Constant { + raw, + upper: raw.to_uppercase(), + } + } + + fn get(&self, i: usize) -> Option<(Cow<'_, str>, Cow<'_, str>)> { match self { - Charsets::Constant(charset) => Some(Cow::Borrowed(charset)), - Charsets::PerRow(array) => { - (!array.is_null(i)).then(|| Cow::Owned(array.value(i).to_uppercase())) + Charsets::Constant { raw, upper } => { + Some((Cow::Borrowed(*raw), Cow::Borrowed(upper.as_str()))) } + Charsets::PerRow(array) => (!array.is_null(i)).then(|| { + let raw = array.value(i); + (Cow::Borrowed(raw), Cow::Owned(raw.to_uppercase())) + }), } } } @@ -153,13 +268,22 @@ fn encode_rows<'a>( is_null: impl Fn(usize) -> bool, decode: impl Fn(usize) -> Cow<'a, str>, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { let mut builder = BinaryBuilder::with_capacity(len, len * 4); let mut bytes = Vec::new(); for i in 0..len { match (is_null(i), charsets.get(i)) { - (false, Some(charset)) => { - encode_string(&decode(i), &charset, &mut bytes)?; + (false, Some((raw, upper))) => { + validate_charset(&raw, &upper, legacy_java_charsets)?; + encode_string( + &decode(i), + &raw, + &upper, + legacy_coding_error_action, + &mut bytes, + )?; builder.append_value(&bytes); } _ => builder.append_null(), @@ -171,45 +295,88 @@ fn encode_rows<'a>( fn encode_array<'a, S: StringArrayType<'a>>( array: &S, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| Cow::Borrowed(array.value(i)), charsets, + legacy_java_charsets, + legacy_coding_error_action, ) } fn encode_binary_array<'a, B: arrow::array::BinaryArrayType<'a>>( array: &'a B, charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| String::from_utf8_lossy(array.value(i)), charsets, + legacy_java_charsets, + legacy_coding_error_action, ) } -fn encode_dispatch(arr: &ArrayRef, charsets: &Charsets) -> Result { +fn encode_dispatch( + arr: &ArrayRef, + charsets: &Charsets, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, +) -> Result { match arr.data_type() { - DataType::Utf8 => encode_array(&arr.as_string::(), charsets), - DataType::LargeUtf8 => encode_array(&arr.as_string::(), charsets), - DataType::Utf8View => encode_array(&arr.as_string_view(), charsets), - DataType::Binary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::LargeBinary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::BinaryView => encode_binary_array(&arr.as_binary_view(), charsets), + DataType::Utf8 => encode_array( + &arr.as_string::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::LargeUtf8 => encode_array( + &arr.as_string::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::Utf8View => encode_array( + &arr.as_string_view(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::Binary => encode_binary_array( + &arr.as_binary::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::LargeBinary => encode_binary_array( + &arr.as_binary::(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), + DataType::BinaryView => encode_binary_array( + &arr.as_binary_view(), + charsets, + legacy_java_charsets, + legacy_coding_error_action, + ), DataType::Null => Ok(Arc::new(BinaryArray::new_null(arr.len()))), dt => exec_err!("encode expects a string or binary argument, got {dt:?}"), } } -/// The uppercased charset from a scalar, or `None` if it is null. -fn scalar_charset(scalar: &ScalarValue) -> Result> { +/// The raw charset string from a scalar, or `None` if it is null. +fn scalar_charset(scalar: &ScalarValue) -> Result> { match scalar { ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s) => { - Ok(s.as_ref().map(|s| s.to_uppercase())) + Ok(s.as_deref()) } ScalarValue::Null => Ok(None), other => exec_err!("encode charset argument must be a string, got {other:?}"), @@ -282,6 +449,9 @@ impl ScalarUDFImpl for SparkEncode { fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [value, charset_arg] = take_function_args(self.name(), args.args)?; + let legacy_java_charsets = self.legacy_java_charsets; + let legacy_coding_error_action = self.legacy_coding_error_action; + let len = [&value, &charset_arg] .into_iter() .find_map(|arg| match arg { @@ -300,7 +470,17 @@ impl ScalarUDFImpl for SparkEncode { let result = match &charset_arg { ColumnarValue::Scalar(scalar) => match scalar_charset(scalar)? { Some(charset) => { - encode_dispatch(&value_arr, &Charsets::Constant(&charset))? + let charsets = Charsets::constant(charset); + // Upfront validation for scalar charset so invalid charset fails even if array is empty + if let Some((raw, upper)) = charsets.get(0) { + validate_charset(&raw, &upper, legacy_java_charsets)?; + } + encode_dispatch( + &value_arr, + &charsets, + legacy_java_charsets, + legacy_coding_error_action, + )? } None => { return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))); @@ -309,6 +489,8 @@ impl ScalarUDFImpl for SparkEncode { ColumnarValue::Array(charset_array) => encode_dispatch( &value_arr, &Charsets::PerRow(charset_array.as_string::()), + legacy_java_charsets, + legacy_coding_error_action, )?, }; @@ -323,6 +505,329 @@ impl ScalarUDFImpl for SparkEncode { #[cfg(test)] mod tests { use super::*; + use arrow::array::AsArray; + use datafusion_common::config::ConfigOptions; + + fn invoke_encode( + input: &str, + charset: &str, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, + ) -> Result> { + let func = + SparkEncode::with_options(legacy_java_charsets, legacy_coding_error_action); + let config = Arc::new(ConfigOptions::default()); + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some(input.to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(charset.to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args)? { + ColumnarValue::Scalar(ScalarValue::Binary(Some(b))) => Ok(b), + other => panic!("expected binary scalar, got {other:?}"), + } + } + + #[test] + fn test_canonical_charsets_strict_mode() { + assert_eq!( + invoke_encode("Spark SQL", "utf-8", false, false).unwrap(), + b"Spark SQL" + ); + assert_eq!( + invoke_encode("Hello", "us-ascii", false, false).unwrap(), + b"Hello" + ); + assert_eq!( + invoke_encode("naïve", "iso-8859-1", false, false).unwrap(), + &[0x6E, 0x61, 0xEF, 0x76, 0x65] + ); + assert_eq!( + invoke_encode("AB", "utf-16be", false, false).unwrap(), + &[0x00, 0x41, 0x00, 0x42] + ); + assert_eq!( + invoke_encode("AB", "utf-16le", false, false).unwrap(), + &[0x41, 0x00, 0x42, 0x00] + ); + assert_eq!( + invoke_encode("A", "utf-16", false, false).unwrap(), + &[0xFE, 0xFF, 0x00, 0x41] + ); + assert_eq!( + invoke_encode("A", "utf-32", false, false).unwrap(), + &[0x00, 0x00, 0x00, 0x41] + ); + } + + #[test] + fn test_canonical_case_insensitivity() { + assert_eq!( + invoke_encode("hello", "Utf-8", false, false).unwrap(), + b"hello" + ); + assert_eq!( + invoke_encode("hello", "US-ASCII", false, false).unwrap(), + b"hello" + ); + assert_eq!( + invoke_encode("hello", "Iso-8859-1", false, false).unwrap(), + b"hello" + ); + assert_eq!( + invoke_encode("AB", "Utf-16Be", false, false).unwrap(), + &[0x00, 0x41, 0x00, 0x42] + ); + assert_eq!( + invoke_encode("AB", "Utf-16Le", false, false).unwrap(), + &[0x41, 0x00, 0x42, 0x00] + ); + assert_eq!( + invoke_encode("A", "Utf-16", false, false).unwrap(), + &[0xFE, 0xFF, 0x00, 0x41] + ); + assert_eq!( + invoke_encode("A", "Utf-32", false, false).unwrap(), + &[0x00, 0x00, 0x00, 0x41] + ); + } + + #[test] + fn test_alias_rejection_strict_mode() { + let aliases = [ + "utf8", "ascii", "latin1", "iso88591", "utf-32be", "utf-32le", "utf32be", + "utf32le", "utf16be", "utf16le", "utf16", "utf32", + ]; + for alias in aliases { + let err = invoke_encode("A", alias, false, false).unwrap_err(); + assert!( + err.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]"), + "expected alias '{alias}' to be rejected in strict mode, got: {err}" + ); + } + + // Unknown charset + let err = invoke_encode("A", "EBCDIC", false, false).unwrap_err(); + assert!( + err.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]"), + "expected unknown charset to be rejected in strict mode, got: {err}" + ); + } + + #[test] + fn test_alias_acceptance_legacy_mode() { + assert_eq!( + invoke_encode("Spark", "utf8", true, false).unwrap(), + b"Spark" + ); + assert_eq!( + invoke_encode("Hello", "ascii", true, false).unwrap(), + b"Hello" + ); + assert_eq!( + invoke_encode("naïve", "latin1", true, false).unwrap(), + &[0x6E, 0x61, 0xEF, 0x76, 0x65] + ); + assert_eq!( + invoke_encode("naïve", "iso88591", true, false).unwrap(), + &[0x6E, 0x61, 0xEF, 0x76, 0x65] + ); + assert_eq!( + invoke_encode("A", "utf-32be", true, false).unwrap(), + &[0x00, 0x00, 0x00, 0x41] + ); + assert_eq!( + invoke_encode("A", "utf-32le", true, false).unwrap(), + &[0x41, 0x00, 0x00, 0x00] + ); + + // Truly unsupported charset in legacy mode + let err = invoke_encode("A", "EBCDIC", true, false).unwrap_err(); + assert!( + err.to_string().contains("Unsupported charset for encode"), + "expected unsupported charset error, got: {err}" + ); + } + + #[test] + fn test_unmappable_characters_strict_mode() { + // 'é' (U+00E9) in US-ASCII + let err = invoke_encode("é", "us-ascii", false, false).unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains("[MALFORMED_CHARACTER_CODING]"), + "expected MALFORMED_CHARACTER_CODING, got: {err_msg}" + ); + assert!( + err_msg.contains("SQLSTATE: 22000"), + "expected SQLSTATE: 22000, got: {err_msg}" + ); + assert!( + err_msg.contains("U+00E9"), + "expected code point U+00E9 in message, got: {err_msg}" + ); + + // 'Ā' (U+0100) in ISO-8859-1 + let err2 = invoke_encode("Ā", "iso-8859-1", false, false).unwrap_err(); + let err2_msg = err2.to_string(); + assert!( + err2_msg.contains("[MALFORMED_CHARACTER_CODING]"), + "expected MALFORMED_CHARACTER_CODING, got: {err2_msg}" + ); + assert!( + err2_msg.contains("SQLSTATE: 22000"), + "expected SQLSTATE: 22000, got: {err2_msg}" + ); + assert!( + err2_msg.contains("U+0100"), + "expected code point U+0100 in message, got: {err2_msg}" + ); + } + + #[test] + fn test_unmappable_characters_legacy_mode() { + // In legacy mode, 'é' in US-ASCII becomes '?' (0x3F) + let res = invoke_encode("é", "us-ascii", false, true).unwrap(); + assert_eq!(res, b"?"); + + // 'Ā' in ISO-8859-1 becomes '?' + let res2 = invoke_encode("Ā", "iso-8859-1", false, true).unwrap(); + assert_eq!(res2, b"?"); + } + + #[test] + fn test_independent_flags() { + // 1. legacy_java_charsets = true, legacy_coding_error_action = false: + // 'ascii' alias accepted, but 'é' fails with MALFORMED_CHARACTER_CODING + let err = invoke_encode("é", "ascii", true, false).unwrap_err(); + assert!(err.to_string().contains("[MALFORMED_CHARACTER_CODING]")); + + // 2. legacy_java_charsets = false, legacy_coding_error_action = true: + // 'us-ascii' accepted, and 'é' replaced with '?' + let res = invoke_encode("é", "us-ascii", false, true).unwrap(); + assert_eq!(res, b"?"); + + // But non-canonical 'ascii' is still rejected + let err2 = invoke_encode("é", "ascii", false, true).unwrap_err(); + assert!( + err2.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + } + + #[test] + fn test_empty_and_null_inputs() { + let func = SparkEncode::new(); + let config = Arc::new(ConfigOptions::default()); + + // Empty string + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::new()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("utf-8".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(Some(b))) => { + assert_eq!(b, Vec::::new()) + } + other => panic!("expected empty binary, got {other:?}"), + } + + // Null value yields null + let args_null_val = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("utf-8".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_null_val).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(None)) => {} + other => panic!("expected null binary, got {other:?}"), + } + + // Null charset yields null (NullIntolerant) + let args_null_charset = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some("hello".to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_null_charset).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(None)) => {} + other => panic!("expected null binary, got {other:?}"), + } + } + + #[test] + fn test_per_row_charsets_array() { + let func = SparkEncode::new(); + let config = Arc::new(ConfigOptions::default()); + + let values = Arc::new(StringArray::from(vec![Some("hello"), Some("world")])); + let charsets = Arc::new(StringArray::from(vec![Some("utf-8"), Some("utf-16be")])); + + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ColumnarValue::Array(values), ColumnarValue::Array(charsets)], + number_rows: 2, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + + let res = func.invoke_with_args(args).unwrap(); + match res { + ColumnarValue::Array(arr) => { + let bin = arr.as_binary::(); + assert_eq!(bin.value(0), b"hello"); + assert_eq!( + bin.value(1), + &[0x00, 0x77, 0x00, 0x6F, 0x00, 0x72, 0x00, 0x6C, 0x00, 0x64] + ); + } + other => panic!("expected array, got {other:?}"), + } + + // Array with invalid alias in strict mode should fail + let bad_charsets = Arc::new(StringArray::from(vec![Some("utf-8"), Some("utf8")])); + let bad_args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Array(Arc::new(StringArray::from(vec![ + Some("hello"), + Some("world"), + ]))), + ColumnarValue::Array(bad_charsets), + ], + number_rows: 2, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + let err = func.invoke_with_args(bad_args).unwrap_err(); + assert!( + err.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + } #[test] fn test_encode_return_field_nullable() { diff --git a/crates/sqllogictest/test_files/string/encode.slt b/crates/sqllogictest/test_files/string/encode.slt index 93df7e80..f3f6eb8f 100644 --- a/crates/sqllogictest/test_files/string/encode.slt +++ b/crates/sqllogictest/test_files/string/encode.slt @@ -38,17 +38,13 @@ SELECT encode('Spark SQL'::string, 'utf-16'::string); ---- feff0053007000610072006b002000530051004c -# UTF-32BE encoding -query ? +# UTF-32BE encoding (alias rejected in Spark 4 strict mode) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('A'::string, 'utf-32be'::string); ----- -00000041 -# UTF-32LE encoding -query ? +# UTF-32LE encoding (alias rejected in Spark 4 strict mode) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('AB'::string, 'utf-32le'::string); ----- -4100000042000000 # UTF-32 (Spark 3.5 / 4.1: no BOM, identical to UTF-32BE) query ? @@ -62,17 +58,13 @@ SELECT encode('hello'::string, 'Utf-8'::string); ---- 68656c6c6f -# US-ASCII: unmappable characters are silently replaced with '?' (Spark 3.5 behavior) -query ? +# US-ASCII: unmappable characters fail in Spark 4 strict mode +statement error MALFORMED_CHARACTER_CODING SELECT encode('é'::string, 'us-ascii'::string); ----- -3f -# ISO-8859-1: code points above U+00FF are replaced with '?' (Ā is U+0100) -query ? +# ISO-8859-1: code points above U+00FF fail in Spark 4 strict mode (Ā is U+0100) +statement error MALFORMED_CHARACTER_CODING SELECT encode('Ā'::string, 'iso-8859-1'::string); ----- -3f # Multi-byte input (emoji 😀 = U+1F600) across charsets query ? @@ -95,15 +87,11 @@ SELECT encode('😀'::string, 'utf-16'::string); ---- feffd83dde00 -query ? +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32be'::string); ----- -0001f600 -query ? +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('😀'::string, 'utf-32le'::string); ----- -00f60100 query ? SELECT encode('😀'::string, 'utf-32'::string); @@ -174,8 +162,8 @@ SELECT encode(a, b) FROM VALUES (X'0a', 'US-ASCII'), (X'deadbeef', 'UTF-16be') A 0a 07adfffdfffd -# Error: unsupported charset -statement error Unsupported charset for encode +# Error: unsupported charset (strict Spark 4 rejects unknown charsets with INVALID_PARAMETER_VALUE.CHARSET) +statement error INVALID_PARAMETER_VALUE\.CHARSET SELECT encode('hello'::string, 'EBCDIC'::string); # Error: no arguments From 5addd2f9005f702eb093bea806f7af5a3fe2c341 Mon Sep 17 00:00:00 2001 From: Subodh Bansode Date: Mon, 16 Nov 2026 10:28:20 +0530 Subject: [PATCH 2/2] fix(string): align encode charset validation with Spark Remove up-front scalar charset validation to ensure NULL inputs return NULL without raising errors (NullIntolerant). Return [INVALID_PARAMETER_VALUE.CHARSET] with SQLSTATE 22023 for unknown charsets in legacy mode. Match Spark 4.2 error formatting with backticks, lowercase sorted canonical list, and standard MALFORMED_CHARACTER_CODING message. Add strict-mode test for invalid UTF-8 binary input and NULL propagation cases. Signed-off-by: Subodh Bansode --- crates/spark/src/function/string/encode.rs | 245 +++++++++++++++--- .../sqllogictest/test_files/string/encode.slt | 29 +++ 2 files changed, 242 insertions(+), 32 deletions(-) diff --git a/crates/spark/src/function/string/encode.rs b/crates/spark/src/function/string/encode.rs index b8dbb49d..51b0c0b0 100644 --- a/crates/spark/src/function/string/encode.rs +++ b/crates/spark/src/function/string/encode.rs @@ -33,7 +33,8 @@ use datafusion_macros::user_doc; Characters the character set cannot represent (non-ASCII in `US-ASCII`, code points above U+00FF in \ `ISO-8859-1`) raise `[MALFORMED_CHARACTER_CODING]`. Binary input is decoded as UTF-8 first, with each \ invalid byte replaced by U+FFFD. Returns NULL if either input is NULL. \ - This follows Spark 4.0. Pre-Spark 4.0 accepted Java aliases (`UTF8`, `ASCII`, `LATIN1`, `UTF-32BE`, ...) \ + This follows Spark 4.0. Pre-Spark 4.0 accepted Java aliases (legacy mode supports a fixed set of aliases: \ + `UTF8`, `ASCII`, `LATIN1`, `ISO88591`, `UTF16`, `UTF16BE`, `UTF16LE`, `UTF32`, `UTF32BE`, `UTF32LE`) \ and replaced unmappable characters with `?`; those legacy behaviors can be enabled programmatically \ via `with_legacy_java_charsets(true)` and `with_legacy_coding_error_action(true)`.", syntax_example = "encode(str, charset)", @@ -112,12 +113,16 @@ fn is_canonical_charset(upper: &str) -> bool { CANONICAL_CHARSETS.contains(&upper) } +fn invalid_charset_error(raw: &str) -> datafusion_common::DataFusionError { + datafusion_common::DataFusionError::Execution(format!( + "[INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) `charset` in `encode` is invalid: expects one of the iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8, but got {raw}. SQLSTATE: 22023" + )) +} + fn validate_charset(raw: &str, upper: &str, legacy_java_charsets: bool) -> Result<()> { if !legacy_java_charsets { if !is_canonical_charset(upper) { - return exec_err!( - "[INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) 'charset' in 'encode' is invalid: expects one of the [US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32], but got '{raw}'." - ); + return Err(invalid_charset_error(raw)); } } else { match upper { @@ -126,9 +131,7 @@ fn validate_charset(raw: &str, upper: &str, legacy_java_charsets: bool) -> Resul | "UTF16" | "UTF-32" | "UTF32" | "UTF-32BE" | "UTF32BE" | "UTF-32LE" | "UTF32LE" => {} _ => { - return exec_err!( - "Unsupported charset for encode: '{raw}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE" - ); + return Err(invalid_charset_error(raw)); } } } @@ -158,8 +161,7 @@ fn encode_string( bytes.push(c as u8); } else { return exec_err!( - "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", - c as u32 + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing `encode` with {raw_charset} SQLSTATE: 22000" ); } } @@ -179,8 +181,7 @@ fn encode_string( bytes.push(cp as u8); } else { return exec_err!( - "[MALFORMED_CHARACTER_CODING] Invalid value found when performing 'encode' with {raw_charset}: character '{c}' (U+{:04X}) cannot be encoded. SQLSTATE: 22000", - cp + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing `encode` with {raw_charset} SQLSTATE: 22000" ); } } @@ -220,9 +221,7 @@ fn encode_string( } } _ => { - return exec_err!( - "Unsupported charset for encode: '{raw_charset}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE" - ); + return Err(invalid_charset_error(raw_charset)); } } Ok(()) @@ -471,10 +470,6 @@ impl ScalarUDFImpl for SparkEncode { ColumnarValue::Scalar(scalar) => match scalar_charset(scalar)? { Some(charset) => { let charsets = Charsets::constant(charset); - // Upfront validation for scalar charset so invalid charset fails even if array is empty - if let Some((raw, upper)) = charsets.get(0) { - validate_charset(&raw, &upper, legacy_java_charsets)?; - } encode_dispatch( &value_arr, &charsets, @@ -483,15 +478,28 @@ impl ScalarUDFImpl for SparkEncode { )? } None => { - return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))); + if is_scalar { + return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))); + } else { + return Ok(ColumnarValue::Array(Arc::new( + BinaryArray::new_null(inferred_length), + ))); + } } }, - ColumnarValue::Array(charset_array) => encode_dispatch( - &value_arr, - &Charsets::PerRow(charset_array.as_string::()), - legacy_java_charsets, - legacy_coding_error_action, - )?, + ColumnarValue::Array(charset_array) => { + let charset_array = if charset_array.data_type() != &DataType::Utf8 { + arrow::compute::cast(charset_array, &DataType::Utf8)? + } else { + Arc::clone(charset_array) + }; + encode_dispatch( + &value_arr, + &Charsets::PerRow(charset_array.as_string::()), + legacy_java_charsets, + legacy_coding_error_action, + )? + } }; if is_scalar { @@ -533,6 +541,31 @@ mod tests { } } + fn invoke_encode_binary( + input: &[u8], + charset: &str, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, + ) -> Result> { + let func = + SparkEncode::with_options(legacy_java_charsets, legacy_coding_error_action); + let config = Arc::new(ConfigOptions::default()); + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Binary(Some(input.to_vec()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some(charset.to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args)? { + ColumnarValue::Scalar(ScalarValue::Binary(Some(b))) => Ok(b), + other => panic!("expected binary scalar, got {other:?}"), + } + } + #[test] fn test_canonical_charsets_strict_mode() { assert_eq!( @@ -648,11 +681,16 @@ mod tests { &[0x41, 0x00, 0x00, 0x00] ); - // Truly unsupported charset in legacy mode + // Unknown charset in legacy mode raises INVALID_PARAMETER_VALUE.CHARSET with SQLSTATE 22023 let err = invoke_encode("A", "EBCDIC", true, false).unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains("[INVALID_PARAMETER_VALUE.CHARSET]"), + "expected INVALID_PARAMETER_VALUE.CHARSET in legacy mode, got: {err_msg}" + ); assert!( - err.to_string().contains("Unsupported charset for encode"), - "expected unsupported charset error, got: {err}" + err_msg.contains("SQLSTATE: 22023"), + "expected SQLSTATE: 22023, got: {err_msg}" ); } @@ -670,8 +708,10 @@ mod tests { "expected SQLSTATE: 22000, got: {err_msg}" ); assert!( - err_msg.contains("U+00E9"), - "expected code point U+00E9 in message, got: {err_msg}" + err_msg.contains( + "Invalid value found when performing `encode` with us-ascii SQLSTATE: 22000" + ), + "expected exact error message, got: {err_msg}" ); // 'Ā' (U+0100) in ISO-8859-1 @@ -686,8 +726,10 @@ mod tests { "expected SQLSTATE: 22000, got: {err2_msg}" ); assert!( - err2_msg.contains("U+0100"), - "expected code point U+0100 in message, got: {err2_msg}" + err2_msg.contains( + "Invalid value found when performing `encode` with iso-8859-1 SQLSTATE: 22000" + ), + "expected exact error message, got: {err2_msg}" ); } @@ -856,4 +898,143 @@ mod tests { .unwrap(); assert!(!non_nullable.is_nullable()); } + + #[test] + fn test_binary_input_malformed_utf8_strict_and_legacy() { + // X'ff' with us-ascii in strict mode raises MALFORMED_CHARACTER_CODING + let err = invoke_encode_binary(&[0xFF], "us-ascii", false, false).unwrap_err(); + let err_msg = err.to_string(); + assert!( + err_msg.contains("[MALFORMED_CHARACTER_CODING]"), + "expected MALFORMED_CHARACTER_CODING, got: {err_msg}" + ); + assert!( + err_msg.contains("SQLSTATE: 22000"), + "expected SQLSTATE: 22000, got: {err_msg}" + ); + assert!( + err_msg.contains( + "Invalid value found when performing `encode` with us-ascii SQLSTATE: 22000" + ), + "expected exact error text, got: {err_msg}" + ); + + // X'ff' with us-ascii in legacy coding error action produces '?' (0x3F) + let res = invoke_encode_binary(&[0xFF], "us-ascii", false, true).unwrap(); + assert_eq!(res, b"?"); + } + + #[test] + fn test_invalid_charset_error_message_format() { + let err = invoke_encode("hello", "bogus", false, false).unwrap_err(); + let err_msg = err.to_string(); + assert_eq!( + err_msg, + "Execution error: [INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) `charset` in `encode` is invalid: expects one of the iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8, but got bogus. SQLSTATE: 22023" + ); + + // Also verify case preservation of the raw charset argument + let err2 = invoke_encode("hello", "My_Bogus_Charset", false, false).unwrap_err(); + let err2_msg = err2.to_string(); + assert!( + err2_msg.contains("but got My_Bogus_Charset. SQLSTATE: 22023"), + "expected preserved raw charset casing, got: {err2_msg}" + ); + } + + #[test] + fn test_null_propagation_with_invalid_charset() { + let func = SparkEncode::new(); + let config = Arc::new(ConfigOptions::default()); + + // 1. encode(NULL, 'bogus') returns NULL + let args = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(None)), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("bogus".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args).unwrap() { + ColumnarValue::Scalar(ScalarValue::Binary(None)) => {} + other => panic!("expected null binary, got {other:?}"), + } + + // 2. Column where all values are NULL with invalid charset returns NULL array + let null_values = Arc::new(StringArray::from(vec![None as Option<&str>, None])); + let args_all_nulls = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Array(null_values), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("bogus".to_string()))), + ], + number_rows: 2, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_all_nulls).unwrap() { + ColumnarValue::Array(arr) => { + assert_eq!(arr.len(), 2); + assert!(arr.is_null(0)); + assert!(arr.is_null(1)); + } + other => panic!("expected null binary array, got {other:?}"), + } + + // 3. Empty input batch with invalid charset returns empty array without error + let empty_values = Arc::new(StringArray::from(Vec::>::new())); + let args_empty = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Array(empty_values), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("bogus".to_string()))), + ], + number_rows: 0, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + match func.invoke_with_args(args_empty).unwrap() { + ColumnarValue::Array(arr) => { + assert_eq!(arr.len(), 0); + } + other => panic!("expected empty binary array, got {other:?}"), + } + + // 4. Empty string with invalid charset raises error + let args_empty_str = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some(String::new()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("bogus".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + let err = func.invoke_with_args(args_empty_str).unwrap_err(); + assert!( + err.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + + // 5. Non-null string with invalid charset raises error + let args_non_null = ScalarFunctionArgs { + arg_fields: vec![], + args: vec![ + ColumnarValue::Scalar(ScalarValue::Utf8(Some("hello".to_string()))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("bogus".to_string()))), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::clone(&config), + }; + let err2 = func.invoke_with_args(args_non_null).unwrap_err(); + assert!( + err2.to_string() + .contains("[INVALID_PARAMETER_VALUE.CHARSET]") + ); + } } diff --git a/crates/sqllogictest/test_files/string/encode.slt b/crates/sqllogictest/test_files/string/encode.slt index f3f6eb8f..a2a06b10 100644 --- a/crates/sqllogictest/test_files/string/encode.slt +++ b/crates/sqllogictest/test_files/string/encode.slt @@ -128,12 +128,41 @@ SELECT encode(X'FEFF00610073', 'utf-8'::string); ---- efbfbdefbfbd00610073 +# Strict mode: binary input with invalid UTF-8 byte 0xFF becomes U+FFFD, which cannot be encoded in US-ASCII +query error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii SQLSTATE: 22000 +SELECT encode(X'ff', 'us-ascii'::string); + # NULL input query ? SELECT encode(NULL::string, 'utf-8'::string); ---- NULL +# NULL input with invalid charset returns NULL (Spark NullIntolerant) +query ? +SELECT encode(NULL::string, 'bogus'::string); +---- +NULL + +# Column input where every value is NULL with invalid charset returns NULL +query ? +SELECT encode(a, 'bogus'::string) FROM VALUES (CAST(NULL AS STRING)) t(a); +---- +NULL + +# Empty input batch with invalid charset returns no rows without error +query ? +SELECT encode(a, 'bogus'::string) FROM (SELECT 'test'::string AS a WHERE false); +---- + +# Empty string with invalid charset raises error +query error \[INVALID_PARAMETER_VALUE\.CHARSET\] The value of parameter\(s\) `charset` in `encode` is invalid: expects one of the iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8, but got bogus\. SQLSTATE: 22023 +SELECT encode(''::string, 'bogus'::string); + +# Non-null value with invalid charset raises error +query error \[INVALID_PARAMETER_VALUE\.CHARSET\] The value of parameter\(s\) `charset` in `encode` is invalid: expects one of the iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8, but got bogus\. SQLSTATE: 22023 +SELECT encode('hello'::string, 'bogus'::string); + # Array input with NULLs query ? SELECT encode(s, 'utf-8'::string) FROM (VALUES ('hello'::string), (NULL::string), ('world'::string)) AS t(s);