From e50999f79bef7842213a98a76acb4acbac8ec5a4 Mon Sep 17 00:00:00 2001 From: Roopesh S <119874392+TechWizard9999@users.noreply.github.com> Date: Thu, 19 Nov 2026 08:33:09 +0530 Subject: [PATCH 1/2] fix url_decode utf-8 --- crates/spark/src/function/url/url_decode.rs | 24 +++++++++---------- .../test_files/url/try_url_decode.slt | 5 ++-- .../test_files/url/url_decode.slt | 14 ++++++++--- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/crates/spark/src/function/url/url_decode.rs b/crates/spark/src/function/url/url_decode.rs index 90a0112e..0bed5df6 100644 --- a/crates/spark/src/function/url/url_decode.rs +++ b/crates/spark/src/function/url/url_decode.rs @@ -13,7 +13,7 @@ use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; use datafusion_common::utils::offset_span_len; -use datafusion_common::{Result, exec_datafusion_err, exec_err, plan_err}; +use datafusion_common::{Result, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -29,8 +29,8 @@ use percent_encoding::percent_decode; doc_section(label = "URL Functions"), description = "Decodes a string in `application/x-www-form-urlencoded` format, the inverse of `url_encode`. \ DataFusion has no built-in `url_decode`. `+` becomes a space and each `%XX` escape becomes the byte \ - it encodes, and the bytes are read as UTF-8. A `%` that is not followed by two hex digits, or bytes \ - that are not valid UTF-8, are an error; use `try_url_decode` to get NULL instead. \ + it encodes, and the bytes are read as UTF-8, with U+FFFD for bytes that are not valid UTF-8. \ + A `%` that is not followed by two hex digits is an error; use `try_url_decode` to get NULL instead. \ Returns NULL if the input is NULL.", syntax_example = "url_decode(str)", sql_example = r#"```sql @@ -79,23 +79,21 @@ impl UrlDecode { /// /// * `Ok(Cow)` - The decoded string, borrowed from `value` when there /// was nothing to rewrite and owned otherwise - /// * `Err(DataFusionError)` - If the input is malformed or contains invalid UTF-8 + /// * `Err(DataFusionError)` - If the input is malformed fn decode(value: &str) -> Result> { // Check if the string has valid percent encoding Self::validate_percent_encoding(value)?; - match Self::replace_plus(value.as_bytes()) { + // Bytes that are not valid UTF-8 become U+FFFD, like Java's `URLDecoder` + Ok(match Self::replace_plus(value.as_bytes()) { // No '+' was rewritten, so the decode can borrow from `value` itself. - Cow::Borrowed(bytes) => percent_decode(bytes) - .decode_utf8() - .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")), + Cow::Borrowed(bytes) => percent_decode(bytes).decode_utf8_lossy(), // Rewriting '+' already allocated, so owning the decoded form here // costs nothing beyond what has been spent. - Cow::Owned(bytes) => percent_decode(&bytes) - .decode_utf8() - .map(|decoded| Cow::Owned(decoded.into_owned())) - .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")), - } + Cow::Owned(bytes) => { + Cow::Owned(percent_decode(&bytes).decode_utf8_lossy().into_owned()) + } + }) } /// Replace b'+' with b' ' diff --git a/crates/sqllogictest/test_files/url/try_url_decode.slt b/crates/sqllogictest/test_files/url/try_url_decode.slt index 81a7d065..4c179eaa 100644 --- a/crates/sqllogictest/test_files/url/try_url_decode.slt +++ b/crates/sqllogictest/test_files/url/try_url_decode.slt @@ -55,9 +55,8 @@ SELECT try_url_decode('https%3%2F%2Fspark.apache.org'::string); ---- NULL -# Bytes that are not valid UTF-8 give NULL -# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366) +# Bytes that are not valid UTF-8 become U+FFFD, like Spark query T SELECT try_url_decode('%FF'::string); ---- -NULL +� diff --git a/crates/sqllogictest/test_files/url/url_decode.slt b/crates/sqllogictest/test_files/url/url_decode.slt index cf8fe0c6..e770365d 100644 --- a/crates/sqllogictest/test_files/url/url_decode.slt +++ b/crates/sqllogictest/test_files/url/url_decode.slt @@ -53,7 +53,15 @@ Spark SQL! query error DataFusion error: Execution error: Invalid percent\-encoding: invalid hex sequence '%3%' at position 5 SELECT url_decode('https%3%2F%2Fspark.apache.org'::string); -# Bytes that are not valid UTF-8 are an error -# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366) -query error DataFusion error: Execution error: Invalid UTF\-8 sequence +# Bytes that are not valid UTF-8 become U+FFFD, like Spark +query T SELECT url_decode('%FF'::string); +---- +� + +query T +SELECT url_decode(column1) FROM VALUES ('a%FFb%C3%28'), ('a%C3%A9'), (NULL); +---- +a�b�( +aé +NULL From 5c45167f9ccee98e836a47e3620f35f6ff933f36 Mon Sep 17 00:00:00 2001 From: Roopesh S <119874392+TechWizard9999@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:38:26 +0530 Subject: [PATCH 2/2] fold encoded surrogates --- crates/spark/src/function/url/url_decode.rs | 46 +++++++++++++++++-- .../test_files/url/try_url_decode.slt | 12 +++-- .../test_files/url/url_decode.slt | 33 ++++++++++--- 3 files changed, 77 insertions(+), 14 deletions(-) diff --git a/crates/spark/src/function/url/url_decode.rs b/crates/spark/src/function/url/url_decode.rs index 0bed5df6..eaacd251 100644 --- a/crates/spark/src/function/url/url_decode.rs +++ b/crates/spark/src/function/url/url_decode.rs @@ -84,18 +84,54 @@ impl UrlDecode { // Check if the string has valid percent encoding Self::validate_percent_encoding(value)?; - // Bytes that are not valid UTF-8 become U+FFFD, like Java's `URLDecoder` Ok(match Self::replace_plus(value.as_bytes()) { // No '+' was rewritten, so the decode can borrow from `value` itself. - Cow::Borrowed(bytes) => percent_decode(bytes).decode_utf8_lossy(), + Cow::Borrowed(bytes) => Self::decode_utf8_lossy(percent_decode(bytes).into()), // Rewriting '+' already allocated, so owning the decoded form here // costs nothing beyond what has been spent. - Cow::Owned(bytes) => { - Cow::Owned(percent_decode(&bytes).decode_utf8_lossy().into_owned()) - } + Cow::Owned(bytes) => Cow::Owned( + Self::decode_utf8_lossy(percent_decode(&bytes).into()).into_owned(), + ), }) } + /// Reads `bytes` as UTF-8 the way Java's `URLDecoder` does: each invalid + /// sequence becomes U+FFFD. Unlike `String::from_utf8_lossy`, an encoded + /// surrogate (`ED` followed by `A0..=BF`, plus one more continuation byte + /// if present) becomes a single U+FFFD rather than one per byte. + fn decode_utf8_lossy(bytes: Cow<'_, [u8]>) -> Cow<'_, str> { + match bytes { + Cow::Borrowed(bytes) => match std::str::from_utf8(bytes) { + Ok(s) => Cow::Borrowed(s), + Err(_) => Cow::Owned(Self::replace_invalid_utf8(bytes)), + }, + Cow::Owned(bytes) => match String::from_utf8(bytes) { + Ok(s) => Cow::Owned(s), + Err(e) => Cow::Owned(Self::replace_invalid_utf8(e.as_bytes())), + }, + } + } + + fn replace_invalid_utf8(mut rest: &[u8]) -> String { + let mut out = String::with_capacity(rest.len()); + while let Some(chunk) = rest.utf8_chunks().next() { + out.push_str(chunk.valid()); + let mut end = chunk.valid().len() + chunk.invalid().len(); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + if chunk.invalid() == [0xED] && matches!(rest.get(end), Some(0xA0..=0xBF)) + { + end += 1; + if matches!(rest.get(end), Some(0x80..=0xBF)) { + end += 1; + } + } + } + rest = &rest[end..]; + } + out + } + /// Replace b'+' with b' ' /// See: fn replace_plus(input: &[u8]) -> Cow<'_, [u8]> { diff --git a/crates/sqllogictest/test_files/url/try_url_decode.slt b/crates/sqllogictest/test_files/url/try_url_decode.slt index 4c179eaa..773cf252 100644 --- a/crates/sqllogictest/test_files/url/try_url_decode.slt +++ b/crates/sqllogictest/test_files/url/try_url_decode.slt @@ -55,8 +55,14 @@ SELECT try_url_decode('https%3%2F%2Fspark.apache.org'::string); ---- NULL -# Bytes that are not valid UTF-8 become U+FFFD, like Spark +# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark query T -SELECT try_url_decode('%FF'::string); +SELECT hex(try_url_decode('%FF'::string)); ---- -� +EFBFBD + +# An encoded surrogate becomes a single U+FFFD, like Spark +query T +SELECT hex(try_url_decode('%ED%A0%80'::string)); +---- +EFBFBD diff --git a/crates/sqllogictest/test_files/url/url_decode.slt b/crates/sqllogictest/test_files/url/url_decode.slt index e770365d..433dab11 100644 --- a/crates/sqllogictest/test_files/url/url_decode.slt +++ b/crates/sqllogictest/test_files/url/url_decode.slt @@ -53,15 +53,36 @@ Spark SQL! query error DataFusion error: Execution error: Invalid percent\-encoding: invalid hex sequence '%3%' at position 5 SELECT url_decode('https%3%2F%2Fspark.apache.org'::string); -# Bytes that are not valid UTF-8 become U+FFFD, like Spark +# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark query T -SELECT url_decode('%FF'::string); +SELECT hex(url_decode('%FF'::string)); ---- -� +EFBFBD query T -SELECT url_decode(column1) FROM VALUES ('a%FFb%C3%28'), ('a%C3%A9'), (NULL); +SELECT hex(url_decode(column1)) FROM VALUES ('a%FFb%C3%28'), ('a%C3%A9'), (NULL); ---- -a�b�( -aé +61EFBFBD62EFBFBD28 +61C3A9 NULL + +# An encoded surrogate becomes a single U+FFFD, like Spark +query T +SELECT hex(url_decode('%ED%A0%80'::string)); +---- +EFBFBD + +query T +SELECT hex(url_decode('%ED%A0%80a'::string)); +---- +EFBFBD61 + +query T +SELECT hex(url_decode('%ED%A0'::string)); +---- +EFBFBD + +query T +SELECT hex(url_decode('%ED%A0%80%ED%B0%80'::string)); +---- +EFBFBDEFBFBD