diff --git a/crates/spark/src/function/url/url_decode.rs b/crates/spark/src/function/url/url_decode.rs index 90a0112e..eaacd251 100644 --- a/crates/spark/src/function/url/url_decode.rs +++ b/crates/spark/src/function/url/url_decode.rs @@ -13,7 +13,7 @@ use datafusion_common::cast::{ as_large_string_array, as_string_array, as_string_view_array, }; use datafusion_common::utils::offset_span_len; -use datafusion_common::{Result, exec_datafusion_err, exec_err, plan_err}; +use datafusion_common::{Result, exec_err, plan_err}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, @@ -29,8 +29,8 @@ use percent_encoding::percent_decode; doc_section(label = "URL Functions"), description = "Decodes a string in `application/x-www-form-urlencoded` format, the inverse of `url_encode`. \ DataFusion has no built-in `url_decode`. `+` becomes a space and each `%XX` escape becomes the byte \ - it encodes, and the bytes are read as UTF-8. A `%` that is not followed by two hex digits, or bytes \ - that are not valid UTF-8, are an error; use `try_url_decode` to get NULL instead. \ + it encodes, and the bytes are read as UTF-8, with U+FFFD for bytes that are not valid UTF-8. \ + A `%` that is not followed by two hex digits is an error; use `try_url_decode` to get NULL instead. \ Returns NULL if the input is NULL.", syntax_example = "url_decode(str)", sql_example = r#"```sql @@ -79,23 +79,57 @@ impl UrlDecode { /// /// * `Ok(Cow)` - The decoded string, borrowed from `value` when there /// was nothing to rewrite and owned otherwise - /// * `Err(DataFusionError)` - If the input is malformed or contains invalid UTF-8 + /// * `Err(DataFusionError)` - If the input is malformed fn decode(value: &str) -> Result> { // Check if the string has valid percent encoding Self::validate_percent_encoding(value)?; - match Self::replace_plus(value.as_bytes()) { + Ok(match Self::replace_plus(value.as_bytes()) { // No '+' was rewritten, so the decode can borrow from `value` itself. - Cow::Borrowed(bytes) => percent_decode(bytes) - .decode_utf8() - .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")), + Cow::Borrowed(bytes) => Self::decode_utf8_lossy(percent_decode(bytes).into()), // Rewriting '+' already allocated, so owning the decoded form here // costs nothing beyond what has been spent. - Cow::Owned(bytes) => percent_decode(&bytes) - .decode_utf8() - .map(|decoded| Cow::Owned(decoded.into_owned())) - .map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")), + Cow::Owned(bytes) => Cow::Owned( + Self::decode_utf8_lossy(percent_decode(&bytes).into()).into_owned(), + ), + }) + } + + /// Reads `bytes` as UTF-8 the way Java's `URLDecoder` does: each invalid + /// sequence becomes U+FFFD. Unlike `String::from_utf8_lossy`, an encoded + /// surrogate (`ED` followed by `A0..=BF`, plus one more continuation byte + /// if present) becomes a single U+FFFD rather than one per byte. + fn decode_utf8_lossy(bytes: Cow<'_, [u8]>) -> Cow<'_, str> { + match bytes { + Cow::Borrowed(bytes) => match std::str::from_utf8(bytes) { + Ok(s) => Cow::Borrowed(s), + Err(_) => Cow::Owned(Self::replace_invalid_utf8(bytes)), + }, + Cow::Owned(bytes) => match String::from_utf8(bytes) { + Ok(s) => Cow::Owned(s), + Err(e) => Cow::Owned(Self::replace_invalid_utf8(e.as_bytes())), + }, + } + } + + fn replace_invalid_utf8(mut rest: &[u8]) -> String { + let mut out = String::with_capacity(rest.len()); + while let Some(chunk) = rest.utf8_chunks().next() { + out.push_str(chunk.valid()); + let mut end = chunk.valid().len() + chunk.invalid().len(); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + if chunk.invalid() == [0xED] && matches!(rest.get(end), Some(0xA0..=0xBF)) + { + end += 1; + if matches!(rest.get(end), Some(0x80..=0xBF)) { + end += 1; + } + } + } + rest = &rest[end..]; } + out } /// Replace b'+' with b' ' diff --git a/crates/sqllogictest/test_files/url/try_url_decode.slt b/crates/sqllogictest/test_files/url/try_url_decode.slt index 81a7d065..773cf252 100644 --- a/crates/sqllogictest/test_files/url/try_url_decode.slt +++ b/crates/sqllogictest/test_files/url/try_url_decode.slt @@ -55,9 +55,14 @@ SELECT try_url_decode('https%3%2F%2Fspark.apache.org'::string); ---- NULL -# Bytes that are not valid UTF-8 give NULL -# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366) +# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark query T -SELECT try_url_decode('%FF'::string); +SELECT hex(try_url_decode('%FF'::string)); ---- -NULL +EFBFBD + +# An encoded surrogate becomes a single U+FFFD, like Spark +query T +SELECT hex(try_url_decode('%ED%A0%80'::string)); +---- +EFBFBD diff --git a/crates/sqllogictest/test_files/url/url_decode.slt b/crates/sqllogictest/test_files/url/url_decode.slt index cf8fe0c6..433dab11 100644 --- a/crates/sqllogictest/test_files/url/url_decode.slt +++ b/crates/sqllogictest/test_files/url/url_decode.slt @@ -53,7 +53,36 @@ Spark SQL! query error DataFusion error: Execution error: Invalid percent\-encoding: invalid hex sequence '%3%' at position 5 SELECT url_decode('https%3%2F%2Fspark.apache.org'::string); -# Bytes that are not valid UTF-8 are an error -# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366) -query error DataFusion error: Execution error: Invalid UTF\-8 sequence -SELECT url_decode('%FF'::string); +# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark +query T +SELECT hex(url_decode('%FF'::string)); +---- +EFBFBD + +query T +SELECT hex(url_decode(column1)) FROM VALUES ('a%FFb%C3%28'), ('a%C3%A9'), (NULL); +---- +61EFBFBD62EFBFBD28 +61C3A9 +NULL + +# An encoded surrogate becomes a single U+FFFD, like Spark +query T +SELECT hex(url_decode('%ED%A0%80'::string)); +---- +EFBFBD + +query T +SELECT hex(url_decode('%ED%A0%80a'::string)); +---- +EFBFBD61 + +query T +SELECT hex(url_decode('%ED%A0'::string)); +---- +EFBFBD + +query T +SELECT hex(url_decode('%ED%A0%80%ED%B0%80'::string)); +---- +EFBFBDEFBFBD