Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 46 additions & 12 deletions crates/spark/src/function/url/url_decode.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use datafusion_common::cast::{
as_large_string_array, as_string_array, as_string_view_array,
};
use datafusion_common::utils::offset_span_len;
use datafusion_common::{Result, exec_datafusion_err, exec_err, plan_err};
use datafusion_common::{Result, exec_err, plan_err};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
Expand All @@ -29,8 +29,8 @@ use percent_encoding::percent_decode;
doc_section(label = "URL Functions"),
description = "Decodes a string in `application/x-www-form-urlencoded` format, the inverse of `url_encode`. \
DataFusion has no built-in `url_decode`. `+` becomes a space and each `%XX` escape becomes the byte \
it encodes, and the bytes are read as UTF-8. A `%` that is not followed by two hex digits, or bytes \
that are not valid UTF-8, are an error; use `try_url_decode` to get NULL instead. \
it encodes, and the bytes are read as UTF-8, with U+FFFD for bytes that are not valid UTF-8. \
A `%` that is not followed by two hex digits is an error; use `try_url_decode` to get NULL instead. \
Returns NULL if the input is NULL.",
syntax_example = "url_decode(str)",
sql_example = r#"```sql
Expand Down Expand Up @@ -79,23 +79,57 @@ impl UrlDecode {
///
/// * `Ok(Cow<str>)` - The decoded string, borrowed from `value` when there
/// was nothing to rewrite and owned otherwise
/// * `Err(DataFusionError)` - If the input is malformed or contains invalid UTF-8
/// * `Err(DataFusionError)` - If the input is malformed
fn decode(value: &str) -> Result<Cow<'_, str>> {
// Check if the string has valid percent encoding
Self::validate_percent_encoding(value)?;

match Self::replace_plus(value.as_bytes()) {
Ok(match Self::replace_plus(value.as_bytes()) {
// No '+' was rewritten, so the decode can borrow from `value` itself.
Cow::Borrowed(bytes) => percent_decode(bytes)
.decode_utf8()
.map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")),
Cow::Borrowed(bytes) => Self::decode_utf8_lossy(percent_decode(bytes).into()),
// Rewriting '+' already allocated, so owning the decoded form here
// costs nothing beyond what has been spent.
Cow::Owned(bytes) => percent_decode(&bytes)
.decode_utf8()
.map(|decoded| Cow::Owned(decoded.into_owned()))
.map_err(|e| exec_datafusion_err!("Invalid UTF-8 sequence: {e}")),
Cow::Owned(bytes) => Cow::Owned(
Self::decode_utf8_lossy(percent_decode(&bytes).into()).into_owned(),
),
})
}

/// Reads `bytes` as UTF-8 the way Java's `URLDecoder` does: each invalid
/// sequence becomes U+FFFD. Unlike `String::from_utf8_lossy`, an encoded
/// surrogate (`ED` followed by `A0..=BF`, plus one more continuation byte
/// if present) becomes a single U+FFFD rather than one per byte.
fn decode_utf8_lossy(bytes: Cow<'_, [u8]>) -> Cow<'_, str> {
match bytes {
Cow::Borrowed(bytes) => match std::str::from_utf8(bytes) {
Ok(s) => Cow::Borrowed(s),
Err(_) => Cow::Owned(Self::replace_invalid_utf8(bytes)),
},
Cow::Owned(bytes) => match String::from_utf8(bytes) {
Ok(s) => Cow::Owned(s),
Err(e) => Cow::Owned(Self::replace_invalid_utf8(e.as_bytes())),
},
}
}

fn replace_invalid_utf8(mut rest: &[u8]) -> String {
let mut out = String::with_capacity(rest.len());
while let Some(chunk) = rest.utf8_chunks().next() {
out.push_str(chunk.valid());
let mut end = chunk.valid().len() + chunk.invalid().len();
if !chunk.invalid().is_empty() {
out.push(char::REPLACEMENT_CHARACTER);
if chunk.invalid() == [0xED] && matches!(rest.get(end), Some(0xA0..=0xBF))
{
end += 1;
if matches!(rest.get(end), Some(0x80..=0xBF)) {
end += 1;
}
}
}
rest = &rest[end..];
}
out
}

/// Replace b'+' with b' '
Expand Down
13 changes: 9 additions & 4 deletions crates/sqllogictest/test_files/url/try_url_decode.slt
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,14 @@ SELECT try_url_decode('https%3%2F%2Fspark.apache.org'::string);
----
NULL

# Bytes that are not valid UTF-8 give NULL
# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366)
# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark
query T
SELECT try_url_decode('%FF'::string);
SELECT hex(try_url_decode('%FF'::string));
----
NULL
EFBFBD

# An encoded surrogate becomes a single U+FFFD, like Spark
query T
SELECT hex(try_url_decode('%ED%A0%80'::string));
----
EFBFBD
37 changes: 33 additions & 4 deletions crates/sqllogictest/test_files/url/url_decode.slt
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,36 @@ Spark SQL!
query error DataFusion error: Execution error: Invalid percent\-encoding: invalid hex sequence '%3%' at position 5
SELECT url_decode('https%3%2F%2Fspark.apache.org'::string);

# Bytes that are not valid UTF-8 are an error
# xfail spark: Spark replaces bytes that are not valid UTF-8 with U+FFFD, datafusion-spark raises an error or returns NULL (https://github.com/apache/datafusion-spark/issues/366)
query error DataFusion error: Execution error: Invalid UTF\-8 sequence
SELECT url_decode('%FF'::string);
# Bytes that are not valid UTF-8 become U+FFFD (EFBFBD), like Spark
query T
SELECT hex(url_decode('%FF'::string));
----
EFBFBD

query T
SELECT hex(url_decode(column1)) FROM VALUES ('a%FFb%C3%28'), ('a%C3%A9'), (NULL);
----
61EFBFBD62EFBFBD28
61C3A9
NULL

# An encoded surrogate becomes a single U+FFFD, like Spark
query T
SELECT hex(url_decode('%ED%A0%80'::string));
----
EFBFBD

query T
SELECT hex(url_decode('%ED%A0%80a'::string));
----
EFBFBD61

query T
SELECT hex(url_decode('%ED%A0'::string));
----
EFBFBD

query T
SELECT hex(url_decode('%ED%A0%80%ED%B0%80'::string));
----
EFBFBDEFBFBD