Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 66 additions & 5 deletions native/spark-expr/src/url_funcs/parse_url.rs
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,20 @@ fn extract_query_value<'a>(query: &'a str, key: &str) -> Option<&'a str> {
})
}

fn is_escape_at(bytes: &[u8], i: usize) -> bool {
matches!(bytes.get(i + 1..i + 3), Some([h, l]) if h.is_ascii_hexdigit() && l.is_ascii_hexdigit())
}

// Characters java.net.URI rejects in every component, plus '%' not followed by two hex digits.
fn has_invalid_uri_chars(s: &str) -> bool {
s.bytes()
.any(|b| b == b' ' || b == b'{' || b == b'}' || b == b'<' || b == b'>' || b < 0x20)
let bytes = s.as_bytes();
let bad_byte = bytes.iter().enumerate().any(|(i, &b)| match b {
b' ' | b'"' | b'<' | b'>' | b'\\' | b'^' | b'`' | b'{' | b'|' | b'}' | 0x7f => true,
b'%' => !is_escape_at(bytes, i),
_ => b < 0x20,
});
// Non-ASCII characters are allowed unless they are control or space characters.
bad_byte || (!s.is_ascii() && s.chars().any(|c| c.is_control() || c.is_whitespace()))
}

fn invalid_url_err(value: &str) -> datafusion::common::DataFusionError {
Expand Down Expand Up @@ -181,11 +192,48 @@ fn parse_url_component<'a>(
fragment,
} = decompose_uri(value);

if scheme.is_none() {
if value.contains("://") {
// A value without a scheme is a relative reference; a leading ':' is an empty
// scheme name, which java.net.URI rejects ("Expected scheme name at index 0").
if value.starts_with(':') {
return Err(invalid_url_err(value));
}

// java.net.URI: scheme = alpha *( alpha | digit | "+" | "-" | "." ), followed by a
// non-empty scheme-specific part; one not starting with '/' makes the URI opaque.
let mut opaque = false;
if let Some(scheme) = scheme {
let mut bytes = scheme.bytes();
let valid_scheme = bytes.next().is_some_and(|b| b.is_ascii_alphabetic())
&& bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'));
let scheme_specific = &value[scheme.len() + 1..];
if !valid_scheme || scheme_specific.is_empty() || scheme_specific.starts_with('#') {
return Err(invalid_url_err(value));
}
return Ok(None);
opaque = !scheme_specific.starts_with('/');
}

// java.net.URI rejects an empty authority ("//", "http://") unless a path,
// query or fragment follows it ("Expected authority").
if authority == Some("") && file.is_empty() && fragment.is_none() {
return Err(invalid_url_err(value));
}

if fragment.is_some_and(|f| f.contains('#')) {
return Err(invalid_url_err(value));
}

// An opaque URI (mailto:x) has only a scheme, a scheme-specific part and a fragment.
if opaque {
return Ok(match part {
"PROTOCOL" => scheme,
"REF" => fragment,
_ => None,
});
}

// java.net.URI allows '[' and ']' in a query or fragment, but not in a hierarchical path.
if path.contains(['[', ']']) {
return Err(invalid_url_err(value));
}
Comment on lines +195 to 237

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

scheme.is_none() && is redundant, since decompose_uri only reports a scheme when the : is at an index above 0, so value.starts_with(':') is enough. In the second check, file is the path plus any ?query, so file.is_empty() && fragment.is_none() says the same thing. I read this from decompose_uri and have not run it.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, both simplified as you suggested.


// java.net.URI rejects unbalanced brackets in authority
Expand Down Expand Up @@ -795,4 +843,17 @@ mod tests {
let result = parse_url_component("http://[::1/path", "HOST", None);
assert!(result.is_err());
}

#[test]
fn test_invisible_chars() -> Result<()> {
// DEL, NBSP, NEL and U+2028 are illegal for java.net.URI; other non-ASCII is allowed.
for url in ["/p\u{7f}q", "/p\u{a0}q", "/p\u{85}q", "/p?a=\u{2028}"] {
assert!(parse_url_component(url, "PATH", None).is_err(), "{url:?}");
}
assert_eq!(
parse_url_component("/p\u{e9}q", "PATH", None)?,
Some("/p\u{e9}q")
);
Ok(())
}
}
96 changes: 96 additions & 0 deletions spark/src/test/resources/sql-tests/expressions/url/parse_url.sql
Original file line number Diff line number Diff line change
Expand Up @@ -170,3 +170,99 @@ SELECT parse_url('http://host:abc/', 'AUTHORITY')
-- unbalanced IPv6 bracket is invalid
query
SELECT parse_url('http://[::1/path', 'AUTHORITY')

-- relative references (no scheme) are parsed like java.net.URI
statement
CREATE TABLE test_parse_url_relative(url string) USING parquet

statement
INSERT INTO test_parse_url_relative VALUES
('/'),
('notaurl'),
('a/b:c?id=1'),
('?id=7'),
('#frag'),
('/p#a?id=1'),
('/shop/item/123?ref=abc&id=42#top'),
('/r?to=http://x.com/a&id=1'),
('//example.com'),
('//user:pw@example.com:8080/p?id=1#f'),
('///p?id=3'),
('//#f'),
('//?id=4'),
(':'),
(':foo'),
('//'),
('http://'),
('http://example.com/r?to=https://y.com&id=9'),
(''),
(NULL)

query
SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'),
parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'),
parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id')
FROM test_parse_url_relative

-- characters java.net.URI rejects, and nearby ones it accepts
statement
CREATE TABLE test_parse_url_chars(url string) USING parquet

statement
INSERT INTO test_parse_url_chars VALUES
('/p|q'),
('?a=b|c'),
('/p"q'),
('/p\\q'),
('/p^q'),
('/p`q'),
('/p%zz'),
('/p%4'),
('/p%41?id=%2F'),
('/p[1]'),
('?a=[1]#[b]'),
('#a#b'),
('/p?q#f?'),
('/péq?id=é'),
('http://example.com/p|q'),
('http://example.com/p?id=%zz'),
('http://example.com/p[1]'),
('http://example.com/p?id=[1]'),
('http://example.com/p#a#b'),
('http://[::1]:8080/p'),
(NULL)

query
SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'),
parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'),
parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id')
FROM test_parse_url_chars

-- scheme names and opaque URIs (scheme-specific part not starting with '/')
statement
CREATE TABLE test_parse_url_schemes(url string) USING parquet

statement
INSERT INTO test_parse_url_schemes VALUES
('mailto:x'),
('mailto:a[b]'),
('mailto:x#f'),
('mailto:x#f#g'),
('urn:isbn:123'),
('http:p?id=1'),
('http:?id=1'),
('http:'),
('http:#f'),
('user_id:5'),
('5:30'),
('1http://example.com/p'),
('h.t-t+p://example.com/p?id=1'),
('HTTP://example.com/p'),
('http:/p?id=1'),
(NULL)

query
SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'),
parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'),
parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id')
FROM test_parse_url_schemes
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,41 @@ SELECT parse_url('', 'HOST')
-- unbalanced IPv6 bracket throws in ANSI mode
query expect_error(http://[::1/path)
SELECT parse_url('http://[::1/path', 'AUTHORITY')

-- relative references do not throw in ANSI mode
query
SELECT parse_url('/shop?ref=abc&id=42', 'QUERY', 'id')

query
SELECT parse_url('//example.com/p', 'HOST')

-- a leading ':' or an empty authority with nothing after it throws in ANSI mode
query expect_error(:foo)
SELECT parse_url(':foo', 'PATH')

query expect_error(http://)
SELECT parse_url('http://', 'PROTOCOL')

-- characters java.net.URI rejects throw in ANSI mode
query expect_error(/p|q)
SELECT parse_url('/p|q', 'PATH')

query expect_error(/p%zz)
SELECT parse_url('/p%zz', 'PATH')

query expect_error(http://example.com/p[1])
SELECT parse_url('http://example.com/p[1]', 'HOST')

query expect_error(#a#b)
SELECT parse_url('#a#b', 'REF')

-- an invalid scheme name or an empty scheme-specific part throws in ANSI mode
query expect_error(user_id:5)
SELECT parse_url('user_id:5', 'PROTOCOL')

query expect_error(http:)
SELECT parse_url('http:', 'PROTOCOL')

-- an opaque URI does not throw in ANSI mode
query
SELECT parse_url('mailto:x#f', 'REF')