diff --git a/native/spark-expr/src/url_funcs/parse_url.rs b/native/spark-expr/src/url_funcs/parse_url.rs index c60ed1247a8..4a005f8ff91 100644 --- a/native/spark-expr/src/url_funcs/parse_url.rs +++ b/native/spark-expr/src/url_funcs/parse_url.rs @@ -99,9 +99,20 @@ fn extract_query_value<'a>(query: &'a str, key: &str) -> Option<&'a str> { }) } +fn is_escape_at(bytes: &[u8], i: usize) -> bool { + matches!(bytes.get(i + 1..i + 3), Some([h, l]) if h.is_ascii_hexdigit() && l.is_ascii_hexdigit()) +} + +// Characters java.net.URI rejects in every component, plus '%' not followed by two hex digits. fn has_invalid_uri_chars(s: &str) -> bool { - s.bytes() - .any(|b| b == b' ' || b == b'{' || b == b'}' || b == b'<' || b == b'>' || b < 0x20) + let bytes = s.as_bytes(); + let bad_byte = bytes.iter().enumerate().any(|(i, &b)| match b { + b' ' | b'"' | b'<' | b'>' | b'\\' | b'^' | b'`' | b'{' | b'|' | b'}' | 0x7f => true, + b'%' => !is_escape_at(bytes, i), + _ => b < 0x20, + }); + // Non-ASCII characters are allowed unless they are control or space characters. + bad_byte || (!s.is_ascii() && s.chars().any(|c| c.is_control() || c.is_whitespace())) } fn invalid_url_err(value: &str) -> datafusion::common::DataFusionError { @@ -181,11 +192,48 @@ fn parse_url_component<'a>( fragment, } = decompose_uri(value); - if scheme.is_none() { - if value.contains("://") { + // A value without a scheme is a relative reference; a leading ':' is an empty + // scheme name, which java.net.URI rejects ("Expected scheme name at index 0"). + if value.starts_with(':') { + return Err(invalid_url_err(value)); + } + + // java.net.URI: scheme = alpha *( alpha | digit | "+" | "-" | "." ), followed by a + // non-empty scheme-specific part; one not starting with '/' makes the URI opaque. + let mut opaque = false; + if let Some(scheme) = scheme { + let mut bytes = scheme.bytes(); + let valid_scheme = bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) + && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')); + let scheme_specific = &value[scheme.len() + 1..]; + if !valid_scheme || scheme_specific.is_empty() || scheme_specific.starts_with('#') { return Err(invalid_url_err(value)); } - return Ok(None); + opaque = !scheme_specific.starts_with('/'); + } + + // java.net.URI rejects an empty authority ("//", "http://") unless a path, + // query or fragment follows it ("Expected authority"). + if authority == Some("") && file.is_empty() && fragment.is_none() { + return Err(invalid_url_err(value)); + } + + if fragment.is_some_and(|f| f.contains('#')) { + return Err(invalid_url_err(value)); + } + + // An opaque URI (mailto:x) has only a scheme, a scheme-specific part and a fragment. + if opaque { + return Ok(match part { + "PROTOCOL" => scheme, + "REF" => fragment, + _ => None, + }); + } + + // java.net.URI allows '[' and ']' in a query or fragment, but not in a hierarchical path. + if path.contains(['[', ']']) { + return Err(invalid_url_err(value)); } // java.net.URI rejects unbalanced brackets in authority @@ -795,4 +843,17 @@ mod tests { let result = parse_url_component("http://[::1/path", "HOST", None); assert!(result.is_err()); } + + #[test] + fn test_invisible_chars() -> Result<()> { + // DEL, NBSP, NEL and U+2028 are illegal for java.net.URI; other non-ASCII is allowed. + for url in ["/p\u{7f}q", "/p\u{a0}q", "/p\u{85}q", "/p?a=\u{2028}"] { + assert!(parse_url_component(url, "PATH", None).is_err(), "{url:?}"); + } + assert_eq!( + parse_url_component("/p\u{e9}q", "PATH", None)?, + Some("/p\u{e9}q") + ); + Ok(()) + } } diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql index fc87e3d77c2..58cc1ddc2af 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql @@ -170,3 +170,99 @@ SELECT parse_url('http://host:abc/', 'AUTHORITY') -- unbalanced IPv6 bracket is invalid query SELECT parse_url('http://[::1/path', 'AUTHORITY') + +-- relative references (no scheme) are parsed like java.net.URI +statement +CREATE TABLE test_parse_url_relative(url string) USING parquet + +statement +INSERT INTO test_parse_url_relative VALUES + ('/'), + ('notaurl'), + ('a/b:c?id=1'), + ('?id=7'), + ('#frag'), + ('/p#a?id=1'), + ('/shop/item/123?ref=abc&id=42#top'), + ('/r?to=http://x.com/a&id=1'), + ('//example.com'), + ('//user:pw@example.com:8080/p?id=1#f'), + ('///p?id=3'), + ('//#f'), + ('//?id=4'), + (':'), + (':foo'), + ('//'), + ('http://'), + ('http://example.com/r?to=https://y.com&id=9'), + (''), + (NULL) + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_relative + +-- characters java.net.URI rejects, and nearby ones it accepts +statement +CREATE TABLE test_parse_url_chars(url string) USING parquet + +statement +INSERT INTO test_parse_url_chars VALUES + ('/p|q'), + ('?a=b|c'), + ('/p"q'), + ('/p\\q'), + ('/p^q'), + ('/p`q'), + ('/p%zz'), + ('/p%4'), + ('/p%41?id=%2F'), + ('/p[1]'), + ('?a=[1]#[b]'), + ('#a#b'), + ('/p?q#f?'), + ('/péq?id=é'), + ('http://example.com/p|q'), + ('http://example.com/p?id=%zz'), + ('http://example.com/p[1]'), + ('http://example.com/p?id=[1]'), + ('http://example.com/p#a#b'), + ('http://[::1]:8080/p'), + (NULL) + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_chars + +-- scheme names and opaque URIs (scheme-specific part not starting with '/') +statement +CREATE TABLE test_parse_url_schemes(url string) USING parquet + +statement +INSERT INTO test_parse_url_schemes VALUES + ('mailto:x'), + ('mailto:a[b]'), + ('mailto:x#f'), + ('mailto:x#f#g'), + ('urn:isbn:123'), + ('http:p?id=1'), + ('http:?id=1'), + ('http:'), + ('http:#f'), + ('user_id:5'), + ('5:30'), + ('1http://example.com/p'), + ('h.t-t+p://example.com/p?id=1'), + ('HTTP://example.com/p'), + ('http:/p?id=1'), + (NULL) + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_schemes diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql index b3a0ed779da..d621ddf51ed 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql @@ -55,3 +55,41 @@ SELECT parse_url('', 'HOST') -- unbalanced IPv6 bracket throws in ANSI mode query expect_error(http://[::1/path) SELECT parse_url('http://[::1/path', 'AUTHORITY') + +-- relative references do not throw in ANSI mode +query +SELECT parse_url('/shop?ref=abc&id=42', 'QUERY', 'id') + +query +SELECT parse_url('//example.com/p', 'HOST') + +-- a leading ':' or an empty authority with nothing after it throws in ANSI mode +query expect_error(:foo) +SELECT parse_url(':foo', 'PATH') + +query expect_error(http://) +SELECT parse_url('http://', 'PROTOCOL') + +-- characters java.net.URI rejects throw in ANSI mode +query expect_error(/p|q) +SELECT parse_url('/p|q', 'PATH') + +query expect_error(/p%zz) +SELECT parse_url('/p%zz', 'PATH') + +query expect_error(http://example.com/p[1]) +SELECT parse_url('http://example.com/p[1]', 'HOST') + +query expect_error(#a#b) +SELECT parse_url('#a#b', 'REF') + +-- an invalid scheme name or an empty scheme-specific part throws in ANSI mode +query expect_error(user_id:5) +SELECT parse_url('user_id:5', 'PROTOCOL') + +query expect_error(http:) +SELECT parse_url('http:', 'PROTOCOL') + +-- an opaque URI does not throw in ANSI mode +query +SELECT parse_url('mailto:x#f', 'REF')