From 778984ca49abbcaff3975b088ad187423faf661f Mon Sep 17 00:00:00 2001 From: Valentin Rul Date: Mon, 2 Nov 2026 10:19:04 +0000 Subject: [PATCH 1/2] fix: parse_url handles relative references like java.net.URI Co-Authored-By: Claude Opus 5.5 --- native/spark-expr/src/url_funcs/parse_url.rs | 63 +++++++++++++++++-- .../sql-tests/expressions/url/parse_url.sql | 30 +++++++++ .../expressions/url/parse_url_ansi.sql | 14 +++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/native/spark-expr/src/url_funcs/parse_url.rs b/native/spark-expr/src/url_funcs/parse_url.rs index c60ed1247a8..b4ff94302c0 100644 --- a/native/spark-expr/src/url_funcs/parse_url.rs +++ b/native/spark-expr/src/url_funcs/parse_url.rs @@ -181,11 +181,16 @@ fn parse_url_component<'a>( fragment, } = decompose_uri(value); - if scheme.is_none() { - if value.contains("://") { - return Err(invalid_url_err(value)); - } - return Ok(None); + // A value without a scheme is a relative reference; a leading ':' is an empty + // scheme name, which java.net.URI rejects ("Expected scheme name at index 0"). + if scheme.is_none() && value.starts_with(':') { + return Err(invalid_url_err(value)); + } + + // java.net.URI rejects an empty authority ("//", "http://") unless a path, + // query or fragment follows it ("Expected authority"). + if authority == Some("") && path.is_empty() && query.is_none() && fragment.is_none() { + return Err(invalid_url_err(value)); } // java.net.URI rejects unbalanced brackets in authority @@ -795,4 +800,52 @@ mod tests { let result = parse_url_component("http://[::1/path", "HOST", None); assert!(result.is_err()); } + + #[test] + fn test_relative_reference() -> Result<()> { + // Expected values are from Spark 4.1.1 (java.net.URI). + for (url, part, key, expected) in [ + ("/shop?ref=abc&id=42#top", "PATH", None, Some("/shop")), + ("/shop?ref=abc&id=42#top", "QUERY", Some("id"), Some("42")), + ("/shop?ref=abc&id=42#top", "REF", None, Some("top")), + ("/shop?ref=abc&id=42#top", "HOST", None, None), + ("?id=7", "FILE", None, Some("?id=7")), + ("notaurl", "PATH", None, Some("notaurl")), + ("a/b:c?id=1", "PATH", None, Some("a/b:c")), + ( + "//user:pw@example.com:8080/p", + "HOST", + None, + Some("example.com"), + ), + ( + "//user:pw@example.com:8080/p", + "USERINFO", + None, + Some("user:pw"), + ), + ("///p?id=3", "AUTHORITY", None, None), + ("//?id=4", "QUERY", None, Some("id=4")), + ("/r?to=http://x.com/a&id=1", "QUERY", Some("id"), Some("1")), + ] { + assert_eq!( + parse_url_component(url, part, key)?, + expected, + "{url} {part}" + ); + } + Ok(()) + } + + #[test] + fn test_invalid_relative_reference() { + for url in [":", ":foo", "://missing-scheme", "//", "http://"] { + for part in ["HOST", "PATH", "PROTOCOL", "FILE"] { + assert!( + parse_url_component(url, part, None).is_err(), + "{url} {part}" + ); + } + } + } } diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql index fc87e3d77c2..fe6a42871e5 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql @@ -170,3 +170,33 @@ SELECT parse_url('http://host:abc/', 'AUTHORITY') -- unbalanced IPv6 bracket is invalid query SELECT parse_url('http://[::1/path', 'AUTHORITY') + +-- relative references (no scheme) are parsed like java.net.URI +statement +CREATE TABLE test_parse_url_relative(url string) USING parquet + +statement +INSERT INTO test_parse_url_relative VALUES + ('/'), + ('notaurl'), + ('a/b:c?id=1'), + ('?id=7'), + ('#frag'), + ('/p#a?id=1'), + ('/shop/item/123?ref=abc&id=42#top'), + ('/r?to=http://x.com/a&id=1'), + ('//example.com'), + ('//user:pw@example.com:8080/p?id=1#f'), + ('///p?id=3'), + ('//#f'), + ('//?id=4'), + (':foo'), + ('//'), + ('http://'), + ('http://example.com/r?to=https://y.com&id=9') + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_relative diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql index b3a0ed779da..a8edbcb4f5d 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql @@ -55,3 +55,17 @@ SELECT parse_url('', 'HOST') -- unbalanced IPv6 bracket throws in ANSI mode query expect_error(http://[::1/path) SELECT parse_url('http://[::1/path', 'AUTHORITY') + +-- relative references do not throw in ANSI mode +query +SELECT parse_url('/shop?ref=abc&id=42', 'QUERY', 'id') + +query +SELECT parse_url('//example.com/p', 'HOST') + +-- a leading ':' or an empty authority with nothing after it throws in ANSI mode +query expect_error(:foo) +SELECT parse_url(':foo', 'PATH') + +query expect_error(http://) +SELECT parse_url('http://', 'PROTOCOL') From 81e5b3b2cac4f17d9276c7a658c6271046f42d69 Mon Sep 17 00:00:00 2001 From: Valentin Rul <33987698+valentinsoe@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:31:33 +0000 Subject: [PATCH 2/2] fix: parse_url rejects what java.net.URI rejects and handles opaque URIs Co-Authored-By: Claude Opus 5.5 --- native/spark-expr/src/url_funcs/parse_url.rs | 102 ++++++++++-------- .../sql-tests/expressions/url/parse_url.sql | 68 +++++++++++- .../expressions/url/parse_url_ansi.sql | 24 +++++ 3 files changed, 146 insertions(+), 48 deletions(-) diff --git a/native/spark-expr/src/url_funcs/parse_url.rs b/native/spark-expr/src/url_funcs/parse_url.rs index b4ff94302c0..4a005f8ff91 100644 --- a/native/spark-expr/src/url_funcs/parse_url.rs +++ b/native/spark-expr/src/url_funcs/parse_url.rs @@ -99,9 +99,20 @@ fn extract_query_value<'a>(query: &'a str, key: &str) -> Option<&'a str> { }) } +fn is_escape_at(bytes: &[u8], i: usize) -> bool { + matches!(bytes.get(i + 1..i + 3), Some([h, l]) if h.is_ascii_hexdigit() && l.is_ascii_hexdigit()) +} + +// Characters java.net.URI rejects in every component, plus '%' not followed by two hex digits. fn has_invalid_uri_chars(s: &str) -> bool { - s.bytes() - .any(|b| b == b' ' || b == b'{' || b == b'}' || b == b'<' || b == b'>' || b < 0x20) + let bytes = s.as_bytes(); + let bad_byte = bytes.iter().enumerate().any(|(i, &b)| match b { + b' ' | b'"' | b'<' | b'>' | b'\\' | b'^' | b'`' | b'{' | b'|' | b'}' | 0x7f => true, + b'%' => !is_escape_at(bytes, i), + _ => b < 0x20, + }); + // Non-ASCII characters are allowed unless they are control or space characters. + bad_byte || (!s.is_ascii() && s.chars().any(|c| c.is_control() || c.is_whitespace())) } fn invalid_url_err(value: &str) -> datafusion::common::DataFusionError { @@ -183,13 +194,45 @@ fn parse_url_component<'a>( // A value without a scheme is a relative reference; a leading ':' is an empty // scheme name, which java.net.URI rejects ("Expected scheme name at index 0"). - if scheme.is_none() && value.starts_with(':') { + if value.starts_with(':') { return Err(invalid_url_err(value)); } + // java.net.URI: scheme = alpha *( alpha | digit | "+" | "-" | "." ), followed by a + // non-empty scheme-specific part; one not starting with '/' makes the URI opaque. + let mut opaque = false; + if let Some(scheme) = scheme { + let mut bytes = scheme.bytes(); + let valid_scheme = bytes.next().is_some_and(|b| b.is_ascii_alphabetic()) + && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')); + let scheme_specific = &value[scheme.len() + 1..]; + if !valid_scheme || scheme_specific.is_empty() || scheme_specific.starts_with('#') { + return Err(invalid_url_err(value)); + } + opaque = !scheme_specific.starts_with('/'); + } + // java.net.URI rejects an empty authority ("//", "http://") unless a path, // query or fragment follows it ("Expected authority"). - if authority == Some("") && path.is_empty() && query.is_none() && fragment.is_none() { + if authority == Some("") && file.is_empty() && fragment.is_none() { + return Err(invalid_url_err(value)); + } + + if fragment.is_some_and(|f| f.contains('#')) { + return Err(invalid_url_err(value)); + } + + // An opaque URI (mailto:x) has only a scheme, a scheme-specific part and a fragment. + if opaque { + return Ok(match part { + "PROTOCOL" => scheme, + "REF" => fragment, + _ => None, + }); + } + + // java.net.URI allows '[' and ']' in a query or fragment, but not in a hierarchical path. + if path.contains(['[', ']']) { return Err(invalid_url_err(value)); } @@ -802,50 +845,15 @@ mod tests { } #[test] - fn test_relative_reference() -> Result<()> { - // Expected values are from Spark 4.1.1 (java.net.URI). - for (url, part, key, expected) in [ - ("/shop?ref=abc&id=42#top", "PATH", None, Some("/shop")), - ("/shop?ref=abc&id=42#top", "QUERY", Some("id"), Some("42")), - ("/shop?ref=abc&id=42#top", "REF", None, Some("top")), - ("/shop?ref=abc&id=42#top", "HOST", None, None), - ("?id=7", "FILE", None, Some("?id=7")), - ("notaurl", "PATH", None, Some("notaurl")), - ("a/b:c?id=1", "PATH", None, Some("a/b:c")), - ( - "//user:pw@example.com:8080/p", - "HOST", - None, - Some("example.com"), - ), - ( - "//user:pw@example.com:8080/p", - "USERINFO", - None, - Some("user:pw"), - ), - ("///p?id=3", "AUTHORITY", None, None), - ("//?id=4", "QUERY", None, Some("id=4")), - ("/r?to=http://x.com/a&id=1", "QUERY", Some("id"), Some("1")), - ] { - assert_eq!( - parse_url_component(url, part, key)?, - expected, - "{url} {part}" - ); + fn test_invisible_chars() -> Result<()> { + // DEL, NBSP, NEL and U+2028 are illegal for java.net.URI; other non-ASCII is allowed. + for url in ["/p\u{7f}q", "/p\u{a0}q", "/p\u{85}q", "/p?a=\u{2028}"] { + assert!(parse_url_component(url, "PATH", None).is_err(), "{url:?}"); } + assert_eq!( + parse_url_component("/p\u{e9}q", "PATH", None)?, + Some("/p\u{e9}q") + ); Ok(()) } - - #[test] - fn test_invalid_relative_reference() { - for url in [":", ":foo", "://missing-scheme", "//", "http://"] { - for part in ["HOST", "PATH", "PROTOCOL", "FILE"] { - assert!( - parse_url_component(url, part, None).is_err(), - "{url} {part}" - ); - } - } - } } diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql index fe6a42871e5..58cc1ddc2af 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url.sql @@ -190,13 +190,79 @@ INSERT INTO test_parse_url_relative VALUES ('///p?id=3'), ('//#f'), ('//?id=4'), + (':'), (':foo'), ('//'), ('http://'), - ('http://example.com/r?to=https://y.com&id=9') + ('http://example.com/r?to=https://y.com&id=9'), + (''), + (NULL) query SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') FROM test_parse_url_relative + +-- characters java.net.URI rejects, and nearby ones it accepts +statement +CREATE TABLE test_parse_url_chars(url string) USING parquet + +statement +INSERT INTO test_parse_url_chars VALUES + ('/p|q'), + ('?a=b|c'), + ('/p"q'), + ('/p\\q'), + ('/p^q'), + ('/p`q'), + ('/p%zz'), + ('/p%4'), + ('/p%41?id=%2F'), + ('/p[1]'), + ('?a=[1]#[b]'), + ('#a#b'), + ('/p?q#f?'), + ('/péq?id=é'), + ('http://example.com/p|q'), + ('http://example.com/p?id=%zz'), + ('http://example.com/p[1]'), + ('http://example.com/p?id=[1]'), + ('http://example.com/p#a#b'), + ('http://[::1]:8080/p'), + (NULL) + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_chars + +-- scheme names and opaque URIs (scheme-specific part not starting with '/') +statement +CREATE TABLE test_parse_url_schemes(url string) USING parquet + +statement +INSERT INTO test_parse_url_schemes VALUES + ('mailto:x'), + ('mailto:a[b]'), + ('mailto:x#f'), + ('mailto:x#f#g'), + ('urn:isbn:123'), + ('http:p?id=1'), + ('http:?id=1'), + ('http:'), + ('http:#f'), + ('user_id:5'), + ('5:30'), + ('1http://example.com/p'), + ('h.t-t+p://example.com/p?id=1'), + ('HTTP://example.com/p'), + ('http:/p?id=1'), + (NULL) + +query +SELECT url, parse_url(url, 'HOST'), parse_url(url, 'PATH'), parse_url(url, 'QUERY'), + parse_url(url, 'REF'), parse_url(url, 'PROTOCOL'), parse_url(url, 'FILE'), + parse_url(url, 'AUTHORITY'), parse_url(url, 'USERINFO'), parse_url(url, 'QUERY', 'id') +FROM test_parse_url_schemes diff --git a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql index a8edbcb4f5d..d621ddf51ed 100644 --- a/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql +++ b/spark/src/test/resources/sql-tests/expressions/url/parse_url_ansi.sql @@ -69,3 +69,27 @@ SELECT parse_url(':foo', 'PATH') query expect_error(http://) SELECT parse_url('http://', 'PROTOCOL') + +-- characters java.net.URI rejects throw in ANSI mode +query expect_error(/p|q) +SELECT parse_url('/p|q', 'PATH') + +query expect_error(/p%zz) +SELECT parse_url('/p%zz', 'PATH') + +query expect_error(http://example.com/p[1]) +SELECT parse_url('http://example.com/p[1]', 'HOST') + +query expect_error(#a#b) +SELECT parse_url('#a#b', 'REF') + +-- an invalid scheme name or an empty scheme-specific part throws in ANSI mode +query expect_error(user_id:5) +SELECT parse_url('user_id:5', 'PROTOCOL') + +query expect_error(http:) +SELECT parse_url('http:', 'PROTOCOL') + +-- an opaque URI does not throw in ANSI mode +query +SELECT parse_url('mailto:x#f', 'REF')