From d22832c56cc84602f413d1038273f7a156b9ec48 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:56:31 +0000 Subject: [PATCH 1/6] Fix json_get_array on dictionary-encoded JSON columns `return_type_check` promised a `Dictionary(Int64, )` return type whenever the first argument was a dictionary and the value type was non-primitive, but `invoke_array_array` / `invoke_array_scalars` only wrap the result back into a dictionary when the function's `InvokeResult::ACCEPT_DICT_RETURN` is set. `json_get_array` is the only UDF in the crate with a non-primitive value type (`List(Utf8)`) and `ACCEPT_DICT_RETURN = false`, so every call against a dictionary-encoded column failed the planned-vs-actual return type check. Make `return_type_check` generic over the `InvokeResult` implementation and key the dictionary wrapping off `R::ACCEPT_DICT_RETURN`, so the declared return type and the array `invoke` actually builds are derived from the same fact and cannot drift apart again. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- src/common.rs | 16 +++++++++++-- src/json_as_text.rs | 2 +- src/json_contains.rs | 2 +- src/json_get.rs | 2 +- src/json_get_array.rs | 2 +- src/json_get_bool.rs | 2 +- src/json_get_float.rs | 2 +- src/json_get_int.rs | 2 +- src/json_get_json.rs | 2 +- src/json_get_str.rs | 2 +- src/json_length.rs | 2 +- src/json_object_keys.rs | 2 +- tests/main.rs | 50 +++++++++++++++++++++++++++++++++++++++++ 13 files changed, 75 insertions(+), 13 deletions(-) diff --git a/src/common.rs b/src/common.rs index 6b48d7e..4194a27 100644 --- a/src/common.rs +++ b/src/common.rs @@ -18,13 +18,23 @@ use crate::common_union::{ /// General implementation of `ScalarUDFImpl::return_type`. /// +/// # Type parameters +/// +/// * `R` - the `InvokeResult` implementation the function passes to [`invoke`]; its +/// `ACCEPT_DICT_RETURN` decides whether a dictionary input produces a dictionary output, so reading +/// it here keeps the declared return type in step with what `invoke` actually builds +/// /// # Arguments /// /// * `args` - The arguments to the function /// * `fn_name` - The name of the function /// * `value_type` - The general return type of the function, might be wrapped in a dictionary depending /// on the first argument -pub fn return_type_check(args: &[DataType], fn_name: &str, value_type: DataType) -> DataFusionResult { +pub fn return_type_check( + args: &[DataType], + fn_name: &str, + value_type: DataType, +) -> DataFusionResult { let Some(first) = args.first() else { return plan_err!("The '{fn_name}' function requires one or more arguments."); }; @@ -43,7 +53,9 @@ pub fn return_type_check(args: &[DataType], fn_name: &str, value_type: DataType) ) } })?; - if first_dict_key_type.is_some() && !value_type.is_primitive() { + // this must mirror the dictionary handling in `invoke_array_array` and `invoke_array_scalars`, + // which wrap the result back into a dictionary if and only if `R::ACCEPT_DICT_RETURN` is set + if first_dict_key_type.is_some() && R::ACCEPT_DICT_RETURN { Ok(DataType::Dictionary(Box::new(DataType::Int64), Box::new(value_type))) } else { Ok(value_type) diff --git a/src/json_as_text.rs b/src/json_as_text.rs index 5ce0500..0f72e21 100644 --- a/src/json_as_text.rs +++ b/src/json_as_text.rs @@ -41,7 +41,7 @@ impl ScalarUDFImpl for JsonAsText { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Utf8) + return_type_check::(arg_types, self.name(), DataType::Utf8) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_contains.rs b/src/json_contains.rs index eca36d5..38ca7aa 100644 --- a/src/json_contains.rs +++ b/src/json_contains.rs @@ -44,7 +44,7 @@ impl ScalarUDFImpl for JsonContains { if arg_types.len() < 2 { plan_err!("The 'json_contains' function requires two or more arguments.") } else { - return_type_check(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + return_type_check::(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) } } diff --git a/src/json_get.rs b/src/json_get.rs index 0e04c79..38231b8 100644 --- a/src/json_get.rs +++ b/src/json_get.rs @@ -47,7 +47,7 @@ impl ScalarUDFImpl for JsonGet { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), JsonUnion::data_type()) + return_type_check::(arg_types, self.name(), JsonUnion::data_type()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_get_array.rs b/src/json_get_array.rs index 860fb3b..ea066a1 100644 --- a/src/json_get_array.rs +++ b/src/json_get_array.rs @@ -46,7 +46,7 @@ impl ScalarUDFImpl for JsonGetArray { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::List(Arc::new(list_item_field()))) + return_type_check::(arg_types, self.name(), DataType::List(Arc::new(list_item_field()))) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_get_bool.rs b/src/json_get_bool.rs index b7e7901..9b9667b 100644 --- a/src/json_get_bool.rs +++ b/src/json_get_bool.rs @@ -39,7 +39,7 @@ impl ScalarUDFImpl for JsonGetBool { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + return_type_check::(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_get_float.rs b/src/json_get_float.rs index 4076ee2..4a85eed 100644 --- a/src/json_get_float.rs +++ b/src/json_get_float.rs @@ -41,7 +41,7 @@ impl ScalarUDFImpl for JsonGetFloat { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Float64) + return_type_check::(arg_types, self.name(), DataType::Float64) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_get_int.rs b/src/json_get_int.rs index 7fc702a..bc62293 100644 --- a/src/json_get_int.rs +++ b/src/json_get_int.rs @@ -41,7 +41,7 @@ impl ScalarUDFImpl for JsonGetInt { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Int64) + return_type_check::(arg_types, self.name(), DataType::Int64) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_get_json.rs b/src/json_get_json.rs index 89d6622..b748ea5 100644 --- a/src/json_get_json.rs +++ b/src/json_get_json.rs @@ -43,7 +43,7 @@ impl ScalarUDFImpl for JsonGetJson { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Utf8) + return_type_check::(arg_types, self.name(), DataType::Utf8) } fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { diff --git a/src/json_get_str.rs b/src/json_get_str.rs index cdea9cf..cf7d846 100644 --- a/src/json_get_str.rs +++ b/src/json_get_str.rs @@ -39,7 +39,7 @@ impl ScalarUDFImpl for JsonGetStr { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::Utf8) + return_type_check::(arg_types, self.name(), DataType::Utf8) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_length.rs b/src/json_length.rs index c6fcbc8..0c43d25 100644 --- a/src/json_length.rs +++ b/src/json_length.rs @@ -41,7 +41,7 @@ impl ScalarUDFImpl for JsonLength { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check(arg_types, self.name(), DataType::UInt64) + return_type_check::(arg_types, self.name(), DataType::UInt64) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/src/json_object_keys.rs b/src/json_object_keys.rs index a9c1b9f..f304061 100644 --- a/src/json_object_keys.rs +++ b/src/json_object_keys.rs @@ -41,7 +41,7 @@ impl ScalarUDFImpl for JsonObjectKeys { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check( + return_type_check::( arg_types, self.name(), DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), diff --git a/tests/main.rs b/tests/main.rs index 7ee4bda..3bfac85 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -162,6 +162,56 @@ async fn test_json_get_array_with_path() { assert_eq!(value_repr, "[1, 2, 3]"); } +#[tokio::test] +async fn test_json_get_array_all_datatypes() { + for_all_json_datatypes(async |dt| { + // no path argument: the whole document is the array + let batches = run_query_datatype("select json_get_array(json_data) from test where name='list_foo'", dt) + .await + .unwrap(); + let (value_type, value_repr) = display_val(batches).await; + assert!( + matches!(value_type, DataType::List(_)), + "unexpected type {value_type} for {dt}" + ); + assert_eq!(value_repr, r#"["foo"]"#, "unexpected value for {dt}"); + + // with a path argument + let batches = run_query_datatype( + "select json_get_array(json_data, 'foo') from test where name='object_foo_array'", + dt, + ) + .await + .unwrap(); + let (value_type, value_repr) = display_val(batches).await; + assert!( + matches!(value_type, DataType::List(_)), + "unexpected type {value_type} for {dt}" + ); + assert_eq!(value_repr, "[1]", "unexpected value for {dt}"); + + // documents which are not arrays (or not JSON at all) come back as null + let expected = [ + "+------------------+--------------------------------------------+", + "| name | json_get_array(test.json_data,Utf8(\"foo\")) |", + "+------------------+--------------------------------------------+", + "| object_foo | |", + "| object_foo_array | [1] |", + "| object_foo_obj | |", + "| object_foo_null | |", + "| object_bar | |", + "| list_foo | |", + "| invalid_json | |", + "+------------------+--------------------------------------------+", + ]; + let batches = run_query_datatype("select name, json_get_array(json_data, 'foo') from test", dt) + .await + .unwrap(); + assert_batches_eq!(expected, &batches); + }) + .await; +} + #[tokio::test] async fn test_json_get_array_inner_field_json_metadata() { let sql = r#"select json_get_array('[{"a": 1}, {"b": 2}]') as v"#; From 7f7d5f395a8592563970d8ad620901870469329b Mon Sep 17 00:00:00 2001 From: Huaijin Date: Thu, 10 Sep 2026 23:09:24 +0800 Subject: [PATCH 2/6] fix cargo audit by upgrade jiter to 0.16.0 (#122) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 304872e..52fa633 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ rust-version = "1.94.0" datafusion = { version = "55", default-features = false, features = [ "sql", ] } -jiter = "0.15.0" +jiter = "0.16.0" log = "0.4" paste = "1" serde_json = "1" From 30bc9dfb394ad2c5348c929bdf762f3fbf0d18ce Mon Sep 17 00:00:00 2001 From: Peter L Date: Sat, 12 Sep 2026 00:50:56 +0930 Subject: [PATCH 3/6] Add the postgres `#>` and `#>>` path operators (#128) Co-authored-by: Claude Fable 5.1 --- README.md | 17 ++++ benches/main.rs | 90 ++++++++++++++++- src/common.rs | 181 ++++++++++++++++++++++++---------- src/json_length.rs | 15 +-- src/rewrite.rs | 12 ++- tests/main.rs | 235 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 487 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 61e8713..206aaaa 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,23 @@ SELECT id, json_col->'a' as json_col_a FROM test_table - [x] `->` operator - alias for `json_get` - [x] `->>` operator - alias for `json_as_text` - [x] `?` operator - alias for `json_contains` +- [x] `#>` operator - `json_get` over a path, so `a #> array['x', 'y']` is `a -> 'x' -> 'y'` +- [x] `#>>` operator - `json_as_text` over a path, so `a #>> array['x', 'y']` is `a -> 'x' ->> 'y'` + +The right-hand side of a path operator is a `text[]` list expression: + +```sql +select json_col #>> array['a', 'b'] from test_table; +select json_col #>> array['a', 'b']::text[] from test_table; +select json_col #>> $1::text[] from test_table; +``` + +Postgres's array literal syntax, `'{a,b}'`, is not supported. As in postgres, each element is +a key when it reaches an object and an index when it reaches an array, so +`array['items', '0', 'name']` walks to `items`, then element zero, then `name`. A list of +integers is a path of indices, and a negative index counts from the end of the array. The +`json_get*` and `json_as_text` functions accept the same list as a path, e.g. +`json_as_text(json_col, array['a', 'b'])`. ### Notes Cast expressions with `json_get` are rewritten to the appropriate method, e.g. diff --git a/benches/main.rs b/benches/main.rs index 521e0c6..4aafb94 100644 --- a/benches/main.rs +++ b/benches/main.rs @@ -6,7 +6,7 @@ use datafusion::arrow::array::{StringArray, StringViewArray}; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::logical_expr::ColumnarValue; use datafusion::{common::ScalarValue, logical_expr::ScalarFunctionArgs}; -use datafusion_functions_json::udfs::{json_contains_udf, json_get_str_udf}; +use datafusion_functions_json::udfs::{json_contains_udf, json_get_str_udf, json_length_udf}; fn bench_json_contains(b: &mut Bencher) { let json_contains = json_contains_udf(); @@ -134,7 +134,95 @@ fn bench_json_get_str_view_array(b: &mut Bencher) { }); } +/// A document with a 64 element array, so that stepping through it is measurable. +fn array_doc() -> String { + let items: Vec = (0..64).map(|i| format!(r#""v{i}""#)).collect(); + format!(r#"{{"a": [{}]}}"#, items.join(", ")) +} + +fn bench_array_path(b: &mut Bencher, udf: &datafusion::logical_expr::ScalarUDF, index: &ScalarValue) { + let args = &[ + ColumnarValue::Array(Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + array_doc(), + 1024, + )))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("a".to_string()))), + ColumnarValue::Scalar(index.clone()), + ]; + + let arg_fields = vec![ + Arc::new(Field::new("arg0", DataType::Utf8, false)), + Arc::new(Field::new("arg1", DataType::Utf8, false)), + Arc::new(Field::new("arg2", index.data_type(), false)), + ]; + + let return_field = Arc::new(Field::new( + "out", + udf.return_type(&[DataType::Utf8, DataType::Utf8, index.data_type()]) + .unwrap(), + false, + )); + + b.iter(|| { + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: 1024, + return_field: return_field.clone(), + config_options: Arc::new(datafusion::config::ConfigOptions::default()), + }) + .unwrap(); + }); +} + +fn bench_json_get_str_index(b: &mut Bencher) { + bench_array_path(b, &json_get_str_udf(), &ScalarValue::Int64(Some(32))); +} + +fn bench_json_get_str_index_last(b: &mut Bencher) { + bench_array_path(b, &json_get_str_udf(), &ScalarValue::Int64(Some(63))); +} + +fn bench_json_get_str_negative_index(b: &mut Bencher) { + bench_array_path(b, &json_get_str_udf(), &ScalarValue::Int64(Some(-32))); +} + +fn bench_json_get_str_text_index(b: &mut Bencher) { + bench_array_path(b, &json_get_str_udf(), &ScalarValue::Utf8(Some("32".to_string()))); +} + +fn bench_json_length_array(b: &mut Bencher) { + let args = &[ + ColumnarValue::Array(Arc::new(StringArray::from_iter_values(std::iter::repeat_n( + array_doc(), + 1024, + )))), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("a".to_string()))), + ]; + let arg_fields = vec![ + Arc::new(Field::new("arg0", DataType::Utf8, false)), + Arc::new(Field::new("arg1", DataType::Utf8, false)), + ]; + let return_field = Arc::new(Field::new("json_length", DataType::UInt64, false)); + let udf = json_length_udf(); + b.iter(|| { + udf.invoke_with_args(ScalarFunctionArgs { + args: args.to_vec(), + arg_fields: arg_fields.clone(), + number_rows: 1024, + return_field: return_field.clone(), + config_options: Arc::new(datafusion::config::ConfigOptions::default()), + }) + .unwrap(); + }); +} + fn criterion_benchmark(c: &mut Criterion) { + c.bench_function("json_get_str_index", bench_json_get_str_index); + c.bench_function("json_get_str_index_last", bench_json_get_str_index_last); + c.bench_function("json_get_str_negative_index", bench_json_get_str_negative_index); + c.bench_function("json_get_str_text_index", bench_json_get_str_text_index); + c.bench_function("json_length_array", bench_json_length_array); c.bench_function("json_contains", bench_json_contains); c.bench_function("json_get_str_scalar", bench_json_get_str_scalar); c.bench_function("json_get_str_array", bench_json_get_str_array); diff --git a/src/common.rs b/src/common.rs index 6b48d7e..a5393c7 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::str::Utf8Error; use std::sync::Arc; @@ -34,7 +35,7 @@ pub fn return_type_check(args: &[DataType], fn_name: &str, value_type: DataType) return plan_err!("Unexpected argument type to '{fn_name}' at position 1, expected a string, got {first:?}."); } args.iter().skip(1).enumerate().try_for_each(|(index, arg)| { - if is_str(arg) || is_int(arg) || dict_key_type(arg).is_some() { + if is_str(arg) || is_int(arg) || is_path_list(arg) || dict_key_type(arg).is_some() { Ok(()) } else { plan_err!( @@ -59,6 +60,13 @@ fn is_int(d: &DataType) -> bool { matches!(d, DataType::UInt64 | DataType::Int64) } +/// A list usable as a whole path, e.g. `array['a', 'b']` or `array[0, 1]`, with one step per +/// element. `Null` elements are what an unbound placeholder such as `array[$1]` has at +/// planning time. +fn is_path_list(d: &DataType) -> bool { + matches!(d, DataType::List(field) if is_str(field.data_type()) || is_int(field.data_type()) || field.data_type() == &DataType::Null) +} + fn dict_key_type(d: &DataType) -> Option { if let DataType::Dictionary(key, value) = d { if is_str(value) || is_json_union(value) { @@ -70,29 +78,37 @@ fn dict_key_type(d: &DataType) -> Option { #[derive(Debug)] pub enum JsonPath<'s> { - Key(&'s str), - Index(usize), + Key(Cow<'s, str>), + /// A negative index counts from the end of the array, as in postgres. + Index(isize), None, } +impl JsonPath<'_> { + fn into_owned(self) -> JsonPath<'static> { + match self { + JsonPath::Key(key) => JsonPath::Key(Cow::Owned(key.into_owned())), + JsonPath::Index(index) => JsonPath::Index(index), + JsonPath::None => JsonPath::None, + } + } +} + impl<'a> From<&'a str> for JsonPath<'a> { fn from(key: &'a str) -> Self { - JsonPath::Key(key) + JsonPath::Key(Cow::Borrowed(key)) } } impl From for JsonPath<'_> { fn from(index: u64) -> Self { - JsonPath::Index(usize::try_from(index).unwrap()) + isize::try_from(index).map_or(JsonPath::None, JsonPath::Index) } } impl From for JsonPath<'_> { fn from(index: i64) -> Self { - match usize::try_from(index) { - Ok(i) => Self::Index(i), - Err(_) => Self::None, - } + isize::try_from(index).map_or(JsonPath::None, JsonPath::Index) } } @@ -109,35 +125,49 @@ impl<'s> JsonPathArgs<'s> { return Ok(Self::Array(array)); } - path_args - .iter() - .enumerate() - .map(|(pos, arg)| match arg { - ColumnarValue::Scalar( - ScalarValue::Utf8(Some(s)) | ScalarValue::Utf8View(Some(s)) | ScalarValue::LargeUtf8(Some(s)), - ) => Ok(JsonPath::Key(s)), - ColumnarValue::Scalar(ScalarValue::UInt64(Some(i))) => Ok((*i).into()), - ColumnarValue::Scalar(ScalarValue::Int64(Some(i))) => Ok((*i).into()), - ColumnarValue::Scalar( - ScalarValue::Null - | ScalarValue::Utf8(None) - | ScalarValue::Utf8View(None) - | ScalarValue::LargeUtf8(None) - | ScalarValue::UInt64(None) - | ScalarValue::Int64(None), - ) => Ok(JsonPath::None), + let mut path = Vec::with_capacity(path_args.len()); + for (pos, arg) in path_args.iter().enumerate() { + match arg { + ColumnarValue::Scalar(ScalarValue::List(list)) => { + // a whole path as one list, e.g. `array['a', 'b']`, with one step per element. + // There are two levels here: a scalar list is a `ListArray` holding exactly + // one row, and `value(0)` unwraps that row into the array of its elements + if list.len() != 1 { + return exec_err!("Expected a scalar list as a JSON path, got {} rows.", list.len()); + } + let elements = list.value(0); + for i in 0..elements.len() { + let element = ScalarValue::try_from_array(&elements, i)?; + path.push(scalar_path(&element, pos)?.into_owned()); + } + } + ColumnarValue::Scalar(scalar) => path.push(scalar_path(scalar, pos)?), ColumnarValue::Array(_) => { // if there was a single arg, which is an array, handled above in the // split_first case. So this is multiple args of which one is an array - exec_err!("More than 1 path element is not supported when querying JSON using an array.") + return exec_err!("More than 1 path element is not supported when querying JSON using an array."); } - ColumnarValue::Scalar(arg) => exec_err!( - "Unexpected argument type at position {}, expected string or int, got {arg:?}.", - pos + 1 - ), - }) - .collect::>() - .map(JsonPathArgs::Scalars) + } + } + Ok(Self::Scalars(path)) + } +} + +/// One path step from a scalar: a string is a key, an integer an index, and a null a null step. +fn scalar_path(scalar: &ScalarValue, pos: usize) -> DataFusionResult> { + if scalar.is_null() { + return Ok(JsonPath::None); + } + if let Some(Some(key)) = scalar.try_as_str() { + return Ok(key.into()); + } + match scalar { + ScalarValue::UInt64(Some(i)) => Ok((*i).into()), + ScalarValue::Int64(Some(i)) => Ok((*i).into()), + other => exec_err!( + "Unexpected argument type at position {}, expected string or int, got {other:?}.", + pos + 1 + ), } } @@ -527,8 +557,8 @@ pub fn jiter_json_find<'j>(opt_json: Option<&'j str>, path: &[JsonPath]) -> Opti let mut jiter = Jiter::new(json_str.as_bytes()); let mut peek = jiter.peek().ok()?; for element in path { - match element { - JsonPath::Key(key) if peek == Peek::Object => { + match (element, peek) { + (JsonPath::Key(key), Peek::Object) => { let mut next_key = jiter.known_object().ok()??; while next_key != *key { @@ -538,24 +568,50 @@ pub fn jiter_json_find<'j>(opt_json: Option<&'j str>, path: &[JsonPath]) -> Opti peek = jiter.peek().ok()?; } - JsonPath::Index(index) if peek == Peek::Array => { - let mut array_item = jiter.known_array().ok()??; - - for _ in 0..*index { - jiter.known_skip(array_item).ok()?; - array_item = jiter.array_step().ok()??; - } - - peek = array_item; - } - _ => { - return None; - } + (JsonPath::Index(index), Peek::Array) => peek = jiter_array_step(&mut jiter, *index)?, + // as in postgres's path operators, a text element which reaches an array is + // used as an index if it spells one, e.g. `j #> '{a,0}'` + (JsonPath::Key(key), Peek::Array) => peek = jiter_array_step(&mut jiter, key.parse().ok()?)?, + _ => return None, } } Some((jiter, peek)) } +/// Step into the array `jiter` is positioned at and advance to element `index`. +/// +/// A negative index counts from the end. jiter only streams forward, so that case first +/// counts the elements on a clone of the parser; a non-negative index is a single pass. +fn jiter_array_step(jiter: &mut Jiter, index: isize) -> Option { + let index = if let Ok(index) = usize::try_from(index) { + index + } else { + let len = jiter_array_len(&mut jiter.clone()).ok()?; + len.checked_add_signed(index)? + }; + + let mut array_item = jiter.known_array().ok()??; + + for _ in 0..index { + jiter.known_skip(array_item).ok()?; + array_item = jiter.array_step().ok()??; + } + + Some(array_item) +} + +/// Count the elements of the array `jiter` is positioned at, consuming it. +pub(crate) fn jiter_array_len(jiter: &mut Jiter) -> Result { + let mut peek_opt = jiter.known_array()?; + let mut len = 0; + while let Some(peek) = peek_opt { + jiter.known_skip(peek)?; + len += 1; + peek_opt = jiter.array_step()?; + } + Ok(len) +} + macro_rules! get_err { () => { Err(GetError) @@ -596,3 +652,30 @@ fn mask_dictionary_keys(keys: &PrimitiveArray, type_ids: &[i8]) -> Pr } PrimitiveArray::new(keys.values().clone(), Some(null_mask.into())) } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::ListArray; + + #[test] + fn into_owned_keeps_non_key_steps() { + assert!(matches!(JsonPath::Index(-1).into_owned(), JsonPath::Index(-1))); + assert!(matches!(JsonPath::None.into_owned(), JsonPath::None)); + assert!(matches!(JsonPath::from("a").into_owned(), JsonPath::Key(Cow::Owned(k)) if k == "a")); + } + + #[test] + fn list_path_must_be_scalar() { + // two rows, `[[1], [2]]`, where a scalar list has one + let two_rows = ListArray::from_iter_primitive::([Some([Some(1)]), Some([Some(2)])]); + let args = [ColumnarValue::Scalar(ScalarValue::List(Arc::new(two_rows)))]; + // a single array argument is a column of keys and takes the other path, so add a key + let args_with_key = [args[0].clone(), ColumnarValue::Scalar(ScalarValue::from("k"))]; + let err = JsonPathArgs::extract_path(&args_with_key).unwrap_err(); + assert_eq!( + err.to_string(), + "Execution error: Expected a scalar list as a JSON path, got 2 rows." + ); + } +} diff --git a/src/json_length.rs b/src/json_length.rs index c6fcbc8..55f1254 100644 --- a/src/json_length.rs +++ b/src/json_length.rs @@ -6,7 +6,9 @@ use datafusion::common::{Result as DataFusionResult, ScalarValue}; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use jiter::Peek; -use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; +use crate::common::{ + get_err, invoke, jiter_array_len, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath, +}; use crate::common_macros::make_udf_function; make_udf_function!( @@ -99,16 +101,7 @@ impl InvokeResult for UInt64Array { fn jiter_json_length(opt_json: Option<&str>, path: &[JsonPath]) -> Result { if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { match peek { - Peek::Array => { - let mut peek_opt = jiter.known_array()?; - let mut length: u64 = 0; - while let Some(peek) = peek_opt { - jiter.known_skip(peek)?; - length += 1; - peek_opt = jiter.array_step()?; - } - Ok(length) - } + Peek::Array => Ok(jiter_array_len(&mut jiter)? as u64), Peek::Object => { let mut opt_key = jiter.known_object()?; diff --git a/src/rewrite.rs b/src/rewrite.rs index 1dea562..81f836e 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -107,6 +107,8 @@ enum JsonOperator { Arrow, LongArrow, Question, + HashArrow, + HashLongArrow, } impl TryFrom<&BinaryOperator> for JsonOperator { @@ -117,6 +119,8 @@ impl TryFrom<&BinaryOperator> for JsonOperator { BinaryOperator::Arrow => Ok(JsonOperator::Arrow), BinaryOperator::LongArrow => Ok(JsonOperator::LongArrow), BinaryOperator::Question => Ok(JsonOperator::Question), + BinaryOperator::HashArrow => Ok(JsonOperator::HashArrow), + BinaryOperator::HashLongArrow => Ok(JsonOperator::HashLongArrow), _ => Err(()), } } @@ -125,8 +129,10 @@ impl TryFrom<&BinaryOperator> for JsonOperator { impl From for Arc { fn from(op: JsonOperator) -> Arc { match op { - JsonOperator::Arrow => crate::udfs::json_get_udf(), - JsonOperator::LongArrow => crate::udfs::json_as_text_udf(), + // the path operators use the same functions as their single key counterparts, with + // the path passed as a list + JsonOperator::Arrow | JsonOperator::HashArrow => crate::udfs::json_get_udf(), + JsonOperator::LongArrow | JsonOperator::HashLongArrow => crate::udfs::json_as_text_udf(), JsonOperator::Question => crate::udfs::json_contains_udf(), } } @@ -138,6 +144,8 @@ impl std::fmt::Display for JsonOperator { JsonOperator::Arrow => write!(f, "->"), JsonOperator::LongArrow => write!(f, "->>"), JsonOperator::Question => write!(f, "?"), + JsonOperator::HashArrow => write!(f, "#>"), + JsonOperator::HashLongArrow => write!(f, "#>>"), } } } diff --git a/tests/main.rs b/tests/main.rs index 7ee4bda..1facea1 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -3106,3 +3106,238 @@ async fn test_json_union_to_text_arms() { .await; assert_eq!(repr, "true"); } + +// `#>` and `#>>`, the postgres JSON path operators. +// +// `a #> array['x', 'y']` is `a -> 'x' -> 'y'`, and `a #>> array['x', 'y']` is `a -> 'x' ->> 'y'`. + +const NESTED: &str = r#"'{"a": {"b": {"c": "x"}}}'"#; + +#[tokio::test] +async fn test_hash_long_arrow_path() { + let batches = run_query(&format!("select {NESTED} #>> array['a', 'b', 'c']")) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "x".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_single_element() { + let batches = run_query(&format!("select {NESTED} #>> array['a']")).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Utf8, r#"{"b": {"c": "x"}}"#.to_string()) + ); +} + +#[tokio::test] +async fn test_hash_arrow_path() { + // `#>` keeps the value as JSON, where `#>>` renders it as text + let batches = run_query(&format!("select {NESTED} #> array['a', 'b']")).await.unwrap(); + assert_eq!(display_val(batches).await.1, r#"{object={"c": "x"}}"#); +} + +#[tokio::test] +async fn test_hash_long_arrow_cast_array() { + let sql = format!("select {NESTED} #>> array['a', 'b', 'c']::text[]"); + let batches = run_query(&sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "x".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_missing_path() { + // a missing path is null, never an error + let batches = run_query(&format!("select {NESTED} #>> array['nope', 'nope']")) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); +} + +#[tokio::test] +async fn test_hash_long_arrow_dotted_key() { + // a key containing dots is a single element + let sql = r#"select '{"code.file.path": "main.py"}' #>> array['code.file.path']"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "main.py".to_string())); +} + +#[tokio::test] +async fn test_hash_arrow_equivalent_to_arrow_chain() { + let sql = format!("select ({NESTED} #>> array['a', 'b', 'c']) = ({NESTED} -> 'a' -> 'b' ->> 'c')"); + let batches = run_query(&sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Boolean, "true".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_integer_list() { + // a list of integers is a path of indices; a list is homogeneous, so a mixed path spells + // its indices as text and relies on the index-or-key rule instead + let batches = run_query("select '[[1, 2], [3, 4]]' #>> array[1, 0]").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "3".to_string())); + + let batches = run_query("select '[[1, 2], [3, 4]]' #>> array[-1, -1]").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "4".to_string())); + + let sql = r#"select '{"a": [{"b": "x"}]}' #>> array['a', '0', 'b']"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "x".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_null_element() { + // a null path element is null overall, as in postgres + let batches = run_query(&format!("select {NESTED} #>> array['a', null]")) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); +} + +#[tokio::test] +async fn test_hash_long_arrow_param() { + let params = || vec![ScalarValue::Utf8(Some("a".to_string()))]; + + // with a cast to `text[]`... + let sql = format!("select {NESTED} #>> array[$1]::text[]"); + let batches = run_query_params(&sql, &DataType::Utf8View, params()).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Utf8, r#"{"b": {"c": "x"}}"#.to_string()) + ); + + // ...and without it: the placeholder has no type until bound, and the list is accepted anyway + let sql = format!("select {NESTED} #>> array[$1]"); + let batches = run_query_params(&sql, &DataType::Utf8View, params()).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Utf8, r#"{"b": {"c": "x"}}"#.to_string()) + ); + + // mixed placeholders and literals + let sql = format!("select {NESTED} #>> array[$1, 'b', 'c']::text[]"); + let batches = run_query_params(&sql, &DataType::Utf8View, params()).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "x".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_list_param() { + // the whole path as one bound parameter + let sql = format!("select {NESTED} #>> $1::text[]"); + let path = ScalarValue::List(ScalarValue::new_list_nullable( + &[ + ScalarValue::Utf8(Some("a".to_string())), + ScalarValue::Utf8(Some("b".to_string())), + ], + &DataType::Utf8, + )); + let batches = run_query_params(&sql, &DataType::Utf8View, vec![path]).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Utf8, r#"{"c": "x"}"#.to_string()) + ); +} + +#[tokio::test] +async fn test_json_as_text_list_path() { + // the list form is a property of the functions, not just the operators + let sql = r#"select json_as_text('{"a": {"b": 1}}', array['a', 'b'])"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "1".to_string())); +} + +#[tokio::test] +async fn test_hash_long_arrow_list_column() { + // a list-valued column is one path per row, which is not supported + let err = run_query("select json_data #>> array[name] from test") + .await + .unwrap_err(); + assert!( + err.to_string() + .starts_with("Execution error: unexpected second argument type, expected string or int array, got List"), + "{err}" + ); +} + +#[tokio::test] +async fn test_plan_hash_long_arrow() { + // the path is passed through whole, and DataFusion folds a constant array into a list literal + let lines = logical_plan(r"explain select json_data #>> array['a', 'b']::text[] from test").await; + assert_eq!( + lines, + [ + "Projection: json_as_text(test.json_data, List([a, b])) AS json_data #>> make_array(Utf8(\"a\"), Utf8(\"b\"))", + " TableScan: test projection=[json_data]", + ] + ); + + let lines = logical_plan(r"explain select json_data #> array['a', 'b'] from test").await; + assert_eq!( + lines, + [ + "Projection: json_get(test.json_data, List([a, b])) AS json_data #> make_array(Utf8(\"a\"), Utf8(\"b\"))", + " TableScan: test projection=[json_data]", + ] + ); +} + +#[tokio::test] +async fn test_hash_long_arrow_index_or_key() { + // as in postgres, a path element is resolved against the value it reaches: an index + // for an array, a key for an object, even when the element spells an integer + let sql = r#"select '{"items": [{"n": "first"}, {"n": "second"}]}' #>> array['items', '1', 'n']"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "second".to_string())); + + let sql = r#"select '{"0": "v"}' #>> array['0']"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "v".to_string())); + + // a non-integer element reaching an array is null, as is an out of range index + let batches = run_query("select '[1, 2]' #>> array['x']").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); + + let batches = run_query("select '[1, 2]' #>> array['5']").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); + + // a negative index counts from the end, and one past the start is null + let batches = run_query("select '[1, 2]' #>> array['-1']").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "2".to_string())); + + let batches = run_query("select '[1, 2]' #>> array['-3']").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); +} + +#[tokio::test] +async fn test_long_arrow_text_index() { + // the same rule applies to a text key which reaches an array; postgres returns null + // here because its text `->` only looks at objects, so this is more lenient + let batches = run_query("select '[1, 2]' ->> '0'").await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "1".to_string())); +} + +#[tokio::test] +async fn test_json_get_unsigned_index() { + let batches = run_query("select '[1, 2, 3]' -> arrow_cast(1, 'UInt64')") + .await + .unwrap(); + assert_eq!(display_val(batches).await.1, "{int=2}"); +} + +#[tokio::test] +async fn test_json_get_negative_index() { + let batches = run_query("select '[1, 2, 3]' -> -1").await.unwrap(); + assert_eq!(display_val(batches).await.1, "{int=3}"); + + let batches = run_query("select '[1, 2, 3]' -> -3").await.unwrap(); + assert_eq!(display_val(batches).await.1, "{int=1}"); + + let batches = run_query("select '[1, 2, 3]' -> -4").await.unwrap(); + assert_eq!(display_val(batches).await.1, "{null=}"); + + let batches = run_query("select '[]' -> -1").await.unwrap(); + assert_eq!(display_val(batches).await.1, "{null=}"); + + let batches = run_query(r#"select json_get_str('{"a": ["x", "y"]}', 'a', -1)"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, "y".to_string())); +} From 2f68bf58093af694f11f9c3ec50fa181c0c52de9 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:23:13 -0500 Subject: [PATCH 4/6] Keep the cast when folding json_get, plus property tests for the rewriter (#130) Co-authored-by: Claude Opus 5 --- Cargo.toml | 1 + src/rewrite.rs | 72 ++-- tests/json_robustness.rs | 322 ++++++++++++++++++ tests/main.rs | 31 +- tests/rewrite_differential.rs | 614 ++++++++++++++++++++++++++++++++++ 5 files changed, 1007 insertions(+), 33 deletions(-) create mode 100644 tests/json_robustness.rs create mode 100644 tests/rewrite_differential.rs diff --git a/Cargo.toml b/Cargo.toml index 52fa633..3a4329b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ datafusion = { version = "55", default-features = false, features = [ "nested_expressions", "sql", ] } +proptest = "1.11.0" tokio = { version = "1.43", features = ["full"] } [lints.clippy] diff --git a/src/rewrite.rs b/src/rewrite.rs index 81f836e..8a6c890 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -23,38 +23,72 @@ impl FunctionRewrite for JsonFunctionRewriter { fn rewrite(&self, expr: Expr, _schema: &DFSchema, _config: &ConfigOptions) -> Result> { let transform = match &expr { - Expr::Cast(cast) => optimise_json_get_cast(cast), + Expr::Cast(cast) => optimise_json_get(cast.field.data_type(), &cast.expr).map(|folded| match folded { + Folded::Exact(accessor) => accessor, + Folded::Narrowing(accessor) => Expr::Cast(Cast { + expr: Box::new(accessor), + field: cast.field.clone(), + }), + }), Expr::ScalarFunction(func) => unnest_json_calls(func), _ => None, }; - Ok(transform.unwrap_or_else(|| Transformed::no(expr))) + Ok(transform.map_or_else(|| Transformed::no(expr), Transformed::yes)) } } -/// This replaces `get_json(foo, bar)::int` with `json_get_int(foo, bar)` so the JSON function can take care of -/// extracting the right value type from JSON without the need to materialize the JSON union. -fn optimise_json_get_cast(cast: &Cast) -> Option> { - let scalar_func = extract_scalar_function(&cast.expr)?; +/// The accessor call that replaces a `json_get` under a cast, and whether the cast is still needed. +enum Folded { + /// The accessor already returns the type the query asked for; the cast can go. + Exact(Expr), + /// The accessor returns a wider type; the cast has to stay or both the type of the expression + /// and, for a narrowing cast, its value would change. + Narrowing(Expr), +} + +/// Replace the `json_get` under a cast to `cast_to` with the typed accessor that reads that type +/// out of the JSON directly, so the JSON union never has to be materialized just to be cast away. +/// +/// The accessors return one Arrow type per JSON type — `json_get_int` is always `Int64` — which is +/// not necessarily the type that was asked for. Where it is, the cast is dropped and this is the +/// plain substitution it has always been. Where it is not, only the cast's *input* is replaced: +/// dropping the cast there would silently change the type of the expression, and for a narrowing +/// cast its value too, while keeping it costs one primitive-to-primitive cast and still never +/// materializes the union. +fn optimise_json_get(cast_to: &DataType, cast_expr: &Expr) -> Option { + let scalar_func = extract_scalar_function(cast_expr)?; if !is_json_get(scalar_func) { return None; } - let func = match cast.field.data_type() { - DataType::Boolean => crate::json_get_bool::json_get_bool_udf(), + let (func, returns) = typed_accessor(cast_to)?; + let accessor = Expr::ScalarFunction(ScalarFunction { + func, + args: scalar_func.args.clone(), + }); + Some(if returns == *cast_to { + Folded::Exact(accessor) + } else { + Folded::Narrowing(accessor) + }) +} + +/// The accessor that reads `cast_to` out of JSON, if there is one, and the type it returns. +fn typed_accessor(cast_to: &DataType) -> Option<(Arc, DataType)> { + Some(match cast_to { + DataType::Boolean => (crate::json_get_bool::json_get_bool_udf(), DataType::Boolean), DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => { - crate::json_get_float::json_get_float_udf() + (crate::json_get_float::json_get_float_udf(), DataType::Float64) + } + DataType::Int64 | DataType::Int32 => (crate::json_get_int::json_get_int_udf(), DataType::Int64), + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => { + (crate::json_get_str::json_get_str_udf(), DataType::Utf8) } - DataType::Int64 | DataType::Int32 => crate::json_get_int::json_get_int_udf(), - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => crate::json_get_str::json_get_str_udf(), _ => return None, - }; - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { - func, - args: scalar_func.args.clone(), - }))) + }) } // Replace nested JSON functions e.g. `json_get(json_get(col, 'foo'), 'bar')` with `json_get(col, 'foo', 'bar')` -fn unnest_json_calls(func: &ScalarFunction) -> Option> { +fn unnest_json_calls(func: &ScalarFunction) -> Option { if !matches!( func.func.name(), "json_get" @@ -81,10 +115,10 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { args.extend(outer_args_iter.cloned()); // See #23, unnest only when all lookup arguments are literals if args.iter().skip(1).all(|arg| matches!(arg, Expr::Literal(_, _))) { - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { + Some(Expr::ScalarFunction(ScalarFunction { func: func.func.clone(), args, - }))) + })) } else { None } diff --git a/tests/json_robustness.rs b/tests/json_robustness.rs new file mode 100644 index 0000000..fac5bf7 --- /dev/null +++ b/tests/json_robustness.rs @@ -0,0 +1,322 @@ +//! Properties that hold for every JSON UDF, checked over generated input. +//! +//! These are deliberately cheap and broad rather than targeted: they cover the accessors as a +//! group, so a change to any one of them — or to the shared scanning code in `common.rs` — +//! has to keep them true. +//! +//! * [`prop_no_panic_on_arbitrary_json`] — malformed input yields NULL, never a panic and +//! never an error. This is the bug shape that produced the `todo!("BigInt not supported +//! yet")` panic fixed in #124: a JSON integer too wide for jiter's fast path. +//! * [`prop_scalar_and_array_paths_agree`] — a literal argument is const-evaluated through +//! the `ScalarValue` path while a column goes through the array path. They are separate +//! code paths in every UDF here, and they have to agree. +//! * [`prop_input_encoding_does_not_change_result`] — `Utf8`, `LargeUtf8`, `Utf8View` and +//! dictionary-encoded input are four more separate paths that have to agree. + +use std::sync::Arc; + +use datafusion::arrow::array::{ + ArrayRef, DictionaryArray, Int32Array, LargeStringArray, RecordBatch, StringArray, StringViewArray, +}; +use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +/// Every UDF that takes a JSON document and a lookup path. +const PATH_FUNCS: &[&str] = &[ + "json_get", + "json_get_bool", + "json_get_float", + "json_get_int", + "json_get_json", + "json_get_str", + "json_get_array", + "json_as_text", + "json_contains", + "json_length", + "json_object_keys", +]; + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +/// Each case here runs a query per UDF, so the defaults are kept low enough that the suite +/// stays fast. `PROPTEST_CASES` still wins when it is set, which is how these get run as a +/// fuzzer: `PROPTEST_CASES=100000 cargo test --release --test json_robustness`. +fn config(cases: u32) -> ProptestConfig { + let mut config = ProptestConfig::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = cases; + } + config +} + +/// Build the single-row table `t` holding `json` in the given encoding. +fn set_doc(ctx: &SessionContext, json: &str, encoding: &DataType) { + let array: ArrayRef = match encoding { + DataType::Utf8 => Arc::new(StringArray::from(vec![json])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec![json])), + DataType::Utf8View => Arc::new(StringViewArray::from(vec![json])), + DataType::Dictionary(_, _) => Arc::new(DictionaryArray::::new( + Int32Array::from(vec![0]), + Arc::new(StringArray::from(vec![json])), + )), + other => panic!("unsupported JSON encoding {other}"), + }; + let schema = Schema::new(vec![Field::new(JSON_COLUMN, encoding.clone(), false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +const ENCODINGS: &[DataType] = &[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View]; + +fn dict_encoding() -> DataType { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) +} + +/// Run a single-row query, formatting the result. `Err` carries only the fact that the query +/// failed, so the properties are not brittle about error text. +async fn outcome(ctx: &SessionContext, sql: &str) -> std::result::Result { + let df = ctx.sql(sql).await.map_err(|e| e.to_string())?; + let batches = df.collect().await.map_err(|e| e.to_string())?; + let batch = &batches[0]; + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).map_err(|e| e.to_string())?; + Ok(format!( + "{}={}", + logical_type(batch.schema().field(0).data_type()), + formatter.value(0).try_to_string().map_err(|e| e.to_string())? + )) +} + +/// Dictionary encoding of the input carries through to the output, so compare the type it +/// wraps rather than the wrapper. +fn logical_type(data_type: &DataType) -> &DataType { + match data_type { + DataType::Dictionary(_, value) => value, + other => other, + } +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// Well-formed JSON, then mangled: truncated, or with a byte dropped. Most of the interesting +/// malformed input is "nearly valid", which random text never produces. +fn json_text() -> impl Strategy { + prop_oneof![ + 6 => valid_json(), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let cut = j.char_indices().nth(n).map_or(j.len(), |(i, _)| i); + j[..cut].to_string() + }), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let mut chars: Vec = j.chars().collect(); + if !chars.is_empty() { + chars.remove(n % chars.len()); + } + chars.into_iter().collect() + }), + 1 => ".{0,24}".prop_map(|s| s), + ] +} + +fn valid_json() -> impl Strategy { + let leaf = prop_oneof![ + Just("null".to_string()), + any::().prop_map(|b| b.to_string()), + any::().prop_map(|i| i.to_string()), + any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + "[a-zA-Z0-9 ._+-]{0,6}".prop_map(|s| format!("\"{s}\"")), + // integers and floats outside the fast paths + Just("123456789012345678901234567890".to_string()), + Just("-99999999999999999999999999999999".to_string()), + Just("1e309".to_string()), + Just("-1e-309".to_string()), + ]; + leaf.prop_recursive(3, 12, 3, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..3).prop_map(|items| format!("[{}]", items.join(", "))), + prop::collection::vec(("[a-c]{1,2}", inner), 0..3).prop_map(|entries| { + let body = entries + .into_iter() + .map(|(k, v)| format!("\"{k}\": {v}")) + .collect::>() + .join(", "); + format!("{{{body}}}") + }), + ] + }) +} + +/// A lookup path: string keys and array indices, rendered as SQL arguments. +fn path() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + "[a-c]{1,2}".prop_map(|k| format!("'{k}'")), + (0i64..4).prop_map(|i| i.to_string()), + ], + 0..3, + ) +} + +fn call(func: &str, target: &str, path: &[String]) -> String { + let args = std::iter::once(target.to_string()) + .chain(path.iter().cloned()) + .collect::>(); + format!("{func}({})", args.join(", ")) +} + +/// `json_object_keys` and `json_get_array` return lists, and `json_length` takes no path +/// beyond the document, but every function still accepts the same argument shape. +fn callable(func: &str, path: &[String]) -> bool { + // json_contains needs at least one path element to mean anything + !(func == "json_contains" && path.is_empty()) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// No input, however malformed, may panic or raise an error. The accessors report "not +/// found" as NULL, and invalid JSON is just another way of not finding anything. +#[test] +fn prop_no_panic_on_arbitrary_json() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let result = rt.block_on(outcome(&ctx, &sql)); + prop_assert!(result.is_ok(), "{sql}\n json: {json:?}\n {result:?}"); + } + }); +} + +/// A literal document is const-evaluated through each UDF's `ScalarValue` path; a column goes +/// through its array path. The two must not disagree. +#[test] +fn prop_scalar_and_array_paths_agree() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + let literal = format!("'{}'", json.replace('\'', "''")); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let from_column = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let from_literal = format!("select {} as v", call(func, &literal, &path)); + let column = rt.block_on(outcome(&ctx, &from_column)); + let scalar = rt.block_on(outcome(&ctx, &from_literal)); + prop_assert_eq!(&column, &scalar, + "\n column: {} => {:?}\n literal: {} => {:?}\n json: {:?}\n", + from_column, column, from_literal, scalar, json); + } + }); +} + +/// The four string encodings a JSON column can arrive in are four separate code paths, and +/// they must all produce the same answer. +/// +/// Dictionary-encoded input keeps its encoding in the output, so the dictionary wrapper is +/// stripped before comparing types. `json_get` is left out: it returns the JSON union, which +/// has no meaningful text form to compare — `json_as_text` and `json_get_json` cover the same +/// scanning code and do return text. `json_get_array` is left out too, but for a different +/// reason — see [`json_get_array_errors_on_dictionary_input`]. +#[test] +fn prop_input_encoding_does_not_change_result() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(48), |(json in json_text(), path in path())| { + for func in PATH_FUNCS { + if !callable(func, &path) || matches!(*func, "json_get" | "json_get_array") { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + + set_doc(&ctx, &json, &DataType::Utf8View); + let baseline = rt.block_on(outcome(&ctx, &sql)); + + for encoding in ENCODINGS.iter().cloned().chain(std::iter::once(dict_encoding())) { + set_doc(&ctx, &json, &encoding); + let got = rt.block_on(outcome(&ctx, &sql)); + prop_assert_eq!(&got, &baseline, + "\n {} in {} => {:?}\n in Utf8View => {:?}\n json: {:?}\n", + sql, encoding, got, baseline, json); + } + } + }); +} + +/// `json_get_array` is unusable on a dictionary-encoded JSON column: every call fails with an +/// internal error, whatever the document and whatever the path. +/// +/// `return_type_check` wraps the declared return type in `Dictionary(Int64, ..)` whenever the +/// first argument is a dictionary and the value type is not primitive. `json_get_array` is the +/// only UDF here whose value type is not primitive (`List(Utf8)`) *and* whose `InvokeResult` +/// sets `ACCEPT_DICT_RETURN = false`, so `invoke_array_array` hands back a bare `List` while +/// `return_type` promised a dictionary, and `DataFusion` rejects the mismatch. +/// +/// The fix is to stop promising the wrapper for this one function, i.e. teach +/// `return_type_check` about `ACCEPT_DICT_RETURN` (or unwrap the dictionary in +/// `JsonGetArray::return_type`). **Delete this test once that lands**, and drop +/// `json_get_array` from the exclusion in [`prop_input_encoding_does_not_change_result`], +/// which then covers it. +#[test] +fn json_get_array_errors_on_dictionary_input() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + for json in [r"[1, 2, 3]", r#"{"a": [1, 2]}"#, "null", "not json"] { + set_doc(&ctx, json, &dict_encoding()); + let err = rt + .block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect_err("json_get_array over a dictionary column should still be failing"); + // debug builds trip DataFusion's own assertion, release builds reach Arrow's schema + // check; both name the dictionary type that `return_type` promised and did not deliver + assert!( + err.contains("Dictionary(Int64, List("), + "unexpected error for {json:?}: {err}" + ); + + // the same document in a plain string column works + set_doc(&ctx, json, &DataType::Utf8View); + rt.block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect("json_get_array over a string column works"); + } +} diff --git a/tests/main.rs b/tests/main.rs index 1facea1..afe94ec 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -312,13 +312,13 @@ async fn test_json_get_str_null() { #[tokio::test] async fn test_json_get_no_path() { let batches = run_query(r#"select json_get('"foo"')::string"#).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, "foo".to_string())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, "foo".to_string())); let batches = run_query(r"select json_get('123')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "123".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "123".to_string())); let batches = run_query(r"select json_get('true')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] @@ -398,11 +398,11 @@ async fn test_json_get_out_of_range_int() { // casts of the null union stay null rather than erroring let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::string"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, String::new())); } #[tokio::test] @@ -524,19 +524,19 @@ async fn test_json_get_path() { async fn test_json_get_cast_int() { let sql = r#"select json_get('{"foo": 42}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); // floats not allowed let sql = r#"select json_get('{"foo": 4.2}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] async fn test_json_get_cast_int_path() { let sql = r#"select json_get('{"foo": [null, {"x": false, "bar": 73}}', 'foo', 1, 'bar')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "73".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "73".to_string())); } #[tokio::test] @@ -579,14 +579,17 @@ async fn test_json_get_float() { async fn test_json_get_cast_float() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::float"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!(display_val(batches).await, (DataType::Float32, "420.0".to_string())); } #[tokio::test] async fn test_json_get_cast_numeric() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::numeric"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!( + display_val(batches).await, + (DataType::Decimal128(38, 10), "420.0000000000".to_string()) + ); } #[tokio::test] @@ -1214,7 +1217,7 @@ async fn test_json_get_int_unnest() { async fn test_plan_json_get_int_unnest() { let sql = "explain select json_get(json_get(json_data, 'foo'), 0)::int v from test"; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS v", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS v", " TableScan: test projection=[json_data]", ]; @@ -1438,7 +1441,7 @@ async fn test_arrow_cast_int() { for_all_json_datatypes(async |dt| { let batches = run_query_datatype(sql, dt).await.unwrap(); assert_batches_eq!(expected, &batches); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); }) .await; } @@ -1448,7 +1451,7 @@ async fn test_plan_arrow_cast_int() { let lines = logical_plan(r"explain select (json_data->'foo')::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\")) AS json_data -> 'foo'", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\")) AS Int32) AS json_data -> 'foo'", " TableScan: test projection=[json_data]", ]; @@ -1575,7 +1578,7 @@ async fn test_plan_arrow_double_nested_cast() { let lines = logical_plan(r"explain select (json_data->'foo'->0)::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS json_data -> 'foo' -> 0", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS json_data -> 'foo' -> 0", " TableScan: test projection=[json_data]", ]; diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs new file mode 100644 index 0000000..55239e7 --- /dev/null +++ b/tests/rewrite_differential.rs @@ -0,0 +1,614 @@ +//! Differential tests for the expression folding done by `JsonFunctionRewriter`. +//! +//! The rewriter folds `CAST(json_get(x, 'k') AS BIGINT)` down to `json_get_int(x, 'k')` so the +//! JSON union is never materialized just to be cast away, and it flattens nested `json_get` +//! calls into a single multi-argument call. Both are optimizations, so both are only correct +//! if they are invisible to the caller. These tests generate input and check that. +//! +//! # Running the same query with and without the rewrite +//! +//! Function rewrites run in the *analyzer*, before the optimizer merges projections. A cast +//! applied to a column of a subquery therefore never sees a `json_get` underneath it and is +//! not folded — and then `merge_projection` inlines the `json_get` back under the cast. So +//! [`unfolded_sql`] gives the plan we would have had with no rewriter registered at all, +//! without needing a second `SessionContext`. [`barrier_really_blocks_folding`] keeps that +//! trick honest. +//! +//! For nested `json_get` flattening this gives a straightforward invariant, checked by +//! [`prop_unnest_is_invisible`]: the flattened and un-flattened plans must agree. +//! +//! For cast folding it does **not**: casting the JSON union is much weaker than the typed +//! accessors (`CAST(union AS Float64)` is NULL even for `42` — see +//! [`unfolded_union_cast_is_weaker`]), so the rewrite is deliberately *better* than the +//! expression it replaces and equality is the wrong assertion. What the fold does promise is +//! +//! ```text +//! CAST(json_get(x, 'k') AS T) == CAST(json_get_(x, 'k') AS T) +//! ``` +//! +//! i.e. "use the typed accessor, then apply the cast the user actually wrote". The +//! right-hand side is not itself folded — the rewriter only folds casts over `json_get` — so +//! it serves as the reference implementation. That is [`prop_fold_is_invisible`]. +//! +//! That is also, literally, what the rewriter emits: it replaces the cast's *input*, and drops +//! the cast only where the accessor already returns the type that was asked for. The tests keep +//! both sides anyway, because the reference is built from the SQL rather than from the rewriter, +//! so a change to which accessor a type maps to still shows up here. + +use std::sync::Arc; + +use datafusion::arrow::array::{RecordBatch, StringViewArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +// --------------------------------------------------------------------------------------- +// the cast matrix +// --------------------------------------------------------------------------------------- + +/// What the rewriter does with a given cast of a `json_get` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fold { + /// Left alone: the cast is applied to the JSON union. + None, + /// Folded to the typed accessor, whose return type *is* the type that was asked for, so + /// the cast is dropped entirely. + Exact, + /// Folded to the typed accessor, which returns a wider type, so the cast to the type that + /// was asked for is kept on top of it. The union is still never materialized. + Narrowing, +} + +/// A cast target: how it is spelled in SQL, the typed accessor the rewriter folds it to, and +/// what the fold does with it. +#[derive(Debug, Clone, Copy)] +struct Target { + sql: &'static str, + accessor: &'static str, + sql_cast: Fold, +} + +const TARGETS: &[Target] = &[ + Target { + sql: "bigint", + accessor: "json_get_int", + sql_cast: Fold::Exact, + }, + Target { + sql: "double", + accessor: "json_get_float", + sql_cast: Fold::Exact, + }, + Target { + sql: "boolean", + accessor: "json_get_bool", + sql_cast: Fold::Exact, + }, + // SQL `VARCHAR` is `Utf8View` in DataFusion, but `json_get_str` returns `Utf8`. + Target { + sql: "varchar", + accessor: "json_get_str", + sql_cast: Fold::Narrowing, + }, + // Narrowing targets: the type asked for is narrower than what the accessor returns, so the + // fold keeps the cast on top of the accessor. + Target { + sql: "int", + accessor: "json_get_int", + sql_cast: Fold::Narrowing, + }, + Target { + sql: "real", + accessor: "json_get_float", + sql_cast: Fold::Narrowing, + }, + Target { + sql: "decimal(10,2)", + accessor: "json_get_float", + sql_cast: Fold::Narrowing, + }, + // A type with no accessor, so it is not folded. + Target { + sql: "smallint", + accessor: "json_get_int", + sql_cast: Fold::None, + }, +]; + +/// The spellings of a cast that all mean the same thing to a user. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Spelling { + Cast, + DoubleColon, +} + +const SPELLINGS: &[Spelling] = &[Spelling::Cast, Spelling::DoubleColon]; + +impl Spelling { + /// Write `inner` cast to `target`. + fn apply(self, inner: &str, target: Target) -> String { + match self { + Spelling::Cast => format!("cast({inner} as {})", target.sql), + Spelling::DoubleColon => format!("({inner})::{}", target.sql), + } + } + + /// What the rewriter does with this spelling of this target. Every spelling here is a SQL + /// cast, so they all fold alike; the argument stays for the call sites that pair the two. + fn fold(self, target: Target) -> Fold { + let _ = self; + target.sql_cast + } +} + +// --------------------------------------------------------------------------------------- +// documents and SQL +// --------------------------------------------------------------------------------------- + +/// A JSON document and a lookup path into it that reaches the interesting value. +#[derive(Debug, Clone)] +struct Doc { + json: String, + path: Vec, +} + +impl Doc { + /// `json_get(, )`, the expression the rewriter folds. + fn json_get(&self) -> String { + self.call("json_get") + } + + /// The same lookup through a typed accessor, which the rewriter leaves alone. + fn accessor(&self, name: &str) -> String { + self.call(name) + } + + fn call(&self, func: &str) -> String { + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(self.path.iter().map(|p| path_arg(p))) + .collect::>(); + format!("{func}({})", args.join(", ")) + } + + /// `json_get(j, k1, .., k(n-1))`: the lookup for everything but the last path segment, + /// which is what an outer accessor call gets nested around. `None` for a one-element + /// path, where there is nothing to nest. + fn prefix_json_get(&self) -> Option { + let (_, prefix) = self.path.split_last()?; + if prefix.is_empty() { + return None; + } + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(prefix.iter().map(|p| path_arg(p))) + .collect::>(); + Some(format!("json_get({})", args.join(", "))) + } +} + +/// Render a path segment: integers stay integers so array indexing is exercised. +fn path_arg(segment: &str) -> String { + if segment.chars().all(|c| c.is_ascii_digit()) { + segment.to_string() + } else { + sql_str(segment) + } +} + +/// Quote a string as a SQL literal. +fn sql_str(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +/// Point the single-row table `t` at `json`. +/// +/// The document has to arrive as column data rather than as a SQL literal: `DataFusion` +/// const-evaluates a projection over literals down to a single scalar, which erases the very +/// plan shape these tests assert on. +fn set_doc(ctx: &SessionContext, json: &str) { + let schema = Schema::new(vec![Field::new(JSON_COLUMN, DataType::Utf8View, false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(StringViewArray::from(vec![json]))]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +fn select(expr: &str) -> String { + format!("select {expr} as v from {JSON_TABLE}") +} + +/// Wrap `expr` so the rewriter cannot see through to the `json_get` underneath. +/// +/// See the module docs: the analyzer sees a cast over a subquery column and leaves it alone, +/// then the optimizer inlines the `json_get` back underneath. The result evaluates as if no +/// rewrite had been registered. +fn unfolded_sql(inner: &str, wrap: impl FnOnce(&str) -> String) -> String { + format!("select {} as v from (select {inner} as x from {JSON_TABLE})", wrap("x")) +} + +// --------------------------------------------------------------------------------------- +// running +// --------------------------------------------------------------------------------------- + +/// The result of a query: coarse enough not to be brittle about error text, precise enough +/// to catch a changed type, a changed value, or a cast that should have failed but did not. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Outcome { + Value(DataType, String), + Error, +} + +impl std::fmt::Display for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Outcome::Error => write!(f, "ERROR"), + Outcome::Value(dt, v) if v.is_empty() => write!(f, "{dt}=NULL"), + Outcome::Value(dt, v) => write!(f, "{dt}={v}"), + } + } +} + +async fn outcome(ctx: &SessionContext, sql: &str) -> Outcome { + let Ok(df) = ctx.sql(sql).await else { + return Outcome::Error; + }; + let Ok(batches) = df.collect().await else { + return Outcome::Error; + }; + assert_eq!(batches.len(), 1, "expected one batch from {sql}"); + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 1, "expected one row from {sql}"); + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).unwrap(); + Outcome::Value( + batch.schema().field(0).data_type().clone(), + formatter.value(0).try_to_string().unwrap(), + ) +} + +async fn logical_plan(ctx: &SessionContext, sql: &str) -> String { + let batches = ctx + .sql(&format!("explain {sql}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + let options = FormatOptions::default(); + let formatter = ArrayFormatter::try_new(batches[0].column(1).as_ref(), &options).unwrap(); + formatter.value(0).try_to_string().unwrap() +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// A single JSON value, weighted towards the values where the typed accessors and Arrow's +/// cast kernels are most likely to disagree. +fn json_scalar() -> impl Strategy { + prop_oneof![ + 1 => Just("null".to_string()), + 2 => any::().prop_map(|b| b.to_string()), + 4 => any::().prop_map(|i| i.to_string()), + 3 => any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + 2 => "[a-zA-Z0-9 ._+-]{0,8}".prop_map(|s| format!("\"{s}\"")), + // numeric- and boolean-looking strings: the accessors are lenient here, Arrow is not + 2 => any::().prop_map(|i| format!("\"{i}\"")), + 1 => Just("\"true\"".to_string()), + // integers wider than i64, which take jiter's slow path + 1 => Just("123456789012345678901234567890".to_string()), + 1 => Just("-123456789012345678901234567890".to_string()), + // nested values, which the accessors report as absent rather than as a value + 1 => Just("[1, 2]".to_string()), + 1 => Just(r#"{"c": 1}"#.to_string()), + ] +} + +/// A JSON value nested somewhere, together with the path that reaches it. +fn doc() -> impl Strategy { + (json_scalar(), 0usize..4).prop_map(|(value, shape)| match shape { + 0 => Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }, + 1 => Doc { + json: format!(r#"{{"a": {{"b": {value}}}}}"#), + path: vec!["a".to_string(), "b".to_string()], + }, + 2 => Doc { + json: format!("[{value}]"), + path: vec!["0".to_string()], + }, + _ => Doc { + json: format!(r#"{{"a": [1, {value}]}}"#), + path: vec!["a".to_string(), "1".to_string()], + }, + }) +} + +/// A `(spelling, target)` pair the rewriter folds, exactly or with a cast kept on top. Either +/// way the fold has to be invisible. +fn folding_cast() -> impl Strategy { + let pairs: Vec<(Spelling, Target)> = SPELLINGS + .iter() + .flat_map(|&s| TARGETS.iter().map(move |&t| (s, t))) + .filter(|&(s, t)| s.fold(t) != Fold::None) + .collect(); + prop::sample::select(pairs) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// Folding must be indistinguishable from applying the same cast to the typed accessor — +/// which is exactly what the rewriter now emits for a narrowing cast, and what the exact case +/// simplifies down to. +#[test] +fn prop_fold_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc(), (spelling, target) in folding_cast())| { + set_doc(&ctx, &doc.json); + let folded = select(&spelling.apply(&doc.json_get(), target)); + let reference = select(&spelling.apply(&doc.accessor(target.accessor), target)); + + let got = rt.block_on(outcome(&ctx, &folded)); + let want = rt.block_on(outcome(&ctx, &reference)); + + prop_assert_eq!( + &got, &want, + "\n folded: {} => {}\n reference: {} => {}\n", folded, got, reference, want + ); + }); +} + +/// Flattening `json_get(json_get(j, 'a'), 'b')` into `json_get(j, 'a', 'b')` must not change +/// the answer. Here the plan without the rewrite really is the right reference, so this is +/// the plain with/without-optimizer differential the barrier was built for. +#[test] +fn prop_unnest_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc())| { + let Some(prefix) = doc.prefix_json_get() else { + return Ok(()); + }; + set_doc(&ctx, &doc.json); + + // `json_get` returns the JSON union, which formats as an opaque value, so read the + // result out through `json_as_text`. The rewriter flattens the `json_get` underneath + // it into the outer call; the barrier stops it from seeing the nesting at all. + let last = path_arg(doc.path.last().unwrap()); + let unnested = select(&format!("json_as_text({prefix}, {last})")); + let not_unnested = unfolded_sql(&prefix, |x| format!("json_as_text({x}, {last})")); + + // the comparison only means something while each side is the plan it claims to be + let plan = rt.block_on(logical_plan(&ctx, ¬_unnested)); + prop_assert!( + plan.contains("json_as_text(json_get("), + "barrier stopped blocking the unnest:\n{}\n{}", not_unnested, plan + ); + let unnested_plan = rt.block_on(logical_plan(&ctx, &unnested)); + prop_assert!( + !unnested_plan.contains("json_as_text(json_get("), + "expected the nested call to be flattened:\n{}\n{}", unnested, unnested_plan + ); + + let got = rt.block_on(outcome(&ctx, &unnested)); + let want = rt.block_on(outcome(&ctx, ¬_unnested)); + prop_assert_eq!(&got, &want, + "\n unnested: {} => {}\n not unnested: {} => {}\n", + unnested, got, not_unnested, want); + }); +} + +// --------------------------------------------------------------------------------------- +// plan shape: which spellings fold +// --------------------------------------------------------------------------------------- + +/// Every spelling of a cast a user might reach for folds to the same accessor, and the plan +/// keeps the cast exactly where the accessor's type is not the one that was asked for. +#[test] +fn every_cast_spelling_folds_as_documented() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = select(&spelling.apply(&doc.json_get(), target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + let fold = spelling.fold(target); + + assert_eq!( + plan.contains(target.accessor), + fold != Fold::None, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", + target.sql + ); + if fold == Fold::None { + continue; + } + assert!( + !plan.contains("json_get(t."), + "fold left the union accessor behind for {spelling:?} / {}\n{plan}", + target.sql + ); + // "CAST(json_get" also matches "TRY_CAST(json_get" + assert_eq!( + plan.contains(&format!("CAST({}(t.", target.accessor)), + fold == Fold::Narrowing, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } +} + +// --------------------------------------------------------------------------------------- +// narrowing casts +// --------------------------------------------------------------------------------------- + +/// Folding a cast whose type no accessor returns exactly still yields that type, and still +/// narrows. Keeping the cast on top of the accessor is what buys this; folding to the bare +/// accessor would hand back the accessor's wider type and skip the narrowing entirely. +#[test] +fn narrowing_cast_preserves_type_and_narrows() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + // (json value, sql type, CAST outcome) + let cases = [ + ("42", "int", "Int32=42"), + ("42", "real", "Float32=42.0"), + ("42", "decimal(10,2)", "Decimal128(10, 2)=42.00"), + (r#""abc""#, "varchar", "Utf8View=abc"), + // out of the target type's range: CAST fails + ("3000000000", "int", "ERROR"), + ("9223372036854775807", "int", "ERROR"), + ("3000000000", "decimal(10,2)", "ERROR"), + ]; + + for (value, sql_type, want_cast) in cases { + let target = TARGETS.iter().find(|t| t.sql == sql_type).unwrap(); + let doc = Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + let sql = select(&Spelling::Cast.apply(&doc.json_get(), *target)); + assert_eq!(rt.block_on(outcome(&ctx, &sql)).to_string(), want_cast, "{sql}"); + } +} + +/// The narrowing fold is not just correct, it is the whole point: the JSON union never gets +/// materialized, the plan reads the value straight out with the typed accessor and casts that. +#[test] +fn narrowing_cast_still_avoids_the_union() { + let rt = runtime(); + let ctx = create_context().unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + let target = TARGETS.iter().find(|t| t.sql == "int").unwrap(); + + let plan = rt.block_on(logical_plan( + &ctx, + &select(&Spelling::Cast.apply(&doc.json_get(), *target)), + )); + assert!( + plan.contains("CAST(json_get_int(t.j, Utf8(\"a\")) AS Int32)"), + "unexpected plan:\n{plan}" + ); +} + +/// Keeping the cast must not cost the accessor its trip to the leaf nodes. +/// +/// `JsonGet`'s `placement` moves a `json_get` over a column with literal path arguments towards +/// the leaves, which is where the work wants to happen for a filter. A cast sitting on top of the +/// accessor is exactly the sort of thing that could strand it higher up the plan, so pin it. +#[test] +fn narrowing_cast_still_reaches_the_leaf_nodes() { + let rt = runtime(); + let ctx = create_context().unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); + + for target in TARGETS.iter().filter(|t| t.sql_cast != Fold::None) { + let predicate = Spelling::Cast.apply(&format!("json_get({JSON_COLUMN}, 'a')"), *target); + let sql = format!("select {JSON_COLUMN} from {JSON_TABLE} where {predicate} is not null"); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + + let leaf_projection = plan + .lines() + .rfind(|line| line.trim_start().starts_with("Projection:")) + .unwrap_or_default(); + assert!( + leaf_projection.contains(&format!("{}(t.j", target.accessor)), + "accessor did not reach the leaf projection for {}\n{plan}", + target.sql + ); + } +} + +// --------------------------------------------------------------------------------------- +// the barrier, and why the naive differential does not work for casts +// --------------------------------------------------------------------------------------- + +/// The subquery barrier must actually stop the fold, or every test that uses it as the +/// "without the rewrite" reference silently becomes a tautology. +#[test] +fn barrier_really_blocks_folding() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = unfolded_sql(&doc.json_get(), |x| spelling.apply(x, target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + // the json_get survives un-folded, and the optimizer inlined it back under the + // cast, so this is one expression and not an extra projection + assert!( + plan.contains("CAST(json_get(t.") || plan.contains("TRY_CAST(json_get(t."), + "barrier no longer blocks the fold for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } +} + +/// Why cast folding cannot be checked by diffing against the un-folded plan: casting the JSON +/// union is much weaker than the typed accessors, so the two disagree by design. +/// +/// This is the reason [`prop_fold_is_invisible`] compares against the typed accessor instead. +#[test] +fn unfolded_union_cast_is_weaker() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + let target = TARGETS.iter().find(|t| t.sql == "double").unwrap(); + + set_doc(&ctx, &doc.json); + let folded = select(&Spelling::Cast.apply(&doc.json_get(), *target)); + let unfolded = unfolded_sql(&doc.json_get(), |x| Spelling::Cast.apply(x, *target)); + + assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), "Float64=42.0"); + // the union has no Float64 member holding an integer, so the cast yields NULL + assert_eq!(rt.block_on(outcome(&ctx, &unfolded)).to_string(), "Float64=NULL"); +} From 1f301186f548e26530dc373f0988b9fb368bb1b7 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:12:39 -0500 Subject: [PATCH 5/6] Drop the json_get_array dictionary bug-repro test now that it is fixed `json_get_array_errors_on_dictionary_input` asserted the bug fixed in this branch and asked to be deleted once the fix landed. Remove it and stop excluding `json_get_array` from `prop_input_encoding_does_not_change_result`, which now covers dictionary input for it. Co-Authored-By: Claude Opus 5 --- tests/json_robustness.rs | 51 +++------------------------------------- 1 file changed, 3 insertions(+), 48 deletions(-) diff --git a/tests/json_robustness.rs b/tests/json_robustness.rs index fac5bf7..f3a5fe8 100644 --- a/tests/json_robustness.rs +++ b/tests/json_robustness.rs @@ -246,11 +246,10 @@ fn prop_scalar_and_array_paths_agree() { /// The four string encodings a JSON column can arrive in are four separate code paths, and /// they must all produce the same answer. /// -/// Dictionary-encoded input keeps its encoding in the output, so the dictionary wrapper is +/// Dictionary-encoded input may keep its encoding in the output, so the dictionary wrapper is /// stripped before comparing types. `json_get` is left out: it returns the JSON union, which /// has no meaningful text form to compare — `json_as_text` and `json_get_json` cover the same -/// scanning code and do return text. `json_get_array` is left out too, but for a different -/// reason — see [`json_get_array_errors_on_dictionary_input`]. +/// scanning code and do return text. #[test] fn prop_input_encoding_does_not_change_result() { let rt = runtime(); @@ -258,7 +257,7 @@ fn prop_input_encoding_does_not_change_result() { proptest!(config(48), |(json in json_text(), path in path())| { for func in PATH_FUNCS { - if !callable(func, &path) || matches!(*func, "json_get" | "json_get_array") { + if !callable(func, &path) || *func == "json_get" { continue; } let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); @@ -276,47 +275,3 @@ fn prop_input_encoding_does_not_change_result() { } }); } - -/// `json_get_array` is unusable on a dictionary-encoded JSON column: every call fails with an -/// internal error, whatever the document and whatever the path. -/// -/// `return_type_check` wraps the declared return type in `Dictionary(Int64, ..)` whenever the -/// first argument is a dictionary and the value type is not primitive. `json_get_array` is the -/// only UDF here whose value type is not primitive (`List(Utf8)`) *and* whose `InvokeResult` -/// sets `ACCEPT_DICT_RETURN = false`, so `invoke_array_array` hands back a bare `List` while -/// `return_type` promised a dictionary, and `DataFusion` rejects the mismatch. -/// -/// The fix is to stop promising the wrapper for this one function, i.e. teach -/// `return_type_check` about `ACCEPT_DICT_RETURN` (or unwrap the dictionary in -/// `JsonGetArray::return_type`). **Delete this test once that lands**, and drop -/// `json_get_array` from the exclusion in [`prop_input_encoding_does_not_change_result`], -/// which then covers it. -#[test] -fn json_get_array_errors_on_dictionary_input() { - let rt = runtime(); - let ctx = create_context().unwrap(); - - for json in [r"[1, 2, 3]", r#"{"a": [1, 2]}"#, "null", "not json"] { - set_doc(&ctx, json, &dict_encoding()); - let err = rt - .block_on(outcome( - &ctx, - &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), - )) - .expect_err("json_get_array over a dictionary column should still be failing"); - // debug builds trip DataFusion's own assertion, release builds reach Arrow's schema - // check; both name the dictionary type that `return_type` promised and did not deliver - assert!( - err.contains("Dictionary(Int64, List("), - "unexpected error for {json:?}: {err}" - ); - - // the same document in a plain string column works - set_doc(&ctx, json, &DataType::Utf8View); - rt.block_on(outcome( - &ctx, - &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), - )) - .expect("json_get_array over a string column works"); - } -} From adacb32b427bc01bd9959bafa842bd34f65d9b48 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:23:11 -0500 Subject: [PATCH 6/6] Address review: drop dead Boolean maps, clarify docs, test array paths - `json_get_bool` and `json_contains` no longer need `.map(|_| DataType::Boolean)`: `return_type_check::` already declares a bare `Boolean` now that the dictionary decision reads `ACCEPT_DICT_RETURN`. - Document `ACCEPT_DICT_RETURN` properly (fixing a typo), and note on `return_type_check` that dictionary *scalar* inputs are still not re-wrapped (the existing FIXMEs in the scalar invoke paths). - Add `test_dict_get_array`, covering `json_get_array` over a dictionary JSON argument with a column path, i.e. the `invoke_array_array` branch. Co-Authored-By: Claude Opus 5 --- src/common.rs | 8 +++++--- src/json_contains.rs | 2 +- src/json_get_bool.rs | 2 +- tests/main.rs | 29 +++++++++++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/common.rs b/src/common.rs index 20f863f..a8a467b 100644 --- a/src/common.rs +++ b/src/common.rs @@ -22,8 +22,9 @@ use crate::common_union::{ /// # Type parameters /// /// * `R` - the `InvokeResult` implementation the function passes to [`invoke`]; its -/// `ACCEPT_DICT_RETURN` decides whether a dictionary input produces a dictionary output, so reading -/// it here keeps the declared return type in step with what `invoke` actually builds +/// `ACCEPT_DICT_RETURN` decides whether a dictionary input column produces a dictionary output, +/// so reading it here keeps the declared return type in step with what the array paths of +/// `invoke` build (dictionary *scalar* inputs are not yet re-wrapped, see the `FIXME`s below) /// /// # Arguments /// @@ -187,7 +188,8 @@ pub trait InvokeResult { type Item; type Builder; - // Whether the return type should is allowed to be a dictionary + /// Whether a dictionary-encoded JSON column produces a `Dictionary(Int64, _)` result; read by + /// both `return_type_check` (the declared type) and `invoke` (the built array), so they agree const ACCEPT_DICT_RETURN: bool; fn builder(capacity: usize) -> Self::Builder; diff --git a/src/json_contains.rs b/src/json_contains.rs index 38ca7aa..40a0794 100644 --- a/src/json_contains.rs +++ b/src/json_contains.rs @@ -44,7 +44,7 @@ impl ScalarUDFImpl for JsonContains { if arg_types.len() < 2 { plan_err!("The 'json_contains' function requires two or more arguments.") } else { - return_type_check::(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + return_type_check::(arg_types, self.name(), DataType::Boolean) } } diff --git a/src/json_get_bool.rs b/src/json_get_bool.rs index 9b9667b..597b745 100644 --- a/src/json_get_bool.rs +++ b/src/json_get_bool.rs @@ -39,7 +39,7 @@ impl ScalarUDFImpl for JsonGetBool { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check::(arg_types, self.name(), DataType::Boolean).map(|_| DataType::Boolean) + return_type_check::(arg_types, self.name(), DataType::Boolean) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { diff --git a/tests/main.rs b/tests/main.rs index ffd22b2..1f8b5c9 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -2150,6 +2150,35 @@ async fn test_dict_get_int() { .await; } +#[tokio::test] +async fn test_dict_get_array() { + // both calls take a column path, so both go through `invoke_array_array` with a dictionary + // JSON argument; `json_get_array` does not re-wrap its result, so the output is a plain list + let sql = "select json_get_array(json_get_json(json_data, str_key1), str_key2) v from dicts"; + #[rustfmt::skip] + let expected = [ + "+-----+", + "| v |", + "+-----+", + "| [0] |", + "| |", + "| |", + "| |", + "+-----+", + ]; + + for_all_json_datatypes(async |dt| { + let batches = run_query_datatype(sql, dt).await.unwrap(); + let value_type = batches[0].schema().field(0).data_type().clone(); + assert!( + matches!(value_type, DataType::List(_)), + "unexpected type {value_type} for {dt}" + ); + assert_batches_eq!(expected, &batches); + }) + .await; +} + async fn build_dict_schema() -> SessionContext { let mut builder = StringDictionaryBuilder::::new(); builder.append(r#"{"foo": "bar"}"#).unwrap();