diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ffeba4..e193fad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,16 +12,17 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27) with: + toolchain: stable components: rustfmt, clippy - id: cache-rust - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 with: extra_args: --all-files --verbose env: @@ -33,10 +34,10 @@ jobs: outputs: MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: set up python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -61,20 +62,22 @@ jobs: RUST_VERSION: ${{ matrix.rust-version }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - - uses: dtolnay/rust-toolchain@master + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27) with: toolchain: ${{ matrix.rust-version }} - id: cache-rust - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - - uses: taiki-e/install-action@cargo-llvm-cov + - uses: taiki-e/install-action@0631aa6515c7d545823c67cfae7ef4fc7f490154 # v2.81.8 + with: + tool: cargo-llvm-cov - run: cargo llvm-cov --all-features --codecov --output-path codecov.json - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: codecov.json @@ -87,7 +90,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 with: jobs: ${{ toJSON(needs) }} @@ -98,12 +101,14 @@ jobs: environment: release steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: install rust stable - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27) + with: + toolchain: stable - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 - run: cargo publish env: diff --git a/.gitignore b/.gitignore index b471067..c3da986 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target Cargo.lock .idea +.history diff --git a/Cargo.toml b/Cargo.toml index 7e051ee..a1e6e05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafusion-functions-json" -version = "0.52.0" +version = "0.54.3" edition = "2021" description = "JSON functions for DataFusion" readme = "README.md" @@ -11,14 +11,17 @@ repository = "https://github.com/datafusion-contrib/datafusion-functions-json/" rust-version = "1.88.0" [dependencies] -datafusion = { version = "52", default-features = false, features = ["sql"] } -jiter = "0.12.0" +datafusion = { version = "54", default-features = false, features = [ + "sql", +] } +jiter = "0.15.0" log = "0.4" paste = "1" +serde_json = "1" [dev-dependencies] codspeed-criterion-compat = "2.6" -datafusion = { version = "52", default-features = false, features = [ +datafusion = { version = "54", default-features = false, features = [ "nested_expressions", "sql", ] } @@ -35,3 +38,13 @@ pedantic = { level = "deny", priority = -1 } [[bench]] name = "main" harness = false + +# [patch.crates-io] +# datafusion = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } diff --git a/src/common.rs b/src/common.rs index 58a3642..6b48d7e 100644 --- a/src/common.rs +++ b/src/common.rs @@ -181,6 +181,14 @@ pub fn invoke( } } +fn null_result(len: usize) -> DataFusionResult { + let mut builder = R::builder(len); + for _ in 0..len { + R::append_value(&mut builder, None); + } + R::finish(builder) +} + fn invoke_array_array( json_array: &ArrayRef, path_array: &ArrayRef, @@ -236,6 +244,7 @@ fn invoke_array_array( DataType::Utf8 => zip_apply::(json_array.as_string::(), path_array, jiter_find), DataType::LargeUtf8 => zip_apply::(json_array.as_string::(), path_array, jiter_find), DataType::Utf8View => zip_apply::(json_array.as_string_view(), path_array, jiter_find), + DataType::Null => null_result::(json_array.len()), other => { if let Some(string_array) = nested_json_array(json_array, is_object_lookup_array(path_array.data_type())) { zip_apply::(string_array, path_array, jiter_find) @@ -319,6 +328,7 @@ fn invoke_array_scalars( DataType::Utf8 => inner::(json_array.as_string::(), path, jiter_find), DataType::LargeUtf8 => inner::(json_array.as_string::(), path, jiter_find), DataType::Utf8View => inner::(json_array.as_string_view(), path, jiter_find), + DataType::Null => null_result::(json_array.len()), other => { if let Some(string_array) = nested_json_array(json_array, is_object_lookup(path)) { inner::(string_array, path, jiter_find) diff --git a/src/common_union.rs b/src/common_union.rs index 217042b..3061b25 100644 --- a/src/common_union.rs +++ b/src/common_union.rs @@ -9,6 +9,30 @@ use datafusion::arrow::datatypes::{DataType, Field, UnionFields, UnionMode}; use datafusion::arrow::error::ArrowError; use datafusion::common::ScalarValue; +/// Field metadata used to mark a `Utf8` field as containing raw JSON. +/// +/// Attach this to any Arrow `Field` whose values are JSON-encoded strings so +/// downstream consumers can recognize them as JSON rather than opaque text. +/// +/// Emits Arrow's canonical JSON extension type keys +/// (`ARROW:extension:name` = `arrow.json`, `ARROW:extension:metadata` = `{}`), +/// see . +/// +/// Also emits a legacy `is_json` = `true` key. This key predates this crate's +/// adoption of the canonical extension and is non-standard — no other Arrow +/// tool recognizes it. It is kept only for back-compat with existing +/// downstream consumers of this crate and will be removed in a future +/// release; new consumers should key off `ARROW:extension:name` instead. +#[must_use] +pub fn json_field_metadata() -> HashMap { + HashMap::from([ + ("ARROW:extension:name".to_string(), "arrow.json".to_string()), + ("ARROW:extension:metadata".to_string(), "{}".to_string()), + // Legacy, non-standard. Remove in a future release — see doc comment above. + ("is_json".to_string(), "true".to_string()), + ]) +} + pub fn is_json_union(data_type: &DataType) -> bool { match data_type { DataType::Union(fields, UnionMode::Sparse) => fields == &union_fields(), @@ -161,8 +185,6 @@ fn union_fields() -> UnionFields { static FIELDS: OnceLock = OnceLock::new(); FIELDS .get_or_init(|| { - let json_metadata: HashMap = - HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())]); UnionFields::from_iter([ (TYPE_ID_NULL, Arc::new(Field::new("null", DataType::Null, true))), (TYPE_ID_BOOL, Arc::new(Field::new("bool", DataType::Boolean, false))), @@ -171,11 +193,11 @@ fn union_fields() -> UnionFields { (TYPE_ID_STR, Arc::new(Field::new("str", DataType::Utf8, false))), ( TYPE_ID_ARRAY, - Arc::new(Field::new("array", DataType::Utf8, false).with_metadata(json_metadata.clone())), + Arc::new(Field::new("array", DataType::Utf8, false).with_metadata(json_field_metadata())), ), ( TYPE_ID_OBJECT, - Arc::new(Field::new("object", DataType::Utf8, false).with_metadata(json_metadata.clone())), + Arc::new(Field::new("object", DataType::Utf8, false).with_metadata(json_field_metadata())), ), ]) }) diff --git a/src/json_as_text.rs b/src/json_as_text.rs index bfb4cb1..5ce0500 100644 --- a/src/json_as_text.rs +++ b/src/json_as_text.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, StringArray, StringBuilder}; @@ -33,10 +32,6 @@ impl Default for JsonAsText { } impl ScalarUDFImpl for JsonAsText { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -56,6 +51,24 @@ impl ScalarUDFImpl for JsonAsText { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for StringArray { diff --git a/src/json_contains.rs b/src/json_contains.rs index 01b1120..eca36d5 100644 --- a/src/json_contains.rs +++ b/src/json_contains.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::BooleanBuilder; @@ -33,10 +32,6 @@ impl Default for JsonContains { } impl ScalarUDFImpl for JsonContains { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -60,6 +55,24 @@ impl ScalarUDFImpl for JsonContains { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for BooleanArray { diff --git a/src/json_from_scalar.rs b/src/json_from_scalar.rs index f83b71f..221bf55 100644 --- a/src/json_from_scalar.rs +++ b/src/json_from_scalar.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{Array, ArrayRef, AsArray, UnionArray}; @@ -35,10 +34,6 @@ impl Default for JsonFromScalar { } impl ScalarUDFImpl for JsonFromScalar { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } diff --git a/src/json_get.rs b/src/json_get.rs index 4c7f9bc..0e04c79 100644 --- a/src/json_get.rs +++ b/src/json_get.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::ArrayRef; @@ -39,10 +38,6 @@ impl Default for JsonGet { } impl ScalarUDFImpl for JsonGet { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -62,6 +57,24 @@ impl ScalarUDFImpl for JsonGet { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for JsonUnion { @@ -131,7 +144,14 @@ fn build_union(jiter: &mut Jiter, peek: Peek) -> Result match jiter.known_number(peek)? { NumberAny::Int(NumberInt::Int(value)) => Ok(JsonUnionField::Int(value)), - NumberAny::Int(NumberInt::BigInt(_)) => todo!("BigInt not supported yet"), + // jiter returns `BigInt` for any integer its fast path couldn't decode, which includes + // values that do fit in `i64`, hence the conversion attempt. Values genuinely outside + // `i64` range have no representation in the union, so they're returned as null rather + // than silently losing precision as a `Float`. + NumberAny::Int(NumberInt::BigInt(value)) => match i64::try_from(value) { + Ok(value) => Ok(JsonUnionField::Int(value)), + Err(_) => get_err!(), + }, NumberAny::Float(value) => Ok(JsonUnionField::Float(value)), }, } diff --git a/src/json_get_array.rs b/src/json_get_array.rs index 88c680d..860fb3b 100644 --- a/src/json_get_array.rs +++ b/src/json_get_array.rs @@ -1,14 +1,18 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, ListBuilder, StringBuilder}; -use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::{Result as DataFusionResult, ScalarValue}; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use jiter::Peek; use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; use crate::common_macros::make_udf_function; +use crate::common_union::json_field_metadata; + +fn list_item_field() -> Field { + Field::new("item", DataType::Utf8, true).with_metadata(json_field_metadata()) +} make_udf_function!( JsonGetArray, @@ -33,10 +37,6 @@ impl Default for JsonGetArray { } impl ScalarUDFImpl for JsonGetArray { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -46,15 +46,7 @@ impl ScalarUDFImpl for JsonGetArray { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check( - arg_types, - self.name(), - DataType::List(Arc::new(datafusion::arrow::datatypes::Field::new( - "item", - DataType::Utf8, - true, - ))), - ) + return_type_check(arg_types, self.name(), DataType::List(Arc::new(list_item_field()))) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { @@ -64,6 +56,24 @@ impl ScalarUDFImpl for JsonGetArray { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } #[derive(Debug)] @@ -78,7 +88,7 @@ impl InvokeResult for BuildArrayList { fn builder(capacity: usize) -> Self::Builder { let values_builder = StringBuilder::new(); - ListBuilder::with_capacity(values_builder, capacity) + ListBuilder::with_capacity(values_builder, capacity).with_field(list_item_field()) } fn append_value(builder: &mut Self::Builder, value: Option) { @@ -90,7 +100,7 @@ impl InvokeResult for BuildArrayList { } fn scalar(value: Option) -> ScalarValue { - let mut builder = ListBuilder::new(StringBuilder::new()); + let mut builder = ListBuilder::new(StringBuilder::new()).with_field(list_item_field()); if let Some(array_items) = value { for item in array_items { diff --git a/src/json_get_bool.rs b/src/json_get_bool.rs index 17be9b0..b7e7901 100644 --- a/src/json_get_bool.rs +++ b/src/json_get_bool.rs @@ -1,5 +1,3 @@ -use std::any::Any; - use datafusion::arrow::array::BooleanArray; use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DataFusionResult; @@ -32,10 +30,6 @@ impl Default for JsonGetBool { } impl ScalarUDFImpl for JsonGetBool { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -55,12 +49,34 @@ impl ScalarUDFImpl for JsonGetBool { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } fn jiter_json_get_bool(json_data: Option<&str>, path: &[JsonPath]) -> Result { if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { match peek { Peek::True | Peek::False => Ok(jiter.known_bool(peek)?), + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } _ => get_err!(), } } else { diff --git a/src/json_get_float.rs b/src/json_get_float.rs index aff252b..15b6418 100644 --- a/src/json_get_float.rs +++ b/src/json_get_float.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, Float64Array, Float64Builder}; @@ -33,10 +32,6 @@ impl Default for JsonGetFloat { } impl ScalarUDFImpl for JsonGetFloat { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -56,6 +51,24 @@ impl ScalarUDFImpl for JsonGetFloat { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for Float64Array { @@ -86,6 +99,10 @@ impl InvokeResult for Float64Array { fn jiter_json_get_float(json_data: Option<&str>, path: &[JsonPath]) -> Result { if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { match peek { + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } // numbers are represented by everything else in peek, hence doing it this way Peek::Null | Peek::True @@ -93,7 +110,6 @@ fn jiter_json_get_float(json_data: Option<&str>, path: &[JsonPath]) -> Result get_err!(), _ => match jiter.known_number(peek)? { diff --git a/src/json_get_int.rs b/src/json_get_int.rs index 5788957..f266d23 100644 --- a/src/json_get_int.rs +++ b/src/json_get_int.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, Int64Array, Int64Builder}; @@ -33,10 +32,6 @@ impl Default for JsonGetInt { } impl ScalarUDFImpl for JsonGetInt { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -56,6 +51,24 @@ impl ScalarUDFImpl for JsonGetInt { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for Int64Array { @@ -86,19 +99,23 @@ impl InvokeResult for Int64Array { fn jiter_json_get_int(json_data: Option<&str>, path: &[JsonPath]) -> Result { if let Some((mut jiter, peek)) = jiter_json_find(json_data, path) { match peek { - // numbers are represented by everything else in peek, hence doing it this way + Peek::String => { + let s = jiter.known_str()?; + s.parse::().map_err(|_| GetError) + } Peek::Null | Peek::True | Peek::False | Peek::Minus | Peek::Infinity | Peek::NaN - | Peek::String | Peek::Array | Peek::Object => get_err!(), _ => match jiter.known_int(peek)? { NumberInt::Int(i) => Ok(i), - NumberInt::BigInt(_) => get_err!(), + // jiter returns `BigInt` for any integer its fast path couldn't decode, which + // includes values that do fit in `i64`, hence the conversion attempt + NumberInt::BigInt(b) => i64::try_from(b).map_err(|_| GetError), }, } } else { diff --git a/src/json_get_json.rs b/src/json_get_json.rs index 5907b2b..89d6622 100644 --- a/src/json_get_json.rs +++ b/src/json_get_json.rs @@ -1,12 +1,15 @@ -use std::any::Any; +use std::sync::Arc; use datafusion::arrow::array::StringArray; -use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion::common::Result as DataFusionResult; -use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; +use datafusion::logical_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, JsonPath}; use crate::common_macros::make_udf_function; +use crate::common_union::json_field_metadata; make_udf_function!( JsonGetJson, @@ -31,10 +34,6 @@ impl Default for JsonGetJson { } impl ScalarUDFImpl for JsonGetJson { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -47,6 +46,14 @@ impl ScalarUDFImpl for JsonGetJson { return_type_check(arg_types, self.name(), DataType::Utf8) } + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { + let arg_types: Vec = args.arg_fields.iter().map(|f| f.data_type().clone()).collect(); + let return_type = self.return_type(&arg_types)?; + Ok(Arc::new( + Field::new(self.name(), return_type, true).with_metadata(json_field_metadata()), + )) + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { invoke::(&args.args, jiter_json_get_json) } @@ -54,6 +61,24 @@ impl ScalarUDFImpl for JsonGetJson { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } fn jiter_json_get_json(opt_json: Option<&str>, path: &[JsonPath]) -> Result { diff --git a/src/json_get_str.rs b/src/json_get_str.rs index 658f1e3..cdea9cf 100644 --- a/src/json_get_str.rs +++ b/src/json_get_str.rs @@ -1,5 +1,3 @@ -use std::any::Any; - use datafusion::arrow::array::StringArray; use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DataFusionResult; @@ -32,10 +30,6 @@ impl Default for JsonGetStr { } impl ScalarUDFImpl for JsonGetStr { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -55,6 +49,24 @@ impl ScalarUDFImpl for JsonGetStr { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } fn jiter_json_get_str(json_data: Option<&str>, path: &[JsonPath]) -> Result { diff --git a/src/json_length.rs b/src/json_length.rs index 8bd657d..c6fcbc8 100644 --- a/src/json_length.rs +++ b/src/json_length.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, UInt64Array, UInt64Builder}; @@ -33,10 +32,6 @@ impl Default for JsonLength { } impl ScalarUDFImpl for JsonLength { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -56,6 +51,24 @@ impl ScalarUDFImpl for JsonLength { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } impl InvokeResult for UInt64Array { diff --git a/src/json_object_keys.rs b/src/json_object_keys.rs index 8ea040d..a9c1b9f 100644 --- a/src/json_object_keys.rs +++ b/src/json_object_keys.rs @@ -1,4 +1,3 @@ -use std::any::Any; use std::sync::Arc; use datafusion::arrow::array::{ArrayRef, ListBuilder, StringBuilder}; @@ -33,10 +32,6 @@ impl Default for JsonObjectKeys { } impl ScalarUDFImpl for JsonObjectKeys { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.aliases[0].as_str() } @@ -60,6 +55,24 @@ impl ScalarUDFImpl for JsonObjectKeys { fn aliases(&self) -> &[String] { &self.aliases } + + fn placement( + &self, + args: &[datafusion::logical_expr::ExpressionPlacement], + ) -> datafusion::logical_expr::ExpressionPlacement { + // If the first argument is a column and the remaining arguments are literals (a path) + // then we can push this UDF down to the leaf nodes. + if args.len() >= 2 + && matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column) + && args[1..] + .iter() + .all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal)) + { + datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes + } else { + datafusion::logical_expr::ExpressionPlacement::KeepInPlace + } + } } /// Struct used to build a `ListArray` from the result of `jiter_json_object_keys`. diff --git a/src/json_union_to_text.rs b/src/json_union_to_text.rs new file mode 100644 index 0000000..d3e8ccf --- /dev/null +++ b/src/json_union_to_text.rs @@ -0,0 +1,176 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{Array, ArrayRef, StringViewBuilder, UnionArray}; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion::common::{exec_datafusion_err, exec_err, plan_err, Result as DataFusionResult}; +use datafusion::logical_expr::{ + ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, +}; + +use crate::common_macros::make_udf_function; +use crate::common_union::{is_json_union, json_field_metadata, JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE}; + +make_udf_function!( + JsonUnionToText, + json_union_to_text, + json_union, + "Flatten a JSON union value (produced by `json_get`) into its canonical JSON text" +); + +/// Flattens the heterogeneous JSON union that `json_get` produces into a single +/// `Utf8View` column of canonical JSON text: scalars render as `true` / `42` / +/// `1.5`, strings are JSON-quoted and escaped, and array/object arms (already raw +/// JSON text) pass through. A JSON `null` arm becomes a SQL `NULL`. +/// +/// Useful when a JSON-union column must be materialized somewhere that can't +/// represent an Arrow `Union` — e.g. the Parquet writer, which rejects unions +/// (`arrow_to_parquet_schema` panics with "See ARROW-8817."). +#[derive(Debug, PartialEq, Eq, Hash)] +pub(super) struct JsonUnionToText { + signature: Signature, + aliases: [String; 1], +} + +impl Default for JsonUnionToText { + fn default() -> Self { + Self { + // Exactly the JSON union — any other argument type is a planning error. + signature: Signature::exact(vec![JSON_UNION_DATA_TYPE.clone()], Volatility::Immutable), + aliases: ["json_union_to_text".to_string()], + } + } +} + +impl ScalarUDFImpl for JsonUnionToText { + fn name(&self) -> &str { + self.aliases[0].as_str() + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + match arg_types { + [t] if is_json_union(t) => Ok(DataType::Utf8View), + _ => plan_err!("json_union_to_text expects a single JSON-union argument, got {arg_types:?}"), + } + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { + let arg_types: Vec = args.arg_fields.iter().map(|f| f.data_type().clone()).collect(); + let return_type = self.return_type(&arg_types)?; + Ok(Arc::new( + Field::new(self.name(), return_type, true).with_metadata(json_field_metadata()), + )) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + let Some(arg) = args.args.into_iter().next() else { + return exec_err!("json_union_to_text expects one argument"); + }; + let array = arg.into_array(args.number_rows)?; + Ok(ColumnarValue::Array(json_union_to_text_array(&array)?)) + } + + fn aliases(&self) -> &[String] { + &self.aliases + } +} + +/// Encode a JSON-union array into a `Utf8View` array of canonical JSON text. +fn json_union_to_text_array(array: &ArrayRef) -> DataFusionResult { + let Some(union) = array.as_any().downcast_ref::() else { + return exec_err!("json_union_to_text expects a UnionArray argument"); + }; + let Some(encoder) = JsonUnionEncoder::from_union(union.clone()) else { + return exec_err!("json_union_to_text argument is not the JSON union type"); + }; + + let mut builder = StringViewBuilder::with_capacity(encoder.len()); + // Scalar arms are JSON-encoded with serde_json (string escaping, float + // formatting, …); the array/object arms already hold raw JSON text and pass + // through verbatim. + let mut scratch: Vec = Vec::new(); + for idx in 0..encoder.len() { + scratch.clear(); + let write_result = match encoder.get_value(idx) { + JsonUnionValue::JsonNull => { + builder.append_null(); + continue; + } + JsonUnionValue::Bool(b) => serde_json::to_writer(&mut scratch, &b), + JsonUnionValue::Int(i) => serde_json::to_writer(&mut scratch, &i), + JsonUnionValue::Float(f) => serde_json::to_writer(&mut scratch, &f), + JsonUnionValue::Str(s) => serde_json::to_writer(&mut scratch, s), + JsonUnionValue::Array(s) | JsonUnionValue::Object(s) => { + builder.append_value(s); + continue; + } + }; + write_result.map_err(|e| exec_datafusion_err!("json_union_to_text: failed to encode JSON value: {e}"))?; + // `serde_json` always emits valid UTF-8. + let text = std::str::from_utf8(&scratch) + .map_err(|e| exec_datafusion_err!("json_union_to_text: encoded value was not UTF-8: {e}"))?; + builder.append_value(text); + } + Ok(Arc::new(builder.finish())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::common_union::{JsonUnion, JsonUnionField}; + use datafusion::arrow::array::StringViewArray; + + #[test] + fn flattens_each_arm_to_json_text() { + let union = JsonUnion::from_iter(vec![ + Some(JsonUnionField::JsonNull), + Some(JsonUnionField::Bool(true)), + Some(JsonUnionField::Int(42)), + Some(JsonUnionField::Float(1.5)), + Some(JsonUnionField::Str("foo\"bar\n\u{1}".to_string())), + Some(JsonUnionField::Array("[1,2]".to_string())), + Some(JsonUnionField::Object(r#"{"a":1}"#.to_string())), + None, + ]); + let array: ArrayRef = Arc::new(UnionArray::try_from(union).unwrap()); + + let out = json_union_to_text_array(&array).unwrap(); + let strings = out.as_any().downcast_ref::().unwrap(); + let got: Vec> = (0..strings.len()) + .map(|i| (!strings.is_null(i)).then(|| strings.value(i))) + .collect(); + assert_eq!( + got, + vec![ + None, // JsonNull + Some("true"), // Bool + Some("42"), // Int + Some("1.5"), // Float + Some("\"foo\\\"bar\\n\\u0001\""), // Str: JSON-quoted + escaped (quote, newline, control char) + Some("[1,2]"), // Array (passthrough) + Some(r#"{"a":1}"#), // Object (passthrough) + None, // None + ] + ); + } + + #[test] + fn output_field_is_marked_as_json() { + let udf = JsonUnionToText::default(); + let arg = Arc::new(Field::new("j", JSON_UNION_DATA_TYPE.clone(), true)); + let field = udf + .return_field_from_args(ReturnFieldArgs { + arg_fields: std::slice::from_ref(&arg), + scalar_arguments: &[], + }) + .unwrap(); + assert_eq!(field.data_type(), &DataType::Utf8View); + assert_eq!( + field.metadata().get("ARROW:extension:name").map(String::as_str), + Some("arrow.json") + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 6acc415..d81d9a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,9 +20,10 @@ mod json_get_json; mod json_get_str; mod json_length; mod json_object_keys; +mod json_union_to_text; mod rewrite; -pub use common_union::{JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE}; +pub use common_union::{json_field_metadata, JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE}; pub mod functions { pub use crate::json_as_text::json_as_text; @@ -37,6 +38,7 @@ pub mod functions { pub use crate::json_get_str::json_get_str; pub use crate::json_length::json_length; pub use crate::json_object_keys::json_object_keys; + pub use crate::json_union_to_text::json_union_to_text; } pub mod udfs { @@ -52,6 +54,7 @@ pub mod udfs { pub use crate::json_get_str::json_get_str_udf; pub use crate::json_length::json_length_udf; pub use crate::json_object_keys::json_object_keys_udf; + pub use crate::json_union_to_text::json_union_to_text_udf; } /// Register all JSON UDFs, and [`rewrite::JsonFunctionRewriter`] with the provided [`FunctionRegistry`]. @@ -77,6 +80,7 @@ pub fn register_all(registry: &mut dyn FunctionRegistry) -> Result<()> { json_length::json_length_udf(), json_object_keys::json_object_keys_udf(), json_from_scalar::json_from_scalar_udf(), + json_union_to_text::json_union_to_text_udf(), ]; functions.into_iter().try_for_each(|udf| { let existing_udf = registry.register_udf(udf)?; diff --git a/src/rewrite.rs b/src/rewrite.rs index a0e87d5..1dea562 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -35,10 +35,10 @@ impl FunctionRewrite for JsonFunctionRewriter { /// extracting the right value type from JSON without the need to materialize the JSON union. fn optimise_json_get_cast(cast: &Cast) -> Option> { let scalar_func = extract_scalar_function(&cast.expr)?; - if scalar_func.func.name() != "json_get" { + if !is_json_get(scalar_func) { return None; } - let func = match &cast.data_type { + let func = match cast.field.data_type() { DataType::Boolean => crate::json_get_bool::json_get_bool_udf(), DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => { crate::json_get_float::json_get_float_udf() @@ -71,9 +71,9 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { let first_arg = outer_args_iter.next()?; let inner_func = extract_scalar_function(first_arg)?; - // both json_get and json_as_text would produce new JSON to be processed by the outer - // function so can be inlined - if !matches!(inner_func.func.name(), "json_get" | "json_as_text") { + // only json_get preserves a JSON value for the outer function. json_as_text returns SQL text, + // so flattening through it changes semantics for JSON strings that contain JSON. + if !is_json_get(inner_func) { return None; } @@ -90,6 +90,10 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { } } +fn is_json_get(func: &ScalarFunction) -> bool { + func.func.inner().is::() +} + fn extract_scalar_function(expr: &Expr) -> Option<&ScalarFunction> { match expr { Expr::ScalarFunction(func) => Some(func), diff --git a/tests/main.rs b/tests/main.rs index c340fcd..46f5ef2 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -9,6 +9,7 @@ use datafusion::common::ScalarValue; use datafusion::config::ConfigOptions; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs}; use datafusion::prelude::SessionContext; +use datafusion_functions_json::json_field_metadata; use datafusion_functions_json::udfs::json_get_str_udf; use utils::{create_context, display_val, logical_plan, run_query, run_query_params}; @@ -161,6 +162,36 @@ async fn test_json_get_array_with_path() { assert_eq!(value_repr, "[1, 2, 3]"); } +#[tokio::test] +async fn test_json_get_array_inner_field_json_metadata() { + let sql = r#"select json_get_array('[{"a": 1}, {"b": 2}]') as v"#; + let batches = run_query(sql).await.unwrap(); + let schema = batches[0].schema(); + let field = schema.field(0); + let DataType::List(inner_field) = field.data_type() else { + panic!("expected List, got {:?}", field.data_type()); + }; + assert_json_field_metadata(inner_field.metadata()); + + let array_field = batches[0] + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let DataType::List(produced_inner) = array_field.data_type() else { + panic!("expected List in produced array"); + }; + assert_json_field_metadata(produced_inner.metadata()); +} + +fn assert_json_field_metadata(metadata: &HashMap) { + assert_eq!( + metadata.get("ARROW:extension:name").map(String::as_str), + Some("arrow.json") + ); + assert_eq!(metadata.get("ARROW:extension:metadata").map(String::as_str), Some("{}")); +} + #[tokio::test] async fn test_json_get_equals() { let e = run_query(r"select name, json_get(json_data, 'foo')='abc' from test") @@ -284,6 +315,167 @@ async fn test_json_get_int() { assert_eq!(display_val(batches).await, (DataType::Int64, "2".to_string())); } +#[tokio::test] +async fn test_json_get_int_string_parse() { + // string containing int + let batches = run_query(r#"select json_get_int('{"foo": "123"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, "123".to_string())); + + // negative string + let batches = run_query(r#"select json_get_int('{"foo": "-42"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, "-42".to_string())); + + // non-numeric string returns null + let batches = run_query(r#"select json_get_int('{"foo": "abc"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + + // float string returns null (not a valid int) + let batches = run_query(r#"select json_get_int('{"foo": "1.5"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); +} + +#[tokio::test] +async fn test_json_get_large_int() { + // jiter returns these as `BigInt` even though they fit in i64, they must not be lost + let sql = r#"select json_get('{"foo": 1753200000000000000}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + let (value_type, value_repr) = display_val(batches).await; + assert!(matches!(value_type, DataType::Union(_, _))); + assert_eq!(value_repr, "{int=1753200000000000000}"); + + // the i64 bounds themselves + let sql = r#"select json_get('{"foo": 9223372036854775807}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await.1, "{int=9223372036854775807}"); + + let sql = r#"select json_get('{"foo": -9223372036854775808}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await.1, "{int=-9223372036854775808}"); + + let sql = r#"select json_get_int('{"foo": 9223372036854775807}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Int64, "9223372036854775807".to_string()) + ); +} + +#[tokio::test] +async fn test_json_get_out_of_range_int() { + // integers outside i64 range have no representation in the JSON union, they're returned as + // null rather than panicking (or silently losing precision as a float) + let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + let (value_type, value_repr) = display_val(batches).await; + assert!(matches!(value_type, DataType::Union(_, _))); + assert_eq!(value_repr, "{null=}"); + + // below i64::MIN + let sql = r#"select json_get('{"foo": -9223372036854775809}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await.1, "{null=}"); + + // casts of the null union stay null rather than erroring + let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::int"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + + let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::string"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); +} + +#[tokio::test] +async fn test_json_get_out_of_range_int_neighbours() { + // an out-of-range int must only null out its own row, not poison the rest of the batch + let sql = "select json_get(v, 0) from (values ('[1]'), ('[18446744073709551615]'), ('[3]')) t(v)"; + let batches = run_query(sql).await.unwrap(); + let expected = [ + "+------------------------+", + "| json_get(t.v,Int64(0)) |", + "+------------------------+", + "| {int=1} |", + "| {null=} |", + "| {int=3} |", + "+------------------------+", + ]; + assert_batches_eq!(expected, &batches); +} + +#[tokio::test] +async fn test_json_get_out_of_range_int_typed() { + // json_get_int can't represent it either + let sql = r#"select json_get_int('{"foo": 18446744073709551615}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + + // json_get_float is lossy but returns a value + let sql = r#"select json_get_float('{"foo": 18446744073709551615}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Float64, "1.8446744073709552e19".to_string()) + ); + + // json_as_text reads the raw slice, so it stays exact + let sql = r#"select json_as_text('{"foo": 18446744073709551615}', 'foo')"#; + let batches = run_query(sql).await.unwrap(); + assert_eq!( + display_val(batches).await, + (DataType::Utf8, "18446744073709551615".to_string()) + ); +} + +#[tokio::test] +async fn test_json_get_float_string_parse() { + // string containing float + let batches = run_query(r#"select json_get_float('{"foo": "1.5"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Float64, "1.5".to_string())); + + // string containing int parses as float + let batches = run_query(r#"select json_get_float('{"foo": "42"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Float64, "42.0".to_string())); + + // non-numeric string returns null + let batches = run_query(r#"select json_get_float('{"foo": "abc"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Float64, String::new())); +} + +#[tokio::test] +async fn test_json_get_bool_string_parse() { + // string "true" + let batches = run_query(r#"select json_get_bool('{"foo": "true"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Boolean, "true".to_string())); + + // string "false" + let batches = run_query(r#"select json_get_bool('{"foo": "false"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Boolean, "false".to_string())); + + // non-bool string returns null + let batches = run_query(r#"select json_get_bool('{"foo": "abc"}', 'foo')"#) + .await + .unwrap(); + assert_eq!(display_val(batches).await, (DataType::Boolean, String::new())); +} + #[tokio::test] async fn test_json_get_path() { let batches = run_query(r#"select json_get('{"i": 19}', 'i')::int<20"#).await.unwrap(); @@ -411,6 +603,15 @@ async fn test_json_get_json_float() { assert_eq!(display_val(batches).await, (DataType::Utf8, "4.2e-1".to_string())); } +#[tokio::test] +async fn test_json_get_json_json_metadata() { + let sql = r#"select json_get_json('{"x": [1, 2]}', 'x') as v"#; + let batches = run_query(sql).await.unwrap(); + let schema = batches[0].schema(); + let field = schema.field(0); + assert_json_field_metadata(field.metadata()); +} + #[tokio::test] async fn test_json_length_array() { let sql = "select json_length('[1, 2, 3]')"; @@ -597,10 +798,7 @@ fn test_json_get_utf8() { Arc::new(Field::new("arg_3", DataType::LargeUtf8, false)), ], number_rows: 1, - return_field: Arc::new( - Field::new("ret_field", DataType::Utf8, false) - .with_metadata(HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())])), - ), + return_field: Arc::new(Field::new("ret_field", DataType::Utf8, false).with_metadata(json_field_metadata())), config_options: Arc::new(ConfigOptions::default()), }) .unwrap() @@ -631,10 +829,7 @@ fn test_json_get_large_utf8() { Arc::new(Field::new("arg_3", DataType::LargeUtf8, false)), ], number_rows: 1, - return_field: Arc::new( - Field::new("ret_field", DataType::Utf8, false) - .with_metadata(HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())])), - ), + return_field: Arc::new(Field::new("ret_field", DataType::Utf8, false).with_metadata(json_field_metadata())), config_options: Arc::new(ConfigOptions::default()), }) .unwrap() @@ -645,6 +840,114 @@ fn test_json_get_large_utf8() { assert_eq!(sv, ScalarValue::Utf8(Some("x".to_string()))); } +/// A `NullArray` input (Arrow `DataType::Null`, as opposed to a string array containing null +/// values) must yield an all-null result rather than erroring with "unexpected json array type +/// Null". This mirrors the runtime condition where a column declared as a string type +/// materialises as a `NullArray` for a given batch. +#[test] +fn test_json_as_text_null_array_scalar_path() { + use datafusion::arrow::array::NullArray; + use datafusion_functions_json::udfs::json_as_text_udf; + + let udf = json_as_text_udf(); + let attributes: ArrayRef = Arc::new(NullArray::new(3)); // DataType::Null + + let ColumnarValue::Array(result) = udf + .invoke_with_args(ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(attributes), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("client_name".into()))), + ], + arg_fields: vec![ + Arc::new(Field::new("attributes", DataType::Null, true)), + Arc::new(Field::new("path", DataType::Utf8, true)), + ], + number_rows: 3, + return_field: Arc::new(Field::new("result", DataType::Utf8, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap() + else { + panic!("expected array") + }; + + assert_eq!(result.len(), 3); + assert_eq!(result.null_count(), 3); + assert_eq!(result.data_type(), &DataType::Utf8); +} + +/// As above but exercising the array/array-path kernel (`invoke_array_array`): both the JSON +/// argument and the path argument are arrays, with the JSON argument being a `NullArray`. +#[test] +fn test_json_get_int_null_array_array_path() { + use datafusion::arrow::array::{Int64Array, NullArray, StringArray}; + use datafusion_functions_json::udfs::json_get_int_udf; + + let udf = json_get_int_udf(); + let attributes: ArrayRef = Arc::new(NullArray::new(2)); // DataType::Null + let paths: ArrayRef = Arc::new(StringArray::from(vec!["a", "b"])); + + let ColumnarValue::Array(result) = udf + .invoke_with_args(ScalarFunctionArgs { + args: vec![ColumnarValue::Array(attributes), ColumnarValue::Array(paths)], + arg_fields: vec![ + Arc::new(Field::new("attributes", DataType::Null, true)), + Arc::new(Field::new("path", DataType::Utf8, true)), + ], + number_rows: 2, + return_field: Arc::new(Field::new("result", DataType::Int64, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap() + else { + panic!("expected array") + }; + + assert_eq!(result.len(), 2); + assert_eq!(result.null_count(), 2); + assert_eq!(result.as_any().downcast_ref::().unwrap().len(), 2); +} + +/// `json_get` returns the JSON union type; a `NullArray` input must round-trip to an all-null +/// union result. +#[test] +fn test_json_get_null_array_union_return() { + use datafusion::arrow::array::{NullArray, UnionArray}; + use datafusion_functions_json::udfs::json_get_udf; + + let udf = json_get_udf(); + let attributes: ArrayRef = Arc::new(NullArray::new(4)); // DataType::Null + + // json_get returns the JSON union type; derive it from the udf rather than hard-coding it. + // The declared (plan-time) type is a string even though the runtime array is a NullArray. + let return_type = udf.return_type(&[DataType::Utf8, DataType::Utf8]).unwrap(); + + let ColumnarValue::Array(result) = udf + .invoke_with_args(ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(attributes), + ColumnarValue::Scalar(ScalarValue::Utf8(Some("a".into()))), + ], + arg_fields: vec![ + Arc::new(Field::new("attributes", DataType::Null, true)), + Arc::new(Field::new("path", DataType::Utf8, true)), + ], + number_rows: 4, + return_field: Arc::new(Field::new("result", return_type, true).with_metadata(json_field_metadata())), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap() + else { + panic!("expected array") + }; + + // A union array encodes nulls via the "null" union member (type id 0), not a null buffer, + // so check that every row points at the null member rather than relying on null_count(). + assert_eq!(result.len(), 4); + let union = result.as_any().downcast_ref::().unwrap(); + assert!(union.type_ids().iter().all(|&id| id == 0), "expected all-null union"); +} + #[tokio::test] async fn test_json_get_union_scalar() { let sql = r#"select json_get(json_get('{"x": {"y": 1}}', 'x'), 'y') as v"#; @@ -663,6 +966,22 @@ async fn test_json_get_union_scalar() { .await; } +#[tokio::test] +async fn test_json_as_text_nested_json_string() { + let sql = r#"select json_as_text(json_as_text('{"user_id":"{\"device_id\":\"abc\"}"}', 'user_id'), 'device_id')"#; + let batches = run_query(sql).await.unwrap(); + + assert_eq!(display_val(batches).await, (DataType::Utf8, "abc".to_string())); +} + +#[tokio::test] +async fn test_json_get_str_nested_json_string() { + let sql = r#"select json_get_str(json_as_text('{"user_id":"{\"device_id\":\"abc\"}"}', 'user_id'), 'device_id')"#; + let batches = run_query(sql).await.unwrap(); + + assert_eq!(display_val(batches).await, (DataType::Utf8, "abc".to_string())); +} + #[tokio::test] async fn test_json_get_nested_collapsed() { let sql = "select name, json_get(json_get(json_data, 'foo'), 0) as v from test"; @@ -762,9 +1081,9 @@ async fn test_plan_json_get_cte() { select name, json_get(j, 0) v from t "; let expected = [ - "Projection: t.name, json_get(t.j, Int64(0)) AS v", + "Projection: t.name, __datafusion_extracted_1 AS v", " SubqueryAlias: t", - " Projection: test.name, json_get(test.json_data, Utf8(\"foo\")) AS j", + " Projection: test.name, json_get(json_get(test.json_data, Utf8(\"foo\")), Int64(0)) AS __datafusion_extracted_1", " TableScan: test projection=[name, json_data]", ]; @@ -1182,7 +1501,7 @@ async fn test_plan_double_arrow_double_nested() { let lines = logical_plan(r"explain select json_data->>'foo'->>0 from test").await; let expected = [ - "Projection: json_as_text(test.json_data, Utf8(\"foo\"), Int64(0)) AS json_data ->> 'foo' ->> 0", + "Projection: json_as_text(json_as_text(test.json_data, Utf8(\"foo\")), Int64(0)) AS json_data ->> 'foo' ->> 0", " TableScan: test projection=[json_data]", ]; @@ -1255,7 +1574,7 @@ async fn test_plan_double_arrow_double_nested_cast() { // NB: json_as_text(..)::int is NOT the same as `json_get_int(..)`, hence the cast is not rewritten let expected = [ - "Projection: CAST(json_as_text(test.json_data, Utf8(\"foo\"), Int64(0)) AS json_data ->> 'foo' ->> 0 AS Int32)", + "Projection: CAST(json_as_text(json_as_text(test.json_data, Utf8(\"foo\")), Int64(0)) AS Int32) AS json_data ->> 'foo' ->> 0", " TableScan: test projection=[json_data]", ]; @@ -2677,3 +2996,75 @@ async fn test_json_from_scalar_null_column() { ]; assert_batches_eq!(expected, &batches); } + +#[tokio::test] +async fn test_json_union_to_text() { + // Flatten the heterogeneous union from `json_get` to JSON text, exercising the + // str / array (list member) / object / null arms in a single batch. + let batches = run_query("select name, json_union_to_text(json_get(json_data, 'foo')) as foo from test") + .await + .unwrap(); + + let expected = [ + "+------------------+-------+", + "| name | foo |", + "+------------------+-------+", + "| object_foo | \"abc\" |", + "| object_foo_array | [1] |", + "| object_foo_obj | {} |", + "| object_foo_null | |", + "| object_bar | |", + "| list_foo | |", + "| invalid_json | |", + "+------------------+-------+", + ]; + assert_batches_eq!(expected, &batches); + + // The output column is tagged with the canonical Arrow JSON extension type. + let field = batches[0].schema().field_with_name("foo").unwrap().clone(); + assert_eq!(field.data_type(), &DataType::Utf8View); + assert_eq!( + field.metadata().get("ARROW:extension:name").map(String::as_str), + Some("arrow.json") + ); + assert_eq!(field.metadata(), &json_field_metadata()); +} + +#[tokio::test] +async fn test_json_union_to_text_arms() { + // array and object arms already hold raw JSON text and pass through verbatim, + // including nested structures. + let (dt, repr) = display_val( + run_query(r#"select json_union_to_text(json_get('{"a": [1, {"b": 2}]}', 'a'))"#) + .await + .unwrap(), + ) + .await; + assert_eq!(dt, DataType::Utf8View); + assert_eq!(repr, r#"[1, {"b": 2}]"#); + + // string scalars are JSON-quoted and escaped (here: embedded quote + newline) + let (_, repr) = display_val( + run_query(r#"select json_union_to_text(json_get('{"s": "a\"b\n"}', 's'))"#) + .await + .unwrap(), + ) + .await; + assert_eq!(repr, r#""a\"b\n""#); + + // numbers and booleans render as bare JSON literals + let (_, repr) = display_val( + run_query(r#"select json_union_to_text(json_get('{"n": 42}', 'n'))"#) + .await + .unwrap(), + ) + .await; + assert_eq!(repr, "42"); + let (_, repr) = display_val( + run_query(r#"select json_union_to_text(json_get('{"b": true}', 'b'))"#) + .await + .unwrap(), + ) + .await; + assert_eq!(repr, "true"); +}