diff --git a/src/common_string.rs b/src/common_string.rs new file mode 100644 index 0000000..2ef5415 --- /dev/null +++ b/src/common_string.rs @@ -0,0 +1,175 @@ +use std::sync::Arc; + +use datafusion::arrow::array::{ + ArrayRef, GenericStringArray, GenericStringBuilder, LargeStringArray, OffsetSizeTrait, StringArray, + StringViewArray, StringViewBuilder, +}; +use datafusion::arrow::datatypes::DataType; +use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::logical_expr::ColumnarValue; + +use crate::common::{invoke, invoke_array_scalars_direct, return_type_check, GetError, InvokeResult, JsonPath}; + +/// A JSON function that returns a string, such as `json_as_text`. +/// +/// The result has the string type of the JSON argument (see [`string_result_type`]). Each value is +/// a slice of the JSON text or a decoded JSON string, and a decoded string is never longer than its +/// escaped form. Thus a result never needs more offset space than its argument. For example, a +/// `Utf8View` batch of more than 2 GiB does not overflow the `i32` offsets of a `Utf8` result. +pub(crate) trait StringGetter { + /// Write the value at `path` in `json` into `builder`, or a null if there is none. + fn append(json: Option<&str>, path: &[JsonPath], builder: &mut B); + + /// The value at `path` in `json` as an owned string, for the argument shapes that `invoke` serves. + fn get(json: Option<&str>, path: &[JsonPath]) -> Result; +} + +/// A builder of one of the Arrow string types. +pub(crate) trait StringResultBuilder { + fn append_str(&mut self, value: Option<&str>); +} + +impl StringResultBuilder for GenericStringBuilder { + fn append_str(&mut self, value: Option<&str>) { + self.append_option(value); + } +} + +impl StringResultBuilder for StringViewBuilder { + fn append_str(&mut self, value: Option<&str>) { + self.append_option(value); + } +} + +impl InvokeResult for GenericStringArray { + type Item = String; + + type Builder = GenericStringBuilder; + + const ACCEPT_DICT_RETURN: bool = true; + + fn builder(capacity: usize) -> Self::Builder { + GenericStringBuilder::with_capacity(capacity, 0) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + if O::IS_LARGE { + ScalarValue::LargeUtf8(value) + } else { + ScalarValue::Utf8(value) + } + } +} + +impl InvokeResult for StringViewArray { + type Item = String; + + type Builder = StringViewBuilder; + + const ACCEPT_DICT_RETURN: bool = true; + + fn builder(capacity: usize) -> Self::Builder { + StringViewBuilder::with_capacity(capacity) + } + + fn append_value(builder: &mut Self::Builder, value: Option) { + builder.append_option(value); + } + + fn finish(mut builder: Self::Builder) -> DataFusionResult { + Ok(Arc::new(builder.finish())) + } + + fn scalar(value: Option) -> ScalarValue { + ScalarValue::Utf8View(value) + } +} + +/// The string type of the result for a JSON argument of type `json_type`. +/// +/// A string argument gives its own type, and a dictionary argument gives the type of its values. +/// A JSON union holds its strings as `Utf8`, so a union argument gives `Utf8`. +fn string_result_type(json_type: &DataType) -> DataType { + match json_type { + DataType::LargeUtf8 => DataType::LargeUtf8, + DataType::Utf8View => DataType::Utf8View, + DataType::Dictionary(_, value_type) => string_result_type(value_type), + _ => DataType::Utf8, + } +} + +/// `ScalarUDFImpl::return_type` for a [`StringGetter`]. +pub(crate) fn string_return_type(arg_types: &[DataType], fn_name: &str) -> DataFusionResult { + let value_type = arg_types.first().map_or(DataType::Utf8, string_result_type); + return_type_check::(arg_types, fn_name, value_type) +} + +/// `ScalarUDFImpl::invoke_with_args` for a [`StringGetter`]. +pub(crate) fn invoke_string(args: &[ColumnarValue]) -> DataFusionResult { + match args.first().map(|json| string_result_type(&json.data_type())) { + Some(DataType::LargeUtf8) => invoke_string_as::(args), + Some(DataType::Utf8View) => invoke_string_as::(args), + _ => invoke_string_as::(args), + } +} + +fn invoke_string_as(args: &[ColumnarValue]) -> DataFusionResult +where + R: InvokeResult, + R::Builder: StringResultBuilder, +{ + if let Some(result) = invoke_array_scalars_direct::(args, G::append::)? { + return Ok(result); + } + invoke::(args, G::get) +} + +#[cfg(test)] +pub(crate) mod test_util { + use super::*; + use crate::common::test_util::assert_direct_matches_owned; + + /// Check the direct path and `invoke` build equal arrays for each string result type. + pub(crate) fn assert_string_direct_matches_owned( + rows: &[Option<&str>], + paths: &[Vec], + ) { + assert_direct_matches_owned::(rows, paths, G::append, G::get); + assert_direct_matches_owned::(rows, paths, G::append, G::get); + assert_direct_matches_owned::(rows, paths, G::append, G::get); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn result_type_follows_json_type() { + let dict = |value: DataType| DataType::Dictionary(Box::new(DataType::Int32), Box::new(value)); + let large_dict = |value: DataType| DataType::Dictionary(Box::new(DataType::Int64), Box::new(value)); + let union = crate::JSON_UNION_DATA_TYPE.clone(); + let cases = [ + (DataType::Utf8, DataType::Utf8), + (DataType::LargeUtf8, DataType::LargeUtf8), + (DataType::Utf8View, DataType::Utf8View), + (union.clone(), DataType::Utf8), + (dict(DataType::Utf8), large_dict(DataType::Utf8)), + (dict(DataType::LargeUtf8), large_dict(DataType::LargeUtf8)), + (dict(DataType::Utf8View), large_dict(DataType::Utf8View)), + (dict(union), large_dict(DataType::Utf8)), + ]; + for (json_type, want) in cases { + let got = string_return_type(&[json_type.clone(), DataType::Utf8], "f").unwrap(); + assert_eq!(got, want, "{json_type}"); + } + } +} diff --git a/src/json_as_text.rs b/src/json_as_text.rs index 7c614a1..a2d0e47 100644 --- a/src/json_as_text.rs +++ b/src/json_as_text.rs @@ -1,16 +1,11 @@ -use std::sync::Arc; - -use datafusion::arrow::array::{ArrayRef, StringArray, StringBuilder}; use datafusion::arrow::datatypes::DataType; -use datafusion::common::{Result as DataFusionResult, ScalarValue}; +use datafusion::common::Result as DataFusionResult; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use jiter::{Jiter, Peek}; -use crate::common::{ - get_err, invoke, invoke_array_scalars_direct, jiter_json_find, jiter_skip_str, return_type_check, GetError, - InvokeResult, JsonPath, -}; +use crate::common::{get_err, jiter_json_find, jiter_skip_str, GetError, JsonPath}; use crate::common_macros::make_udf_function; +use crate::common_string::{invoke_string, string_return_type, StringGetter, StringResultBuilder}; make_udf_function!( JsonAsText, @@ -44,14 +39,11 @@ impl ScalarUDFImpl for JsonAsText { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check::(arg_types, self.name(), DataType::Utf8) + string_return_type(arg_types, self.name()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { - if let Some(result) = invoke_array_scalars_direct::(&args.args, append_json_as_text)? { - return Ok(result); - } - invoke::(&args.args, jiter_json_as_text) + invoke_string::(&args.args) } fn aliases(&self) -> &[String] { @@ -77,42 +69,20 @@ impl ScalarUDFImpl for JsonAsText { } } -impl InvokeResult for StringArray { - type Item = String; - - type Builder = StringBuilder; - - const ACCEPT_DICT_RETURN: bool = true; - - fn builder(capacity: usize) -> Self::Builder { - StringBuilder::with_capacity(capacity, 0) - } - - fn append_value(builder: &mut Self::Builder, value: Option) { - builder.append_option(value); - } - - fn finish(mut builder: Self::Builder) -> DataFusionResult { - Ok(Arc::new(builder.finish())) - } - - fn scalar(value: Option) -> ScalarValue { - ScalarValue::Utf8(value) +impl StringGetter for JsonAsText { + fn append(opt_json: Option<&str>, path: &[JsonPath], builder: &mut B) { + if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) { + builder.append_str(text_value(json, &mut jiter, peek).ok()); + } else { + builder.append_str(None); + } } -} - -fn jiter_json_as_text(opt_json: Option<&str>, path: &[JsonPath]) -> Result { - let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else { - return get_err!(); - }; - text_value(json, &mut jiter, peek).map(str::to_owned) -} -fn append_json_as_text(opt_json: Option<&str>, path: &[JsonPath], builder: &mut StringBuilder) { - if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) { - builder.append_option(text_value(json, &mut jiter, peek).ok()); - } else { - builder.append_null(); + fn get(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else { + return get_err!(); + }; + text_value(json, &mut jiter, peek).map(str::to_owned) } } @@ -129,7 +99,8 @@ fn text_value<'a, 'j: 'a>(json: &'j str, jiter: &'a mut Jiter<'j>, peek: Peek) - #[cfg(test)] mod tests { use super::*; - use crate::common::test_util::assert_direct_matches_owned; + use crate::common_string::test_util::assert_string_direct_matches_owned; + use datafusion::common::ScalarValue; #[test] fn direct_builder_matches_owned_results() { @@ -153,6 +124,6 @@ mod tests { ], vec![ScalarValue::Utf8(None)], ]; - assert_direct_matches_owned::(&rows, &paths, append_json_as_text, jiter_json_as_text); + assert_string_direct_matches_owned::(&rows, &paths); } } diff --git a/src/json_get_json.rs b/src/json_get_json.rs index 653ee57..763c863 100644 --- a/src/json_get_json.rs +++ b/src/json_get_json.rs @@ -1,17 +1,14 @@ use std::sync::Arc; -use datafusion::arrow::array::{StringArray, StringBuilder}; use datafusion::arrow::datatypes::{DataType, Field, FieldRef}; use datafusion::common::Result as DataFusionResult; use datafusion::logical_expr::{ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use crate::common::{ - get_err, invoke, invoke_array_scalars_direct, jiter_json_find, jiter_skip_str, return_type_check, GetError, - JsonPath, -}; +use crate::common::{get_err, jiter_json_find, jiter_skip_str, GetError, JsonPath}; use crate::common_macros::make_udf_function; +use crate::common_string::{invoke_string, string_return_type, StringGetter, StringResultBuilder}; use crate::common_union::json_field_metadata; make_udf_function!( @@ -46,7 +43,7 @@ impl ScalarUDFImpl for JsonGetJson { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check::(arg_types, self.name(), DataType::Utf8) + string_return_type(arg_types, self.name()) } fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult { @@ -58,10 +55,7 @@ impl ScalarUDFImpl for JsonGetJson { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { - if let Some(result) = invoke_array_scalars_direct::(&args.args, append_json_get_json)? { - return Ok(result); - } - invoke::(&args.args, jiter_json_get_json) + invoke_string::(&args.args) } fn aliases(&self) -> &[String] { @@ -87,25 +81,27 @@ impl ScalarUDFImpl for JsonGetJson { } } -fn jiter_json_get_json(opt_json: Option<&str>, path: &[JsonPath]) -> Result { - let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else { - return get_err!(); - }; - jiter_skip_str(json, &mut jiter, peek).map(str::to_owned) -} +impl StringGetter for JsonGetJson { + fn append(opt_json: Option<&str>, path: &[JsonPath], builder: &mut B) { + if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) { + builder.append_str(jiter_skip_str(json, &mut jiter, peek).ok()); + } else { + builder.append_str(None); + } + } -fn append_json_get_json(opt_json: Option<&str>, path: &[JsonPath], builder: &mut StringBuilder) { - if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) { - builder.append_option(jiter_skip_str(json, &mut jiter, peek).ok()); - } else { - builder.append_null(); + fn get(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else { + return get_err!(); + }; + jiter_skip_str(json, &mut jiter, peek).map(str::to_owned) } } #[cfg(test)] mod tests { use super::*; - use crate::common::test_util::assert_direct_matches_owned; + use crate::common_string::test_util::assert_string_direct_matches_owned; use datafusion::common::ScalarValue; #[test] @@ -125,6 +121,6 @@ mod tests { vec![ScalarValue::Utf8(Some("c".to_owned())), ScalarValue::Int64(Some(1))], vec![ScalarValue::Utf8(None)], ]; - assert_direct_matches_owned::(&rows, &paths, append_json_get_json, jiter_json_get_json); + assert_string_direct_matches_owned::(&rows, &paths); } } diff --git a/src/json_get_str.rs b/src/json_get_str.rs index bb768b0..9facf07 100644 --- a/src/json_get_str.rs +++ b/src/json_get_str.rs @@ -1,13 +1,11 @@ -use datafusion::arrow::array::{StringArray, StringBuilder}; use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DataFusionResult; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use jiter::{Jiter, Peek}; -use crate::common::{ - get_err, invoke, invoke_array_scalars_direct, jiter_json_find, return_type_check, GetError, JsonPath, -}; +use crate::common::{get_err, jiter_json_find, GetError, JsonPath}; use crate::common_macros::make_udf_function; +use crate::common_string::{invoke_string, string_return_type, StringGetter, StringResultBuilder}; make_udf_function!( JsonGetStr, @@ -41,14 +39,11 @@ impl ScalarUDFImpl for JsonGetStr { } fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { - return_type_check::(arg_types, self.name(), DataType::Utf8) + string_return_type(arg_types, self.name()) } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { - if let Some(result) = invoke_array_scalars_direct::(&args.args, append_json_get_str)? { - return Ok(result); - } - invoke::(&args.args, jiter_json_get_str) + invoke_string::(&args.args) } fn aliases(&self) -> &[String] { @@ -74,18 +69,20 @@ impl ScalarUDFImpl for JsonGetStr { } } -fn jiter_json_get_str(opt_json: Option<&str>, path: &[JsonPath]) -> Result { - let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) else { - return get_err!(); - }; - str_value(&mut jiter, peek).map(str::to_owned) -} +impl StringGetter for JsonGetStr { + fn append(opt_json: Option<&str>, path: &[JsonPath], builder: &mut B) { + if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { + builder.append_str(str_value(&mut jiter, peek).ok()); + } else { + builder.append_str(None); + } + } -fn append_json_get_str(opt_json: Option<&str>, path: &[JsonPath], builder: &mut StringBuilder) { - if let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) { - builder.append_option(str_value(&mut jiter, peek).ok()); - } else { - builder.append_null(); + fn get(opt_json: Option<&str>, path: &[JsonPath]) -> Result { + let Some((mut jiter, peek)) = jiter_json_find(opt_json, path) else { + return get_err!(); + }; + str_value(&mut jiter, peek).map(str::to_owned) } } @@ -99,7 +96,7 @@ fn str_value<'a>(jiter: &'a mut Jiter<'_>, peek: Peek) -> Result<&'a str, GetErr #[cfg(test)] mod tests { use super::*; - use crate::common::test_util::assert_direct_matches_owned; + use crate::common_string::test_util::assert_string_direct_matches_owned; use datafusion::common::ScalarValue; #[test] @@ -124,6 +121,6 @@ mod tests { ], vec![ScalarValue::Utf8(None)], ]; - assert_direct_matches_owned::(&rows, &paths, append_json_get_str, jiter_json_get_str); + assert_string_direct_matches_owned::(&rows, &paths); } } diff --git a/src/lib.rs b/src/lib.rs index d81d9a0..42d12d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use datafusion::logical_expr::ScalarUDF; mod common; mod common_macros; +mod common_string; mod common_union; mod json_as_text; mod json_contains; diff --git a/tests/json_robustness.rs b/tests/json_robustness.rs index 92a4cde..8348108 100644 --- a/tests/json_robustness.rs +++ b/tests/json_robustness.rs @@ -118,7 +118,7 @@ async fn outcome(ctx: &SessionContext, sql: &str) -> std::result::Result std::result::Result { let (data_type, value) = run_single(ctx, sql).await?; Ok(format!("{}={value}", logical_type(&data_type))) @@ -135,11 +135,12 @@ async fn run_single(ctx: &SessionContext, sql: &str) -> std::result::Result<(Dat Ok((batch.schema().field(0).data_type().clone(), value)) } -/// Dictionary encoding of the input carries through to the output, so compare the type it -/// wraps rather than the wrapper. +/// The dictionary encoding and the string type of the input carry through to the output, so +/// compare the type the dictionary wraps, with every string type as `Utf8`. fn logical_type(data_type: &DataType) -> &DataType { match data_type { - DataType::Dictionary(_, value) => value, + DataType::Dictionary(_, value) => logical_type(value), + DataType::LargeUtf8 | DataType::Utf8View => &DataType::Utf8, other => other, } } @@ -300,8 +301,9 @@ fn prop_scalar_and_array_paths_agree() { proptest!(config(96), |(json in json_text(), path in path())| { let literal = format!("'{}'", json.replace('\'', "''")); + let view_literal = format!("arrow_cast({literal}, '{}')", DataType::Utf8View); let dict_literal = format!("arrow_cast({literal}, '{}')", dict_encoding()); - for (encoding, literal) in [(DataType::Utf8View, literal), (dict_encoding(), dict_literal)] { + for (encoding, literal) in [(DataType::Utf8View, view_literal), (dict_encoding(), dict_literal)] { set_doc(&ctx, &json, &encoding); for func in PATH_FUNCS { if !callable(func, &path) { diff --git a/tests/main.rs b/tests/main.rs index edd2175..f607ead 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -970,7 +970,9 @@ fn test_json_get_large_utf8() { Arc::new(Field::new("arg_3", DataType::LargeUtf8, false)), ], number_rows: 1, - return_field: Arc::new(Field::new("ret_field", DataType::Utf8, false).with_metadata(json_field_metadata())), + return_field: Arc::new( + Field::new("ret_field", DataType::LargeUtf8, false).with_metadata(json_field_metadata()), + ), config_options: Arc::new(ConfigOptions::default()), }) .unwrap() @@ -978,7 +980,7 @@ fn test_json_get_large_utf8() { panic!("expected scalar") }; - assert_eq!(sv, ScalarValue::Utf8(Some("x".to_string()))); + assert_eq!(sv, ScalarValue::LargeUtf8(Some("x".to_string()))); } /// A `NullArray` input (Arrow `DataType::Null`, as opposed to a string array containing null @@ -2179,7 +2181,8 @@ async fn test_dict_column_path_no_null_values() { } } -/// A dictionary JSON column of every string value type, with a column path. +/// A dictionary JSON column of every string value type, with a column path. The result is a +/// dictionary of the same string type. #[tokio::test] async fn test_dict_value_types_column_path() { for value_type in ["Utf8", "LargeUtf8", "Utf8View"] { @@ -2190,7 +2193,7 @@ async fn test_dict_value_types_column_path() { assert_eq!( display_rows(&batches), ( - DataType::Dictionary(Box::new(DataType::Int64), Box::new(DataType::Utf8)), + DataType::Dictionary(Box::new(DataType::Int64), Box::new(value_type.parse().unwrap())), vec!["42".to_string(), String::new(), String::new(), String::new()] ), "{sql}" @@ -2201,6 +2204,37 @@ async fn test_dict_value_types_column_path() { } } +/// The string functions return the string type of their JSON argument, and a dictionary of it for +/// a dictionary argument. A value is never longer than the JSON it comes from, so the result never +/// needs larger offsets than the argument. +#[tokio::test] +async fn test_string_result_follows_json_type() { + let doc = r#"'{"s": "x\ny", "o": {"a": 1}}'"#; + let dict = |value| DataType::Dictionary(Box::new(DataType::Int64), Box::new(value)); + for (json_type, want_type) in [ + ("Utf8", DataType::Utf8), + ("LargeUtf8", DataType::LargeUtf8), + ("Utf8View", DataType::Utf8View), + ("Dictionary(Int32, Utf8View)", dict(DataType::Utf8View)), + ] { + let json = format!("arrow_cast(j, '{json_type}')"); + for (expr, want) in [ + (format!("json_as_text({json}, 's')"), "x\ny"), + (format!("{json} ->> 'o'"), r#"{"a": 1}"#), + (format!("json_get_str({json}, 's')"), "x\ny"), + (format!("json_get_json({json}, 'o')"), r#"{"a": 1}"#), + ] { + let sql = format!("select {expr} from (values ({doc})) t(j)"); + let batches = run_query(&sql).await.unwrap(); + assert_eq!( + display_rows(&batches), + (want_type.clone(), vec![want.to_string()]), + "{sql}" + ); + } + } +} + /// With `varchar` planned as `Utf8`, `json_get(..)::varchar` folds to `json_as_text` and drops the /// cast, since `json_as_text` returns `Utf8` too. For a dictionary JSON argument it returns /// `Dictionary(Int64, Utf8)`, so there the cast has to stay. @@ -2258,9 +2292,10 @@ async fn test_cast_json_get_to_text() { } } + // `json_data` is `Utf8View`, the type `::text` asks for, so the cast goes let plan = logical_plan(r"explain select (json_data -> 'foo')::text from test").await; assert!( - plan[0].contains("CAST(json_as_text(test.json_data, Utf8(\"foo\")) AS Utf8View)"), + plan[0].contains("Projection: json_as_text(test.json_data, Utf8(\"foo\")) AS"), "{plan:?}" ); } @@ -2281,14 +2316,14 @@ async fn test_cast_json_get_nested_to_text() { let plan = logical_plan(r"explain select (json_data -> 'foo' -> 'bar')::text from test").await; assert!( - plan[0].contains("CAST(json_as_text(test.json_data, Utf8(\"foo\"), Utf8(\"bar\")) AS Utf8View)"), + plan[0].contains("Projection: json_as_text(test.json_data, Utf8(\"foo\"), Utf8(\"bar\")) AS"), "{plan:?}" ); } /// Every string cast target folds to `json_as_text`, and the result has the type that was asked -/// for. `json_as_text` returns `Utf8`, or `Dictionary(Int64, Utf8)` for a dictionary JSON argument, -/// so only `Utf8` over a plain string drops the cast. +/// for. Here `json_as_text` returns `Utf8`, or `Dictionary(Int64, Utf8)` for the dictionary JSON +/// argument, so only `Utf8` over a plain string drops the cast. #[tokio::test] async fn test_cast_json_get_to_every_string_type() { use datafusion::logical_expr::{cast as cast_expr, col, lit}; diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs index f5ba6ea..33bb8a3 100644 --- a/tests/rewrite_differential.rs +++ b/tests/rewrite_differential.rs @@ -97,13 +97,14 @@ const TARGETS: &[Target] = &[ sql_cast: Fold::Exact, arrow_cast: Fold::Exact, }, - // SQL `VARCHAR` is `Utf8View` in DataFusion, but `json_as_text` returns `Utf8`. + // SQL `VARCHAR` is `Utf8View` in DataFusion. `json_as_text` returns the string type of the + // `Utf8View` document, so the `Utf8` that `arrow_cast` names here is the narrowing one. Target { sql: "varchar", arrow: "Utf8", accessor: "json_as_text", - sql_cast: Fold::Narrowing, - arrow_cast: Fold::Exact, + sql_cast: Fold::Exact, + arrow_cast: Fold::Narrowing, }, // Narrowing targets: the type asked for is narrower than what the accessor returns, so the // fold keeps the cast on top of the accessor. @@ -280,6 +281,16 @@ enum Outcome { Error, } +impl Outcome { + /// The same outcome, with a `LargeUtf8` or `Utf8View` value given as `Utf8`. + fn with_string_type_as_utf8(self) -> Self { + match self { + Outcome::Value(DataType::LargeUtf8 | DataType::Utf8View, value) => Outcome::Value(DataType::Utf8, value), + other => other, + } + } +} + impl std::fmt::Display for Outcome { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -423,6 +434,9 @@ fn prop_fold_is_invisible() { /// Flattening `json_get(json_get(j, 'a'), 'b')` into `json_get(j, 'a', 'b')` must not change /// the answer. Here the plan without the rewrite really is the right reference, so this is /// the plain with/without-optimizer differential the barrier was built for. +/// +/// The string type can change. The nested call reads the strings of a JSON union, which are +/// `Utf8`, but the flattened call reads `j` and returns the string type of `j`. #[test] fn prop_unnest_is_invisible() { let rt = runtime(); @@ -453,8 +467,8 @@ fn prop_unnest_is_invisible() { "expected the nested call to be flattened:\n{}\n{}", unnested, unnested_plan ); - let got = rt.block_on(outcome(&ctx, &unnested)); - let want = rt.block_on(outcome(&ctx, ¬_unnested)); + let got = rt.block_on(outcome(&ctx, &unnested)).with_string_type_as_utf8(); + let want = rt.block_on(outcome(&ctx, ¬_unnested)).with_string_type_as_utf8(); prop_assert_eq!(&got, &want, "\n unnested: {} => {}\n not unnested: {} => {}\n", unnested, got, not_unnested, want);