diff --git a/Cargo.toml b/Cargo.toml index 249d5e5..3078a43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ datafusion = { version = "54", default-features = false, features = [ "nested_expressions", "sql", ] } +proptest = "1.11.0" tokio = { version = "1.43", features = ["full"] } [lints.clippy] diff --git a/src/rewrite.rs b/src/rewrite.rs index df6586e..1cc423d 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -6,7 +6,7 @@ use datafusion::common::tree_node::Transformed; use datafusion::common::Column; use datafusion::common::DFSchema; use datafusion::common::Result; -use datafusion::logical_expr::expr::{Alias, Expr, ScalarFunction}; +use datafusion::logical_expr::expr::{Alias, Cast, Expr, ScalarFunction, TryCast}; use datafusion::logical_expr::expr_rewriter::FunctionRewrite; use datafusion::logical_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr}; use datafusion::logical_expr::sqlparser::ast::BinaryOperator; @@ -23,38 +23,80 @@ impl FunctionRewrite for JsonFunctionRewriter { fn rewrite(&self, expr: Expr, _schema: &DFSchema, _config: &ConfigOptions) -> Result> { let transform = match &expr { - Expr::Cast(cast) => optimise_json_get_cast(cast.field.data_type(), &cast.expr), - Expr::TryCast(try_cast) => optimise_json_get_cast(try_cast.field.data_type(), &try_cast.expr), + Expr::Cast(cast) => optimise_json_get(cast.field.data_type(), &cast.expr).map(|folded| match folded { + Folded::Exact(accessor) => accessor, + Folded::Narrowing(accessor) => Expr::Cast(Cast { + expr: Box::new(accessor), + field: cast.field.clone(), + }), + }), + Expr::TryCast(try_cast) => { + optimise_json_get(try_cast.field.data_type(), &try_cast.expr).map(|folded| match folded { + Folded::Exact(accessor) => accessor, + Folded::Narrowing(accessor) => Expr::TryCast(TryCast { + expr: Box::new(accessor), + field: try_cast.field.clone(), + }), + }) + } Expr::ScalarFunction(func) => optimise_json_get_arrow_cast(func).or_else(|| unnest_json_calls(func)), _ => None, }; - Ok(transform.unwrap_or_else(|| Transformed::no(expr))) + Ok(transform.map_or_else(|| Transformed::no(expr), Transformed::yes)) } } -/// This replaces `get_json(foo, bar)::int` with `json_get_int(foo, bar)` so the JSON function can take care of -/// extracting the right value type from JSON without the need to materialize the JSON union. +/// The accessor call that replaces a `json_get` under a cast, and whether the cast is still needed. +enum Folded { + /// The accessor already returns the type the query asked for; the cast can go. + Exact(Expr), + /// The accessor returns a wider type; the cast has to stay or both the type of the expression + /// and, for a narrowing cast, its value would change. + Narrowing(Expr), +} + +/// Replace the `json_get` under a cast to `cast_to` with the typed accessor that reads that type +/// out of the JSON directly, so the JSON union never has to be materialized just to be cast away. +/// +/// The accessors return one Arrow type per JSON type — `json_get_int` is always `Int64` — which is +/// not necessarily the type that was asked for. Where it is, the cast is dropped and this is the +/// plain substitution it has always been. Where it is not, only the cast's *input* is replaced: +/// dropping the cast there would silently change the type of the expression, and for a narrowing +/// cast its value too, while keeping it costs one primitive-to-primitive cast and still never +/// materializes the union. /// -/// `TRY_CAST` is folded the same way. The typed accessors already yield NULL for a value of another -/// type instead of failing, which is exactly what `TRY_CAST` asks for. -fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option> { +/// `TRY_CAST` gets the same treatment, so a value outside the target type's range still becomes +/// NULL rather than being handed back as the accessor's wider type. +fn optimise_json_get(cast_to: &DataType, cast_expr: &Expr) -> Option { let scalar_func = extract_scalar_function(cast_expr)?; if !is_json_get(scalar_func) { return None; } - let func = match cast_to { - DataType::Boolean => crate::json_get_bool::json_get_bool_udf(), + let (func, returns) = typed_accessor(cast_to)?; + let accessor = Expr::ScalarFunction(ScalarFunction { + func, + args: scalar_func.args.clone(), + }); + Some(if returns == *cast_to { + Folded::Exact(accessor) + } else { + Folded::Narrowing(accessor) + }) +} + +/// The accessor that reads `cast_to` out of JSON, if there is one, and the type it returns. +fn typed_accessor(cast_to: &DataType) -> Option<(Arc, DataType)> { + Some(match cast_to { + DataType::Boolean => (crate::json_get_bool::json_get_bool_udf(), DataType::Boolean), DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => { - crate::json_get_float::json_get_float_udf() + (crate::json_get_float::json_get_float_udf(), DataType::Float64) + } + DataType::Int64 | DataType::Int32 => (crate::json_get_int::json_get_int_udf(), DataType::Int64), + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => { + (crate::json_get_str::json_get_str_udf(), DataType::Utf8) } - DataType::Int64 | DataType::Int32 => crate::json_get_int::json_get_int_udf(), - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => crate::json_get_str::json_get_str_udf(), _ => return None, - }; - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { - func, - args: scalar_func.args.clone(), - }))) + }) } /// The same rewrite for `arrow_cast(json_get(foo, bar), 'Int64')` and its `arrow_try_cast` sibling. @@ -64,10 +106,9 @@ fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option Option> { +/// Only the call's first argument is replaced, so the named type is still what comes out. That +/// lowering then drops the cast by itself when the accessor already returns the named type. +fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option { if !matches!(func.func.name(), "arrow_cast" | "arrow_try_cast") { return None; } @@ -77,25 +118,22 @@ fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option crate::json_get_bool::json_get_bool_udf(), - "Float64" => crate::json_get_float::json_get_float_udf(), - "Int64" => crate::json_get_int::json_get_int_udf(), - "Utf8" => crate::json_get_str::json_get_str_udf(), - _ => return None, + // `arrow_cast` names its target as an Arrow type string, which is how DataFusion itself reads + // it back in `ArrowCastFunc::return_field_from_args`. + let cast_to = type_name.parse::().ok()?; + // the call keeps its type argument either way, and `ArrowCastFunc::simplify` drops the cast + // itself when the accessor already returns that type + let accessor = match optimise_json_get(&cast_to, cast_expr)? { + Folded::Exact(accessor) | Folded::Narrowing(accessor) => accessor, }; - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { - func, - args: scalar_func.args.clone(), - }))) + Some(Expr::ScalarFunction(ScalarFunction { + func: func.func.clone(), + args: vec![accessor, type_arg.clone()], + })) } // Replace nested JSON functions e.g. `json_get(json_get(col, 'foo'), 'bar')` with `json_get(col, 'foo', 'bar')` -fn unnest_json_calls(func: &ScalarFunction) -> Option> { +fn unnest_json_calls(func: &ScalarFunction) -> Option { if !matches!( func.func.name(), "json_get" @@ -122,10 +160,10 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { args.extend(outer_args_iter.cloned()); // See #23, unnest only when all lookup arguments are literals if args.iter().skip(1).all(|arg| matches!(arg, Expr::Literal(_, _))) { - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { + Some(Expr::ScalarFunction(ScalarFunction { func: func.func.clone(), args, - }))) + })) } else { None } diff --git a/tests/json_robustness.rs b/tests/json_robustness.rs new file mode 100644 index 0000000..fac5bf7 --- /dev/null +++ b/tests/json_robustness.rs @@ -0,0 +1,322 @@ +//! Properties that hold for every JSON UDF, checked over generated input. +//! +//! These are deliberately cheap and broad rather than targeted: they cover the accessors as a +//! group, so a change to any one of them — or to the shared scanning code in `common.rs` — +//! has to keep them true. +//! +//! * [`prop_no_panic_on_arbitrary_json`] — malformed input yields NULL, never a panic and +//! never an error. This is the bug shape that produced the `todo!("BigInt not supported +//! yet")` panic fixed in #124: a JSON integer too wide for jiter's fast path. +//! * [`prop_scalar_and_array_paths_agree`] — a literal argument is const-evaluated through +//! the `ScalarValue` path while a column goes through the array path. They are separate +//! code paths in every UDF here, and they have to agree. +//! * [`prop_input_encoding_does_not_change_result`] — `Utf8`, `LargeUtf8`, `Utf8View` and +//! dictionary-encoded input are four more separate paths that have to agree. + +use std::sync::Arc; + +use datafusion::arrow::array::{ + ArrayRef, DictionaryArray, Int32Array, LargeStringArray, RecordBatch, StringArray, StringViewArray, +}; +use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +/// Every UDF that takes a JSON document and a lookup path. +const PATH_FUNCS: &[&str] = &[ + "json_get", + "json_get_bool", + "json_get_float", + "json_get_int", + "json_get_json", + "json_get_str", + "json_get_array", + "json_as_text", + "json_contains", + "json_length", + "json_object_keys", +]; + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +/// Each case here runs a query per UDF, so the defaults are kept low enough that the suite +/// stays fast. `PROPTEST_CASES` still wins when it is set, which is how these get run as a +/// fuzzer: `PROPTEST_CASES=100000 cargo test --release --test json_robustness`. +fn config(cases: u32) -> ProptestConfig { + let mut config = ProptestConfig::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = cases; + } + config +} + +/// Build the single-row table `t` holding `json` in the given encoding. +fn set_doc(ctx: &SessionContext, json: &str, encoding: &DataType) { + let array: ArrayRef = match encoding { + DataType::Utf8 => Arc::new(StringArray::from(vec![json])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec![json])), + DataType::Utf8View => Arc::new(StringViewArray::from(vec![json])), + DataType::Dictionary(_, _) => Arc::new(DictionaryArray::::new( + Int32Array::from(vec![0]), + Arc::new(StringArray::from(vec![json])), + )), + other => panic!("unsupported JSON encoding {other}"), + }; + let schema = Schema::new(vec![Field::new(JSON_COLUMN, encoding.clone(), false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +const ENCODINGS: &[DataType] = &[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View]; + +fn dict_encoding() -> DataType { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) +} + +/// Run a single-row query, formatting the result. `Err` carries only the fact that the query +/// failed, so the properties are not brittle about error text. +async fn outcome(ctx: &SessionContext, sql: &str) -> std::result::Result { + let df = ctx.sql(sql).await.map_err(|e| e.to_string())?; + let batches = df.collect().await.map_err(|e| e.to_string())?; + let batch = &batches[0]; + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).map_err(|e| e.to_string())?; + Ok(format!( + "{}={}", + logical_type(batch.schema().field(0).data_type()), + formatter.value(0).try_to_string().map_err(|e| e.to_string())? + )) +} + +/// Dictionary encoding of the input carries through to the output, so compare the type it +/// wraps rather than the wrapper. +fn logical_type(data_type: &DataType) -> &DataType { + match data_type { + DataType::Dictionary(_, value) => value, + other => other, + } +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// Well-formed JSON, then mangled: truncated, or with a byte dropped. Most of the interesting +/// malformed input is "nearly valid", which random text never produces. +fn json_text() -> impl Strategy { + prop_oneof![ + 6 => valid_json(), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let cut = j.char_indices().nth(n).map_or(j.len(), |(i, _)| i); + j[..cut].to_string() + }), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let mut chars: Vec = j.chars().collect(); + if !chars.is_empty() { + chars.remove(n % chars.len()); + } + chars.into_iter().collect() + }), + 1 => ".{0,24}".prop_map(|s| s), + ] +} + +fn valid_json() -> impl Strategy { + let leaf = prop_oneof![ + Just("null".to_string()), + any::().prop_map(|b| b.to_string()), + any::().prop_map(|i| i.to_string()), + any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + "[a-zA-Z0-9 ._+-]{0,6}".prop_map(|s| format!("\"{s}\"")), + // integers and floats outside the fast paths + Just("123456789012345678901234567890".to_string()), + Just("-99999999999999999999999999999999".to_string()), + Just("1e309".to_string()), + Just("-1e-309".to_string()), + ]; + leaf.prop_recursive(3, 12, 3, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..3).prop_map(|items| format!("[{}]", items.join(", "))), + prop::collection::vec(("[a-c]{1,2}", inner), 0..3).prop_map(|entries| { + let body = entries + .into_iter() + .map(|(k, v)| format!("\"{k}\": {v}")) + .collect::>() + .join(", "); + format!("{{{body}}}") + }), + ] + }) +} + +/// A lookup path: string keys and array indices, rendered as SQL arguments. +fn path() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + "[a-c]{1,2}".prop_map(|k| format!("'{k}'")), + (0i64..4).prop_map(|i| i.to_string()), + ], + 0..3, + ) +} + +fn call(func: &str, target: &str, path: &[String]) -> String { + let args = std::iter::once(target.to_string()) + .chain(path.iter().cloned()) + .collect::>(); + format!("{func}({})", args.join(", ")) +} + +/// `json_object_keys` and `json_get_array` return lists, and `json_length` takes no path +/// beyond the document, but every function still accepts the same argument shape. +fn callable(func: &str, path: &[String]) -> bool { + // json_contains needs at least one path element to mean anything + !(func == "json_contains" && path.is_empty()) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// No input, however malformed, may panic or raise an error. The accessors report "not +/// found" as NULL, and invalid JSON is just another way of not finding anything. +#[test] +fn prop_no_panic_on_arbitrary_json() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let result = rt.block_on(outcome(&ctx, &sql)); + prop_assert!(result.is_ok(), "{sql}\n json: {json:?}\n {result:?}"); + } + }); +} + +/// A literal document is const-evaluated through each UDF's `ScalarValue` path; a column goes +/// through its array path. The two must not disagree. +#[test] +fn prop_scalar_and_array_paths_agree() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + let literal = format!("'{}'", json.replace('\'', "''")); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let from_column = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let from_literal = format!("select {} as v", call(func, &literal, &path)); + let column = rt.block_on(outcome(&ctx, &from_column)); + let scalar = rt.block_on(outcome(&ctx, &from_literal)); + prop_assert_eq!(&column, &scalar, + "\n column: {} => {:?}\n literal: {} => {:?}\n json: {:?}\n", + from_column, column, from_literal, scalar, json); + } + }); +} + +/// The four string encodings a JSON column can arrive in are four separate code paths, and +/// they must all produce the same answer. +/// +/// Dictionary-encoded input keeps its encoding in the output, so the dictionary wrapper is +/// stripped before comparing types. `json_get` is left out: it returns the JSON union, which +/// has no meaningful text form to compare — `json_as_text` and `json_get_json` cover the same +/// scanning code and do return text. `json_get_array` is left out too, but for a different +/// reason — see [`json_get_array_errors_on_dictionary_input`]. +#[test] +fn prop_input_encoding_does_not_change_result() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(48), |(json in json_text(), path in path())| { + for func in PATH_FUNCS { + if !callable(func, &path) || matches!(*func, "json_get" | "json_get_array") { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + + set_doc(&ctx, &json, &DataType::Utf8View); + let baseline = rt.block_on(outcome(&ctx, &sql)); + + for encoding in ENCODINGS.iter().cloned().chain(std::iter::once(dict_encoding())) { + set_doc(&ctx, &json, &encoding); + let got = rt.block_on(outcome(&ctx, &sql)); + prop_assert_eq!(&got, &baseline, + "\n {} in {} => {:?}\n in Utf8View => {:?}\n json: {:?}\n", + sql, encoding, got, baseline, json); + } + } + }); +} + +/// `json_get_array` is unusable on a dictionary-encoded JSON column: every call fails with an +/// internal error, whatever the document and whatever the path. +/// +/// `return_type_check` wraps the declared return type in `Dictionary(Int64, ..)` whenever the +/// first argument is a dictionary and the value type is not primitive. `json_get_array` is the +/// only UDF here whose value type is not primitive (`List(Utf8)`) *and* whose `InvokeResult` +/// sets `ACCEPT_DICT_RETURN = false`, so `invoke_array_array` hands back a bare `List` while +/// `return_type` promised a dictionary, and `DataFusion` rejects the mismatch. +/// +/// The fix is to stop promising the wrapper for this one function, i.e. teach +/// `return_type_check` about `ACCEPT_DICT_RETURN` (or unwrap the dictionary in +/// `JsonGetArray::return_type`). **Delete this test once that lands**, and drop +/// `json_get_array` from the exclusion in [`prop_input_encoding_does_not_change_result`], +/// which then covers it. +#[test] +fn json_get_array_errors_on_dictionary_input() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + for json in [r"[1, 2, 3]", r#"{"a": [1, 2]}"#, "null", "not json"] { + set_doc(&ctx, json, &dict_encoding()); + let err = rt + .block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect_err("json_get_array over a dictionary column should still be failing"); + // debug builds trip DataFusion's own assertion, release builds reach Arrow's schema + // check; both name the dictionary type that `return_type` promised and did not deliver + assert!( + err.contains("Dictionary(Int64, List("), + "unexpected error for {json:?}: {err}" + ); + + // the same document in a plain string column works + set_doc(&ctx, json, &DataType::Utf8View); + rt.block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect("json_get_array over a string column works"); + } +} diff --git a/tests/main.rs b/tests/main.rs index b909ec0..f118295 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -300,13 +300,13 @@ async fn test_json_get_str_null() { #[tokio::test] async fn test_json_get_no_path() { let batches = run_query(r#"select json_get('"foo"')::string"#).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, "foo".to_string())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, "foo".to_string())); let batches = run_query(r"select json_get('123')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "123".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "123".to_string())); let batches = run_query(r"select json_get('true')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] @@ -386,11 +386,11 @@ async fn test_json_get_out_of_range_int() { // casts of the null union stay null rather than erroring let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::string"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, String::new())); } #[tokio::test] @@ -512,19 +512,19 @@ async fn test_json_get_path() { async fn test_json_get_cast_int() { let sql = r#"select json_get('{"foo": 42}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); // floats not allowed let sql = r#"select json_get('{"foo": 4.2}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] async fn test_json_get_cast_int_path() { let sql = r#"select json_get('{"foo": [null, {"x": false, "bar": 73}}', 'foo', 1, 'bar')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "73".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "73".to_string())); } #[tokio::test] @@ -567,14 +567,17 @@ async fn test_json_get_float() { async fn test_json_get_cast_float() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::float"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!(display_val(batches).await, (DataType::Float32, "420.0".to_string())); } #[tokio::test] async fn test_json_get_cast_numeric() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::numeric"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!( + display_val(batches).await, + (DataType::Decimal128(38, 10), "420.0000000000".to_string()) + ); } #[tokio::test] @@ -1202,7 +1205,7 @@ async fn test_json_get_int_unnest() { async fn test_plan_json_get_int_unnest() { let sql = "explain select json_get(json_get(json_data, 'foo'), 0)::int v from test"; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS v", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS v", " TableScan: test projection=[json_data]", ]; @@ -1426,7 +1429,7 @@ async fn test_arrow_cast_int() { for_all_json_datatypes(async |dt| { let batches = run_query_datatype(sql, dt).await.unwrap(); assert_batches_eq!(expected, &batches); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); }) .await; } @@ -1436,7 +1439,7 @@ async fn test_plan_arrow_cast_int() { let lines = logical_plan(r"explain select (json_data->'foo')::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\")) AS json_data -> 'foo'", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\")) AS Int32) AS json_data -> 'foo'", " TableScan: test projection=[json_data]", ]; @@ -1508,14 +1511,15 @@ async fn test_arrow_try_cast_fn_str() { assert_eq!(display_val(batches).await, (DataType::Utf8, "bar".to_string())); } -/// `arrow_cast` names an exact Arrow type, so a type no accessor returns exactly is left alone -/// rather than widened to the accessor's type. +/// `arrow_cast` names an exact Arrow type. A type no accessor returns exactly is still folded, so +/// the union is not materialized, but the cast to that type is kept so the output type is the one +/// that was named. #[tokio::test] -async fn test_plan_arrow_cast_fn_inexact_type_not_folded() { +async fn test_plan_arrow_cast_fn_narrowing_type_keeps_cast() { let lines = logical_plan(r"explain select arrow_cast((json_data->'foo'), 'Int32') from test").await; let expected = [ - "Projection: CAST(json_get(test.json_data, Utf8(\"foo\")) AS Int32) AS arrow_cast(json_data -> 'foo',Utf8(\"Int32\"))", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\")) AS Int32) AS arrow_cast(json_data -> 'foo',Utf8(\"Int32\"))", " TableScan: test projection=[json_data]", ]; @@ -1642,7 +1646,7 @@ async fn test_plan_arrow_double_nested_cast() { let lines = logical_plan(r"explain select (json_data->'foo'->0)::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS json_data -> 'foo' -> 0", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS json_data -> 'foo' -> 0", " TableScan: test projection=[json_data]", ]; diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs new file mode 100644 index 0000000..437d0a6 --- /dev/null +++ b/tests/rewrite_differential.rs @@ -0,0 +1,658 @@ +//! Differential tests for the expression folding done by `JsonFunctionRewriter`. +//! +//! The rewriter folds `CAST(json_get(x, 'k') AS BIGINT)` down to `json_get_int(x, 'k')` so the +//! JSON union is never materialized just to be cast away, and it flattens nested `json_get` +//! calls into a single multi-argument call. Both are optimizations, so both are only correct +//! if they are invisible to the caller. These tests generate input and check that. +//! +//! # Running the same query with and without the rewrite +//! +//! Function rewrites run in the *analyzer*, before the optimizer merges projections. A cast +//! applied to a column of a subquery therefore never sees a `json_get` underneath it and is +//! not folded — and then `merge_projection` inlines the `json_get` back under the cast. So +//! [`unfolded_sql`] gives the plan we would have had with no rewriter registered at all, +//! without needing a second `SessionContext`. [`barrier_really_blocks_folding`] keeps that +//! trick honest. +//! +//! For nested `json_get` flattening this gives a straightforward invariant, checked by +//! [`prop_unnest_is_invisible`]: the flattened and un-flattened plans must agree. +//! +//! For cast folding it does **not**: casting the JSON union is much weaker than the typed +//! accessors (`CAST(union AS Float64)` is NULL even for `42` — see +//! [`unfolded_union_cast_is_weaker`]), so the rewrite is deliberately *better* than the +//! expression it replaces and equality is the wrong assertion. What the fold does promise is +//! +//! ```text +//! CAST(json_get(x, 'k') AS T) == CAST(json_get_(x, 'k') AS T) +//! ``` +//! +//! i.e. "use the typed accessor, then apply the cast the user actually wrote". The +//! right-hand side is not itself folded — the rewriter only folds casts over `json_get` — so +//! it serves as the reference implementation. That is [`prop_fold_is_invisible`]. +//! +//! That is also, literally, what the rewriter emits: it replaces the cast's *input*, and drops +//! the cast only where the accessor already returns the type that was asked for. The tests keep +//! both sides anyway, because the reference is built from the SQL rather than from the rewriter, +//! so a change to which accessor a type maps to still shows up here. + +use std::sync::Arc; + +use datafusion::arrow::array::{RecordBatch, StringViewArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +// --------------------------------------------------------------------------------------- +// the cast matrix +// --------------------------------------------------------------------------------------- + +/// What the rewriter does with a given cast of a `json_get` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fold { + /// Left alone: the cast is applied to the JSON union. + None, + /// Folded to the typed accessor, whose return type *is* the type that was asked for, so + /// the cast is dropped entirely. + Exact, + /// Folded to the typed accessor, which returns a wider type, so the cast to the type that + /// was asked for is kept on top of it. The union is still never materialized. + Narrowing, +} + +/// A cast target: how it is spelled in SQL, the exact Arrow type `arrow_cast` names for it, +/// the typed accessor the rewriter folds it to, and what each of the two folding paths does +/// with it. +#[derive(Debug, Clone, Copy)] +struct Target { + sql: &'static str, + arrow: &'static str, + accessor: &'static str, + sql_cast: Fold, + arrow_cast: Fold, +} + +const TARGETS: &[Target] = &[ + Target { + sql: "bigint", + arrow: "Int64", + accessor: "json_get_int", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + Target { + sql: "double", + arrow: "Float64", + accessor: "json_get_float", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + Target { + sql: "boolean", + arrow: "Boolean", + accessor: "json_get_bool", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + // SQL `VARCHAR` is `Utf8View` in DataFusion, but `json_get_str` returns `Utf8`. + Target { + sql: "varchar", + arrow: "Utf8", + accessor: "json_get_str", + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Exact, + }, + // Narrowing targets: the type asked for is narrower than what the accessor returns, so the + // fold keeps the cast on top of the accessor. + Target { + sql: "int", + arrow: "Int32", + accessor: "json_get_int", + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, + }, + Target { + sql: "real", + arrow: "Float32", + accessor: "json_get_float", + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, + }, + Target { + sql: "decimal(10,2)", + arrow: "Decimal128(10, 2)", + accessor: "json_get_float", + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, + }, + // A type neither path has an accessor for, so neither folds it. + Target { + sql: "smallint", + arrow: "Int16", + accessor: "json_get_int", + sql_cast: Fold::None, + arrow_cast: Fold::None, + }, +]; + +/// The spellings of a cast that all mean the same thing to a user. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Spelling { + Cast, + DoubleColon, + TryCast, + ArrowCast, + ArrowTryCast, +} + +const SPELLINGS: &[Spelling] = &[ + Spelling::Cast, + Spelling::DoubleColon, + Spelling::TryCast, + Spelling::ArrowCast, + Spelling::ArrowTryCast, +]; + +impl Spelling { + /// Write `inner` cast to `target`. + fn apply(self, inner: &str, target: Target) -> String { + match self { + Spelling::Cast => format!("cast({inner} as {})", target.sql), + Spelling::DoubleColon => format!("({inner})::{}", target.sql), + Spelling::TryCast => format!("try_cast({inner} as {})", target.sql), + Spelling::ArrowCast => format!("arrow_cast({inner}, '{}')", target.arrow), + Spelling::ArrowTryCast => format!("arrow_try_cast({inner}, '{}')", target.arrow), + } + } + + fn names_an_arrow_type(self) -> bool { + matches!(self, Spelling::ArrowCast | Spelling::ArrowTryCast) + } + + /// What the rewriter does with this spelling of this target. + fn fold(self, target: Target) -> Fold { + if self.names_an_arrow_type() { + target.arrow_cast + } else { + target.sql_cast + } + } +} + +// --------------------------------------------------------------------------------------- +// documents and SQL +// --------------------------------------------------------------------------------------- + +/// A JSON document and a lookup path into it that reaches the interesting value. +#[derive(Debug, Clone)] +struct Doc { + json: String, + path: Vec, +} + +impl Doc { + /// `json_get(, )`, the expression the rewriter folds. + fn json_get(&self) -> String { + self.call("json_get") + } + + /// The same lookup through a typed accessor, which the rewriter leaves alone. + fn accessor(&self, name: &str) -> String { + self.call(name) + } + + fn call(&self, func: &str) -> String { + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(self.path.iter().map(|p| path_arg(p))) + .collect::>(); + format!("{func}({})", args.join(", ")) + } + + /// `json_get(j, k1, .., k(n-1))`: the lookup for everything but the last path segment, + /// which is what an outer accessor call gets nested around. `None` for a one-element + /// path, where there is nothing to nest. + fn prefix_json_get(&self) -> Option { + let (_, prefix) = self.path.split_last()?; + if prefix.is_empty() { + return None; + } + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(prefix.iter().map(|p| path_arg(p))) + .collect::>(); + Some(format!("json_get({})", args.join(", "))) + } +} + +/// Render a path segment: integers stay integers so array indexing is exercised. +fn path_arg(segment: &str) -> String { + if segment.chars().all(|c| c.is_ascii_digit()) { + segment.to_string() + } else { + sql_str(segment) + } +} + +/// Quote a string as a SQL literal. +fn sql_str(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +/// Point the single-row table `t` at `json`. +/// +/// The document has to arrive as column data rather than as a SQL literal: `DataFusion` +/// const-evaluates a projection over literals down to a single scalar, which erases the very +/// plan shape these tests assert on. +fn set_doc(ctx: &SessionContext, json: &str) { + let schema = Schema::new(vec![Field::new(JSON_COLUMN, DataType::Utf8View, false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(StringViewArray::from(vec![json]))]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +fn select(expr: &str) -> String { + format!("select {expr} as v from {JSON_TABLE}") +} + +/// Wrap `expr` so the rewriter cannot see through to the `json_get` underneath. +/// +/// See the module docs: the analyzer sees a cast over a subquery column and leaves it alone, +/// then the optimizer inlines the `json_get` back underneath. The result evaluates as if no +/// rewrite had been registered. +fn unfolded_sql(inner: &str, wrap: impl FnOnce(&str) -> String) -> String { + format!("select {} as v from (select {inner} as x from {JSON_TABLE})", wrap("x")) +} + +// --------------------------------------------------------------------------------------- +// running +// --------------------------------------------------------------------------------------- + +/// The result of a query: coarse enough not to be brittle about error text, precise enough +/// to catch a changed type, a changed value, or a cast that should have failed but did not. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Outcome { + Value(DataType, String), + Error, +} + +impl std::fmt::Display for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Outcome::Error => write!(f, "ERROR"), + Outcome::Value(dt, v) if v.is_empty() => write!(f, "{dt}=NULL"), + Outcome::Value(dt, v) => write!(f, "{dt}={v}"), + } + } +} + +async fn outcome(ctx: &SessionContext, sql: &str) -> Outcome { + let Ok(df) = ctx.sql(sql).await else { + return Outcome::Error; + }; + let Ok(batches) = df.collect().await else { + return Outcome::Error; + }; + assert_eq!(batches.len(), 1, "expected one batch from {sql}"); + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 1, "expected one row from {sql}"); + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).unwrap(); + Outcome::Value( + batch.schema().field(0).data_type().clone(), + formatter.value(0).try_to_string().unwrap(), + ) +} + +async fn logical_plan(ctx: &SessionContext, sql: &str) -> String { + let batches = ctx + .sql(&format!("explain {sql}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + let options = FormatOptions::default(); + let formatter = ArrayFormatter::try_new(batches[0].column(1).as_ref(), &options).unwrap(); + formatter.value(0).try_to_string().unwrap() +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// A single JSON value, weighted towards the values where the typed accessors and Arrow's +/// cast kernels are most likely to disagree. +fn json_scalar() -> impl Strategy { + prop_oneof![ + 1 => Just("null".to_string()), + 2 => any::().prop_map(|b| b.to_string()), + 4 => any::().prop_map(|i| i.to_string()), + 3 => any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + 2 => "[a-zA-Z0-9 ._+-]{0,8}".prop_map(|s| format!("\"{s}\"")), + // numeric- and boolean-looking strings: the accessors are lenient here, Arrow is not + 2 => any::().prop_map(|i| format!("\"{i}\"")), + 1 => Just("\"true\"".to_string()), + // integers wider than i64, which take jiter's slow path + 1 => Just("123456789012345678901234567890".to_string()), + 1 => Just("-123456789012345678901234567890".to_string()), + // nested values, which the accessors report as absent rather than as a value + 1 => Just("[1, 2]".to_string()), + 1 => Just(r#"{"c": 1}"#.to_string()), + ] +} + +/// A JSON value nested somewhere, together with the path that reaches it. +fn doc() -> impl Strategy { + (json_scalar(), 0usize..4).prop_map(|(value, shape)| match shape { + 0 => Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }, + 1 => Doc { + json: format!(r#"{{"a": {{"b": {value}}}}}"#), + path: vec!["a".to_string(), "b".to_string()], + }, + 2 => Doc { + json: format!("[{value}]"), + path: vec!["0".to_string()], + }, + _ => Doc { + json: format!(r#"{{"a": [1, {value}]}}"#), + path: vec!["a".to_string(), "1".to_string()], + }, + }) +} + +/// A `(spelling, target)` pair the rewriter folds, exactly or with a cast kept on top. Either +/// way the fold has to be invisible. +fn folding_cast() -> impl Strategy { + let pairs: Vec<(Spelling, Target)> = SPELLINGS + .iter() + .flat_map(|&s| TARGETS.iter().map(move |&t| (s, t))) + .filter(|&(s, t)| s.fold(t) != Fold::None) + .collect(); + prop::sample::select(pairs) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// Folding must be indistinguishable from applying the same cast to the typed accessor — +/// which is exactly what the rewriter now emits for a narrowing cast, and what the exact case +/// simplifies down to. +#[test] +fn prop_fold_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc(), (spelling, target) in folding_cast())| { + set_doc(&ctx, &doc.json); + let folded = select(&spelling.apply(&doc.json_get(), target)); + let reference = select(&spelling.apply(&doc.accessor(target.accessor), target)); + + let got = rt.block_on(outcome(&ctx, &folded)); + let want = rt.block_on(outcome(&ctx, &reference)); + + prop_assert_eq!( + &got, &want, + "\n folded: {} => {}\n reference: {} => {}\n", folded, got, reference, want + ); + }); +} + +/// Flattening `json_get(json_get(j, 'a'), 'b')` into `json_get(j, 'a', 'b')` must not change +/// the answer. Here the plan without the rewrite really is the right reference, so this is +/// the plain with/without-optimizer differential the barrier was built for. +#[test] +fn prop_unnest_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc())| { + let Some(prefix) = doc.prefix_json_get() else { + return Ok(()); + }; + set_doc(&ctx, &doc.json); + + // `json_get` returns the JSON union, which formats as an opaque value, so read the + // result out through `json_as_text`. The rewriter flattens the `json_get` underneath + // it into the outer call; the barrier stops it from seeing the nesting at all. + let last = path_arg(doc.path.last().unwrap()); + let unnested = select(&format!("json_as_text({prefix}, {last})")); + let not_unnested = unfolded_sql(&prefix, |x| format!("json_as_text({x}, {last})")); + + // the comparison only means something while each side is the plan it claims to be + let plan = rt.block_on(logical_plan(&ctx, ¬_unnested)); + prop_assert!( + plan.contains("json_as_text(json_get("), + "barrier stopped blocking the unnest:\n{}\n{}", not_unnested, plan + ); + let unnested_plan = rt.block_on(logical_plan(&ctx, &unnested)); + prop_assert!( + !unnested_plan.contains("json_as_text(json_get("), + "expected the nested call to be flattened:\n{}\n{}", unnested, unnested_plan + ); + + let got = rt.block_on(outcome(&ctx, &unnested)); + let want = rt.block_on(outcome(&ctx, ¬_unnested)); + prop_assert_eq!(&got, &want, + "\n unnested: {} => {}\n not unnested: {} => {}\n", + unnested, got, not_unnested, want); + }); +} + +// --------------------------------------------------------------------------------------- +// plan shape: which spellings fold +// --------------------------------------------------------------------------------------- + +/// Every spelling of a cast a user might reach for folds to the same accessor, and the plan +/// keeps the cast exactly where the accessor's type is not the one that was asked for. +#[test] +fn every_cast_spelling_folds_as_documented() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = select(&spelling.apply(&doc.json_get(), target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + let fold = spelling.fold(target); + + assert_eq!( + plan.contains(target.accessor), + fold != Fold::None, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", + target.sql + ); + if fold == Fold::None { + continue; + } + assert!( + !plan.contains("json_get(t."), + "fold left the union accessor behind for {spelling:?} / {}\n{plan}", + target.sql + ); + // "CAST(json_get" also matches "TRY_CAST(json_get" + assert_eq!( + plan.contains(&format!("CAST({}(t.", target.accessor)), + fold == Fold::Narrowing, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } +} + +// --------------------------------------------------------------------------------------- +// narrowing casts +// --------------------------------------------------------------------------------------- + +/// Folding a cast whose type no accessor returns exactly still yields that type, and still +/// narrows. Keeping the cast on top of the accessor is what buys this; folding to the bare +/// accessor would hand back the accessor's wider type and skip the narrowing entirely. +#[test] +fn narrowing_cast_preserves_type_and_narrows() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + // (json value, sql type, CAST outcome, TRY_CAST outcome) + let cases = [ + ("42", "int", "Int32=42", "Int32=42"), + ("42", "real", "Float32=42.0", "Float32=42.0"), + ( + "42", + "decimal(10,2)", + "Decimal128(10, 2)=42.00", + "Decimal128(10, 2)=42.00", + ), + (r#""abc""#, "varchar", "Utf8View=abc", "Utf8View=abc"), + // out of the target type's range: CAST fails, TRY_CAST yields NULL + ("3000000000", "int", "ERROR", "Int32=NULL"), + ("9223372036854775807", "int", "ERROR", "Int32=NULL"), + ("3000000000", "decimal(10,2)", "ERROR", "Decimal128(10, 2)=NULL"), + ]; + + for (value, sql_type, want_cast, want_try_cast) in cases { + let target = TARGETS.iter().find(|t| t.sql == sql_type).unwrap(); + let doc = Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for (spelling, want) in [(Spelling::Cast, want_cast), (Spelling::TryCast, want_try_cast)] { + let sql = select(&spelling.apply(&doc.json_get(), *target)); + assert_eq!(rt.block_on(outcome(&ctx, &sql)).to_string(), want, "{sql}"); + } + } +} + +/// The narrowing fold is not just correct, it is the whole point: the JSON union never gets +/// materialized, the plan reads the value straight out with the typed accessor and casts that. +#[test] +fn narrowing_cast_still_avoids_the_union() { + let rt = runtime(); + let ctx = create_context().unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + let target = TARGETS.iter().find(|t| t.sql == "int").unwrap(); + + let plan = rt.block_on(logical_plan( + &ctx, + &select(&Spelling::Cast.apply(&doc.json_get(), *target)), + )); + assert!( + plan.contains("CAST(json_get_int(t.j, Utf8(\"a\")) AS Int32)"), + "unexpected plan:\n{plan}" + ); +} + +/// Keeping the cast must not cost the accessor its trip to the leaf nodes. +/// +/// `JsonGet`'s `placement` moves a `json_get` over a column with literal path arguments towards +/// the leaves, which is where the work wants to happen for a filter. A cast sitting on top of the +/// accessor is exactly the sort of thing that could strand it higher up the plan, so pin it. +#[test] +fn narrowing_cast_still_reaches_the_leaf_nodes() { + let rt = runtime(); + let ctx = create_context().unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); + + for target in TARGETS.iter().filter(|t| t.sql_cast != Fold::None) { + let predicate = Spelling::Cast.apply(&format!("json_get({JSON_COLUMN}, 'a')"), *target); + let sql = format!("select {JSON_COLUMN} from {JSON_TABLE} where {predicate} is not null"); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + + let leaf_projection = plan + .lines() + .rfind(|line| line.trim_start().starts_with("Projection:")) + .unwrap_or_default(); + assert!( + leaf_projection.contains(&format!("{}(t.j", target.accessor)), + "accessor did not reach the leaf projection for {}\n{plan}", + target.sql + ); + } +} + +// --------------------------------------------------------------------------------------- +// the barrier, and why the naive differential does not work for casts +// --------------------------------------------------------------------------------------- + +/// The subquery barrier must actually stop the fold, or every test that uses it as the +/// "without the rewrite" reference silently becomes a tautology. +#[test] +fn barrier_really_blocks_folding() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = unfolded_sql(&doc.json_get(), |x| spelling.apply(x, target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + // the json_get survives un-folded, and the optimizer inlined it back under the + // cast, so this is one expression and not an extra projection + assert!( + plan.contains("CAST(json_get(t.") || plan.contains("TRY_CAST(json_get(t."), + "barrier no longer blocks the fold for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } +} + +/// Why cast folding cannot be checked by diffing against the un-folded plan: casting the JSON +/// union is much weaker than the typed accessors, so the two disagree by design. +/// +/// This is the reason [`prop_fold_is_invisible`] compares against the typed accessor instead. +#[test] +fn unfolded_union_cast_is_weaker() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + let target = TARGETS.iter().find(|t| t.sql == "double").unwrap(); + + set_doc(&ctx, &doc.json); + let folded = select(&Spelling::Cast.apply(&doc.json_get(), *target)); + let unfolded = unfolded_sql(&doc.json_get(), |x| Spelling::Cast.apply(x, *target)); + + assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), "Float64=42.0"); + // the union has no Float64 member holding an integer, so the cast yields NULL + assert_eq!(rt.block_on(outcome(&ctx, &unfolded)).to_string(), "Float64=NULL"); +}