From fdc6d9245bc06306ea8444a8d2cb27a8f6aa8f3c Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:03:36 +0000 Subject: [PATCH 1/5] Add differential property tests for JsonFunctionRewriter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rewriter's cast folding and nested-call flattening are optimizations, so both are only correct if they are invisible to the caller. `tests/main.rs` checks that with hand-written examples, one per spelling; these generate input instead. Function rewrites run in the analyzer, before the optimizer merges projections, so a cast applied to a subquery column never sees the `json_get` underneath it and is not folded — and `merge_projection` then inlines the `json_get` back under the cast. That yields the un-rewritten plan without needing a second `SessionContext`, and `barrier_really_blocks_folding` keeps it honest. For nested `json_get` flattening that gives the plain with/without-optimizer differential, `prop_unnest_is_invisible`. For cast folding it does not: casting the JSON union is far weaker than the typed accessors (`CAST(union AS Float64)` is NULL even for `42`), so the fold is deliberately better than what it replaces. `prop_fold_is_invisible` instead checks the equivalence the fold actually promises, that folding equals using the typed accessor and then applying the cast that was written. That equivalence holds only where the accessor returns exactly the type asked for. `narrowing_cast_fold_is_not_type_preserving` and `narrowing_try_cast_fold_returns_value_instead_of_null` pin down where it does not: the `CAST`/`TRY_CAST` table folds `int`, `real`, `decimal` and `varchar` to accessors returning `Int64`, `Float64` and `Utf8`, so the expression takes the accessor's type and the narrowing never happens. Both tests characterize current behaviour so they stay green, and say what to do when it is fixed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- Cargo.toml | 1 + tests/rewrite_differential.rs | 622 ++++++++++++++++++++++++++++++++++ 2 files changed, 623 insertions(+) create mode 100644 tests/rewrite_differential.rs diff --git a/Cargo.toml b/Cargo.toml index 249d5e5..3078a43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ datafusion = { version = "54", default-features = false, features = [ "nested_expressions", "sql", ] } +proptest = "1.11.0" tokio = { version = "1.43", features = ["full"] } [lints.clippy] diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs new file mode 100644 index 0000000..c9b8a86 --- /dev/null +++ b/tests/rewrite_differential.rs @@ -0,0 +1,622 @@ +//! Differential tests for the expression folding done by `JsonFunctionRewriter`. +//! +//! The rewriter folds `CAST(json_get(x, 'k') AS BIGINT)` down to `json_get_int(x, 'k')` so the +//! JSON union is never materialized just to be cast away, and it flattens nested `json_get` +//! calls into a single multi-argument call. Both are optimizations, so both are only correct +//! if they are invisible to the caller. These tests generate input and check that. +//! +//! # Running the same query with and without the rewrite +//! +//! Function rewrites run in the *analyzer*, before the optimizer merges projections. A cast +//! applied to a column of a subquery therefore never sees a `json_get` underneath it and is +//! not folded — and then `merge_projection` inlines the `json_get` back under the cast. So +//! [`unfolded_sql`] gives the plan we would have had with no rewriter registered at all, +//! without needing a second `SessionContext`. [`barrier_really_blocks_folding`] keeps that +//! trick honest. +//! +//! For nested `json_get` flattening this gives a straightforward invariant, checked by +//! [`prop_unnest_is_invisible`]: the flattened and un-flattened plans must agree. +//! +//! For cast folding it does **not**: casting the JSON union is much weaker than the typed +//! accessors (`CAST(union AS Float64)` is NULL even for `42` — see +//! [`unfolded_union_cast_is_weaker`]), so the rewrite is deliberately *better* than the +//! expression it replaces and equality is the wrong assertion. What the fold does promise is +//! +//! ```text +//! CAST(json_get(x, 'k') AS T) == CAST(json_get_(x, 'k') AS T) +//! ``` +//! +//! i.e. "use the typed accessor, then apply the cast the user actually wrote". The +//! right-hand side is not itself folded — the rewriter only folds casts over `json_get` — so +//! it serves as the reference implementation. That is [`prop_fold_is_invisible`]. + +use std::sync::Arc; + +use datafusion::arrow::array::{RecordBatch, StringViewArray}; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +// --------------------------------------------------------------------------------------- +// the cast matrix +// --------------------------------------------------------------------------------------- + +/// What the rewriter does with a given cast of a `json_get` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Fold { + /// Left alone: the cast is applied to the JSON union. + None, + /// Folded to the typed accessor, whose return type *is* the type that was asked for. + Exact, + /// Folded to the typed accessor, whose return type is **not** the type that was asked + /// for. The expression silently changes type and the narrowing never happens. + Widening, +} + +/// A cast target: how it is spelled in SQL, the exact Arrow type `arrow_cast` names for it, +/// the typed accessor the rewriter folds it to, and what each of the two folding paths does +/// with it. +#[derive(Debug, Clone, Copy)] +struct Target { + sql: &'static str, + arrow: &'static str, + accessor: &'static str, + sql_cast: Fold, + arrow_cast: Fold, +} + +const TARGETS: &[Target] = &[ + Target { + sql: "bigint", + arrow: "Int64", + accessor: "json_get_int", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + Target { + sql: "double", + arrow: "Float64", + accessor: "json_get_float", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + Target { + sql: "boolean", + arrow: "Boolean", + accessor: "json_get_bool", + sql_cast: Fold::Exact, + arrow_cast: Fold::Exact, + }, + // SQL `VARCHAR` is `Utf8View` in DataFusion, but `json_get_str` returns `Utf8`. + Target { + sql: "varchar", + arrow: "Utf8", + accessor: "json_get_str", + sql_cast: Fold::Widening, + arrow_cast: Fold::Exact, + }, + // Narrowing targets: the type asked for is narrower than what the accessor returns. The + // `CAST` table folds them anyway; the `arrow_*` handler deliberately does not. + Target { + sql: "int", + arrow: "Int32", + accessor: "json_get_int", + sql_cast: Fold::Widening, + arrow_cast: Fold::None, + }, + Target { + sql: "real", + arrow: "Float32", + accessor: "json_get_float", + sql_cast: Fold::Widening, + arrow_cast: Fold::None, + }, + Target { + sql: "decimal(10,2)", + arrow: "Decimal128(10, 2)", + accessor: "json_get_float", + sql_cast: Fold::Widening, + arrow_cast: Fold::None, + }, + // A type neither path has an accessor for, so neither folds it. + Target { + sql: "smallint", + arrow: "Int16", + accessor: "json_get_int", + sql_cast: Fold::None, + arrow_cast: Fold::None, + }, +]; + +/// The spellings of a cast that all mean the same thing to a user. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Spelling { + Cast, + DoubleColon, + TryCast, + ArrowCast, + ArrowTryCast, +} + +const SPELLINGS: &[Spelling] = &[ + Spelling::Cast, + Spelling::DoubleColon, + Spelling::TryCast, + Spelling::ArrowCast, + Spelling::ArrowTryCast, +]; + +impl Spelling { + /// Write `inner` cast to `target`. + fn apply(self, inner: &str, target: Target) -> String { + match self { + Spelling::Cast => format!("cast({inner} as {})", target.sql), + Spelling::DoubleColon => format!("({inner})::{}", target.sql), + Spelling::TryCast => format!("try_cast({inner} as {})", target.sql), + Spelling::ArrowCast => format!("arrow_cast({inner}, '{}')", target.arrow), + Spelling::ArrowTryCast => format!("arrow_try_cast({inner}, '{}')", target.arrow), + } + } + + fn names_an_arrow_type(self) -> bool { + matches!(self, Spelling::ArrowCast | Spelling::ArrowTryCast) + } + + /// What the rewriter does with this spelling of this target. + fn fold(self, target: Target) -> Fold { + if self.names_an_arrow_type() { + target.arrow_cast + } else { + target.sql_cast + } + } +} + +// --------------------------------------------------------------------------------------- +// documents and SQL +// --------------------------------------------------------------------------------------- + +/// A JSON document and a lookup path into it that reaches the interesting value. +#[derive(Debug, Clone)] +struct Doc { + json: String, + path: Vec, +} + +impl Doc { + /// `json_get(, )`, the expression the rewriter folds. + fn json_get(&self) -> String { + self.call("json_get") + } + + /// The same lookup through a typed accessor, which the rewriter leaves alone. + fn accessor(&self, name: &str) -> String { + self.call(name) + } + + fn call(&self, func: &str) -> String { + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(self.path.iter().map(|p| path_arg(p))) + .collect::>(); + format!("{func}({})", args.join(", ")) + } + + /// `json_get(j, k1, .., k(n-1))`: the lookup for everything but the last path segment, + /// which is what an outer accessor call gets nested around. `None` for a one-element + /// path, where there is nothing to nest. + fn prefix_json_get(&self) -> Option { + let (_, prefix) = self.path.split_last()?; + if prefix.is_empty() { + return None; + } + let args = std::iter::once(JSON_COLUMN.to_string()) + .chain(prefix.iter().map(|p| path_arg(p))) + .collect::>(); + Some(format!("json_get({})", args.join(", "))) + } +} + +/// Render a path segment: integers stay integers so array indexing is exercised. +fn path_arg(segment: &str) -> String { + if segment.chars().all(|c| c.is_ascii_digit()) { + segment.to_string() + } else { + sql_str(segment) + } +} + +/// Quote a string as a SQL literal. +fn sql_str(s: &str) -> String { + format!("'{}'", s.replace('\'', "''")) +} + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +/// Point the single-row table `t` at `json`. +/// +/// The document has to arrive as column data rather than as a SQL literal: `DataFusion` +/// const-evaluates a projection over literals down to a single scalar, which erases the very +/// plan shape these tests assert on. +fn set_doc(ctx: &SessionContext, json: &str) { + let schema = Schema::new(vec![Field::new(JSON_COLUMN, DataType::Utf8View, false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(StringViewArray::from(vec![json]))]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +fn select(expr: &str) -> String { + format!("select {expr} as v from {JSON_TABLE}") +} + +/// Wrap `expr` so the rewriter cannot see through to the `json_get` underneath. +/// +/// See the module docs: the analyzer sees a cast over a subquery column and leaves it alone, +/// then the optimizer inlines the `json_get` back underneath. The result evaluates as if no +/// rewrite had been registered. +fn unfolded_sql(inner: &str, wrap: impl FnOnce(&str) -> String) -> String { + format!("select {} as v from (select {inner} as x from {JSON_TABLE})", wrap("x")) +} + +// --------------------------------------------------------------------------------------- +// running +// --------------------------------------------------------------------------------------- + +/// The result of a query: coarse enough not to be brittle about error text, precise enough +/// to catch a changed type, a changed value, or a cast that should have failed but did not. +#[derive(Debug, Clone, PartialEq, Eq)] +enum Outcome { + Value(DataType, String), + Error, +} + +impl std::fmt::Display for Outcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Outcome::Error => write!(f, "ERROR"), + Outcome::Value(dt, v) if v.is_empty() => write!(f, "{dt}=NULL"), + Outcome::Value(dt, v) => write!(f, "{dt}={v}"), + } + } +} + +async fn outcome(ctx: &SessionContext, sql: &str) -> Outcome { + let Ok(df) = ctx.sql(sql).await else { + return Outcome::Error; + }; + let Ok(batches) = df.collect().await else { + return Outcome::Error; + }; + assert_eq!(batches.len(), 1, "expected one batch from {sql}"); + let batch = &batches[0]; + assert_eq!(batch.num_rows(), 1, "expected one row from {sql}"); + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).unwrap(); + Outcome::Value( + batch.schema().field(0).data_type().clone(), + formatter.value(0).try_to_string().unwrap(), + ) +} + +async fn logical_plan(ctx: &SessionContext, sql: &str) -> String { + let batches = ctx + .sql(&format!("explain {sql}")) + .await + .unwrap() + .collect() + .await + .unwrap(); + let options = FormatOptions::default(); + let formatter = ArrayFormatter::try_new(batches[0].column(1).as_ref(), &options).unwrap(); + formatter.value(0).try_to_string().unwrap() +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// A single JSON value, weighted towards the values where the typed accessors and Arrow's +/// cast kernels are most likely to disagree. +fn json_scalar() -> impl Strategy { + prop_oneof![ + 1 => Just("null".to_string()), + 2 => any::().prop_map(|b| b.to_string()), + 4 => any::().prop_map(|i| i.to_string()), + 3 => any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + 2 => "[a-zA-Z0-9 ._+-]{0,8}".prop_map(|s| format!("\"{s}\"")), + // numeric- and boolean-looking strings: the accessors are lenient here, Arrow is not + 2 => any::().prop_map(|i| format!("\"{i}\"")), + 1 => Just("\"true\"".to_string()), + // integers wider than i64, which take jiter's slow path + 1 => Just("123456789012345678901234567890".to_string()), + 1 => Just("-123456789012345678901234567890".to_string()), + // nested values, which the accessors report as absent rather than as a value + 1 => Just("[1, 2]".to_string()), + 1 => Just(r#"{"c": 1}"#.to_string()), + ] +} + +/// A JSON value nested somewhere, together with the path that reaches it. +fn doc() -> impl Strategy { + (json_scalar(), 0usize..4).prop_map(|(value, shape)| match shape { + 0 => Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }, + 1 => Doc { + json: format!(r#"{{"a": {{"b": {value}}}}}"#), + path: vec!["a".to_string(), "b".to_string()], + }, + 2 => Doc { + json: format!("[{value}]"), + path: vec!["0".to_string()], + }, + _ => Doc { + json: format!(r#"{{"a": [1, {value}]}}"#), + path: vec!["a".to_string(), "1".to_string()], + }, + }) +} + +/// A `(spelling, target)` pair whose fold is type-exact, so the fold must be invisible. +fn exact_cast() -> impl Strategy { + let pairs: Vec<(Spelling, Target)> = SPELLINGS + .iter() + .flat_map(|&s| TARGETS.iter().map(move |&t| (s, t))) + .filter(|&(s, t)| s.fold(t) == Fold::Exact) + .collect(); + prop::sample::select(pairs) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// Folding a type-exact cast must be indistinguishable from applying that same cast to the +/// typed accessor. +#[test] +fn prop_fold_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc(), (spelling, target) in exact_cast())| { + set_doc(&ctx, &doc.json); + let folded = select(&spelling.apply(&doc.json_get(), target)); + let reference = select(&spelling.apply(&doc.accessor(target.accessor), target)); + + let got = rt.block_on(outcome(&ctx, &folded)); + let want = rt.block_on(outcome(&ctx, &reference)); + + prop_assert_eq!( + &got, &want, + "\n folded: {} => {}\n reference: {} => {}\n", folded, got, reference, want + ); + }); +} + +/// Flattening `json_get(json_get(j, 'a'), 'b')` into `json_get(j, 'a', 'b')` must not change +/// the answer. Here the plan without the rewrite really is the right reference, so this is +/// the plain with/without-optimizer differential the barrier was built for. +#[test] +fn prop_unnest_is_invisible() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(|(doc in doc())| { + let Some(prefix) = doc.prefix_json_get() else { + return Ok(()); + }; + set_doc(&ctx, &doc.json); + + // `json_get` returns the JSON union, which formats as an opaque value, so read the + // result out through `json_as_text`. The rewriter flattens the `json_get` underneath + // it into the outer call; the barrier stops it from seeing the nesting at all. + let last = path_arg(doc.path.last().unwrap()); + let unnested = select(&format!("json_as_text({prefix}, {last})")); + let not_unnested = unfolded_sql(&prefix, |x| format!("json_as_text({x}, {last})")); + + // the comparison only means something while each side is the plan it claims to be + let plan = rt.block_on(logical_plan(&ctx, ¬_unnested)); + prop_assert!( + plan.contains("json_as_text(json_get("), + "barrier stopped blocking the unnest:\n{}\n{}", not_unnested, plan + ); + let unnested_plan = rt.block_on(logical_plan(&ctx, &unnested)); + prop_assert!( + !unnested_plan.contains("json_as_text(json_get("), + "expected the nested call to be flattened:\n{}\n{}", unnested, unnested_plan + ); + + let got = rt.block_on(outcome(&ctx, &unnested)); + let want = rt.block_on(outcome(&ctx, ¬_unnested)); + prop_assert_eq!(&got, &want, + "\n unnested: {} => {}\n not unnested: {} => {}\n", + unnested, got, not_unnested, want); + }); +} + +// --------------------------------------------------------------------------------------- +// plan shape: which spellings fold +// --------------------------------------------------------------------------------------- + +/// Every spelling of a cast a user might reach for folds to the same accessor — that is what +/// PR #127 added — except `arrow_cast`/`arrow_try_cast` of a type no accessor returns +/// exactly, which are deliberately left alone. +#[test] +fn every_cast_spelling_folds_as_documented() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = select(&spelling.apply(&doc.json_get(), target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + let folded = plan.contains(target.accessor); + let expected = spelling.fold(target) != Fold::None; + assert_eq!( + folded, expected, + "expected folded={expected} for {spelling:?} / {}\n{plan}", + target.sql + ); + if folded { + assert!( + !plan.contains("json_get(t."), + "fold left the union accessor behind for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } + } +} + +// --------------------------------------------------------------------------------------- +// characterization: what folding a narrowing cast currently does +// --------------------------------------------------------------------------------------- + +/// The `CAST`/`TRY_CAST` path folds *every* type in its table down to the bare accessor, +/// including types the accessor does not return. The result is that the expression's type is +/// not the type the user asked for, and the narrowing the cast promised never happens. +/// +/// This test pins the current behaviour rather than asserting the desired behaviour, so that +/// it stays green while the divergence is open. **If this test starts failing because the +/// fold became type-preserving, delete it** — [`prop_fold_is_invisible`] covers the fixed +/// behaviour once these targets' `sql_cast` becomes `Fold::Exact`. +/// +/// The fix that makes all of this go away is to keep the cast the user wrote and fold only +/// its input, `CAST(json_get_int(x, 'k') AS Int32)` instead of `json_get_int(x, 'k')`. That +/// keeps the whole point of the rewrite — the union is never materialized — while leaving +/// the declared type and the narrowing intact, and `SimplifyExpressions` drops the cast +/// where it is a no-op. +#[test] +fn narrowing_cast_fold_is_not_type_preserving() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + // (json value, sql type, folded outcome, what the same cast over the typed accessor gives) + let cases = [ + // the type of the expression is the accessor's, not the one the user asked for + ("42", "int", "Int64=42", "Int32=42"), + ("42", "real", "Float64=42.0", "Float32=42.0"), + ("42", "decimal(10,2)", "Float64=42.0", "Decimal128(10, 2)=42.00"), + (r#""abc""#, "varchar", "Utf8=abc", "Utf8View=abc"), + // and the narrowing the cast promised never happens + ("3000000000", "int", "Int64=3000000000", "ERROR"), + ("9223372036854775807", "int", "Int64=9223372036854775807", "ERROR"), + ("3000000000", "decimal(10,2)", "Float64=3000000000.0", "ERROR"), + ]; + + for (value, sql_type, want_folded, want_reference) in cases { + let target = TARGETS.iter().find(|t| t.sql == sql_type).unwrap(); + let doc = Doc { + json: format!(r#"{{"a": {value}}}"#), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + let folded = select(&Spelling::Cast.apply(&doc.json_get(), *target)); + let reference = select(&Spelling::Cast.apply(&doc.accessor(target.accessor), *target)); + + assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), want_folded, "{folded}"); + assert_eq!( + rt.block_on(outcome(&ctx, &reference)).to_string(), + want_reference, + "{reference}" + ); + } +} + +/// `TRY_CAST`, which PR #127 routes through the same type table, inherits the same gap: it +/// returns a value where the cast it stands for would have returned NULL. +#[test] +fn narrowing_try_cast_fold_returns_value_instead_of_null() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let target = TARGETS.iter().find(|t| t.sql == "int").unwrap(); + let doc = Doc { + json: r#"{"a": 3000000000}"#.to_string(), + path: vec!["a".to_string()], + }; + + set_doc(&ctx, &doc.json); + let folded = select(&Spelling::TryCast.apply(&doc.json_get(), *target)); + let reference = select(&Spelling::TryCast.apply(&doc.accessor(target.accessor), *target)); + + assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), "Int64=3000000000"); + assert_eq!(rt.block_on(outcome(&ctx, &reference)).to_string(), "Int32=NULL"); +} + +// --------------------------------------------------------------------------------------- +// the barrier, and why the naive differential does not work for casts +// --------------------------------------------------------------------------------------- + +/// The subquery barrier must actually stop the fold, or every test that uses it as the +/// "without the rewrite" reference silently becomes a tautology. +#[test] +fn barrier_really_blocks_folding() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + set_doc(&ctx, &doc.json); + + for &target in TARGETS { + for &spelling in SPELLINGS { + let sql = unfolded_sql(&doc.json_get(), |x| spelling.apply(x, target)); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + // the json_get survives un-folded, and the optimizer inlined it back under the + // cast, so this is one expression and not an extra projection + assert!( + plan.contains("CAST(json_get(t.") || plan.contains("TRY_CAST(json_get(t."), + "barrier no longer blocks the fold for {spelling:?} / {}\n{plan}", + target.sql + ); + } + } +} + +/// Why cast folding cannot be checked by diffing against the un-folded plan: casting the JSON +/// union is much weaker than the typed accessors, so the two disagree by design. +/// +/// This is the reason [`prop_fold_is_invisible`] compares against the typed accessor instead. +#[test] +fn unfolded_union_cast_is_weaker() { + let rt = runtime(); + let ctx = create_context().unwrap(); + let doc = Doc { + json: r#"{"a": 42}"#.to_string(), + path: vec!["a".to_string()], + }; + let target = TARGETS.iter().find(|t| t.sql == "double").unwrap(); + + set_doc(&ctx, &doc.json); + let folded = select(&Spelling::Cast.apply(&doc.json_get(), *target)); + let unfolded = unfolded_sql(&doc.json_get(), |x| Spelling::Cast.apply(x, *target)); + + assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), "Float64=42.0"); + // the union has no Float64 member holding an integer, so the cast yields NULL + assert_eq!(rt.block_on(outcome(&ctx, &unfolded)).to_string(), "Float64=NULL"); +} From 0ba0201d2a6f7eec1069fdfcf65cab717ecbaf30 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:03:49 +0000 Subject: [PATCH 2/5] Add cross-cutting property tests for the JSON UDFs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three properties that hold for every accessor, so a change to any one of them or to the shared scanning code in `common.rs` has to keep them true. `prop_no_panic_on_arbitrary_json` feeds each UDF valid JSON, truncated JSON, JSON with a byte removed, and arbitrary text. Malformed input has to come back as NULL rather than a panic or an error. That is the shape of the bug fixed in 774c570 (#124): a JSON integer too wide for jiter's fast path reached a `todo!()`. The generator emits those integers, so this would have caught it. `prop_scalar_and_array_paths_agree` checks a literal document against the same document in a column. A literal is const-evaluated through each UDF's `ScalarValue` path while a column goes through the array path; they are separate code in every UDF here. `prop_input_encoding_does_not_change_result` checks `Utf8`, `LargeUtf8`, `Utf8View` and dictionary-encoded input against each other — four more separate paths, currently covered only by hand-written cases via `for_all_json_datatypes`. That last one found `json_get_array_errors_on_dictionary_input`: every `json_get_array` call on a dictionary-encoded JSON column fails, whatever the document and whatever the path. `return_type_check` promises `Dictionary(Int64, List(Utf8))` because the value type is not primitive, but `BuildArrayList` sets `ACCEPT_DICT_RETURN = false` so `invoke_array_array` returns a bare `List`. It is the only UDF with that combination. Debug builds trip DataFusion's own assertion and release builds reach Arrow's record batch schema check, so the test matches on the promised type that appears in both. It characterizes the failure and says what to delete when it is fixed. Case counts default low so the suite stays fast, and `PROPTEST_CASES` still wins, which is how these get run as a fuzzer. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- tests/json_robustness.rs | 322 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 tests/json_robustness.rs diff --git a/tests/json_robustness.rs b/tests/json_robustness.rs new file mode 100644 index 0000000..fac5bf7 --- /dev/null +++ b/tests/json_robustness.rs @@ -0,0 +1,322 @@ +//! Properties that hold for every JSON UDF, checked over generated input. +//! +//! These are deliberately cheap and broad rather than targeted: they cover the accessors as a +//! group, so a change to any one of them — or to the shared scanning code in `common.rs` — +//! has to keep them true. +//! +//! * [`prop_no_panic_on_arbitrary_json`] — malformed input yields NULL, never a panic and +//! never an error. This is the bug shape that produced the `todo!("BigInt not supported +//! yet")` panic fixed in #124: a JSON integer too wide for jiter's fast path. +//! * [`prop_scalar_and_array_paths_agree`] — a literal argument is const-evaluated through +//! the `ScalarValue` path while a column goes through the array path. They are separate +//! code paths in every UDF here, and they have to agree. +//! * [`prop_input_encoding_does_not_change_result`] — `Utf8`, `LargeUtf8`, `Utf8View` and +//! dictionary-encoded input are four more separate paths that have to agree. + +use std::sync::Arc; + +use datafusion::arrow::array::{ + ArrayRef, DictionaryArray, Int32Array, LargeStringArray, RecordBatch, StringArray, StringViewArray, +}; +use datafusion::arrow::datatypes::{DataType, Field, Int32Type, Schema}; +use datafusion::arrow::util::display::{ArrayFormatter, FormatOptions}; +use datafusion::error::Result; +use datafusion::execution::context::SessionContext; +use datafusion::prelude::SessionConfig; +use datafusion_functions_json::register_all; +use proptest::prelude::*; + +/// Every UDF that takes a JSON document and a lookup path. +const PATH_FUNCS: &[&str] = &[ + "json_get", + "json_get_bool", + "json_get_float", + "json_get_int", + "json_get_json", + "json_get_str", + "json_get_array", + "json_as_text", + "json_contains", + "json_length", + "json_object_keys", +]; + +const JSON_TABLE: &str = "t"; +const JSON_COLUMN: &str = "j"; + +fn create_context() -> Result { + let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", "postgres"); + let mut ctx = SessionContext::new_with_config(config); + register_all(&mut ctx)?; + Ok(ctx) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Runtime::new().unwrap() +} + +/// Each case here runs a query per UDF, so the defaults are kept low enough that the suite +/// stays fast. `PROPTEST_CASES` still wins when it is set, which is how these get run as a +/// fuzzer: `PROPTEST_CASES=100000 cargo test --release --test json_robustness`. +fn config(cases: u32) -> ProptestConfig { + let mut config = ProptestConfig::default(); + if std::env::var_os("PROPTEST_CASES").is_none() { + config.cases = cases; + } + config +} + +/// Build the single-row table `t` holding `json` in the given encoding. +fn set_doc(ctx: &SessionContext, json: &str, encoding: &DataType) { + let array: ArrayRef = match encoding { + DataType::Utf8 => Arc::new(StringArray::from(vec![json])), + DataType::LargeUtf8 => Arc::new(LargeStringArray::from(vec![json])), + DataType::Utf8View => Arc::new(StringViewArray::from(vec![json])), + DataType::Dictionary(_, _) => Arc::new(DictionaryArray::::new( + Int32Array::from(vec![0]), + Arc::new(StringArray::from(vec![json])), + )), + other => panic!("unsupported JSON encoding {other}"), + }; + let schema = Schema::new(vec![Field::new(JSON_COLUMN, encoding.clone(), false)]); + let batch = RecordBatch::try_new(Arc::new(schema), vec![array]).unwrap(); + let _ = ctx.deregister_table(JSON_TABLE); + ctx.register_batch(JSON_TABLE, batch).unwrap(); +} + +const ENCODINGS: &[DataType] = &[DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View]; + +fn dict_encoding() -> DataType { + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)) +} + +/// Run a single-row query, formatting the result. `Err` carries only the fact that the query +/// failed, so the properties are not brittle about error text. +async fn outcome(ctx: &SessionContext, sql: &str) -> std::result::Result { + let df = ctx.sql(sql).await.map_err(|e| e.to_string())?; + let batches = df.collect().await.map_err(|e| e.to_string())?; + let batch = &batches[0]; + let column = batch.column(0); + let options = FormatOptions::default().with_display_error(true); + let formatter = ArrayFormatter::try_new(column.as_ref(), &options).map_err(|e| e.to_string())?; + Ok(format!( + "{}={}", + logical_type(batch.schema().field(0).data_type()), + formatter.value(0).try_to_string().map_err(|e| e.to_string())? + )) +} + +/// Dictionary encoding of the input carries through to the output, so compare the type it +/// wraps rather than the wrapper. +fn logical_type(data_type: &DataType) -> &DataType { + match data_type { + DataType::Dictionary(_, value) => value, + other => other, + } +} + +// --------------------------------------------------------------------------------------- +// generators +// --------------------------------------------------------------------------------------- + +/// Well-formed JSON, then mangled: truncated, or with a byte dropped. Most of the interesting +/// malformed input is "nearly valid", which random text never produces. +fn json_text() -> impl Strategy { + prop_oneof![ + 6 => valid_json(), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let cut = j.char_indices().nth(n).map_or(j.len(), |(i, _)| i); + j[..cut].to_string() + }), + 2 => (valid_json(), 0usize..40).prop_map(|(j, n)| { + let mut chars: Vec = j.chars().collect(); + if !chars.is_empty() { + chars.remove(n % chars.len()); + } + chars.into_iter().collect() + }), + 1 => ".{0,24}".prop_map(|s| s), + ] +} + +fn valid_json() -> impl Strategy { + let leaf = prop_oneof![ + Just("null".to_string()), + any::().prop_map(|b| b.to_string()), + any::().prop_map(|i| i.to_string()), + any::() + .prop_filter("JSON has no inf or nan", |f| f.is_finite()) + .prop_map(|f| format!("{f:?}")), + "[a-zA-Z0-9 ._+-]{0,6}".prop_map(|s| format!("\"{s}\"")), + // integers and floats outside the fast paths + Just("123456789012345678901234567890".to_string()), + Just("-99999999999999999999999999999999".to_string()), + Just("1e309".to_string()), + Just("-1e-309".to_string()), + ]; + leaf.prop_recursive(3, 12, 3, |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..3).prop_map(|items| format!("[{}]", items.join(", "))), + prop::collection::vec(("[a-c]{1,2}", inner), 0..3).prop_map(|entries| { + let body = entries + .into_iter() + .map(|(k, v)| format!("\"{k}\": {v}")) + .collect::>() + .join(", "); + format!("{{{body}}}") + }), + ] + }) +} + +/// A lookup path: string keys and array indices, rendered as SQL arguments. +fn path() -> impl Strategy> { + prop::collection::vec( + prop_oneof![ + "[a-c]{1,2}".prop_map(|k| format!("'{k}'")), + (0i64..4).prop_map(|i| i.to_string()), + ], + 0..3, + ) +} + +fn call(func: &str, target: &str, path: &[String]) -> String { + let args = std::iter::once(target.to_string()) + .chain(path.iter().cloned()) + .collect::>(); + format!("{func}({})", args.join(", ")) +} + +/// `json_object_keys` and `json_get_array` return lists, and `json_length` takes no path +/// beyond the document, but every function still accepts the same argument shape. +fn callable(func: &str, path: &[String]) -> bool { + // json_contains needs at least one path element to mean anything + !(func == "json_contains" && path.is_empty()) +} + +// --------------------------------------------------------------------------------------- +// properties +// --------------------------------------------------------------------------------------- + +/// No input, however malformed, may panic or raise an error. The accessors report "not +/// found" as NULL, and invalid JSON is just another way of not finding anything. +#[test] +fn prop_no_panic_on_arbitrary_json() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let result = rt.block_on(outcome(&ctx, &sql)); + prop_assert!(result.is_ok(), "{sql}\n json: {json:?}\n {result:?}"); + } + }); +} + +/// A literal document is const-evaluated through each UDF's `ScalarValue` path; a column goes +/// through its array path. The two must not disagree. +#[test] +fn prop_scalar_and_array_paths_agree() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(96), |(json in json_text(), path in path())| { + set_doc(&ctx, &json, &DataType::Utf8View); + let literal = format!("'{}'", json.replace('\'', "''")); + for func in PATH_FUNCS { + if !callable(func, &path) { + continue; + } + let from_column = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + let from_literal = format!("select {} as v", call(func, &literal, &path)); + let column = rt.block_on(outcome(&ctx, &from_column)); + let scalar = rt.block_on(outcome(&ctx, &from_literal)); + prop_assert_eq!(&column, &scalar, + "\n column: {} => {:?}\n literal: {} => {:?}\n json: {:?}\n", + from_column, column, from_literal, scalar, json); + } + }); +} + +/// The four string encodings a JSON column can arrive in are four separate code paths, and +/// they must all produce the same answer. +/// +/// Dictionary-encoded input keeps its encoding in the output, so the dictionary wrapper is +/// stripped before comparing types. `json_get` is left out: it returns the JSON union, which +/// has no meaningful text form to compare — `json_as_text` and `json_get_json` cover the same +/// scanning code and do return text. `json_get_array` is left out too, but for a different +/// reason — see [`json_get_array_errors_on_dictionary_input`]. +#[test] +fn prop_input_encoding_does_not_change_result() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + proptest!(config(48), |(json in json_text(), path in path())| { + for func in PATH_FUNCS { + if !callable(func, &path) || matches!(*func, "json_get" | "json_get_array") { + continue; + } + let sql = format!("select {} as v from {JSON_TABLE}", call(func, JSON_COLUMN, &path)); + + set_doc(&ctx, &json, &DataType::Utf8View); + let baseline = rt.block_on(outcome(&ctx, &sql)); + + for encoding in ENCODINGS.iter().cloned().chain(std::iter::once(dict_encoding())) { + set_doc(&ctx, &json, &encoding); + let got = rt.block_on(outcome(&ctx, &sql)); + prop_assert_eq!(&got, &baseline, + "\n {} in {} => {:?}\n in Utf8View => {:?}\n json: {:?}\n", + sql, encoding, got, baseline, json); + } + } + }); +} + +/// `json_get_array` is unusable on a dictionary-encoded JSON column: every call fails with an +/// internal error, whatever the document and whatever the path. +/// +/// `return_type_check` wraps the declared return type in `Dictionary(Int64, ..)` whenever the +/// first argument is a dictionary and the value type is not primitive. `json_get_array` is the +/// only UDF here whose value type is not primitive (`List(Utf8)`) *and* whose `InvokeResult` +/// sets `ACCEPT_DICT_RETURN = false`, so `invoke_array_array` hands back a bare `List` while +/// `return_type` promised a dictionary, and `DataFusion` rejects the mismatch. +/// +/// The fix is to stop promising the wrapper for this one function, i.e. teach +/// `return_type_check` about `ACCEPT_DICT_RETURN` (or unwrap the dictionary in +/// `JsonGetArray::return_type`). **Delete this test once that lands**, and drop +/// `json_get_array` from the exclusion in [`prop_input_encoding_does_not_change_result`], +/// which then covers it. +#[test] +fn json_get_array_errors_on_dictionary_input() { + let rt = runtime(); + let ctx = create_context().unwrap(); + + for json in [r"[1, 2, 3]", r#"{"a": [1, 2]}"#, "null", "not json"] { + set_doc(&ctx, json, &dict_encoding()); + let err = rt + .block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect_err("json_get_array over a dictionary column should still be failing"); + // debug builds trip DataFusion's own assertion, release builds reach Arrow's schema + // check; both name the dictionary type that `return_type` promised and did not deliver + assert!( + err.contains("Dictionary(Int64, List("), + "unexpected error for {json:?}: {err}" + ); + + // the same document in a plain string column works + set_doc(&ctx, json, &DataType::Utf8View); + rt.block_on(outcome( + &ctx, + &format!("select json_get_array({JSON_COLUMN}) as v from {JSON_TABLE}"), + )) + .expect("json_get_array over a string column works"); + } +} From da2461a14765a8483b6f31dddfe3d53d3927537f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:38:30 +0000 Subject: [PATCH 3/5] Keep the cast when folding json_get, so the declared type survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold replaced `CAST(json_get(x,'k') AS T)` with the bare accessor for `T`. The accessors return one Arrow type per JSON type — `json_get_int` is always `Int64` — so whenever `T` was not that type, the expression silently came out as the accessor's type and the narrowing the cast promised never happened: CAST(json_get(x,'k') AS INT) -> Int64, not Int32 CAST(json_get(x,'k') AS REAL) -> Float64, not Float32 CAST(json_get(x,'k') AS DECIMAL(10,2)) -> Float64, not Decimal128(10,2) CAST(json_get(x,'k') AS VARCHAR) -> Utf8, not Utf8View and on `{"a": 3000000000}`, `CAST(.. AS INT)` returned 3000000000 where a real cast raises an overflow error, while `TRY_CAST(.. AS INT)` returned it instead of NULL. Now only the cast's input is replaced. The cast stays unless the accessor already returns the type asked for, in which case it is dropped and this is the substitution it has always been — so `::bigint`, `::double` and `::boolean` plan exactly as before. Where it stays it costs one primitive-to-primitive cast and the JSON union is still never materialized, which was the whole point. That also removes the reason `arrow_cast` folded only the four types an accessor returns exactly: `arrow_cast(x,'Int32')` now folds to `CAST(json_get_int(x) AS Int32)` and keeps its `Int32`. Both paths share one type table, with the `arrow_*` type argument parsed the way DataFusion parses it itself in `ArrowCastFunc::return_field_from_args`. `tests/main.rs` expectations change from the accessor's type to the type the query asked for. `prop_fold_is_invisible` now covers every folding (spelling, type) pair rather than only the type-exact ones, and the two tests that characterized the old behaviour are replaced by `narrowing_cast_preserves_type_and_narrows` and `narrowing_cast_still_avoids_the_union`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- src/rewrite.rs | 118 ++++++++++++++++-------- tests/main.rs | 40 ++++---- tests/rewrite_differential.rs | 166 ++++++++++++++++++---------------- 3 files changed, 187 insertions(+), 137 deletions(-) diff --git a/src/rewrite.rs b/src/rewrite.rs index df6586e..1cc423d 100644 --- a/src/rewrite.rs +++ b/src/rewrite.rs @@ -6,7 +6,7 @@ use datafusion::common::tree_node::Transformed; use datafusion::common::Column; use datafusion::common::DFSchema; use datafusion::common::Result; -use datafusion::logical_expr::expr::{Alias, Expr, ScalarFunction}; +use datafusion::logical_expr::expr::{Alias, Cast, Expr, ScalarFunction, TryCast}; use datafusion::logical_expr::expr_rewriter::FunctionRewrite; use datafusion::logical_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr}; use datafusion::logical_expr::sqlparser::ast::BinaryOperator; @@ -23,38 +23,80 @@ impl FunctionRewrite for JsonFunctionRewriter { fn rewrite(&self, expr: Expr, _schema: &DFSchema, _config: &ConfigOptions) -> Result> { let transform = match &expr { - Expr::Cast(cast) => optimise_json_get_cast(cast.field.data_type(), &cast.expr), - Expr::TryCast(try_cast) => optimise_json_get_cast(try_cast.field.data_type(), &try_cast.expr), + Expr::Cast(cast) => optimise_json_get(cast.field.data_type(), &cast.expr).map(|folded| match folded { + Folded::Exact(accessor) => accessor, + Folded::Narrowing(accessor) => Expr::Cast(Cast { + expr: Box::new(accessor), + field: cast.field.clone(), + }), + }), + Expr::TryCast(try_cast) => { + optimise_json_get(try_cast.field.data_type(), &try_cast.expr).map(|folded| match folded { + Folded::Exact(accessor) => accessor, + Folded::Narrowing(accessor) => Expr::TryCast(TryCast { + expr: Box::new(accessor), + field: try_cast.field.clone(), + }), + }) + } Expr::ScalarFunction(func) => optimise_json_get_arrow_cast(func).or_else(|| unnest_json_calls(func)), _ => None, }; - Ok(transform.unwrap_or_else(|| Transformed::no(expr))) + Ok(transform.map_or_else(|| Transformed::no(expr), Transformed::yes)) } } -/// This replaces `get_json(foo, bar)::int` with `json_get_int(foo, bar)` so the JSON function can take care of -/// extracting the right value type from JSON without the need to materialize the JSON union. +/// The accessor call that replaces a `json_get` under a cast, and whether the cast is still needed. +enum Folded { + /// The accessor already returns the type the query asked for; the cast can go. + Exact(Expr), + /// The accessor returns a wider type; the cast has to stay or both the type of the expression + /// and, for a narrowing cast, its value would change. + Narrowing(Expr), +} + +/// Replace the `json_get` under a cast to `cast_to` with the typed accessor that reads that type +/// out of the JSON directly, so the JSON union never has to be materialized just to be cast away. +/// +/// The accessors return one Arrow type per JSON type — `json_get_int` is always `Int64` — which is +/// not necessarily the type that was asked for. Where it is, the cast is dropped and this is the +/// plain substitution it has always been. Where it is not, only the cast's *input* is replaced: +/// dropping the cast there would silently change the type of the expression, and for a narrowing +/// cast its value too, while keeping it costs one primitive-to-primitive cast and still never +/// materializes the union. /// -/// `TRY_CAST` is folded the same way. The typed accessors already yield NULL for a value of another -/// type instead of failing, which is exactly what `TRY_CAST` asks for. -fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option> { +/// `TRY_CAST` gets the same treatment, so a value outside the target type's range still becomes +/// NULL rather than being handed back as the accessor's wider type. +fn optimise_json_get(cast_to: &DataType, cast_expr: &Expr) -> Option { let scalar_func = extract_scalar_function(cast_expr)?; if !is_json_get(scalar_func) { return None; } - let func = match cast_to { - DataType::Boolean => crate::json_get_bool::json_get_bool_udf(), + let (func, returns) = typed_accessor(cast_to)?; + let accessor = Expr::ScalarFunction(ScalarFunction { + func, + args: scalar_func.args.clone(), + }); + Some(if returns == *cast_to { + Folded::Exact(accessor) + } else { + Folded::Narrowing(accessor) + }) +} + +/// The accessor that reads `cast_to` out of JSON, if there is one, and the type it returns. +fn typed_accessor(cast_to: &DataType) -> Option<(Arc, DataType)> { + Some(match cast_to { + DataType::Boolean => (crate::json_get_bool::json_get_bool_udf(), DataType::Boolean), DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => { - crate::json_get_float::json_get_float_udf() + (crate::json_get_float::json_get_float_udf(), DataType::Float64) + } + DataType::Int64 | DataType::Int32 => (crate::json_get_int::json_get_int_udf(), DataType::Int64), + DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => { + (crate::json_get_str::json_get_str_udf(), DataType::Utf8) } - DataType::Int64 | DataType::Int32 => crate::json_get_int::json_get_int_udf(), - DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => crate::json_get_str::json_get_str_udf(), _ => return None, - }; - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { - func, - args: scalar_func.args.clone(), - }))) + }) } /// The same rewrite for `arrow_cast(json_get(foo, bar), 'Int64')` and its `arrow_try_cast` sibling. @@ -64,10 +106,9 @@ fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option Option> { +/// Only the call's first argument is replaced, so the named type is still what comes out. That +/// lowering then drops the cast by itself when the accessor already returns the named type. +fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option { if !matches!(func.func.name(), "arrow_cast" | "arrow_try_cast") { return None; } @@ -77,25 +118,22 @@ fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option crate::json_get_bool::json_get_bool_udf(), - "Float64" => crate::json_get_float::json_get_float_udf(), - "Int64" => crate::json_get_int::json_get_int_udf(), - "Utf8" => crate::json_get_str::json_get_str_udf(), - _ => return None, + // `arrow_cast` names its target as an Arrow type string, which is how DataFusion itself reads + // it back in `ArrowCastFunc::return_field_from_args`. + let cast_to = type_name.parse::().ok()?; + // the call keeps its type argument either way, and `ArrowCastFunc::simplify` drops the cast + // itself when the accessor already returns that type + let accessor = match optimise_json_get(&cast_to, cast_expr)? { + Folded::Exact(accessor) | Folded::Narrowing(accessor) => accessor, }; - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { - func, - args: scalar_func.args.clone(), - }))) + Some(Expr::ScalarFunction(ScalarFunction { + func: func.func.clone(), + args: vec![accessor, type_arg.clone()], + })) } // Replace nested JSON functions e.g. `json_get(json_get(col, 'foo'), 'bar')` with `json_get(col, 'foo', 'bar')` -fn unnest_json_calls(func: &ScalarFunction) -> Option> { +fn unnest_json_calls(func: &ScalarFunction) -> Option { if !matches!( func.func.name(), "json_get" @@ -122,10 +160,10 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option> { args.extend(outer_args_iter.cloned()); // See #23, unnest only when all lookup arguments are literals if args.iter().skip(1).all(|arg| matches!(arg, Expr::Literal(_, _))) { - Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction { + Some(Expr::ScalarFunction(ScalarFunction { func: func.func.clone(), args, - }))) + })) } else { None } diff --git a/tests/main.rs b/tests/main.rs index b909ec0..f118295 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -300,13 +300,13 @@ async fn test_json_get_str_null() { #[tokio::test] async fn test_json_get_no_path() { let batches = run_query(r#"select json_get('"foo"')::string"#).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, "foo".to_string())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, "foo".to_string())); let batches = run_query(r"select json_get('123')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "123".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "123".to_string())); let batches = run_query(r"select json_get('true')::int").await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] @@ -386,11 +386,11 @@ async fn test_json_get_out_of_range_int() { // casts of the null union stay null rather than erroring let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); let sql = r#"select json_get('{"foo": 18446744073709551615}', 'foo')::string"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Utf8, String::new())); + assert_eq!(display_val(batches).await, (DataType::Utf8View, String::new())); } #[tokio::test] @@ -512,19 +512,19 @@ async fn test_json_get_path() { async fn test_json_get_cast_int() { let sql = r#"select json_get('{"foo": 42}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); // floats not allowed let sql = r#"select json_get('{"foo": 4.2}', 'foo')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); + assert_eq!(display_val(batches).await, (DataType::Int32, String::new())); } #[tokio::test] async fn test_json_get_cast_int_path() { let sql = r#"select json_get('{"foo": [null, {"x": false, "bar": 73}}', 'foo', 1, 'bar')::int"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Int64, "73".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "73".to_string())); } #[tokio::test] @@ -567,14 +567,17 @@ async fn test_json_get_float() { async fn test_json_get_cast_float() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::float"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!(display_val(batches).await, (DataType::Float32, "420.0".to_string())); } #[tokio::test] async fn test_json_get_cast_numeric() { let sql = r#"select json_get('{"foo": 4.2e2}', 'foo')::numeric"#; let batches = run_query(sql).await.unwrap(); - assert_eq!(display_val(batches).await, (DataType::Float64, "420.0".to_string())); + assert_eq!( + display_val(batches).await, + (DataType::Decimal128(38, 10), "420.0000000000".to_string()) + ); } #[tokio::test] @@ -1202,7 +1205,7 @@ async fn test_json_get_int_unnest() { async fn test_plan_json_get_int_unnest() { let sql = "explain select json_get(json_get(json_data, 'foo'), 0)::int v from test"; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS v", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS v", " TableScan: test projection=[json_data]", ]; @@ -1426,7 +1429,7 @@ async fn test_arrow_cast_int() { for_all_json_datatypes(async |dt| { let batches = run_query_datatype(sql, dt).await.unwrap(); assert_batches_eq!(expected, &batches); - assert_eq!(display_val(batches).await, (DataType::Int64, "42".to_string())); + assert_eq!(display_val(batches).await, (DataType::Int32, "42".to_string())); }) .await; } @@ -1436,7 +1439,7 @@ async fn test_plan_arrow_cast_int() { let lines = logical_plan(r"explain select (json_data->'foo')::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\")) AS json_data -> 'foo'", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\")) AS Int32) AS json_data -> 'foo'", " TableScan: test projection=[json_data]", ]; @@ -1508,14 +1511,15 @@ async fn test_arrow_try_cast_fn_str() { assert_eq!(display_val(batches).await, (DataType::Utf8, "bar".to_string())); } -/// `arrow_cast` names an exact Arrow type, so a type no accessor returns exactly is left alone -/// rather than widened to the accessor's type. +/// `arrow_cast` names an exact Arrow type. A type no accessor returns exactly is still folded, so +/// the union is not materialized, but the cast to that type is kept so the output type is the one +/// that was named. #[tokio::test] -async fn test_plan_arrow_cast_fn_inexact_type_not_folded() { +async fn test_plan_arrow_cast_fn_narrowing_type_keeps_cast() { let lines = logical_plan(r"explain select arrow_cast((json_data->'foo'), 'Int32') from test").await; let expected = [ - "Projection: CAST(json_get(test.json_data, Utf8(\"foo\")) AS Int32) AS arrow_cast(json_data -> 'foo',Utf8(\"Int32\"))", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\")) AS Int32) AS arrow_cast(json_data -> 'foo',Utf8(\"Int32\"))", " TableScan: test projection=[json_data]", ]; @@ -1642,7 +1646,7 @@ async fn test_plan_arrow_double_nested_cast() { let lines = logical_plan(r"explain select (json_data->'foo'->0)::int from test").await; let expected = [ - "Projection: json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS json_data -> 'foo' -> 0", + "Projection: CAST(json_get_int(test.json_data, Utf8(\"foo\"), Int64(0)) AS Int32) AS json_data -> 'foo' -> 0", " TableScan: test projection=[json_data]", ]; diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs index c9b8a86..4882a6d 100644 --- a/tests/rewrite_differential.rs +++ b/tests/rewrite_differential.rs @@ -29,6 +29,11 @@ //! i.e. "use the typed accessor, then apply the cast the user actually wrote". The //! right-hand side is not itself folded — the rewriter only folds casts over `json_get` — so //! it serves as the reference implementation. That is [`prop_fold_is_invisible`]. +//! +//! That is also, literally, what the rewriter emits: it replaces the cast's *input*, and drops +//! the cast only where the accessor already returns the type that was asked for. The tests keep +//! both sides anyway, because the reference is built from the SQL rather than from the rewriter, +//! so a change to which accessor a type maps to still shows up here. use std::sync::Arc; @@ -50,11 +55,12 @@ use proptest::prelude::*; enum Fold { /// Left alone: the cast is applied to the JSON union. None, - /// Folded to the typed accessor, whose return type *is* the type that was asked for. + /// Folded to the typed accessor, whose return type *is* the type that was asked for, so + /// the cast is dropped entirely. Exact, - /// Folded to the typed accessor, whose return type is **not** the type that was asked - /// for. The expression silently changes type and the narrowing never happens. - Widening, + /// Folded to the typed accessor, which returns a wider type, so the cast to the type that + /// was asked for is kept on top of it. The union is still never materialized. + Narrowing, } /// A cast target: how it is spelled in SQL, the exact Arrow type `arrow_cast` names for it, @@ -96,31 +102,31 @@ const TARGETS: &[Target] = &[ sql: "varchar", arrow: "Utf8", accessor: "json_get_str", - sql_cast: Fold::Widening, + sql_cast: Fold::Narrowing, arrow_cast: Fold::Exact, }, - // Narrowing targets: the type asked for is narrower than what the accessor returns. The - // `CAST` table folds them anyway; the `arrow_*` handler deliberately does not. + // Narrowing targets: the type asked for is narrower than what the accessor returns, so the + // fold keeps the cast on top of the accessor. Target { sql: "int", arrow: "Int32", accessor: "json_get_int", - sql_cast: Fold::Widening, - arrow_cast: Fold::None, + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, }, Target { sql: "real", arrow: "Float32", accessor: "json_get_float", - sql_cast: Fold::Widening, - arrow_cast: Fold::None, + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, }, Target { sql: "decimal(10,2)", arrow: "Decimal128(10, 2)", accessor: "json_get_float", - sql_cast: Fold::Widening, - arrow_cast: Fold::None, + sql_cast: Fold::Narrowing, + arrow_cast: Fold::Narrowing, }, // A type neither path has an accessor for, so neither folds it. Target { @@ -376,12 +382,13 @@ fn doc() -> impl Strategy { }) } -/// A `(spelling, target)` pair whose fold is type-exact, so the fold must be invisible. -fn exact_cast() -> impl Strategy { +/// A `(spelling, target)` pair the rewriter folds, exactly or with a cast kept on top. Either +/// way the fold has to be invisible. +fn folding_cast() -> impl Strategy { let pairs: Vec<(Spelling, Target)> = SPELLINGS .iter() .flat_map(|&s| TARGETS.iter().map(move |&t| (s, t))) - .filter(|&(s, t)| s.fold(t) == Fold::Exact) + .filter(|&(s, t)| s.fold(t) != Fold::None) .collect(); prop::sample::select(pairs) } @@ -390,14 +397,15 @@ fn exact_cast() -> impl Strategy { // properties // --------------------------------------------------------------------------------------- -/// Folding a type-exact cast must be indistinguishable from applying that same cast to the -/// typed accessor. +/// Folding must be indistinguishable from applying the same cast to the typed accessor — +/// which is exactly what the rewriter now emits for a narrowing cast, and what the exact case +/// simplifies down to. #[test] fn prop_fold_is_invisible() { let rt = runtime(); let ctx = create_context().unwrap(); - proptest!(|(doc in doc(), (spelling, target) in exact_cast())| { + proptest!(|(doc in doc(), (spelling, target) in folding_cast())| { set_doc(&ctx, &doc.json); let folded = select(&spelling.apply(&doc.json_get(), target)); let reference = select(&spelling.apply(&doc.accessor(target.accessor), target)); @@ -457,9 +465,8 @@ fn prop_unnest_is_invisible() { // plan shape: which spellings fold // --------------------------------------------------------------------------------------- -/// Every spelling of a cast a user might reach for folds to the same accessor — that is what -/// PR #127 added — except `arrow_cast`/`arrow_try_cast` of a type no accessor returns -/// exactly, which are deliberately left alone. +/// Every spelling of a cast a user might reach for folds to the same accessor, and the plan +/// keeps the cast exactly where the accessor's type is not the one that was asked for. #[test] fn every_cast_spelling_folds_as_documented() { let rt = runtime(); @@ -474,97 +481,98 @@ fn every_cast_spelling_folds_as_documented() { for &spelling in SPELLINGS { let sql = select(&spelling.apply(&doc.json_get(), target)); let plan = rt.block_on(logical_plan(&ctx, &sql)); - let folded = plan.contains(target.accessor); - let expected = spelling.fold(target) != Fold::None; + let fold = spelling.fold(target); + assert_eq!( - folded, expected, - "expected folded={expected} for {spelling:?} / {}\n{plan}", + plan.contains(target.accessor), + fold != Fold::None, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", target.sql ); - if folded { - assert!( - !plan.contains("json_get(t."), - "fold left the union accessor behind for {spelling:?} / {}\n{plan}", - target.sql - ); + if fold == Fold::None { + continue; } + assert!( + !plan.contains("json_get(t."), + "fold left the union accessor behind for {spelling:?} / {}\n{plan}", + target.sql + ); + // "CAST(json_get" also matches "TRY_CAST(json_get" + assert_eq!( + plan.contains(&format!("CAST({}(t.", target.accessor)), + fold == Fold::Narrowing, + "expected fold={fold:?} for {spelling:?} / {}\n{plan}", + target.sql + ); } } } // --------------------------------------------------------------------------------------- -// characterization: what folding a narrowing cast currently does +// narrowing casts // --------------------------------------------------------------------------------------- -/// The `CAST`/`TRY_CAST` path folds *every* type in its table down to the bare accessor, -/// including types the accessor does not return. The result is that the expression's type is -/// not the type the user asked for, and the narrowing the cast promised never happens. -/// -/// This test pins the current behaviour rather than asserting the desired behaviour, so that -/// it stays green while the divergence is open. **If this test starts failing because the -/// fold became type-preserving, delete it** — [`prop_fold_is_invisible`] covers the fixed -/// behaviour once these targets' `sql_cast` becomes `Fold::Exact`. -/// -/// The fix that makes all of this go away is to keep the cast the user wrote and fold only -/// its input, `CAST(json_get_int(x, 'k') AS Int32)` instead of `json_get_int(x, 'k')`. That -/// keeps the whole point of the rewrite — the union is never materialized — while leaving -/// the declared type and the narrowing intact, and `SimplifyExpressions` drops the cast -/// where it is a no-op. +/// Folding a cast whose type no accessor returns exactly still yields that type, and still +/// narrows. Keeping the cast on top of the accessor is what buys this; folding to the bare +/// accessor would hand back the accessor's wider type and skip the narrowing entirely. #[test] -fn narrowing_cast_fold_is_not_type_preserving() { +fn narrowing_cast_preserves_type_and_narrows() { let rt = runtime(); let ctx = create_context().unwrap(); - // (json value, sql type, folded outcome, what the same cast over the typed accessor gives) + // (json value, sql type, CAST outcome, TRY_CAST outcome) let cases = [ - // the type of the expression is the accessor's, not the one the user asked for - ("42", "int", "Int64=42", "Int32=42"), - ("42", "real", "Float64=42.0", "Float32=42.0"), - ("42", "decimal(10,2)", "Float64=42.0", "Decimal128(10, 2)=42.00"), - (r#""abc""#, "varchar", "Utf8=abc", "Utf8View=abc"), - // and the narrowing the cast promised never happens - ("3000000000", "int", "Int64=3000000000", "ERROR"), - ("9223372036854775807", "int", "Int64=9223372036854775807", "ERROR"), - ("3000000000", "decimal(10,2)", "Float64=3000000000.0", "ERROR"), + ("42", "int", "Int32=42", "Int32=42"), + ("42", "real", "Float32=42.0", "Float32=42.0"), + ( + "42", + "decimal(10,2)", + "Decimal128(10, 2)=42.00", + "Decimal128(10, 2)=42.00", + ), + (r#""abc""#, "varchar", "Utf8View=abc", "Utf8View=abc"), + // out of the target type's range: CAST fails, TRY_CAST yields NULL + ("3000000000", "int", "ERROR", "Int32=NULL"), + ("9223372036854775807", "int", "ERROR", "Int32=NULL"), + ("3000000000", "decimal(10,2)", "ERROR", "Decimal128(10, 2)=NULL"), ]; - for (value, sql_type, want_folded, want_reference) in cases { + for (value, sql_type, want_cast, want_try_cast) in cases { let target = TARGETS.iter().find(|t| t.sql == sql_type).unwrap(); let doc = Doc { json: format!(r#"{{"a": {value}}}"#), path: vec!["a".to_string()], }; set_doc(&ctx, &doc.json); - let folded = select(&Spelling::Cast.apply(&doc.json_get(), *target)); - let reference = select(&Spelling::Cast.apply(&doc.accessor(target.accessor), *target)); - - assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), want_folded, "{folded}"); - assert_eq!( - rt.block_on(outcome(&ctx, &reference)).to_string(), - want_reference, - "{reference}" - ); + + for (spelling, want) in [(Spelling::Cast, want_cast), (Spelling::TryCast, want_try_cast)] { + let sql = select(&spelling.apply(&doc.json_get(), *target)); + assert_eq!(rt.block_on(outcome(&ctx, &sql)).to_string(), want, "{sql}"); + } } } -/// `TRY_CAST`, which PR #127 routes through the same type table, inherits the same gap: it -/// returns a value where the cast it stands for would have returned NULL. +/// The narrowing fold is not just correct, it is the whole point: the JSON union never gets +/// materialized, the plan reads the value straight out with the typed accessor and casts that. #[test] -fn narrowing_try_cast_fold_returns_value_instead_of_null() { +fn narrowing_cast_still_avoids_the_union() { let rt = runtime(); let ctx = create_context().unwrap(); - let target = TARGETS.iter().find(|t| t.sql == "int").unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); let doc = Doc { - json: r#"{"a": 3000000000}"#.to_string(), + json: r#"{"a": 42}"#.to_string(), path: vec!["a".to_string()], }; + let target = TARGETS.iter().find(|t| t.sql == "int").unwrap(); - set_doc(&ctx, &doc.json); - let folded = select(&Spelling::TryCast.apply(&doc.json_get(), *target)); - let reference = select(&Spelling::TryCast.apply(&doc.accessor(target.accessor), *target)); - - assert_eq!(rt.block_on(outcome(&ctx, &folded)).to_string(), "Int64=3000000000"); - assert_eq!(rt.block_on(outcome(&ctx, &reference)).to_string(), "Int32=NULL"); + let plan = rt.block_on(logical_plan( + &ctx, + &select(&Spelling::Cast.apply(&doc.json_get(), *target)), + )); + assert!( + plan.contains("CAST(json_get_int(t.j, Utf8(\"a\")) AS Int32)"), + "unexpected plan:\n{plan}" + ); } // --------------------------------------------------------------------------------------- From f9b2ced501b143cff730be86d88c9af96462045f Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:43:19 +0000 Subject: [PATCH 4/5] Pin that the kept cast does not strand the accessor above the leaf nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JsonGet::placement` moves a `json_get` over a column with literal path arguments towards the leaf nodes, which is where a filter wants the work to happen. A cast now sits on top of the accessor for a narrowing target, which is exactly the shape that could have stranded it higher up the plan. It does not — the accessor still lands in the leaf projection and `unwrap_cast_in_comparison` even removes the cast again for an integer comparison — so pin it before some later change quietly gives it up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- tests/rewrite_differential.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs index 4882a6d..dad9477 100644 --- a/tests/rewrite_differential.rs +++ b/tests/rewrite_differential.rs @@ -575,6 +575,35 @@ fn narrowing_cast_still_avoids_the_union() { ); } +/// Keeping the cast must not cost the accessor its trip to the leaf nodes. +/// +/// `JsonGet`'s `placement` moves a `json_get` over a column with literal path arguments towards +/// the leaves, which is where the work wants to happen for a filter. A cast sitting on top of the +/// accessor is exactly the sort of thing that could strand it higher up the plan, so pin it. +#[test] +fn narrowing_cast_still_reaches_the_leaf_nodes() { + let rt = runtime(); + let ctx = create_context().unwrap(); + set_doc(&ctx, r#"{"a": 42}"#); + + for target in TARGETS.iter().filter(|t| t.sql_cast != Fold::None) { + let predicate = Spelling::Cast.apply(&format!("json_get({JSON_COLUMN}, 'a')"), *target); + let sql = format!("select {JSON_COLUMN} from {JSON_TABLE} where {predicate} is not null"); + let plan = rt.block_on(logical_plan(&ctx, &sql)); + + let leaf_projection = plan + .lines() + .filter(|line| line.trim_start().starts_with("Projection:")) + .next_back() + .unwrap_or_default(); + assert!( + leaf_projection.contains(&format!("{}(t.j", target.accessor)), + "accessor did not reach the leaf projection for {}\n{plan}", + target.sql + ); + } +} + // --------------------------------------------------------------------------------------- // the barrier, and why the naive differential does not work for casts // --------------------------------------------------------------------------------------- From e9b5823c2144b7a9d08cd7809200e246a0597dbc Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Thu, 10 Sep 2026 04:06:21 +0000 Subject: [PATCH 5/5] Use rfind for the leaf projection lookup CI runs `cargo clippy --all-targets -- -D warnings`, which promotes the warn-by-default `clippy::filter_next` that the crate's own pedantic-only lint config does not. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VQbTD3AHrpLgGodFEDhJvr --- tests/rewrite_differential.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/rewrite_differential.rs b/tests/rewrite_differential.rs index dad9477..437d0a6 100644 --- a/tests/rewrite_differential.rs +++ b/tests/rewrite_differential.rs @@ -593,8 +593,7 @@ fn narrowing_cast_still_reaches_the_leaf_nodes() { let leaf_projection = plan .lines() - .filter(|line| line.trim_start().starts_with("Projection:")) - .next_back() + .rfind(|line| line.trim_start().starts_with("Projection:")) .unwrap_or_default(); assert!( leaf_projection.contains(&format!("{}(t.j", target.accessor)),