Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ datafusion = { version = "54", default-features = false, features = [
"nested_expressions",
"sql",
] }
proptest = "1.11.0"
tokio = { version = "1.43", features = ["full"] }

[lints.clippy]
Expand Down
118 changes: 78 additions & 40 deletions src/rewrite.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use datafusion::common::tree_node::Transformed;
use datafusion::common::Column;
use datafusion::common::DFSchema;
use datafusion::common::Result;
use datafusion::logical_expr::expr::{Alias, Expr, ScalarFunction};
use datafusion::logical_expr::expr::{Alias, Cast, Expr, ScalarFunction, TryCast};
use datafusion::logical_expr::expr_rewriter::FunctionRewrite;
use datafusion::logical_expr::planner::{ExprPlanner, PlannerResult, RawBinaryExpr};
use datafusion::logical_expr::sqlparser::ast::BinaryOperator;
Expand All @@ -23,38 +23,80 @@ impl FunctionRewrite for JsonFunctionRewriter {

fn rewrite(&self, expr: Expr, _schema: &DFSchema, _config: &ConfigOptions) -> Result<Transformed<Expr>> {
let transform = match &expr {
Expr::Cast(cast) => optimise_json_get_cast(cast.field.data_type(), &cast.expr),
Expr::TryCast(try_cast) => optimise_json_get_cast(try_cast.field.data_type(), &try_cast.expr),
Expr::Cast(cast) => optimise_json_get(cast.field.data_type(), &cast.expr).map(|folded| match folded {
Folded::Exact(accessor) => accessor,
Folded::Narrowing(accessor) => Expr::Cast(Cast {
expr: Box::new(accessor),
field: cast.field.clone(),
}),
}),
Expr::TryCast(try_cast) => {
optimise_json_get(try_cast.field.data_type(), &try_cast.expr).map(|folded| match folded {
Folded::Exact(accessor) => accessor,
Folded::Narrowing(accessor) => Expr::TryCast(TryCast {
expr: Box::new(accessor),
field: try_cast.field.clone(),
}),
})
}
Expr::ScalarFunction(func) => optimise_json_get_arrow_cast(func).or_else(|| unnest_json_calls(func)),
_ => None,
};
Ok(transform.unwrap_or_else(|| Transformed::no(expr)))
Ok(transform.map_or_else(|| Transformed::no(expr), Transformed::yes))
}
}

/// This replaces `get_json(foo, bar)::int` with `json_get_int(foo, bar)` so the JSON function can take care of
/// extracting the right value type from JSON without the need to materialize the JSON union.
/// The accessor call that replaces a `json_get` under a cast, and whether the cast is still needed.
enum Folded {
/// The accessor already returns the type the query asked for; the cast can go.
Exact(Expr),
/// The accessor returns a wider type; the cast has to stay or both the type of the expression
/// and, for a narrowing cast, its value would change.
Narrowing(Expr),
}

/// Replace the `json_get` under a cast to `cast_to` with the typed accessor that reads that type
/// out of the JSON directly, so the JSON union never has to be materialized just to be cast away.
///
/// The accessors return one Arrow type per JSON type — `json_get_int` is always `Int64` — which is
/// not necessarily the type that was asked for. Where it is, the cast is dropped and this is the
/// plain substitution it has always been. Where it is not, only the cast's *input* is replaced:
/// dropping the cast there would silently change the type of the expression, and for a narrowing
/// cast its value too, while keeping it costs one primitive-to-primitive cast and still never
/// materializes the union.
///
/// `TRY_CAST` is folded the same way. The typed accessors already yield NULL for a value of another
/// type instead of failing, which is exactly what `TRY_CAST` asks for.
fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option<Transformed<Expr>> {
/// `TRY_CAST` gets the same treatment, so a value outside the target type's range still becomes
/// NULL rather than being handed back as the accessor's wider type.
fn optimise_json_get(cast_to: &DataType, cast_expr: &Expr) -> Option<Folded> {
let scalar_func = extract_scalar_function(cast_expr)?;
if !is_json_get(scalar_func) {
return None;
}
let func = match cast_to {
DataType::Boolean => crate::json_get_bool::json_get_bool_udf(),
let (func, returns) = typed_accessor(cast_to)?;
let accessor = Expr::ScalarFunction(ScalarFunction {
func,
args: scalar_func.args.clone(),
});
Some(if returns == *cast_to {
Folded::Exact(accessor)
} else {
Folded::Narrowing(accessor)
})
}

/// The accessor that reads `cast_to` out of JSON, if there is one, and the type it returns.
fn typed_accessor(cast_to: &DataType) -> Option<(Arc<ScalarUDF>, DataType)> {
Some(match cast_to {
DataType::Boolean => (crate::json_get_bool::json_get_bool_udf(), DataType::Boolean),
DataType::Float64 | DataType::Float32 | DataType::Decimal128(_, _) | DataType::Decimal256(_, _) => {
crate::json_get_float::json_get_float_udf()
(crate::json_get_float::json_get_float_udf(), DataType::Float64)
}
DataType::Int64 | DataType::Int32 => (crate::json_get_int::json_get_int_udf(), DataType::Int64),
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => {
(crate::json_get_str::json_get_str_udf(), DataType::Utf8)
}
DataType::Int64 | DataType::Int32 => crate::json_get_int::json_get_int_udf(),
DataType::Utf8 | DataType::Utf8View | DataType::LargeUtf8 => crate::json_get_str::json_get_str_udf(),
_ => return None,
};
Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction {
func,
args: scalar_func.args.clone(),
})))
})
}

/// The same rewrite for `arrow_cast(json_get(foo, bar), 'Int64')` and its `arrow_try_cast` sibling.
Expand All @@ -64,10 +106,9 @@ fn optimise_json_get_cast(cast_to: &DataType, cast_expr: &Expr) -> Option<Transf
/// rewrites are applied by `ApplyFunctionRewrites` at the start of the analyzer. The analyzer never
/// runs again afterwards, so without this the JSON union is materialized only to be cast away.
///
/// Unlike `CAST`, `arrow_cast` names an exact Arrow type, so only the types an accessor returns
/// exactly are folded here. `arrow_cast(x, 'Int32')` keeps its `Int32` output rather than widening
/// to `json_get_int`'s `Int64`.
fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option<Transformed<Expr>> {
/// Only the call's first argument is replaced, so the named type is still what comes out. That
/// lowering then drops the cast by itself when the accessor already returns the named type.
fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option<Expr> {
if !matches!(func.func.name(), "arrow_cast" | "arrow_try_cast") {
return None;
}
Expand All @@ -77,25 +118,22 @@ fn optimise_json_get_arrow_cast(func: &ScalarFunction) -> Option<Transformed<Exp
let Expr::Literal(ScalarValue::Utf8(Some(type_name)), _) = type_arg else {
return None;
};
let scalar_func = extract_scalar_function(cast_expr)?;
if !is_json_get(scalar_func) {
return None;
}
let func = match type_name.as_str() {
"Boolean" => crate::json_get_bool::json_get_bool_udf(),
"Float64" => crate::json_get_float::json_get_float_udf(),
"Int64" => crate::json_get_int::json_get_int_udf(),
"Utf8" => crate::json_get_str::json_get_str_udf(),
_ => return None,
// `arrow_cast` names its target as an Arrow type string, which is how DataFusion itself reads
// it back in `ArrowCastFunc::return_field_from_args`.
let cast_to = type_name.parse::<DataType>().ok()?;
// the call keeps its type argument either way, and `ArrowCastFunc::simplify` drops the cast
// itself when the accessor already returns that type
let accessor = match optimise_json_get(&cast_to, cast_expr)? {
Folded::Exact(accessor) | Folded::Narrowing(accessor) => accessor,
};
Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction {
func,
args: scalar_func.args.clone(),
})))
Some(Expr::ScalarFunction(ScalarFunction {
func: func.func.clone(),
args: vec![accessor, type_arg.clone()],
}))
}

// Replace nested JSON functions e.g. `json_get(json_get(col, 'foo'), 'bar')` with `json_get(col, 'foo', 'bar')`
fn unnest_json_calls(func: &ScalarFunction) -> Option<Transformed<Expr>> {
fn unnest_json_calls(func: &ScalarFunction) -> Option<Expr> {
if !matches!(
func.func.name(),
"json_get"
Expand All @@ -122,10 +160,10 @@ fn unnest_json_calls(func: &ScalarFunction) -> Option<Transformed<Expr>> {
args.extend(outer_args_iter.cloned());
// See #23, unnest only when all lookup arguments are literals
if args.iter().skip(1).all(|arg| matches!(arg, Expr::Literal(_, _))) {
Some(Transformed::yes(Expr::ScalarFunction(ScalarFunction {
Some(Expr::ScalarFunction(ScalarFunction {
func: func.func.clone(),
args,
})))
}))
} else {
None
}
Expand Down
Loading
Loading