From 932021e6fada3b582446e00896fec8496e4c6e1c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 25 Jul 2026 03:36:54 +0000 Subject: [PATCH] Build against apache/datafusion main to prep for the next release Points the datafusion dependencies at apache/datafusion main (currently 54.1.0, rev 4a40101) so breakage from unreleased DataFusion changes shows up here rather than during the upgrade after a release. One behaviour change needed fixing: comparing a JSON union to a string, e.g. `json_get(json_data, 'foo')='abc'`, used to fail to plan (apache/datafusion#10180). DataFusion now coerces the union to Utf8, so the query plans and runs, yielding null for values that aren't JSON strings. test_json_get_equals asserted the old planning error and has been rewritten to assert the new results, with test_plan_json_get_equals pinning the plan the coercion produces. Also raises rust-version to 1.94.0 to match DataFusion main's MSRV, which the CI MSRV job resolves from Cargo.toml. This branch cannot be published as-is: cargo publish rejects git dependencies. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CShJj2bykoAodw9WnMQuRY --- Cargo.toml | 30 ++++++++++++++++++------------ tests/main.rs | 39 +++++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a1e6e05..2ba5c8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,12 @@ license = "Apache-2.0" keywords = ["datafusion", "JSON", "SQL"] categories = ["database-implementations", "parsing"] repository = "https://github.com/datafusion-contrib/datafusion-functions-json/" -rust-version = "1.88.0" +# TEMPORARY: raised from 1.88.0 to match the MSRV of `apache/datafusion` main +rust-version = "1.94.0" [dependencies] -datafusion = { version = "54", default-features = false, features = [ +# TEMPORARY: track unreleased DataFusion, see the note at the bottom of this file +datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main", default-features = false, features = [ "sql", ] } jiter = "0.15.0" @@ -21,7 +23,8 @@ serde_json = "1" [dev-dependencies] codspeed-criterion-compat = "2.6" -datafusion = { version = "54", default-features = false, features = [ +# TEMPORARY: track unreleased DataFusion, see the note at the bottom of this file +datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main", default-features = false, features = [ "nested_expressions", "sql", ] } @@ -39,12 +42,15 @@ pedantic = { level = "deny", priority = -1 } name = "main" harness = false -# [patch.crates-io] -# datafusion = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } -# datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" } +# TEMPORARY — do not release from this branch. +# +# The `datafusion` dependencies above point at `apache/datafusion` main rather than a published +# version, so breakage from unreleased DataFusion changes shows up here instead of during the +# upgrade after a release. `cargo publish` rejects git dependencies, so this branch cannot be +# released as-is: when the next DataFusion is out, swap both entries back to +# `{ version = "", default-features = false, features = [...] }` and restore `rust-version`. +# +# A `[patch.crates-io]` entry (as previously stubbed out here) is deliberately not used: a patch +# only applies while the git version still satisfies the `version = "..."` requirement, so the +# moment main bumps its major the patch would be silently dropped and CI would go back to testing +# the released version. diff --git a/tests/main.rs b/tests/main.rs index 46f5ef2..3d3eae9 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -194,14 +194,41 @@ fn assert_json_field_metadata(metadata: &HashMap) { #[tokio::test] async fn test_json_get_equals() { - let e = run_query(r"select name, json_get(json_data, 'foo')='abc' from test") + // this used to fail to plan, see https://github.com/apache/datafusion/issues/10180, DataFusion + // now coerces the union to a string, so the comparison is equivalent to `json_get(...)::string` + // and yields null for any value that isn't a JSON string + let batches = run_query(r"select name, json_get(json_data, 'foo')='abc' from test") .await - .unwrap_err(); + .unwrap(); - // see https://github.com/apache/datafusion/issues/10180 - assert!(e - .to_string() - .starts_with("Error during planning: Cannot infer common argument type for comparison operation Union")); + let expected = [ + "+------------------+----------------------------------------------------+", + "| name | json_get(test.json_data,Utf8(\"foo\")) = Utf8(\"abc\") |", + "+------------------+----------------------------------------------------+", + "| object_foo | true |", + "| object_foo_array | |", + "| object_foo_obj | |", + "| object_foo_null | |", + "| object_bar | |", + "| list_foo | |", + "| invalid_json | |", + "+------------------+----------------------------------------------------+", + ]; + assert_batches_eq!(expected, &batches); +} + +#[tokio::test] +async fn test_plan_json_get_equals() { + // the implicit cast inserted by type coercion is not rewritten to `json_get_str`, unlike the + // explicit cast in `test_plan_arrow_cast_int` and friends + let lines = logical_plan(r"explain select name, json_get(json_data, 'foo')='abc' from test").await; + + let expected = [ + "Projection: test.name, CAST(json_get(test.json_data, Utf8(\"foo\")) AS Utf8) = Utf8(\"abc\")", + " TableScan: test projection=[name, json_data]", + ]; + + assert_eq!(lines, expected); } #[tokio::test]