Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 20 additions & 15 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,17 @@ jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- uses: dtolnay/rust-toolchain@stable
- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27)
with:
toolchain: stable
components: rustfmt, clippy

- id: cache-rust
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

- uses: pre-commit/action@v3.0.0
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
with:
extra_args: --all-files --verbose
env:
Expand All @@ -33,10 +34,10 @@ jobs:
outputs:
MSRV: ${{ steps.resolve-msrv.outputs.MSRV }}
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- name: set up python
uses: actions/setup-python@v5
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.12'

Expand All @@ -61,20 +62,22 @@ jobs:
RUST_VERSION: ${{ matrix.rust-version }}

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- uses: dtolnay/rust-toolchain@master
- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27)
with:
toolchain: ${{ matrix.rust-version }}

- id: cache-rust
uses: Swatinem/rust-cache@v2
uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

- uses: taiki-e/install-action@cargo-llvm-cov
- uses: taiki-e/install-action@0631aa6515c7d545823c67cfae7ef4fc7f490154 # v2.81.8
with:
tool: cargo-llvm-cov

- run: cargo llvm-cov --all-features --codecov --output-path codecov.json

- uses: codecov/codecov-action@v3
- uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: codecov.json
Expand All @@ -87,7 +90,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
with:
jobs: ${{ toJSON(needs) }}

Expand All @@ -98,12 +101,14 @@ jobs:
environment: release

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3

- name: install rust stable
uses: dtolnay/rust-toolchain@stable
uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 # master (2026-03-27)
with:
toolchain: stable

- uses: Swatinem/rust-cache@v2
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1

- run: cargo publish
env:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/target
Cargo.lock
.idea
.history
21 changes: 17 additions & 4 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "datafusion-functions-json"
version = "0.52.0"
version = "0.54.3"
edition = "2021"
description = "JSON functions for DataFusion"
readme = "README.md"
Expand All @@ -11,14 +11,17 @@ repository = "https://github.com/datafusion-contrib/datafusion-functions-json/"
rust-version = "1.88.0"

[dependencies]
datafusion = { version = "52", default-features = false, features = ["sql"] }
jiter = "0.12.0"
datafusion = { version = "54", default-features = false, features = [
"sql",
] }
jiter = "0.15.0"
log = "0.4"
paste = "1"
serde_json = "1"

[dev-dependencies]
codspeed-criterion-compat = "2.6"
datafusion = { version = "52", default-features = false, features = [
datafusion = { version = "54", default-features = false, features = [
"nested_expressions",
"sql",
] }
Expand All @@ -35,3 +38,13 @@ pedantic = { level = "deny", priority = -1 }
[[bench]]
name = "main"
harness = false

# [patch.crates-io]
# datafusion = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
# datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "d47bd599ee911eb4be018595b860341ff437ccb1" }
10 changes: 10 additions & 0 deletions src/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,14 @@ pub fn invoke<R: InvokeResult>(
}
}

fn null_result<R: InvokeResult>(len: usize) -> DataFusionResult<ArrayRef> {
let mut builder = R::builder(len);
for _ in 0..len {
R::append_value(&mut builder, None);
}
R::finish(builder)
}

fn invoke_array_array<R: InvokeResult>(
json_array: &ArrayRef,
path_array: &ArrayRef,
Expand Down Expand Up @@ -236,6 +244,7 @@ fn invoke_array_array<R: InvokeResult>(
DataType::Utf8 => zip_apply::<R>(json_array.as_string::<i32>(), path_array, jiter_find),
DataType::LargeUtf8 => zip_apply::<R>(json_array.as_string::<i64>(), path_array, jiter_find),
DataType::Utf8View => zip_apply::<R>(json_array.as_string_view(), path_array, jiter_find),
DataType::Null => null_result::<R>(json_array.len()),
other => {
if let Some(string_array) = nested_json_array(json_array, is_object_lookup_array(path_array.data_type())) {
zip_apply::<R>(string_array, path_array, jiter_find)
Expand Down Expand Up @@ -319,6 +328,7 @@ fn invoke_array_scalars<R: InvokeResult>(
DataType::Utf8 => inner::<R>(json_array.as_string::<i32>(), path, jiter_find),
DataType::LargeUtf8 => inner::<R>(json_array.as_string::<i64>(), path, jiter_find),
DataType::Utf8View => inner::<R>(json_array.as_string_view(), path, jiter_find),
DataType::Null => null_result::<R>(json_array.len()),
other => {
if let Some(string_array) = nested_json_array(json_array, is_object_lookup(path)) {
inner::<R>(string_array, path, jiter_find)
Expand Down
30 changes: 26 additions & 4 deletions src/common_union.rs

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium src/common_union.rs:184

Unions produced by previous releases are no longer recognized as JSON unions. The code identifies a union by comparing the full UnionFields for equality, and union_fields() now adds ARROW:extension:name and ARROW:extension:metadata keys to the array and object child fields. Existing unions whose child fields carry only the legacy is_json=true metadata fail this equality check, so passing them into JsonUnionEncoder::from_union or json_union_to_text is rejected as a non-JSON union — even though the legacy is_json key was explicitly retained for backward compatibility. Recognition should accept both the legacy and canonical metadata shapes (e.g., ignore these additive metadata keys when identifying the union type), not require an exact match.

🚀 Reply "fix it for me" or copy this AI Prompt for your agent:
In file @src/common_union.rs around line 184:

Unions produced by previous releases are no longer recognized as JSON unions. The code identifies a union by comparing the full `UnionFields` for equality, and `union_fields()` now adds `ARROW:extension:name` and `ARROW:extension:metadata` keys to the `array` and `object` child fields. Existing unions whose child fields carry only the legacy `is_json=true` metadata fail this equality check, so passing them into `JsonUnionEncoder::from_union` or `json_union_to_text` is rejected as a non-JSON union — even though the legacy `is_json` key was explicitly retained for backward compatibility. Recognition should accept both the legacy and canonical metadata shapes (e.g., ignore these additive metadata keys when identifying the union type), not require an exact match.

Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,30 @@ use datafusion::arrow::datatypes::{DataType, Field, UnionFields, UnionMode};
use datafusion::arrow::error::ArrowError;
use datafusion::common::ScalarValue;

/// Field metadata used to mark a `Utf8` field as containing raw JSON.
///
/// Attach this to any Arrow `Field` whose values are JSON-encoded strings so
/// downstream consumers can recognize them as JSON rather than opaque text.
///
/// Emits Arrow's canonical JSON extension type keys
/// (`ARROW:extension:name` = `arrow.json`, `ARROW:extension:metadata` = `{}`),
/// see <https://arrow.apache.org/docs/format/CanonicalExtensions.html#json>.
///
/// Also emits a legacy `is_json` = `true` key. This key predates this crate's
/// adoption of the canonical extension and is non-standard — no other Arrow
/// tool recognizes it. It is kept only for back-compat with existing
/// downstream consumers of this crate and will be removed in a future
/// release; new consumers should key off `ARROW:extension:name` instead.
#[must_use]
pub fn json_field_metadata() -> HashMap<String, String> {
HashMap::from([
("ARROW:extension:name".to_string(), "arrow.json".to_string()),
("ARROW:extension:metadata".to_string(), "{}".to_string()),
// Legacy, non-standard. Remove in a future release — see doc comment above.
("is_json".to_string(), "true".to_string()),
])
}

pub fn is_json_union(data_type: &DataType) -> bool {
match data_type {
DataType::Union(fields, UnionMode::Sparse) => fields == &union_fields(),
Expand Down Expand Up @@ -161,8 +185,6 @@ fn union_fields() -> UnionFields {
static FIELDS: OnceLock<UnionFields> = OnceLock::new();
FIELDS
.get_or_init(|| {
let json_metadata: HashMap<String, String> =
HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())]);
UnionFields::from_iter([
(TYPE_ID_NULL, Arc::new(Field::new("null", DataType::Null, true))),
(TYPE_ID_BOOL, Arc::new(Field::new("bool", DataType::Boolean, false))),
Expand All @@ -171,11 +193,11 @@ fn union_fields() -> UnionFields {
(TYPE_ID_STR, Arc::new(Field::new("str", DataType::Utf8, false))),
(
TYPE_ID_ARRAY,
Arc::new(Field::new("array", DataType::Utf8, false).with_metadata(json_metadata.clone())),
Arc::new(Field::new("array", DataType::Utf8, false).with_metadata(json_field_metadata())),
),
(
TYPE_ID_OBJECT,
Arc::new(Field::new("object", DataType::Utf8, false).with_metadata(json_metadata.clone())),
Arc::new(Field::new("object", DataType::Utf8, false).with_metadata(json_field_metadata())),
),
])
})
Expand Down
23 changes: 18 additions & 5 deletions src/json_as_text.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::any::Any;
use std::sync::Arc;

use datafusion::arrow::array::{ArrayRef, StringArray, StringBuilder};
Expand Down Expand Up @@ -33,10 +32,6 @@ impl Default for JsonAsText {
}

impl ScalarUDFImpl for JsonAsText {
fn as_any(&self) -> &dyn Any {
self
}

fn name(&self) -> &str {
self.aliases[0].as_str()
}
Expand All @@ -56,6 +51,24 @@ impl ScalarUDFImpl for JsonAsText {
fn aliases(&self) -> &[String] {
&self.aliases
}

fn placement(
&self,
args: &[datafusion::logical_expr::ExpressionPlacement],
) -> datafusion::logical_expr::ExpressionPlacement {
// If the first argument is a column and the remaining arguments are literals (a path)
// then we can push this UDF down to the leaf nodes.
if args.len() >= 2
&& matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column)
&& args[1..]
.iter()
.all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal))
{
datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes
} else {
datafusion::logical_expr::ExpressionPlacement::KeepInPlace
}
}
}

impl InvokeResult for StringArray {
Expand Down
23 changes: 18 additions & 5 deletions src/json_contains.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::any::Any;
use std::sync::Arc;

use datafusion::arrow::array::BooleanBuilder;
Expand Down Expand Up @@ -33,10 +32,6 @@ impl Default for JsonContains {
}

impl ScalarUDFImpl for JsonContains {
fn as_any(&self) -> &dyn Any {
self
}

fn name(&self) -> &str {
self.aliases[0].as_str()
}
Expand All @@ -60,6 +55,24 @@ impl ScalarUDFImpl for JsonContains {
fn aliases(&self) -> &[String] {
&self.aliases
}

fn placement(
&self,
args: &[datafusion::logical_expr::ExpressionPlacement],
) -> datafusion::logical_expr::ExpressionPlacement {
// If the first argument is a column and the remaining arguments are literals (a path)
// then we can push this UDF down to the leaf nodes.
if args.len() >= 2
&& matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column)
&& args[1..]
.iter()
.all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal))
{
datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes
} else {
datafusion::logical_expr::ExpressionPlacement::KeepInPlace
}
}
}

impl InvokeResult for BooleanArray {
Expand Down
5 changes: 0 additions & 5 deletions src/json_from_scalar.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::any::Any;
use std::sync::Arc;

use datafusion::arrow::array::{Array, ArrayRef, AsArray, UnionArray};
Expand Down Expand Up @@ -35,10 +34,6 @@ impl Default for JsonFromScalar {
}

impl ScalarUDFImpl for JsonFromScalar {
fn as_any(&self) -> &dyn Any {
self
}

fn name(&self) -> &str {
self.aliases[0].as_str()
}
Expand Down
32 changes: 26 additions & 6 deletions src/json_get.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
use std::any::Any;
use std::sync::Arc;

use datafusion::arrow::array::ArrayRef;
Expand Down Expand Up @@ -39,10 +38,6 @@ impl Default for JsonGet {
}

impl ScalarUDFImpl for JsonGet {
fn as_any(&self) -> &dyn Any {
self
}

fn name(&self) -> &str {
self.aliases[0].as_str()
}
Expand All @@ -62,6 +57,24 @@ impl ScalarUDFImpl for JsonGet {
fn aliases(&self) -> &[String] {
&self.aliases
}

fn placement(
&self,
args: &[datafusion::logical_expr::ExpressionPlacement],
) -> datafusion::logical_expr::ExpressionPlacement {
// If the first argument is a column and the remaining arguments are literals (a path)
// then we can push this UDF down to the leaf nodes.
if args.len() >= 2
&& matches!(args[0], datafusion::logical_expr::ExpressionPlacement::Column)
&& args[1..]
.iter()
.all(|arg| matches!(arg, datafusion::logical_expr::ExpressionPlacement::Literal))
{
datafusion::logical_expr::ExpressionPlacement::MoveTowardsLeafNodes
} else {
datafusion::logical_expr::ExpressionPlacement::KeepInPlace
}
}
}

impl InvokeResult for JsonUnion {
Expand Down Expand Up @@ -131,7 +144,14 @@ fn build_union(jiter: &mut Jiter, peek: Peek) -> Result<JsonUnionField, GetError
}
_ => match jiter.known_number(peek)? {
NumberAny::Int(NumberInt::Int(value)) => Ok(JsonUnionField::Int(value)),
NumberAny::Int(NumberInt::BigInt(_)) => todo!("BigInt not supported yet"),
// jiter returns `BigInt` for any integer its fast path couldn't decode, which includes
// values that do fit in `i64`, hence the conversion attempt. Values genuinely outside
// `i64` range have no representation in the union, so they're returned as null rather
// than silently losing precision as a `Float`.
NumberAny::Int(NumberInt::BigInt(value)) => match i64::try_from(value) {
Ok(value) => Ok(JsonUnionField::Int(value)),
Err(_) => get_err!(),
},
NumberAny::Float(value) => Ok(JsonUnionField::Float(value)),
},
}
Expand Down
Loading
Loading