Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ SELECT id, json_col->'a' as json_col_a FROM test_table
* [x] `json_contains(json: str, *keys: str | int) -> bool` - true if a JSON string has a specific key (used for the `?` operator)
* [x] `json_get(json: str, *keys: str | int) -> JsonUnion` - Get a value from a JSON string by its "path"
* [x] `json_get_str(json: str, *keys: str | int) -> str` - Get a string value from a JSON string by its "path"
* [x] `json_get_int(json: str, *keys: str | int) -> int` - Get an integer value from a JSON string by its "path"
* [x] `json_get_int(json: str, *keys: str | int) -> int` - Get an integer value from a JSON string by its "path", a float with an integral value such as `1.0` or `2e3` included
* [x] `json_get_float(json: str, *keys: str | int) -> float` - Get a float value from a JSON string by its "path"
* [x] `json_get_bool(json: str, *keys: str | int) -> bool` - Get a boolean value from a JSON string by its "path"
* [x] `json_get_json(json: str, *keys: str | int) -> str` - Get a nested raw JSON string from a JSON string by its "path"
Expand Down
49 changes: 45 additions & 4 deletions src/json_get_int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use datafusion::arrow::array::{ArrayRef, Int64Array, Int64Builder};
use datafusion::arrow::datatypes::DataType;
use datafusion::common::{Result as DataFusionResult, ScalarValue};
use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
use jiter::{NumberInt, Peek};
use jiter::{NumberAny, NumberInt, Peek};

use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath};
use crate::common_macros::make_udf_function;
Expand Down Expand Up @@ -108,14 +108,55 @@ fn jiter_json_get_int(json_data: Option<&str>, path: &[JsonPath]) -> Result<i64,
Peek::Null | Peek::True | Peek::False | Peek::Infinity | Peek::NaN | Peek::Array | Peek::Object => {
get_err!()
}
_ => match jiter.known_int(peek)? {
NumberInt::Int(i) => Ok(i),
_ => match jiter.known_number(peek)? {
NumberAny::Int(NumberInt::Int(i)) => Ok(i),
// jiter returns `BigInt` for any integer its fast path couldn't decode, which
// includes values that do fit in `i64`, hence the conversion attempt
NumberInt::BigInt(b) => i64::try_from(b).map_err(|_| GetError),
NumberAny::Int(NumberInt::BigInt(b)) => i64::try_from(b).map_err(|_| GetError),
NumberAny::Float(f) => float_to_int(f),
},
}
} else {
get_err!()
}
}

/// A float with an integral value, such as `1.0` or `2e3`, as that integer. A fractional value is
/// not rounded, and a value outside the `i64` range is not saturated: both are an error.
fn float_to_int(f: f64) -> Result<i64, GetError> {
// -2^63 and 2^63 are exact as f64, so the range check is exact; a NaN or infinite value has a
// NaN fractional part and fails the first check
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
if f.fract() == 0.0 && f >= i64::MIN as f64 && f < -(i64::MIN as f64) {
Ok(f as i64)
} else {
get_err!()
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn float_to_int_edges() {
assert_eq!(float_to_int(1.0).ok(), Some(1));
assert_eq!(float_to_int(-0.0).ok(), Some(0));
assert_eq!(float_to_int(2e3).ok(), Some(2000));
// the largest f64 below 2^63, and -2^63 itself, both fit
assert_eq!(
float_to_int(9_223_372_036_854_774_784.0).ok(),
Some(9_223_372_036_854_774_784)
);
assert_eq!(float_to_int(-9_223_372_036_854_775_808.0).ok(), Some(i64::MIN));
// 2^63 is one past i64::MAX, and the next f64 below -2^63 is out of range too
assert!(float_to_int(9_223_372_036_854_775_808.0).is_err());
assert!(float_to_int(-9_223_372_036_854_777_856.0).is_err());
assert!(float_to_int(1.5).is_err());
assert!(float_to_int(-0.5).is_err());
assert!(float_to_int(1e300).is_err());
assert!(float_to_int(f64::NAN).is_err());
assert!(float_to_int(f64::INFINITY).is_err());
assert!(float_to_int(f64::NEG_INFINITY).is_err());
}
}
44 changes: 44 additions & 0 deletions tests/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -407,6 +407,50 @@ async fn test_json_get_int_string_parse() {
assert_eq!(display_val(batches).await, (DataType::Int64, String::new()));
}

/// A JSON float with an integral value reads as that integer. A fractional value, a value outside
/// the `i64` range, and everything else that is not an integer stay NULL.
#[tokio::test]
async fn test_json_get_int_integral_float() {
let cases = [
("1.0", "1"),
("-2.0", "-2"),
("-0.0", "0"),
("2e3", "2000"),
("1.5E1", "15"),
("-9.2233720368547748e18", "-9223372036854774784"),
("1.5", ""),
("-0.5", ""),
("1e300", ""),
// rounds to 2^63, one past `i64::MAX`
("9.223372036854776e18", ""),
// decoded by jiter as a `BigInt`, outside the `i64` range
("123456789012345678901234567890", ""),
// unchanged: integers and numeric strings
("42", "42"),
("-9223372036854775808", "-9223372036854775808"),
(r#""42""#, "42"),
(r#""1.0""#, ""),
("true", ""),
("null", ""),
("[1.0]", ""),
];
for (value, want) in cases {
let doc = format!(r#"{{"foo": {value}}}"#);
for sql in [
format!("select json_get_int('{doc}', 'foo')"),
format!("select cast(json_get('{doc}', 'foo') as bigint)"),
format!("select ('{doc}' -> 'foo')::bigint"),
] {
let batches = run_query(&sql).await.unwrap();
assert_eq!(display_val(batches).await, (DataType::Int64, want.to_string()), "{sql}");
}
}

// a narrowing cast still applies on top of the integer
let batches = run_query(r#"select ('{"foo": 7.0}' -> 'foo')::int"#).await.unwrap();
assert_eq!(display_val(batches).await, (DataType::Int32, "7".to_string()));
}

#[tokio::test]
async fn test_json_get_large_int() {
// jiter returns these as `BigInt` even though they fit in i64, they must not be lost
Expand Down
Loading