diff --git a/README.md b/README.md index 206aaaa..17e3cb4 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ SELECT id, json_col->'a' as json_col_a FROM test_table * [x] `json_contains(json: str, *keys: str | int) -> bool` - true if a JSON string has a specific key (used for the `?` operator) * [x] `json_get(json: str, *keys: str | int) -> JsonUnion` - Get a value from a JSON string by its "path" * [x] `json_get_str(json: str, *keys: str | int) -> str` - Get a string value from a JSON string by its "path" -* [x] `json_get_int(json: str, *keys: str | int) -> int` - Get an integer value from a JSON string by its "path" +* [x] `json_get_int(json: str, *keys: str | int) -> int` - Get an integer value from a JSON string by its "path", a float with an integral value such as `1.0` or `2e3` included * [x] `json_get_float(json: str, *keys: str | int) -> float` - Get a float value from a JSON string by its "path" * [x] `json_get_bool(json: str, *keys: str | int) -> bool` - Get a boolean value from a JSON string by its "path" * [x] `json_get_json(json: str, *keys: str | int) -> str` - Get a nested raw JSON string from a JSON string by its "path" diff --git a/src/json_get_int.rs b/src/json_get_int.rs index bc62293..5b415b5 100644 --- a/src/json_get_int.rs +++ b/src/json_get_int.rs @@ -4,7 +4,7 @@ use datafusion::arrow::array::{ArrayRef, Int64Array, Int64Builder}; use datafusion::arrow::datatypes::DataType; use datafusion::common::{Result as DataFusionResult, ScalarValue}; use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; -use jiter::{NumberInt, Peek}; +use jiter::{NumberAny, NumberInt, Peek}; use crate::common::{get_err, invoke, jiter_json_find, return_type_check, GetError, InvokeResult, JsonPath}; use crate::common_macros::make_udf_function; @@ -108,14 +108,55 @@ fn jiter_json_get_int(json_data: Option<&str>, path: &[JsonPath]) -> Result { get_err!() } - _ => match jiter.known_int(peek)? { - NumberInt::Int(i) => Ok(i), + _ => match jiter.known_number(peek)? { + NumberAny::Int(NumberInt::Int(i)) => Ok(i), // jiter returns `BigInt` for any integer its fast path couldn't decode, which // includes values that do fit in `i64`, hence the conversion attempt - NumberInt::BigInt(b) => i64::try_from(b).map_err(|_| GetError), + NumberAny::Int(NumberInt::BigInt(b)) => i64::try_from(b).map_err(|_| GetError), + NumberAny::Float(f) => float_to_int(f), }, } } else { get_err!() } } + +/// A float with an integral value, such as `1.0` or `2e3`, as that integer. A fractional value is +/// not rounded, and a value outside the `i64` range is not saturated: both are an error. +fn float_to_int(f: f64) -> Result { + // -2^63 and 2^63 are exact as f64, so the range check is exact; a NaN or infinite value has a + // NaN fractional part and fails the first check + #[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)] + if f.fract() == 0.0 && f >= i64::MIN as f64 && f < -(i64::MIN as f64) { + Ok(f as i64) + } else { + get_err!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn float_to_int_edges() { + assert_eq!(float_to_int(1.0).ok(), Some(1)); + assert_eq!(float_to_int(-0.0).ok(), Some(0)); + assert_eq!(float_to_int(2e3).ok(), Some(2000)); + // the largest f64 below 2^63, and -2^63 itself, both fit + assert_eq!( + float_to_int(9_223_372_036_854_774_784.0).ok(), + Some(9_223_372_036_854_774_784) + ); + assert_eq!(float_to_int(-9_223_372_036_854_775_808.0).ok(), Some(i64::MIN)); + // 2^63 is one past i64::MAX, and the next f64 below -2^63 is out of range too + assert!(float_to_int(9_223_372_036_854_775_808.0).is_err()); + assert!(float_to_int(-9_223_372_036_854_777_856.0).is_err()); + assert!(float_to_int(1.5).is_err()); + assert!(float_to_int(-0.5).is_err()); + assert!(float_to_int(1e300).is_err()); + assert!(float_to_int(f64::NAN).is_err()); + assert!(float_to_int(f64::INFINITY).is_err()); + assert!(float_to_int(f64::NEG_INFINITY).is_err()); + } +} diff --git a/tests/main.rs b/tests/main.rs index 7079ee3..6e6a8ce 100644 --- a/tests/main.rs +++ b/tests/main.rs @@ -407,6 +407,50 @@ async fn test_json_get_int_string_parse() { assert_eq!(display_val(batches).await, (DataType::Int64, String::new())); } +/// A JSON float with an integral value reads as that integer. A fractional value, a value outside +/// the `i64` range, and everything else that is not an integer stay NULL. +#[tokio::test] +async fn test_json_get_int_integral_float() { + let cases = [ + ("1.0", "1"), + ("-2.0", "-2"), + ("-0.0", "0"), + ("2e3", "2000"), + ("1.5E1", "15"), + ("-9.2233720368547748e18", "-9223372036854774784"), + ("1.5", ""), + ("-0.5", ""), + ("1e300", ""), + // rounds to 2^63, one past `i64::MAX` + ("9.223372036854776e18", ""), + // decoded by jiter as a `BigInt`, outside the `i64` range + ("123456789012345678901234567890", ""), + // unchanged: integers and numeric strings + ("42", "42"), + ("-9223372036854775808", "-9223372036854775808"), + (r#""42""#, "42"), + (r#""1.0""#, ""), + ("true", ""), + ("null", ""), + ("[1.0]", ""), + ]; + for (value, want) in cases { + let doc = format!(r#"{{"foo": {value}}}"#); + for sql in [ + format!("select json_get_int('{doc}', 'foo')"), + format!("select cast(json_get('{doc}', 'foo') as bigint)"), + format!("select ('{doc}' -> 'foo')::bigint"), + ] { + let batches = run_query(&sql).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int64, want.to_string()), "{sql}"); + } + } + + // a narrowing cast still applies on top of the integer + let batches = run_query(r#"select ('{"foo": 7.0}' -> 'foo')::int"#).await.unwrap(); + assert_eq!(display_val(batches).await, (DataType::Int32, "7".to_string())); +} + #[tokio::test] async fn test_json_get_large_int() { // jiter returns these as `BigInt` even though they fit in i64, they must not be lost