Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
175 changes: 175 additions & 0 deletions src/common_string.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,175 @@
use std::sync::Arc;

use datafusion::arrow::array::{
ArrayRef, GenericStringArray, GenericStringBuilder, LargeStringArray, OffsetSizeTrait, StringArray,
StringViewArray, StringViewBuilder,
};
use datafusion::arrow::datatypes::DataType;
use datafusion::common::{Result as DataFusionResult, ScalarValue};
use datafusion::logical_expr::ColumnarValue;

use crate::common::{invoke, invoke_array_scalars_direct, return_type_check, GetError, InvokeResult, JsonPath};

/// A JSON function that returns a string, such as `json_as_text`.
///
/// The result has the string type of the JSON argument (see [`string_result_type`]). Each value is
/// a slice of the JSON text or a decoded JSON string, and a decoded string is never longer than its
/// escaped form. Thus a result never needs more offset space than its argument. For example, a
/// `Utf8View` batch of more than 2 GiB does not overflow the `i32` offsets of a `Utf8` result.
pub(crate) trait StringGetter {
/// Write the value at `path` in `json` into `builder`, or a null if there is none.
fn append<B: StringResultBuilder>(json: Option<&str>, path: &[JsonPath], builder: &mut B);

/// The value at `path` in `json` as an owned string, for the argument shapes that `invoke` serves.
fn get(json: Option<&str>, path: &[JsonPath]) -> Result<String, GetError>;
}

/// A builder of one of the Arrow string types.
pub(crate) trait StringResultBuilder {
fn append_str(&mut self, value: Option<&str>);
}

impl<O: OffsetSizeTrait> StringResultBuilder for GenericStringBuilder<O> {
fn append_str(&mut self, value: Option<&str>) {
self.append_option(value);
}
}

impl StringResultBuilder for StringViewBuilder {
fn append_str(&mut self, value: Option<&str>) {
self.append_option(value);
}
}

impl<O: OffsetSizeTrait> InvokeResult for GenericStringArray<O> {
type Item = String;

type Builder = GenericStringBuilder<O>;

const ACCEPT_DICT_RETURN: bool = true;

fn builder(capacity: usize) -> Self::Builder {
GenericStringBuilder::with_capacity(capacity, 0)
}

fn append_value(builder: &mut Self::Builder, value: Option<Self::Item>) {
builder.append_option(value);
}

fn finish(mut builder: Self::Builder) -> DataFusionResult<ArrayRef> {
Ok(Arc::new(builder.finish()))
}

fn scalar(value: Option<Self::Item>) -> ScalarValue {
if O::IS_LARGE {
ScalarValue::LargeUtf8(value)
} else {
ScalarValue::Utf8(value)
}
}
}

impl InvokeResult for StringViewArray {
type Item = String;

type Builder = StringViewBuilder;

const ACCEPT_DICT_RETURN: bool = true;

fn builder(capacity: usize) -> Self::Builder {
StringViewBuilder::with_capacity(capacity)
}

fn append_value(builder: &mut Self::Builder, value: Option<Self::Item>) {
builder.append_option(value);
}

fn finish(mut builder: Self::Builder) -> DataFusionResult<ArrayRef> {
Ok(Arc::new(builder.finish()))
}

fn scalar(value: Option<Self::Item>) -> ScalarValue {
ScalarValue::Utf8View(value)
}
}

/// The string type of the result for a JSON argument of type `json_type`.
///
/// A string argument gives its own type, and a dictionary argument gives the type of its values.
/// A JSON union holds its strings as `Utf8`, so a union argument gives `Utf8`.
fn string_result_type(json_type: &DataType) -> DataType {
match json_type {
DataType::LargeUtf8 => DataType::LargeUtf8,
DataType::Utf8View => DataType::Utf8View,
DataType::Dictionary(_, value_type) => string_result_type(value_type),
_ => DataType::Utf8,
}
}

/// `ScalarUDFImpl::return_type` for a [`StringGetter`].
pub(crate) fn string_return_type(arg_types: &[DataType], fn_name: &str) -> DataFusionResult<DataType> {
let value_type = arg_types.first().map_or(DataType::Utf8, string_result_type);
return_type_check::<StringArray>(arg_types, fn_name, value_type)
}

/// `ScalarUDFImpl::invoke_with_args` for a [`StringGetter`].
pub(crate) fn invoke_string<G: StringGetter>(args: &[ColumnarValue]) -> DataFusionResult<ColumnarValue> {
match args.first().map(|json| string_result_type(&json.data_type())) {
Some(DataType::LargeUtf8) => invoke_string_as::<G, LargeStringArray>(args),
Some(DataType::Utf8View) => invoke_string_as::<G, StringViewArray>(args),
_ => invoke_string_as::<G, StringArray>(args),
}
}

fn invoke_string_as<G: StringGetter, R>(args: &[ColumnarValue]) -> DataFusionResult<ColumnarValue>
where
R: InvokeResult<Item = String>,
R::Builder: StringResultBuilder,
{
if let Some(result) = invoke_array_scalars_direct::<R>(args, G::append::<R::Builder>)? {
return Ok(result);
}
invoke::<R>(args, G::get)
}

#[cfg(test)]
pub(crate) mod test_util {
use super::*;
use crate::common::test_util::assert_direct_matches_owned;

/// Check the direct path and `invoke` build equal arrays for each string result type.
pub(crate) fn assert_string_direct_matches_owned<G: StringGetter>(
rows: &[Option<&str>],
paths: &[Vec<ScalarValue>],
) {
assert_direct_matches_owned::<StringArray>(rows, paths, G::append, G::get);
assert_direct_matches_owned::<LargeStringArray>(rows, paths, G::append, G::get);
assert_direct_matches_owned::<StringViewArray>(rows, paths, G::append, G::get);
}
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn result_type_follows_json_type() {
let dict = |value: DataType| DataType::Dictionary(Box::new(DataType::Int32), Box::new(value));
let large_dict = |value: DataType| DataType::Dictionary(Box::new(DataType::Int64), Box::new(value));
let union = crate::JSON_UNION_DATA_TYPE.clone();
let cases = [
(DataType::Utf8, DataType::Utf8),
(DataType::LargeUtf8, DataType::LargeUtf8),
(DataType::Utf8View, DataType::Utf8View),
(union.clone(), DataType::Utf8),
(dict(DataType::Utf8), large_dict(DataType::Utf8)),
(dict(DataType::LargeUtf8), large_dict(DataType::LargeUtf8)),
(dict(DataType::Utf8View), large_dict(DataType::Utf8View)),
(dict(union), large_dict(DataType::Utf8)),
];
for (json_type, want) in cases {
let got = string_return_type(&[json_type.clone(), DataType::Utf8], "f").unwrap();
assert_eq!(got, want, "{json_type}");
}
}
}
69 changes: 20 additions & 49 deletions src/json_as_text.rs
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
use std::sync::Arc;

use datafusion::arrow::array::{ArrayRef, StringArray, StringBuilder};
use datafusion::arrow::datatypes::DataType;
use datafusion::common::{Result as DataFusionResult, ScalarValue};
use datafusion::common::Result as DataFusionResult;
use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility};
use jiter::{Jiter, Peek};

use crate::common::{
get_err, invoke, invoke_array_scalars_direct, jiter_json_find, jiter_skip_str, return_type_check, GetError,
InvokeResult, JsonPath,
};
use crate::common::{get_err, jiter_json_find, jiter_skip_str, GetError, JsonPath};
use crate::common_macros::make_udf_function;
use crate::common_string::{invoke_string, string_return_type, StringGetter, StringResultBuilder};

make_udf_function!(
JsonAsText,
Expand Down Expand Up @@ -44,14 +39,11 @@ impl ScalarUDFImpl for JsonAsText {
}

fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult<DataType> {
return_type_check::<StringArray>(arg_types, self.name(), DataType::Utf8)
string_return_type(arg_types, self.name())
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult<ColumnarValue> {
if let Some(result) = invoke_array_scalars_direct::<StringArray>(&args.args, append_json_as_text)? {
return Ok(result);
}
invoke::<StringArray>(&args.args, jiter_json_as_text)
invoke_string::<Self>(&args.args)
}

fn aliases(&self) -> &[String] {
Expand All @@ -77,42 +69,20 @@ impl ScalarUDFImpl for JsonAsText {
}
}

impl InvokeResult for StringArray {
type Item = String;

type Builder = StringBuilder;

const ACCEPT_DICT_RETURN: bool = true;

fn builder(capacity: usize) -> Self::Builder {
StringBuilder::with_capacity(capacity, 0)
}

fn append_value(builder: &mut Self::Builder, value: Option<Self::Item>) {
builder.append_option(value);
}

fn finish(mut builder: Self::Builder) -> DataFusionResult<ArrayRef> {
Ok(Arc::new(builder.finish()))
}

fn scalar(value: Option<Self::Item>) -> ScalarValue {
ScalarValue::Utf8(value)
impl StringGetter for JsonAsText {
fn append<B: StringResultBuilder>(opt_json: Option<&str>, path: &[JsonPath], builder: &mut B) {
if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) {
builder.append_str(text_value(json, &mut jiter, peek).ok());
} else {
builder.append_str(None);
}
}
}

fn jiter_json_as_text(opt_json: Option<&str>, path: &[JsonPath]) -> Result<String, GetError> {
let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else {
return get_err!();
};
text_value(json, &mut jiter, peek).map(str::to_owned)
}

fn append_json_as_text(opt_json: Option<&str>, path: &[JsonPath], builder: &mut StringBuilder) {
if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) {
builder.append_option(text_value(json, &mut jiter, peek).ok());
} else {
builder.append_null();
fn get(opt_json: Option<&str>, path: &[JsonPath]) -> Result<String, GetError> {
let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else {
return get_err!();
};
text_value(json, &mut jiter, peek).map(str::to_owned)
}
}

Expand All @@ -129,7 +99,8 @@ fn text_value<'a, 'j: 'a>(json: &'j str, jiter: &'a mut Jiter<'j>, peek: Peek) -
#[cfg(test)]
mod tests {
use super::*;
use crate::common::test_util::assert_direct_matches_owned;
use crate::common_string::test_util::assert_string_direct_matches_owned;
use datafusion::common::ScalarValue;

#[test]
fn direct_builder_matches_owned_results() {
Expand All @@ -153,6 +124,6 @@ mod tests {
],
vec![ScalarValue::Utf8(None)],
];
assert_direct_matches_owned::<StringArray>(&rows, &paths, append_json_as_text, jiter_json_as_text);
assert_string_direct_matches_owned::<JsonAsText>(&rows, &paths);
}
}
42 changes: 19 additions & 23 deletions src/json_get_json.rs
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
use std::sync::Arc;

use datafusion::arrow::array::{StringArray, StringBuilder};
use datafusion::arrow::datatypes::{DataType, Field, FieldRef};
use datafusion::common::Result as DataFusionResult;
use datafusion::logical_expr::{
ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
};

use crate::common::{
get_err, invoke, invoke_array_scalars_direct, jiter_json_find, jiter_skip_str, return_type_check, GetError,
JsonPath,
};
use crate::common::{get_err, jiter_json_find, jiter_skip_str, GetError, JsonPath};
use crate::common_macros::make_udf_function;
use crate::common_string::{invoke_string, string_return_type, StringGetter, StringResultBuilder};
use crate::common_union::json_field_metadata;

make_udf_function!(
Expand Down Expand Up @@ -46,7 +43,7 @@ impl ScalarUDFImpl for JsonGetJson {
}

fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult<DataType> {
return_type_check::<StringArray>(arg_types, self.name(), DataType::Utf8)
string_return_type(arg_types, self.name())
}

fn return_field_from_args(&self, args: ReturnFieldArgs) -> DataFusionResult<FieldRef> {
Expand All @@ -58,10 +55,7 @@ impl ScalarUDFImpl for JsonGetJson {
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult<ColumnarValue> {
if let Some(result) = invoke_array_scalars_direct::<StringArray>(&args.args, append_json_get_json)? {
return Ok(result);
}
invoke::<StringArray>(&args.args, jiter_json_get_json)
invoke_string::<Self>(&args.args)
}

fn aliases(&self) -> &[String] {
Expand All @@ -87,25 +81,27 @@ impl ScalarUDFImpl for JsonGetJson {
}
}

fn jiter_json_get_json(opt_json: Option<&str>, path: &[JsonPath]) -> Result<String, GetError> {
let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else {
return get_err!();
};
jiter_skip_str(json, &mut jiter, peek).map(str::to_owned)
}
impl StringGetter for JsonGetJson {
fn append<B: StringResultBuilder>(opt_json: Option<&str>, path: &[JsonPath], builder: &mut B) {
if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) {
builder.append_str(jiter_skip_str(json, &mut jiter, peek).ok());
} else {
builder.append_str(None);
}
}

fn append_json_get_json(opt_json: Option<&str>, path: &[JsonPath], builder: &mut StringBuilder) {
if let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) {
builder.append_option(jiter_skip_str(json, &mut jiter, peek).ok());
} else {
builder.append_null();
fn get(opt_json: Option<&str>, path: &[JsonPath]) -> Result<String, GetError> {
let (Some(json), Some((mut jiter, peek))) = (opt_json, jiter_json_find(opt_json, path)) else {
return get_err!();
};
jiter_skip_str(json, &mut jiter, peek).map(str::to_owned)
}
}

#[cfg(test)]
mod tests {
use super::*;
use crate::common::test_util::assert_direct_matches_owned;
use crate::common_string::test_util::assert_string_direct_matches_owned;
use datafusion::common::ScalarValue;

#[test]
Expand All @@ -125,6 +121,6 @@ mod tests {
vec![ScalarValue::Utf8(Some("c".to_owned())), ScalarValue::Int64(Some(1))],
vec![ScalarValue::Utf8(None)],
];
assert_direct_matches_owned::<StringArray>(&rows, &paths, append_json_get_json, jiter_json_get_json);
assert_string_direct_matches_owned::<JsonGetJson>(&rows, &paths);
}
}
Loading
Loading