Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions crates/spark/src/function/aggregate/avg.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,36 @@ use datafusion_common::{Result, ScalarValue, not_impl_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,
Signature, TypeSignatureClass, Volatility,
Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, GroupsAccumulator,
ReversedUDAF, Signature, TypeSignatureClass, Volatility,
};
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::{
filtered_null_mask, set_nulls,
};
use datafusion_macros::user_doc;
use std::sync::Arc;

/// AVG aggregate expression
/// Spark average aggregate expression. Differs from standard DataFusion average aggregate
/// in that it uses an `i64` for the count (DataFusion version uses `u64`); also there is ANSI mode
/// support planned in the future for Spark version.

// TODO: see if can deduplicate with DF version
// https://github.com/apache/datafusion/issues/17964

/// Spark-compatible `avg` aggregate expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#avg>
#[user_doc(
doc_section(label = "General Functions"),
description = "Returns the average of the non-NULL values as a DOUBLE, or NULL if there are none. \
All numeric input is cast to DOUBLE first. So unlike DataFusion's `avg` and Spark's, \
DECIMAL input returns a DOUBLE rather than a DECIMAL ([#111](https://github.com/apache/datafusion-spark/issues/111)).",
syntax_example = "avg(expr)",
sql_example = r#"```sql
> SELECT avg(a) FROM (VALUES (40), (23), (17), (40), (NULL)) AS t(a);
30
```"#,
argument(
name = "expr",
description = "Numeric expression to average. NULL values are skipped."
)
)]
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SparkAvg {
signature: Signature,
Expand Down Expand Up @@ -100,6 +115,10 @@ impl AggregateUDFImpl for SparkAvg {
"avg"
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn reverse_expr(&self) -> ReversedUDAF {
ReversedUDAF::Identical
}
Expand Down
63 changes: 54 additions & 9 deletions crates/spark/src/function/aggregate/collect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,15 @@ use datafusion_common::utils::SingleRowListArrayBuilder;
use datafusion_common::{Result, ScalarValue, internal_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
};
use datafusion_functions_aggregate::array_agg::{
ArrayAggAccumulator, DistinctArrayAggAccumulator,
};
use datafusion_macros::user_doc;
use std::sync::Arc;

// Spark implementation of collect_list/collect_set aggregate function.
// Differs from DataFusion ArrayAgg in the following ways:
// - ignores NULL inputs
// - returns an empty list when all inputs are NULL
// - does not support ordering

/// Build an empty list `ScalarValue` for a `List(element_type)` data type.
/// Used as the result for empty window frames and for groups whose inputs
/// were all NULL, matching Spark's `collect_list` / `collect_set` semantics.
Expand All @@ -33,7 +30,28 @@ fn empty_list_scalar(list_type: &DataType) -> Result<ScalarValue> {
Ok(SingleRowListArrayBuilder::new(empty).build_list_scalar())
}

// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_list>
/// Spark-compatible `collect_list` aggregate expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_list>
#[user_doc(
doc_section(label = "General Functions"),
description = "Returns an array of the non-NULL values, including duplicates. \
Unlike DataFusion's `array_agg`, NULL values are skipped, so a group or window frame that is empty \
or holds only NULLs returns an empty array, where `array_agg` returns NULL or an array of NULLs. \
The order of the elements depends on the order of the rows, which may not be deterministic. \
`ORDER BY` inside the call is not supported and fails with an internal error ([#112](https://github.com/apache/datafusion-spark/issues/112)).",
syntax_example = "collect_list(expr)",
sql_example = r#"```sql
> SELECT collect_list(a) FROM (VALUES (1), (2), (NULL), (1)) AS t(a);
[1, 2, 1]
> SELECT collect_list(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a);
[]
```"#,
argument(
name = "expr",
description = "Expression of any type to collect. NULL values are skipped."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkCollectList {
signature: Signature,
Expand All @@ -58,6 +76,10 @@ impl AggregateUDFImpl for SparkCollectList {
"collect_list"
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn signature(&self) -> &Signature {
&self.signature
}
Expand Down Expand Up @@ -94,7 +116,26 @@ impl AggregateUDFImpl for SparkCollectList {
}
}

// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_set>
/// Spark-compatible `collect_set` aggregate expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#collect_set>
#[user_doc(
doc_section(label = "General Functions"),
description = "Returns an array of the distinct non-NULL values, in no particular order. \
Unlike DataFusion's `array_agg(DISTINCT ...)`, NULL values are skipped, so a group or window frame that is empty \
or holds only NULLs returns an empty array, where `array_agg` returns NULL or `[NULL]`.",
syntax_example = "collect_set(expr)",
sql_example = r#"```sql
> SELECT array_sort(collect_set(a)) FROM (VALUES (1), (2), (NULL), (2)) AS t(a);
[1, 2]
> SELECT collect_set(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a);
[]
```"#,
argument(
name = "expr",
description = "Expression of any type to collect. NULL values are skipped."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkCollectSet {
signature: Signature,
Expand All @@ -119,6 +160,10 @@ impl AggregateUDFImpl for SparkCollectSet {
"collect_set"
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn signature(&self) -> &Signature {
&self.signature
}
Expand Down
30 changes: 29 additions & 1 deletion crates/spark/src/function/aggregate/try_sum.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,34 @@ use arrow::datatypes::{
use datafusion_common::{Result, ScalarValue, downcast_value, exec_err, not_impl_err};
use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion_expr::utils::format_state_name;
use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility};
use datafusion_expr::{
Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility,
};
use datafusion_macros::user_doc;
use std::fmt::{Debug, Formatter};
use std::mem::size_of_val;

/// Spark-compatible `try_sum` aggregate expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#try_sum>
#[user_doc(
doc_section(label = "General Functions"),
description = "Returns the sum of the non-NULL values, or NULL if the sum overflows. DataFusion has no built-in `try_sum`. \
Integer input returns BIGINT, floating-point input returns DOUBLE, and DECIMAL(p, s) returns DECIMAL(min(p + 10, 38), s). \
BIGINT and DECIMAL overflow returns NULL, while DOUBLE sums follow IEEE 754 and can return Infinity or NaN. \
Returns NULL if all values are NULL. Other input types, such as intervals, are not supported.",
syntax_example = "try_sum(expr)",
sql_example = r#"```sql
> SELECT try_sum(x) FROM VALUES (1), (2), (3) AS tab(x);
6
> SELECT try_sum(x) FROM VALUES (CAST(9223372036854775807 AS BIGINT)), (1) AS tab(x);
NULL
```"#,
argument(
name = "expr",
description = "Integer, floating-point or decimal expression to sum. NULL values are skipped."
)
)]
#[derive(PartialEq, Eq, Hash)]
pub struct SparkTrySum {
signature: Signature,
Expand Down Expand Up @@ -238,6 +262,10 @@ impl AggregateUDFImpl for SparkTrySum {
"try_sum"
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn signature(&self) -> &Signature {
&self.signature
}
Expand Down
34 changes: 32 additions & 2 deletions crates/spark/src/function/array/repeat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,43 @@ use arrow::datatypes::{DataType, Field};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue, exec_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
use datafusion_functions_nested::repeat::ArrayRepeat;
use datafusion_macros::user_doc;
use std::sync::Arc;

use crate::function::null_utils::{
NullMaskResolution, apply_null_mask, compute_null_mask,
};

/// Spark-compatible `array_repeat` expression. The difference with DataFusion's `array_repeat` is the handling of NULL count: in Spark if the count is NULL, the result is NULL.
/// Spark-compatible `array_repeat` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#array_repeat>
#[user_doc(
doc_section(label = "Array Functions"),
description = "Returns an array that contains `element` `count` times. \
Unlike DataFusion's `array_repeat`, a NULL `count` returns NULL. \
A NULL `element` is repeated like any other value, and a `count` of 0 or less returns an empty array.",
syntax_example = "array_repeat(element, count)",
sql_example = r#"```sql
> SELECT array_repeat('123', 2);
[123, 123]
> SELECT array_repeat(NULL, 2);
[NULL, NULL]
> SELECT array_repeat('123', NULL);
NULL
```"#,
argument(
name = "element",
description = "Value to repeat, of any type. NULL is repeated."
),
argument(
name = "count",
description = "Integer number of times to repeat `element`. Non-integer types are an error. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkArrayRepeat {
signature: Signature,
Expand Down Expand Up @@ -56,6 +82,10 @@ impl ScalarUDFImpl for SparkArrayRepeat {
spark_array_repeat(args)
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn coerce_types(&self, arg_types: &[DataType]) -> Result<Vec<DataType>> {
let [first_type, second_type] = take_function_args(self.name(), arg_types)?;

Expand Down
31 changes: 29 additions & 2 deletions crates/spark/src/function/array/shuffle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,37 @@ use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
};
use datafusion_expr::{
ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignature, Volatility,
ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation,
ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
};
use datafusion_macros::user_doc;
use rand::rng;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng, seq::SliceRandom};
use std::sync::Arc;

/// Spark-compatible `shuffle` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#shuffle>
#[user_doc(
doc_section(label = "Array Functions"),
description = "Returns a random permutation of the array. DataFusion has no built-in `shuffle`. \
NULL elements are kept and shuffled with the others, and a NULL array returns NULL. \
With a `seed`, the order is the same on every run. It comes from a different random number generator \
than Spark's, so it does not match Spark's order for the same seed.",
syntax_example = "shuffle(array[, seed])",
sql_example = r#"```sql
> SELECT shuffle(array(1, 2, 3, 4, 5, NULL), 1);
[1, 4, NULL, 2, 5, 3]
> SELECT shuffle(NULL);
NULL
```"#,
argument(name = "array", description = "Array to shuffle. NULL returns NULL."),
argument(
name = "seed",
description = "Optional constant integer seed. A NULL seed is the same as no seed, which gives a different order on each call."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkShuffle {
signature: Signature,
Expand Down Expand Up @@ -84,6 +107,10 @@ impl ScalarUDFImpl for SparkShuffle {
Ok(Arc::clone(&args.arg_fields[0]))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
if args.args.is_empty() || args.args.len() > 2 {
return exec_err!("shuffle expects 1 or 2 argument(s)");
Expand Down
41 changes: 37 additions & 4 deletions crates/spark/src/function/array/slice.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,44 @@ use datafusion_common::{
Result, ScalarValue, exec_err, internal_err, utils::take_function_args,
};
use datafusion_expr::{
ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs,
ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility,
ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation,
ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
};
use datafusion_functions_nested::extract::array_slice_udf;
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark slice function implementation
/// Main difference from DataFusion's array_slice is that the third argument is the length of the slice and not the end index.
/// Spark-compatible `slice` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#slice>
#[user_doc(
doc_section(label = "Array Functions"),
description = "Returns the part of array `x` that starts at `start` and is at most `length` elements long. \
Unlike DataFusion's `array_slice`, the third argument is a length, not an end index. \
Positions are 1-based, and a negative `start` counts from the end of the array (-1 is the last element). \
A `start` past the end, or one that counts back past the first element, returns an empty array, \
and a `length` that runs past the end is clamped. \
A `start` of 0 or a negative `length` is an error. Returns NULL if any argument is NULL.",
syntax_example = "slice(x, start, length)",
sql_example = r#"```sql
> SELECT slice(array(1, 2, 3, 4), 2, 2);
[2, 3]
> SELECT slice(array(1, 2, 3, 4), -2, 2);
[3, 4]
> SELECT slice(array(1, 2, 3, 4), 100, 2);
[]
```"#,
argument(name = "x", description = "Array to slice. NULL returns NULL."),
argument(
name = "start",
description = "Integer start position, 1-based. A negative value counts from the end of the array. 0 is an error."
),
argument(
name = "length",
description = "Integer maximum number of elements to return. A negative value is an error."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkSlice {
signature: Signature,
Expand Down Expand Up @@ -77,6 +106,10 @@ impl ScalarUDFImpl for SparkSlice {
Ok(Arc::new(Field::new("slice", data_type, nullable)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(
&self,
mut func_args: ScalarFunctionArgs,
Expand Down
Loading
Loading