From 5ff43f51f6abee65f645dd3a1086fb63dae15d7e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:39:01 -0700 Subject: [PATCH 1/2] docs(spark): document array, collection and aggregate functions with #[user_doc] Document these functions with #[user_doc] and implement documentation(), following the guidelines in CONTRIBUTING.md: - array: slice, array, shuffle, array_repeat - collection: size - json: json_tuple - generator: stack - aggregate: try_sum, avg, collect_list, collect_set Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/aggregate/avg.rs | 33 +++++++--- .../spark/src/function/aggregate/collect.rs | 62 ++++++++++++++++--- .../spark/src/function/aggregate/try_sum.rs | 30 ++++++++- crates/spark/src/function/array/repeat.rs | 34 +++++++++- crates/spark/src/function/array/shuffle.rs | 31 +++++++++- crates/spark/src/function/array/slice.rs | 41 ++++++++++-- .../spark/src/function/array/spark_array.rs | 31 +++++++++- crates/spark/src/function/collection/size.rs | 36 +++++++++-- crates/spark/src/function/generator/stack.rs | 58 +++++++++++------ crates/spark/src/function/json/json_tuple.rs | 49 ++++++++++----- 10 files changed, 337 insertions(+), 68 deletions(-) diff --git a/crates/spark/src/function/aggregate/avg.rs b/crates/spark/src/function/aggregate/avg.rs index 0ad8ceff..2c0b5d14 100644 --- a/crates/spark/src/function/aggregate/avg.rs +++ b/crates/spark/src/function/aggregate/avg.rs @@ -16,21 +16,36 @@ use datafusion_common::{Result, ScalarValue, not_impl_err}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name}; use datafusion_expr::{ - Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF, - Signature, TypeSignatureClass, Volatility, + Accumulator, AggregateUDFImpl, Coercion, Documentation, EmitTo, GroupsAccumulator, + ReversedUDAF, Signature, TypeSignatureClass, Volatility, }; use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::{ filtered_null_mask, set_nulls, }; +use datafusion_macros::user_doc; use std::sync::Arc; -/// AVG aggregate expression -/// Spark average aggregate expression. Differs from standard DataFusion average aggregate -/// in that it uses an `i64` for the count (DataFusion version uses `u64`); also there is ANSI mode -/// support planned in the future for Spark version. - // TODO: see if can deduplicate with DF version // https://github.com/apache/datafusion/issues/17964 + +/// Spark-compatible `avg` aggregate expression. +/// +/// +#[user_doc( + doc_section(label = "General Functions"), + description = "Returns the average of the non-NULL values as a DOUBLE, or NULL if there are none. \ + All numeric input is cast to DOUBLE first. So unlike DataFusion's `avg` and Spark's, \ + DECIMAL input returns a DOUBLE rather than a DECIMAL.", + syntax_example = "avg(expr)", + sql_example = r#"```sql +> SELECT avg(a) FROM (VALUES (40), (23), (17), (40), (NULL)) AS t(a); +30 +```"#, + argument( + name = "expr", + description = "Numeric expression to average. NULL values are skipped." + ) +)] #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct SparkAvg { signature: Signature, @@ -100,6 +115,10 @@ impl AggregateUDFImpl for SparkAvg { "avg" } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn reverse_expr(&self) -> ReversedUDAF { ReversedUDAF::Identical } diff --git a/crates/spark/src/function/aggregate/collect.rs b/crates/spark/src/function/aggregate/collect.rs index d29e3460..9ca585c1 100644 --- a/crates/spark/src/function/aggregate/collect.rs +++ b/crates/spark/src/function/aggregate/collect.rs @@ -8,18 +8,15 @@ use datafusion_common::utils::SingleRowListArrayBuilder; use datafusion_common::{Result, ScalarValue, internal_err}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; -use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; +use datafusion_expr::{ + Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility, +}; use datafusion_functions_aggregate::array_agg::{ ArrayAggAccumulator, DistinctArrayAggAccumulator, }; +use datafusion_macros::user_doc; use std::sync::Arc; -// Spark implementation of collect_list/collect_set aggregate function. -// Differs from DataFusion ArrayAgg in the following ways: -// - ignores NULL inputs -// - returns an empty list when all inputs are NULL -// - does not support ordering - /// Build an empty list `ScalarValue` for a `List(element_type)` data type. /// Used as the result for empty window frames and for groups whose inputs /// were all NULL, matching Spark's `collect_list` / `collect_set` semantics. @@ -33,7 +30,27 @@ fn empty_list_scalar(list_type: &DataType) -> Result { Ok(SingleRowListArrayBuilder::new(empty).build_list_scalar()) } -// +/// Spark-compatible `collect_list` aggregate expression. +/// +/// +#[user_doc( + doc_section(label = "General Functions"), + description = "Returns an array of the non-NULL values, including duplicates. \ + Unlike DataFusion's `array_agg`, NULL values are skipped, so a group or window frame that is empty \ + or holds only NULLs returns an empty array, where `array_agg` returns NULL or an array of NULLs. \ + The order of the elements depends on the order of the rows, which may not be deterministic.", + syntax_example = "collect_list(expr)", + sql_example = r#"```sql +> SELECT collect_list(a) FROM (VALUES (1), (2), (NULL), (1)) AS t(a); +[1, 2, 1] +> SELECT collect_list(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a); +[] +```"#, + argument( + name = "expr", + description = "Expression of any type to collect. NULL values are skipped." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkCollectList { signature: Signature, @@ -58,6 +75,10 @@ impl AggregateUDFImpl for SparkCollectList { "collect_list" } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn signature(&self) -> &Signature { &self.signature } @@ -94,7 +115,26 @@ impl AggregateUDFImpl for SparkCollectList { } } -// +/// Spark-compatible `collect_set` aggregate expression. +/// +/// +#[user_doc( + doc_section(label = "General Functions"), + description = "Returns an array of the distinct non-NULL values, in no particular order. \ + Unlike DataFusion's `array_agg(DISTINCT ...)`, NULL values are skipped, so a group or window frame that is empty \ + or holds only NULLs returns an empty array, where `array_agg` returns NULL or `[NULL]`.", + syntax_example = "collect_set(expr)", + sql_example = r#"```sql +> SELECT array_sort(collect_set(a)) FROM (VALUES (1), (2), (NULL), (2)) AS t(a); +[1, 2] +> SELECT collect_set(a) FROM (VALUES (CAST(NULL AS INT))) AS t(a); +[] +```"#, + argument( + name = "expr", + description = "Expression of any type to collect. NULL values are skipped." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkCollectSet { signature: Signature, @@ -119,6 +159,10 @@ impl AggregateUDFImpl for SparkCollectSet { "collect_set" } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn signature(&self) -> &Signature { &self.signature } diff --git a/crates/spark/src/function/aggregate/try_sum.rs b/crates/spark/src/function/aggregate/try_sum.rs index 92649629..a79bf633 100644 --- a/crates/spark/src/function/aggregate/try_sum.rs +++ b/crates/spark/src/function/aggregate/try_sum.rs @@ -10,10 +10,34 @@ use arrow::datatypes::{ use datafusion_common::{Result, ScalarValue, downcast_value, exec_err, not_impl_err}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; -use datafusion_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; +use datafusion_expr::{ + Accumulator, AggregateUDFImpl, Documentation, Signature, Volatility, +}; +use datafusion_macros::user_doc; use std::fmt::{Debug, Formatter}; use std::mem::size_of_val; +/// Spark-compatible `try_sum` aggregate expression. +/// +/// +#[user_doc( + doc_section(label = "General Functions"), + description = "Returns the sum of the non-NULL values, or NULL if the sum overflows. DataFusion has no built-in `try_sum`. \ + Integer input returns BIGINT, floating-point input returns DOUBLE, and DECIMAL(p, s) returns DECIMAL(min(p + 10, 38), s). \ + BIGINT and DECIMAL overflow returns NULL, while DOUBLE sums follow IEEE 754 and can return Infinity or NaN. \ + Returns NULL if all values are NULL. Other input types, such as intervals, are not supported.", + syntax_example = "try_sum(expr)", + sql_example = r#"```sql +> SELECT try_sum(x) FROM VALUES (1), (2), (3) AS tab(x); +6 +> SELECT try_sum(x) FROM VALUES (CAST(9223372036854775807 AS BIGINT)), (1) AS tab(x); +NULL +```"#, + argument( + name = "expr", + description = "Integer, floating-point or decimal expression to sum. NULL values are skipped." + ) +)] #[derive(PartialEq, Eq, Hash)] pub struct SparkTrySum { signature: Signature, @@ -238,6 +262,10 @@ impl AggregateUDFImpl for SparkTrySum { "try_sum" } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn signature(&self) -> &Signature { &self.signature } diff --git a/crates/spark/src/function/array/repeat.rs b/crates/spark/src/function/array/repeat.rs index 9016c9df..e9f6f31b 100644 --- a/crates/spark/src/function/array/repeat.rs +++ b/crates/spark/src/function/array/repeat.rs @@ -6,17 +6,43 @@ use arrow::datatypes::{DataType, Field}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, }; use datafusion_functions_nested::repeat::ArrayRepeat; +use datafusion_macros::user_doc; use std::sync::Arc; use crate::function::null_utils::{ NullMaskResolution, apply_null_mask, compute_null_mask, }; -/// Spark-compatible `array_repeat` expression. The difference with DataFusion's `array_repeat` is the handling of NULL count: in Spark if the count is NULL, the result is NULL. +/// Spark-compatible `array_repeat` expression. +/// /// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns an array that contains `element` `count` times. \ + Unlike DataFusion's `array_repeat`, a NULL `count` returns NULL. \ + A NULL `element` is repeated like any other value, and a `count` of 0 or less returns an empty array.", + syntax_example = "array_repeat(element, count)", + sql_example = r#"```sql +> SELECT array_repeat('123', 2); +[123, 123] +> SELECT array_repeat(NULL, 2); +[NULL, NULL] +> SELECT array_repeat('123', NULL); +NULL +```"#, + argument( + name = "element", + description = "Value to repeat, of any type. NULL is repeated." + ), + argument( + name = "count", + description = "Integer number of times to repeat `element`. Non-integer types are an error. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkArrayRepeat { signature: Signature, @@ -56,6 +82,10 @@ impl ScalarUDFImpl for SparkArrayRepeat { spark_array_repeat(args) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn coerce_types(&self, arg_types: &[DataType]) -> Result> { let [first_type, second_type] = take_function_args(self.name(), arg_types)?; diff --git a/crates/spark/src/function/array/shuffle.rs b/crates/spark/src/function/array/shuffle.rs index 00706133..97cef47e 100644 --- a/crates/spark/src/function/array/shuffle.rs +++ b/crates/spark/src/function/array/shuffle.rs @@ -18,14 +18,37 @@ use datafusion_common::{ Result, ScalarValue, exec_err, internal_err, utils::take_function_args, }; use datafusion_expr::{ - ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ScalarFunctionArgs, - ScalarUDFImpl, Signature, TypeSignature, Volatility, + ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, + ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; +use datafusion_macros::user_doc; use rand::rng; use rand::rngs::StdRng; use rand::{Rng, SeedableRng, seq::SliceRandom}; use std::sync::Arc; +/// Spark-compatible `shuffle` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns a random permutation of the array. DataFusion has no built-in `shuffle`. \ + NULL elements are kept and shuffled with the others, and a NULL array returns NULL. \ + With a `seed`, the order is the same on every run. It comes from a different random number generator \ + than Spark's, so it does not match Spark's order for the same seed.", + syntax_example = "shuffle(array[, seed])", + sql_example = r#"```sql +> SELECT shuffle(array(1, 2, 3, 4, 5, NULL), 1); +[1, 4, NULL, 2, 5, 3] +> SELECT shuffle(NULL); +NULL +```"#, + argument(name = "array", description = "Array to shuffle. NULL returns NULL."), + argument( + name = "seed", + description = "Optional constant integer seed. A NULL seed is the same as no seed, which gives a different order on each call." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkShuffle { signature: Signature, @@ -84,6 +107,10 @@ impl ScalarUDFImpl for SparkShuffle { Ok(Arc::clone(&args.arg_fields[0])) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.is_empty() || args.args.len() > 2 { return exec_err!("shuffle expects 1 or 2 argument(s)"); diff --git a/crates/spark/src/function/array/slice.rs b/crates/spark/src/function/array/slice.rs index 70069e03..5e6f7274 100644 --- a/crates/spark/src/function/array/slice.rs +++ b/crates/spark/src/function/array/slice.rs @@ -10,15 +10,44 @@ use datafusion_common::{ Result, ScalarValue, exec_err, internal_err, utils::take_function_args, }; use datafusion_expr::{ - ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs, - ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, + ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, + ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + Volatility, }; use datafusion_functions_nested::extract::array_slice_udf; +use datafusion_macros::user_doc; use std::sync::Arc; -/// Spark slice function implementation -/// Main difference from DataFusion's array_slice is that the third argument is the length of the slice and not the end index. +/// Spark-compatible `slice` expression. +/// /// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the part of array `x` that starts at `start` and is at most `length` elements long. \ + Unlike DataFusion's `array_slice`, the third argument is a length, not an end index. \ + Positions are 1-based, and a negative `start` counts from the end of the array (-1 is the last element). \ + A `start` past the end, or one that counts back past the first element, returns an empty array, \ + and a `length` that runs past the end is clamped. \ + A `start` of 0 or a negative `length` is an error. Returns NULL if any argument is NULL.", + syntax_example = "slice(x, start, length)", + sql_example = r#"```sql +> SELECT slice(array(1, 2, 3, 4), 2, 2); +[2, 3] +> SELECT slice(array(1, 2, 3, 4), -2, 2); +[3, 4] +> SELECT slice(array(1, 2, 3, 4), 100, 2); +[] +```"#, + argument(name = "x", description = "Array to slice. NULL returns NULL."), + argument( + name = "start", + description = "Integer start position, 1-based. A negative value counts from the end of the array. 0 is an error." + ), + argument( + name = "length", + description = "Integer maximum number of elements to return. A negative value is an error." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkSlice { signature: Signature, @@ -77,6 +106,10 @@ impl ScalarUDFImpl for SparkSlice { Ok(Arc::new(Field::new("slice", data_type, nullable))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args( &self, mut func_args: ScalarFunctionArgs, diff --git a/crates/spark/src/function/array/spark_array.rs b/crates/spark/src/function/array/spark_array.rs index 5ff40ed6..e2f0efe6 100644 --- a/crates/spark/src/function/array/spark_array.rs +++ b/crates/spark/src/function/array/spark_array.rs @@ -9,15 +9,38 @@ use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::utils::SingleRowListArrayBuilder; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; use datafusion_functions_nested::make_array::{array_array, coerce_types_inner}; +use datafusion_macros::user_doc; use crate::function::functions_nested_utils::make_scalar_function; const ARRAY_FIELD_DEFAULT_NAME: &str = "element"; +/// Spark-compatible `array` expression. +/// +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns an array of the given elements, coerced to a common type. NULL elements are kept, \ + and with no arguments the result is an empty array. \ + Unlike DataFusion's `make_array`, the list's element field is named `element`, as in Spark, rather than `item`.", + syntax_example = "array(expr, ...)", + sql_example = r#"```sql +> SELECT array(1, 2, 3); +[1, 2, 3] +> SELECT array(1, NULL, 3); +[1, NULL, 3] +> SELECT array(); +[] +```"#, + argument( + name = "expr", + description = "Zero or more elements of any type that can be coerced to a common type." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkArray { signature: Signature, @@ -79,6 +102,10 @@ impl ScalarUDFImpl for SparkArray { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let ScalarFunctionArgs { args, .. } = args; make_scalar_function(make_array_inner)(args.as_slice()) diff --git a/crates/spark/src/function/collection/size.rs b/crates/spark/src/function/collection/size.rs index d860286e..417c9d9f 100644 --- a/crates/spark/src/function/collection/size.rs +++ b/crates/spark/src/function/collection/size.rs @@ -7,16 +7,38 @@ use arrow::compute::kernels::length::length as arrow_length; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::{Result, plan_err}; use datafusion_expr::{ - ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, ReturnFieldArgs, - ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, + ArrayFunctionArgument, ArrayFunctionSignature, ColumnarValue, Documentation, + ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, + Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; use std::sync::Arc; -/// Spark-compatible `size` function. +/// Spark-compatible `size` expression. /// -/// Returns the number of elements in an array or the number of key-value pairs in a map. -/// Returns -1 for null input (Spark behavior). +/// +#[user_doc( + doc_section(label = "Array Functions"), + description = "Returns the number of elements in an array, or the number of entries in a map, as an INT. \ + NULL elements are counted. DataFusion has no built-in `size`. \ + A NULL input returns -1 whatever the ANSI mode setting, so the result is never NULL. \ + This is Spark's legacy behavior (`spark.sql.legacy.sizeOfNull` true and ANSI mode off). \ + Spark with ANSI mode on, the default since Spark 4.0, returns NULL instead.", + syntax_example = "size(expr)", + sql_example = r#"```sql +> SELECT size(array(1, NULL, 3)); +3 +> SELECT size(map(array('a', 'b'), array(1, 2))); +2 +> SELECT size(CAST(NULL AS ARRAY)); +-1 +```"#, + argument( + name = "expr", + description = "Array or map expression. NULL returns -1." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkSize { signature: Signature, @@ -65,6 +87,10 @@ impl ScalarUDFImpl for SparkSize { Ok(Arc::new(Field::new(self.name(), DataType::Int32, false))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_size_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/generator/stack.rs b/crates/spark/src/function/generator/stack.rs index 1b611b59..3254cfc8 100644 --- a/crates/spark/src/function/generator/stack.rs +++ b/crates/spark/src/function/generator/stack.rs @@ -16,35 +16,49 @@ use datafusion_common::{ Result, ScalarValue, internal_err, plan_err, resources_datafusion_err, }; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; +use datafusion_macros::user_doc; /// The most rows a call generates per batch. Spark streams the rows, but they are /// materialized here, so a large `n` such as `stack(2000000000, 1)` fails instead of /// exhausting memory. const MAX_GENERATED_ROWS: usize = 1 << 24; -/// Spark-compatible `stack` expression +/// Spark-compatible `stack` expression. /// /// -/// -/// `stack(n, expr1, ..., exprk)` separates `expr1`, ..., `exprk` into `n` rows of -/// `ceil(k / n)` columns named `col0`, `col1`, ... The values fill the rows in -/// order, and the last rows are padded with NULLs. -/// -/// Note: In Spark, `stack` is a Generator that produces rows. In DataFusion, a -/// ScalarUDF can only return one value per row, so the `n` rows are returned as a -/// `List>`, which is the data type of Spark's Generator. As -/// in Spark, the list is not nullable while its elements and fields are. The -/// caller (e.g. Comet, or `unnest` in SQL) is expected to unnest the list. -/// -/// As in Spark, `n` is an `INT` in `[1, 2147483647]`, and the values of a column -/// must have the same type, which is the type of its first non-NULL value. NULLs, -/// Arrow types of the same Spark type (such as `Utf8` and `Utf8View`), and integer -/// literals that fit an `INT` column are cast to it, wherever they are in the -/// column. Unlike Spark, which accepts any foldable `n`, `n` must be an integer -/// literal, because the return type is planned before expressions are folded. +#[user_doc( + doc_section(label = "Other Functions"), + description = "Separates `expr1`, ..., `exprk` into `n` rows of `ceil(k / n)` columns named `col0`, `col1`, ... \ + The values fill the rows in order, and the last rows are padded with NULLs. DataFusion has no built-in `stack`. \ + In Spark, `stack` is a generator that returns rows. A scalar function returns one value per row, so here the rows \ + are returned as a list of structs, which the caller (e.g. Comet, or `unnest` in SQL) is expected to expand. \ + As in Spark, the list is never NULL, but its elements and fields can be. \ + As in Spark, `n` must be an INT from 1 to 2147483647, and the values of a column must have the same type, \ + which is the type of its first non-NULL value. NULLs, Arrow types of the same Spark type (such as `Utf8` and `Utf8View`), \ + and integer literals that fit an INT column are cast to it. \ + Unlike Spark, which accepts any constant `n` (such as `3 - 1`), `n` must be an integer literal, because the \ + return type is planned before expressions are folded. The rows are materialized, so a very large `n`, \ + such as `stack(2000000000, 1)`, is an error.", + syntax_example = "stack(n, expr1[, ..., exprk])", + sql_example = r#"```sql +> SELECT stack(2, 1, 2, 3); +[{col0: 1, col1: 2}, {col0: 3, col1: NULL}] +> SELECT unnest(stack(2, 'a', 1, 'b', 2)); +{col0: a, col1: 1} +{col0: b, col1: 2} +```"#, + argument( + name = "n", + description = "Integer literal number of rows, from 1 to 2147483647." + ), + argument( + name = "expr", + description = "One or more values to separate into rows. The values of each column must have the same type." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkStack { signature: Signature, @@ -115,6 +129,10 @@ impl ScalarUDFImpl for SparkStack { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let ScalarFunctionArgs { args, diff --git a/crates/spark/src/function/json/json_tuple.rs b/crates/spark/src/function/json/json_tuple.rs index 17be42d2..da4f4436 100644 --- a/crates/spark/src/function/json/json_tuple.rs +++ b/crates/spark/src/function/json/json_tuple.rs @@ -10,26 +10,39 @@ use arrow::datatypes::{DataType, Field, FieldRef, Fields}; use datafusion_common::cast::as_string_array; use datafusion_common::{Result, exec_err, internal_err}; use datafusion_expr::{ - ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature, - Volatility, + ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, Volatility, }; +use datafusion_macros::user_doc; -/// Spark-compatible `json_tuple` expression +/// Spark-compatible `json_tuple` expression. /// /// -/// -/// Extracts top-level fields from a JSON string and returns them as a struct. -/// -/// `json_tuple(json_string, field1, field2, ...) -> Struct` -/// -/// Note: In Spark, `json_tuple` is a Generator that produces multiple columns directly. -/// In DataFusion, a ScalarUDF can only return one value per row, so the result is wrapped -/// in a Struct. The caller (e.g. Comet) is expected to destructure the struct fields. -/// -/// - Returns NULL for each field that is missing from the JSON object -/// - Returns NULL for all fields if the input is NULL or not valid JSON -/// - Non-string JSON values are converted to their JSON string representation -/// - JSON `null` values are returned as NULL (not the string "null") +#[user_doc( + doc_section(label = "JSON Functions"), + description = "Returns the values of the given top-level fields of a JSON object, as a struct with one string \ + field per key, named `c0`, `c1`, ... DataFusion has no built-in `json_tuple`. \ + In Spark, `json_tuple` is a generator that returns one column per key. A scalar function returns one value \ + per row, so here the columns are wrapped in a struct, which the caller (e.g. Comet) is expected to unpack. \ + Keys are case-sensitive. A missing field, a JSON `null` value or a NULL key gives a NULL field, \ + and non-string values such as numbers are returned as their JSON text. \ + Returns NULL if the input is NULL, is not valid JSON, or is not a JSON object.", + syntax_example = "json_tuple(json_str, key1[, key2, ...])", + sql_example = r#"```sql +> SELECT json_tuple('{"a":1, "b":"x"}', 'a', 'b', 'c'); +{c0: 1, c1: x, c2: NULL} +> SELECT json_tuple('[1, 2]', 'a'); +NULL +```"#, + argument( + name = "json_str", + description = "String containing a JSON object. NULL, invalid JSON or JSON that is not an object returns NULL." + ), + argument( + name = "key", + description = "One or more string keys of top-level fields to return. A NULL key gives a NULL field." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct JsonTuple { signature: Signature, @@ -83,6 +96,10 @@ impl ScalarUDFImpl for JsonTuple { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let ScalarFunctionArgs { args: arg_values, From 6a0237be812173a0100774842678a79e105488c3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:52:00 -0700 Subject: [PATCH 2/2] docs(spark): note avg and collect_list gaps with issue links avg returns DOUBLE for DECIMAL input where Spark returns DECIMAL (#111), and collect_list(x ORDER BY ...) fails with an internal error (#112). Add sqllogictest cases that pin the current behavior. Co-Authored-By: Claude Opus 5.5 --- crates/spark/src/function/aggregate/avg.rs | 2 +- crates/spark/src/function/aggregate/collect.rs | 3 ++- crates/sqllogictest/test_files/aggregate/avg.slt | 6 ++++++ crates/sqllogictest/test_files/aggregate/collect.slt | 4 ++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/spark/src/function/aggregate/avg.rs b/crates/spark/src/function/aggregate/avg.rs index 2c0b5d14..752f9c04 100644 --- a/crates/spark/src/function/aggregate/avg.rs +++ b/crates/spark/src/function/aggregate/avg.rs @@ -35,7 +35,7 @@ use std::sync::Arc; doc_section(label = "General Functions"), description = "Returns the average of the non-NULL values as a DOUBLE, or NULL if there are none. \ All numeric input is cast to DOUBLE first. So unlike DataFusion's `avg` and Spark's, \ - DECIMAL input returns a DOUBLE rather than a DECIMAL.", + DECIMAL input returns a DOUBLE rather than a DECIMAL ([#111](https://github.com/apache/datafusion-spark/issues/111)).", syntax_example = "avg(expr)", sql_example = r#"```sql > SELECT avg(a) FROM (VALUES (40), (23), (17), (40), (NULL)) AS t(a); diff --git a/crates/spark/src/function/aggregate/collect.rs b/crates/spark/src/function/aggregate/collect.rs index 9ca585c1..1a972988 100644 --- a/crates/spark/src/function/aggregate/collect.rs +++ b/crates/spark/src/function/aggregate/collect.rs @@ -38,7 +38,8 @@ fn empty_list_scalar(list_type: &DataType) -> Result { description = "Returns an array of the non-NULL values, including duplicates. \ Unlike DataFusion's `array_agg`, NULL values are skipped, so a group or window frame that is empty \ or holds only NULLs returns an empty array, where `array_agg` returns NULL or an array of NULLs. \ - The order of the elements depends on the order of the rows, which may not be deterministic.", + The order of the elements depends on the order of the rows, which may not be deterministic. \ + `ORDER BY` inside the call is not supported and fails with an internal error ([#112](https://github.com/apache/datafusion-spark/issues/112)).", syntax_example = "collect_list(expr)", sql_example = r#"```sql > SELECT collect_list(a) FROM (VALUES (1), (2), (NULL), (1)) AS t(a); diff --git a/crates/sqllogictest/test_files/aggregate/avg.slt b/crates/sqllogictest/test_files/aggregate/avg.slt index a3bc3a1f..8fdd68ef 100644 --- a/crates/sqllogictest/test_files/aggregate/avg.slt +++ b/crates/sqllogictest/test_files/aggregate/avg.slt @@ -51,3 +51,9 @@ ORDER BY g; ---- 0 36.666666666666664 1 20 + +# DECIMAL input is cast to DOUBLE. Spark returns 1.500000 as DECIMAL(14, 6) (https://github.com/apache/datafusion-spark/issues/111) +query TR +SELECT arrow_typeof(avg(a)), avg(a) FROM (VALUES (1.00::DECIMAL(10, 2)), (2.00::DECIMAL(10, 2))) AS t(a); +---- +Float64 1.5 diff --git a/crates/sqllogictest/test_files/aggregate/collect.slt b/crates/sqllogictest/test_files/aggregate/collect.slt index 97bdd235..fad47f9e 100644 --- a/crates/sqllogictest/test_files/aggregate/collect.slt +++ b/crates/sqllogictest/test_files/aggregate/collect.slt @@ -109,3 +109,7 @@ ORDER BY g; 1 [5] 2 [10, 20] 3 [] + +# ORDER BY inside the call is not supported (https://github.com/apache/datafusion-spark/issues/112) +query error expects single batch +SELECT collect_list(a ORDER BY a DESC) FROM (VALUES (3), (1), (2)) AS t(a);