From 2fb3c2d2aee4cbc96d1716aa18008f2b7e4be7de Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 13 Nov 2026 14:38:32 -0700 Subject: [PATCH] docs(spark): document hash, bitwise, bitmap and conditional functions with #[user_doc] Add #[user_doc] documentation and documentation() to: - hash: sha1, crc32 - bitwise: bit_get, bit_count, bitwise_not, shiftleft, shiftright, shiftrightunsigned - bitmap: bitmap_count, bitmap_bucket_number, bitmap_bit_position - misc: equal_null - conditional: if The three shift functions share one struct, so each gets its own documentation via a private marker type, selected by shift direction. Co-Authored-By: Claude Opus 5.5 --- .../function/bitmap/bitmap_bit_position.rs | 33 +++++- .../function/bitmap/bitmap_bucket_number.rs | 34 +++++- .../spark/src/function/bitmap/bitmap_count.rs | 30 +++++- .../spark/src/function/bitwise/bit_count.rs | 35 +++++- crates/spark/src/function/bitwise/bit_get.rs | 38 ++++++- .../spark/src/function/bitwise/bit_shift.rs | 102 +++++++++++++++++- .../spark/src/function/bitwise/bitwise_not.rs | 29 ++++- crates/spark/src/function/conditional/if.rs | 41 ++++++- crates/spark/src/function/hash/crc32.rs | 30 +++++- crates/spark/src/function/hash/sha1.rs | 28 ++++- crates/spark/src/function/misc/equal_null.rs | 35 +++++- 11 files changed, 409 insertions(+), 26 deletions(-) diff --git a/crates/spark/src/function/bitmap/bitmap_bit_position.rs b/crates/spark/src/function/bitmap/bitmap_bit_position.rs index ffe1a32e..b730f139 100644 --- a/crates/spark/src/function/bitmap/bitmap_bit_position.rs +++ b/crates/spark/src/function/bitmap/bitmap_bit_position.rs @@ -8,14 +8,37 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64 use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, - Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; use std::sync::Arc; -/// Spark-compatible `bitmap_bit_position` expression +/// Spark-compatible `bitmap_bit_position` expression. +/// /// +#[user_doc( + doc_section(label = "Other Functions"), + description = "Returns the bit position of a value within its bitmap bucket, as a BIGINT from 0 to 32767. \ + Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); \ + `bitmap_bucket_number` gives the bucket. \ + A positive value maps to `(value - 1) % 32768`, and zero or a negative value maps to `-value % 32768`. \ + NULL returns NULL. DataFusion has no built-in `bitmap_bit_position`.", + syntax_example = "bitmap_bit_position(expr)", + sql_example = r#"```sql +> SELECT bitmap_bit_position(1); +0 +> SELECT bitmap_bit_position(32769); +0 +> SELECT bitmap_bit_position(-1); +1 +```"#, + argument( + name = "expr", + description = "TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct BitmapBitPosition { signature: Signature, @@ -67,6 +90,10 @@ impl ScalarUDFImpl for BitmapBitPosition { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(bitmap_bit_position_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/bitmap/bitmap_bucket_number.rs b/crates/spark/src/function/bitmap/bitmap_bucket_number.rs index 3ea547db..59fabe18 100644 --- a/crates/spark/src/function/bitmap/bitmap_bucket_number.rs +++ b/crates/spark/src/function/bitmap/bitmap_bucket_number.rs @@ -8,14 +8,38 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64 use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, - Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; use std::sync::Arc; -/// Spark-compatible `bitmap_bucket_number` expression +/// Spark-compatible `bitmap_bucket_number` expression. +/// /// +#[user_doc( + doc_section(label = "Other Functions"), + description = "Returns the number of the bitmap bucket that a value falls in, as a BIGINT. \ + Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); \ + `bitmap_bit_position` gives the position within the bucket. \ + A positive value maps to `1 + (value - 1) / 32768`, so 1 to 32768 are bucket 1. \ + Zero or a negative value maps to `value / 32768`, rounded toward zero, so -32767 to 0 are bucket 0. \ + NULL returns NULL. DataFusion has no built-in `bitmap_bucket_number`.", + syntax_example = "bitmap_bucket_number(expr)", + sql_example = r#"```sql +> SELECT bitmap_bucket_number(1); +1 +> SELECT bitmap_bucket_number(65537); +3 +> SELECT bitmap_bucket_number(-65536); +-2 +```"#, + argument( + name = "expr", + description = "TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct BitmapBucketNumber { signature: Signature, @@ -67,6 +91,10 @@ impl ScalarUDFImpl for BitmapBucketNumber { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(bitmap_bucket_number_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/bitmap/bitmap_count.rs b/crates/spark/src/function/bitmap/bitmap_count.rs index b1b48a71..ff53e2e2 100644 --- a/crates/spark/src/function/bitmap/bitmap_count.rs +++ b/crates/spark/src/function/bitmap/bitmap_count.rs @@ -15,12 +15,32 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64 use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_functions::downcast_arg; use datafusion_functions::utils::make_scalar_function; - +use datafusion_macros::user_doc; + +/// Spark-compatible `bitmap_count` expression. +/// +/// +#[user_doc( + doc_section(label = "Other Functions"), + description = "Returns the number of bits that are set in a binary bitmap, as a BIGINT. \ + NULL returns NULL. DataFusion has no built-in `bitmap_count`.", + syntax_example = "bitmap_count(bitmap)", + sql_example = r#"```sql +> SELECT bitmap_count(X'1010'); +2 +> SELECT bitmap_count(X'FFFF'); +16 +```"#, + argument( + name = "bitmap", + description = "Binary expression, such as a bitmap built by Spark's `bitmap_construct_agg`. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct BitmapCount { signature: Signature, @@ -72,6 +92,10 @@ impl ScalarUDFImpl for BitmapCount { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(bitmap_count_inner, vec![])(&args.args) } diff --git a/crates/spark/src/function/bitwise/bit_count.rs b/crates/spark/src/function/bitwise/bit_count.rs index d6885945..9c63274a 100644 --- a/crates/spark/src/function/bitwise/bit_count.rs +++ b/crates/spark/src/function/bitwise/bit_count.rs @@ -12,11 +12,36 @@ use arrow::datatypes::{ use datafusion_common::cast::as_boolean_array; use datafusion_common::{Result, internal_err, plan_err}; use datafusion_expr::{ - ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, - Volatility, + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, Volatility, }; use datafusion_functions::utils::make_scalar_function; - +use datafusion_macros::user_doc; + +/// Spark-compatible `bit_count` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the number of bits that are set in `expr`, as an INT. \ + As in Spark, signed integers are sign-extended to 64 bits before counting, \ + so a negative value of any signed type counts the extra high bits (`bit_count(-1)` is 64 for TINYINT, SMALLINT, INT and BIGINT). \ + Unsigned integers are counted at their own width. A boolean counts as 1 for true and 0 for false. \ + NULL returns NULL. DataFusion has no built-in `bit_count`.", + syntax_example = "bit_count(expr)", + sql_example = r#"```sql +> SELECT bit_count(7); +3 +> SELECT bit_count(-1::int); +64 +> SELECT bit_count(NULL::int); +NULL +```"#, + argument( + name = "expr", + description = "Boolean or integer expression (signed or unsigned, 8 to 64 bits). NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkBitCount { signature: Signature, @@ -75,6 +100,10 @@ impl ScalarUDFImpl for SparkBitCount { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.len() != 1 { return plan_err!("bit_count expects exactly 1 argument"); diff --git a/crates/spark/src/function/bitwise/bit_get.rs b/crates/spark/src/function/bitwise/bit_get.rs index 1e0c93ee..64b6e50a 100644 --- a/crates/spark/src/function/bitwise/bit_get.rs +++ b/crates/spark/src/function/bitwise/bit_get.rs @@ -15,11 +15,39 @@ use datafusion_common::types::{NativeType, logical_int32}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; - +use datafusion_macros::user_doc; + +/// Spark-compatible `bit_get` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the bit (0 or 1) at position `pos` of an integer, as a TINYINT. \ + Positions count from 0 at the least significant bit. \ + Returns NULL if either input is NULL. \ + Errors if `pos` is negative or not less than the bit width of the type of `expr`. \ + Integer literals are BIGINT in DataFusion but INT in Spark, so for an uncast literal the limit is 64 here and 32 in Spark. \ + DataFusion has no built-in `bit_get`.", + syntax_example = "bit_get(expr, pos)", + alternative_syntax = "getbit(expr, pos)", + sql_example = r#"```sql +> SELECT bit_get(11, 0); +1 +> SELECT bit_get(11, 2); +0 +> SELECT bit_get(11, NULL); +NULL +```"#, + argument(name = "expr", description = "Integer expression. NULL returns NULL."), + argument( + name = "pos", + description = "Integer bit position, from 0 to the bit width of `expr` minus 1. Any other value is an error. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkBitGet { signature: Signature, @@ -74,6 +102,10 @@ impl ScalarUDFImpl for SparkBitGet { Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_bit_get, vec![])(&args.args) } diff --git a/crates/spark/src/function/bitwise/bit_shift.rs b/crates/spark/src/function/bitwise/bit_shift.rs index cf56b153..a50d7282 100644 --- a/crates/spark/src/function/bitwise/bit_shift.rs +++ b/crates/spark/src/function/bitwise/bit_shift.rs @@ -17,10 +17,11 @@ use datafusion_common::types::{ use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; /// Bitwise left shift on elements in `value` by corresponding `shift` amount. /// The shift amount is normalized to the bit width of the type, matching Spark/Java @@ -179,6 +180,12 @@ enum BitShiftType { RightUnsigned, } +/// Spark-compatible `shiftleft`, `shiftright` and `shiftrightunsigned` expressions. +/// +/// One struct implements all three functions, so each function's documentation is +/// attached to its own marker type below. +/// +/// #[derive(Debug, Hash, Eq, PartialEq)] pub struct SparkBitShift { signature: Signature, @@ -249,6 +256,89 @@ impl SparkBitShift { } } +#[user_doc( + doc_section(label = "Math Functions"), + description = "Shifts the bits of `expr` left by `n` positions. Bits shifted past the top are lost, \ + so the result can change sign or wrap to 0. \ + As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \ + so `shiftleft(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \ + TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \ + Returns NULL if either input is NULL. DataFusion has no built-in `shiftleft` function.", + syntax_example = "shiftleft(expr, n)", + sql_example = r#"```sql +> SELECT shiftleft(2::int, 1); +4 +> SELECT shiftleft(2147483647::int, 1); +-2 +> SELECT shiftleft(1::int, 32); +1 +```"#, + argument( + name = "expr", + description = "Integer expression to shift. NULL returns NULL." + ), + argument( + name = "n", + description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL." + ) +)] +struct ShiftLeftDoc; + +#[user_doc( + doc_section(label = "Math Functions"), + description = "Shifts the bits of `expr` right by `n` positions, copying the sign bit into the vacated high bits \ + (an arithmetic shift, so negative values stay negative). \ + As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \ + so `shiftright(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \ + TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \ + Returns NULL if either input is NULL. See `shiftrightunsigned` for a shift that fills with zeros. \ + DataFusion has no built-in `shiftright` function.", + syntax_example = "shiftright(expr, n)", + sql_example = r#"```sql +> SELECT shiftright(16::int, 3); +2 +> SELECT shiftright(-16::int, 3); +-2 +> SELECT shiftright(4::int, -1); +0 +```"#, + argument( + name = "expr", + description = "Integer expression to shift. NULL returns NULL." + ), + argument( + name = "n", + description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL." + ) +)] +struct ShiftRightDoc; + +#[user_doc( + doc_section(label = "Math Functions"), + description = "Shifts the bits of `expr` right by `n` positions, filling the vacated high bits with zeros \ + (a logical shift, Java's `>>>`), so a negative value becomes a large positive one. \ + As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \ + so `shiftrightunsigned(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \ + TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \ + Returns NULL if either input is NULL. DataFusion has no built-in `shiftrightunsigned` function.", + syntax_example = "shiftrightunsigned(expr, n)", + sql_example = r#"```sql +> SELECT shiftrightunsigned(16::int, 3); +2 +> SELECT shiftrightunsigned(-16::int, 3); +536870910 +```"#, + argument( + name = "expr", + description = "Integer expression to shift. NULL returns NULL." + ), + argument( + name = "n", + description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL." + ) +)] +struct ShiftRightUnsignedDoc; + impl ScalarUDFImpl for SparkBitShift { fn name(&self) -> &str { self.name @@ -268,6 +358,14 @@ impl ScalarUDFImpl for SparkBitShift { Ok(Arc::new(Field::new(self.name(), data_type, nullable))) } + fn documentation(&self) -> Option<&Documentation> { + match self.bit_shift_type { + BitShiftType::Left => ShiftLeftDoc.doc(), + BitShiftType::Right => ShiftRightDoc.doc(), + BitShiftType::RightUnsigned => ShiftRightUnsignedDoc.doc(), + } + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let inner = |arr: &[ArrayRef]| -> Result { shift_inner(arr, self.name(), self.bit_shift_type) diff --git a/crates/spark/src/function/bitwise/bitwise_not.rs b/crates/spark/src/function/bitwise/bitwise_not.rs index 0637698c..758fc321 100644 --- a/crates/spark/src/function/bitwise/bitwise_not.rs +++ b/crates/spark/src/function/bitwise/bitwise_not.rs @@ -8,11 +8,34 @@ use arrow::datatypes::{ DataType, Field, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, }; use datafusion_common::{Result, internal_err, plan_err}; -use datafusion_expr::{ColumnarValue, TypeSignature, Volatility}; +use datafusion_expr::{ColumnarValue, Documentation, TypeSignature, Volatility}; use datafusion_expr::{ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, Signature}; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; use std::sync::Arc; +/// Spark-compatible `bitwise_not` expression (the `~` operator in Spark SQL). +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the bitwise NOT of a signed integer, with every bit flipped, in the same type as the input. \ + In two's complement this is `-expr - 1`, and it never overflows (`bitwise_not` of the maximum value is the minimum value). \ + NULL returns NULL. Spark SQL spells this `~expr`. DataFusion has no built-in `bitwise_not` function.", + syntax_example = "bitwise_not(expr)", + sql_example = r#"```sql +> SELECT bitwise_not(0::int); +-1 +> SELECT bitwise_not(-3::int); +2 +> SELECT bitwise_not(2147483647::int); +-2147483648 +```"#, + argument( + name = "expr", + description = "TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkBitwiseNot { signature: Signature, @@ -63,6 +86,10 @@ impl ScalarUDFImpl for SparkBitwiseNot { ))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { if args.args.len() != 1 { return plan_err!("bitwise_not expects exactly 1 argument"); diff --git a/crates/spark/src/function/conditional/if.rs b/crates/spark/src/function/conditional/if.rs index 73e630e0..6caaad9a 100644 --- a/crates/spark/src/function/conditional/if.rs +++ b/crates/spark/src/function/conditional/if.rs @@ -5,10 +5,43 @@ use arrow::datatypes::DataType; use datafusion_common::{Result, internal_err, plan_err}; use datafusion_expr::{ - ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, - binary::try_type_union_resolution, simplify::ExprSimplifyResult, when, + ColumnarValue, Documentation, Expr, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, binary::try_type_union_resolution, simplify::ExprSimplifyResult, when, }; +use datafusion_macros::user_doc; +/// Spark-compatible `if` expression. +/// +/// +#[user_doc( + doc_section(label = "Conditional Functions"), + description = "Returns `expr1` if `cond` is true, and `expr2` otherwise. A NULL condition counts as false. \ + It is rewritten to `CASE WHEN cond THEN expr1 ELSE expr2 END`, so an error such as division by zero \ + in the branch that isn't taken is not raised. \ + `expr1` and `expr2` are coerced to a common type (`if(true, 10, 20.5)` returns a DOUBLE). \ + It is an error if `cond` is not a boolean or NULL. DataFusion has no built-in `if`.", + syntax_example = "if(cond, expr1, expr2)", + sql_example = r#"```sql +> SELECT if(1 < 2, 'a', 'b'); +a +> SELECT if(NULL, 'yes', 'no'); +no +> SELECT if(true, 1 / 1, 1 / 0); +1 +```"#, + argument( + name = "cond", + description = "Boolean expression. NULL is treated as false." + ), + argument( + name = "expr1", + description = "Expression returned when `cond` is true." + ), + argument( + name = "expr2", + description = "Expression returned when `cond` is false or NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkIf { signature: Signature, @@ -62,6 +95,10 @@ impl ScalarUDFImpl for SparkIf { Ok(arg_types[1].clone()) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { internal_err!("if should have been simplified to case") } diff --git a/crates/spark/src/function/hash/crc32.rs b/crates/spark/src/function/hash/crc32.rs index 6a517505..a6b436d6 100644 --- a/crates/spark/src/function/hash/crc32.rs +++ b/crates/spark/src/function/hash/crc32.rs @@ -15,12 +15,34 @@ use datafusion_common::types::{NativeType, logical_string}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; +/// Spark-compatible `crc32` expression. +/// /// +#[user_doc( + doc_section(label = "Hashing Functions"), + description = "Returns the CRC-32 checksum of the input as a BIGINT. \ + The checksum is an unsigned 32-bit value, so the result is never negative. \ + DataFusion has no built-in `crc32`.", + syntax_example = "crc32(expr)", + sql_example = r#"```sql +> SELECT crc32('Spark'); +1557323817 +> SELECT crc32(''); +0 +> SELECT crc32(NULL); +NULL +```"#, + argument( + name = "expr", + description = "String or binary expression. Strings are checksummed as UTF-8 bytes. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkCrc32 { signature: Signature, @@ -65,6 +87,10 @@ impl ScalarUDFImpl for SparkCrc32 { Ok(Arc::new(Field::new(self.name(), DataType::Int64, nullable))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_crc32, vec![])(&args.args) } diff --git a/crates/spark/src/function/hash/sha1.rs b/crates/spark/src/function/hash/sha1.rs index 9c15a64b..42f44454 100644 --- a/crates/spark/src/function/hash/sha1.rs +++ b/crates/spark/src/function/hash/sha1.rs @@ -15,13 +15,33 @@ use datafusion_common::utils::hex::{HexCase, encode_bytes_into}; use datafusion_common::utils::take_function_args; use datafusion_common::{Result, internal_err}; use datafusion_expr::{ - Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, TypeSignatureClass, Volatility, + Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; use sha1::{Digest, Sha1}; +/// Spark-compatible `sha1` expression. +/// /// +#[user_doc( + doc_section(label = "Hashing Functions"), + description = "Returns the SHA-1 digest of the input as a 40-character lowercase hex string. \ + DataFusion has no built-in `sha1`.", + syntax_example = "sha1(expr)", + alternative_syntax = "sha(expr)", + sql_example = r#"```sql +> SELECT sha1('Spark'); +85f5955f4b27a9a4c2aab6ffe5d7189fc298b92c +> SELECT sha1(NULL); +NULL +```"#, + argument( + name = "expr", + description = "String or binary expression to hash. Strings are hashed as UTF-8 bytes. NULL returns NULL." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkSha1 { signature: Signature, @@ -72,6 +92,10 @@ impl ScalarUDFImpl for SparkSha1 { Ok(Arc::new(Field::new(self.name(), DataType::Utf8, nullable))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { make_scalar_function(spark_sha1, vec![])(&args.args) } diff --git a/crates/spark/src/function/misc/equal_null.rs b/crates/spark/src/function/misc/equal_null.rs index f9053919..cf81c6b1 100644 --- a/crates/spark/src/function/misc/equal_null.rs +++ b/crates/spark/src/function/misc/equal_null.rs @@ -8,12 +8,39 @@ use datafusion_common::{Result, plan_err}; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; use datafusion_expr::type_coercion::binary::comparison_coercion; use datafusion_expr::{ - ColumnarValue, Expr, Operator, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, - Signature, Volatility, binary_expr, + ColumnarValue, Documentation, Expr, Operator, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDFImpl, Signature, Volatility, binary_expr, }; +use datafusion_macros::user_doc; use datafusion_physical_expr_common::datum::apply_cmp; use std::sync::Arc; +/// Spark-compatible `equal_null` expression. +/// +/// +#[user_doc( + doc_section(label = "Other Functions"), + description = "Returns true if the two arguments are equal or both NULL, and false otherwise. Never returns NULL. \ + Same as Spark's `<=>` operator and DataFusion's `IS NOT DISTINCT FROM`, which it is rewritten to. \ + The arguments are coerced to a common type, so `equal_null(1::int, 1.0::double)` is true; \ + it is an error if they can't be compared. \ + For floating-point values NaN equals NaN and 0.0 equals -0.0, including inside arrays and structs. \ + DataFusion has no built-in `equal_null`.", + syntax_example = "equal_null(expr1, expr2)", + sql_example = r#"```sql +> SELECT equal_null(3, 3); +true +> SELECT equal_null(1, NULL); +false +> SELECT equal_null(NULL, NULL); +true +```"#, + argument(name = "expr1", description = "Expression of any comparable type."), + argument( + name = "expr2", + description = "Expression of a type comparable with `expr1`." + ) +)] #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkEqualNull { signature: Signature, @@ -66,6 +93,10 @@ impl ScalarUDFImpl for SparkEqualNull { Ok(Arc::new(Field::new(self.name(), DataType::Boolean, false))) } + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { let [lhs, rhs] = take_function_args(self.name(), args.args)?; apply_cmp(Operator::IsNotDistinctFrom, &lhs, &rhs)