Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions crates/spark/src/function/bitmap/bitmap_bit_position.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,37 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark-compatible `bitmap_bit_position` expression
/// Spark-compatible `bitmap_bit_position` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bitmap_bit_position>
#[user_doc(
doc_section(label = "Other Functions"),
description = "Returns the bit position of a value within its bitmap bucket, as a BIGINT from 0 to 32767. \
Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); \
`bitmap_bucket_number` gives the bucket. \
A positive value maps to `(value - 1) % 32768`, and zero or a negative value maps to `-value % 32768`. \
NULL returns NULL. DataFusion has no built-in `bitmap_bit_position`.",
syntax_example = "bitmap_bit_position(expr)",
sql_example = r#"```sql
> SELECT bitmap_bit_position(1);
0
> SELECT bitmap_bit_position(32769);
0
> SELECT bitmap_bit_position(-1);
1
```"#,
argument(
name = "expr",
description = "TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct BitmapBitPosition {
signature: Signature,
Expand Down Expand Up @@ -67,6 +90,10 @@ impl ScalarUDFImpl for BitmapBitPosition {
)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
make_scalar_function(bitmap_bit_position_inner, vec![])(&args.args)
}
Expand Down
34 changes: 31 additions & 3 deletions crates/spark/src/function/bitmap/bitmap_bucket_number.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,38 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark-compatible `bitmap_bucket_number` expression
/// Spark-compatible `bitmap_bucket_number` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bitmap_bucket_number>
#[user_doc(
doc_section(label = "Other Functions"),
description = "Returns the number of the bitmap bucket that a value falls in, as a BIGINT. \
Spark's bitmap functions split integers into buckets of 32768 values (one 4 KiB bitmap each); \
`bitmap_bit_position` gives the position within the bucket. \
A positive value maps to `1 + (value - 1) / 32768`, so 1 to 32768 are bucket 1. \
Zero or a negative value maps to `value / 32768`, rounded toward zero, so -32767 to 0 are bucket 0. \
NULL returns NULL. DataFusion has no built-in `bitmap_bucket_number`.",
syntax_example = "bitmap_bucket_number(expr)",
sql_example = r#"```sql
> SELECT bitmap_bucket_number(1);
1
> SELECT bitmap_bucket_number(65537);
3
> SELECT bitmap_bucket_number(-65536);
-2
```"#,
argument(
name = "expr",
description = "TINYINT, SMALLINT, INT or BIGINT expression. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct BitmapBucketNumber {
signature: Signature,
Expand Down Expand Up @@ -67,6 +91,10 @@ impl ScalarUDFImpl for BitmapBucketNumber {
)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
make_scalar_function(bitmap_bucket_number_inner, vec![])(&args.args)
}
Expand Down
30 changes: 27 additions & 3 deletions crates/spark/src/function/bitmap/bitmap_count.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,32 @@ use arrow::datatypes::{DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, EncodingPreservation, ScalarFunctionArgs, ScalarUDFImpl,
Signature, TypeSignatureClass, Volatility,
Coercion, ColumnarValue, Documentation, EncodingPreservation, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
};
use datafusion_functions::downcast_arg;
use datafusion_functions::utils::make_scalar_function;

use datafusion_macros::user_doc;

/// Spark-compatible `bitmap_count` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bitmap_count>
#[user_doc(
doc_section(label = "Other Functions"),
description = "Returns the number of bits that are set in a binary bitmap, as a BIGINT. \
NULL returns NULL. DataFusion has no built-in `bitmap_count`.",
syntax_example = "bitmap_count(bitmap)",
sql_example = r#"```sql
> SELECT bitmap_count(X'1010');
2
> SELECT bitmap_count(X'FFFF');
16
```"#,
argument(
name = "bitmap",
description = "Binary expression, such as a bitmap built by Spark's `bitmap_construct_agg`. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct BitmapCount {
signature: Signature,
Expand Down Expand Up @@ -72,6 +92,10 @@ impl ScalarUDFImpl for BitmapCount {
)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
make_scalar_function(bitmap_count_inner, vec![])(&args.args)
}
Expand Down
35 changes: 32 additions & 3 deletions crates/spark/src/function/bitwise/bit_count.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,36 @@ use arrow::datatypes::{
use datafusion_common::cast::as_boolean_array;
use datafusion_common::{Result, internal_err, plan_err};
use datafusion_expr::{
ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature,
Volatility,
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
TypeSignature, Volatility,
};
use datafusion_functions::utils::make_scalar_function;

use datafusion_macros::user_doc;

/// Spark-compatible `bit_count` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bit_count>
#[user_doc(
doc_section(label = "Math Functions"),
description = "Returns the number of bits that are set in `expr`, as an INT. \
As in Spark, signed integers are sign-extended to 64 bits before counting, \
so a negative value of any signed type counts the extra high bits (`bit_count(-1)` is 64 for TINYINT, SMALLINT, INT and BIGINT). \
Unsigned integers are counted at their own width. A boolean counts as 1 for true and 0 for false. \
NULL returns NULL. DataFusion has no built-in `bit_count`.",
syntax_example = "bit_count(expr)",
sql_example = r#"```sql
> SELECT bit_count(7);
3
> SELECT bit_count(-1::int);
64
> SELECT bit_count(NULL::int);
NULL
```"#,
argument(
name = "expr",
description = "Boolean or integer expression (signed or unsigned, 8 to 64 bits). NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkBitCount {
signature: Signature,
Expand Down Expand Up @@ -75,6 +100,10 @@ impl ScalarUDFImpl for SparkBitCount {
)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
if args.args.len() != 1 {
return plan_err!("bit_count expects exactly 1 argument");
Expand Down
38 changes: 35 additions & 3 deletions crates/spark/src/function/bitwise/bit_get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,39 @@ use datafusion_common::types::{NativeType, logical_int32};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, TypeSignatureClass, Volatility,
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
};
use datafusion_functions::utils::make_scalar_function;

use datafusion_macros::user_doc;

/// Spark-compatible `bit_get` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#bit_get>
#[user_doc(
doc_section(label = "Math Functions"),
description = "Returns the bit (0 or 1) at position `pos` of an integer, as a TINYINT. \
Positions count from 0 at the least significant bit. \
Returns NULL if either input is NULL. \
Errors if `pos` is negative or not less than the bit width of the type of `expr`. \
Integer literals are BIGINT in DataFusion but INT in Spark, so for an uncast literal the limit is 64 here and 32 in Spark. \
DataFusion has no built-in `bit_get`.",
syntax_example = "bit_get(expr, pos)",
alternative_syntax = "getbit(expr, pos)",
sql_example = r#"```sql
> SELECT bit_get(11, 0);
1
> SELECT bit_get(11, 2);
0
> SELECT bit_get(11, NULL);
NULL
```"#,
argument(name = "expr", description = "Integer expression. NULL returns NULL."),
argument(
name = "pos",
description = "Integer bit position, from 0 to the bit width of `expr` minus 1. Any other value is an error. NULL returns NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkBitGet {
signature: Signature,
Expand Down Expand Up @@ -74,6 +102,10 @@ impl ScalarUDFImpl for SparkBitGet {
Ok(Arc::new(Field::new(self.name(), DataType::Int8, nullable)))
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
make_scalar_function(spark_bit_get, vec![])(&args.args)
}
Expand Down
102 changes: 100 additions & 2 deletions crates/spark/src/function/bitwise/bit_shift.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@ use datafusion_common::types::{
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, internal_err};
use datafusion_expr::{
Coercion, ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature, TypeSignature, TypeSignatureClass, Volatility,
Coercion, ColumnarValue, Documentation, ReturnFieldArgs, ScalarFunctionArgs,
ScalarUDFImpl, Signature, TypeSignature, TypeSignatureClass, Volatility,
};
use datafusion_functions::utils::make_scalar_function;
use datafusion_macros::user_doc;

/// Bitwise left shift on elements in `value` by corresponding `shift` amount.
/// The shift amount is normalized to the bit width of the type, matching Spark/Java
Expand Down Expand Up @@ -179,6 +180,12 @@ enum BitShiftType {
RightUnsigned,
}

/// Spark-compatible `shiftleft`, `shiftright` and `shiftrightunsigned` expressions.
///
/// One struct implements all three functions, so each function's documentation is
/// attached to its own marker type below.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#shiftleft>
#[derive(Debug, Hash, Eq, PartialEq)]
pub struct SparkBitShift {
signature: Signature,
Expand Down Expand Up @@ -249,6 +256,89 @@ impl SparkBitShift {
}
}

#[user_doc(
doc_section(label = "Math Functions"),
description = "Shifts the bits of `expr` left by `n` positions. Bits shifted past the top are lost, \
so the result can change sign or wrap to 0. \
As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \
so `shiftleft(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \
TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \
Returns NULL if either input is NULL. DataFusion has no built-in `shiftleft` function.",
syntax_example = "shiftleft(expr, n)",
sql_example = r#"```sql
> SELECT shiftleft(2::int, 1);
4
> SELECT shiftleft(2147483647::int, 1);
-2
> SELECT shiftleft(1::int, 32);
1
```"#,
argument(
name = "expr",
description = "Integer expression to shift. NULL returns NULL."
),
argument(
name = "n",
description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL."
)
)]
struct ShiftLeftDoc;

#[user_doc(
doc_section(label = "Math Functions"),
description = "Shifts the bits of `expr` right by `n` positions, copying the sign bit into the vacated high bits \
(an arithmetic shift, so negative values stay negative). \
As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \
so `shiftright(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \
TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \
Returns NULL if either input is NULL. See `shiftrightunsigned` for a shift that fills with zeros. \
DataFusion has no built-in `shiftright` function.",
syntax_example = "shiftright(expr, n)",
sql_example = r#"```sql
> SELECT shiftright(16::int, 3);
2
> SELECT shiftright(-16::int, 3);
-2
> SELECT shiftright(4::int, -1);
0
```"#,
argument(
name = "expr",
description = "Integer expression to shift. NULL returns NULL."
),
argument(
name = "n",
description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL."
)
)]
struct ShiftRightDoc;

#[user_doc(
doc_section(label = "Math Functions"),
description = "Shifts the bits of `expr` right by `n` positions, filling the vacated high bits with zeros \
(a logical shift, Java's `>>>`), so a negative value becomes a large positive one. \
As in Spark (and Java), the shift amount is taken modulo the bit width of `expr` (32 for INT, 64 for BIGINT), \
so `shiftrightunsigned(1::int, 32)` is 1 and a negative `n` shifts by the bit width plus `n`. \
TINYINT and SMALLINT are widened to INT, and the result has the type of the widened `expr`. \
Returns NULL if either input is NULL. DataFusion has no built-in `shiftrightunsigned` function.",
syntax_example = "shiftrightunsigned(expr, n)",
sql_example = r#"```sql
> SELECT shiftrightunsigned(16::int, 3);
2
> SELECT shiftrightunsigned(-16::int, 3);
536870910
```"#,
argument(
name = "expr",
description = "Integer expression to shift. NULL returns NULL."
),
argument(
name = "n",
description = "Integer number of bit positions, taken modulo the bit width of `expr`. NULL returns NULL."
)
)]
struct ShiftRightUnsignedDoc;

impl ScalarUDFImpl for SparkBitShift {
fn name(&self) -> &str {
self.name
Expand All @@ -268,6 +358,14 @@ impl ScalarUDFImpl for SparkBitShift {
Ok(Arc::new(Field::new(self.name(), data_type, nullable)))
}

fn documentation(&self) -> Option<&Documentation> {
match self.bit_shift_type {
BitShiftType::Left => ShiftLeftDoc.doc(),
BitShiftType::Right => ShiftRightDoc.doc(),
BitShiftType::RightUnsigned => ShiftRightUnsignedDoc.doc(),
}
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let inner = |arr: &[ArrayRef]| -> Result<ArrayRef> {
shift_inner(arr, self.name(), self.bit_shift_type)
Expand Down
Loading
Loading