Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 112 additions & 0 deletions crates/spark/src/function/math/log1p.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

use crate::function::error_utils::unsupported_data_type_exec_err;
use arrow::array::{ArrayRef, AsArray, Float64Array};
use arrow::datatypes::{DataType, Float64Type};
use datafusion_common::utils::take_function_args;
use datafusion_common::{Result, ScalarValue};
use datafusion_expr::{
ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
Volatility,
};
use datafusion_macros::user_doc;
use std::sync::Arc;

/// Spark-compatible `log1p` expression.
///
/// <https://spark.apache.org/docs/latest/api/sql/index.html#log1p>
#[user_doc(
doc_section(label = "Math Functions"),
description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is \
close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.",
syntax_example = "log1p(expr)",
sql_example = r#"```sql
> SELECT log1p(0);
0
> SELECT log1p(-1);
NULL
```"#,
argument(
name = "expr",
description = "Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL."
)
)]
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct SparkLog1p {
signature: Signature,
}

impl Default for SparkLog1p {
fn default() -> Self {
Self::new()
}
}

impl SparkLog1p {
pub fn new() -> Self {
Self {
signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spark's Log1p declares inputTypes = Seq(DoubleType) with ImplicitCastInputTypes. That means it accepts strings (and errors on a malformed one under ANSI), and it rejects BOOLEAN, DATE and BINARY with UNEXPECTED_INPUT_TYPE. Signature::exact(Float64) rejects strings, so log1p('0.5') in this PR's own test fails.

Since #350 we have SparkInputTypes for exactly this case. Could you switch to SparkInputTypes::new(vec![SparkInputType::Double]) and add coerce_types and with_updated_config the way hash/sha2.rs on main does? Then the argument is always Float64, and the two unsupported_data_type_exec_err arms can go.

}
}
}

impl ScalarUDFImpl for SparkLog1p {
fn name(&self) -> &str {
"log1p"
}

fn signature(&self) -> &Signature {
&self.signature
}

fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
Ok(DataType::Float64)
}

fn documentation(&self) -> Option<&Documentation> {
self.doc()
}

fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
let [arg] = take_function_args(self.name(), args.args)?;

match arg {
ColumnarValue::Scalar(ScalarValue::Float64(value)) => {
Ok(ColumnarValue::Scalar(ScalarValue::Float64(
value.and_then(log1p_or_null),
)))
}
ColumnarValue::Array(array) => match array.data_type() {
DataType::Float64 => {
let result: Float64Array = array
.as_primitive::<Float64Type>()
.iter()
.map(|value| value.and_then(log1p_or_null))
.collect();

Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef))
}
other => Err(unsupported_data_type_exec_err(
"log1p",
format!("{}", DataType::Float64).as_str(),
other,
)),
},
other => Err(unsupported_data_type_exec_err(
"log1p",
format!("{}", DataType::Float64).as_str(),
&other.data_type(),
)),
}
}
}

fn log1p_or_null(value: f64) -> Option<f64> {
if value <= -1.0 {
None
} else {
Some(value.ln_1p())
}
}
8 changes: 8 additions & 0 deletions crates/spark/src/function/math/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ pub mod factorial;
pub mod floor;
pub mod hex;
pub mod hypot;
pub mod log1p;
pub mod modulus;
pub mod negative;
pub mod positive;
Expand Down Expand Up @@ -51,6 +52,7 @@ make_udf_function!(negative::SparkNegative, negative);
make_udf_function!(bin::SparkBin, bin);
make_udf_function!(bround::SparkBRound, bround);
make_udf_function!(divide::SparkDivide, spark_divide);
make_udf_function!(log1p::SparkLog1p, log1p);

pub mod expr_fn {
use datafusion_functions::export_functions;
Expand Down Expand Up @@ -104,6 +106,11 @@ pub mod expr_fn {
"Rounds the value of expr to scale decimal places using HALF_EVEN rounding mode.",
arg1 arg2
));
export_functions!((
log1p,
"Returns the natural logarithm of 1 plus the given value, computed accurately for values near zero. Returns NULL when the input is less than or equal to -1.",
arg1
));
}

pub fn functions() -> Vec<Arc<ScalarUDF>> {
Expand All @@ -130,5 +137,6 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
bin(),
bround(),
spark_divide(),
log1p(),
]
}
111 changes: 101 additions & 10 deletions crates/sqllogictest/test_files/math/log1p.slt
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,104 @@
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

# This file was originally created by a porting script from:
# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function
# This file is part of the implementation of the datafusion-spark function library.
# For more information, please see:
# https://github.com/apache/datafusion/issues/15914

## Original Query: SELECT log1p(0);
## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'}
#query
#SELECT log1p(0::int);
# Basic values and a negative input within the domain
query R
SELECT log1p(0.5::DOUBLE);
----
0.4054651081081644

query R
SELECT log1p(1.0::DOUBLE);
----
0.6931471805599453

query R
SELECT log1p(-0.5::DOUBLE);
----
-0.6931471805599453

# A value immediately greater than the boundary -1
query R
SELECT log1p(-0.9999999999999999::DOUBLE);
----
-36.7368005696771

# The boundary and values outside the domain return NULL
query R
SELECT log1p(-1.0::DOUBLE);
----
NULL

query R
SELECT log1p(-2.0::DOUBLE);
----
NULL

query R
SELECT log1p(CAST('-Infinity' AS DOUBLE));
----
NULL

# NULL input returns NULL
query R
SELECT log1p(CAST(NULL AS DOUBLE));
----
NULL

# Evaluate multiple values in a column
query R
SELECT log1p(a)
FROM (VALUES
(0.0::DOUBLE),
(0.5::DOUBLE),
(-0.5::DOUBLE),
(-1.0::DOUBLE)
) AS t(a);
----
0
0.4054651081081644
-0.6931471805599453
NULL

# Small inputs retain precision near zero
query T
SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE;
----
true

query T
SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE;
----
true

# NaN and positive infinity are preserved
query R
SELECT log1p(CAST('NaN' AS DOUBLE));
----
NaN

query R
SELECT log1p(CAST('Infinity' AS DOUBLE));
----
Infinity

# Integer, float, decimal, and string inputs
query R
SELECT log1p(CAST(1 AS INT));
----
0.6931471805599453

query R
SELECT log1p(CAST(0.5 AS FLOAT));
----
0.4054651081081644

query R
SELECT log1p(CAST(0.5 AS DECIMAL(10, 2)));
----
0.4054651081081644

query R
SELECT log1p('0.5');
----
0.4054651081081644

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add a few cases for the inputs Spark rejects, like log1p(true), log1p(DATE '2020-01-01') and log1p(X'01'), as query error? A malformed string like log1p('abc') would also be useful. It returns NULL with ANSI off and raises CAST_INVALID_INPUT with ANSI on, so it needs the ANSI wrapper described in test_files/README.md.