diff --git a/crates/spark/src/function/math/log1p.rs b/crates/spark/src/function/math/log1p.rs new file mode 100644 index 00000000..99898e24 --- /dev/null +++ b/crates/spark/src/function/math/log1p.rs @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +use crate::function::error_utils::unsupported_data_type_exec_err; +use arrow::array::{ArrayRef, AsArray, Float64Array}; +use arrow::datatypes::{DataType, Float64Type}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, ScalarValue}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + Volatility, +}; +use datafusion_macros::user_doc; +use std::sync::Arc; + +/// Spark-compatible `log1p` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is \ + close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.", + syntax_example = "log1p(expr)", + sql_example = r#"```sql +> SELECT log1p(0); +0 +> SELECT log1p(-1); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog1p { + signature: Signature, +} + +impl Default for SparkLog1p { + fn default() -> Self { + Self::new() + } +} + +impl SparkLog1p { + pub fn new() -> Self { + Self { + signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for SparkLog1p { + fn name(&self) -> &str { + "log1p" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let [arg] = take_function_args(self.name(), args.args)?; + + match arg { + ColumnarValue::Scalar(ScalarValue::Float64(value)) => { + Ok(ColumnarValue::Scalar(ScalarValue::Float64( + value.and_then(log1p_or_null), + ))) + } + ColumnarValue::Array(array) => match array.data_type() { + DataType::Float64 => { + let result: Float64Array = array + .as_primitive::() + .iter() + .map(|value| value.and_then(log1p_or_null)) + .collect(); + + Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef)) + } + other => Err(unsupported_data_type_exec_err( + "log1p", + format!("{}", DataType::Float64).as_str(), + other, + )), + }, + other => Err(unsupported_data_type_exec_err( + "log1p", + format!("{}", DataType::Float64).as_str(), + &other.data_type(), + )), + } + } +} + +fn log1p_or_null(value: f64) -> Option { + if value <= -1.0 { + None + } else { + Some(value.ln_1p()) + } +} diff --git a/crates/spark/src/function/math/mod.rs b/crates/spark/src/function/math/mod.rs index 1367a84f..0cf87e99 100644 --- a/crates/spark/src/function/math/mod.rs +++ b/crates/spark/src/function/math/mod.rs @@ -15,6 +15,7 @@ pub mod factorial; pub mod floor; pub mod hex; pub mod hypot; +pub mod log1p; pub mod modulus; pub mod negative; pub mod positive; @@ -51,6 +52,7 @@ make_udf_function!(negative::SparkNegative, negative); make_udf_function!(bin::SparkBin, bin); make_udf_function!(bround::SparkBRound, bround); make_udf_function!(divide::SparkDivide, spark_divide); +make_udf_function!(log1p::SparkLog1p, log1p); pub mod expr_fn { use datafusion_functions::export_functions; @@ -104,6 +106,11 @@ pub mod expr_fn { "Rounds the value of expr to scale decimal places using HALF_EVEN rounding mode.", arg1 arg2 )); + export_functions!(( + log1p, + "Returns the natural logarithm of 1 plus the given value, computed accurately for values near zero. Returns NULL when the input is less than or equal to -1.", + arg1 + )); } pub fn functions() -> Vec> { @@ -130,5 +137,6 @@ pub fn functions() -> Vec> { bin(), bround(), spark_divide(), + log1p(), ] } diff --git a/crates/sqllogictest/test_files/math/log1p.slt b/crates/sqllogictest/test_files/math/log1p.slt index 1bde6500..dbe56b7b 100644 --- a/crates/sqllogictest/test_files/math/log1p.slt +++ b/crates/sqllogictest/test_files/math/log1p.slt @@ -2,13 +2,104 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -# This file was originally created by a porting script from: -# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function -# This file is part of the implementation of the datafusion-spark function library. -# For more information, please see: -# https://github.com/apache/datafusion/issues/15914 - -## Original Query: SELECT log1p(0); -## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'} -#query -#SELECT log1p(0::int); +# Basic values and a negative input within the domain +query R +SELECT log1p(0.5::DOUBLE); +---- +0.4054651081081644 + +query R +SELECT log1p(1.0::DOUBLE); +---- +0.6931471805599453 + +query R +SELECT log1p(-0.5::DOUBLE); +---- +-0.6931471805599453 + +# A value immediately greater than the boundary -1 +query R +SELECT log1p(-0.9999999999999999::DOUBLE); +---- +-36.7368005696771 + +# The boundary and values outside the domain return NULL +query R +SELECT log1p(-1.0::DOUBLE); +---- +NULL + +query R +SELECT log1p(-2.0::DOUBLE); +---- +NULL + +query R +SELECT log1p(CAST('-Infinity' AS DOUBLE)); +---- +NULL + +# NULL input returns NULL +query R +SELECT log1p(CAST(NULL AS DOUBLE)); +---- +NULL + +# Evaluate multiple values in a column +query R +SELECT log1p(a) +FROM (VALUES + (0.0::DOUBLE), + (0.5::DOUBLE), + (-0.5::DOUBLE), + (-1.0::DOUBLE) +) AS t(a); +---- +0 +0.4054651081081644 +-0.6931471805599453 +NULL + +# Small inputs retain precision near zero +query T +SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE; +---- +true + +query T +SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE; +---- +true + +# NaN and positive infinity are preserved +query R +SELECT log1p(CAST('NaN' AS DOUBLE)); +---- +NaN + +query R +SELECT log1p(CAST('Infinity' AS DOUBLE)); +---- +Infinity + +# Integer, float, decimal, and string inputs +query R +SELECT log1p(CAST(1 AS INT)); +---- +0.6931471805599453 + +query R +SELECT log1p(CAST(0.5 AS FLOAT)); +---- +0.4054651081081644 + +query R +SELECT log1p(CAST(0.5 AS DECIMAL(10, 2))); +---- +0.4054651081081644 + +query R +SELECT log1p('0.5'); +---- +0.4054651081081644