Repository navigation
feat(spark): implement Spark-compatible log1p #344
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership | ||
| // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements | ||
|
|
||
| use crate::function::error_utils::unsupported_data_type_exec_err; | ||
| use arrow::array::{ArrayRef, AsArray, Float64Array}; | ||
| use arrow::datatypes::{DataType, Float64Type}; | ||
| use datafusion_common::utils::take_function_args; | ||
| use datafusion_common::{Result, ScalarValue}; | ||
| use datafusion_expr::{ | ||
| ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, | ||
| Volatility, | ||
| }; | ||
| use datafusion_macros::user_doc; | ||
| use std::sync::Arc; | ||
|
|
||
| /// Spark-compatible `log1p` expression. | ||
| /// | ||
| /// <https://spark.apache.org/docs/latest/api/sql/index.html#log1p> | ||
| #[user_doc( | ||
| doc_section(label = "Math Functions"), | ||
| description = "Returns `ln(1 + expr)` as a DOUBLE, computed accurately when `expr` is \ | ||
| close to 0. Values less than or equal to -1 return NULL. NULL input returns NULL.", | ||
| syntax_example = "log1p(expr)", | ||
| sql_example = r#"```sql | ||
| > SELECT log1p(0); | ||
| 0 | ||
| > SELECT log1p(-1); | ||
| NULL | ||
| ```"#, | ||
| argument( | ||
| name = "expr", | ||
| description = "Numeric expression evaluated as DOUBLE. Values <= -1 and NULL return NULL." | ||
| ) | ||
| )] | ||
| #[derive(Debug, PartialEq, Eq, Hash)] | ||
| pub struct SparkLog1p { | ||
| signature: Signature, | ||
| } | ||
|
|
||
| impl Default for SparkLog1p { | ||
| fn default() -> Self { | ||
| Self::new() | ||
| } | ||
| } | ||
|
|
||
| impl SparkLog1p { | ||
| pub fn new() -> Self { | ||
| Self { | ||
| signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl ScalarUDFImpl for SparkLog1p { | ||
| fn name(&self) -> &str { | ||
| "log1p" | ||
| } | ||
|
|
||
| fn signature(&self) -> &Signature { | ||
| &self.signature | ||
| } | ||
|
|
||
| fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> { | ||
| Ok(DataType::Float64) | ||
| } | ||
|
|
||
| fn documentation(&self) -> Option<&Documentation> { | ||
| self.doc() | ||
| } | ||
|
|
||
| fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> { | ||
| let [arg] = take_function_args(self.name(), args.args)?; | ||
|
|
||
| match arg { | ||
| ColumnarValue::Scalar(ScalarValue::Float64(value)) => { | ||
| Ok(ColumnarValue::Scalar(ScalarValue::Float64( | ||
| value.and_then(log1p_or_null), | ||
| ))) | ||
| } | ||
| ColumnarValue::Array(array) => match array.data_type() { | ||
| DataType::Float64 => { | ||
| let result: Float64Array = array | ||
| .as_primitive::<Float64Type>() | ||
| .iter() | ||
| .map(|value| value.and_then(log1p_or_null)) | ||
| .collect(); | ||
|
|
||
| Ok(ColumnarValue::Array(Arc::new(result) as ArrayRef)) | ||
| } | ||
| other => Err(unsupported_data_type_exec_err( | ||
| "log1p", | ||
| format!("{}", DataType::Float64).as_str(), | ||
| other, | ||
| )), | ||
| }, | ||
| other => Err(unsupported_data_type_exec_err( | ||
| "log1p", | ||
| format!("{}", DataType::Float64).as_str(), | ||
| &other.data_type(), | ||
| )), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn log1p_or_null(value: f64) -> Option<f64> { | ||
| if value <= -1.0 { | ||
| None | ||
| } else { | ||
| Some(value.ln_1p()) | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,13 +2,104 @@ | |
| # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership | ||
| # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements | ||
|
|
||
| # This file was originally created by a porting script from: | ||
| # https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function | ||
| # This file is part of the implementation of the datafusion-spark function library. | ||
| # For more information, please see: | ||
| # https://github.com/apache/datafusion/issues/15914 | ||
|
|
||
| ## Original Query: SELECT log1p(0); | ||
| ## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'} | ||
| #query | ||
| #SELECT log1p(0::int); | ||
| # Basic values and a negative input within the domain | ||
| query R | ||
| SELECT log1p(0.5::DOUBLE); | ||
| ---- | ||
| 0.4054651081081644 | ||
|
|
||
| query R | ||
| SELECT log1p(1.0::DOUBLE); | ||
| ---- | ||
| 0.6931471805599453 | ||
|
|
||
| query R | ||
| SELECT log1p(-0.5::DOUBLE); | ||
| ---- | ||
| -0.6931471805599453 | ||
|
|
||
| # A value immediately greater than the boundary -1 | ||
| query R | ||
| SELECT log1p(-0.9999999999999999::DOUBLE); | ||
| ---- | ||
| -36.7368005696771 | ||
|
|
||
| # The boundary and values outside the domain return NULL | ||
| query R | ||
| SELECT log1p(-1.0::DOUBLE); | ||
| ---- | ||
| NULL | ||
|
|
||
| query R | ||
| SELECT log1p(-2.0::DOUBLE); | ||
| ---- | ||
| NULL | ||
|
|
||
| query R | ||
| SELECT log1p(CAST('-Infinity' AS DOUBLE)); | ||
| ---- | ||
| NULL | ||
|
|
||
| # NULL input returns NULL | ||
| query R | ||
| SELECT log1p(CAST(NULL AS DOUBLE)); | ||
| ---- | ||
| NULL | ||
|
|
||
| # Evaluate multiple values in a column | ||
| query R | ||
| SELECT log1p(a) | ||
| FROM (VALUES | ||
| (0.0::DOUBLE), | ||
| (0.5::DOUBLE), | ||
| (-0.5::DOUBLE), | ||
| (-1.0::DOUBLE) | ||
| ) AS t(a); | ||
| ---- | ||
| 0 | ||
| 0.4054651081081644 | ||
| -0.6931471805599453 | ||
| NULL | ||
|
|
||
| # Small inputs retain precision near zero | ||
| query T | ||
| SELECT log1p(1e-300::DOUBLE) = 1e-300::DOUBLE; | ||
| ---- | ||
| true | ||
|
|
||
| query T | ||
| SELECT log1p(1e-20::DOUBLE) = 1e-20::DOUBLE; | ||
| ---- | ||
| true | ||
|
|
||
| # NaN and positive infinity are preserved | ||
| query R | ||
| SELECT log1p(CAST('NaN' AS DOUBLE)); | ||
| ---- | ||
| NaN | ||
|
|
||
| query R | ||
| SELECT log1p(CAST('Infinity' AS DOUBLE)); | ||
| ---- | ||
| Infinity | ||
|
|
||
| # Integer, float, decimal, and string inputs | ||
| query R | ||
| SELECT log1p(CAST(1 AS INT)); | ||
| ---- | ||
| 0.6931471805599453 | ||
|
|
||
| query R | ||
| SELECT log1p(CAST(0.5 AS FLOAT)); | ||
| ---- | ||
| 0.4054651081081644 | ||
|
|
||
| query R | ||
| SELECT log1p(CAST(0.5 AS DECIMAL(10, 2))); | ||
| ---- | ||
| 0.4054651081081644 | ||
|
|
||
| query R | ||
| SELECT log1p('0.5'); | ||
| ---- | ||
| 0.4054651081081644 | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you add a few cases for the inputs Spark rejects, like |
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Spark's
Log1pdeclaresinputTypes = Seq(DoubleType)withImplicitCastInputTypes. That means it accepts strings (and errors on a malformed one under ANSI), and it rejects BOOLEAN, DATE and BINARY withUNEXPECTED_INPUT_TYPE.Signature::exact(Float64)rejects strings, solog1p('0.5')in this PR's own test fails.Since #350 we have
SparkInputTypesfor exactly this case. Could you switch toSparkInputTypes::new(vec![SparkInputType::Double])and addcoerce_typesandwith_updated_configthe wayhash/sha2.rsonmaindoes? Then the argument is always Float64, and the twounsupported_data_type_exec_errarms can go.