From f6dd8dc60176978da54ad80d1457a891228133cc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 06:36:00 -0700 Subject: [PATCH 1/6] feat(spark): add Spark-compatible ln, log, log10, log2 and log1p Spark returns NULL for an input at or below the asymptote (0, or -1 for log1p) where DataFusion's built-ins return -Infinity or NaN, and Spark's one-argument log is the natural logarithm rather than base 10. Spark computes these with java.lang.StrictMath, which is fdlibm. Add a translation of fdlibm 5.3's e_log.c, e_log10.c and s_log1p.c, keeping Sun's notice and adding it to LICENSE.txt, so the results match Spark bit for bit. Rust's f64::ln differs in the last digit for some inputs. --- LICENSE.txt | 12 + crates/spark/src/function/math/fdlibm.rs | 354 ++++++++++++++++++ crates/spark/src/function/math/log.rs | 317 ++++++++++++++++ crates/spark/src/function/math/mod.rs | 33 ++ crates/sqllogictest/test_files/math/ln.slt | 68 +++- crates/sqllogictest/test_files/math/log.slt | 99 ++++- crates/sqllogictest/test_files/math/log10.slt | 48 ++- crates/sqllogictest/test_files/math/log1p.slt | 55 ++- crates/sqllogictest/test_files/math/log2.slt | 48 ++- 9 files changed, 1024 insertions(+), 10 deletions(-) create mode 100644 crates/spark/src/function/math/fdlibm.rs create mode 100644 crates/spark/src/function/math/log.rs diff --git a/LICENSE.txt b/LICENSE.txt index d6456956..7f409929 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -200,3 +200,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes code translated from fdlibm 5.3 (Freely Distributable +LIBM), in crates/spark/src/function/math/fdlibm.rs, under the following terms: + + Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. + + Developed at SunSoft, a Sun Microsystems, Inc. business. + Permission to use, copy, modify, and distribute this + software is freely granted, provided that this notice + is preserved. diff --git a/crates/spark/src/function/math/fdlibm.rs b/crates/spark/src/function/math/fdlibm.rs new file mode 100644 index 00000000..83a99fcb --- /dev/null +++ b/crates/spark/src/function/math/fdlibm.rs @@ -0,0 +1,354 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Logarithms that return the same bits as Java's `StrictMath`. +//! +//! Spark evaluates `ln`, `log`, `log10`, `log2` and `log1p` with +//! `java.lang.StrictMath`, which the Java specification requires to return +//! exactly the results of fdlibm 5.3. Rust's `f64::ln` calls the platform's +//! libm, which differs from fdlibm in the last bit for a few percent of inputs, +//! and differently on each platform. The `libm` crate follows musl, which +//! simplified fdlibm's final steps and so also rounds differently. +//! +//! These functions are a direct translation of fdlibm 5.3's `e_log.c`, +//! `e_log10.c` and `s_log1p.c` from C to Rust, keeping its order of operations. +//! They match `StrictMath` bit for bit on 2 million random inputs, on JDK 17 +//! and JDK 21. Special values (0, negative numbers, NaN and infinities) give +//! the IEEE 754 results. The callers map the inputs that Spark sends to NULL +//! before calling these functions. +//! +//! The original fdlibm source carries this notice, which must be preserved: +//! +//! ```text +//! ==================================================== +//! Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. +//! +//! Developed at SunSoft, a Sun Microsystems, Inc. business. +//! Permission to use, copy, modify, and distribute this +//! software is freely granted, provided that this notice +//! is preserved. +//! ==================================================== +//! ``` + +// fdlibm's constants, as the bit patterns given in its comments. +// `log1p`'s Lp1-Lp7 have the same values as `log`'s Lg1-Lg7. +const LN2_HI: f64 = f64::from_bits(0x3fe62e42_fee00000); +const LN2_LO: f64 = f64::from_bits(0x3dea39ef_35793c76); +const TWO54: f64 = f64::from_bits(0x43500000_00000000); +const LG1: f64 = f64::from_bits(0x3fe55555_55555593); +const LG2: f64 = f64::from_bits(0x3fd99999_9997fa04); +const LG3: f64 = f64::from_bits(0x3fd24924_94229359); +const LG4: f64 = f64::from_bits(0x3fcc71c5_1d8e78af); +const LG5: f64 = f64::from_bits(0x3fc74664_96cb03de); +const LG6: f64 = f64::from_bits(0x3fc39a09_d078c69f); +const LG7: f64 = f64::from_bits(0x3fc2f112_df3e5244); + +/// The high 32 bits of `x`, as fdlibm's `__HI`. +fn hi(x: f64) -> i32 { + (x.to_bits() >> 32) as i32 +} +/// The low 32 bits of `x`, as fdlibm's `__LO`. +fn lo(x: f64) -> u32 { + x.to_bits() as u32 +} +/// `x` with its high 32 bits replaced, as an assignment to fdlibm's `__HI`. +fn with_hi(x: f64, hi: i32) -> f64 { + f64::from_bits(((hi as u32 as u64) << 32) | (x.to_bits() & 0xffff_ffff)) +} + +/// `StrictMath.log`: the natural logarithm (fdlibm `__ieee754_log`). +pub(crate) fn log(mut x: f64) -> f64 { + let mut hx = hi(x); + let lx = lo(x); + let mut k: i32 = 0; + if hx < 0x0010_0000 { + if ((hx & 0x7fff_ffff) as u32 | lx) == 0 { + return f64::NEG_INFINITY; + } + if hx < 0 { + return f64::NAN; + } + k -= 54; + x *= TWO54; + hx = hi(x); + } + if hx >= 0x7ff0_0000 { + return x + x; + } + k += (hx >> 20) - 1023; + hx &= 0x000f_ffff; + let i = (hx + 0x95f64) & 0x10_0000; + x = with_hi(x, hx | (i ^ 0x3ff0_0000)); + k += i >> 20; + let f = x - 1.0; + let dk = k as f64; + if (0x000f_ffff & (2 + hx)) < 3 { + if f == 0.0 { + return if k == 0 { + 0.0 + } else { + dk * LN2_HI + dk * LN2_LO + }; + } + let r = f * f * (0.5 - f64::from_bits(0x3fd55555_55555555) * f); + return if k == 0 { + f - r + } else { + dk * LN2_HI - ((r - dk * LN2_LO) - f) + }; + } + let s = f / (2.0 + f); + let z = s * s; + let mut i = hx - 0x6147a; + let w = z * z; + let j = 0x6b851 - hx; + let t1 = w * (LG2 + w * (LG4 + w * LG6)); + let t2 = z * (LG1 + w * (LG3 + w * (LG5 + w * LG7))); + i |= j; + let r = t2 + t1; + if i > 0 { + let hfsq = 0.5 * f * f; + if k == 0 { + f - (hfsq - s * (hfsq + r)) + } else { + dk * LN2_HI - ((hfsq - (s * (hfsq + r) + dk * LN2_LO)) - f) + } + } else if k == 0 { + f - s * (f - r) + } else { + dk * LN2_HI - ((s * (f - r) - dk * LN2_LO) - f) + } +} + +const IVLN10: f64 = f64::from_bits(0x3fdbcb7b_1526e50e); +const LOG10_2HI: f64 = f64::from_bits(0x3fd34413_509f6000); +const LOG10_2LO: f64 = f64::from_bits(0x3d59fef3_11f12b36); + +/// `StrictMath.log10`: the base 10 logarithm (fdlibm `__ieee754_log10`). +pub(crate) fn log10(mut x: f64) -> f64 { + let mut hx = hi(x); + let lx = lo(x); + let mut k: i32 = 0; + if hx < 0x0010_0000 { + if ((hx & 0x7fff_ffff) as u32 | lx) == 0 { + return f64::NEG_INFINITY; + } + if hx < 0 { + return f64::NAN; + } + k -= 54; + x *= TWO54; + hx = hi(x); + } + if hx >= 0x7ff0_0000 { + return x + x; + } + k += (hx >> 20) - 1023; + let i = ((k as u32) & 0x8000_0000) >> 31; + hx = (hx & 0x000f_ffff) | ((0x3ff - i as i32) << 20); + let y = (k + i as i32) as f64; + x = with_hi(x, hx); + let z = y * LOG10_2LO + IVLN10 * log(x); + z + y * LOG10_2HI +} + +/// `StrictMath.log1p`: `ln(1 + x)`, accurate near 0 (fdlibm `log1p`). +pub(crate) fn log1p(x: f64) -> f64 { + let hx = hi(x); + let ax = hx & 0x7fff_ffff; + let mut k: i32 = 1; + let mut f = 0.0; + let mut hu: i32 = 0; + let mut c = 0.0; + if hx < 0x3FDA_827A { + if ax >= 0x3ff0_0000 { + return if x == -1.0 { + f64::NEG_INFINITY + } else { + f64::NAN + }; + } + if ax < 0x3e20_0000 { + return if TWO54 + x > 0.0 && ax < 0x3c90_0000 { + x + } else { + x - x * x * 0.5 + }; + } + if hx > 0 || hx <= 0xbfd2_bec3_u32 as i32 { + k = 0; + f = x; + hu = 1; + } + } + if hx >= 0x7ff0_0000 { + return x + x; + } + if k != 0 { + let mut u; + if hx < 0x4340_0000 { + u = 1.0 + x; + hu = hi(u); + k = (hu >> 20) - 1023; + c = if k > 0 { 1.0 - (u - x) } else { x - (u - 1.0) }; + c /= u; + } else { + u = x; + hu = hi(u); + k = (hu >> 20) - 1023; + c = 0.0; + } + hu &= 0x000f_ffff; + if hu < 0x6a09e { + u = with_hi(u, hu | 0x3ff0_0000); + } else { + k += 1; + u = with_hi(u, hu | 0x3fe0_0000); + hu = (0x0010_0000 - hu) >> 2; + } + f = u - 1.0; + } + let hfsq = 0.5 * f * f; + let dk = k as f64; + if hu == 0 { + if f == 0.0 { + if k == 0 { + return 0.0; + } + c += dk * LN2_LO; + return dk * LN2_HI + c; + } + let r = hfsq * (1.0 - f64::from_bits(0x3fe55555_55555555) * f); + return if k == 0 { + f - r + } else { + dk * LN2_HI - ((r - (dk * LN2_LO + c)) - f) + }; + } + let s = f / (2.0 + f); + let z = s * s; + let r = + z * (LG1 + z * (LG2 + z * (LG3 + z * (LG4 + z * (LG5 + z * (LG6 + z * LG7)))))); + if k == 0 { + f - (hfsq - s * (hfsq + r)) + } else { + dk * LN2_HI - ((hfsq - (s * (hfsq + r) + (dk * LN2_LO + c))) - f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Inputs with the raw bits of `StrictMath.log`, `log10` and `log1p` on + /// JDK 17. The first two are inputs where Rust's `f64::ln` is 1 ulp off. + const STRICT_MATH: &[(f64, i64, i64, i64)] = &[ + ( + 0.670311110015402, + -4622494417381967792, + -4627944964877657846, + 4602796001956754528, + ), + ( + 0.801786884658205, + -4626244807238519389, + -4631794334810663820, + 4603478468290144808, + ), + ( + 81.0, + 4616633728321258251, + 4611273871592510973, + 4616647543217364325, + ), + ( + 3.0, + 4607626529066517258, + 4602266672337769981, + 4608922133940812271, + ), + ( + 1000.0, + 4619463459452485536, + 4613937818241073152, + 4619464584789817444, + ), + ( + 2.0, + 4604418534313441775, + 4599094494223104511, + 4607626529066517258, + ), + (4.9e-324, -4573612656913714749, -4578976757573532812, 1), + ( + f64::MAX, + 4649454530587146735, + 4644130490496809471, + 4649454530587146735, + ), + ( + 0.5658035575800715, + -4620067142282017771, + -4625292858330671666, + 4601749260853035305, + ), + ( + 806916.0, + 4623846286214522504, + 4618336515890487656, + 4623846286912178276, + ), + ( + 0.4111496273080162, + -4617191235959499470, + -4622746857327404785, + 4599875863624971816, + ), + ( + 1e-20, + -4591694018691028389, + -4597049319638433792, + 4307583784117748259, + ), + ( + 1e-300, + -4574084695234936907, + -4579386764849840128, + 118622047889322841, + ), + ]; + + #[test] + fn matches_strict_math() { + for &(x, ln, lg10, ln1p) in STRICT_MATH { + assert_eq!(log(x).to_bits() as i64, ln, "log({x})"); + assert_eq!(log10(x).to_bits() as i64, lg10, "log10({x})"); + assert_eq!(log1p(x).to_bits() as i64, ln1p, "log1p({x})"); + } + } + + #[test] + fn log1p_negative_inputs() { + assert_eq!(log1p(-0.5), -std::f64::consts::LN_2); + assert_eq!(log1p(-1e-10), -1.00000000005e-10); + } + + #[test] + fn special_values() { + for f in [log, log10] { + assert_eq!(f(0.0), f64::NEG_INFINITY); + assert_eq!(f(-0.0), f64::NEG_INFINITY); + assert!(f(-1.0).is_nan()); + assert!(f(f64::NAN).is_nan()); + assert_eq!(f(f64::INFINITY), f64::INFINITY); + assert_eq!(f(1.0), 0.0); + } + assert_eq!(log1p(-1.0), f64::NEG_INFINITY); + assert!(log1p(-2.0).is_nan()); + assert!(log1p(f64::NAN).is_nan()); + assert_eq!(log1p(f64::INFINITY), f64::INFINITY); + assert_eq!(log1p(0.0), 0.0); + assert_eq!(log1p(-0.0).to_bits(), (-0.0f64).to_bits()); + } +} diff --git a/crates/spark/src/function/math/log.rs b/crates/spark/src/function/math/log.rs new file mode 100644 index 00000000..8abd38ef --- /dev/null +++ b/crates/spark/src/function/math/log.rs @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership +// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements + +//! Spark-compatible logarithm functions: `ln`, `log10`, `log2`, `log1p` and `log`. +//! +//! Spark follows Hive here: an input at or below the function's asymptote +//! (0 for `ln`, `log10`, `log2` and both arguments of `log`, -1 for `log1p`) +//! returns NULL instead of the NaN or -Infinity that IEEE 754 gives. NaN is +//! not below the asymptote, so a NaN input returns NaN. +//! +//! DataFusion's built-ins return -Infinity or NaN for those inputs, and its +//! one-argument `log` is base 10 where Spark's is the natural logarithm. +//! +//! Spark computes the logarithms with `java.lang.StrictMath`, so these use the +//! fdlibm translation in [`super::fdlibm`] to return the same bits. Rust's +//! `f64::ln` differs in the last digit for some inputs. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, AsArray, Float64Array}; +use arrow::datatypes::{DataType, Float64Type}; +use datafusion_common::utils::take_function_args; +use datafusion_common::{Result, exec_err}; +use datafusion_expr::{ + ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, Volatility, +}; +use datafusion_functions::utils::make_scalar_function; +use datafusion_macros::user_doc; + +use super::fdlibm; + +/// Applies `f` to every value of a Float64 array, returning NULL where the +/// value is at or below `asymptote`, as Spark's `UnaryLogExpression` does. +fn unary_log( + name: &str, + args: &[ArrayRef], + asymptote: f64, + f: impl Fn(f64) -> f64, +) -> Result { + let [arg] = take_function_args(name, args)?; + let result: Float64Array = arg + .as_primitive::() + .unary_opt(|x| if x <= asymptote { None } else { Some(f(x)) }); + Ok(Arc::new(result)) +} + +/// Defines a one-argument Spark logarithm UDF over DOUBLE. +macro_rules! unary_log_udf { + ($struct:ident, $name:literal, $asymptote:expr, $f:expr) => { + impl Default for $struct { + fn default() -> Self { + Self::new() + } + } + + impl $struct { + pub fn new() -> Self { + Self { + signature: Signature::exact( + vec![DataType::Float64], + Volatility::Immutable, + ), + } + } + } + + impl ScalarUDFImpl for $struct { + fn name(&self) -> &str { + $name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args( + &self, + args: ScalarFunctionArgs, + ) -> Result { + make_scalar_function( + |args: &[ArrayRef]| unary_log($name, args, $asymptote, $f), + vec![], + )(&args.args) + } + } + }; +} + +/// Spark-compatible `ln` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the natural logarithm (base e) of `expr` as a DOUBLE. The argument is \ + cast to DOUBLE. Unlike DataFusion's built-in `ln`, which returns -Infinity for 0 and NaN for \ + a negative number, an input less than or equal to 0 returns NULL. Returns NULL if the input \ + is NULL, and NaN if it is NaN.", + syntax_example = "ln(expr)", + sql_example = r#"```sql +> SELECT ln(1); +0 +> SELECT ln(0); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLn { + signature: Signature, +} + +unary_log_udf!(SparkLn, "ln", 0.0, fdlibm::log); + +/// Spark-compatible `log10` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the base 10 logarithm of `expr` as a DOUBLE. The argument is cast to \ + DOUBLE. Unlike DataFusion's built-in `log10`, which returns -Infinity for 0 and NaN for a \ + negative number, an input less than or equal to 0 returns NULL. Returns NULL if the input \ + is NULL, and NaN if it is NaN.", + syntax_example = "log10(expr)", + sql_example = r#"```sql +> SELECT log10(10); +1 +> SELECT log10(-1); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog10 { + signature: Signature, +} + +unary_log_udf!(SparkLog10, "log10", 0.0, fdlibm::log10); + +/// Spark-compatible `log2` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the base 2 logarithm of `expr` as a DOUBLE. The argument is cast to \ + DOUBLE. Like Spark, it is computed as `ln(expr) / ln(2)`, so it can differ from an exact \ + base 2 logarithm in the last digit. Unlike DataFusion's built-in `log2`, which returns \ + -Infinity for 0 and NaN for a negative number, an input less than or equal to 0 returns \ + NULL. Returns NULL if the input is NULL, and NaN if it is NaN.", + syntax_example = "log2(expr)", + sql_example = r#"```sql +> SELECT log2(2); +1 +> SELECT log2(0); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog2 { + signature: Signature, +} + +// Spark computes `StrictMath.log(x) / StrictMath.log(2)` rather than a true +// base 2 logarithm, which differs in the last bit for some inputs. +unary_log_udf!(SparkLog2, "log2", 0.0, |x: f64| { + fdlibm::log(x) / fdlibm::log(2.0) +}); + +/// Spark-compatible `log1p` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns `ln(1 + expr)` as a DOUBLE, computed without the precision loss of \ + adding 1 to `expr` when `expr` is close to 0. DataFusion has no built-in `log1p`. The \ + argument is cast to DOUBLE. An input less than or equal to -1 returns NULL. Returns NULL \ + if the input is NULL, and NaN if it is NaN.", + syntax_example = "log1p(expr)", + sql_example = r#"```sql +> SELECT log1p(0); +0 +> SELECT log1p(-1); +NULL +```"#, + argument( + name = "expr", + description = "Numeric expression, cast to DOUBLE. NULL or a number less than or equal to -1 returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog1p { + signature: Signature, +} + +unary_log_udf!(SparkLog1p, "log1p", -1.0, fdlibm::log1p); + +/// Spark-compatible `log` expression. +/// +/// +#[user_doc( + doc_section(label = "Math Functions"), + description = "Returns the logarithm of `expr` in base `base` as a DOUBLE, computed as \ + `ln(expr) / ln(base)`. With one argument it returns the natural logarithm of `expr`, \ + where DataFusion's built-in `log` returns the base 10 logarithm. Both arguments are cast \ + to DOUBLE. Returns NULL if either argument is NULL or less than or equal to 0, otherwise \ + NaN if either is NaN. A base of 1 divides by 0, so it returns Infinity, -Infinity, or NaN \ + when `expr` is also 1.", + syntax_example = "log([base, ]expr)", + sql_example = r#"```sql +> SELECT log(10, 100); +2 +> SELECT log(1); +0 +> SELECT log(0, 10); +NULL +```"#, + argument( + name = "base", + description = "Optional numeric base, cast to DOUBLE. Defaults to e. NULL, 0 or a negative number returns NULL." + ), + argument( + name = "expr", + description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + ) +)] +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkLog { + signature: Signature, +} + +impl Default for SparkLog { + fn default() -> Self { + Self::new() + } +} + +impl SparkLog { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Float64]), + TypeSignature::Exact(vec![DataType::Float64, DataType::Float64]), + ], + Volatility::Immutable, + ), + } + } +} + +impl ScalarUDFImpl for SparkLog { + fn name(&self) -> &str { + "log" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(DataType::Float64) + } + + fn documentation(&self) -> Option<&Documentation> { + self.doc() + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + make_scalar_function(spark_log, vec![])(&args.args) + } +} + +fn spark_log(args: &[ArrayRef]) -> Result { + match args { + // Spark's one-argument `log` is `Logarithm(EulerNumber(), expr)`, + // which its generated code evaluates as `StrictMath.log(expr)`. + [_] => unary_log("log", args, 0.0, fdlibm::log), + [base, value] => { + let base = base.as_primitive::(); + let value = value.as_primitive::(); + let result: Float64Array = base + .iter() + .zip(value.iter()) + .map(|(base, value)| match (base, value) { + // NaN is not <= 0, so a NaN argument yields NaN + // unless the other argument is <= 0. + (Some(base), Some(value)) if base <= 0.0 || value <= 0.0 => None, + (Some(base), Some(value)) => { + Some(fdlibm::log(value) / fdlibm::log(base)) + } + _ => None, + }) + .collect(); + Ok(Arc::new(result)) + } + _ => exec_err!("log expects 1 or 2 arguments, got {}", args.len()), + } +} diff --git a/crates/spark/src/function/math/mod.rs b/crates/spark/src/function/math/mod.rs index 1367a84f..a2e111d5 100644 --- a/crates/spark/src/function/math/mod.rs +++ b/crates/spark/src/function/math/mod.rs @@ -12,9 +12,11 @@ pub mod ceil; pub mod divide; pub mod expm1; pub mod factorial; +mod fdlibm; pub mod floor; pub mod hex; pub mod hypot; +pub mod log; pub mod modulus; pub mod negative; pub mod positive; @@ -37,6 +39,11 @@ make_udf_function!(factorial::SparkFactorial, factorial); make_udf_function!(floor::SparkFloor, floor); make_udf_function!(hex::SparkHex, hex); make_udf_function!(hypot::SparkHypot, hypot); +make_udf_function!(log::SparkLn, ln); +make_udf_function!(log::SparkLog, log); +make_udf_function!(log::SparkLog10, log10); +make_udf_function!(log::SparkLog1p, log1p); +make_udf_function!(log::SparkLog2, log2); make_udf_function!(modulus::SparkMod, modulus); make_udf_function!(modulus::SparkPmod, pmod); make_udf_function!(positive::SparkPositive, positive); @@ -67,6 +74,27 @@ pub mod expr_fn { export_functions!((floor, "Returns floor of expr.", arg1)); export_functions!((hex, "Computes hex value of the given column.", arg1)); export_functions!((hypot, "Returns sqrt(a^2 + b^2) without intermediate overflow or underflow.", arg1 arg2)); + export_functions!(( + ln, + "Returns the natural logarithm of expr, or NULL if expr <= 0.", + arg1 + )); + export_functions!(( + log, + "Returns the logarithm of expr in the given base (default e), or NULL if either argument <= 0.", + args + )); + export_functions!(( + log10, + "Returns the base 10 logarithm of expr, or NULL if expr <= 0.", + arg1 + )); + export_functions!((log1p, "Returns ln(1 + expr), or NULL if expr <= -1.", arg1)); + export_functions!(( + log2, + "Returns the base 2 logarithm of expr, or NULL if expr <= 0.", + arg1 + )); export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2)); export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2)); export_functions!(( @@ -116,6 +144,11 @@ pub fn functions() -> Vec> { floor(), hex(), hypot(), + ln(), + log(), + log10(), + log1p(), + log2(), modulus(), pmod(), pow(), diff --git a/crates/sqllogictest/test_files/math/ln.slt b/crates/sqllogictest/test_files/math/ln.slt index 454c26cd..5b1f0995 100644 --- a/crates/sqllogictest/test_files/math/ln.slt +++ b/crates/sqllogictest/test_files/math/ln.slt @@ -8,7 +8,71 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 + ## Original Query: SELECT ln(1); ## PySpark 3.5.5 Result: {'ln(1)': 0.0, 'typeof(ln(1))': 'double', 'typeof(1)': 'int'} -#query -#SELECT ln(1::int); +query R +SELECT ln(1::int); +---- +0 + +# Result is always DOUBLE +query T +SELECT arrow_typeof(ln(1::int)); +---- +Float64 + +query R +SELECT ln(2::double); +---- +0.6931471805599453 + +query R +SELECT ln(0.5::double); +---- +-0.6931471805599453 + +# Integer, float and decimal inputs are cast to DOUBLE +query RRRR +SELECT ln(10::tinyint) AS a, ln(10::bigint) AS b, ln(10::float) AS c, ln(10::decimal(5, 2)) AS d; +---- +2.302585092994046 2.302585092994046 2.302585092994046 2.302585092994046 + +# Spark returns NULL, not -Infinity or NaN, for an input <= 0 +query RRRR +SELECT ln(0::int) AS a, ln(-0.0::double) AS b, ln(-1::int) AS c, ln('-Infinity'::double) AS d; +---- +NULL NULL NULL NULL + +query R +SELECT ln(NULL::double); +---- +NULL + +# NaN is not <= 0, so NaN returns NaN +query R +SELECT ln('NaN'::double); +---- +NaN + +query RRR +SELECT ln('Infinity'::double), ln(1.7976931348623157e308::double), ln(4.9e-324::double); +---- +Infinity 709.782712893384 -744.4400719213812 + +# Array path, including NULL and non-positive rows +query R +SELECT ln(a) FROM (VALUES (1::double), (0::double), (NULL::double), (-5::double), (2::double)) AS t(a); +---- +0 +NULL +NULL +NULL +0.6931471805599453 + +# Spark uses StrictMath.log (fdlibm), which differs from Rust's f64::ln in the +# last digit for some inputs (f64::ln gives -0.4000133310645469 here) +query R +SELECT ln(0.670311110015402::double); +---- +-0.40001333106454684 diff --git a/crates/sqllogictest/test_files/math/log.slt b/crates/sqllogictest/test_files/math/log.slt index 669f5683..cce753ac 100644 --- a/crates/sqllogictest/test_files/math/log.slt +++ b/crates/sqllogictest/test_files/math/log.slt @@ -8,7 +8,102 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 + ## Original Query: SELECT log(10, 100); ## PySpark 3.5.5 Result: {'LOG(10, 100)': 2.0, 'typeof(LOG(10, 100))': 'double', 'typeof(10)': 'int', 'typeof(100)': 'int'} -#query -#SELECT log(10::int, 100::int); +query R +SELECT log(10::int, 100::int); +---- +2 + +query T +SELECT arrow_typeof(log(10::int, 100::int)); +---- +Float64 + +query RR +SELECT log(2::bigint, 1024::bigint), log(0.5::double, 8::double); +---- +10 -3 + +# Computed as ln(expr) / ln(base), so results can be off in the last digit. +# Rust's f64::ln gives 4 for log(3, 81); StrictMath.log, which Spark uses, +# gives 4.000000000000001. +query RR +SELECT log(10::int, 1000::int), log(3::int, 81::int); +---- +2.9999999999999996 4.000000000000001 + +query R +SELECT log(10::decimal(5, 1), 100::decimal(5, 1)); +---- +2 + +# One argument is the natural logarithm, not base 10 as in DataFusion +query RRR +SELECT log(1::int), log(2::double), log(10::decimal(5, 2)); +---- +0 0.6931471805599453 2.302585092994046 + +query T +SELECT arrow_typeof(log(1::int)); +---- +Float64 + +# NULL if either argument is <= 0 +query RRRRRR +SELECT log(0::int, 10::int), log(-2::int, 10::int), log(2::int, 0::int), log(2::int, -8::int), log(-1::double), log(0::double); +---- +NULL NULL NULL NULL NULL NULL + +# ...even when the other argument is NaN +query RR +SELECT log(0::double, 'NaN'::double), log('NaN'::double, -1::double); +---- +NULL NULL + +query RRRR +SELECT log(NULL::double, 2::double), log(2::double, NULL::double), log(NULL::double), log(NULL::double, NULL::double); +---- +NULL NULL NULL NULL + +query RRR +SELECT log('NaN'::double, 2::double), log(2::double, 'NaN'::double), log('NaN'::double); +---- +NaN NaN NaN + +# A base of 1 divides by ln(1) = 0 +query RRR +SELECT log(1::int, 2::int), log(1::double, 0.5::double), log(1::int, 1::int); +---- +Infinity -Infinity NaN + +query RRR +SELECT log('Infinity'::double, 2::double), log(2::double, 'Infinity'::double), log('Infinity'::double, 'Infinity'::double); +---- +0 Infinity NaN + +# Array path, including NULL and non-positive rows +query R +SELECT log(b, x) FROM (VALUES (10::double, 100::double), (2::double, 0::double), (NULL::double, 3::double), (0::double, 3::double), (2::double, 8::double)) AS t(b, x); +---- +2 +NULL +NULL +NULL +3 + +# Scalar base with an array expr +query R +SELECT log(2::int, x) FROM (VALUES (8::double), (-1::double), (NULL::double)) AS t(x); +---- +3 +NULL +NULL + +query R +SELECT log(x) FROM (VALUES (1::double), (0::double), (NULL::double)) AS t(x); +---- +0 +NULL +NULL diff --git a/crates/sqllogictest/test_files/math/log10.slt b/crates/sqllogictest/test_files/math/log10.slt index d99dfebb..82028eee 100644 --- a/crates/sqllogictest/test_files/math/log10.slt +++ b/crates/sqllogictest/test_files/math/log10.slt @@ -8,7 +8,51 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 + ## Original Query: SELECT log10(10); ## PySpark 3.5.5 Result: {'LOG10(10)': 1.0, 'typeof(LOG10(10))': 'double', 'typeof(10)': 'int'} -#query -#SELECT log10(10::int); +query R +SELECT log10(10::int); +---- +1 + +query T +SELECT arrow_typeof(log10(10::int)); +---- +Float64 + +query RRR +SELECT log10(1000::bigint), log10(0.001::double), log10(2::double); +---- +3 -3 0.3010299956639812 + +query R +SELECT log10(100::decimal(5, 1)); +---- +2 + +# Spark returns NULL, not -Infinity or NaN, for an input <= 0 +query RRR +SELECT log10(0::int), log10(-10::int), log10('-Infinity'::double); +---- +NULL NULL NULL + +query RRR +SELECT log10(NULL::double), log10('NaN'::double), log10('Infinity'::double); +---- +NULL NaN Infinity + +query R +SELECT log10(a) FROM (VALUES (100::double), (0::double), (NULL::double), (-1::double)) AS t(a); +---- +2 +NULL +NULL +NULL + +# Spark uses StrictMath.log10 (fdlibm), which differs from Rust's f64::log10 in +# the last digit for some inputs (f64::log10 gives -0.17372358236907134 here) +query R +SELECT log10(0.670311110015402::double); +---- +-0.17372358236907132 diff --git a/crates/sqllogictest/test_files/math/log1p.slt b/crates/sqllogictest/test_files/math/log1p.slt index 1bde6500..895c20ef 100644 --- a/crates/sqllogictest/test_files/math/log1p.slt +++ b/crates/sqllogictest/test_files/math/log1p.slt @@ -8,7 +8,58 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 + ## Original Query: SELECT log1p(0); ## PySpark 3.5.5 Result: {'LOG1P(0)': 0.0, 'typeof(LOG1P(0))': 'double', 'typeof(0)': 'int'} -#query -#SELECT log1p(0::int); +query R +SELECT log1p(0::int); +---- +0 + +query T +SELECT arrow_typeof(log1p(0::int)); +---- +Float64 + +query RR +SELECT log1p(1::int), log1p(-0.5::double); +---- +0.6931471805599453 -0.6931471805599453 + +# Accurate near 0, where ln(1 + x) would round to 0 +query R +SELECT log1p(1e-20::double); +---- +0.00000000000000000001 + +# Spark returns NULL, not -Infinity or NaN, for an input <= -1 +query RRR +SELECT log1p(-1::int), log1p(-2::int), log1p('-Infinity'::double); +---- +NULL NULL NULL + +query RRR +SELECT log1p(NULL::double), log1p('NaN'::double), log1p('Infinity'::double); +---- +NULL NaN Infinity + +query R +SELECT log1p(a) FROM (VALUES (0::double), (-1::double), (NULL::double), (-0.5::double)) AS t(a); +---- +0 +NULL +NULL +-0.6931471805599453 + +# Spark uses StrictMath.log1p (fdlibm), which differs from Rust's f64::ln_1p in +# the last digit for some inputs (f64::ln_1p gives 0.5130099025005953 here) +query R +SELECT log1p(0.670311110015402::double); +---- +0.5130099025005954 + +# -0.0 is above the asymptote, and log1p(-0.0) is -0.0 +query R +SELECT log1p(-0.0::double); +---- +-0 diff --git a/crates/sqllogictest/test_files/math/log2.slt b/crates/sqllogictest/test_files/math/log2.slt index ef5a343c..c43aacc8 100644 --- a/crates/sqllogictest/test_files/math/log2.slt +++ b/crates/sqllogictest/test_files/math/log2.slt @@ -8,7 +8,51 @@ # For more information, please see: # https://github.com/apache/datafusion/issues/15914 + ## Original Query: SELECT log2(2); ## PySpark 3.5.5 Result: {'LOG2(2)': 1.0, 'typeof(LOG2(2))': 'double', 'typeof(2)': 'int'} -#query -#SELECT log2(2::int); +query R +SELECT log2(2::int); +---- +1 + +query T +SELECT arrow_typeof(log2(2::int)); +---- +Float64 + +query RRRR +SELECT log2(8::int), log2(1024::bigint), log2(0.5::double), log2(3::double); +---- +3 10 -1 1.584962500721156 + +# Spark computes ln(x) / ln(2), not an exact base 2 logarithm, so some +# powers of two are off in the last digit +query RR +SELECT log2(536870912::int), log2(2147483648::bigint); +---- +29.000000000000004 31.000000000000004 + +query R +SELECT log2(4.9e-324::double); +---- +-1074 + +# Spark returns NULL, not -Infinity or NaN, for an input <= 0 +query RRR +SELECT log2(0::int), log2(-2::int), log2('-Infinity'::double); +---- +NULL NULL NULL + +query RRR +SELECT log2(NULL::double), log2('NaN'::double), log2('Infinity'::double); +---- +NULL NaN Infinity + +query R +SELECT log2(a) FROM (VALUES (8::double), (0::double), (NULL::double), (-1::double)) AS t(a); +---- +3 +NULL +NULL +NULL From 69c97d12a0880950de2338816cfa83c05146cf2f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 16 Nov 2026 09:11:38 -0700 Subject: [PATCH 2/6] fix(spark): compute expm1 with a translation of fdlibm's s_expm1.c Spark computes expm1 with java.lang.StrictMath, which is fdlibm. f64::exp_m1 calls the platform's libm, so expm1(1) gave 1.7182818284590453 on macOS where Spark and Linux give 1.718281828459045. Translate fdlibm 5.3's s_expm1.c next to the logarithms. It matches StrictMath.expm1 bit for bit on 1.24 million inputs on JDK 17. --- crates/spark/src/function/math/expm1.rs | 9 +- crates/spark/src/function/math/fdlibm.rs | 144 ++++++++++++++++++++++- 2 files changed, 148 insertions(+), 5 deletions(-) diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index b0c280df..00406069 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -3,6 +3,7 @@ // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements use crate::function::error_utils::unsupported_data_type_exec_err; +use crate::function::math::fdlibm; use arrow::array::{ArrayRef, AsArray}; use arrow::datatypes::{DataType, Float64Type}; use datafusion_common::utils::take_function_args; @@ -17,6 +18,10 @@ use std::sync::Arc; /// Spark-compatible `expm1` expression. /// /// +/// +/// Spark computes this with `java.lang.StrictMath.expm1`, so this uses the fdlibm +/// translation in [`fdlibm`] rather than `f64::exp_m1`, which calls the platform's +/// libm and differs in the last digit on some platforms. #[user_doc( doc_section(label = "Math Functions"), description = "Returns `exp(expr) - 1` as a DOUBLE, computed without the precision loss of \ @@ -74,13 +79,13 @@ impl ScalarUDFImpl for SparkExpm1 { let [arg] = take_function_args(self.name(), args.args)?; match arg { ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok( - ColumnarValue::Scalar(ScalarValue::Float64(value.map(|x| x.exp_m1()))), + ColumnarValue::Scalar(ScalarValue::Float64(value.map(fdlibm::expm1))), ), ColumnarValue::Array(array) => match array.data_type() { DataType::Float64 => Ok(ColumnarValue::Array(Arc::new( array .as_primitive::() - .unary::<_, Float64Type>(|x| x.exp_m1()), + .unary::<_, Float64Type>(fdlibm::expm1), ) as ArrayRef)), other => Err(unsupported_data_type_exec_err( diff --git a/crates/spark/src/function/math/fdlibm.rs b/crates/spark/src/function/math/fdlibm.rs index 83a99fcb..d66efb65 100644 --- a/crates/spark/src/function/math/fdlibm.rs +++ b/crates/spark/src/function/math/fdlibm.rs @@ -2,9 +2,9 @@ // SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership // SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements -//! Logarithms that return the same bits as Java's `StrictMath`. +//! Logarithms and `expm1` that return the same bits as Java's `StrictMath`. //! -//! Spark evaluates `ln`, `log`, `log10`, `log2` and `log1p` with +//! Spark evaluates `ln`, `log`, `log10`, `log2`, `log1p` and `expm1` with //! `java.lang.StrictMath`, which the Java specification requires to return //! exactly the results of fdlibm 5.3. Rust's `f64::ln` calls the platform's //! libm, which differs from fdlibm in the last bit for a few percent of inputs, @@ -12,7 +12,8 @@ //! simplified fdlibm's final steps and so also rounds differently. //! //! These functions are a direct translation of fdlibm 5.3's `e_log.c`, -//! `e_log10.c` and `s_log1p.c` from C to Rust, keeping its order of operations. +//! `e_log10.c`, `s_log1p.c` and `s_expm1.c` from C to Rust, keeping its order +//! of operations. //! They match `StrictMath` bit for bit on 2 million random inputs, on JDK 17 //! and JDK 21. Special values (0, negative numbers, NaN and infinities) give //! the IEEE 754 results. The callers map the inputs that Spark sends to NULL @@ -237,6 +238,105 @@ pub(crate) fn log1p(x: f64) -> f64 { } } +const O_THRESHOLD: f64 = f64::from_bits(0x40862e42_fefa39ef); +const INVLN2: f64 = f64::from_bits(0x3ff71547_652b82fe); +const HUGE: f64 = 1.0e300; +const TINY: f64 = 1.0e-300; +const Q1: f64 = f64::from_bits(0xbfa11111_111110f4); +const Q2: f64 = f64::from_bits(0x3f5a01a0_19fe5585); +const Q3: f64 = f64::from_bits(0xbf14ce19_9eaadbb7); +const Q4: f64 = f64::from_bits(0x3ed0cfca_86e65239); +const Q5: f64 = f64::from_bits(0xbe8afdb7_6e09c32d); + +/// `StrictMath.expm1`: `exp(x) - 1`, accurate near 0 (fdlibm `expm1`). +pub(crate) fn expm1(mut x: f64) -> f64 { + let mut hx = hi(x) as u32; + let negative = hx & 0x8000_0000 != 0; + hx &= 0x7fff_ffff; + + // Huge and non-finite arguments. + if hx >= 0x4043_687a { + if hx >= 0x4086_2e42 { + if hx >= 0x7ff0_0000 { + if ((hx & 0x000f_ffff) | lo(x)) != 0 { + return x + x; + } + return if negative { -1.0 } else { x }; + } + if x > O_THRESHOLD { + return HUGE * HUGE; + } + } + if negative && x + TINY < 0.0 { + return TINY - 1.0; + } + } + + // Argument reduction. + let k: i32; + let mut c = 0.0; + if hx > 0x3fd6_2e42 { + let (hi_part, lo_part); + if hx < 0x3ff0_a2b2 { + if negative { + hi_part = x + LN2_HI; + lo_part = -LN2_LO; + k = -1; + } else { + hi_part = x - LN2_HI; + lo_part = LN2_LO; + k = 1; + } + } else { + k = (INVLN2 * x + if negative { -0.5 } else { 0.5 }) as i32; + let t = k as f64; + hi_part = x - t * LN2_HI; + lo_part = t * LN2_LO; + } + x = hi_part - lo_part; + c = (hi_part - x) - lo_part; + } else if hx < 0x3c90_0000 { + let t = HUGE + x; + return x - (t - (HUGE + x)); + } else { + k = 0; + } + + // x is now in the primary range. + let hfx = 0.5 * x; + let hxs = x * hfx; + let r1 = 1.0 + hxs * (Q1 + hxs * (Q2 + hxs * (Q3 + hxs * (Q4 + hxs * Q5)))); + let t = 3.0 - r1 * hfx; + let mut e = hxs * ((r1 - t) / (6.0 - x * t)); + if k == 0 { + return x - (x * e - hxs); + } + e = x * (e - c) - c; + e -= hxs; + if k == -1 { + return 0.5 * (x - e) - 0.5; + } + if k == 1 { + return if x < -0.25 { + -2.0 * (e - (x + 0.5)) + } else { + 1.0 + 2.0 * (x - e) + }; + } + if k <= -2 || k > 56 { + let y = 1.0 - (e - x); + return with_hi(y, hi(y).wrapping_add(k << 20)) - 1.0; + } + let y = if k < 20 { + let t = with_hi(1.0, 0x3ff0_0000 - (0x20_0000 >> k)); + t - (e - x) + } else { + let t = with_hi(1.0, (0x3ff - k) << 20); + (x - (e + t)) + 1.0 + }; + with_hi(y, hi(y).wrapping_add(k << 20)) +} + #[cfg(test)] mod tests { use super::*; @@ -351,4 +451,42 @@ mod tests { assert_eq!(log1p(0.0), 0.0); assert_eq!(log1p(-0.0).to_bits(), (-0.0f64).to_bits()); } + + /// Inputs with the raw bits of `StrictMath.expm1` on JDK 17, covering each + /// branch of the argument reduction. macOS's `f64::exp_m1(1.0)` is 1 ulp off. + const STRICT_MATH_EXPM1: &[(f64, i64)] = &[ + (1.0, 4610417272575012562), + (-1.0, -4619503181483112348), + (1e-10, 4457293557087970531), + (0.3, 4599974115899632214), + (-0.3, -4625031413077833738), + (0.6, 4605580207391462366), + (2.5, 4622484802509070523), + (-2.5, -4616928973993188996), + (15.0, 4704274570552355463), + (30.0, 4801805078135318253), + (50.0, 4931882512844065605), + (100.0, 5256625774849891317), + (-40.0, -4616189618054758400), + (709.0, 9213593174447348891), + (1e-300, 118622047889322841), + ]; + + #[test] + fn expm1_matches_strict_math() { + for &(x, bits) in STRICT_MATH_EXPM1 { + assert_eq!(expm1(x).to_bits() as i64, bits, "expm1({x})"); + } + } + + #[test] + fn expm1_special_values() { + assert_eq!(expm1(0.0).to_bits(), 0.0f64.to_bits()); + assert_eq!(expm1(-0.0).to_bits(), (-0.0f64).to_bits()); + assert!(expm1(f64::NAN).is_nan()); + assert_eq!(expm1(f64::INFINITY), f64::INFINITY); + assert_eq!(expm1(f64::NEG_INFINITY), -1.0); + assert_eq!(expm1(710.0), f64::INFINITY); + assert_eq!(expm1(-800.0), -1.0); + } } From 917ac6846685a30589436c9ff8f792a595024778 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:02:41 -0700 Subject: [PATCH 3/6] fix(spark): accept string arguments in the log functions and error on non-Float64 arrays Spark implicitly casts numeric and string arguments to DOUBLE. Return an error instead of panicking when a caller passes a non-Float64 array, divide log2 by LN_2, take the logarithm of a scalar base once, and fix the -0.0 literals that Spark parses as DECIMAL 0.0. --- crates/spark/src/function/math/fdlibm.rs | 6 + crates/spark/src/function/math/log.rs | 192 +++++++++++++++--- crates/sqllogictest/test_files/math/ln.slt | 25 ++- crates/sqllogictest/test_files/math/log.slt | 20 ++ crates/sqllogictest/test_files/math/log1p.slt | 2 +- 5 files changed, 213 insertions(+), 32 deletions(-) diff --git a/crates/spark/src/function/math/fdlibm.rs b/crates/spark/src/function/math/fdlibm.rs index d66efb65..118e8434 100644 --- a/crates/spark/src/function/math/fdlibm.rs +++ b/crates/spark/src/function/math/fdlibm.rs @@ -428,6 +428,12 @@ mod tests { } } + /// `log2` divides by `std::f64::consts::LN_2` instead of `log(2.0)`. + #[test] + fn log_of_two_is_ln_2() { + assert_eq!(log(2.0), std::f64::consts::LN_2); + } + #[test] fn log1p_negative_inputs() { assert_eq!(log1p(-0.5), -std::f64::consts::LN_2); diff --git a/crates/spark/src/function/math/log.rs b/crates/spark/src/function/math/log.rs index 8abd38ef..a9357956 100644 --- a/crates/spark/src/function/math/log.rs +++ b/crates/spark/src/function/math/log.rs @@ -15,21 +15,55 @@ //! Spark computes the logarithms with `java.lang.StrictMath`, so these use the //! fdlibm translation in [`super::fdlibm`] to return the same bits. Rust's //! `f64::ln` differs in the last digit for some inputs. +//! +//! Spark implicitly casts numeric and string arguments to DOUBLE. With ANSI +//! mode off, Spark returns NULL for a malformed string, while these functions +//! raise an error (see ). +use std::f64::consts::LN_2; use std::sync::Arc; use arrow::array::{ArrayRef, AsArray, Float64Array}; use arrow::datatypes::{DataType, Float64Type}; +use datafusion_common::types::{NativeType, logical_float64, logical_string}; use datafusion_common::utils::take_function_args; -use datafusion_common::{Result, exec_err}; +use datafusion_common::{Result, ScalarValue, exec_err}; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, - TypeSignature, Volatility, + Coercion, ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature, + TypeSignature, TypeSignatureClass, Volatility, }; use datafusion_functions::utils::make_scalar_function; use datafusion_macros::user_doc; use super::fdlibm; +use crate::function::error_utils::unsupported_data_type_exec_err; + +/// Spark's `ImplicitCastInputTypes` with `DoubleType`: numeric and string +/// arguments are cast to DOUBLE. +fn float64_coercion() -> Coercion { + Coercion::new_implicit( + TypeSignatureClass::Native(logical_float64()), + vec![ + TypeSignatureClass::Integer, + TypeSignatureClass::Float, + TypeSignatureClass::Decimal, + TypeSignatureClass::Native(logical_string()), + ], + NativeType::Float64, + ) +} + +/// Downcasts to a Float64 array, returning an error rather than panicking +/// when a caller that skips coercion passes another type. +fn as_float64<'a>(name: &str, array: &'a ArrayRef) -> Result<&'a Float64Array> { + array.as_primitive_opt::().ok_or_else(|| { + unsupported_data_type_exec_err( + name, + DataType::Float64.to_string().as_str(), + array.data_type(), + ) + }) +} /// Applies `f` to every value of a Float64 array, returning NULL where the /// value is at or below `asymptote`, as Spark's `UnaryLogExpression` does. @@ -40,8 +74,7 @@ fn unary_log( f: impl Fn(f64) -> f64, ) -> Result { let [arg] = take_function_args(name, args)?; - let result: Float64Array = arg - .as_primitive::() + let result: Float64Array = as_float64(name, arg)? .unary_opt(|x| if x <= asymptote { None } else { Some(f(x)) }); Ok(Arc::new(result)) } @@ -58,8 +91,8 @@ macro_rules! unary_log_udf { impl $struct { pub fn new() -> Self { Self { - signature: Signature::exact( - vec![DataType::Float64], + signature: Signature::coercible( + vec![float64_coercion()], Volatility::Immutable, ), } @@ -101,8 +134,8 @@ macro_rules! unary_log_udf { /// #[user_doc( doc_section(label = "Math Functions"), - description = "Returns the natural logarithm (base e) of `expr` as a DOUBLE. The argument is \ - cast to DOUBLE. Unlike DataFusion's built-in `ln`, which returns -Infinity for 0 and NaN for \ + description = "Returns the natural logarithm (base e) of `expr` as a DOUBLE. Numeric and \ + string arguments are cast to DOUBLE. Unlike DataFusion's built-in `ln`, which returns -Infinity for 0 and NaN for \ a negative number, an input less than or equal to 0 returns NULL. Returns NULL if the input \ is NULL, and NaN if it is NaN.", syntax_example = "ln(expr)", @@ -114,7 +147,7 @@ NULL ```"#, argument( name = "expr", - description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + description = "Numeric or string expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -129,8 +162,8 @@ unary_log_udf!(SparkLn, "ln", 0.0, fdlibm::log); /// #[user_doc( doc_section(label = "Math Functions"), - description = "Returns the base 10 logarithm of `expr` as a DOUBLE. The argument is cast to \ - DOUBLE. Unlike DataFusion's built-in `log10`, which returns -Infinity for 0 and NaN for a \ + description = "Returns the base 10 logarithm of `expr` as a DOUBLE. Numeric and string \ + arguments are cast to DOUBLE. Unlike DataFusion's built-in `log10`, which returns -Infinity for 0 and NaN for a \ negative number, an input less than or equal to 0 returns NULL. Returns NULL if the input \ is NULL, and NaN if it is NaN.", syntax_example = "log10(expr)", @@ -142,7 +175,7 @@ NULL ```"#, argument( name = "expr", - description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + description = "Numeric or string expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -157,8 +190,8 @@ unary_log_udf!(SparkLog10, "log10", 0.0, fdlibm::log10); /// #[user_doc( doc_section(label = "Math Functions"), - description = "Returns the base 2 logarithm of `expr` as a DOUBLE. The argument is cast to \ - DOUBLE. Like Spark, it is computed as `ln(expr) / ln(2)`, so it can differ from an exact \ + description = "Returns the base 2 logarithm of `expr` as a DOUBLE. Numeric and string \ + arguments are cast to DOUBLE. Like Spark, it is computed as `ln(expr) / ln(2)`, so it can differ from an exact \ base 2 logarithm in the last digit. Unlike DataFusion's built-in `log2`, which returns \ -Infinity for 0 and NaN for a negative number, an input less than or equal to 0 returns \ NULL. Returns NULL if the input is NULL, and NaN if it is NaN.", @@ -171,7 +204,7 @@ NULL ```"#, argument( name = "expr", - description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + description = "Numeric or string expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -181,9 +214,8 @@ pub struct SparkLog2 { // Spark computes `StrictMath.log(x) / StrictMath.log(2)` rather than a true // base 2 logarithm, which differs in the last bit for some inputs. -unary_log_udf!(SparkLog2, "log2", 0.0, |x: f64| { - fdlibm::log(x) / fdlibm::log(2.0) -}); +// `StrictMath.log(2.0)` is exactly `LN_2`. +unary_log_udf!(SparkLog2, "log2", 0.0, |x: f64| fdlibm::log(x) / LN_2); /// Spark-compatible `log1p` expression. /// @@ -191,8 +223,8 @@ unary_log_udf!(SparkLog2, "log2", 0.0, |x: f64| { #[user_doc( doc_section(label = "Math Functions"), description = "Returns `ln(1 + expr)` as a DOUBLE, computed without the precision loss of \ - adding 1 to `expr` when `expr` is close to 0. DataFusion has no built-in `log1p`. The \ - argument is cast to DOUBLE. An input less than or equal to -1 returns NULL. Returns NULL \ + adding 1 to `expr` when `expr` is close to 0. DataFusion has no built-in `log1p`. Numeric \ + and string arguments are cast to DOUBLE. An input less than or equal to -1 returns NULL. Returns NULL \ if the input is NULL, and NaN if it is NaN.", syntax_example = "log1p(expr)", sql_example = r#"```sql @@ -203,7 +235,7 @@ NULL ```"#, argument( name = "expr", - description = "Numeric expression, cast to DOUBLE. NULL or a number less than or equal to -1 returns NULL." + description = "Numeric or string expression, cast to DOUBLE. NULL or a number less than or equal to -1 returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -220,8 +252,8 @@ unary_log_udf!(SparkLog1p, "log1p", -1.0, fdlibm::log1p); doc_section(label = "Math Functions"), description = "Returns the logarithm of `expr` in base `base` as a DOUBLE, computed as \ `ln(expr) / ln(base)`. With one argument it returns the natural logarithm of `expr`, \ - where DataFusion's built-in `log` returns the base 10 logarithm. Both arguments are cast \ - to DOUBLE. Returns NULL if either argument is NULL or less than or equal to 0, otherwise \ + where DataFusion's built-in `log` returns the base 10 logarithm. Numeric and string \ + arguments are cast to DOUBLE. Returns NULL if either argument is NULL or less than or equal to 0, otherwise \ NaN if either is NaN. A base of 1 divides by 0, so it returns Infinity, -Infinity, or NaN \ when `expr` is also 1.", syntax_example = "log([base, ]expr)", @@ -235,11 +267,11 @@ NULL ```"#, argument( name = "base", - description = "Optional numeric base, cast to DOUBLE. Defaults to e. NULL, 0 or a negative number returns NULL." + description = "Optional numeric or string base, cast to DOUBLE. Defaults to e. NULL, 0 or a negative number returns NULL." ), argument( name = "expr", - description = "Numeric expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." + description = "Numeric or string expression, cast to DOUBLE. NULL, 0 or a negative number returns NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -258,8 +290,11 @@ impl SparkLog { Self { signature: Signature::one_of( vec![ - TypeSignature::Exact(vec![DataType::Float64]), - TypeSignature::Exact(vec![DataType::Float64, DataType::Float64]), + TypeSignature::Coercible(vec![float64_coercion()]), + TypeSignature::Coercible(vec![ + float64_coercion(), + float64_coercion(), + ]), ], Volatility::Immutable, ), @@ -285,6 +320,36 @@ impl ScalarUDFImpl for SparkLog { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + // A literal base is the common case. Take its logarithm once instead + // of expanding it into an array. + if let [ColumnarValue::Scalar(base), ColumnarValue::Array(value)] = + args.args.as_slice() + { + let ScalarValue::Float64(base) = base else { + return Err(unsupported_data_type_exec_err( + "log", + DataType::Float64.to_string().as_str(), + &base.data_type(), + )); + }; + let value = as_float64("log", value)?; + let result = match base { + // NaN is not <= 0, so a NaN base takes the second arm. + Some(base) if *base <= 0.0 => Float64Array::new_null(value.len()), + Some(base) => { + let ln_base = fdlibm::log(*base); + value.unary_opt(|x| { + if x <= 0.0 { + None + } else { + Some(fdlibm::log(x) / ln_base) + } + }) + } + None => Float64Array::new_null(value.len()), + }; + return Ok(ColumnarValue::Array(Arc::new(result))); + } make_scalar_function(spark_log, vec![])(&args.args) } } @@ -295,8 +360,8 @@ fn spark_log(args: &[ArrayRef]) -> Result { // which its generated code evaluates as `StrictMath.log(expr)`. [_] => unary_log("log", args, 0.0, fdlibm::log), [base, value] => { - let base = base.as_primitive::(); - let value = value.as_primitive::(); + let base = as_float64("log", base)?; + let value = as_float64("log", value)?; let result: Float64Array = base .iter() .zip(value.iter()) @@ -315,3 +380,70 @@ fn spark_log(args: &[ArrayRef]) -> Result { _ => exec_err!("log expects 1 or 2 arguments, got {}", args.len()), } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::Int32Array; + use arrow::datatypes::Field; + + fn invoke( + udf: &dyn ScalarUDFImpl, + args: Vec, + ) -> Result { + let arg_fields = args + .iter() + .map(|arg| Arc::new(Field::new("a", arg.data_type(), true))) + .collect(); + udf.invoke_with_args(ScalarFunctionArgs { + args, + arg_fields, + number_rows: 1, + return_field: Arc::new(Field::new(udf.name(), DataType::Float64, true)), + config_options: Arc::new(Default::default()), + }) + } + + /// A caller that builds the physical expression directly can skip + /// coercion. A non-Float64 argument must error, not panic. + #[test] + fn non_float64_arguments_error() { + let int: ArrayRef = Arc::new(Int32Array::from(vec![1])); + let float: ArrayRef = Arc::new(Float64Array::from(vec![1.0])); + let ln = SparkLn::new(); + let log = SparkLog::new(); + let cases: Vec<(&dyn ScalarUDFImpl, Vec)> = vec![ + (&ln, vec![ColumnarValue::Array(Arc::clone(&int))]), + (&log, vec![ColumnarValue::Array(Arc::clone(&int))]), + ( + &log, + vec![ + ColumnarValue::Array(Arc::clone(&int)), + ColumnarValue::Array(Arc::clone(&float)), + ], + ), + ( + &log, + vec![ + ColumnarValue::Scalar(ScalarValue::Int32(Some(2))), + ColumnarValue::Array(Arc::clone(&float)), + ], + ), + ( + &log, + vec![ + ColumnarValue::Scalar(ScalarValue::Float64(Some(2.0))), + ColumnarValue::Array(Arc::clone(&int)), + ], + ), + ]; + for (udf, args) in cases { + let err = invoke(udf, args).unwrap_err().strip_backtrace(); + assert!( + err.contains("expects Float64, got Int32"), + "{}: {err}", + udf.name() + ); + } + } +} diff --git a/crates/sqllogictest/test_files/math/ln.slt b/crates/sqllogictest/test_files/math/ln.slt index 5b1f0995..58fb2624 100644 --- a/crates/sqllogictest/test_files/math/ln.slt +++ b/crates/sqllogictest/test_files/math/ln.slt @@ -40,7 +40,7 @@ SELECT ln(10::tinyint) AS a, ln(10::bigint) AS b, ln(10::float) AS c, ln(10::dec # Spark returns NULL, not -Infinity or NaN, for an input <= 0 query RRRR -SELECT ln(0::int) AS a, ln(-0.0::double) AS b, ln(-1::int) AS c, ln('-Infinity'::double) AS d; +SELECT ln(0::int) AS a, ln('-0.0'::double) AS b, ln(-1::int) AS c, ln('-Infinity'::double) AS d; ---- NULL NULL NULL NULL @@ -76,3 +76,26 @@ query R SELECT ln(0.670311110015402::double); ---- -0.40001333106454684 + +# Spark implicitly casts a string argument to DOUBLE +query RR +SELECT ln('10'), ln(a) FROM (VALUES ('1'), (NULL)) AS t(a); +---- +2.302585092994046 0 +2.302585092994046 NULL + +statement ok +set datafusion.execution.enable_ansi_mode = true; + +query error Cannot cast string 'abc' to value of Float64 type +SELECT ln('abc'); + +statement ok +set datafusion.execution.enable_ansi_mode = false; + +# Spark 4.2.0 returns NULL for a malformed string when ANSI mode is off; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query R +# SELECT ln('abc'); +# ---- +# NULL diff --git a/crates/sqllogictest/test_files/math/log.slt b/crates/sqllogictest/test_files/math/log.slt index cce753ac..973925b9 100644 --- a/crates/sqllogictest/test_files/math/log.slt +++ b/crates/sqllogictest/test_files/math/log.slt @@ -101,6 +101,26 @@ SELECT log(2::int, x) FROM (VALUES (8::double), (-1::double), (NULL::double)) AS NULL NULL +# NULL, non-positive and NaN scalar bases with an array expr +query RRRR +SELECT log(NULL::double, x), log(0::double, x), log(-2::double, x), log('NaN'::double, x) FROM (VALUES (8::double)) AS t(x); +---- +NULL NULL NULL NaN + +# Array base with a scalar expr +query R +SELECT log(b, 8::double) FROM (VALUES (2::double), (0::double), (NULL::double)) AS t(b); +---- +3 +NULL +NULL + +# Spark implicitly casts a string argument to DOUBLE +query RR +SELECT log('2', '8'), log('10'); +---- +3 2.302585092994046 + query R SELECT log(x) FROM (VALUES (1::double), (0::double), (NULL::double)) AS t(x); ---- diff --git a/crates/sqllogictest/test_files/math/log1p.slt b/crates/sqllogictest/test_files/math/log1p.slt index 895c20ef..3ab1cda8 100644 --- a/crates/sqllogictest/test_files/math/log1p.slt +++ b/crates/sqllogictest/test_files/math/log1p.slt @@ -60,6 +60,6 @@ SELECT log1p(0.670311110015402::double); # -0.0 is above the asymptote, and log1p(-0.0) is -0.0 query R -SELECT log1p(-0.0::double); +SELECT log1p('-0.0'::double); ---- -0 From 2afb9339fbc78ded694093f3999d6b016dd08a58 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 10:54:28 -0700 Subject: [PATCH 4/6] docs(spark): link the shared input-type coercion issue from float64_coercion --- crates/spark/src/function/math/log.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/spark/src/function/math/log.rs b/crates/spark/src/function/math/log.rs index a9357956..8c9600c7 100644 --- a/crates/spark/src/function/math/log.rs +++ b/crates/spark/src/function/math/log.rs @@ -40,6 +40,8 @@ use crate::function::error_utils::unsupported_data_type_exec_err; /// Spark's `ImplicitCastInputTypes` with `DoubleType`: numeric and string /// arguments are cast to DOUBLE. +/// A stopgap until input types are declared and coerced in one place +/// (). fn float64_coercion() -> Coercion { Coercion::new_implicit( TypeSignatureClass::Native(logical_float64()), From e520f45b086261c52eeca3f4e8a34848a9cca39c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 16:54:35 -0700 Subject: [PATCH 5/6] docs(spark): fix private fdlibm doc links and note Spark's d/f string suffix `cargo doc` with `-D warnings` failed because the public `log` module and `SparkExpm1` linked to the private `fdlibm` module. Name it in plain text. Spark parses a string argument with Java's `Double.valueOf`, which accepts a trailing d, D, f or F in both ANSI modes. The cast DataFusion uses here errors instead. Record the difference in the `log` module doc and as a commented-out query in ln.slt, next to the ANSI-off NULL case, both waiting on #88. --- crates/spark/src/function/math/expm1.rs | 6 ++++-- crates/spark/src/function/math/log.rs | 18 +++++++++++++----- crates/sqllogictest/test_files/math/ln.slt | 8 ++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index 00406069..13792ca7 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -20,8 +20,10 @@ use std::sync::Arc; /// /// /// Spark computes this with `java.lang.StrictMath.expm1`, so this uses the fdlibm -/// translation in [`fdlibm`] rather than `f64::exp_m1`, which calls the platform's -/// libm and differs in the last digit on some platforms. +/// translation in the private `math::fdlibm` module rather than `f64::exp_m1`, +/// which calls the platform's libm and differs in the last digit on some platforms. +/// The module is named in plain text because rustdoc rejects a public item's link +/// to a private one. #[user_doc( doc_section(label = "Math Functions"), description = "Returns `exp(expr) - 1` as a DOUBLE, computed without the precision loss of \ diff --git a/crates/spark/src/function/math/log.rs b/crates/spark/src/function/math/log.rs index 8c9600c7..9b0936ca 100644 --- a/crates/spark/src/function/math/log.rs +++ b/crates/spark/src/function/math/log.rs @@ -13,12 +13,20 @@ //! one-argument `log` is base 10 where Spark's is the natural logarithm. //! //! Spark computes the logarithms with `java.lang.StrictMath`, so these use the -//! fdlibm translation in [`super::fdlibm`] to return the same bits. Rust's -//! `f64::ln` differs in the last digit for some inputs. +//! fdlibm translation in the private `math::fdlibm` module to return the same +//! bits. Rust's `f64::ln` differs in the last digit for some inputs. The module +//! is named in plain text because rustdoc rejects a public item's link to a +//! private one. //! -//! Spark implicitly casts numeric and string arguments to DOUBLE. With ANSI -//! mode off, Spark returns NULL for a malformed string, while these functions -//! raise an error (see ). +//! Spark implicitly casts numeric and string arguments to DOUBLE. The string +//! cast differs from Spark in two ways until it is ported +//! (): +//! +//! - With ANSI mode off, Spark returns NULL for a malformed string, while these +//! functions raise an error. +//! - Spark parses strings with Java's `Double.valueOf`, which accepts a trailing +//! `d`, `D`, `f` or `F` (`ln('10d')` is `ln(10)`) in both ANSI modes. These +//! functions raise an error for such a string. use std::f64::consts::LN_2; use std::sync::Arc; diff --git a/crates/sqllogictest/test_files/math/ln.slt b/crates/sqllogictest/test_files/math/ln.slt index 58fb2624..8e3d8f8e 100644 --- a/crates/sqllogictest/test_files/math/ln.slt +++ b/crates/sqllogictest/test_files/math/ln.slt @@ -99,3 +99,11 @@ set datafusion.execution.enable_ansi_mode = false; # SELECT ln('abc'); # ---- # NULL + +# Spark parses strings with Java's Double.valueOf, which accepts a d or f suffix in both +# ANSI modes; DataFusion errors +# https://github.com/apache/datafusion-spark/issues/88 +# query R +# SELECT ln('10d'); +# ---- +# 2.302585092994046 From f1c3998022699394056a6269f77cd03f3ec0739e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 19 Nov 2026 06:02:27 -0700 Subject: [PATCH 6/6] test(spark): mark ln's string cast differences with xfail comments Turn the commented-out ln('abc') (ANSI off) and ln('10d') records into live records that assert DataFusion's current cast error, each with a # xfail spark: marker that links #88. --- crates/sqllogictest/test_files/math/ln.slt | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/crates/sqllogictest/test_files/math/ln.slt b/crates/sqllogictest/test_files/math/ln.slt index 8e3d8f8e..d937ecac 100644 --- a/crates/sqllogictest/test_files/math/ln.slt +++ b/crates/sqllogictest/test_files/math/ln.slt @@ -93,17 +93,10 @@ SELECT ln('abc'); statement ok set datafusion.execution.enable_ansi_mode = false; -# Spark 4.2.0 returns NULL for a malformed string when ANSI mode is off; DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query R -# SELECT ln('abc'); -# ---- -# NULL - -# Spark parses strings with Java's Double.valueOf, which accepts a d or f suffix in both -# ANSI modes; DataFusion errors -# https://github.com/apache/datafusion-spark/issues/88 -# query R -# SELECT ln('10d'); -# ---- -# 2.302585092994046 +# xfail spark: Spark returns NULL for a malformed string when ANSI mode is off, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Cannot cast string 'abc' to value of Float64 type +SELECT ln('abc'); + +# xfail spark: Spark parses strings with Java's Double.valueOf, which accepts a d or f suffix in both ANSI modes, DataFusion's cast errors (https://github.com/apache/datafusion-spark/issues/88) +query error Cannot cast string '10d' to value of Float64 type +SELECT ln('10d');