diff --git a/Cargo.lock b/Cargo.lock index 65f1326f..2d19f2a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1610,6 +1610,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "datafusion-session", + "libm", "log", "num-traits", "percent-encoding", diff --git a/crates/spark/Cargo.toml b/crates/spark/Cargo.toml index ef542cfb..ae69e4a8 100644 --- a/crates/spark/Cargo.toml +++ b/crates/spark/Cargo.toml @@ -53,6 +53,7 @@ datafusion-functions-nested = { workspace = true } datafusion-macros = { workspace = true } datafusion-physical-expr-common = { workspace = true } datafusion-session = { workspace = true } +libm = "0.2.16" log = { workspace = true } num-traits = { workspace = true } percent-encoding = "2.3.2" diff --git a/crates/spark/src/function/math/expm1.rs b/crates/spark/src/function/math/expm1.rs index b0c280df..fd5a5ad6 100644 --- a/crates/spark/src/function/math/expm1.rs +++ b/crates/spark/src/function/math/expm1.rs @@ -74,13 +74,13 @@ impl ScalarUDFImpl for SparkExpm1 { let [arg] = take_function_args(self.name(), args.args)?; match arg { ColumnarValue::Scalar(ScalarValue::Float64(value)) => Ok( - ColumnarValue::Scalar(ScalarValue::Float64(value.map(|x| x.exp_m1()))), + ColumnarValue::Scalar(ScalarValue::Float64(value.map(spark_expm1))), ), ColumnarValue::Array(array) => match array.data_type() { DataType::Float64 => Ok(ColumnarValue::Array(Arc::new( array .as_primitive::() - .unary::<_, Float64Type>(|x| x.exp_m1()), + .unary::<_, Float64Type>(spark_expm1), ) as ArrayRef)), other => Err(unsupported_data_type_exec_err( @@ -97,3 +97,11 @@ impl ScalarUDFImpl for SparkExpm1 { } } } + +/// Spark's `Expm1` calls Java's `StrictMath.expm1`, which returns fdlibm's +/// result on every platform. `f64::exp_m1` calls the platform's C math library, +/// which can differ in the last bit, as on macOS. The `libm` crate is a port of +/// musl's math library, which comes from fdlibm, so it matches Spark. +fn spark_expm1(x: f64) -> f64 { + libm::expm1(x) +} diff --git a/crates/sqllogictest/test_files/math/expm1.slt b/crates/sqllogictest/test_files/math/expm1.slt index 8e21b971..f0b55d50 100644 --- a/crates/sqllogictest/test_files/math/expm1.slt +++ b/crates/sqllogictest/test_files/math/expm1.slt @@ -22,3 +22,20 @@ query R SELECT expm1(0.0::double); ---- 0 + +# Spark returns fdlibm's result. For each of these, macOS's math library +# returns a result that differs in the last bit. +query R +SELECT expm1(a) FROM (VALUES (-2.5::DOUBLE), (0.9::DOUBLE), (1.7::DOUBLE)) AS t(a); +---- +-0.9179150013761013 +1.4596031111569499 +4.473947391727199 + +query R +SELECT expm1(a) FROM (VALUES (-0.5::DOUBLE), (1e-10::DOUBLE), (710.0::DOUBLE), (-800.0::DOUBLE)) AS t(a); +---- +-0.3934693402873666 +0.000000000100000000005 +Infinity +-1