Skip to content

Commit 5cd9bf3

Browse files
committed
feat(session): register Spark-compatible functions when withSparkFunctions is set
1 parent a36c5b3 commit 5cd9bf3

2 files changed

Lines changed: 102 additions & 3 deletions

File tree

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertThrows;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import org.apache.arrow.memory.BufferAllocator;
27+
import org.apache.arrow.memory.RootAllocator;
28+
import org.apache.arrow.vector.BigIntVector;
29+
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.junit.jupiter.api.Test;
31+
32+
class SessionContextSparkFunctionsTest {
33+
34+
// crc32 is a Spark function provided by datafusion-spark and is NOT a core
35+
// DataFusion built-in. Spark's crc32('Spark') == 1557323817 (standard
36+
// CRC-32 of the ASCII bytes), so it doubles as a correctness check.
37+
@Test
38+
void sparkFunctionIsCallableWhenEnabled() throws Exception {
39+
try (BufferAllocator allocator = new RootAllocator();
40+
SessionContext ctx = SessionContext.builder().withSparkFunctions().build();
41+
DataFrame df = ctx.sql("SELECT crc32('Spark') AS c");
42+
ArrowReader reader = df.collect(allocator)) {
43+
assertTrue(reader.loadNextBatch());
44+
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector("c");
45+
assertEquals(1, v.getValueCount());
46+
assertEquals(1557323817L, v.get(0));
47+
}
48+
}
49+
50+
// Without the flag, crc32 is unregistered, so planning the same SQL must
51+
// fail. This is the proof that withSparkFunctions() is what enabled it.
52+
@Test
53+
void sparkFunctionIsAbsentByDefault() {
54+
try (SessionContext ctx = SessionContext.builder().build()) {
55+
RuntimeException thrown =
56+
assertThrows(RuntimeException.class, () -> ctx.sql("SELECT crc32('Spark')"));
57+
assertTrue(
58+
thrown.getMessage() != null && thrown.getMessage().toLowerCase().contains("crc32"),
59+
"expected error to mention the unknown function, got: " + thrown.getMessage());
60+
}
61+
}
62+
}

‎native/src/lib.rs‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ use datafusion::dataframe::DataFrame;
5050
use datafusion::dataframe::DataFrameWriteOptions;
5151
use datafusion::error::DataFusionError;
5252
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
53-
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
53+
use datafusion::execution::runtime_env::{RuntimeEnv, RuntimeEnvBuilder};
5454
use datafusion::execution::SendableRecordBatchStream;
5555
use datafusion::logical_expr::Expr;
5656
use datafusion::logical_expr::{col, Partitioning, ScalarUDF, Signature, SortExpr};
@@ -117,6 +117,42 @@ fn install_memory_tracker(
117117
tracker
118118
}
119119

120+
/// Build a `SessionContext` from a prepared config + runtime env. When
121+
/// `spark_functions` is set, register Apache Spark-compatible functions and
122+
/// expression planners via the `datafusion-spark` crate (requires the `spark`
123+
/// Cargo feature); otherwise build the plain context. Returns an error if the
124+
/// caller asked for Spark functions in a build that compiled the feature off.
125+
fn build_session_context(
126+
config: SessionConfig,
127+
runtime_env: Arc<RuntimeEnv>,
128+
spark_functions: bool,
129+
) -> JniResult<SessionContext> {
130+
if spark_functions {
131+
#[cfg(feature = "spark")]
132+
{
133+
use datafusion::execution::SessionStateBuilder;
134+
use datafusion_spark::SessionStateBuilderSpark;
135+
// Order matters: `with_spark_features` runs after
136+
// `with_default_features` so Spark implementations override the
137+
// DataFusion built-ins of the same name.
138+
let state = SessionStateBuilder::new()
139+
.with_config(config)
140+
.with_runtime_env(runtime_env)
141+
.with_default_features()
142+
.with_spark_features()
143+
.build();
144+
Ok(SessionContext::new_with_state(state))
145+
}
146+
#[cfg(not(feature = "spark"))]
147+
{
148+
let _ = (config, runtime_env);
149+
Err("spark Cargo feature is not enabled in this build of datafusion-jni".into())
150+
}
151+
} else {
152+
Ok(SessionContext::new_with_config_rt(config, runtime_env))
153+
}
154+
}
155+
120156
#[no_mangle]
121157
pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionContext<'local>(
122158
mut env: JNIEnv<'local>,
@@ -225,8 +261,9 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionCo
225261
// update two atomics.
226262
let tracker = install_memory_tracker(&mut runtime_builder);
227263

228-
let runtime_env = runtime_builder.build()?;
229-
let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env));
264+
let runtime_env = Arc::new(runtime_builder.build()?);
265+
let ctx =
266+
build_session_context(config, runtime_env, opts.spark_functions.unwrap_or(false))?;
230267

231268
// Object-store registrations come last because they need a built
232269
// RuntimeEnv to register against. A failure here drops `ctx` (and its

0 commit comments

Comments
 (0)