Skip to content

Commit a36c5b3

Browse files
committed
feat(session): add SessionContextBuilder.withSparkFunctions flag
1 parent f889243 commit a36c5b3

3 files changed

Lines changed: 39 additions & 0 deletions

File tree

‎core/src/main/java/org/apache/datafusion/SessionContextBuilder.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public final class SessionContextBuilder {
4444
private boolean spillDisabled;
4545
private Long maxTempDirectorySize;
4646
private CacheManagerOptions cacheManager;
47+
private boolean sparkFunctions;
4748
private final LinkedHashMap<String, String> options = new LinkedHashMap<>();
4849
private final List<ObjectStoreOptions> objectStores = new ArrayList<>();
4950

@@ -227,6 +228,20 @@ public SessionContextBuilder cacheManager(CacheManagerOptions options) {
227228
return this;
228229
}
229230

231+
/**
232+
* Register Apache Spark-compatible functions and expression planners on the new context, using
233+
* the {@code datafusion-spark} crate. Once enabled, Spark-compatible functions (e.g. {@code
234+
* crc32}) are callable from SQL and override any DataFusion built-in of the same name.
235+
*
236+
* <p>Requires the native library to be built with the {@code spark} Cargo feature, which is
237+
* enabled in the default build. If it is not, {@link #build()} throws a {@link RuntimeException}
238+
* explaining the feature is missing.
239+
*/
240+
public SessionContextBuilder withSparkFunctions() {
241+
this.sparkFunctions = true;
242+
return this;
243+
}
244+
230245
/**
231246
* Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}.
232247
* Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3},
@@ -309,6 +324,9 @@ byte[] toBytes() {
309324
for (Map.Entry<String, String> e : options.entrySet()) {
310325
b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build());
311326
}
327+
if (sparkFunctions) {
328+
b.setSparkFunctions(true);
329+
}
312330
for (ObjectStoreOptions os : objectStores) {
313331
b.addObjectStores(os.toRegistration());
314332
}

‎core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -426,6 +426,21 @@ void diskManagerFieldIsAbsentWhenNothingSet() throws Exception {
426426
assertFalse(parsed.hasDiskManager());
427427
}
428428

429+
@Test
430+
void withSparkFunctionsRoundTripsThroughProto() throws Exception {
431+
byte[] bytes = SessionContext.builder().withSparkFunctions().toBytes();
432+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
433+
assertTrue(parsed.hasSparkFunctions());
434+
assertTrue(parsed.getSparkFunctions());
435+
}
436+
437+
@Test
438+
void sparkFunctionsAbsentWhenNotRequested() throws Exception {
439+
byte[] bytes = SessionContext.builder().batchSize(8192).toBytes();
440+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
441+
assertFalse(parsed.hasSparkFunctions());
442+
}
443+
429444
@Test
430445
void tempDirectoryStaysOnLegacyField() throws Exception {
431446
// tempDirectory(String) writes the existing SessionOptions.temp_directory

‎proto/session_options.proto‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ message SessionOptions {
6464
// Disk-manager configuration (disable spill, size cap). Unset fields
6565
// leave the upstream `RuntimeEnvBuilder` default in place.
6666
optional DiskManagerOptions disk_manager = 10;
67+
// Register Apache Spark-compatible functions and expression planners on the
68+
// new context (via the datafusion-spark crate). Requires the native library
69+
// to be built with the `spark` Cargo feature (on in the default build);
70+
// otherwise construction fails with a clear error. Spark functions override
71+
// DataFusion built-ins of the same name.
72+
optional bool spark_functions = 11;
6773
}
6874

6975
// Disk-manager configuration. The two fields are independent on the wire;

0 commit comments

Comments
 (0)