Skip to content

Commit f9cdc15

Browse files
authored
feat(session): expose disableSpill and maxTempDirectorySize on SessionContextBuilder (#84)
1 parent d078c12 commit f9cdc15

4 files changed

Lines changed: 203 additions & 0 deletions

File tree

‎core/src/main/java/org/apache/datafusion/SessionContextBuilder.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.Map;
2626

2727
import org.apache.datafusion.protobuf.ConfigOption;
28+
import org.apache.datafusion.protobuf.DiskManagerOptions;
2829
import org.apache.datafusion.protobuf.MemoryLimit;
2930
import org.apache.datafusion.protobuf.SessionOptions;
3031

@@ -40,6 +41,8 @@ public final class SessionContextBuilder {
4041
private Long memoryLimitBytes;
4142
private Double memoryLimitFraction;
4243
private String tempDirectory;
44+
private boolean spillDisabled;
45+
private Long maxTempDirectorySize;
4346
private CacheManagerOptions cacheManager;
4447
private final LinkedHashMap<String, String> options = new LinkedHashMap<>();
4548
private final List<ObjectStoreOptions> objectStores = new ArrayList<>();
@@ -94,6 +97,39 @@ public SessionContextBuilder tempDirectory(String path) {
9497
return this;
9598
}
9699

100+
/**
101+
* Disable on-disk spill entirely. Queries that need spill fail with a {@link
102+
* ResourcesExhaustedException} rather than going to disk; useful for memory-only execution
103+
* profiles or environments without writable disk.
104+
*
105+
* <p>Mutually exclusive with {@link #tempDirectory(String)} — the combination throws at {@link
106+
* #build()} time. {@link #maxTempDirectorySize(long)} is allowed alongside this setter but is a
107+
* no-op (no directory to cap).
108+
*/
109+
public SessionContextBuilder disableSpill() {
110+
this.spillDisabled = true;
111+
return this;
112+
}
113+
114+
/**
115+
* Cap the cumulative bytes used by spill files under {@link #tempDirectory(String)}. Mirrors
116+
* upstream {@code RuntimeEnvBuilder::with_max_temp_directory_size} 1:1. Once exceeded, queries
117+
* that need more spill space fail with a {@link ResourcesExhaustedException}. Combinable with
118+
* {@link #disableSpill()} but a no-op there.
119+
*
120+
* <p>{@code 0} is accepted — upstream documents zero as legal and equivalent to "no spill
121+
* allowed". Negative values are rejected.
122+
*
123+
* @throws IllegalArgumentException if {@code bytes} is negative.
124+
*/
125+
public SessionContextBuilder maxTempDirectorySize(long bytes) {
126+
if (bytes < 0) {
127+
throw new IllegalArgumentException("maxTempDirectorySize must be non-negative, got " + bytes);
128+
}
129+
this.maxTempDirectorySize = bytes;
130+
return this;
131+
}
132+
97133
/**
98134
* Set an arbitrary {@code datafusion.*} config option by string key. Mirrors DataFusion's {@code
99135
* ConfigOptions::set(key, value)} API — see the DataFusion configuration reference for the full
@@ -219,9 +255,15 @@ public SessionContextBuilder registerObjectStore(ObjectStoreOptions options) {
219255
/**
220256
* Construct a {@link SessionContext} with the configured options.
221257
*
258+
* @throws IllegalStateException if {@link #disableSpill()} was called alongside {@link
259+
* #tempDirectory(String)} — the combination is contradictory.
222260
* @throws RuntimeException if the native side fails to construct the context.
223261
*/
224262
public SessionContext build() {
263+
if (spillDisabled && tempDirectory != null) {
264+
throw new IllegalStateException(
265+
"disableSpill() is mutually exclusive with tempDirectory(...)");
266+
}
225267
return new SessionContext(toBytes());
226268
}
227269

@@ -249,6 +291,18 @@ byte[] toBytes() {
249291
if (tempDirectory != null) {
250292
b.setTempDirectory(tempDirectory);
251293
}
294+
DiskManagerOptions.Builder dm = null;
295+
if (spillDisabled) {
296+
dm = DiskManagerOptions.newBuilder().setDisabled(true);
297+
}
298+
if (maxTempDirectorySize != null) {
299+
dm =
300+
(dm != null ? dm : DiskManagerOptions.newBuilder())
301+
.setMaxTempDirectorySize(maxTempDirectorySize);
302+
}
303+
if (dm != null) {
304+
b.setDiskManager(dm.build());
305+
}
252306
if (cacheManager != null) {
253307
b.setCacheManager(cacheManager.toProto());
254308
}

‎core/src/test/java/org/apache/datafusion/SessionContextBuilderTest.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,4 +347,119 @@ void getOptionRejectsRuntimeKeys() {
347347
"expected runtime-key error, got: " + thrown.getMessage());
348348
}
349349
}
350+
351+
// -------------------------------------------------------------------------
352+
// disk_manager surface (disable, size cap)
353+
// -------------------------------------------------------------------------
354+
355+
@Test
356+
void disableSpillRoundTripsThroughProto() throws Exception {
357+
byte[] bytes = SessionContext.builder().disableSpill().toBytes();
358+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
359+
assertTrue(parsed.hasDiskManager());
360+
assertTrue(parsed.getDiskManager().getDisabled());
361+
}
362+
363+
@Test
364+
void disableSpillProducesUsableContext() throws Exception {
365+
// SELECT 1 doesn't spill, so disabling spill must not break it.
366+
try (BufferAllocator allocator = new RootAllocator();
367+
SessionContext ctx = SessionContext.builder().disableSpill().build();
368+
DataFrame df = ctx.sql("SELECT 1");
369+
ArrowReader reader = df.collect(allocator)) {
370+
assertTrue(reader.loadNextBatch());
371+
}
372+
}
373+
374+
@Test
375+
void disableSpillAndTempDirectoryConflictThrowsAtBuild() {
376+
SessionContextBuilder b = SessionContext.builder().disableSpill().tempDirectory("/tmp/x");
377+
assertThrows(IllegalStateException.class, b::build);
378+
}
379+
380+
@Test
381+
void maxTempDirectorySizeRoundTripsThroughProto() throws Exception {
382+
byte[] bytes = SessionContext.builder().maxTempDirectorySize(20L << 30).toBytes();
383+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
384+
assertTrue(parsed.hasDiskManager());
385+
assertTrue(parsed.getDiskManager().hasMaxTempDirectorySize());
386+
assertEquals(20L << 30, parsed.getDiskManager().getMaxTempDirectorySize());
387+
}
388+
389+
@Test
390+
void maxTempDirectorySizeRejectsNegative() {
391+
SessionContextBuilder b = SessionContext.builder();
392+
assertThrows(IllegalArgumentException.class, () -> b.maxTempDirectorySize(-1));
393+
}
394+
395+
@Test
396+
void maxTempDirectorySizeZeroBuildsCleanly() throws Exception {
397+
// Upstream allows 0 -- "no spill allowed" -- so the Java setter mirrors
398+
// that. Sanity-check that the context constructs and SELECT 1 (which
399+
// doesn't spill) still works.
400+
try (BufferAllocator allocator = new RootAllocator();
401+
SessionContext ctx = SessionContext.builder().maxTempDirectorySize(0).build();
402+
DataFrame df = ctx.sql("SELECT 1");
403+
ArrowReader reader = df.collect(allocator)) {
404+
assertTrue(reader.loadNextBatch());
405+
}
406+
}
407+
408+
@Test
409+
void disableSpillCombinesWithMaxTempDirectorySize() throws Exception {
410+
// The cap is a no-op when spill is disabled (no directory to cap), but
411+
// setting both must not throw -- callers may have code that always
412+
// configures the cap and conditionally disables spill.
413+
byte[] bytes = SessionContext.builder().disableSpill().maxTempDirectorySize(1L << 30).toBytes();
414+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
415+
assertTrue(parsed.getDiskManager().getDisabled());
416+
assertTrue(parsed.getDiskManager().hasMaxTempDirectorySize());
417+
}
418+
419+
@Test
420+
void diskManagerFieldIsAbsentWhenNothingSet() throws Exception {
421+
// Existing-callers-see-no-change contract: a builder that doesn't touch
422+
// any of the new disk_manager setters produces no disk_manager field on
423+
// the wire.
424+
byte[] bytes = SessionContext.builder().batchSize(8192).toBytes();
425+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
426+
assertFalse(parsed.hasDiskManager());
427+
}
428+
429+
@Test
430+
void tempDirectoryStaysOnLegacyField() throws Exception {
431+
// tempDirectory(String) writes the existing SessionOptions.temp_directory
432+
// field, not disk_manager -- bytes identical to pre-PR behaviour for
433+
// builders that touch only that setter.
434+
byte[] bytes = SessionContext.builder().tempDirectory("/tmp/df").toBytes();
435+
SessionOptions parsed = SessionOptions.parseFrom(bytes);
436+
assertTrue(parsed.hasTempDirectory());
437+
assertEquals("/tmp/df", parsed.getTempDirectory());
438+
assertFalse(parsed.hasDiskManager());
439+
}
440+
441+
@Test
442+
void disableSpillUnderMemoryPressureThrowsResourcesExhausted() throws java.io.IOException {
443+
// Pin the Javadoc claim on disableSpill() / maxTempDirectorySize that
444+
// out-of-memory queries surface as ResourcesExhaustedException -- not
445+
// the parent DataFusionException, not a generic RuntimeException. A 1 MiB
446+
// pool plus a 200k-row sort plus no spill forces the OOM path
447+
// deterministically.
448+
try (BufferAllocator allocator = new RootAllocator();
449+
SessionContext ctx =
450+
SessionContext.builder().memoryLimit(1L << 20, 1.0).disableSpill().build()) {
451+
DataFusionException e =
452+
assertThrows(
453+
DataFusionException.class,
454+
() -> {
455+
try (DataFrame df =
456+
ctx.sql("SELECT i FROM generate_series(1, 200000) AS t(i) ORDER BY i DESC")) {
457+
df.collect(allocator).close();
458+
}
459+
});
460+
assertTrue(
461+
e instanceof ResourcesExhaustedException,
462+
"expected ResourcesExhaustedException, got " + e.getClass().getName());
463+
}
464+
}
350465
}

‎native/src/lib.rs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ use datafusion::config::TableParquetOptions;
4949
use datafusion::dataframe::DataFrame;
5050
use datafusion::dataframe::DataFrameWriteOptions;
5151
use datafusion::error::DataFusionError;
52+
use datafusion::execution::disk_manager::{DiskManagerBuilder, DiskManagerMode};
5253
use datafusion::execution::runtime_env::RuntimeEnvBuilder;
5354
use datafusion::execution::SendableRecordBatchStream;
5455
use datafusion::logical_expr::Expr;
@@ -166,6 +167,22 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_createSessionCo
166167
if let Some(dir) = opts.temp_directory {
167168
runtime_builder = runtime_builder.with_temp_file_path(PathBuf::from(dir));
168169
}
170+
// disk_manager carries the disable / size-cap surface added on top of
171+
// the legacy temp_directory field. Java-side builder enforces that
172+
// disabled and tempDirectory aren't both set; the Rust layer doesn't
173+
// re-validate because there's no path that produces a contradictory
174+
// mode here -- with_disk_manager_builder(Disabled) wholesale
175+
// replaces any prior with_temp_file_path call, and that's the
176+
// semantics callers using the typed Java setters can already see.
177+
if let Some(dm) = opts.disk_manager.as_ref() {
178+
if dm.disabled() {
179+
let builder = DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled);
180+
runtime_builder = runtime_builder.with_disk_manager_builder(builder);
181+
}
182+
if let Some(size) = dm.max_temp_directory_size {
183+
runtime_builder = runtime_builder.with_max_temp_directory_size(size);
184+
}
185+
}
169186

170187
if let Some(cm_opts) = opts.cache_manager.as_ref() {
171188
if let Some(cm_config) = crate::cache_manager::build_config(cm_opts)? {

‎proto/session_options.proto‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,23 @@ message SessionOptions {
6161
// Configuration for DataFusion's built-in CacheManager. Unset = leave
6262
// the upstream default in place (matches existing callers).
6363
optional CacheManagerOptionsProto cache_manager = 9;
64+
// Disk-manager configuration (disable spill, size cap). Unset fields
65+
// leave the upstream `RuntimeEnvBuilder` default in place.
66+
optional DiskManagerOptions disk_manager = 10;
67+
}
68+
69+
// Disk-manager configuration. The two fields are independent on the wire;
70+
// mutual-exclusion of `disabled` against the legacy
71+
// `SessionOptions.temp_directory` field (slot 6) is enforced at the Java
72+
// builder level.
73+
message DiskManagerOptions {
74+
// Disable spill entirely (`DiskManagerMode::Disabled`).
75+
optional bool disabled = 1;
76+
// Cap on bytes used by spill files. Mirrors upstream
77+
// `RuntimeEnvBuilder::with_max_temp_directory_size` 1:1. Zero is a
78+
// legal value upstream (means "no spill allowed"); the Java setter
79+
// mirrors that contract.
80+
optional uint64 max_temp_directory_size = 2;
6481
}
6582

6683
message ConfigOption {

0 commit comments

Comments
 (0)