2626
2727/**
2828 * A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
29- * by {@link SessionContext#sql(String)} or other planning entry points and executed by {@link
30- * #collect}.
29+ * by {@link SessionContext#sql(String)} or other planning entry points and executed by either
30+ * {@link #collect} (materializes every batch on the native heap before returning) or {@link
31+ * #executeStream} (yields one batch at a time as Java drains the reader).
3132 *
32- * <p>Instances are <strong>not thread-safe</strong> and must be closed. {@link #collect} consumes
33- * the DataFrame: a successfully collected DataFrame cannot be collected again, and {@link #close()}
34- * on an already-collected instance is a no-op.
33+ * <p>Instances are <strong>not thread-safe</strong> and must be closed. Both {@link #collect} and
34+ * {@link #executeStream} consume the DataFrame: a successfully consumed DataFrame cannot be
35+ * consumed again by either method (or by other executors such as {@link #count}), and {@link
36+ * #close()} on an already-consumed instance is a no-op.
3537 */
3638public final class DataFrame implements AutoCloseable {
3739 static {
@@ -53,6 +55,10 @@ public final class DataFrame implements AutoCloseable {
5355 * <p>Consumes this DataFrame: the native plan is released as soon as the stream is established.
5456 * The caller is responsible for closing the returned reader, and the supplied allocator must
5557 * outlive it.
58+ *
59+ * <p>This method materializes every batch on the native heap before the first batch crosses the
60+ * FFI boundary, which can OOM the Rust side for unbounded or very large result sets. Prefer
61+ * {@link #executeStream(BufferAllocator)} for analytics-scale queries.
5662 */
5763 public ArrowReader collect (BufferAllocator allocator ) {
5864 if (nativeHandle == 0 ) {
@@ -70,6 +76,36 @@ public ArrowReader collect(BufferAllocator allocator) {
7076 }
7177 }
7278
79+ /**
80+ * Execute the plan and return its record batches as a streaming {@link ArrowReader}. Each call to
81+ * {@link ArrowReader#loadNextBatch} drives one async {@code stream.next()} on the native side, so
82+ * memory pressure stays bounded by the executor pipeline plus one in-flight batch instead of the
83+ * full result set.
84+ *
85+ * <p>Consumes this DataFrame with the same lifecycle rules as {@link #collect(BufferAllocator)}:
86+ * the native plan is released as soon as the stream is established, the caller closes the
87+ * returned reader, and the supplied allocator must outlive it.
88+ *
89+ * <p>For result sets that fit comfortably in native memory and are read in their entirety, {@link
90+ * #collect(BufferAllocator)} remains a reasonable choice. For TB-scale or unbounded result sets,
91+ * use this method.
92+ */
93+ public ArrowReader executeStream (BufferAllocator allocator ) {
94+ if (nativeHandle == 0 ) {
95+ throw new IllegalStateException ("DataFrame is closed or already collected" );
96+ }
97+ ArrowArrayStream stream = ArrowArrayStream .allocateNew (allocator );
98+ long handle = nativeHandle ;
99+ nativeHandle = 0 ;
100+ try {
101+ executeStreamDataFrame (handle , stream .memoryAddress ());
102+ return Data .importArrayStream (allocator , stream );
103+ } catch (Throwable e ) {
104+ stream .close ();
105+ throw e ;
106+ }
107+ }
108+
73109 /** Execute the plan and return the number of rows. */
74110 public long count () {
75111 if (nativeHandle == 0 ) {
@@ -250,6 +286,70 @@ public void writeParquet(String path, ParquetWriteOptions options) {
250286 options .singleFileOutput () != null && options .singleFileOutput ());
251287 }
252288
289+ /**
290+ * Materialize this DataFrame as CSV at {@code path}. The path is treated as a directory unless
291+ * overridden via {@link CsvWriteOptions#singleFileOutput(boolean)}. The receiver remains usable
292+ * and must still be closed independently.
293+ *
294+ * @throws RuntimeException if the write fails.
295+ */
296+ public void writeCsv (String path ) {
297+ writeCsv (path , new CsvWriteOptions ());
298+ }
299+
300+ /**
301+ * Materialize this DataFrame as CSV at {@code path} with the supplied {@link CsvWriteOptions}.
302+ * The receiver remains usable and must still be closed independently.
303+ *
304+ * @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
305+ * @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
306+ * etc.).
307+ */
308+ public void writeCsv (String path , CsvWriteOptions options ) {
309+ if (nativeHandle == 0 ) {
310+ throw new IllegalStateException ("DataFrame is closed or already collected" );
311+ }
312+ if (path == null ) {
313+ throw new IllegalArgumentException ("writeCsv path must be non-null" );
314+ }
315+ if (options == null ) {
316+ throw new IllegalArgumentException ("writeCsv options must be non-null" );
317+ }
318+ writeCsvWithOptions (nativeHandle , path , options .toBytes ());
319+ }
320+
321+ /**
322+ * Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
323+ * directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
324+ * receiver remains usable and must still be closed independently.
325+ *
326+ * @throws RuntimeException if the write fails.
327+ */
328+ public void writeJson (String path ) {
329+ writeJson (path , new JsonWriteOptions ());
330+ }
331+
332+ /**
333+ * Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
334+ * JsonWriteOptions}. The receiver remains usable and must still be closed independently.
335+ *
336+ * @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
337+ * @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
338+ * etc.).
339+ */
340+ public void writeJson (String path , JsonWriteOptions options ) {
341+ if (nativeHandle == 0 ) {
342+ throw new IllegalStateException ("DataFrame is closed or already collected" );
343+ }
344+ if (path == null ) {
345+ throw new IllegalArgumentException ("writeJson path must be non-null" );
346+ }
347+ if (options == null ) {
348+ throw new IllegalArgumentException ("writeJson options must be non-null" );
349+ }
350+ writeJsonWithOptions (nativeHandle , path , options .toBytes ());
351+ }
352+
253353 @ Override
254354 public void close () {
255355 if (nativeHandle != 0 ) {
@@ -260,6 +360,8 @@ public void close() {
260360
261361 private static native void collectDataFrame (long handle , long ffiStreamAddr );
262362
363+ private static native void executeStreamDataFrame (long handle , long ffiStreamAddr );
364+
263365 private static native void closeDataFrame (long handle );
264366
265367 private static native long countRows (long handle );
@@ -290,4 +392,8 @@ private static native void writeParquetWithOptions(
290392 String compression ,
291393 boolean singleFileOutputSet ,
292394 boolean singleFileOutputValue );
395+
396+ private static native void writeCsvWithOptions (long handle , String path , byte [] optionsBytes );
397+
398+ private static native void writeJsonWithOptions (long handle , String path , byte [] optionsBytes );
293399}
0 commit comments