1919
2020package org .apache .datafusion ;
2121
22+ import java .io .ByteArrayInputStream ;
23+ import java .io .IOException ;
24+ import java .nio .channels .Channels ;
25+
2226import org .apache .arrow .c .ArrowArrayStream ;
2327import org .apache .arrow .c .Data ;
2428import org .apache .arrow .memory .BufferAllocator ;
2529import org .apache .arrow .vector .ipc .ArrowReader ;
30+ import org .apache .arrow .vector .ipc .ReadChannel ;
31+ import org .apache .arrow .vector .ipc .message .MessageSerializer ;
32+ import org .apache .arrow .vector .types .pojo .Schema ;
2633
2734/**
2835 * A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113 }
107114 }
108115
116+ /**
117+ * Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+ * usable and must still be closed independently. Schema inspection does not execute the plan.
119+ *
120+ * <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+ * schema carries no buffer data.
122+ */
123+ public Schema schema () {
124+ if (nativeHandle == 0 ) {
125+ throw new IllegalStateException ("DataFrame is closed or already collected" );
126+ }
127+ byte [] ipcBytes = schemaIpc (nativeHandle );
128+ try {
129+ return MessageSerializer .deserializeSchema (
130+ new ReadChannel (Channels .newChannel (new ByteArrayInputStream (ipcBytes ))));
131+ } catch (IOException e ) {
132+ throw new RuntimeException ("Failed to deserialize IPC schema" , e );
133+ }
134+ }
135+
136+ /**
137+ * Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+ * Non-consuming: the receiver remains usable and must still be closed independently.
139+ *
140+ * <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+ * contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+ * rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+ * #show()} or {@link #collect(BufferAllocator)}.
144+ */
145+ public DataFrame explain (boolean verbose , boolean analyze ) {
146+ if (nativeHandle == 0 ) {
147+ throw new IllegalStateException ("DataFrame is closed or already collected" );
148+ }
149+ return new DataFrame (explainPlan (nativeHandle , verbose , analyze ));
150+ }
151+
152+ /**
153+ * Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+ * Non-consuming: the receiver remains usable and must still be closed independently.
155+ *
156+ * <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+ * DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+ * downstream queries.
159+ *
160+ * @throws RuntimeException if execution fails.
161+ */
162+ public DataFrame cache () {
163+ if (nativeHandle == 0 ) {
164+ throw new IllegalStateException ("DataFrame is closed or already collected" );
165+ }
166+ return new DataFrame (cachePlan (nativeHandle ));
167+ }
168+
169+ /**
170+ * Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+ * DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+ * usable and must still be closed independently.
173+ *
174+ * <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+ * the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+ * count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+ *
178+ * @throws RuntimeException if execution fails.
179+ */
180+ public DataFrame describe () {
181+ if (nativeHandle == 0 ) {
182+ throw new IllegalStateException ("DataFrame is closed or already collected" );
183+ }
184+ return new DataFrame (describePlan (nativeHandle ));
185+ }
186+
109187 /** Execute the plan and return the number of rows. */
110188 public long count () {
111189 if (nativeHandle == 0 ) {
@@ -256,6 +334,121 @@ public DataFrame unnestColumns(UnnestOptions options, String... columns) {
256334 return new DataFrame (unnestColumns (nativeHandle , columns , options .preserveNulls ()));
257335 }
258336
337+ /**
338+ * Equi-join this DataFrame with {@code right} on the named columns, using the given {@link
339+ * JoinType}. The receiver and {@code right} both remain usable and must still be closed
340+ * independently.
341+ *
342+ * <p>Equivalent to SQL {@code left <type> JOIN right ON l.leftCols[0] = r.rightCols[0] AND ...}.
343+ * {@code leftCols} and {@code rightCols} must have the same length.
344+ *
345+ * @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
346+ * rightCols.length}.
347+ * @throws IllegalStateException if either DataFrame is closed or already collected.
348+ * @throws RuntimeException if join planning fails (column collision in the combined schema,
349+ * unknown column names, etc.).
350+ */
351+ public DataFrame join (DataFrame right , JoinType type , String [] leftCols , String [] rightCols ) {
352+ checkJoinArgs (right , type , leftCols , rightCols );
353+ return new DataFrame (
354+ joinDataFrame (nativeHandle , right .nativeHandle , type .code (), leftCols , rightCols , null ));
355+ }
356+
357+ /**
358+ * Equi-join this DataFrame with {@code right}, restricting the result with a residual SQL filter
359+ * parsed against the <em>combined</em> schema (left columns followed by right columns; columns
360+ * may be qualified with the relation alias when ambiguous). The receiver and {@code right} both
361+ * remain usable and must still be closed independently.
362+ *
363+ * <p>For outer joins, the filter is applied only to matched rows; unmatched rows are passed
364+ * through with nulls on the unmatched side, matching DataFusion's semantics.
365+ *
366+ * @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
367+ * rightCols.length}.
368+ * @throws IllegalStateException if either DataFrame is closed or already collected.
369+ * @throws RuntimeException if join planning or filter parsing fails.
370+ */
371+ public DataFrame join (
372+ DataFrame right , JoinType type , String [] leftCols , String [] rightCols , String filter ) {
373+ checkJoinArgs (right , type , leftCols , rightCols );
374+ if (filter == null ) {
375+ throw new IllegalArgumentException ("join filter must be non-null" );
376+ }
377+ return new DataFrame (
378+ joinDataFrame (nativeHandle , right .nativeHandle , type .code (), leftCols , rightCols , filter ));
379+ }
380+
381+ /**
382+ * Join this DataFrame with {@code right} using arbitrary SQL predicates parsed against the
383+ * <em>combined</em> schema. Each predicate is parsed independently and the join evaluates their
384+ * conjunction. Predicates may reference columns from either side and may be qualified with the
385+ * relation alias when ambiguous (e.g. {@code "left.x = right.x"}). The receiver and {@code right}
386+ * both remain usable and must still be closed independently.
387+ *
388+ * <p>DataFusion's optimiser identifies and rewrites equality predicates into hash-join keys
389+ * automatically, so {@code joinOn(right, INNER, "l.id = r.id")} plans equivalently to {@link
390+ * #join(DataFrame, JoinType, String[], String[])} with a single key. Use {@code joinOn} when the
391+ * predicate is not a simple equality, e.g. inequality joins or range conditions.
392+ *
393+ * @throws IllegalArgumentException if {@code right} or {@code type} is {@code null}, or {@code
394+ * predicates} is {@code null} or empty, or any predicate is {@code null}.
395+ * @throws IllegalStateException if either DataFrame is closed or already collected.
396+ * @throws RuntimeException if predicate parsing or join planning fails.
397+ */
398+ public DataFrame joinOn (DataFrame right , JoinType type , String ... predicates ) {
399+ if (right == null ) {
400+ throw new IllegalArgumentException ("joinOn right must be non-null" );
401+ }
402+ if (type == null ) {
403+ throw new IllegalArgumentException ("joinOn type must be non-null" );
404+ }
405+ if (predicates == null || predicates .length == 0 ) {
406+ throw new IllegalArgumentException ("joinOn predicates must be non-null and non-empty" );
407+ }
408+ for (String p : predicates ) {
409+ if (p == null ) {
410+ throw new IllegalArgumentException ("joinOn predicates must not contain null" );
411+ }
412+ }
413+ if (nativeHandle == 0 ) {
414+ throw new IllegalStateException ("DataFrame is closed or already collected" );
415+ }
416+ if (right .nativeHandle == 0 ) {
417+ throw new IllegalStateException ("right DataFrame is closed or already collected" );
418+ }
419+ return new DataFrame (
420+ joinOnDataFrame (nativeHandle , right .nativeHandle , type .code (), predicates ));
421+ }
422+
423+ private void checkJoinArgs (
424+ DataFrame right , JoinType type , String [] leftCols , String [] rightCols ) {
425+ if (right == null ) {
426+ throw new IllegalArgumentException ("join right must be non-null" );
427+ }
428+ if (type == null ) {
429+ throw new IllegalArgumentException ("join type must be non-null" );
430+ }
431+ if (leftCols == null ) {
432+ throw new IllegalArgumentException ("join leftCols must be non-null" );
433+ }
434+ if (rightCols == null ) {
435+ throw new IllegalArgumentException ("join rightCols must be non-null" );
436+ }
437+ if (leftCols .length != rightCols .length ) {
438+ throw new IllegalArgumentException (
439+ "join leftCols and rightCols must have the same length, got "
440+ + leftCols .length
441+ + " and "
442+ + rightCols .length );
443+ }
444+ if (nativeHandle == 0 ) {
445+ throw new IllegalStateException ("DataFrame is closed or already collected" );
446+ }
447+ if (right .nativeHandle == 0 ) {
448+ throw new IllegalStateException ("right DataFrame is closed or already collected" );
449+ }
450+ }
451+
259452 /**
260453 * Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
261454 * unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -318,6 +511,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
318511 writeCsvWithOptions (nativeHandle , path , options .toBytes ());
319512 }
320513
514+ /**
515+ * Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
516+ * directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
517+ * receiver remains usable and must still be closed independently.
518+ *
519+ * @throws RuntimeException if the write fails.
520+ */
521+ public void writeJson (String path ) {
522+ writeJson (path , new JsonWriteOptions ());
523+ }
524+
525+ /**
526+ * Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
527+ * JsonWriteOptions}. The receiver remains usable and must still be closed independently.
528+ *
529+ * @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
530+ * @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
531+ * etc.).
532+ */
533+ public void writeJson (String path , JsonWriteOptions options ) {
534+ if (nativeHandle == 0 ) {
535+ throw new IllegalStateException ("DataFrame is closed or already collected" );
536+ }
537+ if (path == null ) {
538+ throw new IllegalArgumentException ("writeJson path must be non-null" );
539+ }
540+ if (options == null ) {
541+ throw new IllegalArgumentException ("writeJson options must be non-null" );
542+ }
543+ writeJsonWithOptions (nativeHandle , path , options .toBytes ());
544+ }
545+
321546 @ Override
322547 public void close () {
323548 if (nativeHandle != 0 ) {
@@ -334,6 +559,14 @@ public void close() {
334559
335560 private static native long countRows (long handle );
336561
562+ private static native byte [] schemaIpc (long handle );
563+
564+ private static native long explainPlan (long handle , boolean verbose , boolean analyze );
565+
566+ private static native long cachePlan (long handle );
567+
568+ private static native long describePlan (long handle );
569+
337570 private static native void showDataFrame (long handle );
338571
339572 private static native void showDataFrameWithLimit (long handle , int limit );
@@ -354,6 +587,17 @@ public void close() {
354587
355588 private static native long unnestColumns (long handle , String [] columns , boolean preserveNulls );
356589
590+ private static native long joinDataFrame (
591+ long leftHandle ,
592+ long rightHandle ,
593+ byte joinType ,
594+ String [] leftCols ,
595+ String [] rightCols ,
596+ String filter );
597+
598+ private static native long joinOnDataFrame (
599+ long leftHandle , long rightHandle , byte joinType , String [] predicates );
600+
357601 private static native void writeParquetWithOptions (
358602 long handle ,
359603 String path ,
@@ -362,4 +606,6 @@ private static native void writeParquetWithOptions(
362606 boolean singleFileOutputValue );
363607
364608 private static native void writeCsvWithOptions (long handle , String path , byte [] optionsBytes );
609+
610+ private static native void writeJsonWithOptions (long handle , String path , byte [] optionsBytes );
365611}
0 commit comments