1919
2020package org .apache .datafusion ;
2121
22+ import java .io .ByteArrayInputStream ;
23+ import java .io .IOException ;
24+ import java .nio .channels .Channels ;
25+
2226import org .apache .arrow .c .ArrowArrayStream ;
2327import org .apache .arrow .c .Data ;
2428import org .apache .arrow .memory .BufferAllocator ;
2529import org .apache .arrow .vector .ipc .ArrowReader ;
30+ import org .apache .arrow .vector .ipc .ReadChannel ;
31+ import org .apache .arrow .vector .ipc .message .MessageSerializer ;
32+ import org .apache .arrow .vector .types .pojo .Schema ;
2633
2734/**
2835 * A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113 }
107114 }
108115
116+ /**
117+ * Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+ * usable and must still be closed independently. Schema inspection does not execute the plan.
119+ *
120+ * <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+ * schema carries no buffer data.
122+ */
123+ public Schema schema () {
124+ if (nativeHandle == 0 ) {
125+ throw new IllegalStateException ("DataFrame is closed or already collected" );
126+ }
127+ byte [] ipcBytes = schemaIpc (nativeHandle );
128+ try {
129+ return MessageSerializer .deserializeSchema (
130+ new ReadChannel (Channels .newChannel (new ByteArrayInputStream (ipcBytes ))));
131+ } catch (IOException e ) {
132+ throw new RuntimeException ("Failed to deserialize IPC schema" , e );
133+ }
134+ }
135+
136+ /**
137+ * Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+ * Non-consuming: the receiver remains usable and must still be closed independently.
139+ *
140+ * <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+ * contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+ * rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+ * #show()} or {@link #collect(BufferAllocator)}.
144+ */
145+ public DataFrame explain (boolean verbose , boolean analyze ) {
146+ if (nativeHandle == 0 ) {
147+ throw new IllegalStateException ("DataFrame is closed or already collected" );
148+ }
149+ return new DataFrame (explainPlan (nativeHandle , verbose , analyze ));
150+ }
151+
152+ /**
153+ * Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+ * Non-consuming: the receiver remains usable and must still be closed independently.
155+ *
156+ * <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+ * DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+ * downstream queries.
159+ *
160+ * @throws RuntimeException if execution fails.
161+ */
162+ public DataFrame cache () {
163+ if (nativeHandle == 0 ) {
164+ throw new IllegalStateException ("DataFrame is closed or already collected" );
165+ }
166+ return new DataFrame (cachePlan (nativeHandle ));
167+ }
168+
169+ /**
170+ * Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+ * DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+ * usable and must still be closed independently.
173+ *
174+ * <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+ * the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+ * count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+ *
178+ * @throws RuntimeException if execution fails.
179+ */
180+ public DataFrame describe () {
181+ if (nativeHandle == 0 ) {
182+ throw new IllegalStateException ("DataFrame is closed or already collected" );
183+ }
184+ return new DataFrame (describePlan (nativeHandle ));
185+ }
186+
109187 /** Execute the plan and return the number of rows. */
110188 public long count () {
111189 if (nativeHandle == 0 ) {
@@ -405,6 +483,121 @@ private long otherHandle(String op, DataFrame other) {
405483 return other .nativeHandle ;
406484 }
407485
486+ /**
487+ * Equi-join this DataFrame with {@code right} on the named columns, using the given {@link
488+ * JoinType}. The receiver and {@code right} both remain usable and must still be closed
489+ * independently.
490+ *
491+ * <p>Equivalent to SQL {@code left <type> JOIN right ON l.leftCols[0] = r.rightCols[0] AND ...}.
492+ * {@code leftCols} and {@code rightCols} must have the same length.
493+ *
494+ * @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
495+ * rightCols.length}.
496+ * @throws IllegalStateException if either DataFrame is closed or already collected.
497+ * @throws RuntimeException if join planning fails (column collision in the combined schema,
498+ * unknown column names, etc.).
499+ */
500+ public DataFrame join (DataFrame right , JoinType type , String [] leftCols , String [] rightCols ) {
501+ checkJoinArgs (right , type , leftCols , rightCols );
502+ return new DataFrame (
503+ joinDataFrame (nativeHandle , right .nativeHandle , type .code (), leftCols , rightCols , null ));
504+ }
505+
506+ /**
507+ * Equi-join this DataFrame with {@code right}, restricting the result with a residual SQL filter
508+ * parsed against the <em>combined</em> schema (left columns followed by right columns; columns
509+ * may be qualified with the relation alias when ambiguous). The receiver and {@code right} both
510+ * remain usable and must still be closed independently.
511+ *
512+ * <p>For outer joins, the filter is applied only to matched rows; unmatched rows are passed
513+ * through with nulls on the unmatched side, matching DataFusion's semantics.
514+ *
515+ * @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
516+ * rightCols.length}.
517+ * @throws IllegalStateException if either DataFrame is closed or already collected.
518+ * @throws RuntimeException if join planning or filter parsing fails.
519+ */
520+ public DataFrame join (
521+ DataFrame right , JoinType type , String [] leftCols , String [] rightCols , String filter ) {
522+ checkJoinArgs (right , type , leftCols , rightCols );
523+ if (filter == null ) {
524+ throw new IllegalArgumentException ("join filter must be non-null" );
525+ }
526+ return new DataFrame (
527+ joinDataFrame (nativeHandle , right .nativeHandle , type .code (), leftCols , rightCols , filter ));
528+ }
529+
530+ /**
531+ * Join this DataFrame with {@code right} using arbitrary SQL predicates parsed against the
532+ * <em>combined</em> schema. Each predicate is parsed independently and the join evaluates their
533+ * conjunction. Predicates may reference columns from either side and may be qualified with the
534+ * relation alias when ambiguous (e.g. {@code "left.x = right.x"}). The receiver and {@code right}
535+ * both remain usable and must still be closed independently.
536+ *
537+ * <p>DataFusion's optimiser identifies and rewrites equality predicates into hash-join keys
538+ * automatically, so {@code joinOn(right, INNER, "l.id = r.id")} plans equivalently to {@link
539+ * #join(DataFrame, JoinType, String[], String[])} with a single key. Use {@code joinOn} when the
540+ * predicate is not a simple equality, e.g. inequality joins or range conditions.
541+ *
542+ * @throws IllegalArgumentException if {@code right} or {@code type} is {@code null}, or {@code
543+ * predicates} is {@code null} or empty, or any predicate is {@code null}.
544+ * @throws IllegalStateException if either DataFrame is closed or already collected.
545+ * @throws RuntimeException if predicate parsing or join planning fails.
546+ */
547+ public DataFrame joinOn (DataFrame right , JoinType type , String ... predicates ) {
548+ if (right == null ) {
549+ throw new IllegalArgumentException ("joinOn right must be non-null" );
550+ }
551+ if (type == null ) {
552+ throw new IllegalArgumentException ("joinOn type must be non-null" );
553+ }
554+ if (predicates == null || predicates .length == 0 ) {
555+ throw new IllegalArgumentException ("joinOn predicates must be non-null and non-empty" );
556+ }
557+ for (String p : predicates ) {
558+ if (p == null ) {
559+ throw new IllegalArgumentException ("joinOn predicates must not contain null" );
560+ }
561+ }
562+ if (nativeHandle == 0 ) {
563+ throw new IllegalStateException ("DataFrame is closed or already collected" );
564+ }
565+ if (right .nativeHandle == 0 ) {
566+ throw new IllegalStateException ("right DataFrame is closed or already collected" );
567+ }
568+ return new DataFrame (
569+ joinOnDataFrame (nativeHandle , right .nativeHandle , type .code (), predicates ));
570+ }
571+
572+ private void checkJoinArgs (
573+ DataFrame right , JoinType type , String [] leftCols , String [] rightCols ) {
574+ if (right == null ) {
575+ throw new IllegalArgumentException ("join right must be non-null" );
576+ }
577+ if (type == null ) {
578+ throw new IllegalArgumentException ("join type must be non-null" );
579+ }
580+ if (leftCols == null ) {
581+ throw new IllegalArgumentException ("join leftCols must be non-null" );
582+ }
583+ if (rightCols == null ) {
584+ throw new IllegalArgumentException ("join rightCols must be non-null" );
585+ }
586+ if (leftCols .length != rightCols .length ) {
587+ throw new IllegalArgumentException (
588+ "join leftCols and rightCols must have the same length, got "
589+ + leftCols .length
590+ + " and "
591+ + rightCols .length );
592+ }
593+ if (nativeHandle == 0 ) {
594+ throw new IllegalStateException ("DataFrame is closed or already collected" );
595+ }
596+ if (right .nativeHandle == 0 ) {
597+ throw new IllegalStateException ("right DataFrame is closed or already collected" );
598+ }
599+ }
600+
408601 /**
409602 * Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
410603 * unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -467,6 +660,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
467660 writeCsvWithOptions (nativeHandle , path , options .toBytes ());
468661 }
469662
663+ /**
664+ * Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
665+ * directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
666+ * receiver remains usable and must still be closed independently.
667+ *
668+ * @throws RuntimeException if the write fails.
669+ */
670+ public void writeJson (String path ) {
671+ writeJson (path , new JsonWriteOptions ());
672+ }
673+
674+ /**
675+ * Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
676+ * JsonWriteOptions}. The receiver remains usable and must still be closed independently.
677+ *
678+ * @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
679+ * @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
680+ * etc.).
681+ */
682+ public void writeJson (String path , JsonWriteOptions options ) {
683+ if (nativeHandle == 0 ) {
684+ throw new IllegalStateException ("DataFrame is closed or already collected" );
685+ }
686+ if (path == null ) {
687+ throw new IllegalArgumentException ("writeJson path must be non-null" );
688+ }
689+ if (options == null ) {
690+ throw new IllegalArgumentException ("writeJson options must be non-null" );
691+ }
692+ writeJsonWithOptions (nativeHandle , path , options .toBytes ());
693+ }
694+
470695 @ Override
471696 public void close () {
472697 if (nativeHandle != 0 ) {
@@ -483,6 +708,14 @@ public void close() {
483708
484709 private static native long countRows (long handle );
485710
711+ private static native byte [] schemaIpc (long handle );
712+
713+ private static native long explainPlan (long handle , boolean verbose , boolean analyze );
714+
715+ private static native long cachePlan (long handle );
716+
717+ private static native long describePlan (long handle );
718+
486719 private static native void showDataFrame (long handle );
487720
488721 private static native void showDataFrameWithLimit (long handle , int limit );
@@ -519,6 +752,17 @@ public void close() {
519752
520753 private static native long exceptDistinctRows (long handle , long otherHandle );
521754
755+ private static native long joinDataFrame (
756+ long leftHandle ,
757+ long rightHandle ,
758+ byte joinType ,
759+ String [] leftCols ,
760+ String [] rightCols ,
761+ String filter );
762+
763+ private static native long joinOnDataFrame (
764+ long leftHandle , long rightHandle , byte joinType , String [] predicates );
765+
522766 private static native void writeParquetWithOptions (
523767 long handle ,
524768 String path ,
@@ -527,4 +771,6 @@ private static native void writeParquetWithOptions(
527771 boolean singleFileOutputValue );
528772
529773 private static native void writeCsvWithOptions (long handle , String path , byte [] optionsBytes );
774+
775+ private static native void writeJsonWithOptions (long handle , String path , byte [] optionsBytes );
530776}
0 commit comments