Skip to content

Commit e74dd44

Browse files
authored
Merge branch 'main' into feat/dataframe-cancellation
2 parents 4bd2ad1 + 7f49541 commit e74dd44

13 files changed

Lines changed: 2385 additions & 17 deletions

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,17 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayInputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
2226
import org.apache.arrow.c.ArrowArrayStream;
2327
import org.apache.arrow.c.Data;
2428
import org.apache.arrow.memory.BufferAllocator;
2529
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.apache.arrow.vector.ipc.ReadChannel;
31+
import org.apache.arrow.vector.ipc.message.MessageSerializer;
32+
import org.apache.arrow.vector.types.pojo.Schema;
2633

2734
/**
2835
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -170,6 +177,77 @@ private static long resolveTokenHandle(CancellationToken token) {
170177
return handle;
171178
}
172179

180+
/**
181+
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
182+
* usable and must still be closed independently. Schema inspection does not execute the plan.
183+
*
184+
* <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
185+
* schema carries no buffer data.
186+
*/
187+
public Schema schema() {
188+
if (nativeHandle == 0) {
189+
throw new IllegalStateException("DataFrame is closed or already collected");
190+
}
191+
byte[] ipcBytes = schemaIpc(nativeHandle);
192+
try {
193+
return MessageSerializer.deserializeSchema(
194+
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
195+
} catch (IOException e) {
196+
throw new RuntimeException("Failed to deserialize IPC schema", e);
197+
}
198+
}
199+
200+
/**
201+
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
202+
* Non-consuming: the receiver remains usable and must still be closed independently.
203+
*
204+
* <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
205+
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
206+
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
207+
* #show()} or {@link #collect(BufferAllocator)}.
208+
*/
209+
public DataFrame explain(boolean verbose, boolean analyze) {
210+
if (nativeHandle == 0) {
211+
throw new IllegalStateException("DataFrame is closed or already collected");
212+
}
213+
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
214+
}
215+
216+
/**
217+
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
218+
* Non-consuming: the receiver remains usable and must still be closed independently.
219+
*
220+
* <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
221+
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
222+
* downstream queries.
223+
*
224+
* @throws RuntimeException if execution fails.
225+
*/
226+
public DataFrame cache() {
227+
if (nativeHandle == 0) {
228+
throw new IllegalStateException("DataFrame is closed or already collected");
229+
}
230+
return new DataFrame(cachePlan(nativeHandle));
231+
}
232+
233+
/**
234+
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
235+
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
236+
* usable and must still be closed independently.
237+
*
238+
* <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
239+
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
240+
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
241+
*
242+
* @throws RuntimeException if execution fails.
243+
*/
244+
public DataFrame describe() {
245+
if (nativeHandle == 0) {
246+
throw new IllegalStateException("DataFrame is closed or already collected");
247+
}
248+
return new DataFrame(describePlan(nativeHandle));
249+
}
250+
173251
/** Execute the plan and return the number of rows. */
174252
public long count() {
175253
if (nativeHandle == 0) {
@@ -431,6 +509,14 @@ private static native void executeStreamDataFrame(
431509

432510
private static native long countRows(long handle);
433511

512+
private static native byte[] schemaIpc(long handle);
513+
514+
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
515+
516+
private static native long cachePlan(long handle);
517+
518+
private static native long describePlan(long handle);
519+
434520
private static native void showDataFrame(long handle);
435521

436522
private static native void showDataFrameWithLimit(long handle, int limit);

0 commit comments

Comments
 (0)