Skip to content

Commit 2de8dbb

Browse files
authored
Merge branch 'main' into feat/dataframe-join
2 parents 9067b52 + 7f49541 commit 2de8dbb

13 files changed

Lines changed: 2385 additions & 17 deletions

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,17 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayInputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
2226
import org.apache.arrow.c.ArrowArrayStream;
2327
import org.apache.arrow.c.Data;
2428
import org.apache.arrow.memory.BufferAllocator;
2529
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.apache.arrow.vector.ipc.ReadChannel;
31+
import org.apache.arrow.vector.ipc.message.MessageSerializer;
32+
import org.apache.arrow.vector.types.pojo.Schema;
2633

2734
/**
2835
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113
}
107114
}
108115

116+
/**
117+
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+
* usable and must still be closed independently. Schema inspection does not execute the plan.
119+
*
120+
* <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+
* schema carries no buffer data.
122+
*/
123+
public Schema schema() {
124+
if (nativeHandle == 0) {
125+
throw new IllegalStateException("DataFrame is closed or already collected");
126+
}
127+
byte[] ipcBytes = schemaIpc(nativeHandle);
128+
try {
129+
return MessageSerializer.deserializeSchema(
130+
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
131+
} catch (IOException e) {
132+
throw new RuntimeException("Failed to deserialize IPC schema", e);
133+
}
134+
}
135+
136+
/**
137+
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+
* Non-consuming: the receiver remains usable and must still be closed independently.
139+
*
140+
* <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+
* #show()} or {@link #collect(BufferAllocator)}.
144+
*/
145+
public DataFrame explain(boolean verbose, boolean analyze) {
146+
if (nativeHandle == 0) {
147+
throw new IllegalStateException("DataFrame is closed or already collected");
148+
}
149+
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
150+
}
151+
152+
/**
153+
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+
* Non-consuming: the receiver remains usable and must still be closed independently.
155+
*
156+
* <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+
* downstream queries.
159+
*
160+
* @throws RuntimeException if execution fails.
161+
*/
162+
public DataFrame cache() {
163+
if (nativeHandle == 0) {
164+
throw new IllegalStateException("DataFrame is closed or already collected");
165+
}
166+
return new DataFrame(cachePlan(nativeHandle));
167+
}
168+
169+
/**
170+
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+
* usable and must still be closed independently.
173+
*
174+
* <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+
*
178+
* @throws RuntimeException if execution fails.
179+
*/
180+
public DataFrame describe() {
181+
if (nativeHandle == 0) {
182+
throw new IllegalStateException("DataFrame is closed or already collected");
183+
}
184+
return new DataFrame(describePlan(nativeHandle));
185+
}
186+
109187
/** Execute the plan and return the number of rows. */
110188
public long count() {
111189
if (nativeHandle == 0) {
@@ -481,6 +559,14 @@ public void close() {
481559

482560
private static native long countRows(long handle);
483561

562+
private static native byte[] schemaIpc(long handle);
563+
564+
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
565+
566+
private static native long cachePlan(long handle);
567+
568+
private static native long describePlan(long handle);
569+
484570
private static native void showDataFrame(long handle);
485571

486572
private static native void showDataFrameWithLimit(long handle, int limit);

0 commit comments

Comments
 (0)