Skip to content

Commit 6e05e35

Browse files
committed
feat(dataframe): add schema, explain, cache, and describe introspection methods
1 parent 89d5496 commit 6e05e35

3 files changed

Lines changed: 402 additions & 1 deletion

File tree

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,17 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayInputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
2226
import org.apache.arrow.c.ArrowArrayStream;
2327
import org.apache.arrow.c.Data;
2428
import org.apache.arrow.memory.BufferAllocator;
2529
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.apache.arrow.vector.ipc.ReadChannel;
31+
import org.apache.arrow.vector.ipc.message.MessageSerializer;
32+
import org.apache.arrow.vector.types.pojo.Schema;
2633

2734
/**
2835
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113
}
107114
}
108115

116+
/**
117+
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+
* usable and must still be closed independently. Schema inspection does not execute the plan.
119+
*
120+
* <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+
* schema carries no buffer data.
122+
*/
123+
public Schema schema() {
124+
if (nativeHandle == 0) {
125+
throw new IllegalStateException("DataFrame is closed or already collected");
126+
}
127+
byte[] ipcBytes = schemaIpc(nativeHandle);
128+
try {
129+
return MessageSerializer.deserializeSchema(
130+
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
131+
} catch (IOException e) {
132+
throw new RuntimeException("Failed to deserialize IPC schema", e);
133+
}
134+
}
135+
136+
/**
137+
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+
* Non-consuming: the receiver remains usable and must still be closed independently.
139+
*
140+
* <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+
* #show()} or {@link #collect(BufferAllocator)}.
144+
*/
145+
public DataFrame explain(boolean verbose, boolean analyze) {
146+
if (nativeHandle == 0) {
147+
throw new IllegalStateException("DataFrame is closed or already collected");
148+
}
149+
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
150+
}
151+
152+
/**
153+
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+
* Non-consuming: the receiver remains usable and must still be closed independently.
155+
*
156+
* <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+
* downstream queries.
159+
*
160+
* @throws RuntimeException if execution fails.
161+
*/
162+
public DataFrame cache() {
163+
if (nativeHandle == 0) {
164+
throw new IllegalStateException("DataFrame is closed or already collected");
165+
}
166+
return new DataFrame(cachePlan(nativeHandle));
167+
}
168+
169+
/**
170+
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+
* usable and must still be closed independently.
173+
*
174+
* <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+
*
178+
* @throws RuntimeException if execution fails.
179+
*/
180+
public DataFrame describe() {
181+
if (nativeHandle == 0) {
182+
throw new IllegalStateException("DataFrame is closed or already collected");
183+
}
184+
return new DataFrame(describePlan(nativeHandle));
185+
}
186+
109187
/** Execute the plan and return the number of rows. */
110188
public long count() {
111189
if (nativeHandle == 0) {
@@ -366,6 +444,14 @@ public void close() {
366444

367445
private static native long countRows(long handle);
368446

447+
private static native byte[] schemaIpc(long handle);
448+
449+
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
450+
451+
private static native long cachePlan(long handle);
452+
453+
private static native long describePlan(long handle);
454+
369455
private static native void showDataFrame(long handle);
370456

371457
private static native void showDataFrameWithLimit(long handle, int limit);
Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertNotNull;
24+
import static org.junit.jupiter.api.Assertions.assertThrows;
25+
import static org.junit.jupiter.api.Assertions.assertTrue;
26+
27+
import java.util.HashSet;
28+
import java.util.Set;
29+
30+
import org.apache.arrow.memory.BufferAllocator;
31+
import org.apache.arrow.memory.RootAllocator;
32+
import org.apache.arrow.vector.VarCharVector;
33+
import org.apache.arrow.vector.VectorSchemaRoot;
34+
import org.apache.arrow.vector.ipc.ArrowReader;
35+
import org.apache.arrow.vector.types.pojo.ArrowType;
36+
import org.apache.arrow.vector.types.pojo.Schema;
37+
import org.junit.jupiter.api.Test;
38+
39+
class DataFrameIntrospectionTest {
40+
@Test
41+
void schemaReturnsArrowSchema() {
42+
try (SessionContext ctx = new SessionContext();
43+
DataFrame df = ctx.sql("SELECT 1 AS x, 'foo' AS s")) {
44+
Schema schema = df.schema();
45+
assertEquals(2, schema.getFields().size());
46+
assertEquals("x", schema.getFields().get(0).getName());
47+
assertTrue(schema.getFields().get(0).getType() instanceof ArrowType.Int);
48+
assertEquals("s", schema.getFields().get(1).getName());
49+
assertTrue(schema.getFields().get(1).getType() instanceof ArrowType.Utf8);
50+
}
51+
}
52+
53+
@Test
54+
void schemaPreservesReceiver() {
55+
try (SessionContext ctx = new SessionContext();
56+
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
57+
Schema schema = df.schema();
58+
assertNotNull(schema);
59+
// Receiver still usable after schema().
60+
assertEquals(3L, df.count());
61+
}
62+
}
63+
64+
@Test
65+
void schemaThrowsAfterClose() {
66+
try (SessionContext ctx = new SessionContext()) {
67+
DataFrame df = ctx.sql("SELECT 1");
68+
df.close();
69+
assertThrows(IllegalStateException.class, df::schema);
70+
}
71+
}
72+
73+
@Test
74+
void explainReturnsHumanReadablePlan() throws Exception {
75+
try (BufferAllocator allocator = new RootAllocator();
76+
SessionContext ctx = new SessionContext();
77+
DataFrame df = ctx.sql("SELECT x FROM (VALUES (1), (2), (3)) AS t(x) WHERE x > 1");
78+
DataFrame explained = df.explain(false, false);
79+
ArrowReader reader = explained.collect(allocator)) {
80+
StringBuilder labels = new StringBuilder();
81+
StringBuilder bodies = new StringBuilder();
82+
while (reader.loadNextBatch()) {
83+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
84+
// Column 0 = plan-type label, Column 1 = plan text.
85+
VarCharVector planType = (VarCharVector) root.getVector(0);
86+
VarCharVector planText = (VarCharVector) root.getVector(1);
87+
for (int i = 0; i < root.getRowCount(); i++) {
88+
labels.append(new String(planType.get(i))).append('\n');
89+
bodies.append(new String(planText.get(i))).append('\n');
90+
}
91+
}
92+
String labelsDump = labels.toString();
93+
String bodiesDump = bodies.toString();
94+
assertTrue(labelsDump.contains("logical_plan"), () -> "labels: " + labelsDump);
95+
assertTrue(bodiesDump.contains("Filter"), () -> "bodies: " + bodiesDump);
96+
assertTrue(bodiesDump.contains("Projection"), () -> "bodies: " + bodiesDump);
97+
}
98+
}
99+
100+
@Test
101+
void explainVerboseAnalyzeProducesMoreRowsThanLazy() throws Exception {
102+
try (BufferAllocator allocator = new RootAllocator();
103+
SessionContext ctx = new SessionContext()) {
104+
long lazyRows;
105+
try (DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)");
106+
DataFrame e = df.explain(false, false);
107+
ArrowReader reader = e.collect(allocator)) {
108+
lazyRows = countAllRows(reader);
109+
}
110+
long verboseRows;
111+
try (DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)");
112+
DataFrame e = df.explain(true, true);
113+
ArrowReader reader = e.collect(allocator)) {
114+
verboseRows = countAllRows(reader);
115+
}
116+
assertTrue(
117+
verboseRows > lazyRows,
118+
() -> "verbose+analyze rows " + verboseRows + " not greater than lazy " + lazyRows);
119+
}
120+
}
121+
122+
@Test
123+
void explainPreservesReceiver() {
124+
try (SessionContext ctx = new SessionContext();
125+
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)")) {
126+
try (DataFrame e = df.explain(false, false)) {
127+
// Just ensure it's a valid handle.
128+
assertNotNull(e.schema());
129+
}
130+
// Receiver still usable.
131+
assertEquals(2L, df.count());
132+
}
133+
}
134+
135+
@Test
136+
void explainThrowsAfterClose() {
137+
try (SessionContext ctx = new SessionContext()) {
138+
DataFrame df = ctx.sql("SELECT 1");
139+
df.close();
140+
assertThrows(IllegalStateException.class, () -> df.explain(false, false));
141+
}
142+
}
143+
144+
@Test
145+
void cacheMaterializesPlan() {
146+
try (SessionContext ctx = new SessionContext();
147+
DataFrame df = ctx.sql("SELECT x FROM (VALUES (1), (2), (3), (4)) AS t(x) WHERE x > 1");
148+
DataFrame cached = df.cache()) {
149+
assertEquals(3L, cached.count());
150+
// Schema is preserved after caching.
151+
Schema schema = cached.schema();
152+
assertEquals(1, schema.getFields().size());
153+
assertEquals("x", schema.getFields().get(0).getName());
154+
}
155+
}
156+
157+
@Test
158+
void cachePreservesReceiver() {
159+
try (SessionContext ctx = new SessionContext();
160+
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2)) AS t(x)")) {
161+
try (DataFrame cached = df.cache()) {
162+
assertEquals(2L, cached.count());
163+
}
164+
// Receiver still usable after cache().
165+
assertEquals(2L, df.count());
166+
}
167+
}
168+
169+
@Test
170+
void cacheThrowsAfterClose() {
171+
try (SessionContext ctx = new SessionContext()) {
172+
DataFrame df = ctx.sql("SELECT 1");
173+
df.close();
174+
assertThrows(IllegalStateException.class, df::cache);
175+
}
176+
}
177+
178+
@Test
179+
void describeReturnsSummaryStats() throws Exception {
180+
try (BufferAllocator allocator = new RootAllocator();
181+
SessionContext ctx = new SessionContext();
182+
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1, 'a'), (2, 'b'), (3, 'c')) AS t(x, s)");
183+
DataFrame stats = df.describe()) {
184+
// First column of describe() output is named "describe" and lists the statistic name.
185+
Schema schema = stats.schema();
186+
assertEquals("describe", schema.getFields().get(0).getName());
187+
188+
Set<String> seen = new HashSet<>();
189+
try (ArrowReader reader = stats.collect(allocator)) {
190+
while (reader.loadNextBatch()) {
191+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
192+
VarCharVector labels = (VarCharVector) root.getVector(0);
193+
for (int i = 0; i < root.getRowCount(); i++) {
194+
seen.add(new String(labels.get(i)));
195+
}
196+
}
197+
}
198+
// DataFusion 53.1 reports these seven labels.
199+
assertTrue(seen.contains("count"), () -> "labels=" + seen);
200+
assertTrue(seen.contains("null_count"), () -> "labels=" + seen);
201+
assertTrue(seen.contains("mean"), () -> "labels=" + seen);
202+
assertTrue(seen.contains("std"), () -> "labels=" + seen);
203+
assertTrue(seen.contains("min"), () -> "labels=" + seen);
204+
assertTrue(seen.contains("max"), () -> "labels=" + seen);
205+
assertTrue(seen.contains("median"), () -> "labels=" + seen);
206+
}
207+
}
208+
209+
@Test
210+
void describePreservesReceiver() {
211+
try (SessionContext ctx = new SessionContext();
212+
DataFrame df = ctx.sql("SELECT * FROM (VALUES (1), (2), (3)) AS t(x)")) {
213+
try (DataFrame stats = df.describe()) {
214+
assertNotNull(stats.schema());
215+
}
216+
// Receiver still usable after describe().
217+
assertEquals(3L, df.count());
218+
}
219+
}
220+
221+
@Test
222+
void describeThrowsAfterClose() {
223+
try (SessionContext ctx = new SessionContext()) {
224+
DataFrame df = ctx.sql("SELECT 1");
225+
df.close();
226+
assertThrows(IllegalStateException.class, df::describe);
227+
}
228+
}
229+
230+
private static long countAllRows(ArrowReader reader) throws Exception {
231+
long n = 0;
232+
while (reader.loadNextBatch()) {
233+
n += reader.getVectorSchemaRoot().getRowCount();
234+
}
235+
return n;
236+
}
237+
}

0 commit comments

Comments
 (0)