Skip to content

Commit 2421315

Browse files
authored
Merge branch 'main' into fix/udf-nested-arrow-types
2 parents 305da43 + af78e61 commit 2421315

45 files changed

Lines changed: 6183 additions & 66 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ under the License.
5757
<groupId>com.google.protobuf</groupId>
5858
<artifactId>protobuf-java</artifactId>
5959
</dependency>
60+
<dependency>
61+
<groupId>org.apache.avro</groupId>
62+
<artifactId>avro</artifactId>
63+
<scope>test</scope>
64+
</dependency>
6065
</dependencies>
6166

6267
<build>
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
import org.apache.datafusion.protobuf.AvroReadOptionsProto;
24+
25+
/**
26+
* Configuration knobs for Avro sources passed to {@link SessionContext#registerAvro(String, String,
27+
* AvroReadOptions)} and {@link SessionContext#readAvro(String, AvroReadOptions)}.
28+
*
29+
* <p>Mirrors the subset of DataFusion's {@code AvroReadOptions} that maps onto the Java surface
30+
* today: {@code fileExtension} (default {@code ".avro"}) and an explicit Arrow {@code schema} that
31+
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred -- no
32+
* other Java reader exposes Hive-style partitioning yet.
33+
*
34+
* <p>Avro carries its own per-block compression (snappy, deflate, bzip2, xz, zstandard) inside the
35+
* object container itself, negotiated when the file is written, so unlike CSV / NDJSON there is no
36+
* {@code FileCompressionType} setter.
37+
*/
38+
public final class AvroReadOptions {
39+
40+
private String fileExtension = ".avro";
41+
private Schema schema;
42+
43+
public AvroReadOptions fileExtension(String ext) {
44+
this.fileExtension = ext;
45+
return this;
46+
}
47+
48+
public AvroReadOptions schema(Schema schema) {
49+
this.schema = schema;
50+
return this;
51+
}
52+
53+
byte[] toBytes() {
54+
return AvroReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
55+
}
56+
57+
Schema schema() {
58+
return schema;
59+
}
60+
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,17 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayInputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
2226
import org.apache.arrow.c.ArrowArrayStream;
2327
import org.apache.arrow.c.Data;
2428
import org.apache.arrow.memory.BufferAllocator;
2529
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.apache.arrow.vector.ipc.ReadChannel;
31+
import org.apache.arrow.vector.ipc.message.MessageSerializer;
32+
import org.apache.arrow.vector.types.pojo.Schema;
2633

2734
/**
2835
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113
}
107114
}
108115

116+
/**
117+
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+
* usable and must still be closed independently. Schema inspection does not execute the plan.
119+
*
120+
* <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+
* schema carries no buffer data.
122+
*/
123+
public Schema schema() {
124+
if (nativeHandle == 0) {
125+
throw new IllegalStateException("DataFrame is closed or already collected");
126+
}
127+
byte[] ipcBytes = schemaIpc(nativeHandle);
128+
try {
129+
return MessageSerializer.deserializeSchema(
130+
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
131+
} catch (IOException e) {
132+
throw new RuntimeException("Failed to deserialize IPC schema", e);
133+
}
134+
}
135+
136+
/**
137+
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+
* Non-consuming: the receiver remains usable and must still be closed independently.
139+
*
140+
* <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+
* #show()} or {@link #collect(BufferAllocator)}.
144+
*/
145+
public DataFrame explain(boolean verbose, boolean analyze) {
146+
if (nativeHandle == 0) {
147+
throw new IllegalStateException("DataFrame is closed or already collected");
148+
}
149+
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
150+
}
151+
152+
/**
153+
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+
* Non-consuming: the receiver remains usable and must still be closed independently.
155+
*
156+
* <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+
* downstream queries.
159+
*
160+
* @throws RuntimeException if execution fails.
161+
*/
162+
public DataFrame cache() {
163+
if (nativeHandle == 0) {
164+
throw new IllegalStateException("DataFrame is closed or already collected");
165+
}
166+
return new DataFrame(cachePlan(nativeHandle));
167+
}
168+
169+
/**
170+
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+
* usable and must still be closed independently.
173+
*
174+
* <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+
*
178+
* @throws RuntimeException if execution fails.
179+
*/
180+
public DataFrame describe() {
181+
if (nativeHandle == 0) {
182+
throw new IllegalStateException("DataFrame is closed or already collected");
183+
}
184+
return new DataFrame(describePlan(nativeHandle));
185+
}
186+
109187
/** Execute the plan and return the number of rows. */
110188
public long count() {
111189
if (nativeHandle == 0) {
@@ -256,6 +334,121 @@ public DataFrame unnestColumns(UnnestOptions options, String... columns) {
256334
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
257335
}
258336

337+
/**
338+
* Equi-join this DataFrame with {@code right} on the named columns, using the given {@link
339+
* JoinType}. The receiver and {@code right} both remain usable and must still be closed
340+
* independently.
341+
*
342+
* <p>Equivalent to SQL {@code left <type> JOIN right ON l.leftCols[0] = r.rightCols[0] AND ...}.
343+
* {@code leftCols} and {@code rightCols} must have the same length.
344+
*
345+
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
346+
* rightCols.length}.
347+
* @throws IllegalStateException if either DataFrame is closed or already collected.
348+
* @throws RuntimeException if join planning fails (column collision in the combined schema,
349+
* unknown column names, etc.).
350+
*/
351+
public DataFrame join(DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
352+
checkJoinArgs(right, type, leftCols, rightCols);
353+
return new DataFrame(
354+
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, null));
355+
}
356+
357+
/**
358+
* Equi-join this DataFrame with {@code right}, restricting the result with a residual SQL filter
359+
* parsed against the <em>combined</em> schema (left columns followed by right columns; columns
360+
* may be qualified with the relation alias when ambiguous). The receiver and {@code right} both
361+
* remain usable and must still be closed independently.
362+
*
363+
* <p>For outer joins, the filter is applied only to matched rows; unmatched rows are passed
364+
* through with nulls on the unmatched side, matching DataFusion's semantics.
365+
*
366+
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
367+
* rightCols.length}.
368+
* @throws IllegalStateException if either DataFrame is closed or already collected.
369+
* @throws RuntimeException if join planning or filter parsing fails.
370+
*/
371+
public DataFrame join(
372+
DataFrame right, JoinType type, String[] leftCols, String[] rightCols, String filter) {
373+
checkJoinArgs(right, type, leftCols, rightCols);
374+
if (filter == null) {
375+
throw new IllegalArgumentException("join filter must be non-null");
376+
}
377+
return new DataFrame(
378+
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, filter));
379+
}
380+
381+
/**
382+
* Join this DataFrame with {@code right} using arbitrary SQL predicates parsed against the
383+
* <em>combined</em> schema. Each predicate is parsed independently and the join evaluates their
384+
* conjunction. Predicates may reference columns from either side and may be qualified with the
385+
* relation alias when ambiguous (e.g. {@code "left.x = right.x"}). The receiver and {@code right}
386+
* both remain usable and must still be closed independently.
387+
*
388+
* <p>DataFusion's optimiser identifies and rewrites equality predicates into hash-join keys
389+
* automatically, so {@code joinOn(right, INNER, "l.id = r.id")} plans equivalently to {@link
390+
* #join(DataFrame, JoinType, String[], String[])} with a single key. Use {@code joinOn} when the
391+
* predicate is not a simple equality, e.g. inequality joins or range conditions.
392+
*
393+
* @throws IllegalArgumentException if {@code right} or {@code type} is {@code null}, or {@code
394+
* predicates} is {@code null} or empty, or any predicate is {@code null}.
395+
* @throws IllegalStateException if either DataFrame is closed or already collected.
396+
* @throws RuntimeException if predicate parsing or join planning fails.
397+
*/
398+
public DataFrame joinOn(DataFrame right, JoinType type, String... predicates) {
399+
if (right == null) {
400+
throw new IllegalArgumentException("joinOn right must be non-null");
401+
}
402+
if (type == null) {
403+
throw new IllegalArgumentException("joinOn type must be non-null");
404+
}
405+
if (predicates == null || predicates.length == 0) {
406+
throw new IllegalArgumentException("joinOn predicates must be non-null and non-empty");
407+
}
408+
for (String p : predicates) {
409+
if (p == null) {
410+
throw new IllegalArgumentException("joinOn predicates must not contain null");
411+
}
412+
}
413+
if (nativeHandle == 0) {
414+
throw new IllegalStateException("DataFrame is closed or already collected");
415+
}
416+
if (right.nativeHandle == 0) {
417+
throw new IllegalStateException("right DataFrame is closed or already collected");
418+
}
419+
return new DataFrame(
420+
joinOnDataFrame(nativeHandle, right.nativeHandle, type.code(), predicates));
421+
}
422+
423+
private void checkJoinArgs(
424+
DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
425+
if (right == null) {
426+
throw new IllegalArgumentException("join right must be non-null");
427+
}
428+
if (type == null) {
429+
throw new IllegalArgumentException("join type must be non-null");
430+
}
431+
if (leftCols == null) {
432+
throw new IllegalArgumentException("join leftCols must be non-null");
433+
}
434+
if (rightCols == null) {
435+
throw new IllegalArgumentException("join rightCols must be non-null");
436+
}
437+
if (leftCols.length != rightCols.length) {
438+
throw new IllegalArgumentException(
439+
"join leftCols and rightCols must have the same length, got "
440+
+ leftCols.length
441+
+ " and "
442+
+ rightCols.length);
443+
}
444+
if (nativeHandle == 0) {
445+
throw new IllegalStateException("DataFrame is closed or already collected");
446+
}
447+
if (right.nativeHandle == 0) {
448+
throw new IllegalStateException("right DataFrame is closed or already collected");
449+
}
450+
}
451+
259452
/**
260453
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
261454
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -318,6 +511,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
318511
writeCsvWithOptions(nativeHandle, path, options.toBytes());
319512
}
320513

514+
/**
515+
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
516+
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
517+
* receiver remains usable and must still be closed independently.
518+
*
519+
* @throws RuntimeException if the write fails.
520+
*/
521+
public void writeJson(String path) {
522+
writeJson(path, new JsonWriteOptions());
523+
}
524+
525+
/**
526+
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
527+
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
528+
*
529+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
530+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
531+
* etc.).
532+
*/
533+
public void writeJson(String path, JsonWriteOptions options) {
534+
if (nativeHandle == 0) {
535+
throw new IllegalStateException("DataFrame is closed or already collected");
536+
}
537+
if (path == null) {
538+
throw new IllegalArgumentException("writeJson path must be non-null");
539+
}
540+
if (options == null) {
541+
throw new IllegalArgumentException("writeJson options must be non-null");
542+
}
543+
writeJsonWithOptions(nativeHandle, path, options.toBytes());
544+
}
545+
321546
@Override
322547
public void close() {
323548
if (nativeHandle != 0) {
@@ -334,6 +559,14 @@ public void close() {
334559

335560
private static native long countRows(long handle);
336561

562+
private static native byte[] schemaIpc(long handle);
563+
564+
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
565+
566+
private static native long cachePlan(long handle);
567+
568+
private static native long describePlan(long handle);
569+
337570
private static native void showDataFrame(long handle);
338571

339572
private static native void showDataFrameWithLimit(long handle, int limit);
@@ -354,6 +587,17 @@ public void close() {
354587

355588
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
356589

590+
private static native long joinDataFrame(
591+
long leftHandle,
592+
long rightHandle,
593+
byte joinType,
594+
String[] leftCols,
595+
String[] rightCols,
596+
String filter);
597+
598+
private static native long joinOnDataFrame(
599+
long leftHandle, long rightHandle, byte joinType, String[] predicates);
600+
357601
private static native void writeParquetWithOptions(
358602
long handle,
359603
String path,
@@ -362,4 +606,6 @@ private static native void writeParquetWithOptions(
362606
boolean singleFileOutputValue);
363607

364608
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
609+
610+
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
365611
}

0 commit comments

Comments
 (0)