Skip to content

Commit ffc48ce

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/dataframe-set-operations
2 parents 24ad482 + af78e61 commit ffc48ce

45 files changed

Lines changed: 6183 additions & 66 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎core/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ under the License.
5757
<groupId>com.google.protobuf</groupId>
5858
<artifactId>protobuf-java</artifactId>
5959
</dependency>
60+
<dependency>
61+
<groupId>org.apache.avro</groupId>
62+
<artifactId>avro</artifactId>
63+
<scope>test</scope>
64+
</dependency>
6065
</dependencies>
6166

6267
<build>
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
import org.apache.datafusion.protobuf.AvroReadOptionsProto;
24+
25+
/**
26+
* Configuration knobs for Avro sources passed to {@link SessionContext#registerAvro(String, String,
27+
* AvroReadOptions)} and {@link SessionContext#readAvro(String, AvroReadOptions)}.
28+
*
29+
* <p>Mirrors the subset of DataFusion's {@code AvroReadOptions} that maps onto the Java surface
30+
* today: {@code fileExtension} (default {@code ".avro"}) and an explicit Arrow {@code schema} that
31+
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred -- no
32+
* other Java reader exposes Hive-style partitioning yet.
33+
*
34+
* <p>Avro carries its own per-block compression (snappy, deflate, bzip2, xz, zstandard) inside the
35+
* object container itself, negotiated when the file is written, so unlike CSV / NDJSON there is no
36+
* {@code FileCompressionType} setter.
37+
*/
38+
public final class AvroReadOptions {
39+
40+
private String fileExtension = ".avro";
41+
private Schema schema;
42+
43+
public AvroReadOptions fileExtension(String ext) {
44+
this.fileExtension = ext;
45+
return this;
46+
}
47+
48+
public AvroReadOptions schema(Schema schema) {
49+
this.schema = schema;
50+
return this;
51+
}
52+
53+
byte[] toBytes() {
54+
return AvroReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
55+
}
56+
57+
Schema schema() {
58+
return schema;
59+
}
60+
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,17 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.ByteArrayInputStream;
23+
import java.io.IOException;
24+
import java.nio.channels.Channels;
25+
2226
import org.apache.arrow.c.ArrowArrayStream;
2327
import org.apache.arrow.c.Data;
2428
import org.apache.arrow.memory.BufferAllocator;
2529
import org.apache.arrow.vector.ipc.ArrowReader;
30+
import org.apache.arrow.vector.ipc.ReadChannel;
31+
import org.apache.arrow.vector.ipc.message.MessageSerializer;
32+
import org.apache.arrow.vector.types.pojo.Schema;
2633

2734
/**
2835
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
@@ -106,6 +113,77 @@ public ArrowReader executeStream(BufferAllocator allocator) {
106113
}
107114
}
108115

116+
/**
117+
* Return the Arrow {@link Schema} of this DataFrame's output. Non-consuming: the receiver remains
118+
* usable and must still be closed independently. Schema inspection does not execute the plan.
119+
*
120+
* <p>The schema is transferred via Arrow IPC; no {@link BufferAllocator} is required because a
121+
* schema carries no buffer data.
122+
*/
123+
public Schema schema() {
124+
if (nativeHandle == 0) {
125+
throw new IllegalStateException("DataFrame is closed or already collected");
126+
}
127+
byte[] ipcBytes = schemaIpc(nativeHandle);
128+
try {
129+
return MessageSerializer.deserializeSchema(
130+
new ReadChannel(Channels.newChannel(new ByteArrayInputStream(ipcBytes))));
131+
} catch (IOException e) {
132+
throw new RuntimeException("Failed to deserialize IPC schema", e);
133+
}
134+
}
135+
136+
/**
137+
* Return a new DataFrame whose rows describe the plan that would execute this DataFrame.
138+
* Non-consuming: the receiver remains usable and must still be closed independently.
139+
*
140+
* <p>With {@code verbose=false} and {@code analyze=false} (the cheap, lazy variant), the result
141+
* contains the logical plan only. {@code verbose=true} adds optimised-plan and physical-plan
142+
* rows; {@code analyze=true} runs the plan and attaches per-operator metrics. Render via {@link
143+
* #show()} or {@link #collect(BufferAllocator)}.
144+
*/
145+
public DataFrame explain(boolean verbose, boolean analyze) {
146+
if (nativeHandle == 0) {
147+
throw new IllegalStateException("DataFrame is closed or already collected");
148+
}
149+
return new DataFrame(explainPlan(nativeHandle, verbose, analyze));
150+
}
151+
152+
/**
153+
* Materialise this DataFrame into an in-memory table and return a new DataFrame that scans it.
154+
* Non-consuming: the receiver remains usable and must still be closed independently.
155+
*
156+
* <p>Executes the plan eagerly: the entire result set is held in native memory until the returned
157+
* DataFrame is closed. Suitable for intermediate results that will be reused across multiple
158+
* downstream queries.
159+
*
160+
* @throws RuntimeException if execution fails.
161+
*/
162+
public DataFrame cache() {
163+
if (nativeHandle == 0) {
164+
throw new IllegalStateException("DataFrame is closed or already collected");
165+
}
166+
return new DataFrame(cachePlan(nativeHandle));
167+
}
168+
169+
/**
170+
* Compute summary statistics (count, null_count, mean, std, min, max, median) over this
171+
* DataFrame's columns and return them as a new DataFrame. Non-consuming: the receiver remains
172+
* usable and must still be closed independently.
173+
*
174+
* <p>Executes the plan: DataFusion runs seven aggregate sub-plans against this DataFrame to build
175+
* the summary table. Numeric columns receive every statistic; non-numeric columns receive {@code
176+
* count} / {@code null_count} / {@code min} / {@code max} where applicable.
177+
*
178+
* @throws RuntimeException if execution fails.
179+
*/
180+
public DataFrame describe() {
181+
if (nativeHandle == 0) {
182+
throw new IllegalStateException("DataFrame is closed or already collected");
183+
}
184+
return new DataFrame(describePlan(nativeHandle));
185+
}
186+
109187
/** Execute the plan and return the number of rows. */
110188
public long count() {
111189
if (nativeHandle == 0) {
@@ -405,6 +483,121 @@ private long otherHandle(String op, DataFrame other) {
405483
return other.nativeHandle;
406484
}
407485

486+
/**
487+
* Equi-join this DataFrame with {@code right} on the named columns, using the given {@link
488+
* JoinType}. The receiver and {@code right} both remain usable and must still be closed
489+
* independently.
490+
*
491+
* <p>Equivalent to SQL {@code left <type> JOIN right ON l.leftCols[0] = r.rightCols[0] AND ...}.
492+
* {@code leftCols} and {@code rightCols} must have the same length.
493+
*
494+
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
495+
* rightCols.length}.
496+
* @throws IllegalStateException if either DataFrame is closed or already collected.
497+
* @throws RuntimeException if join planning fails (column collision in the combined schema,
498+
* unknown column names, etc.).
499+
*/
500+
public DataFrame join(DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
501+
checkJoinArgs(right, type, leftCols, rightCols);
502+
return new DataFrame(
503+
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, null));
504+
}
505+
506+
/**
507+
* Equi-join this DataFrame with {@code right}, restricting the result with a residual SQL filter
508+
* parsed against the <em>combined</em> schema (left columns followed by right columns; columns
509+
* may be qualified with the relation alias when ambiguous). The receiver and {@code right} both
510+
* remain usable and must still be closed independently.
511+
*
512+
* <p>For outer joins, the filter is applied only to matched rows; unmatched rows are passed
513+
* through with nulls on the unmatched side, matching DataFusion's semantics.
514+
*
515+
* @throws IllegalArgumentException if any argument is {@code null} or {@code leftCols.length !=
516+
* rightCols.length}.
517+
* @throws IllegalStateException if either DataFrame is closed or already collected.
518+
* @throws RuntimeException if join planning or filter parsing fails.
519+
*/
520+
public DataFrame join(
521+
DataFrame right, JoinType type, String[] leftCols, String[] rightCols, String filter) {
522+
checkJoinArgs(right, type, leftCols, rightCols);
523+
if (filter == null) {
524+
throw new IllegalArgumentException("join filter must be non-null");
525+
}
526+
return new DataFrame(
527+
joinDataFrame(nativeHandle, right.nativeHandle, type.code(), leftCols, rightCols, filter));
528+
}
529+
530+
/**
531+
* Join this DataFrame with {@code right} using arbitrary SQL predicates parsed against the
532+
* <em>combined</em> schema. Each predicate is parsed independently and the join evaluates their
533+
* conjunction. Predicates may reference columns from either side and may be qualified with the
534+
* relation alias when ambiguous (e.g. {@code "left.x = right.x"}). The receiver and {@code right}
535+
* both remain usable and must still be closed independently.
536+
*
537+
* <p>DataFusion's optimiser identifies and rewrites equality predicates into hash-join keys
538+
* automatically, so {@code joinOn(right, INNER, "l.id = r.id")} plans equivalently to {@link
539+
* #join(DataFrame, JoinType, String[], String[])} with a single key. Use {@code joinOn} when the
540+
* predicate is not a simple equality, e.g. inequality joins or range conditions.
541+
*
542+
* @throws IllegalArgumentException if {@code right} or {@code type} is {@code null}, or {@code
543+
* predicates} is {@code null} or empty, or any predicate is {@code null}.
544+
* @throws IllegalStateException if either DataFrame is closed or already collected.
545+
* @throws RuntimeException if predicate parsing or join planning fails.
546+
*/
547+
public DataFrame joinOn(DataFrame right, JoinType type, String... predicates) {
548+
if (right == null) {
549+
throw new IllegalArgumentException("joinOn right must be non-null");
550+
}
551+
if (type == null) {
552+
throw new IllegalArgumentException("joinOn type must be non-null");
553+
}
554+
if (predicates == null || predicates.length == 0) {
555+
throw new IllegalArgumentException("joinOn predicates must be non-null and non-empty");
556+
}
557+
for (String p : predicates) {
558+
if (p == null) {
559+
throw new IllegalArgumentException("joinOn predicates must not contain null");
560+
}
561+
}
562+
if (nativeHandle == 0) {
563+
throw new IllegalStateException("DataFrame is closed or already collected");
564+
}
565+
if (right.nativeHandle == 0) {
566+
throw new IllegalStateException("right DataFrame is closed or already collected");
567+
}
568+
return new DataFrame(
569+
joinOnDataFrame(nativeHandle, right.nativeHandle, type.code(), predicates));
570+
}
571+
572+
private void checkJoinArgs(
573+
DataFrame right, JoinType type, String[] leftCols, String[] rightCols) {
574+
if (right == null) {
575+
throw new IllegalArgumentException("join right must be non-null");
576+
}
577+
if (type == null) {
578+
throw new IllegalArgumentException("join type must be non-null");
579+
}
580+
if (leftCols == null) {
581+
throw new IllegalArgumentException("join leftCols must be non-null");
582+
}
583+
if (rightCols == null) {
584+
throw new IllegalArgumentException("join rightCols must be non-null");
585+
}
586+
if (leftCols.length != rightCols.length) {
587+
throw new IllegalArgumentException(
588+
"join leftCols and rightCols must have the same length, got "
589+
+ leftCols.length
590+
+ " and "
591+
+ rightCols.length);
592+
}
593+
if (nativeHandle == 0) {
594+
throw new IllegalStateException("DataFrame is closed or already collected");
595+
}
596+
if (right.nativeHandle == 0) {
597+
throw new IllegalStateException("right DataFrame is closed or already collected");
598+
}
599+
}
600+
408601
/**
409602
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
410603
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -467,6 +660,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
467660
writeCsvWithOptions(nativeHandle, path, options.toBytes());
468661
}
469662

663+
/**
664+
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
665+
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
666+
* receiver remains usable and must still be closed independently.
667+
*
668+
* @throws RuntimeException if the write fails.
669+
*/
670+
public void writeJson(String path) {
671+
writeJson(path, new JsonWriteOptions());
672+
}
673+
674+
/**
675+
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
676+
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
677+
*
678+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
679+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
680+
* etc.).
681+
*/
682+
public void writeJson(String path, JsonWriteOptions options) {
683+
if (nativeHandle == 0) {
684+
throw new IllegalStateException("DataFrame is closed or already collected");
685+
}
686+
if (path == null) {
687+
throw new IllegalArgumentException("writeJson path must be non-null");
688+
}
689+
if (options == null) {
690+
throw new IllegalArgumentException("writeJson options must be non-null");
691+
}
692+
writeJsonWithOptions(nativeHandle, path, options.toBytes());
693+
}
694+
470695
@Override
471696
public void close() {
472697
if (nativeHandle != 0) {
@@ -483,6 +708,14 @@ public void close() {
483708

484709
private static native long countRows(long handle);
485710

711+
private static native byte[] schemaIpc(long handle);
712+
713+
private static native long explainPlan(long handle, boolean verbose, boolean analyze);
714+
715+
private static native long cachePlan(long handle);
716+
717+
private static native long describePlan(long handle);
718+
486719
private static native void showDataFrame(long handle);
487720

488721
private static native void showDataFrameWithLimit(long handle, int limit);
@@ -519,6 +752,17 @@ public void close() {
519752

520753
private static native long exceptDistinctRows(long handle, long otherHandle);
521754

755+
private static native long joinDataFrame(
756+
long leftHandle,
757+
long rightHandle,
758+
byte joinType,
759+
String[] leftCols,
760+
String[] rightCols,
761+
String filter);
762+
763+
private static native long joinOnDataFrame(
764+
long leftHandle, long rightHandle, byte joinType, String[] predicates);
765+
522766
private static native void writeParquetWithOptions(
523767
long handle,
524768
String path,
@@ -527,4 +771,6 @@ private static native void writeParquetWithOptions(
527771
boolean singleFileOutputValue);
528772

529773
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
774+
775+
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
530776
}

0 commit comments

Comments
 (0)