Skip to content

Commit 9e8279d

Browse files
committed
Merge remote-tracking branch 'apache/main' into feat/columnar-value-udf
# Conflicts: # native/Cargo.toml
2 parents 1df2bd2 + 2ecf3f1 commit 9e8279d

25 files changed

Lines changed: 2100 additions & 14 deletions

‎core/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,11 @@ under the License.
5757
<groupId>com.google.protobuf</groupId>
5858
<artifactId>protobuf-java</artifactId>
5959
</dependency>
60+
<dependency>
61+
<groupId>org.apache.avro</groupId>
62+
<artifactId>avro</artifactId>
63+
<scope>test</scope>
64+
</dependency>
6065
</dependencies>
6166

6267
<build>
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
import org.apache.datafusion.protobuf.AvroReadOptionsProto;
24+
25+
/**
26+
* Configuration knobs for Avro sources passed to {@link SessionContext#registerAvro(String, String,
27+
* AvroReadOptions)} and {@link SessionContext#readAvro(String, AvroReadOptions)}.
28+
*
29+
* <p>Mirrors the subset of DataFusion's {@code AvroReadOptions} that maps onto the Java surface
30+
* today: {@code fileExtension} (default {@code ".avro"}) and an explicit Arrow {@code schema} that
31+
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred -- no
32+
* other Java reader exposes Hive-style partitioning yet.
33+
*
34+
* <p>Avro carries its own per-block compression (snappy, deflate, bzip2, xz, zstandard) inside the
35+
* object container itself, negotiated when the file is written, so unlike CSV / NDJSON there is no
36+
* {@code FileCompressionType} setter.
37+
*/
38+
public final class AvroReadOptions {
39+
40+
private String fileExtension = ".avro";
41+
private Schema schema;
42+
43+
public AvroReadOptions fileExtension(String ext) {
44+
this.fileExtension = ext;
45+
return this;
46+
}
47+
48+
public AvroReadOptions schema(Schema schema) {
49+
this.schema = schema;
50+
return this;
51+
}
52+
53+
byte[] toBytes() {
54+
return AvroReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
55+
}
56+
57+
Schema schema() {
58+
return schema;
59+
}
60+
}
Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Configuration knobs for writing CSV, passed to {@link DataFrame#writeCsv(String,
27+
* CsvWriteOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
30+
* CsvOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
31+
* null} or empty (meaning the DataFusion default is used).
32+
*
33+
* <p>Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
34+
* {@code writeCsv} is the literal output filename. When left unset (the default) and there are no
35+
* partition columns, the path is treated as a directory that DataFusion populates with one or more
36+
* part-files.
37+
*
38+
* <p>Compression reuses {@link FileCompressionType} -- both the read and write sides accept the
39+
* same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}).
40+
*/
41+
public final class CsvWriteOptions {
42+
43+
private Boolean singleFileOutput;
44+
private final List<String> partitionCols = new ArrayList<>();
45+
private Boolean hasHeader;
46+
private Byte delimiter;
47+
private Byte quote;
48+
private Byte escape;
49+
private String nullValue;
50+
private FileCompressionType fileCompressionType;
51+
52+
/**
53+
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
54+
* and no partition columns are configured, the path is treated as a directory and DataFusion
55+
* writes one or more part-files.
56+
*/
57+
public CsvWriteOptions singleFileOutput(boolean v) {
58+
this.singleFileOutput = v;
59+
return this;
60+
}
61+
62+
/**
63+
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
64+
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
65+
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
66+
*/
67+
public CsvWriteOptions partitionCols(String... cols) {
68+
this.partitionCols.clear();
69+
for (String c : cols) {
70+
this.partitionCols.add(c);
71+
}
72+
return this;
73+
}
74+
75+
/** Whether to write a header row. Defaults to DataFusion's setting (typically {@code true}). */
76+
public CsvWriteOptions hasHeader(boolean v) {
77+
this.hasHeader = v;
78+
return this;
79+
}
80+
81+
/** Field delimiter byte. Defaults to {@code ','}. */
82+
public CsvWriteOptions delimiter(byte b) {
83+
this.delimiter = b;
84+
return this;
85+
}
86+
87+
/** Quote character byte. Defaults to {@code '"'}. */
88+
public CsvWriteOptions quote(byte b) {
89+
this.quote = b;
90+
return this;
91+
}
92+
93+
/** Escape character byte. Defaults to none. */
94+
public CsvWriteOptions escape(byte b) {
95+
this.escape = b;
96+
return this;
97+
}
98+
99+
/** String to write for SQL NULL values. Defaults to the empty string. */
100+
public CsvWriteOptions nullValue(String s) {
101+
this.nullValue = s;
102+
return this;
103+
}
104+
105+
/** Output compression codec. Defaults to uncompressed. */
106+
public CsvWriteOptions fileCompressionType(FileCompressionType t) {
107+
this.fileCompressionType = t;
108+
return this;
109+
}
110+
111+
byte[] toBytes() {
112+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.Builder b =
113+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.newBuilder();
114+
if (singleFileOutput != null) {
115+
b.setSingleFileOutput(singleFileOutput);
116+
}
117+
b.addAllPartitionCols(partitionCols);
118+
if (hasHeader != null) {
119+
b.setHasHeader(hasHeader);
120+
}
121+
if (delimiter != null) {
122+
b.setDelimiter(delimiter & 0xFF);
123+
}
124+
if (quote != null) {
125+
b.setQuote(quote & 0xFF);
126+
}
127+
if (escape != null) {
128+
b.setEscape(escape & 0xFF);
129+
}
130+
if (nullValue != null) {
131+
b.setNullValue(nullValue);
132+
}
133+
if (fileCompressionType != null) {
134+
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
135+
}
136+
return b.build().toByteArray();
137+
}
138+
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 111 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,14 @@
2626

2727
/**
2828
* A lazy representation of a query plan, mirroring the Rust DataFusion {@code DataFrame}. Created
29-
* by {@link SessionContext#sql(String)} or other planning entry points and executed by {@link
30-
* #collect}.
29+
* by {@link SessionContext#sql(String)} or other planning entry points and executed by either
30+
* {@link #collect} (materializes every batch on the native heap before returning) or {@link
31+
* #executeStream} (yields one batch at a time as Java drains the reader).
3132
*
32-
* <p>Instances are <strong>not thread-safe</strong> and must be closed. {@link #collect} consumes
33-
* the DataFrame: a successfully collected DataFrame cannot be collected again, and {@link #close()}
34-
* on an already-collected instance is a no-op.
33+
* <p>Instances are <strong>not thread-safe</strong> and must be closed. Both {@link #collect} and
34+
* {@link #executeStream} consume the DataFrame: a successfully consumed DataFrame cannot be
35+
* consumed again by either method (or by other executors such as {@link #count}), and {@link
36+
* #close()} on an already-consumed instance is a no-op.
3537
*/
3638
public final class DataFrame implements AutoCloseable {
3739
static {
@@ -53,6 +55,10 @@ public final class DataFrame implements AutoCloseable {
5355
* <p>Consumes this DataFrame: the native plan is released as soon as the stream is established.
5456
* The caller is responsible for closing the returned reader, and the supplied allocator must
5557
* outlive it.
58+
*
59+
* <p>This method materializes every batch on the native heap before the first batch crosses the
60+
* FFI boundary, which can OOM the Rust side for unbounded or very large result sets. Prefer
61+
* {@link #executeStream(BufferAllocator)} for analytics-scale queries.
5662
*/
5763
public ArrowReader collect(BufferAllocator allocator) {
5864
if (nativeHandle == 0) {
@@ -70,6 +76,36 @@ public ArrowReader collect(BufferAllocator allocator) {
7076
}
7177
}
7278

79+
/**
80+
* Execute the plan and return its record batches as a streaming {@link ArrowReader}. Each call to
81+
* {@link ArrowReader#loadNextBatch} drives one async {@code stream.next()} on the native side, so
82+
* memory pressure stays bounded by the executor pipeline plus one in-flight batch instead of the
83+
* full result set.
84+
*
85+
* <p>Consumes this DataFrame with the same lifecycle rules as {@link #collect(BufferAllocator)}:
86+
* the native plan is released as soon as the stream is established, the caller closes the
87+
* returned reader, and the supplied allocator must outlive it.
88+
*
89+
* <p>For result sets that fit comfortably in native memory and are read in their entirety, {@link
90+
* #collect(BufferAllocator)} remains a reasonable choice. For TB-scale or unbounded result sets,
91+
* use this method.
92+
*/
93+
public ArrowReader executeStream(BufferAllocator allocator) {
94+
if (nativeHandle == 0) {
95+
throw new IllegalStateException("DataFrame is closed or already collected");
96+
}
97+
ArrowArrayStream stream = ArrowArrayStream.allocateNew(allocator);
98+
long handle = nativeHandle;
99+
nativeHandle = 0;
100+
try {
101+
executeStreamDataFrame(handle, stream.memoryAddress());
102+
return Data.importArrayStream(allocator, stream);
103+
} catch (Throwable e) {
104+
stream.close();
105+
throw e;
106+
}
107+
}
108+
73109
/** Execute the plan and return the number of rows. */
74110
public long count() {
75111
if (nativeHandle == 0) {
@@ -250,6 +286,70 @@ public void writeParquet(String path, ParquetWriteOptions options) {
250286
options.singleFileOutput() != null && options.singleFileOutput());
251287
}
252288

289+
/**
290+
* Materialize this DataFrame as CSV at {@code path}. The path is treated as a directory unless
291+
* overridden via {@link CsvWriteOptions#singleFileOutput(boolean)}. The receiver remains usable
292+
* and must still be closed independently.
293+
*
294+
* @throws RuntimeException if the write fails.
295+
*/
296+
public void writeCsv(String path) {
297+
writeCsv(path, new CsvWriteOptions());
298+
}
299+
300+
/**
301+
* Materialize this DataFrame as CSV at {@code path} with the supplied {@link CsvWriteOptions}.
302+
* The receiver remains usable and must still be closed independently.
303+
*
304+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
305+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
306+
* etc.).
307+
*/
308+
public void writeCsv(String path, CsvWriteOptions options) {
309+
if (nativeHandle == 0) {
310+
throw new IllegalStateException("DataFrame is closed or already collected");
311+
}
312+
if (path == null) {
313+
throw new IllegalArgumentException("writeCsv path must be non-null");
314+
}
315+
if (options == null) {
316+
throw new IllegalArgumentException("writeCsv options must be non-null");
317+
}
318+
writeCsvWithOptions(nativeHandle, path, options.toBytes());
319+
}
320+
321+
/**
322+
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
323+
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
324+
* receiver remains usable and must still be closed independently.
325+
*
326+
* @throws RuntimeException if the write fails.
327+
*/
328+
public void writeJson(String path) {
329+
writeJson(path, new JsonWriteOptions());
330+
}
331+
332+
/**
333+
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
334+
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
335+
*
336+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
337+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
338+
* etc.).
339+
*/
340+
public void writeJson(String path, JsonWriteOptions options) {
341+
if (nativeHandle == 0) {
342+
throw new IllegalStateException("DataFrame is closed or already collected");
343+
}
344+
if (path == null) {
345+
throw new IllegalArgumentException("writeJson path must be non-null");
346+
}
347+
if (options == null) {
348+
throw new IllegalArgumentException("writeJson options must be non-null");
349+
}
350+
writeJsonWithOptions(nativeHandle, path, options.toBytes());
351+
}
352+
253353
@Override
254354
public void close() {
255355
if (nativeHandle != 0) {
@@ -260,6 +360,8 @@ public void close() {
260360

261361
private static native void collectDataFrame(long handle, long ffiStreamAddr);
262362

363+
private static native void executeStreamDataFrame(long handle, long ffiStreamAddr);
364+
263365
private static native void closeDataFrame(long handle);
264366

265367
private static native long countRows(long handle);
@@ -290,4 +392,8 @@ private static native void writeParquetWithOptions(
290392
String compression,
291393
boolean singleFileOutputSet,
292394
boolean singleFileOutputValue);
395+
396+
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
397+
398+
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
293399
}

0 commit comments

Comments
 (0)