Skip to content

Commit b727aab

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/dataframe-execute-stream
# Conflicts: # native/src/lib.rs
2 parents e9af25a + 1a81099 commit b727aab

35 files changed

Lines changed: 3390 additions & 49 deletions
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
import org.apache.datafusion.protobuf.ArrowReadOptionsProto;
24+
25+
/**
26+
* Configuration knobs for Arrow IPC sources passed to {@link SessionContext#registerArrow(String,
27+
* String, ArrowReadOptions)} and {@link SessionContext#readArrow(String, ArrowReadOptions)}.
28+
*
29+
* <p>Mirrors the subset of DataFusion's {@code ArrowReadOptions} that maps onto the Java surface
30+
* today: {@code fileExtension} (default {@code ".arrow"}) and an explicit Arrow {@code schema} that
31+
* bypasses on-read schema inference. {@code tablePartitionCols} is intentionally deferred --
32+
* neither Parquet nor CSV expose Hive-style partitioning on the Java side yet.
33+
*
34+
* <p>Arrow IPC files carry their own body compression (LZ4_FRAME / ZSTD per-buffer) inside the file
35+
* format itself, so unlike CSV / NDJSON there is no {@code FileCompressionType} setter.
36+
*/
37+
public final class ArrowReadOptions {
38+
39+
private String fileExtension = ".arrow";
40+
private Schema schema;
41+
42+
public ArrowReadOptions fileExtension(String ext) {
43+
this.fileExtension = ext;
44+
return this;
45+
}
46+
47+
public ArrowReadOptions schema(Schema schema) {
48+
this.schema = schema;
49+
return this;
50+
}
51+
52+
byte[] toBytes() {
53+
return ArrowReadOptionsProto.newBuilder().setFileExtension(fileExtension).build().toByteArray();
54+
}
55+
56+
Schema schema() {
57+
return schema;
58+
}
59+
}

‎core/src/main/java/org/apache/datafusion/CsvReadOptions.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@
3333
*/
3434
public final class CsvReadOptions {
3535

36-
/** Compression of the file. Names match DataFusion's {@code FileCompressionType} variants. */
37-
public enum FileCompressionType {
38-
UNCOMPRESSED,
39-
GZIP,
40-
BZIP2,
41-
XZ,
42-
ZSTD
43-
}
44-
4536
private boolean hasHeader = true;
4637
private byte delimiter = (byte) ',';
4738
private byte quote = (byte) '"';
@@ -90,6 +81,9 @@ public CsvReadOptions newlinesInValues(boolean v) {
9081
}
9182

9283
public CsvReadOptions schemaInferMaxRecords(long n) {
84+
if (n < 0) {
85+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
86+
}
9387
this.schemaInferMaxRecords = n;
9488
return this;
9589
}
@@ -116,7 +110,7 @@ byte[] toBytes() {
116110
.setDelimiter(delimiter & 0xFF)
117111
.setQuote(quote & 0xFF)
118112
.setFileExtension(fileExtension)
119-
.setFileCompressionType(toProto(fileCompressionType));
113+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
120114
if (terminator != null) {
121115
b.setTerminator(terminator & 0xFF);
122116
}
@@ -138,22 +132,4 @@ byte[] toBytes() {
138132
Schema schema() {
139133
return schema;
140134
}
141-
142-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
143-
switch (t) {
144-
case UNCOMPRESSED:
145-
return org.apache.datafusion.protobuf.FileCompressionType
146-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
147-
case GZIP:
148-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
149-
case BZIP2:
150-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
151-
case XZ:
152-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
153-
case ZSTD:
154-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
155-
default:
156-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
157-
}
158-
}
159135
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,55 @@ public DataFrame withColumnRenamed(String oldName, String newName) {
207207
return new DataFrame(renameColumn(nativeHandle, oldName, newName));
208208
}
209209

210+
/**
211+
* Add a column to this DataFrame computed from a SQL expression. If a column with the given name
212+
* already exists, it is replaced in place; otherwise the new column is appended. The expression
213+
* is parsed against this DataFrame's own schema, matching the convention used by {@link
214+
* #filter(String)}. The receiver remains usable and must still be closed independently.
215+
*
216+
* @throws IllegalArgumentException if {@code name} or {@code expr} is {@code null}.
217+
*/
218+
public DataFrame withColumn(String name, String expr) {
219+
if (nativeHandle == 0) {
220+
throw new IllegalStateException("DataFrame is closed or already collected");
221+
}
222+
if (name == null) {
223+
throw new IllegalArgumentException("withColumn name must be non-null");
224+
}
225+
if (expr == null) {
226+
throw new IllegalArgumentException("withColumn expr must be non-null");
227+
}
228+
return new DataFrame(withColumnExpr(nativeHandle, name, expr));
229+
}
230+
231+
/**
232+
* Expand list or struct columns into rows or fields, with default {@link UnnestOptions} (i.e.
233+
* {@code preserveNulls = true}). The receiver remains usable and must still be closed
234+
* independently.
235+
*/
236+
public DataFrame unnestColumns(String... columns) {
237+
return unnestColumns(new UnnestOptions(), columns);
238+
}
239+
240+
/**
241+
* Expand list or struct columns into rows or fields with the supplied {@link UnnestOptions}. The
242+
* receiver remains usable and must still be closed independently.
243+
*
244+
* @throws IllegalArgumentException if {@code options} or {@code columns} is {@code null}.
245+
*/
246+
public DataFrame unnestColumns(UnnestOptions options, String... columns) {
247+
if (nativeHandle == 0) {
248+
throw new IllegalStateException("DataFrame is closed or already collected");
249+
}
250+
if (options == null) {
251+
throw new IllegalArgumentException("unnestColumns options must be non-null");
252+
}
253+
if (columns == null) {
254+
throw new IllegalArgumentException("unnestColumns columns must be non-null");
255+
}
256+
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
257+
}
258+
210259
/**
211260
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
212261
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -269,6 +318,10 @@ public void close() {
269318

270319
private static native long renameColumn(long handle, String oldName, String newName);
271320

321+
private static native long withColumnExpr(long handle, String name, String expr);
322+
323+
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
324+
272325
private static native void writeParquetWithOptions(
273326
long handle,
274327
String path,
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Compression of a file-format source. Shared by {@link CsvReadOptions} and {@link
24+
* NdJsonReadOptions} (and any future format that exposes the same set of compressions). Variant
25+
* names match DataFusion's Rust {@code FileCompressionType} so they round-trip across JNI.
26+
*/
27+
public enum FileCompressionType {
28+
UNCOMPRESSED,
29+
GZIP,
30+
BZIP2,
31+
XZ,
32+
ZSTD
33+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Internal helpers for translating between the user-facing {@link FileCompressionType} enum and the
24+
* protobuf-generated {@code org.apache.datafusion.protobuf.FileCompressionType} enum used on the
25+
* wire. Same variant set, different Java types.
26+
*/
27+
final class FileCompressionTypes {
28+
private FileCompressionTypes() {}
29+
30+
static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
31+
switch (t) {
32+
case UNCOMPRESSED:
33+
return org.apache.datafusion.protobuf.FileCompressionType
34+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
35+
case GZIP:
36+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
37+
case BZIP2:
38+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
39+
case XZ:
40+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
41+
case ZSTD:
42+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
43+
default:
44+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
45+
}
46+
}
47+
}
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for newline-delimited JSON sources passed to {@link
26+
* SessionContext#registerJson(String, String, NdJsonReadOptions)} and {@link
27+
* SessionContext#readJson(String, NdJsonReadOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code NdJsonReadOptions}. All setters return {@code this}
30+
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
31+
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
32+
* is used).
33+
*/
34+
public final class NdJsonReadOptions {
35+
36+
private String fileExtension = ".json";
37+
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
38+
private Long schemaInferMaxRecords;
39+
private Schema schema;
40+
41+
public NdJsonReadOptions fileExtension(String ext) {
42+
this.fileExtension = ext;
43+
return this;
44+
}
45+
46+
public NdJsonReadOptions fileCompressionType(FileCompressionType t) {
47+
this.fileCompressionType = t;
48+
return this;
49+
}
50+
51+
public NdJsonReadOptions schemaInferMaxRecords(long n) {
52+
if (n < 0) {
53+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
54+
}
55+
this.schemaInferMaxRecords = n;
56+
return this;
57+
}
58+
59+
public NdJsonReadOptions schema(Schema schema) {
60+
this.schema = schema;
61+
return this;
62+
}
63+
64+
byte[] toBytes() {
65+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
66+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
67+
.setFileExtension(fileExtension)
68+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
69+
if (schemaInferMaxRecords != null) {
70+
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
71+
}
72+
return b.build().toByteArray();
73+
}
74+
75+
Schema schema() {
76+
return schema;
77+
}
78+
}

0 commit comments

Comments
 (0)