Skip to content

Commit 7ef0a47

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/dataframe-write-csv
# Conflicts: # native/build.rs
2 parents 6c070e8 + dd5d6fc commit 7ef0a47

31 files changed

Lines changed: 2432 additions & 80 deletions

‎core/src/main/java/org/apache/datafusion/CsvReadOptions.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@
3333
*/
3434
public final class CsvReadOptions {
3535

36-
/** Compression of the file. Names match DataFusion's {@code FileCompressionType} variants. */
37-
public enum FileCompressionType {
38-
UNCOMPRESSED,
39-
GZIP,
40-
BZIP2,
41-
XZ,
42-
ZSTD
43-
}
44-
4536
private boolean hasHeader = true;
4637
private byte delimiter = (byte) ',';
4738
private byte quote = (byte) '"';
@@ -90,6 +81,9 @@ public CsvReadOptions newlinesInValues(boolean v) {
9081
}
9182

9283
public CsvReadOptions schemaInferMaxRecords(long n) {
84+
if (n < 0) {
85+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
86+
}
9387
this.schemaInferMaxRecords = n;
9488
return this;
9589
}
@@ -116,7 +110,7 @@ byte[] toBytes() {
116110
.setDelimiter(delimiter & 0xFF)
117111
.setQuote(quote & 0xFF)
118112
.setFileExtension(fileExtension)
119-
.setFileCompressionType(toProto(fileCompressionType));
113+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
120114
if (terminator != null) {
121115
b.setTerminator(terminator & 0xFF);
122116
}
@@ -138,22 +132,4 @@ byte[] toBytes() {
138132
Schema schema() {
139133
return schema;
140134
}
141-
142-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
143-
switch (t) {
144-
case UNCOMPRESSED:
145-
return org.apache.datafusion.protobuf.FileCompressionType
146-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
147-
case GZIP:
148-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
149-
case BZIP2:
150-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
151-
case XZ:
152-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
153-
case ZSTD:
154-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
155-
default:
156-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
157-
}
158-
}
159135
}

‎core/src/main/java/org/apache/datafusion/CsvWriteOptions.java‎

Lines changed: 5 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,8 @@
3535
* partition columns, the path is treated as a directory that DataFusion populates with one or more
3636
* part-files.
3737
*
38-
* <p>Compression reuses {@link CsvReadOptions.FileCompressionType} -- both the read and write sides
39-
* accept the same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code
40-
* ZSTD}).
38+
* <p>Compression reuses {@link FileCompressionType} -- both the read and write sides accept the
39+
* same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}).
4140
*/
4241
public final class CsvWriteOptions {
4342

@@ -48,7 +47,7 @@ public final class CsvWriteOptions {
4847
private Byte quote;
4948
private Byte escape;
5049
private String nullValue;
51-
private CsvReadOptions.FileCompressionType fileCompressionType;
50+
private FileCompressionType fileCompressionType;
5251

5352
/**
5453
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
@@ -104,7 +103,7 @@ public CsvWriteOptions nullValue(String s) {
104103
}
105104

106105
/** Output compression codec. Defaults to uncompressed. */
107-
public CsvWriteOptions fileCompressionType(CsvReadOptions.FileCompressionType t) {
106+
public CsvWriteOptions fileCompressionType(FileCompressionType t) {
108107
this.fileCompressionType = t;
109108
return this;
110109
}
@@ -132,27 +131,8 @@ byte[] toBytes() {
132131
b.setNullValue(nullValue);
133132
}
134133
if (fileCompressionType != null) {
135-
b.setFileCompressionType(toProto(fileCompressionType));
134+
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
136135
}
137136
return b.build().toByteArray();
138137
}
139-
140-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(
141-
CsvReadOptions.FileCompressionType t) {
142-
switch (t) {
143-
case UNCOMPRESSED:
144-
return org.apache.datafusion.protobuf.FileCompressionType
145-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
146-
case GZIP:
147-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
148-
case BZIP2:
149-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
150-
case XZ:
151-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
152-
case ZSTD:
153-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
154-
default:
155-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
156-
}
157-
}
158138
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,55 @@ public DataFrame withColumnRenamed(String oldName, String newName) {
171171
return new DataFrame(renameColumn(nativeHandle, oldName, newName));
172172
}
173173

174+
/**
175+
* Add a column to this DataFrame computed from a SQL expression. If a column with the given name
176+
* already exists, it is replaced in place; otherwise the new column is appended. The expression
177+
* is parsed against this DataFrame's own schema, matching the convention used by {@link
178+
* #filter(String)}. The receiver remains usable and must still be closed independently.
179+
*
180+
* @throws IllegalArgumentException if {@code name} or {@code expr} is {@code null}.
181+
*/
182+
public DataFrame withColumn(String name, String expr) {
183+
if (nativeHandle == 0) {
184+
throw new IllegalStateException("DataFrame is closed or already collected");
185+
}
186+
if (name == null) {
187+
throw new IllegalArgumentException("withColumn name must be non-null");
188+
}
189+
if (expr == null) {
190+
throw new IllegalArgumentException("withColumn expr must be non-null");
191+
}
192+
return new DataFrame(withColumnExpr(nativeHandle, name, expr));
193+
}
194+
195+
/**
196+
* Expand list or struct columns into rows or fields, with default {@link UnnestOptions} (i.e.
197+
* {@code preserveNulls = true}). The receiver remains usable and must still be closed
198+
* independently.
199+
*/
200+
public DataFrame unnestColumns(String... columns) {
201+
return unnestColumns(new UnnestOptions(), columns);
202+
}
203+
204+
/**
205+
* Expand list or struct columns into rows or fields with the supplied {@link UnnestOptions}. The
206+
* receiver remains usable and must still be closed independently.
207+
*
208+
* @throws IllegalArgumentException if {@code options} or {@code columns} is {@code null}.
209+
*/
210+
public DataFrame unnestColumns(UnnestOptions options, String... columns) {
211+
if (nativeHandle == 0) {
212+
throw new IllegalStateException("DataFrame is closed or already collected");
213+
}
214+
if (options == null) {
215+
throw new IllegalArgumentException("unnestColumns options must be non-null");
216+
}
217+
if (columns == null) {
218+
throw new IllegalArgumentException("unnestColumns columns must be non-null");
219+
}
220+
return new DataFrame(unnestColumns(nativeHandle, columns, options.preserveNulls()));
221+
}
222+
174223
/**
175224
* Materialize this DataFrame as Parquet at {@code path}. The path is treated as a directory
176225
* unless overridden via {@link ParquetWriteOptions#singleFileOutput(boolean)}. The receiver
@@ -263,6 +312,10 @@ public void close() {
263312

264313
private static native long renameColumn(long handle, String oldName, String newName);
265314

315+
private static native long withColumnExpr(long handle, String name, String expr);
316+
317+
private static native long unnestColumns(long handle, String[] columns, boolean preserveNulls);
318+
266319
private static native void writeParquetWithOptions(
267320
long handle,
268321
String path,
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Compression of a file-format source. Shared by {@link CsvReadOptions} and {@link
24+
* NdJsonReadOptions} (and any future format that exposes the same set of compressions). Variant
25+
* names match DataFusion's Rust {@code FileCompressionType} so they round-trip across JNI.
26+
*/
27+
public enum FileCompressionType {
28+
UNCOMPRESSED,
29+
GZIP,
30+
BZIP2,
31+
XZ,
32+
ZSTD
33+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Internal helpers for translating between the user-facing {@link FileCompressionType} enum and the
24+
* protobuf-generated {@code org.apache.datafusion.protobuf.FileCompressionType} enum used on the
25+
* wire. Same variant set, different Java types.
26+
*/
27+
final class FileCompressionTypes {
28+
private FileCompressionTypes() {}
29+
30+
static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
31+
switch (t) {
32+
case UNCOMPRESSED:
33+
return org.apache.datafusion.protobuf.FileCompressionType
34+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
35+
case GZIP:
36+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
37+
case BZIP2:
38+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
39+
case XZ:
40+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
41+
case ZSTD:
42+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
43+
default:
44+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
45+
}
46+
}
47+
}
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for newline-delimited JSON sources passed to {@link
26+
* SessionContext#registerJson(String, String, NdJsonReadOptions)} and {@link
27+
* SessionContext#readJson(String, NdJsonReadOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code NdJsonReadOptions}. All setters return {@code this}
30+
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
31+
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
32+
* is used).
33+
*/
34+
public final class NdJsonReadOptions {
35+
36+
private String fileExtension = ".json";
37+
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
38+
private Long schemaInferMaxRecords;
39+
private Schema schema;
40+
41+
public NdJsonReadOptions fileExtension(String ext) {
42+
this.fileExtension = ext;
43+
return this;
44+
}
45+
46+
public NdJsonReadOptions fileCompressionType(FileCompressionType t) {
47+
this.fileCompressionType = t;
48+
return this;
49+
}
50+
51+
public NdJsonReadOptions schemaInferMaxRecords(long n) {
52+
if (n < 0) {
53+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
54+
}
55+
this.schemaInferMaxRecords = n;
56+
return this;
57+
}
58+
59+
public NdJsonReadOptions schema(Schema schema) {
60+
this.schema = schema;
61+
return this;
62+
}
63+
64+
byte[] toBytes() {
65+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
66+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
67+
.setFileExtension(fileExtension)
68+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
69+
if (schemaInferMaxRecords != null) {
70+
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
71+
}
72+
return b.build().toByteArray();
73+
}
74+
75+
Schema schema() {
76+
return schema;
77+
}
78+
}

0 commit comments

Comments
 (0)