Skip to content

Commit 3d9e19f

Browse files
authored
feat(dataframe): add writeCsv with CsvWriteOptions (#53)
1 parent 8e6ab54 commit 3d9e19f

9 files changed

Lines changed: 612 additions & 5 deletions

File tree

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Configuration knobs for writing CSV, passed to {@link DataFrame#writeCsv(String,
27+
* CsvWriteOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
30+
* CsvOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
31+
* null} or empty (meaning the DataFusion default is used).
32+
*
33+
* <p>Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
34+
* {@code writeCsv} is the literal output filename. When left unset (the default) and there are no
35+
* partition columns, the path is treated as a directory that DataFusion populates with one or more
36+
* part-files.
37+
*
38+
* <p>Compression reuses {@link FileCompressionType} -- both the read and write sides accept the
39+
* same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}).
40+
*/
41+
public final class CsvWriteOptions {
42+
43+
private Boolean singleFileOutput;
44+
private final List<String> partitionCols = new ArrayList<>();
45+
private Boolean hasHeader;
46+
private Byte delimiter;
47+
private Byte quote;
48+
private Byte escape;
49+
private String nullValue;
50+
private FileCompressionType fileCompressionType;
51+
52+
/**
53+
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
54+
* and no partition columns are configured, the path is treated as a directory and DataFusion
55+
* writes one or more part-files.
56+
*/
57+
public CsvWriteOptions singleFileOutput(boolean v) {
58+
this.singleFileOutput = v;
59+
return this;
60+
}
61+
62+
/**
63+
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
64+
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
65+
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
66+
*/
67+
public CsvWriteOptions partitionCols(String... cols) {
68+
this.partitionCols.clear();
69+
for (String c : cols) {
70+
this.partitionCols.add(c);
71+
}
72+
return this;
73+
}
74+
75+
/** Whether to write a header row. Defaults to DataFusion's setting (typically {@code true}). */
76+
public CsvWriteOptions hasHeader(boolean v) {
77+
this.hasHeader = v;
78+
return this;
79+
}
80+
81+
/** Field delimiter byte. Defaults to {@code ','}. */
82+
public CsvWriteOptions delimiter(byte b) {
83+
this.delimiter = b;
84+
return this;
85+
}
86+
87+
/** Quote character byte. Defaults to {@code '"'}. */
88+
public CsvWriteOptions quote(byte b) {
89+
this.quote = b;
90+
return this;
91+
}
92+
93+
/** Escape character byte. Defaults to none. */
94+
public CsvWriteOptions escape(byte b) {
95+
this.escape = b;
96+
return this;
97+
}
98+
99+
/** String to write for SQL NULL values. Defaults to the empty string. */
100+
public CsvWriteOptions nullValue(String s) {
101+
this.nullValue = s;
102+
return this;
103+
}
104+
105+
/** Output compression codec. Defaults to uncompressed. */
106+
public CsvWriteOptions fileCompressionType(FileCompressionType t) {
107+
this.fileCompressionType = t;
108+
return this;
109+
}
110+
111+
byte[] toBytes() {
112+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.Builder b =
113+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.newBuilder();
114+
if (singleFileOutput != null) {
115+
b.setSingleFileOutput(singleFileOutput);
116+
}
117+
b.addAllPartitionCols(partitionCols);
118+
if (hasHeader != null) {
119+
b.setHasHeader(hasHeader);
120+
}
121+
if (delimiter != null) {
122+
b.setDelimiter(delimiter & 0xFF);
123+
}
124+
if (quote != null) {
125+
b.setQuote(quote & 0xFF);
126+
}
127+
if (escape != null) {
128+
b.setEscape(escape & 0xFF);
129+
}
130+
if (nullValue != null) {
131+
b.setNullValue(nullValue);
132+
}
133+
if (fileCompressionType != null) {
134+
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
135+
}
136+
return b.build().toByteArray();
137+
}
138+
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,38 @@ public void writeParquet(String path, ParquetWriteOptions options) {
250250
options.singleFileOutput() != null && options.singleFileOutput());
251251
}
252252

253+
/**
254+
* Materialize this DataFrame as CSV at {@code path}. The path is treated as a directory unless
255+
* overridden via {@link CsvWriteOptions#singleFileOutput(boolean)}. The receiver remains usable
256+
* and must still be closed independently.
257+
*
258+
* @throws RuntimeException if the write fails.
259+
*/
260+
public void writeCsv(String path) {
261+
writeCsv(path, new CsvWriteOptions());
262+
}
263+
264+
/**
265+
* Materialize this DataFrame as CSV at {@code path} with the supplied {@link CsvWriteOptions}.
266+
* The receiver remains usable and must still be closed independently.
267+
*
268+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
269+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
270+
* etc.).
271+
*/
272+
public void writeCsv(String path, CsvWriteOptions options) {
273+
if (nativeHandle == 0) {
274+
throw new IllegalStateException("DataFrame is closed or already collected");
275+
}
276+
if (path == null) {
277+
throw new IllegalArgumentException("writeCsv path must be non-null");
278+
}
279+
if (options == null) {
280+
throw new IllegalArgumentException("writeCsv options must be non-null");
281+
}
282+
writeCsvWithOptions(nativeHandle, path, options.toBytes());
283+
}
284+
253285
@Override
254286
public void close() {
255287
if (nativeHandle != 0) {
@@ -290,4 +322,6 @@ private static native void writeParquetWithOptions(
290322
String compression,
291323
boolean singleFileOutputSet,
292324
boolean singleFileOutputValue);
325+
326+
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
293327
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import org.apache.datafusion.protobuf.CsvWriteOptionsProto;
27+
import org.junit.jupiter.api.Test;
28+
29+
class CsvWriteOptionsTest {
30+
31+
@Test
32+
void defaultsLeaveEverythingUnset() throws Exception {
33+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(new CsvWriteOptions().toBytes());
34+
35+
assertFalse(p.hasSingleFileOutput());
36+
assertEquals(0, p.getPartitionColsCount());
37+
assertFalse(p.hasHasHeader());
38+
assertFalse(p.hasDelimiter());
39+
assertFalse(p.hasQuote());
40+
assertFalse(p.hasEscape());
41+
assertFalse(p.hasNullValue());
42+
assertFalse(p.hasFileCompressionType());
43+
}
44+
45+
@Test
46+
void fluentSettersRoundTripThroughProto() throws Exception {
47+
CsvWriteOptions opts =
48+
new CsvWriteOptions()
49+
.singleFileOutput(true)
50+
.partitionCols("region", "year")
51+
.hasHeader(false)
52+
.delimiter((byte) '|')
53+
.quote((byte) '\'')
54+
.escape((byte) '\\')
55+
.nullValue("\\N")
56+
.fileCompressionType(FileCompressionType.GZIP);
57+
58+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
59+
60+
assertTrue(p.getSingleFileOutput());
61+
assertEquals(2, p.getPartitionColsCount());
62+
assertEquals("region", p.getPartitionCols(0));
63+
assertEquals("year", p.getPartitionCols(1));
64+
assertFalse(p.getHasHeader());
65+
assertEquals((int) '|', p.getDelimiter());
66+
assertEquals((int) '\'', p.getQuote());
67+
assertEquals((int) '\\', p.getEscape());
68+
assertEquals("\\N", p.getNullValue());
69+
assertEquals(
70+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
71+
p.getFileCompressionType());
72+
}
73+
74+
@Test
75+
void partitionColsResetOnSubsequentCalls() throws Exception {
76+
CsvWriteOptions opts = new CsvWriteOptions().partitionCols("a", "b", "c").partitionCols("only");
77+
78+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
79+
assertEquals(1, p.getPartitionColsCount());
80+
assertEquals("only", p.getPartitionCols(0));
81+
}
82+
83+
@Test
84+
void delimiterAcceptsHighByteWithoutSignExtension() throws Exception {
85+
CsvWriteOptions opts = new CsvWriteOptions().delimiter((byte) 0xC2);
86+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
87+
assertEquals(0xC2, p.getDelimiter());
88+
}
89+
}

0 commit comments

Comments
 (0)