Skip to content

Commit 6c070e8

Browse files
committed
feat(dataframe): add writeCsv with CsvWriteOptions
Closes #38 Mirrors writeParquet's surface for CSV: writeCsv(path) and writeCsv(path, options). Options class exposes singleFileOutput, partitionCols, hasHeader, delimiter, quote, escape, nullValue, and fileCompressionType. Uses the proto-over-JNI pattern (CsvWriteOptionsProto) rather than writeParquet's wide-JNI signature because the writer-side option set is much wider; a single byte[] keeps the JNI signature stable as more fields are added later. FileCompressionType is reused from csv_read_options.proto since the codec set is identical between read and write. Option<CsvOptions> on the Rust side stays None when no writer-side knob is set, so DataFusion's runtime defaults are preserved. Tests: 4 new options round-trip tests plus 7 integration tests (default round-trip, single-file, custom delimiter, gzip round-trip, retain-after-write, null path/options rejection). make test passes; cargo clippy/fmt and spotless are clean.
1 parent 4a967e4 commit 6c070e8

7 files changed

Lines changed: 579 additions & 1 deletion

File tree

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Configuration knobs for writing CSV, passed to {@link DataFrame#writeCsv(String,
27+
* CsvWriteOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
30+
* CsvOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
31+
* null} or empty (meaning the DataFusion default is used).
32+
*
33+
* <p>Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
34+
* {@code writeCsv} is the literal output filename. When left unset (the default) and there are no
35+
* partition columns, the path is treated as a directory that DataFusion populates with one or more
36+
* part-files.
37+
*
38+
* <p>Compression reuses {@link CsvReadOptions.FileCompressionType} -- both the read and write sides
39+
* accept the same codec set ({@code UNCOMPRESSED}, {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code
40+
* ZSTD}).
41+
*/
42+
public final class CsvWriteOptions {
43+
44+
private Boolean singleFileOutput;
45+
private final List<String> partitionCols = new ArrayList<>();
46+
private Boolean hasHeader;
47+
private Byte delimiter;
48+
private Byte quote;
49+
private Byte escape;
50+
private String nullValue;
51+
private CsvReadOptions.FileCompressionType fileCompressionType;
52+
53+
/**
54+
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
55+
* and no partition columns are configured, the path is treated as a directory and DataFusion
56+
* writes one or more part-files.
57+
*/
58+
public CsvWriteOptions singleFileOutput(boolean v) {
59+
this.singleFileOutput = v;
60+
return this;
61+
}
62+
63+
/**
64+
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
65+
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
66+
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
67+
*/
68+
public CsvWriteOptions partitionCols(String... cols) {
69+
this.partitionCols.clear();
70+
for (String c : cols) {
71+
this.partitionCols.add(c);
72+
}
73+
return this;
74+
}
75+
76+
/** Whether to write a header row. Defaults to DataFusion's setting (typically {@code true}). */
77+
public CsvWriteOptions hasHeader(boolean v) {
78+
this.hasHeader = v;
79+
return this;
80+
}
81+
82+
/** Field delimiter byte. Defaults to {@code ','}. */
83+
public CsvWriteOptions delimiter(byte b) {
84+
this.delimiter = b;
85+
return this;
86+
}
87+
88+
/** Quote character byte. Defaults to {@code '"'}. */
89+
public CsvWriteOptions quote(byte b) {
90+
this.quote = b;
91+
return this;
92+
}
93+
94+
/** Escape character byte. Defaults to none. */
95+
public CsvWriteOptions escape(byte b) {
96+
this.escape = b;
97+
return this;
98+
}
99+
100+
/** String to write for SQL NULL values. Defaults to the empty string. */
101+
public CsvWriteOptions nullValue(String s) {
102+
this.nullValue = s;
103+
return this;
104+
}
105+
106+
/** Output compression codec. Defaults to uncompressed. */
107+
public CsvWriteOptions fileCompressionType(CsvReadOptions.FileCompressionType t) {
108+
this.fileCompressionType = t;
109+
return this;
110+
}
111+
112+
byte[] toBytes() {
113+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.Builder b =
114+
org.apache.datafusion.protobuf.CsvWriteOptionsProto.newBuilder();
115+
if (singleFileOutput != null) {
116+
b.setSingleFileOutput(singleFileOutput);
117+
}
118+
b.addAllPartitionCols(partitionCols);
119+
if (hasHeader != null) {
120+
b.setHasHeader(hasHeader);
121+
}
122+
if (delimiter != null) {
123+
b.setDelimiter(delimiter & 0xFF);
124+
}
125+
if (quote != null) {
126+
b.setQuote(quote & 0xFF);
127+
}
128+
if (escape != null) {
129+
b.setEscape(escape & 0xFF);
130+
}
131+
if (nullValue != null) {
132+
b.setNullValue(nullValue);
133+
}
134+
if (fileCompressionType != null) {
135+
b.setFileCompressionType(toProto(fileCompressionType));
136+
}
137+
return b.build().toByteArray();
138+
}
139+
140+
private static org.apache.datafusion.protobuf.FileCompressionType toProto(
141+
CsvReadOptions.FileCompressionType t) {
142+
switch (t) {
143+
case UNCOMPRESSED:
144+
return org.apache.datafusion.protobuf.FileCompressionType
145+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
146+
case GZIP:
147+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
148+
case BZIP2:
149+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
150+
case XZ:
151+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
152+
case ZSTD:
153+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
154+
default:
155+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
156+
}
157+
}
158+
}

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,38 @@ public void writeParquet(String path, ParquetWriteOptions options) {
201201
options.singleFileOutput() != null && options.singleFileOutput());
202202
}
203203

204+
/**
205+
* Materialize this DataFrame as CSV at {@code path}. The path is treated as a directory unless
206+
* overridden via {@link CsvWriteOptions#singleFileOutput(boolean)}. The receiver remains usable
207+
* and must still be closed independently.
208+
*
209+
* @throws RuntimeException if the write fails.
210+
*/
211+
public void writeCsv(String path) {
212+
writeCsv(path, new CsvWriteOptions());
213+
}
214+
215+
/**
216+
* Materialize this DataFrame as CSV at {@code path} with the supplied {@link CsvWriteOptions}.
217+
* The receiver remains usable and must still be closed independently.
218+
*
219+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
220+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
221+
* etc.).
222+
*/
223+
public void writeCsv(String path, CsvWriteOptions options) {
224+
if (nativeHandle == 0) {
225+
throw new IllegalStateException("DataFrame is closed or already collected");
226+
}
227+
if (path == null) {
228+
throw new IllegalArgumentException("writeCsv path must be non-null");
229+
}
230+
if (options == null) {
231+
throw new IllegalArgumentException("writeCsv options must be non-null");
232+
}
233+
writeCsvWithOptions(nativeHandle, path, options.toBytes());
234+
}
235+
204236
@Override
205237
public void close() {
206238
if (nativeHandle != 0) {
@@ -237,4 +269,6 @@ private static native void writeParquetWithOptions(
237269
String compression,
238270
boolean singleFileOutputSet,
239271
boolean singleFileOutputValue);
272+
273+
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
240274
}
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import org.apache.datafusion.protobuf.CsvWriteOptionsProto;
27+
import org.apache.datafusion.protobuf.FileCompressionType;
28+
import org.junit.jupiter.api.Test;
29+
30+
class CsvWriteOptionsTest {
31+
32+
@Test
33+
void defaultsLeaveEverythingUnset() throws Exception {
34+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(new CsvWriteOptions().toBytes());
35+
36+
assertFalse(p.hasSingleFileOutput());
37+
assertEquals(0, p.getPartitionColsCount());
38+
assertFalse(p.hasHasHeader());
39+
assertFalse(p.hasDelimiter());
40+
assertFalse(p.hasQuote());
41+
assertFalse(p.hasEscape());
42+
assertFalse(p.hasNullValue());
43+
assertFalse(p.hasFileCompressionType());
44+
}
45+
46+
@Test
47+
void fluentSettersRoundTripThroughProto() throws Exception {
48+
CsvWriteOptions opts =
49+
new CsvWriteOptions()
50+
.singleFileOutput(true)
51+
.partitionCols("region", "year")
52+
.hasHeader(false)
53+
.delimiter((byte) '|')
54+
.quote((byte) '\'')
55+
.escape((byte) '\\')
56+
.nullValue("\\N")
57+
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP);
58+
59+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
60+
61+
assertTrue(p.getSingleFileOutput());
62+
assertEquals(2, p.getPartitionColsCount());
63+
assertEquals("region", p.getPartitionCols(0));
64+
assertEquals("year", p.getPartitionCols(1));
65+
assertFalse(p.getHasHeader());
66+
assertEquals((int) '|', p.getDelimiter());
67+
assertEquals((int) '\'', p.getQuote());
68+
assertEquals((int) '\\', p.getEscape());
69+
assertEquals("\\N", p.getNullValue());
70+
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
71+
}
72+
73+
@Test
74+
void partitionColsResetOnSubsequentCalls() throws Exception {
75+
CsvWriteOptions opts = new CsvWriteOptions().partitionCols("a", "b", "c").partitionCols("only");
76+
77+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
78+
assertEquals(1, p.getPartitionColsCount());
79+
assertEquals("only", p.getPartitionCols(0));
80+
}
81+
82+
@Test
83+
void delimiterAcceptsHighByteWithoutSignExtension() throws Exception {
84+
CsvWriteOptions opts = new CsvWriteOptions().delimiter((byte) 0xC2);
85+
CsvWriteOptionsProto p = CsvWriteOptionsProto.parseFrom(opts.toBytes());
86+
assertEquals(0xC2, p.getDelimiter());
87+
}
88+
}

0 commit comments

Comments
 (0)