Skip to content

Commit 2ecf3f1

Browse files
authored
feat(dataframe): add writeJson with JsonWriteOptions (#61)
1 parent fe4414a commit 2ecf3f1

7 files changed

Lines changed: 467 additions & 1 deletion

File tree

‎core/src/main/java/org/apache/datafusion/DataFrame.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -318,6 +318,38 @@ public void writeCsv(String path, CsvWriteOptions options) {
318318
writeCsvWithOptions(nativeHandle, path, options.toBytes());
319319
}
320320

321+
/**
322+
* Materialize this DataFrame as newline-delimited JSON at {@code path}. The path is treated as a
323+
* directory unless overridden via {@link JsonWriteOptions#singleFileOutput(boolean)}. The
324+
* receiver remains usable and must still be closed independently.
325+
*
326+
* @throws RuntimeException if the write fails.
327+
*/
328+
public void writeJson(String path) {
329+
writeJson(path, new JsonWriteOptions());
330+
}
331+
332+
/**
333+
* Materialize this DataFrame as newline-delimited JSON at {@code path} with the supplied {@link
334+
* JsonWriteOptions}. The receiver remains usable and must still be closed independently.
335+
*
336+
* @throws IllegalArgumentException if {@code path} or {@code options} is {@code null}.
337+
* @throws RuntimeException if the write fails (path inaccessible, invalid compression spec,
338+
* etc.).
339+
*/
340+
public void writeJson(String path, JsonWriteOptions options) {
341+
if (nativeHandle == 0) {
342+
throw new IllegalStateException("DataFrame is closed or already collected");
343+
}
344+
if (path == null) {
345+
throw new IllegalArgumentException("writeJson path must be non-null");
346+
}
347+
if (options == null) {
348+
throw new IllegalArgumentException("writeJson options must be non-null");
349+
}
350+
writeJsonWithOptions(nativeHandle, path, options.toBytes());
351+
}
352+
321353
@Override
322354
public void close() {
323355
if (nativeHandle != 0) {
@@ -362,4 +394,6 @@ private static native void writeParquetWithOptions(
362394
boolean singleFileOutputValue);
363395

364396
private static native void writeCsvWithOptions(long handle, String path, byte[] optionsBytes);
397+
398+
private static native void writeJsonWithOptions(long handle, String path, byte[] optionsBytes);
365399
}
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.ArrayList;
23+
import java.util.List;
24+
25+
/**
26+
* Configuration knobs for writing JSON, passed to {@link DataFrame#writeJson(String,
27+
* JsonWriteOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code DataFrameWriteOptions} and the writer-side {@code
30+
* JsonOptions}. All setters return {@code this} for fluent chaining. Defaults: every field {@code
31+
* null} or empty (meaning the DataFusion default is used).
32+
*
33+
* <p>Path semantics: when {@link #singleFileOutput(boolean)} is {@code true}, the path passed to
34+
* {@code writeJson} is the literal output filename. When left unset (the default) and there are no
35+
* partition columns, the path is treated as a directory that DataFusion populates with one or more
36+
* part-files.
37+
*
38+
* <p>The output is always newline-delimited JSON (NDJSON). DataFusion's JSON writer does not emit
39+
* the bracketed array form, so there is no toggle for it here.
40+
*
41+
* <p>Compression reuses {@link FileCompressionType} -- the same codec set ({@code UNCOMPRESSED},
42+
* {@code GZIP}, {@code BZIP2}, {@code XZ}, {@code ZSTD}) the read side and the CSV writer accept.
43+
*/
44+
public final class JsonWriteOptions {
45+
46+
private Boolean singleFileOutput;
47+
private final List<String> partitionCols = new ArrayList<>();
48+
private FileCompressionType fileCompressionType;
49+
50+
/**
51+
* When {@code true}, write to a single file at the supplied path. When left unset (the default)
52+
* and no partition columns are configured, the path is treated as a directory and DataFusion
53+
* writes one or more part-files.
54+
*/
55+
public JsonWriteOptions singleFileOutput(boolean v) {
56+
this.singleFileOutput = v;
57+
return this;
58+
}
59+
60+
/**
61+
* Hive-style partition columns. Each column listed here is removed from the data rows and encoded
62+
* into the directory layout (one subdirectory per distinct value). Mutually exclusive with {@link
63+
* #singleFileOutput(boolean)} -- DataFusion rejects the combination at write time.
64+
*/
65+
public JsonWriteOptions partitionCols(String... cols) {
66+
this.partitionCols.clear();
67+
for (String c : cols) {
68+
this.partitionCols.add(c);
69+
}
70+
return this;
71+
}
72+
73+
/** Output compression codec. Defaults to uncompressed. */
74+
public JsonWriteOptions fileCompressionType(FileCompressionType t) {
75+
this.fileCompressionType = t;
76+
return this;
77+
}
78+
79+
byte[] toBytes() {
80+
org.apache.datafusion.protobuf.JsonWriteOptionsProto.Builder b =
81+
org.apache.datafusion.protobuf.JsonWriteOptionsProto.newBuilder();
82+
if (singleFileOutput != null) {
83+
b.setSingleFileOutput(singleFileOutput);
84+
}
85+
b.addAllPartitionCols(partitionCols);
86+
if (fileCompressionType != null) {
87+
b.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
88+
}
89+
return b.build().toByteArray();
90+
}
91+
}
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertThrows;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.util.List;
29+
import java.util.stream.Stream;
30+
31+
import org.apache.arrow.memory.BufferAllocator;
32+
import org.apache.arrow.memory.RootAllocator;
33+
import org.apache.arrow.vector.BigIntVector;
34+
import org.apache.arrow.vector.VectorSchemaRoot;
35+
import org.apache.arrow.vector.ipc.ArrowReader;
36+
import org.junit.jupiter.api.Test;
37+
import org.junit.jupiter.api.io.TempDir;
38+
39+
class DataFrameWriteJsonTest {
40+
41+
/**
42+
* Build a 3-row DataFrame {@code (id BIGINT, name VARCHAR)}, write it as JSON via the SUT, then
43+
* read the result back through {@link SessionContext#registerJson} and run {@code COUNT(*)}.
44+
* Returns the row count.
45+
*/
46+
private static long countRowsAt(Path dirOrFile, NdJsonReadOptions readOpts) throws Exception {
47+
try (BufferAllocator allocator = new RootAllocator();
48+
SessionContext ctx = new SessionContext()) {
49+
ctx.registerJson("t", dirOrFile.toAbsolutePath().toString(), readOpts);
50+
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
51+
ArrowReader reader = df.collect(allocator)) {
52+
assertTrue(reader.loadNextBatch());
53+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
54+
assertEquals(1, root.getRowCount());
55+
return ((BigIntVector) root.getVector(0)).get(0);
56+
}
57+
}
58+
}
59+
60+
/** Materialise a 3-row table the test can hand to {@code writeJson}. */
61+
private static String threeRowSql() {
62+
return "SELECT * FROM (VALUES (CAST(1 AS BIGINT), 'alice'),"
63+
+ " (CAST(2 AS BIGINT), 'bob'), (CAST(3 AS BIGINT), 'carol')) AS t(id, name)";
64+
}
65+
66+
@Test
67+
void writeJsonRoundTripsRowCount(@TempDir Path tempDir) throws Exception {
68+
Path out = tempDir.resolve("out");
69+
70+
try (SessionContext ctx = new SessionContext();
71+
DataFrame df = ctx.sql(threeRowSql())) {
72+
df.writeJson(out.toString());
73+
}
74+
75+
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
76+
}
77+
78+
@Test
79+
void writeJsonSingleFileProducesOneFile(@TempDir Path tempDir) throws Exception {
80+
Path out = tempDir.resolve("out.json");
81+
82+
try (SessionContext ctx = new SessionContext();
83+
DataFrame df = ctx.sql(threeRowSql())) {
84+
df.writeJson(out.toString(), new JsonWriteOptions().singleFileOutput(true));
85+
}
86+
87+
assertTrue(Files.isRegularFile(out), "expected single file at " + out);
88+
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
89+
}
90+
91+
@Test
92+
void writeJsonWithGzipCompressionRoundTrips(@TempDir Path tempDir) throws Exception {
93+
Path out = tempDir.resolve("gz-out");
94+
95+
try (SessionContext ctx = new SessionContext();
96+
DataFrame df = ctx.sql(threeRowSql())) {
97+
df.writeJson(
98+
out.toString(), new JsonWriteOptions().fileCompressionType(FileCompressionType.GZIP));
99+
}
100+
101+
try (Stream<Path> stream = Files.walk(out)) {
102+
List<Path> files = stream.filter(Files::isRegularFile).toList();
103+
assertTrue(!files.isEmpty(), "expected at least one part-file under " + out);
104+
for (Path p : files) {
105+
assertTrue(
106+
p.getFileName().toString().endsWith(".gz"),
107+
"expected .gz suffix on " + p.getFileName());
108+
}
109+
}
110+
111+
NdJsonReadOptions readOpts =
112+
new NdJsonReadOptions()
113+
.fileCompressionType(FileCompressionType.GZIP)
114+
.fileExtension(".json.gz");
115+
assertEquals(3L, countRowsAt(out, readOpts));
116+
}
117+
118+
@Test
119+
void writeJsonDefaultsToDirectoryEvenWithExtensionInPath(@TempDir Path tempDir) throws Exception {
120+
// The Javadoc promises "directory unless overridden via singleFileOutput(true)". DataFusion's
121+
// own DataFrameWriteOptions defaults to Automatic mode, where an extension in the path
122+
// (".json" here) silently flips the output to a single file. The native handler explicitly
123+
// pins the default to directory mode so this contract holds regardless of path shape.
124+
Path out = tempDir.resolve("out.json");
125+
126+
try (SessionContext ctx = new SessionContext();
127+
DataFrame df = ctx.sql(threeRowSql())) {
128+
df.writeJson(out.toString());
129+
}
130+
131+
assertTrue(Files.isDirectory(out), "expected directory output at " + out + ", got a file");
132+
assertEquals(3L, countRowsAt(out, new NdJsonReadOptions()));
133+
}
134+
135+
@Test
136+
void writeJsonRetainsDataFrame(@TempDir Path tempDir) throws Exception {
137+
Path out = tempDir.resolve("retained");
138+
139+
try (SessionContext ctx = new SessionContext();
140+
DataFrame df = ctx.sql(threeRowSql())) {
141+
df.writeJson(out.toString());
142+
assertEquals(3L, df.count());
143+
}
144+
}
145+
146+
@Test
147+
void writeJsonRejectsNullPath() {
148+
try (SessionContext ctx = new SessionContext();
149+
DataFrame df = ctx.sql(threeRowSql())) {
150+
assertThrows(IllegalArgumentException.class, () -> df.writeJson(null));
151+
}
152+
}
153+
154+
@Test
155+
void writeJsonRejectsNullOptions(@TempDir Path tempDir) {
156+
Path out = tempDir.resolve("out");
157+
try (SessionContext ctx = new SessionContext();
158+
DataFrame df = ctx.sql(threeRowSql())) {
159+
assertThrows(IllegalArgumentException.class, () -> df.writeJson(out.toString(), null));
160+
}
161+
}
162+
}
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
24+
import static org.junit.jupiter.api.Assertions.assertTrue;
25+
26+
import org.apache.datafusion.protobuf.JsonWriteOptionsProto;
27+
import org.junit.jupiter.api.Test;
28+
29+
class JsonWriteOptionsTest {
30+
31+
@Test
32+
void defaultsLeaveEverythingUnset() throws Exception {
33+
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(new JsonWriteOptions().toBytes());
34+
35+
assertFalse(p.hasSingleFileOutput());
36+
assertEquals(0, p.getPartitionColsCount());
37+
assertFalse(p.hasFileCompressionType());
38+
}
39+
40+
@Test
41+
void fluentSettersRoundTripThroughProto() throws Exception {
42+
JsonWriteOptions opts =
43+
new JsonWriteOptions()
44+
.singleFileOutput(true)
45+
.partitionCols("region", "year")
46+
.fileCompressionType(FileCompressionType.GZIP);
47+
48+
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(opts.toBytes());
49+
50+
assertTrue(p.getSingleFileOutput());
51+
assertEquals(2, p.getPartitionColsCount());
52+
assertEquals("region", p.getPartitionCols(0));
53+
assertEquals("year", p.getPartitionCols(1));
54+
assertEquals(
55+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
56+
p.getFileCompressionType());
57+
}
58+
59+
@Test
60+
void partitionColsResetOnSubsequentCalls() throws Exception {
61+
JsonWriteOptions opts =
62+
new JsonWriteOptions().partitionCols("a", "b", "c").partitionCols("only");
63+
64+
JsonWriteOptionsProto p = JsonWriteOptionsProto.parseFrom(opts.toBytes());
65+
assertEquals(1, p.getPartitionColsCount());
66+
assertEquals("only", p.getPartitionCols(0));
67+
}
68+
}

‎native/build.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ fn main() {
2424
"../proto/csv_read_options.proto",
2525
"../proto/csv_write_options.proto",
2626
"../proto/json_read_options.proto",
27+
"../proto/json_write_options.proto",
2728
"../proto/parquet_read_options.proto",
2829
];
2930
for p in PROTOS {

0 commit comments

Comments
 (0)