Skip to content

Commit 29503c0

Browse files
committed
feat(json): expose NdJsonReadOptions via registerJson and readJson
Mirror the parquet/csv reader pattern for newline-delimited JSON. Adds NdJsonReadOptionsProto in proto/json_read_options.proto (reusing FileCompressionType from csv_read_options.proto), an NdJsonReadOptions Java builder, registerJson/readJson overloads on SessionContext, and a native/src/json.rs JNI module that decodes the proto and forwards to SessionContext::register_json / read_json. Surface is intentionally aligned with the CSV PR (#21): builder fields are fileExtension, fileCompressionType, schemaInferMaxRecords, and an explicit Arrow schema. tablePartitionCols, fileSortOrder, and newline_delimited are left for follow-ups -- none have a parquet/csv counterpart on the Java side yet. The Rust side imports JsonReadOptions from prelude; NdJsonReadOptions is a deprecated alias in DataFusion 53.x. The Java/proto/issue surface keeps the user-facing NdJsonReadOptions name from #35.
1 parent 2689011 commit 29503c0

8 files changed

Lines changed: 515 additions & 0 deletions

File tree

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for newline-delimited JSON sources passed to {@link
26+
* SessionContext#registerJson(String, String, NdJsonReadOptions)} and {@link
27+
* SessionContext#readJson(String, NdJsonReadOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code NdJsonReadOptions}. All setters return {@code this}
30+
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
31+
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
32+
* is used).
33+
*
34+
* <p>{@code FileCompressionType} is reused from {@link CsvReadOptions} since both formats accept
35+
* the same set of compressions.
36+
*/
37+
public final class NdJsonReadOptions {
38+
39+
private String fileExtension = ".json";
40+
private CsvReadOptions.FileCompressionType fileCompressionType =
41+
CsvReadOptions.FileCompressionType.UNCOMPRESSED;
42+
private Long schemaInferMaxRecords;
43+
private Schema schema;
44+
45+
public NdJsonReadOptions fileExtension(String ext) {
46+
this.fileExtension = ext;
47+
return this;
48+
}
49+
50+
public NdJsonReadOptions fileCompressionType(CsvReadOptions.FileCompressionType t) {
51+
this.fileCompressionType = t;
52+
return this;
53+
}
54+
55+
public NdJsonReadOptions schemaInferMaxRecords(long n) {
56+
this.schemaInferMaxRecords = n;
57+
return this;
58+
}
59+
60+
public NdJsonReadOptions schema(Schema schema) {
61+
this.schema = schema;
62+
return this;
63+
}
64+
65+
byte[] toBytes() {
66+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
67+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
68+
.setFileExtension(fileExtension)
69+
.setFileCompressionType(toProto(fileCompressionType));
70+
if (schemaInferMaxRecords != null) {
71+
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
72+
}
73+
return b.build().toByteArray();
74+
}
75+
76+
Schema schema() {
77+
return schema;
78+
}
79+
80+
private static org.apache.datafusion.protobuf.FileCompressionType toProto(
81+
CsvReadOptions.FileCompressionType t) {
82+
switch (t) {
83+
case UNCOMPRESSED:
84+
return org.apache.datafusion.protobuf.FileCompressionType
85+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
86+
case GZIP:
87+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
88+
case BZIP2:
89+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
90+
case XZ:
91+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
92+
case ZSTD:
93+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
94+
default:
95+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
96+
}
97+
}
98+
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,52 @@ public DataFrame readCsv(String path, CsvReadOptions options) {
160160
return new DataFrame(dfHandle);
161161
}
162162

163+
public void registerJson(String name, String path) {
164+
registerJson(name, path, new NdJsonReadOptions());
165+
}
166+
167+
/**
168+
* Register a newline-delimited JSON file (or directory of NDJSON files) as a table with the
169+
* supplied {@link NdJsonReadOptions}.
170+
*
171+
* @throws RuntimeException if registration fails (path not found, schema inference error, etc.).
172+
*/
173+
public void registerJson(String name, String path, NdJsonReadOptions options) {
174+
if (nativeHandle == 0) {
175+
throw new IllegalStateException("SessionContext is closed");
176+
}
177+
registerJsonWithOptions(
178+
nativeHandle,
179+
name,
180+
path,
181+
options.toBytes(),
182+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
183+
}
184+
185+
/** Read a newline-delimited JSON file as a {@link DataFrame} without registering it. */
186+
public DataFrame readJson(String path) {
187+
return readJson(path, new NdJsonReadOptions());
188+
}
189+
190+
/**
191+
* Read a newline-delimited JSON file as a {@link DataFrame} with the supplied {@link
192+
* NdJsonReadOptions}.
193+
*
194+
* @throws RuntimeException if the read fails.
195+
*/
196+
public DataFrame readJson(String path, NdJsonReadOptions options) {
197+
if (nativeHandle == 0) {
198+
throw new IllegalStateException("SessionContext is closed");
199+
}
200+
long dfHandle =
201+
readJsonWithOptions(
202+
nativeHandle,
203+
path,
204+
options.toBytes(),
205+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
206+
return new DataFrame(dfHandle);
207+
}
208+
163209
public void registerParquet(String name, String path) {
164210
registerParquet(name, path, new ParquetReadOptions());
165211
}
@@ -247,5 +293,11 @@ private static native void registerCsvWithOptions(
247293
private static native long readCsvWithOptions(
248294
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
249295

296+
private static native void registerJsonWithOptions(
297+
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
298+
299+
private static native long readJsonWithOptions(
300+
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
301+
250302
private static native void closeSessionContext(long handle);
251303
}
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
24+
import static org.junit.jupiter.api.Assertions.assertSame;
25+
26+
import java.util.List;
27+
28+
import org.apache.arrow.vector.types.pojo.ArrowType;
29+
import org.apache.arrow.vector.types.pojo.Field;
30+
import org.apache.arrow.vector.types.pojo.FieldType;
31+
import org.apache.arrow.vector.types.pojo.Schema;
32+
import org.apache.datafusion.protobuf.FileCompressionType;
33+
import org.apache.datafusion.protobuf.NdJsonReadOptionsProto;
34+
import org.junit.jupiter.api.Test;
35+
36+
import com.google.protobuf.InvalidProtocolBufferException;
37+
38+
class NdJsonReadOptionsTest {
39+
40+
@Test
41+
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
42+
NdJsonReadOptionsProto p = NdJsonReadOptionsProto.parseFrom(new NdJsonReadOptions().toBytes());
43+
44+
assertEquals(".json", p.getFileExtension());
45+
assertEquals(
46+
FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED, p.getFileCompressionType());
47+
assertFalse(p.hasSchemaInferMaxRecords());
48+
}
49+
50+
@Test
51+
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
52+
NdJsonReadOptions opts =
53+
new NdJsonReadOptions()
54+
.fileExtension(".ndjson")
55+
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP)
56+
.schemaInferMaxRecords(50L);
57+
58+
NdJsonReadOptionsProto p = NdJsonReadOptionsProto.parseFrom(opts.toBytes());
59+
60+
assertEquals(".ndjson", p.getFileExtension());
61+
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
62+
assertEquals(50L, p.getSchemaInferMaxRecords());
63+
}
64+
65+
@Test
66+
void schemaIsHeldByReferenceAndNotInProto() {
67+
Schema schema =
68+
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
69+
NdJsonReadOptions opts = new NdJsonReadOptions().schema(schema);
70+
71+
assertSame(schema, opts.schema());
72+
}
73+
74+
@Test
75+
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
76+
for (CsvReadOptions.FileCompressionType t : CsvReadOptions.FileCompressionType.values()) {
77+
NdJsonReadOptionsProto p =
78+
NdJsonReadOptionsProto.parseFrom(
79+
new NdJsonReadOptions().fileCompressionType(t).toBytes());
80+
assertEquals(
81+
"FILE_COMPRESSION_TYPE_" + t.name(),
82+
p.getFileCompressionType().name(),
83+
"mismatch for " + t);
84+
}
85+
}
86+
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertTrue;
24+
25+
import java.io.IOException;
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.util.List;
29+
30+
import org.apache.arrow.memory.BufferAllocator;
31+
import org.apache.arrow.memory.RootAllocator;
32+
import org.apache.arrow.vector.BigIntVector;
33+
import org.apache.arrow.vector.VarCharVector;
34+
import org.apache.arrow.vector.VectorSchemaRoot;
35+
import org.apache.arrow.vector.ipc.ArrowReader;
36+
import org.apache.arrow.vector.types.pojo.ArrowType;
37+
import org.apache.arrow.vector.types.pojo.Field;
38+
import org.apache.arrow.vector.types.pojo.FieldType;
39+
import org.apache.arrow.vector.types.pojo.Schema;
40+
import org.junit.jupiter.api.Test;
41+
import org.junit.jupiter.api.io.TempDir;
42+
43+
class SessionContextJsonTest {
44+
45+
private static Path writeJson(Path dir, String name, String contents) throws IOException {
46+
Path file = dir.resolve(name);
47+
Files.writeString(file, contents);
48+
return file;
49+
}
50+
51+
@Test
52+
void registerJsonInfersSchemaAndCounts(@TempDir Path tempDir) throws Exception {
53+
Path file =
54+
writeJson(
55+
tempDir,
56+
"people.json",
57+
"{\"id\":1,\"name\":\"alice\"}\n"
58+
+ "{\"id\":2,\"name\":\"bob\"}\n"
59+
+ "{\"id\":3,\"name\":\"carol\"}\n");
60+
61+
try (BufferAllocator allocator = new RootAllocator();
62+
SessionContext ctx = new SessionContext()) {
63+
ctx.registerJson("people", file.toAbsolutePath().toString());
64+
65+
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM people");
66+
ArrowReader reader = df.collect(allocator)) {
67+
assertTrue(reader.loadNextBatch());
68+
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
69+
assertEquals(3L, count.get(0));
70+
}
71+
72+
try (DataFrame df = ctx.sql("SELECT name FROM people WHERE id = 2");
73+
ArrowReader reader = df.collect(allocator)) {
74+
assertTrue(reader.loadNextBatch());
75+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
76+
assertEquals(1, root.getRowCount());
77+
VarCharVector names = (VarCharVector) root.getVector(0);
78+
assertEquals("bob", new String(names.get(0)));
79+
}
80+
}
81+
}
82+
83+
@Test
84+
void readJsonWithExplicitSchema(@TempDir Path tempDir) throws Exception {
85+
Path file =
86+
writeJson(
87+
tempDir, "headerless.json", "{\"id\":10,\"name\":\"x\"}\n{\"id\":20,\"name\":\"y\"}\n");
88+
89+
Schema schema =
90+
new Schema(
91+
List.of(
92+
new Field("id", FieldType.nullable(new ArrowType.Int(64, true)), null),
93+
new Field("name", FieldType.nullable(new ArrowType.Utf8()), null)));
94+
95+
try (BufferAllocator allocator = new RootAllocator();
96+
SessionContext ctx = new SessionContext();
97+
DataFrame df =
98+
ctx.readJson(file.toAbsolutePath().toString(), new NdJsonReadOptions().schema(schema));
99+
ArrowReader reader = df.collect(allocator)) {
100+
assertTrue(reader.loadNextBatch());
101+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
102+
assertEquals(2, root.getRowCount());
103+
assertEquals("id", root.getSchema().getFields().get(0).getName());
104+
assertEquals("name", root.getSchema().getFields().get(1).getName());
105+
}
106+
}
107+
108+
@Test
109+
void registerJsonWithCustomExtension(@TempDir Path tempDir) throws Exception {
110+
Path file = writeJson(tempDir, "data.ndjson", "{\"x\":10,\"y\":20}\n{\"x\":30,\"y\":40}\n");
111+
112+
try (BufferAllocator allocator = new RootAllocator();
113+
SessionContext ctx = new SessionContext()) {
114+
ctx.registerJson(
115+
"t", file.toAbsolutePath().toString(), new NdJsonReadOptions().fileExtension(".ndjson"));
116+
117+
try (DataFrame df = ctx.sql("SELECT SUM(x) + SUM(y) FROM t");
118+
ArrowReader reader = df.collect(allocator)) {
119+
assertTrue(reader.loadNextBatch());
120+
BigIntVector v = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
121+
assertEquals(100L, v.get(0));
122+
}
123+
}
124+
}
125+
}

‎native/build.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ fn main() {
1919
const PROTOS: &[&str] = &[
2020
"../proto/session_options.proto",
2121
"../proto/csv_read_options.proto",
22+
"../proto/json_read_options.proto",
2223
"../proto/parquet_read_options.proto",
2324
];
2425
for p in PROTOS {

0 commit comments

Comments
 (0)