Skip to content

Commit 63a86a5

Browse files
authored
feat(json): expose NdJsonReadOptions via registerJson and readJson (#47)
1 parent 4a967e4 commit 63a86a5

14 files changed

Lines changed: 689 additions & 42 deletions

‎core/src/main/java/org/apache/datafusion/CsvReadOptions.java‎

Lines changed: 4 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@
3333
*/
3434
public final class CsvReadOptions {
3535

36-
/** Compression of the file. Names match DataFusion's {@code FileCompressionType} variants. */
37-
public enum FileCompressionType {
38-
UNCOMPRESSED,
39-
GZIP,
40-
BZIP2,
41-
XZ,
42-
ZSTD
43-
}
44-
4536
private boolean hasHeader = true;
4637
private byte delimiter = (byte) ',';
4738
private byte quote = (byte) '"';
@@ -90,6 +81,9 @@ public CsvReadOptions newlinesInValues(boolean v) {
9081
}
9182

9283
public CsvReadOptions schemaInferMaxRecords(long n) {
84+
if (n < 0) {
85+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
86+
}
9387
this.schemaInferMaxRecords = n;
9488
return this;
9589
}
@@ -116,7 +110,7 @@ byte[] toBytes() {
116110
.setDelimiter(delimiter & 0xFF)
117111
.setQuote(quote & 0xFF)
118112
.setFileExtension(fileExtension)
119-
.setFileCompressionType(toProto(fileCompressionType));
113+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
120114
if (terminator != null) {
121115
b.setTerminator(terminator & 0xFF);
122116
}
@@ -138,22 +132,4 @@ byte[] toBytes() {
138132
Schema schema() {
139133
return schema;
140134
}
141-
142-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
143-
switch (t) {
144-
case UNCOMPRESSED:
145-
return org.apache.datafusion.protobuf.FileCompressionType
146-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
147-
case GZIP:
148-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
149-
case BZIP2:
150-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
151-
case XZ:
152-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
153-
case ZSTD:
154-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
155-
default:
156-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
157-
}
158-
}
159135
}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Compression of a file-format source. Shared by {@link CsvReadOptions} and {@link
24+
* NdJsonReadOptions} (and any future format that exposes the same set of compressions). Variant
25+
* names match DataFusion's Rust {@code FileCompressionType} so they round-trip across JNI.
26+
*/
27+
public enum FileCompressionType {
28+
UNCOMPRESSED,
29+
GZIP,
30+
BZIP2,
31+
XZ,
32+
ZSTD
33+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Internal helpers for translating between the user-facing {@link FileCompressionType} enum and the
24+
* protobuf-generated {@code org.apache.datafusion.protobuf.FileCompressionType} enum used on the
25+
* wire. Same variant set, different Java types.
26+
*/
27+
final class FileCompressionTypes {
28+
private FileCompressionTypes() {}
29+
30+
static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
31+
switch (t) {
32+
case UNCOMPRESSED:
33+
return org.apache.datafusion.protobuf.FileCompressionType
34+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
35+
case GZIP:
36+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
37+
case BZIP2:
38+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
39+
case XZ:
40+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
41+
case ZSTD:
42+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
43+
default:
44+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
45+
}
46+
}
47+
}
Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.arrow.vector.types.pojo.Schema;
23+
24+
/**
25+
* Configuration knobs for newline-delimited JSON sources passed to {@link
26+
* SessionContext#registerJson(String, String, NdJsonReadOptions)} and {@link
27+
* SessionContext#readJson(String, NdJsonReadOptions)}.
28+
*
29+
* <p>Mirrors a subset of DataFusion's {@code NdJsonReadOptions}. All setters return {@code this}
30+
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
31+
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
32+
* is used).
33+
*/
34+
public final class NdJsonReadOptions {
35+
36+
private String fileExtension = ".json";
37+
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
38+
private Long schemaInferMaxRecords;
39+
private Schema schema;
40+
41+
public NdJsonReadOptions fileExtension(String ext) {
42+
this.fileExtension = ext;
43+
return this;
44+
}
45+
46+
public NdJsonReadOptions fileCompressionType(FileCompressionType t) {
47+
this.fileCompressionType = t;
48+
return this;
49+
}
50+
51+
public NdJsonReadOptions schemaInferMaxRecords(long n) {
52+
if (n < 0) {
53+
throw new IllegalArgumentException("schemaInferMaxRecords must be non-negative, got " + n);
54+
}
55+
this.schemaInferMaxRecords = n;
56+
return this;
57+
}
58+
59+
public NdJsonReadOptions schema(Schema schema) {
60+
this.schema = schema;
61+
return this;
62+
}
63+
64+
byte[] toBytes() {
65+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
66+
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
67+
.setFileExtension(fileExtension)
68+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
69+
if (schemaInferMaxRecords != null) {
70+
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
71+
}
72+
return b.build().toByteArray();
73+
}
74+
75+
Schema schema() {
76+
return schema;
77+
}
78+
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,67 @@ public DataFrame readCsv(String path, CsvReadOptions options) {
188188
return new DataFrame(dfHandle);
189189
}
190190

191+
public void registerJson(String name, String path) {
192+
registerJson(name, path, new NdJsonReadOptions());
193+
}
194+
195+
/**
196+
* Register a newline-delimited JSON file (or directory of NDJSON files) as a table with the
197+
* supplied {@link NdJsonReadOptions}.
198+
*
199+
* @throws RuntimeException if registration fails (path not found, schema inference error, etc.).
200+
*/
201+
public void registerJson(String name, String path, NdJsonReadOptions options) {
202+
if (nativeHandle == 0) {
203+
throw new IllegalStateException("SessionContext is closed");
204+
}
205+
if (name == null) {
206+
throw new IllegalArgumentException("registerJson name must be non-null");
207+
}
208+
if (path == null) {
209+
throw new IllegalArgumentException("registerJson path must be non-null");
210+
}
211+
if (options == null) {
212+
throw new IllegalArgumentException("registerJson options must be non-null");
213+
}
214+
registerJsonWithOptions(
215+
nativeHandle,
216+
name,
217+
path,
218+
options.toBytes(),
219+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
220+
}
221+
222+
/** Read a newline-delimited JSON file as a {@link DataFrame} without registering it. */
223+
public DataFrame readJson(String path) {
224+
return readJson(path, new NdJsonReadOptions());
225+
}
226+
227+
/**
228+
* Read a newline-delimited JSON file as a {@link DataFrame} with the supplied {@link
229+
* NdJsonReadOptions}.
230+
*
231+
* @throws RuntimeException if the read fails.
232+
*/
233+
public DataFrame readJson(String path, NdJsonReadOptions options) {
234+
if (nativeHandle == 0) {
235+
throw new IllegalStateException("SessionContext is closed");
236+
}
237+
if (path == null) {
238+
throw new IllegalArgumentException("readJson path must be non-null");
239+
}
240+
if (options == null) {
241+
throw new IllegalArgumentException("readJson options must be non-null");
242+
}
243+
long dfHandle =
244+
readJsonWithOptions(
245+
nativeHandle,
246+
path,
247+
options.toBytes(),
248+
options.schema() != null ? serializeSchemaIpc(options.schema()) : null);
249+
return new DataFrame(dfHandle);
250+
}
251+
191252
public void registerParquet(String name, String path) {
192253
registerParquet(name, path, new ParquetReadOptions());
193254
}
@@ -277,5 +338,11 @@ private static native void registerCsvWithOptions(
277338
private static native long readCsvWithOptions(
278339
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
279340

341+
private static native void registerJsonWithOptions(
342+
long handle, String name, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
343+
344+
private static native long readJsonWithOptions(
345+
long handle, String path, byte[] optionsBytes, byte[] schemaIpcBytes);
346+
280347
private static native void closeSessionContext(long handle);
281348
}

‎core/src/test/java/org/apache/datafusion/CsvReadOptionsTest.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.junit.jupiter.api.Assertions.assertEquals;
2323
import static org.junit.jupiter.api.Assertions.assertFalse;
2424
import static org.junit.jupiter.api.Assertions.assertSame;
25+
import static org.junit.jupiter.api.Assertions.assertThrows;
2526
import static org.junit.jupiter.api.Assertions.assertTrue;
2627

2728
import java.util.List;
@@ -31,7 +32,6 @@
3132
import org.apache.arrow.vector.types.pojo.FieldType;
3233
import org.apache.arrow.vector.types.pojo.Schema;
3334
import org.apache.datafusion.protobuf.CsvReadOptionsProto;
34-
import org.apache.datafusion.protobuf.FileCompressionType;
3535
import org.junit.jupiter.api.Test;
3636

3737
import com.google.protobuf.InvalidProtocolBufferException;
@@ -47,7 +47,8 @@ void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
4747
assertEquals((int) '"', p.getQuote());
4848
assertEquals(".csv", p.getFileExtension());
4949
assertEquals(
50-
FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED, p.getFileCompressionType());
50+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED,
51+
p.getFileCompressionType());
5152

5253
assertFalse(p.hasTerminator());
5354
assertFalse(p.hasEscape());
@@ -69,7 +70,7 @@ void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferExcepti
6970
.newlinesInValues(true)
7071
.schemaInferMaxRecords(10L)
7172
.fileExtension(".tsv")
72-
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP);
73+
.fileCompressionType(FileCompressionType.GZIP);
7374

7475
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(opts.toBytes());
7576

@@ -82,7 +83,9 @@ void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferExcepti
8283
assertTrue(p.getNewlinesInValues());
8384
assertEquals(10L, p.getSchemaInferMaxRecords());
8485
assertEquals(".tsv", p.getFileExtension());
85-
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
86+
assertEquals(
87+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
88+
p.getFileCompressionType());
8689
}
8790

8891
@Test
@@ -96,7 +99,7 @@ void schemaIsHeldByReferenceAndNotInProto() {
9699

97100
@Test
98101
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
99-
for (CsvReadOptions.FileCompressionType t : CsvReadOptions.FileCompressionType.values()) {
102+
for (FileCompressionType t : FileCompressionType.values()) {
100103
CsvReadOptionsProto p =
101104
CsvReadOptionsProto.parseFrom(new CsvReadOptions().fileCompressionType(t).toBytes());
102105
assertEquals(
@@ -105,4 +108,14 @@ void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException
105108
"mismatch for " + t);
106109
}
107110
}
111+
112+
@Test
113+
void schemaInferMaxRecordsRejectsNegative() {
114+
// The proto wire field is uint64, so a negative long would be reinterpreted
115+
// as a huge unsigned value on the Rust side and silently expand schema
116+
// inference across the full dataset. Reject at the Java setter instead.
117+
CsvReadOptions opts = new CsvReadOptions();
118+
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(-1L));
119+
assertThrows(IllegalArgumentException.class, () -> opts.schemaInferMaxRecords(Long.MIN_VALUE));
120+
}
108121
}

0 commit comments

Comments
 (0)