Skip to content

Commit 9885fc2

Browse files
committed
Promote FileCompressionType into a new shared proto/file_compression_type.proto
1 parent 29503c0 commit 9885fc2

12 files changed

Lines changed: 181 additions & 77 deletions

‎core/src/main/java/org/apache/datafusion/CsvReadOptions.java‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@
3333
*/
3434
public final class CsvReadOptions {
3535

36-
/** Compression of the file. Names match DataFusion's {@code FileCompressionType} variants. */
37-
public enum FileCompressionType {
38-
UNCOMPRESSED,
39-
GZIP,
40-
BZIP2,
41-
XZ,
42-
ZSTD
43-
}
44-
4536
private boolean hasHeader = true;
4637
private byte delimiter = (byte) ',';
4738
private byte quote = (byte) '"';
@@ -116,7 +107,7 @@ byte[] toBytes() {
116107
.setDelimiter(delimiter & 0xFF)
117108
.setQuote(quote & 0xFF)
118109
.setFileExtension(fileExtension)
119-
.setFileCompressionType(toProto(fileCompressionType));
110+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
120111
if (terminator != null) {
121112
b.setTerminator(terminator & 0xFF);
122113
}
@@ -138,22 +129,4 @@ byte[] toBytes() {
138129
Schema schema() {
139130
return schema;
140131
}
141-
142-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
143-
switch (t) {
144-
case UNCOMPRESSED:
145-
return org.apache.datafusion.protobuf.FileCompressionType
146-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
147-
case GZIP:
148-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
149-
case BZIP2:
150-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
151-
case XZ:
152-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
153-
case ZSTD:
154-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
155-
default:
156-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
157-
}
158-
}
159132
}
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Compression of a file-format source. Shared by {@link CsvReadOptions} and {@link
24+
* NdJsonReadOptions} (and any future format that exposes the same set of compressions). Variant
25+
* names match DataFusion's Rust {@code FileCompressionType} so they round-trip across JNI.
26+
*/
27+
public enum FileCompressionType {
28+
UNCOMPRESSED,
29+
GZIP,
30+
BZIP2,
31+
XZ,
32+
ZSTD
33+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Internal helpers for translating between the user-facing {@link FileCompressionType} enum and the
24+
* protobuf-generated {@code org.apache.datafusion.protobuf.FileCompressionType} enum used on the
25+
* wire. Same variant set, different Java types.
26+
*/
27+
final class FileCompressionTypes {
28+
private FileCompressionTypes() {}
29+
30+
static org.apache.datafusion.protobuf.FileCompressionType toProto(FileCompressionType t) {
31+
switch (t) {
32+
case UNCOMPRESSED:
33+
return org.apache.datafusion.protobuf.FileCompressionType
34+
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
35+
case GZIP:
36+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
37+
case BZIP2:
38+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
39+
case XZ:
40+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
41+
case ZSTD:
42+
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
43+
default:
44+
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
45+
}
46+
}
47+
}

‎core/src/main/java/org/apache/datafusion/NdJsonReadOptions.java‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,11 @@
3030
* for fluent chaining. Defaults match the Rust struct: {@code fileExtension = ".json"}, {@code
3131
* fileCompressionType = UNCOMPRESSED}; {@code schemaInferMaxRecords} unset (the DataFusion default
3232
* is used).
33-
*
34-
* <p>{@code FileCompressionType} is reused from {@link CsvReadOptions} since both formats accept
35-
* the same set of compressions.
3633
*/
3734
public final class NdJsonReadOptions {
3835

3936
private String fileExtension = ".json";
40-
private CsvReadOptions.FileCompressionType fileCompressionType =
41-
CsvReadOptions.FileCompressionType.UNCOMPRESSED;
37+
private FileCompressionType fileCompressionType = FileCompressionType.UNCOMPRESSED;
4238
private Long schemaInferMaxRecords;
4339
private Schema schema;
4440

@@ -47,7 +43,7 @@ public NdJsonReadOptions fileExtension(String ext) {
4743
return this;
4844
}
4945

50-
public NdJsonReadOptions fileCompressionType(CsvReadOptions.FileCompressionType t) {
46+
public NdJsonReadOptions fileCompressionType(FileCompressionType t) {
5147
this.fileCompressionType = t;
5248
return this;
5349
}
@@ -66,7 +62,7 @@ byte[] toBytes() {
6662
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.Builder b =
6763
org.apache.datafusion.protobuf.NdJsonReadOptionsProto.newBuilder()
6864
.setFileExtension(fileExtension)
69-
.setFileCompressionType(toProto(fileCompressionType));
65+
.setFileCompressionType(FileCompressionTypes.toProto(fileCompressionType));
7066
if (schemaInferMaxRecords != null) {
7167
b.setSchemaInferMaxRecords(schemaInferMaxRecords);
7268
}
@@ -76,23 +72,4 @@ byte[] toBytes() {
7672
Schema schema() {
7773
return schema;
7874
}
79-
80-
private static org.apache.datafusion.protobuf.FileCompressionType toProto(
81-
CsvReadOptions.FileCompressionType t) {
82-
switch (t) {
83-
case UNCOMPRESSED:
84-
return org.apache.datafusion.protobuf.FileCompressionType
85-
.FILE_COMPRESSION_TYPE_UNCOMPRESSED;
86-
case GZIP:
87-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP;
88-
case BZIP2:
89-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_BZIP2;
90-
case XZ:
91-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_XZ;
92-
case ZSTD:
93-
return org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_ZSTD;
94-
default:
95-
throw new IllegalArgumentException("unhandled FileCompressionType: " + t);
96-
}
97-
}
9875
}

‎core/src/main/java/org/apache/datafusion/SessionContext.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,15 @@ public void registerJson(String name, String path, NdJsonReadOptions options) {
174174
if (nativeHandle == 0) {
175175
throw new IllegalStateException("SessionContext is closed");
176176
}
177+
if (name == null) {
178+
throw new IllegalArgumentException("registerJson name must be non-null");
179+
}
180+
if (path == null) {
181+
throw new IllegalArgumentException("registerJson path must be non-null");
182+
}
183+
if (options == null) {
184+
throw new IllegalArgumentException("registerJson options must be non-null");
185+
}
177186
registerJsonWithOptions(
178187
nativeHandle,
179188
name,
@@ -197,6 +206,12 @@ public DataFrame readJson(String path, NdJsonReadOptions options) {
197206
if (nativeHandle == 0) {
198207
throw new IllegalStateException("SessionContext is closed");
199208
}
209+
if (path == null) {
210+
throw new IllegalArgumentException("readJson path must be non-null");
211+
}
212+
if (options == null) {
213+
throw new IllegalArgumentException("readJson options must be non-null");
214+
}
200215
long dfHandle =
201216
readJsonWithOptions(
202217
nativeHandle,

‎core/src/test/java/org/apache/datafusion/CsvReadOptionsTest.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.apache.arrow.vector.types.pojo.FieldType;
3232
import org.apache.arrow.vector.types.pojo.Schema;
3333
import org.apache.datafusion.protobuf.CsvReadOptionsProto;
34-
import org.apache.datafusion.protobuf.FileCompressionType;
3534
import org.junit.jupiter.api.Test;
3635

3736
import com.google.protobuf.InvalidProtocolBufferException;
@@ -47,7 +46,8 @@ void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
4746
assertEquals((int) '"', p.getQuote());
4847
assertEquals(".csv", p.getFileExtension());
4948
assertEquals(
50-
FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED, p.getFileCompressionType());
49+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED,
50+
p.getFileCompressionType());
5151

5252
assertFalse(p.hasTerminator());
5353
assertFalse(p.hasEscape());
@@ -69,7 +69,7 @@ void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferExcepti
6969
.newlinesInValues(true)
7070
.schemaInferMaxRecords(10L)
7171
.fileExtension(".tsv")
72-
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP);
72+
.fileCompressionType(FileCompressionType.GZIP);
7373

7474
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(opts.toBytes());
7575

@@ -82,7 +82,9 @@ void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferExcepti
8282
assertTrue(p.getNewlinesInValues());
8383
assertEquals(10L, p.getSchemaInferMaxRecords());
8484
assertEquals(".tsv", p.getFileExtension());
85-
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
85+
assertEquals(
86+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
87+
p.getFileCompressionType());
8688
}
8789

8890
@Test
@@ -96,7 +98,7 @@ void schemaIsHeldByReferenceAndNotInProto() {
9698

9799
@Test
98100
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
99-
for (CsvReadOptions.FileCompressionType t : CsvReadOptions.FileCompressionType.values()) {
101+
for (FileCompressionType t : FileCompressionType.values()) {
100102
CsvReadOptionsProto p =
101103
CsvReadOptionsProto.parseFrom(new CsvReadOptions().fileCompressionType(t).toBytes());
102104
assertEquals(

‎core/src/test/java/org/apache/datafusion/NdJsonReadOptionsTest.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import org.apache.arrow.vector.types.pojo.Field;
3030
import org.apache.arrow.vector.types.pojo.FieldType;
3131
import org.apache.arrow.vector.types.pojo.Schema;
32-
import org.apache.datafusion.protobuf.FileCompressionType;
3332
import org.apache.datafusion.protobuf.NdJsonReadOptionsProto;
3433
import org.junit.jupiter.api.Test;
3534

@@ -43,7 +42,8 @@ void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
4342

4443
assertEquals(".json", p.getFileExtension());
4544
assertEquals(
46-
FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED, p.getFileCompressionType());
45+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED,
46+
p.getFileCompressionType());
4747
assertFalse(p.hasSchemaInferMaxRecords());
4848
}
4949

@@ -52,13 +52,15 @@ void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferExcepti
5252
NdJsonReadOptions opts =
5353
new NdJsonReadOptions()
5454
.fileExtension(".ndjson")
55-
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP)
55+
.fileCompressionType(FileCompressionType.GZIP)
5656
.schemaInferMaxRecords(50L);
5757

5858
NdJsonReadOptionsProto p = NdJsonReadOptionsProto.parseFrom(opts.toBytes());
5959

6060
assertEquals(".ndjson", p.getFileExtension());
61-
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
61+
assertEquals(
62+
org.apache.datafusion.protobuf.FileCompressionType.FILE_COMPRESSION_TYPE_GZIP,
63+
p.getFileCompressionType());
6264
assertEquals(50L, p.getSchemaInferMaxRecords());
6365
}
6466

@@ -73,7 +75,7 @@ void schemaIsHeldByReferenceAndNotInProto() {
7375

7476
@Test
7577
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
76-
for (CsvReadOptions.FileCompressionType t : CsvReadOptions.FileCompressionType.values()) {
78+
for (FileCompressionType t : FileCompressionType.values()) {
7779
NdJsonReadOptionsProto p =
7880
NdJsonReadOptionsProto.parseFrom(
7981
new NdJsonReadOptions().fileCompressionType(t).toBytes());

‎core/src/test/java/org/apache/datafusion/SessionContextJsonTest.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.apache.datafusion;
2121

2222
import static org.junit.jupiter.api.Assertions.assertEquals;
23+
import static org.junit.jupiter.api.Assertions.assertThrows;
2324
import static org.junit.jupiter.api.Assertions.assertTrue;
2425

2526
import java.io.IOException;
@@ -122,4 +123,26 @@ void registerJsonWithCustomExtension(@TempDir Path tempDir) throws Exception {
122123
}
123124
}
124125
}
126+
127+
@Test
128+
void registerJsonRejectsNullArguments() {
129+
try (SessionContext ctx = new SessionContext()) {
130+
NdJsonReadOptions opts = new NdJsonReadOptions();
131+
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson(null, "/p"));
132+
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", null));
133+
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson(null, "/p", opts));
134+
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", null, opts));
135+
assertThrows(IllegalArgumentException.class, () -> ctx.registerJson("t", "/p", null));
136+
}
137+
}
138+
139+
@Test
140+
void readJsonRejectsNullArguments() {
141+
try (SessionContext ctx = new SessionContext()) {
142+
NdJsonReadOptions opts = new NdJsonReadOptions();
143+
assertThrows(IllegalArgumentException.class, () -> ctx.readJson(null));
144+
assertThrows(IllegalArgumentException.class, () -> ctx.readJson(null, opts));
145+
assertThrows(IllegalArgumentException.class, () -> ctx.readJson("/p", null));
146+
}
147+
}
125148
}

‎native/build.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
fn main() {
1919
const PROTOS: &[&str] = &[
2020
"../proto/session_options.proto",
21+
"../proto/file_compression_type.proto",
2122
"../proto/csv_read_options.proto",
2223
"../proto/json_read_options.proto",
2324
"../proto/parquet_read_options.proto",

‎proto/csv_read_options.proto‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ syntax = "proto3";
1919

2020
package datafusion_java;
2121

22+
import "file_compression_type.proto";
23+
2224
option java_package = "org.apache.datafusion.protobuf";
2325
option java_multiple_files = true;
2426

@@ -37,12 +39,3 @@ message CsvReadOptionsProto {
3739
string file_extension = 9;
3840
FileCompressionType file_compression_type = 10;
3941
}
40-
41-
enum FileCompressionType {
42-
FILE_COMPRESSION_TYPE_UNSPECIFIED = 0;
43-
FILE_COMPRESSION_TYPE_UNCOMPRESSED = 1;
44-
FILE_COMPRESSION_TYPE_GZIP = 2;
45-
FILE_COMPRESSION_TYPE_BZIP2 = 3;
46-
FILE_COMPRESSION_TYPE_XZ = 4;
47-
FILE_COMPRESSION_TYPE_ZSTD = 5;
48-
}

0 commit comments

Comments
 (0)