Skip to content

Commit f13b51b

Browse files
committed
test: round-trip read options through protobuf
1 parent 111e70e commit f13b51b

2 files changed

Lines changed: 79 additions & 52 deletions

File tree

‎src/test/java/org/apache/datafusion/CsvReadOptionsTest.java‎

Lines changed: 54 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
package org.apache.datafusion;
2121

2222
import static org.junit.jupiter.api.Assertions.assertEquals;
23-
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
2424
import static org.junit.jupiter.api.Assertions.assertSame;
2525
import static org.junit.jupiter.api.Assertions.assertTrue;
2626

@@ -30,31 +30,34 @@
3030
import org.apache.arrow.vector.types.pojo.Field;
3131
import org.apache.arrow.vector.types.pojo.FieldType;
3232
import org.apache.arrow.vector.types.pojo.Schema;
33+
import org.apache.datafusion.protobuf.CsvReadOptionsProto;
34+
import org.apache.datafusion.protobuf.FileCompressionType;
3335
import org.junit.jupiter.api.Test;
3436

37+
import com.google.protobuf.InvalidProtocolBufferException;
38+
3539
class CsvReadOptionsTest {
3640

3741
@Test
38-
void defaultsMatchDataFusion() {
39-
CsvReadOptions opts = new CsvReadOptions();
40-
assertTrue(opts.hasHeader());
41-
assertEquals((byte) ',', opts.delimiter());
42-
assertEquals((byte) '"', opts.quote());
43-
assertNull(opts.terminator());
44-
assertNull(opts.escape());
45-
assertNull(opts.comment());
46-
assertNull(opts.newlinesInValues());
47-
assertNull(opts.schemaInferMaxRecords());
48-
assertEquals(".csv", opts.fileExtension());
49-
assertEquals(CsvReadOptions.FileCompressionType.UNCOMPRESSED, opts.fileCompressionType());
50-
assertNull(opts.schema());
42+
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
43+
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(new CsvReadOptions().toBytes());
44+
45+
assertTrue(p.getHasHeader());
46+
assertEquals((int) ',', p.getDelimiter());
47+
assertEquals((int) '"', p.getQuote());
48+
assertEquals(".csv", p.getFileExtension());
49+
assertEquals(
50+
FileCompressionType.FILE_COMPRESSION_TYPE_UNCOMPRESSED, p.getFileCompressionType());
51+
52+
assertFalse(p.hasTerminator());
53+
assertFalse(p.hasEscape());
54+
assertFalse(p.hasComment());
55+
assertFalse(p.hasNewlinesInValues());
56+
assertFalse(p.hasSchemaInferMaxRecords());
5157
}
5258

5359
@Test
54-
void fluentSettersChainAndMutate() {
55-
Schema schema =
56-
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
57-
60+
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
5861
CsvReadOptions opts =
5962
new CsvReadOptions()
6063
.hasHeader(false)
@@ -66,19 +69,40 @@ void fluentSettersChainAndMutate() {
6669
.newlinesInValues(true)
6770
.schemaInferMaxRecords(10L)
6871
.fileExtension(".tsv")
69-
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP)
70-
.schema(schema);
72+
.fileCompressionType(CsvReadOptions.FileCompressionType.GZIP);
73+
74+
CsvReadOptionsProto p = CsvReadOptionsProto.parseFrom(opts.toBytes());
75+
76+
assertFalse(p.getHasHeader());
77+
assertEquals((int) '|', p.getDelimiter());
78+
assertEquals((int) '\'', p.getQuote());
79+
assertEquals((int) '\n', p.getTerminator());
80+
assertEquals((int) '\\', p.getEscape());
81+
assertEquals((int) '#', p.getComment());
82+
assertTrue(p.getNewlinesInValues());
83+
assertEquals(10L, p.getSchemaInferMaxRecords());
84+
assertEquals(".tsv", p.getFileExtension());
85+
assertEquals(FileCompressionType.FILE_COMPRESSION_TYPE_GZIP, p.getFileCompressionType());
86+
}
87+
88+
@Test
89+
void schemaIsHeldByReferenceAndNotInProto() {
90+
Schema schema =
91+
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
92+
CsvReadOptions opts = new CsvReadOptions().schema(schema);
7193

72-
assertEquals(false, opts.hasHeader());
73-
assertEquals((byte) '|', opts.delimiter());
74-
assertEquals((byte) '\'', opts.quote());
75-
assertEquals(Byte.valueOf((byte) '\n'), opts.terminator());
76-
assertEquals(Byte.valueOf((byte) '\\'), opts.escape());
77-
assertEquals(Byte.valueOf((byte) '#'), opts.comment());
78-
assertEquals(Boolean.TRUE, opts.newlinesInValues());
79-
assertEquals(Long.valueOf(10L), opts.schemaInferMaxRecords());
80-
assertEquals(".tsv", opts.fileExtension());
81-
assertEquals(CsvReadOptions.FileCompressionType.GZIP, opts.fileCompressionType());
8294
assertSame(schema, opts.schema());
8395
}
96+
97+
@Test
98+
void allCompressionTypesMapThroughProto() throws InvalidProtocolBufferException {
99+
for (CsvReadOptions.FileCompressionType t : CsvReadOptions.FileCompressionType.values()) {
100+
CsvReadOptionsProto p =
101+
CsvReadOptionsProto.parseFrom(new CsvReadOptions().fileCompressionType(t).toBytes());
102+
assertEquals(
103+
"FILE_COMPRESSION_TYPE_" + t.name(),
104+
p.getFileCompressionType().name(),
105+
"mismatch for " + t);
106+
}
107+
}
84108
}

‎src/test/java/org/apache/datafusion/ParquetReadOptionsTest.java‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
package org.apache.datafusion;
2121

2222
import static org.junit.jupiter.api.Assertions.assertEquals;
23-
import static org.junit.jupiter.api.Assertions.assertNull;
23+
import static org.junit.jupiter.api.Assertions.assertFalse;
2424
import static org.junit.jupiter.api.Assertions.assertSame;
2525
import static org.junit.jupiter.api.Assertions.assertTrue;
2626

@@ -30,44 +30,47 @@
3030
import org.apache.arrow.vector.types.pojo.Field;
3131
import org.apache.arrow.vector.types.pojo.FieldType;
3232
import org.apache.arrow.vector.types.pojo.Schema;
33+
import org.apache.datafusion.protobuf.ParquetReadOptionsProto;
3334
import org.junit.jupiter.api.Test;
3435

36+
import com.google.protobuf.InvalidProtocolBufferException;
37+
3538
class ParquetReadOptionsTest {
3639

3740
@Test
38-
void defaultsMatchDataFusion() {
39-
ParquetReadOptions opts = new ParquetReadOptions();
40-
assertEquals(".parquet", opts.fileExtension());
41-
assertNull(opts.parquetPruning());
42-
assertNull(opts.skipMetadata());
43-
assertNull(opts.metadataSizeHint());
44-
assertNull(opts.schema());
41+
void defaultsRoundTripThroughProto() throws InvalidProtocolBufferException {
42+
ParquetReadOptionsProto p =
43+
ParquetReadOptionsProto.parseFrom(new ParquetReadOptions().toBytes());
44+
45+
assertEquals(".parquet", p.getFileExtension());
46+
assertFalse(p.hasParquetPruning());
47+
assertFalse(p.hasSkipMetadata());
48+
assertFalse(p.hasMetadataSizeHint());
4549
}
4650

4751
@Test
48-
void fluentSettersChainAndMutate() {
49-
Schema schema =
50-
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
51-
52+
void fullyConfiguredRoundTripsThroughProto() throws InvalidProtocolBufferException {
5253
ParquetReadOptions opts =
5354
new ParquetReadOptions()
54-
.fileExtension(".parq")
55+
.fileExtension(".par")
5556
.parquetPruning(true)
5657
.skipMetadata(false)
57-
.metadataSizeHint(1_048_576L)
58-
.schema(schema);
58+
.metadataSizeHint(4096L);
5959

60-
assertEquals(".parq", opts.fileExtension());
61-
assertEquals(Boolean.TRUE, opts.parquetPruning());
62-
assertEquals(Boolean.FALSE, opts.skipMetadata());
63-
assertEquals(Long.valueOf(1_048_576L), opts.metadataSizeHint());
64-
assertTrue(opts.schema() == schema);
60+
ParquetReadOptionsProto p = ParquetReadOptionsProto.parseFrom(opts.toBytes());
61+
62+
assertEquals(".par", p.getFileExtension());
63+
assertTrue(p.getParquetPruning());
64+
assertFalse(p.getSkipMetadata());
65+
assertEquals(4096L, p.getMetadataSizeHint());
6566
}
6667

6768
@Test
68-
void schemaSetterRetainsReferenceIdentity() {
69-
Schema schema = new Schema(List.of());
69+
void schemaIsHeldByReferenceAndNotInProto() {
70+
Schema schema =
71+
new Schema(List.of(new Field("x", FieldType.nullable(new ArrowType.Int(32, true)), null)));
7072
ParquetReadOptions opts = new ParquetReadOptions().schema(schema);
73+
7174
assertSame(schema, opts.schema());
7275
}
7376
}

0 commit comments

Comments
 (0)