Skip to content

Commit a08c9fd

Browse files
committed
test(parquet): add integration tests for ParquetReadOptions fields
1 parent ce4fd45 commit a08c9fd

1 file changed

Lines changed: 78 additions & 0 deletions

File tree

‎src/test/java/org/apache/datafusion/SessionContextParquetOptionsTest.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,21 @@
2424

2525
import java.nio.file.Files;
2626
import java.nio.file.Path;
27+
import java.nio.file.StandardCopyOption;
28+
import java.util.List;
2729

2830
import org.apache.arrow.memory.BufferAllocator;
2931
import org.apache.arrow.memory.RootAllocator;
32+
import org.apache.arrow.vector.BigIntVector;
3033
import org.apache.arrow.vector.VectorSchemaRoot;
3134
import org.apache.arrow.vector.ipc.ArrowReader;
35+
import org.apache.arrow.vector.types.pojo.ArrowType;
36+
import org.apache.arrow.vector.types.pojo.Field;
37+
import org.apache.arrow.vector.types.pojo.FieldType;
38+
import org.apache.arrow.vector.types.pojo.Schema;
3239
import org.junit.jupiter.api.Assumptions;
3340
import org.junit.jupiter.api.Test;
41+
import org.junit.jupiter.api.io.TempDir;
3442

3543
class SessionContextParquetOptionsTest {
3644

@@ -53,4 +61,74 @@ void readParquetWithDefaultOptionsCountsAllRows() throws Exception {
5361
assertTrue(total > 0);
5462
}
5563
}
64+
65+
@Test
66+
void registerParquetWithOptionsRespectsCustomExtension(@TempDir Path tempDir) throws Exception {
67+
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
68+
Assumptions.assumeTrue(
69+
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
70+
Path renamed = tempDir.resolve("lineitem.parq");
71+
Files.copy(lineitem, renamed, StandardCopyOption.REPLACE_EXISTING);
72+
73+
try (BufferAllocator allocator = new RootAllocator();
74+
SessionContext ctx = new SessionContext()) {
75+
ctx.registerParquet(
76+
"t",
77+
renamed.toAbsolutePath().toString(),
78+
new ParquetReadOptions().fileExtension(".parq"));
79+
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
80+
ArrowReader reader = df.collect(allocator)) {
81+
assertTrue(reader.loadNextBatch());
82+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
83+
assertEquals(1, root.getRowCount());
84+
BigIntVector count = (BigIntVector) root.getVector(0);
85+
assertEquals(6_001_215L, count.get(0));
86+
}
87+
}
88+
}
89+
90+
@Test
91+
void readParquetWithExplicitSchemaUsesProvidedSchema() throws Exception {
92+
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
93+
Assumptions.assumeTrue(
94+
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
95+
96+
Schema custom =
97+
new Schema(
98+
List.of(
99+
new Field("l_orderkey", FieldType.notNullable(new ArrowType.Int(64, true)), null)));
100+
101+
try (BufferAllocator allocator = new RootAllocator();
102+
SessionContext ctx = new SessionContext();
103+
DataFrame df =
104+
ctx.readParquet(
105+
lineitem.toAbsolutePath().toString(), new ParquetReadOptions().schema(custom));
106+
ArrowReader reader = df.collect(allocator)) {
107+
assertTrue(reader.loadNextBatch());
108+
Schema observed = reader.getVectorSchemaRoot().getSchema();
109+
assertEquals(1, observed.getFields().size());
110+
assertEquals("l_orderkey", observed.getFields().get(0).getName());
111+
}
112+
}
113+
114+
@Test
115+
void registerParquetWithMetadataSizeHintIsAccepted() throws Exception {
116+
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
117+
Assumptions.assumeTrue(
118+
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
119+
120+
try (BufferAllocator allocator = new RootAllocator();
121+
SessionContext ctx = new SessionContext()) {
122+
ctx.registerParquet(
123+
"t",
124+
lineitem.toAbsolutePath().toString(),
125+
new ParquetReadOptions().metadataSizeHint(1L << 20));
126+
try (DataFrame df = ctx.sql("SELECT COUNT(*) FROM t");
127+
ArrowReader reader = df.collect(allocator)) {
128+
assertTrue(reader.loadNextBatch());
129+
BigIntVector count = (BigIntVector) reader.getVectorSchemaRoot().getVector(0);
130+
assertEquals(6_001_215L, count.get(0));
131+
}
132+
}
133+
}
56134
}

0 commit comments

Comments
 (0)