Skip to content

Commit 6daabd9

Browse files
committed
test(dataframe): add TPC-H lineitem filter/count smoke test
1 parent 17f5851 commit 6daabd9

1 file changed

Lines changed: 33 additions & 0 deletions

File tree

‎src/test/java/org/apache/datafusion/DataFrameTransformationsTest.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,15 @@
2424
import static org.junit.jupiter.api.Assertions.assertThrows;
2525
import static org.junit.jupiter.api.Assertions.assertTrue;
2626

27+
import java.nio.file.Files;
28+
import java.nio.file.Path;
29+
2730
import org.apache.arrow.memory.BufferAllocator;
2831
import org.apache.arrow.memory.RootAllocator;
32+
import org.apache.arrow.vector.BigIntVector;
2933
import org.apache.arrow.vector.VectorSchemaRoot;
3034
import org.apache.arrow.vector.ipc.ArrowReader;
35+
import org.junit.jupiter.api.Assumptions;
3136
import org.junit.jupiter.api.Test;
3237

3338
class DataFrameTransformationsTest {
@@ -161,4 +166,32 @@ void filterMalformedPredicateThrows() {
161166
assertThrows(RuntimeException.class, () -> df.filter("this is not sql"));
162167
}
163168
}
169+
170+
@Test
171+
void lineitemFilterCountAgainstSqlBaseline() throws Exception {
172+
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
173+
Assumptions.assumeTrue(
174+
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
175+
176+
try (SessionContext ctx = new SessionContext()) {
177+
ctx.registerParquet("lineitem", lineitem.toAbsolutePath().toString());
178+
179+
long viaDataFrame;
180+
try (DataFrame df = ctx.sql("SELECT * FROM lineitem");
181+
DataFrame filtered = df.filter("l_orderkey < 100")) {
182+
viaDataFrame = filtered.count();
183+
}
184+
185+
long viaSql;
186+
try (BufferAllocator allocator = new RootAllocator();
187+
DataFrame df = ctx.sql("SELECT COUNT(*) FROM lineitem WHERE l_orderkey < 100");
188+
ArrowReader reader = df.collect(allocator)) {
189+
assertTrue(reader.loadNextBatch());
190+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
191+
viaSql = ((BigIntVector) root.getVector(0)).get(0);
192+
}
193+
194+
assertEquals(viaSql, viaDataFrame);
195+
}
196+
}
164197
}

0 commit comments

Comments
 (0)