|
27 | 27 |
|
28 | 28 | import org.apache.arrow.memory.BufferAllocator; |
29 | 29 | import org.apache.arrow.memory.RootAllocator; |
| 30 | +import org.apache.arrow.vector.BigIntVector; |
30 | 31 | import org.apache.arrow.vector.IntVector; |
31 | 32 | import org.apache.arrow.vector.VectorSchemaRoot; |
32 | 33 | import org.apache.arrow.vector.ipc.ArrowReader; |
|
35 | 36 | import org.apache.arrow.vector.types.pojo.Schema; |
36 | 37 | import org.apache.datafusion.DataFrame; |
37 | 38 | import org.apache.datafusion.SessionContext; |
| 39 | +import org.apache.datafusion.protobuf.BareTableReference; |
38 | 40 | import org.apache.datafusion.protobuf.EmptyRelationNode; |
| 41 | +import org.apache.datafusion.protobuf.ListingTableScanNode; |
39 | 42 | import org.apache.datafusion.protobuf.LogicalExprNode; |
40 | 43 | import org.apache.datafusion.protobuf.LogicalPlanNode; |
| 44 | +import org.apache.datafusion.protobuf.ProjectionColumns; |
41 | 45 | import org.apache.datafusion.protobuf.ProjectionNode; |
| 46 | +import org.apache.datafusion.protobuf.SortExprNode; |
| 47 | +import org.apache.datafusion.protobuf.SortNode; |
| 48 | +import org.apache.datafusion.protobuf.TableReference; |
42 | 49 | import org.junit.jupiter.api.Assumptions; |
43 | 50 | import org.junit.jupiter.api.Test; |
44 | 51 |
|
@@ -101,4 +108,81 @@ void tableSchemaReturnsExpectedLineitemFields() { |
101 | 108 | assertEquals(DateUnit.DAY, ship.getUnit()); |
102 | 109 | } |
103 | 110 | } |
| 111 | + |
| 112 | + @Test |
| 113 | + void fromProtoListingScanMatchesSql() throws Exception { |
| 114 | + Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet"); |
| 115 | + Assumptions.assumeTrue( |
| 116 | + Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first"); |
| 117 | + String absPath = lineitem.toAbsolutePath().toString(); |
| 118 | + |
| 119 | + try (BufferAllocator allocator = new RootAllocator(); |
| 120 | + SessionContext ctx = new SessionContext()) { |
| 121 | + ctx.registerParquet("lineitem", absPath); |
| 122 | + Schema arrow = ctx.tableSchema("lineitem"); |
| 123 | + DatafusionCommon.Schema schemaProto = SchemaConverter.toProto(arrow); |
| 124 | + |
| 125 | + LogicalExprNode orderKeyExpr = |
| 126 | + LogicalExprNode.newBuilder() |
| 127 | + .setColumn(DatafusionCommon.Column.newBuilder().setName("l_orderkey").build()) |
| 128 | + .build(); |
| 129 | + |
| 130 | + LogicalPlanNode plan = |
| 131 | + LogicalPlanNode.newBuilder() |
| 132 | + .setSort( |
| 133 | + SortNode.newBuilder() |
| 134 | + .setInput( |
| 135 | + LogicalPlanNode.newBuilder() |
| 136 | + .setListingScan( |
| 137 | + ListingTableScanNode.newBuilder() |
| 138 | + .setTableName( |
| 139 | + TableReference.newBuilder() |
| 140 | + .setBare( |
| 141 | + BareTableReference.newBuilder() |
| 142 | + .setTable("lineitem") |
| 143 | + .build()) |
| 144 | + .build()) |
| 145 | + .addPaths(absPath) |
| 146 | + .setFileExtension(".parquet") |
| 147 | + .setSchema(schemaProto) |
| 148 | + .setProjection( |
| 149 | + ProjectionColumns.newBuilder() |
| 150 | + .addColumns("l_orderkey") |
| 151 | + .build()) |
| 152 | + .setParquet( |
| 153 | + DatafusionCommon.ParquetFormat.getDefaultInstance()) |
| 154 | + .setTargetPartitions(1) |
| 155 | + .build()) |
| 156 | + .build()) |
| 157 | + .addExpr( |
| 158 | + SortExprNode.newBuilder() |
| 159 | + .setExpr(orderKeyExpr) |
| 160 | + .setAsc(true) |
| 161 | + .setNullsFirst(false) |
| 162 | + .build()) |
| 163 | + .setFetch(1) |
| 164 | + .build()) |
| 165 | + .build(); |
| 166 | + |
| 167 | + long protoValue; |
| 168 | + try (DataFrame df = ctx.fromProto(plan.toByteArray()); |
| 169 | + ArrowReader reader = df.collect(allocator)) { |
| 170 | + assertTrue(reader.loadNextBatch()); |
| 171 | + VectorSchemaRoot root = reader.getVectorSchemaRoot(); |
| 172 | + assertEquals(1, root.getRowCount()); |
| 173 | + protoValue = ((BigIntVector) root.getVector(0)).get(0); |
| 174 | + } |
| 175 | + |
| 176 | + long sqlValue; |
| 177 | + try (DataFrame df = ctx.sql("SELECT l_orderkey FROM lineitem ORDER BY l_orderkey LIMIT 1"); |
| 178 | + ArrowReader reader = df.collect(allocator)) { |
| 179 | + assertTrue(reader.loadNextBatch()); |
| 180 | + VectorSchemaRoot root = reader.getVectorSchemaRoot(); |
| 181 | + assertEquals(1, root.getRowCount()); |
| 182 | + sqlValue = ((BigIntVector) root.getVector(0)).get(0); |
| 183 | + } |
| 184 | + |
| 185 | + assertEquals(sqlValue, protoValue); |
| 186 | + } |
| 187 | + } |
104 | 188 | } |
0 commit comments