Skip to content

Commit 35f7347

Browse files
committed
test(proto): add Java-built ListingTableScanNode integration test
Also extend SchemaConverter to support Utf8View and LargeUtf8 Arrow types, required for TPC-H parquet files which use Utf8View for string columns.
1 parent c653e39 commit 35f7347

2 files changed

Lines changed: 95 additions & 1 deletion

File tree

‎src/main/java/org/apache/datafusion/proto/SchemaConverter.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
* used by DataFusion plan messages such as {@code ListingTableScanNode.schema}.
3939
*
4040
* <p>Supports the primitive Arrow types this project's tests exercise (Bool, signed/unsigned Int
41-
* 8..64, Float32/64, Utf8, Date32, Decimal128). Anything else raises {@link
41+
* 8..64, Float32/64, Utf8, Utf8View, LargeUtf8, Date32, Decimal128). Anything else raises {@link
4242
* UnsupportedOperationException} with a message naming the offending type.
4343
*/
4444
public final class SchemaConverter {
@@ -136,6 +136,12 @@ static DatafusionCommon.ArrowType arrowTypeToProto(ArrowType t) {
136136
if (t instanceof ArrowType.Utf8) {
137137
return b.setUTF8(empty).build();
138138
}
139+
if (t instanceof ArrowType.Utf8View) {
140+
return b.setUTF8VIEW(empty).build();
141+
}
142+
if (t instanceof ArrowType.LargeUtf8) {
143+
return b.setLARGEUTF8(empty).build();
144+
}
139145
if (t instanceof ArrowType.Date) {
140146
DateUnit u = ((ArrowType.Date) t).getUnit();
141147
if (u == DateUnit.DAY) return b.setDATE32(empty).build();
@@ -187,6 +193,10 @@ static ArrowType arrowTypeFromProto(DatafusionCommon.ArrowType p) {
187193
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
188194
case UTF8:
189195
return ArrowType.Utf8.INSTANCE;
196+
case UTF8_VIEW:
197+
return ArrowType.Utf8View.INSTANCE;
198+
case LARGE_UTF8:
199+
return ArrowType.LargeUtf8.INSTANCE;
190200
case DATE32:
191201
return new ArrowType.Date(DateUnit.DAY);
192202
case DECIMAL128:

‎src/test/java/org/apache/datafusion/proto/SessionContextProtoTest.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828
import org.apache.arrow.memory.BufferAllocator;
2929
import org.apache.arrow.memory.RootAllocator;
30+
import org.apache.arrow.vector.BigIntVector;
3031
import org.apache.arrow.vector.IntVector;
3132
import org.apache.arrow.vector.VectorSchemaRoot;
3233
import org.apache.arrow.vector.ipc.ArrowReader;
@@ -35,10 +36,16 @@
3536
import org.apache.arrow.vector.types.pojo.Schema;
3637
import org.apache.datafusion.DataFrame;
3738
import org.apache.datafusion.SessionContext;
39+
import org.apache.datafusion.protobuf.BareTableReference;
3840
import org.apache.datafusion.protobuf.EmptyRelationNode;
41+
import org.apache.datafusion.protobuf.ListingTableScanNode;
3942
import org.apache.datafusion.protobuf.LogicalExprNode;
4043
import org.apache.datafusion.protobuf.LogicalPlanNode;
44+
import org.apache.datafusion.protobuf.ProjectionColumns;
4145
import org.apache.datafusion.protobuf.ProjectionNode;
46+
import org.apache.datafusion.protobuf.SortExprNode;
47+
import org.apache.datafusion.protobuf.SortNode;
48+
import org.apache.datafusion.protobuf.TableReference;
4249
import org.junit.jupiter.api.Assumptions;
4350
import org.junit.jupiter.api.Test;
4451

@@ -101,4 +108,81 @@ void tableSchemaReturnsExpectedLineitemFields() {
101108
assertEquals(DateUnit.DAY, ship.getUnit());
102109
}
103110
}
111+
112+
@Test
113+
void fromProtoListingScanMatchesSql() throws Exception {
114+
Path lineitem = Path.of("tpch-data/sf1/lineitem.parquet");
115+
Assumptions.assumeTrue(
116+
Files.exists(lineitem), "TPC-H SF1 data not found; run `make tpch-data` first");
117+
String absPath = lineitem.toAbsolutePath().toString();
118+
119+
try (BufferAllocator allocator = new RootAllocator();
120+
SessionContext ctx = new SessionContext()) {
121+
ctx.registerParquet("lineitem", absPath);
122+
Schema arrow = ctx.tableSchema("lineitem");
123+
DatafusionCommon.Schema schemaProto = SchemaConverter.toProto(arrow);
124+
125+
LogicalExprNode orderKeyExpr =
126+
LogicalExprNode.newBuilder()
127+
.setColumn(DatafusionCommon.Column.newBuilder().setName("l_orderkey").build())
128+
.build();
129+
130+
LogicalPlanNode plan =
131+
LogicalPlanNode.newBuilder()
132+
.setSort(
133+
SortNode.newBuilder()
134+
.setInput(
135+
LogicalPlanNode.newBuilder()
136+
.setListingScan(
137+
ListingTableScanNode.newBuilder()
138+
.setTableName(
139+
TableReference.newBuilder()
140+
.setBare(
141+
BareTableReference.newBuilder()
142+
.setTable("lineitem")
143+
.build())
144+
.build())
145+
.addPaths(absPath)
146+
.setFileExtension(".parquet")
147+
.setSchema(schemaProto)
148+
.setProjection(
149+
ProjectionColumns.newBuilder()
150+
.addColumns("l_orderkey")
151+
.build())
152+
.setParquet(
153+
DatafusionCommon.ParquetFormat.getDefaultInstance())
154+
.setTargetPartitions(1)
155+
.build())
156+
.build())
157+
.addExpr(
158+
SortExprNode.newBuilder()
159+
.setExpr(orderKeyExpr)
160+
.setAsc(true)
161+
.setNullsFirst(false)
162+
.build())
163+
.setFetch(1)
164+
.build())
165+
.build();
166+
167+
long protoValue;
168+
try (DataFrame df = ctx.fromProto(plan.toByteArray());
169+
ArrowReader reader = df.collect(allocator)) {
170+
assertTrue(reader.loadNextBatch());
171+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
172+
assertEquals(1, root.getRowCount());
173+
protoValue = ((BigIntVector) root.getVector(0)).get(0);
174+
}
175+
176+
long sqlValue;
177+
try (DataFrame df = ctx.sql("SELECT l_orderkey FROM lineitem ORDER BY l_orderkey LIMIT 1");
178+
ArrowReader reader = df.collect(allocator)) {
179+
assertTrue(reader.loadNextBatch());
180+
VectorSchemaRoot root = reader.getVectorSchemaRoot();
181+
assertEquals(1, root.getRowCount());
182+
sqlValue = ((BigIntVector) root.getVector(0)).get(0);
183+
}
184+
185+
assertEquals(sqlValue, protoValue);
186+
}
187+
}
104188
}

0 commit comments

Comments
 (0)