Skip to content

Commit 39759e6

Browse files
committed
feat(proto): add SchemaConverter for Arrow Schema <-> datafusion_common.Schema
1 parent 8967c90 commit 39759e6

1 file changed

Lines changed: 174 additions & 0 deletions

File tree

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion.proto;
21+
22+
import java.util.ArrayList;
23+
import java.util.LinkedHashMap;
24+
import java.util.List;
25+
import java.util.Map;
26+
27+
import org.apache.arrow.vector.types.DateUnit;
28+
import org.apache.arrow.vector.types.FloatingPointPrecision;
29+
import org.apache.arrow.vector.types.pojo.ArrowType;
30+
import org.apache.arrow.vector.types.pojo.Field;
31+
import org.apache.arrow.vector.types.pojo.FieldType;
32+
import org.apache.arrow.vector.types.pojo.Schema;
33+
34+
import datafusion_common.DatafusionCommon;
35+
36+
/**
37+
* Convert between Arrow Java {@link Schema} and the {@code datafusion_common.Schema} protobuf
38+
* shape used by DataFusion plan messages such as {@code ListingTableScanNode.schema}.
39+
*
40+
* <p>Supports the primitive Arrow types this project's tests exercise (Bool, signed/unsigned Int
41+
* 8..64, Float32/64, Utf8, Date32, Decimal128). Anything else raises
42+
* {@link UnsupportedOperationException} with a message naming the offending type.
43+
*/
44+
public final class SchemaConverter {
45+
46+
private SchemaConverter() {}
47+
48+
public static DatafusionCommon.Schema toProto(Schema arrow) {
49+
DatafusionCommon.Schema.Builder builder = DatafusionCommon.Schema.newBuilder();
50+
for (Field f : arrow.getFields()) {
51+
builder.addColumns(fieldToProto(f));
52+
}
53+
if (arrow.getCustomMetadata() != null) {
54+
builder.putAllMetadata(arrow.getCustomMetadata());
55+
}
56+
return builder.build();
57+
}
58+
59+
public static Schema fromProto(DatafusionCommon.Schema proto) {
60+
List<Field> fields = new ArrayList<>(proto.getColumnsCount());
61+
for (DatafusionCommon.Field f : proto.getColumnsList()) {
62+
fields.add(fieldFromProto(f));
63+
}
64+
Map<String, String> metadata = new LinkedHashMap<>(proto.getMetadataMap());
65+
return new Schema(fields, metadata);
66+
}
67+
68+
static DatafusionCommon.Field fieldToProto(Field f) {
69+
DatafusionCommon.Field.Builder b =
70+
DatafusionCommon.Field.newBuilder()
71+
.setName(f.getName())
72+
.setArrowType(arrowTypeToProto(f.getType()))
73+
.setNullable(f.isNullable());
74+
if (f.getMetadata() != null) {
75+
b.putAllMetadata(f.getMetadata());
76+
}
77+
return b.build();
78+
}
79+
80+
static Field fieldFromProto(DatafusionCommon.Field f) {
81+
ArrowType type = arrowTypeFromProto(f.getArrowType());
82+
FieldType ft = new FieldType(f.getNullable(), type, null, f.getMetadataMap());
83+
return new Field(f.getName(), ft, null);
84+
}
85+
86+
static DatafusionCommon.ArrowType arrowTypeToProto(ArrowType t) {
87+
DatafusionCommon.EmptyMessage empty = DatafusionCommon.EmptyMessage.getDefaultInstance();
88+
DatafusionCommon.ArrowType.Builder b = DatafusionCommon.ArrowType.newBuilder();
89+
90+
if (t instanceof ArrowType.Bool) {
91+
return b.setBOOL(empty).build();
92+
}
93+
if (t instanceof ArrowType.Int) {
94+
ArrowType.Int i = (ArrowType.Int) t;
95+
if (i.getIsSigned()) {
96+
switch (i.getBitWidth()) {
97+
case 8: return b.setINT8(empty).build();
98+
case 16: return b.setINT16(empty).build();
99+
case 32: return b.setINT32(empty).build();
100+
case 64: return b.setINT64(empty).build();
101+
default:
102+
throw new UnsupportedOperationException(
103+
"Arrow type Int signed width " + i.getBitWidth() + " not yet supported by SchemaConverter");
104+
}
105+
} else {
106+
switch (i.getBitWidth()) {
107+
case 8: return b.setUINT8(empty).build();
108+
case 16: return b.setUINT16(empty).build();
109+
case 32: return b.setUINT32(empty).build();
110+
case 64: return b.setUINT64(empty).build();
111+
default:
112+
throw new UnsupportedOperationException(
113+
"Arrow type Int unsigned width " + i.getBitWidth() + " not yet supported by SchemaConverter");
114+
}
115+
}
116+
}
117+
if (t instanceof ArrowType.FloatingPoint) {
118+
FloatingPointPrecision p = ((ArrowType.FloatingPoint) t).getPrecision();
119+
if (p == FloatingPointPrecision.SINGLE) return b.setFLOAT32(empty).build();
120+
if (p == FloatingPointPrecision.DOUBLE) return b.setFLOAT64(empty).build();
121+
throw new UnsupportedOperationException(
122+
"Arrow type FloatingPoint " + p + " not yet supported by SchemaConverter");
123+
}
124+
if (t instanceof ArrowType.Utf8) {
125+
return b.setUTF8(empty).build();
126+
}
127+
if (t instanceof ArrowType.Date) {
128+
DateUnit u = ((ArrowType.Date) t).getUnit();
129+
if (u == DateUnit.DAY) return b.setDATE32(empty).build();
130+
throw new UnsupportedOperationException(
131+
"Arrow type Date " + u + " not yet supported by SchemaConverter");
132+
}
133+
if (t instanceof ArrowType.Decimal) {
134+
ArrowType.Decimal d = (ArrowType.Decimal) t;
135+
if (d.getBitWidth() != 128) {
136+
throw new UnsupportedOperationException(
137+
"Arrow type Decimal bit width " + d.getBitWidth() + " not yet supported by SchemaConverter");
138+
}
139+
return b.setDECIMAL128(
140+
DatafusionCommon.Decimal128Type.newBuilder()
141+
.setPrecision(d.getPrecision())
142+
.setScale(d.getScale())
143+
.build())
144+
.build();
145+
}
146+
throw new UnsupportedOperationException(
147+
"Arrow type " + t.getClass().getSimpleName() + " not yet supported by SchemaConverter");
148+
}
149+
150+
static ArrowType arrowTypeFromProto(DatafusionCommon.ArrowType p) {
151+
switch (p.getArrowTypeEnumCase()) {
152+
case BOOL:
153+
return ArrowType.Bool.INSTANCE;
154+
case INT8: return new ArrowType.Int(8, true);
155+
case INT16: return new ArrowType.Int(16, true);
156+
case INT32: return new ArrowType.Int(32, true);
157+
case INT64: return new ArrowType.Int(64, true);
158+
case UINT8: return new ArrowType.Int(8, false);
159+
case UINT16: return new ArrowType.Int(16, false);
160+
case UINT32: return new ArrowType.Int(32, false);
161+
case UINT64: return new ArrowType.Int(64, false);
162+
case FLOAT32: return new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE);
163+
case FLOAT64: return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
164+
case UTF8: return ArrowType.Utf8.INSTANCE;
165+
case DATE32: return new ArrowType.Date(DateUnit.DAY);
166+
case DECIMAL128:
167+
DatafusionCommon.Decimal128Type d = p.getDECIMAL128();
168+
return new ArrowType.Decimal(d.getPrecision(), d.getScale(), 128);
169+
default:
170+
throw new UnsupportedOperationException(
171+
"datafusion_common.ArrowType " + p.getArrowTypeEnumCase() + " not yet supported by SchemaConverter");
172+
}
173+
}
174+
}

0 commit comments

Comments
 (0)