Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions parquet-avro/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@
<goals>
<goal>schema</goal>
</goals>
<configuration>
<enableDecimalLogicalType>true</enableDecimalLogicalType>
</configuration>
</execution>
</executions>
</plugin>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
import org.apache.avro.Conversion;
import org.apache.avro.LogicalType;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericEnumSymbol;
import org.apache.avro.generic.GenericFixed;
import org.apache.avro.generic.IndexedRecord;

abstract class ParentValueContainer {
Expand Down Expand Up @@ -185,7 +185,7 @@ public void add(Object value) {
return new ParentValueContainer() {
@Override
public void add(Object value) {
parent.add(conversion.fromFixed((GenericData.Fixed) value, schema, logicalType));
parent.add(conversion.fromFixed((GenericFixed) value, schema, logicalType));
}
};
case RECORD:
Expand Down
19 changes: 19 additions & 0 deletions parquet-avro/src/test/avro/decimalFixed.avsc
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"name": "DecimalFixedTest",
"namespace": "org.apache.parquet.avro",
"doc": "Record for testing a decimal logical type on a named fixed type",
"type": "record",
"fields": [
{
"name": "amount",
"type": {
"name": "Decimal9Fixed",
"type": "fixed",
"size": 4,
"logicalType": "decimal",
"precision": 9,
"scale": 2
}
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.common.collect.ImmutableList;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
Expand Down Expand Up @@ -339,6 +340,43 @@ public void testParsesSpecificDataModel(boolean compat) throws IOException {
assertThat(output).containsExactlyElementsOf(records);
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testReadSpecificDecimalFixed(boolean compat) throws IOException {
Configuration testConf = testConf(compat);
// The decimal is stored in the generated Decimal9Fixed class, which is a SpecificFixed and not a
// GenericData.Fixed, so the conversion must accept any GenericFixed
List<DecimalFixedTest> records = IntStream.range(0, 25)
.mapToObj(i -> DecimalFixedTest.newBuilder()
.setAmount(BigDecimal.valueOf(i * 1000 - 12345, 2))
.build())
.collect(Collectors.toList());

File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
tmp.deleteOnExit();
tmp.delete();
Path path = new Path(tmp.getPath());

try (ParquetWriter<DecimalFixedTest> writer = AvroParquetWriter.<DecimalFixedTest>builder(path)
.withSchema(DecimalFixedTest.getClassSchema())
.withConf(new Configuration(false))
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
.build()) {
for (DecimalFixedTest record : records) {
writer.write(record);
}
}

final List<DecimalFixedTest> output = new ArrayList<>();
try (ParquetReader<DecimalFixedTest> reader = new AvroParquetReader<>(testConf, path)) {
for (DecimalFixedTest record = reader.read(); record != null; record = reader.read()) {
output.add(record);
}
}

assertThat(output).containsExactlyElementsOf(records);
}

private Path writeCarsToParquetFile(int num, CompressionCodecName compression, boolean enableDictionary)
throws IOException {
return writeCarsToParquetFile(num, compression, enableDictionary, DEFAULT_BLOCK_SIZE, DEFAULT_PAGE_SIZE);
Expand Down
Loading