From fe8c8e27aa35d29c8c827376203e844b1ba7d939 Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Wed, 18 Nov 2026 12:16:39 +0530 Subject: [PATCH 1/2] GH-3970: Match the file's list and map structure when converting an Avro requested projection --- .../apache/parquet/avro/AvroReadSupport.java | 109 ++++++++- .../parquet/avro/TestArrayCompatibility.java | 226 ++++++++++++++++++ .../org/apache/parquet/cli/ReadCliTest.java | 39 +++ 3 files changed, 372 insertions(+), 2 deletions(-) diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index ee8f9ab272..d2d65d8cbf 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -18,7 +18,9 @@ */ package org.apache.parquet.avro; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -140,8 +142,13 @@ public ReadContext init( String requestedProjectionString = configuration.get(AVRO_REQUESTED_PROJECTION); if (requestedProjectionString != null) { Schema avroRequestedProjection = new Schema.Parser().parse(requestedProjectionString); - projection = new AvroSchemaConverter(getDerivedListEncodingConf(configuration, fileSchema)) - .convert(avroRequestedProjection); + ParquetConfiguration listEncodingConf = getDerivedListEncodingConf(configuration, fileSchema); + projection = new AvroSchemaConverter(listEncodingConf).convert(avroRequestedProjection); + boolean oldListStructure = listEncodingConf.getBoolean( + AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE, AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE_DEFAULT); + projection = new MessageType( + projection.getName(), + alignListStructures(projection, fileSchema, avroRequestedProjection, oldListStructure)); } String avroReadSchema = configuration.get(AVRO_READ_SCHEMA); @@ -269,6 +276,104 @@ private static ParquetConfiguration getDerivedListEncodingConf( return copiedConfiguration; } + // The projection converted from the Avro schema writes every list with the same structure, so it + // only selects a list's columns if the file wrote the list that way too. Use the repeated and + // element names of each list in the file instead; the converters read all the + // backward-compatible list structures. + private static List alignListStructures( + GroupType projected, GroupType file, Schema record, boolean oldListStructure) { + List fields = new ArrayList<>(projected.getFieldCount()); + for (Type field : projected.getFields()) { + Schema.Field avroField = record.getField(field.getName()); + if (avroField != null && file.containsField(field.getName())) { + fields.add(alignListStructures( + field, file.getType(field.getName()), avroField.schema(), oldListStructure)); + } else { + fields.add(field); + } + } + return fields; + } + + private static Type alignListStructures(Type projected, Type file, Schema schema, boolean oldListStructure) { + if (projected.isPrimitive() || file.isPrimitive()) { + return projected; + } + GroupType projectedGroup = projected.asGroupType(); + GroupType fileGroup = file.asGroupType(); + Schema nonNullSchema = AvroSchemaConverter.getNonNull(schema); + switch (nonNullSchema.getType()) { + case RECORD: + return projectedGroup.withNewFields( + alignListStructures(projectedGroup, fileGroup, nonNullSchema, oldListStructure)); + case ARRAY: + return alignList(projectedGroup, fileGroup, nonNullSchema.getElementType(), oldListStructure); + case MAP: + return alignMap(projectedGroup, fileGroup, nonNullSchema.getValueType(), oldListStructure); + default: + return projected; + } + } + + private static Type alignList(GroupType projected, GroupType file, Schema elementSchema, boolean oldListStructure) { + if (!(projected.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || !(file.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) + || file.isRepetition(Type.Repetition.REPEATED) + || file.getFieldCount() != 1 + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + Type projectedRepeated = projected.getType(0); + Type projectedElement = oldListStructure + ? projectedRepeated + : projectedRepeated.asGroupType().getType(0); + Type fileRepeated = file.getType(0); + Type alignedRepeated; + // Decide which field is the element the same way AvroCollectionConverter does + if (AvroRecordConverter.isElementType(fileRepeated, AvroSchemaConverter.getNonNull(elementSchema))) { + alignedRepeated = alignElement(projectedElement, fileRepeated, elementSchema, oldListStructure); + } else { + GroupType fileRepeatedGroup = fileRepeated.asGroupType(); + Type alignedElement = + alignElement(projectedElement, fileRepeatedGroup.getType(0), elementSchema, oldListStructure); + alignedRepeated = alignedElement == null ? null : fileRepeatedGroup.withNewFields(alignedElement); + } + return alignedRepeated == null ? projected : projected.withNewFields(alignedRepeated); + } + + private static Type alignMap(GroupType projected, GroupType file, Schema valueSchema, boolean oldListStructure) { + if (projected.getFieldCount() != 1 + || file.getFieldCount() != 1 + || file.getType(0).isPrimitive() + || !file.getType(0).isRepetition(Type.Repetition.REPEATED)) { + return projected; + } + GroupType projectedKeyValue = projected.getType(0).asGroupType(); + GroupType fileKeyValue = file.getType(0).asGroupType(); + if (projectedKeyValue.getFieldCount() != 2 || fileKeyValue.getFieldCount() != 2) { + return projected; + } + Type alignedValue = + alignElement(projectedKeyValue.getType(1), fileKeyValue.getType(1), valueSchema, oldListStructure); + return alignedValue == null + ? projected + : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); + } + + // Returns the projected element with the name and repetition of the file's element, or null if + // one of them is a group and the other is not. + private static Type alignElement(Type projected, Type file, Schema schema, boolean oldListStructure) { + if (projected.isPrimitive() != file.isPrimitive()) { + return null; + } + if (file.isPrimitive()) { + return file; + } + GroupType aligned = + alignListStructures(projected, file, schema, oldListStructure).asGroupType(); + return file.asGroupType().withNewFields(aligned.getFields()); + } + private static boolean writesNewListStructure(MessageType schema) { return Boolean.TRUE.equals(allListStructuresAreThreeLevel(schema)); } diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index 242fd2fcf0..e4247fef1c 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -36,11 +36,13 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.avro.util.Utf8; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.parquet.DirectWriterTest; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.io.InvalidRecordException; +import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; import org.junit.jupiter.api.BeforeAll; @@ -1610,6 +1612,230 @@ public void testAutoDetectTwoLevelRepeatedPrimitive() throws Exception { assertReaderContains(new AvroParquetReader<>(newAutoDetectConf(), test), expectedSchema, expectedRecord); } + @Test + public void testProjectionReadsTwoLevelListOfPrimitives() throws Exception { + // The projection is converted with the old list structure ("repeated binary array"), which + // must follow the file's repeated field name to find the column. + Path test = writeDirect( + "message ProjectTwoLevelList {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated binary element (UTF8);" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField("element", 0); + rc.addBinary(Binary.fromString("a")); + rc.addBinary(Binary.fromString("b")); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record("ProjectTwoLevelList", optionalField("tags", array(primitive(Schema.Type.STRING)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), new Utf8("b")))); + } + + @Test + public void testProjectionReadsSingleFieldGroupsWithOtherNames() throws Exception { + // A synthetic group that is not named "list" or whose element is not named "element", such as + // "bag" or the "list" / "item" names older PyArrow versions write + for (String repeatedName : Arrays.asList("bag", "list")) { + String elementName = repeatedName.equals("bag") ? "x" : "item"; + Path test = writeDirect( + "message ProjectSingleFieldGroup {" + + " required int32 id;" + + " optional group tags (LIST) {" + + " repeated group " + repeatedName + " {" + + " optional binary " + elementName + " (UTF8);" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("id", 0); + rc.addInteger(1); + rc.endField("id", 0); + rc.startField("tags", 1); + rc.startGroup(); + rc.startField(repeatedName, 0); + rc.startGroup(); + rc.startField(elementName, 0); + rc.addBinary(Binary.fromString("a")); + rc.endField(elementName, 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField(repeatedName, 0); + rc.endGroup(); + rc.endField("tags", 1); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectSingleFieldGroup", optionalField("tags", array(optional(primitive(Schema.Type.STRING))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "tags", Arrays.asList(new Utf8("a"), null))); + } + } + + @Test + public void testProjectionReadsMixedListStructures() throws Exception { + // A standard 3-level list next to a non-standard one disables auto-detection for the file + Path test = writeDirect( + "message ProjectMixedLists {" + + " optional group standard (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + " optional group legacy (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("standard", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("standard", 0); + rc.startField("legacy", 1); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(2); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("legacy", 1); + rc.endMessage(); + }); + + Schema ints = array(optional(primitive(Schema.Type.INT))); + Schema projection = record("ProjectMixedLists", optionalField("standard", ints), optionalField("legacy", ints)); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "standard", Arrays.asList(1), "legacy", Arrays.asList(2))); + } + + @Test + public void testProjectionPrunesRecordsInNonStandardList() throws Exception { + Path test = writeDirect( + "message ProjectRecordsInList {" + + " optional group points (LIST) {" + + " repeated group list {" + + " optional group item {" + + " required int32 x;" + + " required int32 y;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("points", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.startField("y", 1); + rc.addInteger(4); + rc.endField("y", 1); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("points", 0); + rc.endMessage(); + }); + + Schema point = record("item", field("x", primitive(Schema.Type.INT))); + Schema projection = record("ProjectRecordsInList", optionalField("points", array(optional(point)))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "points", Arrays.asList(instance(point, "x", 3)))); + } + + @Test + public void testProjectionReadsMapWithOtherNames() throws Exception { + Path test = writeDirect( + "message ProjectMap {" + + " optional group counts (MAP_KEY_VALUE) {" + + " repeated group map {" + + " required binary key (UTF8);" + + " optional int32 value;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("counts", 0); + rc.startGroup(); + rc.startField("map", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.addInteger(5); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("map", 0); + rc.endGroup(); + rc.endField("counts", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectMap", optionalField("counts", Schema.createMap(optional(primitive(Schema.Type.INT))))); + Map counts = new HashMap<>(); + counts.put(new Utf8("a"), 5); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "counts", counts)); + } + + private AvroParquetReader projectionReader(Path path, Schema projection) + throws IOException { + Configuration conf = new Configuration(); + AvroReadSupport.setRequestedProjection(conf, projection); + AvroReadSupport.setAvroReadSchema(conf, projection); + return new AvroParquetReader<>(conf, path); + } + private static Configuration newAutoDetectConf() { Configuration conf = new Configuration(); conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true); diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java index 42a7feb680..8cc1b43be6 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/ReadCliTest.java @@ -36,6 +36,7 @@ import org.apache.parquet.hadoop.example.ExampleParquetWriter; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; import org.apache.parquet.schema.Types; import org.junit.jupiter.api.BeforeEach; @@ -152,6 +153,44 @@ void catReadsStandardListsWithoutAvroSchema() throws Exception { assertThat(readRecords(result.output())).containsExactlyElementsOf(expected); } + @Test + void catReadsNonStandardListsWithoutAvroSchema() throws Exception { + // A 2-level list and the "list" / "item" names older PyArrow versions write + Path path = getTempFolder().toPath().resolve("legacy-lists.parquet"); + MessageType schema = MessageTypeParser.parseMessageType("message record {" + + " required int32 id;" + + " optional group two_level (LIST) { repeated binary element (STRING); }" + + " optional group items (LIST) { repeated group list { optional binary item (STRING); } }" + + "}"); + SimpleGroupFactory groups = new SimpleGroupFactory(schema); + try (ParquetWriter writer = ExampleParquetWriter.builder(new org.apache.hadoop.fs.Path(path.toUri())) + .withType(schema) + .build()) { + Group record = groups.newGroup().append("id", 0); + record.addGroup("two_level").append("element", "a").append("element", "b"); + Group items = record.addGroup("items"); + items.addGroup("list").append("item", "c"); + items.addGroup("list"); + writer.write(record); + } + + Map expected = new HashMap<>(); + expected.put("id", 0); + expected.put("two_level", List.of("a", "b")); + expected.put("items", Arrays.asList("c", null)); + + CliResult all = cli("cat", path.toString()); + assertThat(all.exitCode()).as(all.output()).isZero(); + assertThat(readRecords(all.output())).containsExactly(expected); + + CliResult projected = cli("cat", "--columns", "id,items", path.toString()); + assertThat(projected.exitCode()).as(projected.output()).isZero(); + Map expectedProjected = new HashMap<>(); + expectedProjected.put("id", 0); + expectedProjected.put("items", Arrays.asList("c", null)); + assertThat(readRecords(projected.output())).containsExactly(expectedProjected); + } + private Path writeRecords(String name, int firstId, int count) throws IOException { Path path = getTempFolder().toPath().resolve(name); MessageType schema = Types.buildMessage() From 6a4dd2f5cd880aa7f64f5f6443ff7b658b86a6e9 Mon Sep 17 00:00:00 2001 From: Shivadarshan Date: Wed, 18 Nov 2026 22:38:45 +0530 Subject: [PATCH 2/2] GH-3970: Keep projected element types and repetition when aligning with the file --- .../apache/parquet/avro/AvroReadSupport.java | 36 +++- .../parquet/avro/TestArrayCompatibility.java | 176 ++++++++++++++++++ 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java index d2d65d8cbf..7db41a5e8a 100644 --- a/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java +++ b/parquet-avro/src/main/java/org/apache/parquet/avro/AvroReadSupport.java @@ -37,7 +37,9 @@ import org.apache.parquet.schema.GroupType; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.PrimitiveType; import org.apache.parquet.schema.Type; +import org.apache.parquet.schema.Types; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -360,18 +362,44 @@ private static Type alignMap(GroupType projected, GroupType file, Schema valueSc : projected.withNewFields(fileKeyValue.withNewFields(fileKeyValue.getType(0), alignedValue)); } - // Returns the projected element with the name and repetition of the file's element, or null if - // one of them is a group and the other is not. + // Returns the projected element under the name of the file's element, or null if one of them is a + // group and the other is not. The type stays the projected one, so the requested schema is still + // checked against the file. The repetition is the one the converted projection gives the element: + // repeated if the file's repeated field is the element, otherwise optional only if the Avro schema + // allows null. private static Type alignElement(Type projected, Type file, Schema schema, boolean oldListStructure) { if (projected.isPrimitive() != file.isPrimitive()) { return null; } + Type.Repetition repetition; + if (file.isRepetition(Type.Repetition.REPEATED)) { + repetition = Type.Repetition.REPEATED; + } else if (isNullable(schema)) { + repetition = Type.Repetition.OPTIONAL; + } else { + repetition = Type.Repetition.REQUIRED; + } if (file.isPrimitive()) { - return file; + PrimitiveType primitive = projected.asPrimitiveType(); + Types.PrimitiveBuilder builder = Types.primitive( + primitive.getPrimitiveTypeName(), repetition) + .as(primitive.getLogicalTypeAnnotation()); + if (primitive.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) { + builder = builder.length(primitive.getTypeLength()); + } + return builder.named(file.getName()); } GroupType aligned = alignListStructures(projected, file, schema, oldListStructure).asGroupType(); - return file.asGroupType().withNewFields(aligned.getFields()); + return Types.buildGroup(repetition) + .as(aligned.getLogicalTypeAnnotation()) + .addFields(aligned.getFields().toArray(new Type[0])) + .named(file.getName()); + } + + private static boolean isNullable(Schema schema) { + return schema.getType() == Schema.Type.UNION + && schema.getTypes().stream().anyMatch(s -> s.getType() == Schema.Type.NULL); } private static boolean writesNewListStructure(MessageType schema) { diff --git a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java index e4247fef1c..5ff658eae3 100644 --- a/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java +++ b/parquet-avro/src/test/java/org/apache/parquet/avro/TestArrayCompatibility.java @@ -42,6 +42,7 @@ import org.apache.parquet.DirectWriterTest; import org.apache.parquet.hadoop.ParquetFileReader; import org.apache.parquet.io.InvalidRecordException; +import org.apache.parquet.io.ParquetDecodingException; import org.apache.parquet.io.api.Binary; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageTypeParser; @@ -1828,6 +1829,181 @@ public void testProjectionReadsMapWithOtherNames() throws Exception { assertReaderContains(projectionReader(test, projection), projection, instance(projection, "counts", counts)); } + @Test + public void testProjectionStillChecksElementTypes() throws Exception { + // The projected element keeps its own type, so a type that doesn't match the file still fails + // the requested-schema check instead of reaching a converter. + Path test = writeDirect( + "message ProjectWrongElementType {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 element;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(5); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectWrongElementType", optionalField("items", array(optional(primitive(Schema.Type.STRING))))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionKeepsNonNullElementsRequired() throws Exception { + // A non-null Avro element type is still requested as required, so optional elements in the file + // fail the requested-schema check as they did before. + Path test = writeDirect( + "message ProjectRequiredElements {" + + " optional group items (LIST) {" + + " repeated group list {" + + " optional int32 item;" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("items", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.addInteger(5); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("items", 0); + rc.endMessage(); + }); + + Schema projection = + record("ProjectRequiredElements", optionalField("items", array(primitive(Schema.Type.INT)))); + + AvroParquetReader reader = projectionReader(test, projection); + assertThatThrownBy(reader::read) + .isInstanceOf(ParquetDecodingException.class) + .cause() + .isInstanceOf(ParquetDecodingException.class) + .hasMessageContaining("incompatible types"); + } + + @Test + public void testProjectionReadsNestedNonStandardLists() throws Exception { + // A "list" / "item" list whose elements are 2-level lists + Path test = writeDirect( + "message ProjectNestedLists {" + + " optional group matrix (LIST) {" + + " repeated group list {" + + " optional group item (LIST) {" + + " repeated int32 element;" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("matrix", 0); + rc.startGroup(); + rc.startField("list", 0); + rc.startGroup(); + rc.startField("item", 0); + rc.startGroup(); + rc.startField("element", 0); + rc.addInteger(1); + rc.addInteger(2); + rc.endField("element", 0); + rc.endGroup(); + rc.endField("item", 0); + rc.endGroup(); + rc.endField("list", 0); + rc.endGroup(); + rc.endField("matrix", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectNestedLists", optionalField("matrix", array(optional(array(primitive(Schema.Type.INT)))))); + + assertReaderContains( + projectionReader(test, projection), + projection, + instance(projection, "matrix", Arrays.asList(Arrays.asList(1, 2)))); + } + + @Test + public void testProjectionReadsMapOfNonStandardLists() throws Exception { + // A map whose values are "bag" lists + Path test = writeDirect( + "message ProjectMapOfLists {" + + " optional group lists (MAP) {" + + " repeated group key_value {" + + " required binary key (UTF8);" + + " optional group value (LIST) {" + + " repeated group bag {" + + " optional int32 x;" + + " }" + + " }" + + " }" + + " }" + + "}", + rc -> { + rc.startMessage(); + rc.startField("lists", 0); + rc.startGroup(); + rc.startField("key_value", 0); + rc.startGroup(); + rc.startField("key", 0); + rc.addBinary(Binary.fromString("a")); + rc.endField("key", 0); + rc.startField("value", 1); + rc.startGroup(); + rc.startField("bag", 0); + rc.startGroup(); + rc.startField("x", 0); + rc.addInteger(3); + rc.endField("x", 0); + rc.endGroup(); + rc.startGroup(); // null element + rc.endGroup(); + rc.endField("bag", 0); + rc.endGroup(); + rc.endField("value", 1); + rc.endGroup(); + rc.endField("key_value", 0); + rc.endGroup(); + rc.endField("lists", 0); + rc.endMessage(); + }); + + Schema projection = record( + "ProjectMapOfLists", + optionalField("lists", Schema.createMap(optional(array(optional(primitive(Schema.Type.INT))))))); + Map lists = new HashMap<>(); + lists.put(new Utf8("a"), Arrays.asList(3, null)); + + assertReaderContains(projectionReader(test, projection), projection, instance(projection, "lists", lists)); + } + private AvroParquetReader projectionReader(Path path, Schema projection) throws IOException { Configuration conf = new Configuration();