From 7cd30ac27088ba713440efaf5e89893e47693c1c Mon Sep 17 00:00:00 2001 From: "raviston.t" Date: Wed, 11 Nov 2026 15:01:32 +0530 Subject: [PATCH] GH-3939: Keep the Avro schema metadata consistent when pruning or renaming columns ParquetRewriter copied the parquet.avro.schema (and legacy avro.schema) footer entry unchanged when pruning or renaming columns. parquet-avro then read the file with a schema that still declared the pruned field, returning nulls even for required fields, so to-avro and convert failed. When columns are pruned or renamed, remove the pruned fields from the Avro schema and rename the renamed top-level fields. The schema is processed as JSON, so parquet-hadoop does not depend on Avro. If the Avro schema can't be adjusted reliably (a pruned record type reused by name, or a mismatch with the Parquet schema), it is removed, and parquet-avro derives the Avro schema from the Parquet schema. Co-Authored-By: Claude Opus 5.5 --- .../cli/commands/RewriteCommandTest.java | 65 ++++ .../hadoop/rewrite/AvroSchemaMetadata.java | 296 ++++++++++++++++++ .../hadoop/rewrite/ParquetRewriter.java | 9 +- .../rewrite/AvroSchemaMetadataTest.java | 247 +++++++++++++++ 4 files changed, 616 insertions(+), 1 deletion(-) create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadata.java create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadataTest.java diff --git a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java index 30cd8144ee..cf373d16f1 100644 --- a/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java +++ b/parquet-cli/src/test/java/org/apache/parquet/cli/commands/RewriteCommandTest.java @@ -24,9 +24,23 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.avro.AvroParquetReader; +import org.apache.parquet.avro.AvroParquetWriter; +import org.apache.parquet.hadoop.ParquetReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.hadoop.util.HadoopOutputFile; import org.junit.jupiter.api.Test; public class RewriteCommandTest extends ParquetFileTest { @@ -99,4 +113,55 @@ public void testRewriteCommandWithCompression_gzip() throws IOException { assertThat(command.run()).isZero(); assertThat(output).exists(); } + + @Test + public void testRewriteCommandWithPruneColumnsUpdatesAvroSchema() throws IOException { + Configuration conf = new Configuration(); + Schema schema = SchemaBuilder.record("people") + .fields() + .optionalLong("id") + .optionalString("name") + .requiredString("ssn") + .endRecord(); + File input = new File(getTempFolder(), "people.parquet"); + try (ParquetWriter writer = AvroParquetWriter.builder( + HadoopOutputFile.fromPath(new Path(input.getAbsolutePath()), conf)) + .withSchema(schema) + .withConf(conf) + .build()) { + GenericRecord alice = new GenericData.Record(schema); + alice.put("id", 1L); + alice.put("name", "Alice"); + alice.put("ssn", "111-22-3333"); + writer.write(alice); + GenericRecord bob = new GenericData.Record(schema); + bob.put("id", 2L); + bob.put("name", "Bob"); + bob.put("ssn", "444-55-6666"); + writer.write(bob); + } + + RewriteCommand command = new RewriteCommand(createLogger()); + command.inputs = Collections.singletonList(input.getAbsolutePath()); + File output = new File(getTempFolder(), "pruned.parquet"); + command.output = output.getAbsolutePath(); + command.pruneColumns = Collections.singletonList("ssn"); + command.setConf(conf); + assertThat(command.run()).isZero(); + + List records = new ArrayList<>(); + try (ParquetReader reader = AvroParquetReader.builder( + HadoopInputFile.fromPath(new Path(output.getAbsolutePath()), conf)) + .withConf(conf) + .build()) { + for (GenericRecord record = reader.read(); record != null; record = reader.read()) { + records.add(record); + } + } + assertThat(records).hasSize(2); + assertThat(records.get(0).getSchema().getField("ssn")).isNull(); + assertThat(records.get(0).getSchema().getFields()).hasSize(2); + assertThat(records.get(0).get("id")).isEqualTo(1L); + assertThat(records.get(1).get("name")).hasToString("Bob"); + } } diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadata.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadata.java new file mode 100644 index 0000000000..6a2e50d32e --- /dev/null +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadata.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop.rewrite; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.parquet.schema.GroupType; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.Type; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Keeps the Avro schema stored in the file metadata by parquet-avro consistent with the Parquet schema of a rewritten + * file, when columns are pruned or renamed. + *

+ * parquet-avro reads files with the Avro schema stored in the key-value metadata, if present. If that schema still + * declares a pruned field, readers return nulls for it - even if the field is required. Therefore, the fields that are + * no longer present in the Parquet schema are removed from the Avro schema, and renamed top-level fields are renamed. + * If the Avro schema can't be adjusted reliably (for example, if a pruned record type is reused by name elsewhere), + * it is removed from the metadata, so that parquet-avro derives the Avro schema from the Parquet schema. + *

+ * This class does not depend on Avro: the schema is processed as JSON, following the Avro to Parquet schema mapping + * of parquet-avro's {@code AvroSchemaConverter}. + */ +final class AvroSchemaMetadata { + + private static final Logger LOG = LoggerFactory.getLogger(AvroSchemaMetadata.class); + + /** Metadata keys of the Avro schema, as written by parquet-avro (current and legacy). */ + static final List AVRO_SCHEMA_KEYS = Arrays.asList("parquet.avro.schema", "avro.schema"); + + private static final Set PRIMITIVE_TYPES = + new HashSet<>(Arrays.asList("null", "boolean", "int", "long", "float", "double", "bytes", "string")); + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private AvroSchemaMetadata() {} + + /** + * @param keyValueMetaData the key-value metadata of the output file + * @param schema the Parquet schema of the output file, after pruning and before renaming + * @param renamedColumns renamed top-level fields: original names to new names + * @return the key-value metadata, with Avro schemas consistent with the Parquet schema + */ + static Map adjust( + Map keyValueMetaData, MessageType schema, Map renamedColumns) { + Map result = new HashMap<>(keyValueMetaData); + for (String key : AVRO_SCHEMA_KEYS) { + String avroSchema = keyValueMetaData.get(key); + if (avroSchema == null) { + continue; + } + try { + result.put(key, adjust(avroSchema, schema, renamedColumns)); + } catch (InconsistentSchemaException | JsonProcessingException e) { + LOG.warn( + "Removing the Avro schema '{}' from the metadata, because it can't be adjusted to the rewritten " + + "Parquet schema: {}. Avro readers will derive the Avro schema from the Parquet schema.", + key, + e.getMessage()); + result.remove(key); + } + } + return result; + } + + static String adjust(String avroSchema, MessageType schema, Map renamedColumns) + throws JsonProcessingException, InconsistentSchemaException { + JsonNode root = MAPPER.readTree(avroSchema); + if (!root.isObject() || !isRecord(root)) { + throw new InconsistentSchemaException("the Avro schema is not a record"); + } + Set recordNames = new HashSet<>(); + Set namedReferences = new HashSet<>(); + adjustRecord((ObjectNode) root, schema, recordNames, namedReferences); + for (String reference : namedReferences) { + if (recordNames.contains(shortName(reference))) { + // The same record definition is shared by several fields, that may have been pruned differently + throw new InconsistentSchemaException("the record type '" + reference + "' is reused by name"); + } + } + renameFields((ObjectNode) root, renamedColumns); + return MAPPER.writeValueAsString(root); + } + + private static void adjustRecord( + ObjectNode record, Type parquetType, Set recordNames, Set namedReferences) + throws InconsistentSchemaException { + if (parquetType.isPrimitive()) { + throw new InconsistentSchemaException( + "the record '" + record.path("name").asText() + "' is not a group in the Parquet schema"); + } + GroupType group = parquetType.asGroupType(); + addRecordNames(record, recordNames); + JsonNode fields = record.get("fields"); + if (fields == null || !fields.isArray()) { + throw new InconsistentSchemaException( + "the record '" + record.path("name").asText() + "' has no fields"); + } + ArrayNode fieldArray = (ArrayNode) fields; + Set avroFieldNames = new HashSet<>(); + for (int i = fieldArray.size() - 1; i >= 0; i--) { + JsonNode field = fieldArray.get(i); + String name = field.path("name").asText(); + if (!group.containsField(name)) { + // Pruned field + fieldArray.remove(i); + continue; + } + avroFieldNames.add(name); + adjustType(field.get("type"), group.getType(name), recordNames, namedReferences); + } + for (Type parquetField : group.getFields()) { + if (!avroFieldNames.contains(parquetField.getName())) { + throw new InconsistentSchemaException( + "the Parquet field '" + parquetField.getName() + "' is missing in the Avro record '" + + record.path("name").asText() + "'"); + } + } + } + + private static void adjustType( + JsonNode avroType, Type parquetType, Set recordNames, Set namedReferences) + throws InconsistentSchemaException { + if (avroType == null) { + throw new InconsistentSchemaException( + "missing Avro type for the Parquet field '" + parquetType.getName() + "'"); + } + if (avroType.isTextual()) { + if (!PRIMITIVE_TYPES.contains(avroType.asText())) { + namedReferences.add(avroType.asText()); + } + return; + } + if (avroType.isArray()) { + adjustUnion((ArrayNode) avroType, parquetType, recordNames, namedReferences); + return; + } + if (!avroType.isObject() || !avroType.path("type").isTextual()) { + throw new InconsistentSchemaException( + "unsupported Avro type for the Parquet field '" + parquetType.getName() + "': " + avroType); + } + switch (avroType.get("type").asText()) { + case "record": + case "error": + adjustRecord((ObjectNode) avroType, parquetType, recordNames, namedReferences); + break; + case "array": + adjustType(avroType.get("items"), listElement(parquetType), recordNames, namedReferences); + break; + case "map": + adjustType(avroType.get("values"), mapValue(parquetType), recordNames, namedReferences); + break; + default: + // Primitive types (possibly with a logical type), enums and fixed: nothing to prune + break; + } + } + + /** + * Unions with a single non-null type are optional Parquet fields; other unions are groups with a + * {@code memberN} field for each non-null type. + */ + private static void adjustUnion( + ArrayNode union, Type parquetType, Set recordNames, Set namedReferences) + throws InconsistentSchemaException { + int nonNullTypes = 0; + for (JsonNode branch : union) { + if (!(branch.isTextual() && "null".equals(branch.asText()))) { + nonNullTypes++; + } + } + if (nonNullTypes <= 1) { + for (JsonNode branch : union) { + if (!(branch.isTextual() && "null".equals(branch.asText()))) { + adjustType(branch, parquetType, recordNames, namedReferences); + } + } + return; + } + if (parquetType.isPrimitive() || parquetType.asGroupType().getFieldCount() != nonNullTypes) { + throw new InconsistentSchemaException( + "the union field '" + parquetType.getName() + "' does not match the Parquet schema"); + } + GroupType group = parquetType.asGroupType(); + int index = 0; + for (JsonNode branch : union) { + if (branch.isTextual() && "null".equals(branch.asText())) { + continue; + } + String member = "member" + index++; + if (!group.containsField(member)) { + throw new InconsistentSchemaException("the union field '" + parquetType.getName() + "' has no '" + + member + "' in the Parquet schema"); + } + adjustType(branch, group.getType(member), recordNames, namedReferences); + } + } + + /** The element of a 2-level ({@code repeated ... array}) or 3-level ({@code list.element}) Parquet list. */ + private static Type listElement(Type parquetType) throws InconsistentSchemaException { + if (!parquetType.isPrimitive() && parquetType.asGroupType().getFieldCount() == 1) { + Type repeated = parquetType.asGroupType().getType(0); + if (repeated.isRepetition(Type.Repetition.REPEATED)) { + if ("array".equals(repeated.getName())) { + return repeated; + } + if ("list".equals(repeated.getName()) + && !repeated.isPrimitive() + && repeated.asGroupType().getFieldCount() == 1 + && repeated.asGroupType().containsField("element")) { + return repeated.asGroupType().getType("element"); + } + } + } + throw new InconsistentSchemaException( + "the array field '" + parquetType.getName() + "' does not match the Parquet schema"); + } + + /** The value of a Parquet map ({@code key_value.value}). */ + private static Type mapValue(Type parquetType) throws InconsistentSchemaException { + if (!parquetType.isPrimitive() && parquetType.asGroupType().getFieldCount() == 1) { + Type repeated = parquetType.asGroupType().getType(0); + if (repeated.isRepetition(Type.Repetition.REPEATED) + && !repeated.isPrimitive() + && repeated.asGroupType().getFieldCount() == 2 + && repeated.asGroupType().containsField("key") + && repeated.asGroupType().containsField("value")) { + return repeated.asGroupType().getType("value"); + } + } + throw new InconsistentSchemaException( + "the map field '" + parquetType.getName() + "' does not match the Parquet schema"); + } + + private static void renameFields(ObjectNode record, Map renamedColumns) { + if (renamedColumns == null || renamedColumns.isEmpty()) { + return; + } + for (JsonNode field : record.get("fields")) { + String newName = renamedColumns.get(field.path("name").asText()); + if (newName != null) { + ((ObjectNode) field).put("name", newName); + } + } + } + + private static boolean isRecord(JsonNode avroType) { + String type = avroType.path("type").asText(); + return "record".equals(type) || "error".equals(type); + } + + /** + * Record names are compared without their namespace, which may be inherited from an enclosing type: a match is + * either a reuse of the record, or a different record with the same name, which also leads to removing the schema. + */ + private static void addRecordNames(ObjectNode record, Set recordNames) { + recordNames.add(shortName(record.path("name").asText())); + } + + private static String shortName(String name) { + return name.substring(name.lastIndexOf('.') + 1); + } + + static final class InconsistentSchemaException extends Exception { + InconsistentSchemaException(String message) { + super(message); + } + } +} diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java index fce8866850..975a6e8f4b 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/rewrite/ParquetRewriter.java @@ -161,11 +161,18 @@ public ParquetRewriter(RewriteOptions options) throws IOException { this.inputFiles.addAll(getFileReaders(options.getParquetInputFiles(), conf)); this.inputFilesToJoin.addAll(getFileReaders(options.getParquetInputFilesToJoin(), conf)); this.outSchema = pruneColumnsInSchema(getSchema(), options.getPruneColumns()); - this.extraMetaData = getExtraMetadata(options); ensureSameSchema(inputFiles); ensureSameSchema(inputFilesToJoin); ensureRowCount(); ensureRenamingCorrectness(outSchema, renamedColumns); + Map inputMetaData = getExtraMetadata(options); + boolean pruned = + options.getPruneColumns() != null && !options.getPruneColumns().isEmpty(); + if (pruned || !renamedColumns.isEmpty()) { + // Keep the Avro schema stored by parquet-avro consistent with the pruned/renamed Parquet schema + inputMetaData = AvroSchemaMetadata.adjust(inputMetaData, outSchema, renamedColumns); + } + this.extraMetaData = inputMetaData; OutputFile out = options.getParquetOutputFile(); LOG.info( "Start rewriting {} input file(s) {} to {}", diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadataTest.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadataTest.java new file mode 100644 index 0000000000..e8b148b814 --- /dev/null +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/rewrite/AvroSchemaMetadataTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.hadoop.rewrite; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.parquet.example.data.Group; +import org.apache.parquet.example.data.simple.SimpleGroupFactory; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.ParquetWriter; +import org.apache.parquet.hadoop.example.ExampleParquetWriter; +import org.apache.parquet.hadoop.util.HadoopInputFile; +import org.apache.parquet.schema.MessageType; +import org.apache.parquet.schema.MessageTypeParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class AvroSchemaMetadataTest { + + private static final String AVRO_SCHEMA_KEY = "parquet.avro.schema"; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + /** The example of the issue: a required field is pruned. */ + private static final String PEOPLE_AVRO_SCHEMA = "{\"type\":\"record\",\"name\":\"people\",\"fields\":[" + + "{\"name\":\"id\",\"type\":[\"null\",\"long\"],\"default\":null}," + + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"ssn\",\"type\":\"string\"}]}"; + + private static final MessageType PEOPLE_SCHEMA = MessageTypeParser.parseMessageType( + "message people { optional int64 id; optional binary name (STRING); required binary ssn (STRING); }"); + + @TempDir + private java.nio.file.Path tempDir; + + @Test + public void testPruneTopLevelField() throws Exception { + MessageType pruned = MessageTypeParser.parseMessageType( + "message people { optional int64 id; optional binary name (STRING); }"); + + assertSameJson( + AvroSchemaMetadata.adjust(PEOPLE_AVRO_SCHEMA, pruned, Collections.emptyMap()), + "{\"type\":\"record\",\"name\":\"people\",\"fields\":[" + + "{\"name\":\"id\",\"type\":[\"null\",\"long\"],\"default\":null}," + + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}]}"); + } + + @Test + public void testRenameTopLevelField() throws Exception { + assertSameJson( + AvroSchemaMetadata.adjust( + PEOPLE_AVRO_SCHEMA, PEOPLE_SCHEMA, Collections.singletonMap("name", "full_name")), + "{\"type\":\"record\",\"name\":\"people\",\"fields\":[" + + "{\"name\":\"id\",\"type\":[\"null\",\"long\"],\"default\":null}," + + "{\"name\":\"full_name\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"ssn\",\"type\":\"string\"}]}"); + } + + @Test + public void testPruneNestedFields() throws Exception { + String avroSchema = "{\"type\":\"record\",\"name\":\"r\",\"namespace\":\"ns\",\"fields\":[" + + "{\"name\":\"opt\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Opt\",\"fields\":[" + + "{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}]}," + + "{\"name\":\"list3\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"L3\"," + + "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}}}," + + "{\"name\":\"list2\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"L2\"," + + "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}}}," + + "{\"name\":\"map\",\"type\":{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"M\"," + + "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}}}," + + "{\"name\":\"union\",\"type\":[\"null\",\"int\",{\"type\":\"record\",\"name\":\"U\"," + + "\"fields\":[{\"name\":\"a\",\"type\":\"int\"},{\"name\":\"b\",\"type\":\"int\"}]}]}," + + "{\"name\":\"date\",\"type\":{\"type\":\"int\",\"logicalType\":\"date\"}}]}"; + // The "b" fields are pruned + MessageType pruned = MessageTypeParser.parseMessageType("message r {" + + " optional group opt { required int32 a; }" + + " required group list3 (LIST) { repeated group list { required group element { required int32 a; } } }" + + " required group list2 (LIST) { repeated group array { required int32 a; } }" + + " required group map (MAP) { repeated group key_value { required binary key (STRING);" + + " required group value { required int32 a; } } }" + + " optional group union { optional int32 member0; optional group member1 { required int32 a; } }" + + " required int32 date (DATE); }"); + + assertSameJson( + AvroSchemaMetadata.adjust(avroSchema, pruned, Collections.emptyMap()), + avroSchema.replace(",{\"name\":\"b\",\"type\":\"int\"}", "")); + } + + @Test + public void testRemoveSchemaIfRecordIsReusedByName() { + String avroSchema = "{\"type\":\"record\",\"name\":\"r\",\"fields\":[" + + "{\"name\":\"home\",\"type\":{\"type\":\"record\",\"name\":\"Address\",\"fields\":[" + + "{\"name\":\"city\",\"type\":\"string\"},{\"name\":\"zip\",\"type\":\"string\"}]}}," + + "{\"name\":\"work\",\"type\":\"Address\"}]}"; + MessageType pruned = MessageTypeParser.parseMessageType("message r {" + + " required group home { required binary city (STRING); }" + + " required group work { required binary city (STRING); required binary zip (STRING); } }"); + + assertThat(AvroSchemaMetadata.adjust(metadata(avroSchema), pruned, Collections.emptyMap())) + .doesNotContainKey(AVRO_SCHEMA_KEY) + .containsEntry("other", "value"); + } + + @Test + public void testRemoveSchemaIfParquetFieldIsMissingInAvroSchema() { + MessageType schema = MessageTypeParser.parseMessageType( + "message people { optional int64 id; optional binary name (STRING); optional int32 age; }"); + + assertThat(AvroSchemaMetadata.adjust(metadata(PEOPLE_AVRO_SCHEMA), schema, Collections.emptyMap())) + .doesNotContainKey(AVRO_SCHEMA_KEY) + .containsEntry("other", "value"); + } + + @Test + public void testRemoveInvalidSchema() { + assertThat(AvroSchemaMetadata.adjust(metadata("{not json"), PEOPLE_SCHEMA, Collections.emptyMap())) + .doesNotContainKey(AVRO_SCHEMA_KEY) + .containsEntry("other", "value"); + } + + @Test + public void testLegacyAvroSchemaKey() throws Exception { + MessageType pruned = MessageTypeParser.parseMessageType( + "message people { optional int64 id; optional binary name (STRING); }"); + Map metadata = Collections.singletonMap("avro.schema", PEOPLE_AVRO_SCHEMA); + + Map adjusted = AvroSchemaMetadata.adjust(metadata, pruned, Collections.emptyMap()); + + assertThat(adjusted).containsKey("avro.schema"); + assertSameJson( + adjusted.get("avro.schema"), + AvroSchemaMetadata.adjust(PEOPLE_AVRO_SCHEMA, pruned, Collections.emptyMap())); + assertThat(json(adjusted.get("avro.schema")).toString()).doesNotContain("ssn"); + } + + @Test + public void testRewriterPrunesAvroSchema() throws Exception { + Path input = writePeople(); + Path output = new Path(tempDir.resolve("pruned.parquet").toUri()); + Configuration conf = new Configuration(); + RewriteOptions options = new RewriteOptions.Builder(conf, input, output) + .prune(Collections.singletonList("ssn")) + .build(); + try (ParquetRewriter rewriter = new ParquetRewriter(options)) { + rewriter.processBlocks(); + } + + Map metadata = keyValueMetaData(output); + assertThat(metadata).containsKey(AVRO_SCHEMA_KEY).containsEntry("other", "value"); + assertSameJson( + metadata.get(AVRO_SCHEMA_KEY), + "{\"type\":\"record\",\"name\":\"people\",\"fields\":[" + + "{\"name\":\"id\",\"type\":[\"null\",\"long\"],\"default\":null}," + + "{\"name\":\"name\",\"type\":[\"null\",\"string\"],\"default\":null}]}"); + } + + @Test + public void testRewriterRenamesAvroSchemaFields() throws Exception { + Path input = writePeople(); + Path output = new Path(tempDir.resolve("renamed.parquet").toUri()); + Configuration conf = new Configuration(); + RewriteOptions options = new RewriteOptions.Builder(conf, input, output) + .renameColumns(Collections.singletonMap("name", "full_name")) + .build(); + try (ParquetRewriter rewriter = new ParquetRewriter(options)) { + rewriter.processBlocks(); + } + + assertSameJson( + keyValueMetaData(output).get(AVRO_SCHEMA_KEY), + "{\"type\":\"record\",\"name\":\"people\",\"fields\":[" + + "{\"name\":\"id\",\"type\":[\"null\",\"long\"],\"default\":null}," + + "{\"name\":\"full_name\",\"type\":[\"null\",\"string\"],\"default\":null}," + + "{\"name\":\"ssn\",\"type\":\"string\"}]}"); + } + + @Test + public void testRewriterKeepsAvroSchemaWithoutPruningOrRenaming() throws Exception { + Path input = writePeople(); + Path output = new Path(tempDir.resolve("copied.parquet").toUri()); + Configuration conf = new Configuration(); + RewriteOptions options = new RewriteOptions.Builder(conf, input, output).build(); + try (ParquetRewriter rewriter = new ParquetRewriter(options)) { + rewriter.processBlocks(); + } + + assertThat(keyValueMetaData(output)).containsEntry(AVRO_SCHEMA_KEY, PEOPLE_AVRO_SCHEMA); + } + + private Path writePeople() throws IOException { + Path path = new Path(tempDir.resolve("people.parquet").toUri()); + SimpleGroupFactory factory = new SimpleGroupFactory(PEOPLE_SCHEMA); + try (ParquetWriter writer = ExampleParquetWriter.builder(path) + .withType(PEOPLE_SCHEMA) + .withExtraMetaData(metadata(PEOPLE_AVRO_SCHEMA)) + .build()) { + writer.write( + factory.newGroup().append("id", 1L).append("name", "Alice").append("ssn", "111-22-3333")); + writer.write( + factory.newGroup().append("id", 2L).append("name", "Bob").append("ssn", "444-55-6666")); + } + return path; + } + + private static Map keyValueMetaData(Path path) throws IOException { + try (ParquetFileReader reader = ParquetFileReader.open(HadoopInputFile.fromPath(path, new Configuration()))) { + return reader.getFooter().getFileMetaData().getKeyValueMetaData(); + } + } + + private static Map metadata(String avroSchema) { + Map metadata = new HashMap<>(); + metadata.put(AVRO_SCHEMA_KEY, avroSchema); + metadata.put("other", "value"); + return metadata; + } + + private static JsonNode json(String json) throws IOException { + return MAPPER.readTree(json); + } + + private static void assertSameJson(String actual, String expected) throws IOException { + assertThat(json(actual)).isEqualTo(json(expected)); + } +}