From 4af7ca31ab5a7de366e811b6bf35cd2d903c5544 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 9 Aug 2026 23:19:56 +0800 Subject: [PATCH 1/6] Parquet: Validate geospatial projection parameters Generated-by: Codex --- .../iceberg/parquet/MessageTypeToType.java | 4 + .../apache/iceberg/parquet/PruneColumns.java | 12 +++ .../iceberg/parquet/TestPruneColumns.java | 93 +++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java index 2b01bf882e75..5d791e66f35c 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/MessageTypeToType.java @@ -149,6 +149,10 @@ public Type map(GroupType map, Type keyType, Type valueType) { @Override public Type primitive(PrimitiveType primitive) { + return convertPrimitive(primitive); + } + + static Type convertPrimitive(PrimitiveType primitive) { // first, use the logical type annotation, if present LogicalTypeAnnotation logicalType = primitive.getLogicalTypeAnnotation(); if (logicalType != null) { diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java index 1675325c7848..6d6f4fe7e174 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java @@ -18,12 +18,16 @@ */ package org.apache.iceberg.parquet; +import static org.apache.iceberg.types.Type.TypeID.GEOGRAPHY; +import static org.apache.iceberg.types.Type.TypeID.GEOMETRY; + import java.util.Collections; import java.util.List; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Objects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.ListType; import org.apache.iceberg.types.Types.MapType; import org.apache.iceberg.types.Types.NestedField; @@ -170,6 +174,14 @@ public Type variant( @Override public Type primitive( org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { + if (expected != null && (expected.typeId() == GEOMETRY || expected.typeId() == GEOGRAPHY)) { + Preconditions.checkArgument( + TypeUtil.isPromotionAllowed(MessageTypeToType.convertPrimitive(primitive), expected), + "Cannot read Parquet type %s as Iceberg type %s", + primitive, + expected); + } + return null; } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index 46b00585d067..bc235e02a9d3 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -19,10 +19,14 @@ package org.apache.iceberg.parquet; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.apache.iceberg.Schema; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.EdgeAlgorithm; import org.apache.iceberg.types.Types.DoubleType; +import org.apache.iceberg.types.Types.GeographyType; +import org.apache.iceberg.types.Types.GeometryType; import org.apache.iceberg.types.Types.IntegerType; import org.apache.iceberg.types.Types.ListType; import org.apache.iceberg.types.Types.LongType; @@ -32,6 +36,7 @@ import org.apache.iceberg.types.Types.StructType; import org.apache.iceberg.types.Types.VariantType; import org.apache.iceberg.variants.Variant; +import org.apache.parquet.column.schema.EdgeInterpolationAlgorithm; import org.apache.parquet.schema.LogicalTypeAnnotation; import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName; @@ -681,6 +686,94 @@ public void testNestedStructInsideMap() { assertThat(wholeActual).as("Whole-map projection is unchanged").isEqualTo(fileSchema); } + @Test + public void acceptsMatchingGeospatialParameters() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType(null)) + .id(1) + .named("geom_default") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .id(2) + .named("geom_projected") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType(null, null)) + .id(3) + .named("geog_default") + .optional(PrimitiveTypeName.BINARY) + .as( + LogicalTypeAnnotation.geographyType( + "EPSG:4326", EdgeInterpolationAlgorithm.ANDOYER)) + .id(4) + .named("geog_custom") + .named("table"); + + Schema projection = + new Schema( + NestedField.optional(1, "geom_default", GeometryType.crs84()), + NestedField.optional(2, "geom_projected", GeometryType.of("epsg:3857")), + NestedField.optional(3, "geog_default", GeographyType.crs84()), + NestedField.optional( + 4, "geog_custom", GeographyType.of("epsg:4326", EdgeAlgorithm.ANDOYER))); + + assertThat(ParquetSchemaUtil.pruneColumns(fileSchema, projection)).isEqualTo(fileSchema); + } + + @Test + public void rejectsGeometryCrsMismatch() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .id(1) + .named("geom") + .named("table"); + Schema projection = new Schema(NestedField.optional(1, "geom", GeometryType.crs84())); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumns(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("geometry(OGC:CRS84)"); + } + + @Test + public void rejectsGeographyCrsMismatch() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as( + LogicalTypeAnnotation.geographyType( + "EPSG:4326", EdgeInterpolationAlgorithm.SPHERICAL)) + .id(1) + .named("geog") + .named("table"); + Schema projection = new Schema(NestedField.optional(1, "geog", GeographyType.crs84())); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumns(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("geography(OGC:CRS84, spherical)"); + } + + @Test + public void rejectsGeographyAlgorithmMismatch() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geographyType("OGC:CRS84", EdgeInterpolationAlgorithm.KARNEY)) + .id(1) + .named("geog") + .named("table"); + Schema projection = new Schema(NestedField.optional(1, "geog", GeographyType.crs84())); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumns(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("geography(OGC:CRS84, spherical)"); + } + private static Type buildVariantType(int id, String name) { return Types.buildGroup(Type.Repetition.OPTIONAL) .as(LogicalTypeAnnotation.variantType(Variant.VARIANT_SPEC_VERSION)) From 2f0729de26c1b5d18aca967adbe8268911da0c05 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 9 Aug 2026 23:29:02 +0800 Subject: [PATCH 2/6] Parquet: Validate fallback geospatial projections Generated-by: Codex --- .../iceberg/parquet/ParquetSchemaUtil.java | 6 ++++++ .../org/apache/iceberg/parquet/PruneColumns.java | 8 ++++++-- .../apache/iceberg/parquet/TestPruneColumns.java | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java index 75e070381e8d..484a503b73c7 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java @@ -267,6 +267,12 @@ public static MessageType pruneColumnsFallback(MessageType fileSchema, Schema ex int ordinal = 1; for (Type type : fileSchema.getFields()) { if (selectedIds.contains(ordinal)) { + Types.NestedField expectedField = expectedSchema.findField(ordinal); + if (type.isPrimitive() && expectedField.type().isPrimitiveType()) { + PruneColumns.validatePrimitive( + expectedField.type().asPrimitiveType(), type.asPrimitiveType()); + } + builder.addField(type.withId(ordinal)); } ordinal += 1; diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java index 6d6f4fe7e174..aaaadaf00d01 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java @@ -174,6 +174,12 @@ public Type variant( @Override public Type primitive( org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { + validatePrimitive(expected, primitive); + return null; + } + + static void validatePrimitive( + org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { if (expected != null && (expected.typeId() == GEOMETRY || expected.typeId() == GEOGRAPHY)) { Preconditions.checkArgument( TypeUtil.isPromotionAllowed(MessageTypeToType.convertPrimitive(primitive), expected), @@ -181,8 +187,6 @@ public Type primitive( primitive, expected); } - - return null; } private Integer getId(Type type) { diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index bc235e02a9d3..e90d99efb97b 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -738,6 +738,22 @@ public void rejectsGeometryCrsMismatch() { .hasMessageContaining("geometry(OGC:CRS84)"); } + @Test + public void rejectsGeometryCrsMismatchWithoutIds() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .named("geom") + .named("table"); + Schema projection = new Schema(NestedField.optional(1, "geom", GeometryType.crs84())); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("geometry(OGC:CRS84)"); + } + @Test public void rejectsGeographyCrsMismatch() { MessageType fileSchema = From fa0795215909eba175f2aa350e24b2420de90f8d Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sun, 9 Aug 2026 23:32:12 +0800 Subject: [PATCH 3/6] Parquet: Fix geospatial validation Checkstyle Generated-by: Codex --- .../main/java/org/apache/iceberg/parquet/PruneColumns.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java index aaaadaf00d01..7370ab4479f0 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java @@ -18,15 +18,13 @@ */ package org.apache.iceberg.parquet; -import static org.apache.iceberg.types.Type.TypeID.GEOGRAPHY; -import static org.apache.iceberg.types.Type.TypeID.GEOMETRY; - import java.util.Collections; import java.util.List; import java.util.Set; import org.apache.iceberg.relocated.com.google.common.base.Objects; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types.ListType; import org.apache.iceberg.types.Types.MapType; @@ -180,7 +178,8 @@ public Type primitive( static void validatePrimitive( org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { - if (expected != null && (expected.typeId() == GEOMETRY || expected.typeId() == GEOGRAPHY)) { + if (expected != null + && (expected.typeId() == TypeID.GEOMETRY || expected.typeId() == TypeID.GEOGRAPHY)) { Preconditions.checkArgument( TypeUtil.isPromotionAllowed(MessageTypeToType.convertPrimitive(primitive), expected), "Cannot read Parquet type %s as Iceberg type %s", From f5d0cabf19f713edf2d4585c111c536f5fc70847 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Mon, 10 Aug 2026 23:42:31 +0800 Subject: [PATCH 4/6] Parquet: Assert mismatched geography algorithm Generated-by: Codex --- .../test/java/org/apache/iceberg/parquet/TestPruneColumns.java | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index e90d99efb97b..a74d54535940 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -787,6 +787,7 @@ public void rejectsGeographyAlgorithmMismatch() { assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumns(fileSchema, projection)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("KARNEY") .hasMessageContaining("geography(OGC:CRS84, spherical)"); } From 188198b83d5ef08f7988f29d8723da48cdc9da72 Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sat, 12 Sep 2026 13:12:11 +0800 Subject: [PATCH 5/6] Parquet: Validate nested fallback geospatial types Validate geospatial parameters recursively for ID-less nested fields and include the field path in incompatible projection errors. Generated-by: Codex --- .../iceberg/parquet/ParquetSchemaUtil.java | 5 +- .../apache/iceberg/parquet/PruneColumns.java | 69 ++++++++++++++- .../iceberg/parquet/TestPruneColumns.java | 83 +++++++++++++++++++ 3 files changed, 149 insertions(+), 8 deletions(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java index 484a503b73c7..ccbf68ec9e6b 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/ParquetSchemaUtil.java @@ -268,10 +268,7 @@ public static MessageType pruneColumnsFallback(MessageType fileSchema, Schema ex for (Type type : fileSchema.getFields()) { if (selectedIds.contains(ordinal)) { Types.NestedField expectedField = expectedSchema.findField(ordinal); - if (type.isPrimitive() && expectedField.type().isPrimitiveType()) { - PruneColumns.validatePrimitive( - expectedField.type().asPrimitiveType(), type.asPrimitiveType()); - } + PruneColumns.validateFallbackType(expectedField.type(), type, type.getName()); builder.addField(type.withId(ordinal)); } diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java index 7370ab4479f0..4a98d5fafb14 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java @@ -172,19 +172,80 @@ public Type variant( @Override public Type primitive( org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { - validatePrimitive(expected, primitive); + validatePrimitive(expected, primitive, String.join(".", currentPath())); return null; } static void validatePrimitive( - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { + org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive, String path) { if (expected != null && (expected.typeId() == TypeID.GEOMETRY || expected.typeId() == TypeID.GEOGRAPHY)) { Preconditions.checkArgument( TypeUtil.isPromotionAllowed(MessageTypeToType.convertPrimitive(primitive), expected), - "Cannot read Parquet type %s as Iceberg type %s", + "Cannot read Parquet type %s as Iceberg type %s for field %s", primitive, - expected); + expected, + path); + } + } + + static void validateFallbackType(org.apache.iceberg.types.Type expected, Type type, String path) { + if (expected == null) { + return; + } + + if (expected.isPrimitiveType() || type.isPrimitive()) { + if (expected.isPrimitiveType() && type.isPrimitive()) { + validatePrimitive(expected.asPrimitiveType(), type.asPrimitiveType(), path); + } + + return; + } + + GroupType group = type.asGroupType(); + if (expected.isStructType()) { + validateFallbackStruct(expected.asStructType(), group, path); + } else if (expected.isListType()) { + validateFallbackList(expected.asListType(), group, path); + } else if (expected.isMapType()) { + validateFallbackMap(expected.asMapType(), group, path); + } + } + + private static void validateFallbackStruct(StructType expected, GroupType struct, String path) { + List expectedFields = expected.fields(); + int fieldCount = Math.min(expectedFields.size(), struct.getFieldCount()); + for (int i = 0; i < fieldCount; i += 1) { + Type field = struct.getType(i); + validateFallbackType(expectedFields.get(i).type(), field, path + "." + field.getName()); + } + } + + private static void validateFallbackList(ListType expected, GroupType list, String path) { + Type element = ParquetSchemaUtil.determineListElementType(list); + String elementPath = path; + if (!element.isRepetition(Type.Repetition.REPEATED)) { + elementPath += "." + list.getFieldName(0); + } + + validateFallbackType(expected.elementType(), element, elementPath + "." + element.getName()); + } + + private static void validateFallbackMap(MapType expected, GroupType map, String path) { + GroupType repeated = map.getType(0).asGroupType(); + String repeatedPath = path + "." + repeated.getName(); + if (repeated.getFieldCount() == 2) { + Type key = repeated.getType(0); + Type value = repeated.getType(1); + validateFallbackType(expected.keyType(), key, repeatedPath + "." + key.getName()); + validateFallbackType(expected.valueType(), value, repeatedPath + "." + value.getName()); + + } else if (repeated.getFieldCount() == 1) { + Type keyOrValue = repeated.getType(0); + org.apache.iceberg.types.Type expectedKeyOrValue = + keyOrValue.getName().equalsIgnoreCase("key") ? expected.keyType() : expected.valueType(); + validateFallbackType( + expectedKeyOrValue, keyOrValue, repeatedPath + "." + keyOrValue.getName()); } } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index a74d54535940..f51bba87e79e 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -735,6 +735,7 @@ public void rejectsGeometryCrsMismatch() { assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumns(fileSchema, projection)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("field geom") .hasMessageContaining("geometry(OGC:CRS84)"); } @@ -751,6 +752,88 @@ public void rejectsGeometryCrsMismatchWithoutIds() { assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("field geom") + .hasMessageContaining("geometry(OGC:CRS84)"); + } + + @Test + public void rejectsNestedStructGeometryCrsMismatchWithoutIds() { + MessageType fileSchema = + Types.buildMessage() + .addField( + Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .named("geom") + .named("location")) + .named("table"); + Schema projection = + new Schema( + NestedField.optional( + 1, + "location", + StructType.of(NestedField.optional(2, "geom", GeometryType.crs84())))); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("field location.geom") + .hasMessageContaining("geometry(OGC:CRS84)"); + } + + @Test + public void rejectsListElementGeometryCrsMismatchWithoutIds() { + MessageType fileSchema = + Types.buildMessage() + .addField( + Types.buildGroup(Type.Repetition.OPTIONAL) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .named("element") + .named("list")) + .as(LogicalTypeAnnotation.listType()) + .named("locations")) + .named("table"); + Schema projection = + new Schema( + NestedField.optional(1, "locations", ListType.ofOptional(2, GeometryType.crs84()))); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("field locations.list.element") + .hasMessageContaining("geometry(OGC:CRS84)"); + } + + @Test + public void rejectsMapValueGeometryCrsMismatchWithoutIds() { + MessageType fileSchema = + Types.buildMessage() + .addField( + Types.buildGroup(Type.Repetition.OPTIONAL) + .addField( + Types.buildGroup(Type.Repetition.REPEATED) + .required(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.stringType()) + .named("key") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .named("value") + .named("key_value")) + .as(LogicalTypeAnnotation.mapType()) + .named("locations")) + .named("table"); + Schema projection = + new Schema( + NestedField.optional( + 1, "locations", MapType.ofOptional(2, 3, StringType.get(), GeometryType.crs84()))); + + assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot read Parquet type") + .hasMessageContaining("field locations.key_value.value") .hasMessageContaining("geometry(OGC:CRS84)"); } From 3239fc629e8b8dba1427dc720caf69ae1d432c7d Mon Sep 17 00:00:00 2001 From: Manu Zhang Date: Sat, 19 Sep 2026 21:50:43 +0800 Subject: [PATCH 6/6] Parquet: Align nested fallback validation Match projected nested fields to ID-less Parquet fields only when their names establish alignment, avoiding shifted validation for compressed struct projections. Generated-by: Codex --- .../apache/iceberg/parquet/PruneColumns.java | 12 ++++--- .../iceberg/parquet/TestPruneColumns.java | 31 +++++++++++++++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java index 4a98d5fafb14..0caa376fbb1d 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/PruneColumns.java @@ -213,11 +213,13 @@ static void validateFallbackType(org.apache.iceberg.types.Type expected, Type ty } private static void validateFallbackStruct(StructType expected, GroupType struct, String path) { - List expectedFields = expected.fields(); - int fieldCount = Math.min(expectedFields.size(), struct.getFieldCount()); - for (int i = 0; i < fieldCount; i += 1) { - Type field = struct.getType(i); - validateFallbackType(expectedFields.get(i).type(), field, path + "." + field.getName()); + for (NestedField expectedField : expected.fields()) { + // Projected structs omit unselected fields, so projected positions may not match the file. + // Validate only when an expected name identifies the corresponding file field. + Type field = ParquetSchemaUtil.fieldType(struct, expectedField.name()); + if (field != null) { + validateFallbackType(expectedField.type(), field, path + "." + field.getName()); + } } } diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index f51bba87e79e..8d2b3210add3 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -757,11 +757,14 @@ public void rejectsGeometryCrsMismatchWithoutIds() { } @Test - public void rejectsNestedStructGeometryCrsMismatchWithoutIds() { + public void rejectsProjectedNestedStructGeometryCrsMismatchWithoutIds() { MessageType fileSchema = Types.buildMessage() .addField( Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType(null)) + .named("other_geom") .optional(PrimitiveTypeName.BINARY) .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) .named("geom") @@ -772,7 +775,7 @@ public void rejectsNestedStructGeometryCrsMismatchWithoutIds() { NestedField.optional( 1, "location", - StructType.of(NestedField.optional(2, "geom", GeometryType.crs84())))); + StructType.of(NestedField.optional(3, "geom", GeometryType.crs84())))); assertThatThrownBy(() -> ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)) .isInstanceOf(IllegalArgumentException.class) @@ -781,6 +784,30 @@ public void rejectsNestedStructGeometryCrsMismatchWithoutIds() { .hasMessageContaining("geometry(OGC:CRS84)"); } + @Test + public void acceptsProjectedNestedStructGeometryParametersWithoutIds() { + MessageType fileSchema = + Types.buildMessage() + .addField( + Types.buildGroup(Type.Repetition.OPTIONAL) + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("EPSG:3857")) + .named("other_geom") + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType(null)) + .named("geom") + .named("location")) + .named("table"); + Schema projection = + new Schema( + NestedField.optional( + 1, + "location", + StructType.of(NestedField.optional(3, "geom", GeometryType.crs84())))); + + assertThat(ParquetSchemaUtil.pruneColumnsFallback(fileSchema, projection)).isNotNull(); + } + @Test public void rejectsListElementGeometryCrsMismatchWithoutIds() { MessageType fileSchema =