From 5ede6dd1a828c41ecfa175b952d18e34e648703b Mon Sep 17 00:00:00 2001 From: Vincent <1729065730@qq.com> Date: Thu, 29 Oct 2026 11:25:15 +0800 Subject: [PATCH 1/2] API: Treat EPSG:4326 and OGC:CRS84 as equivalent CRS identifiers Parquet writers (arrow-rs / Arrow C++) emit the WGS84 lon/lat default as OGC:CRS84. Iceberg tables commonly declare Geometry/Geography as EPSG:4326. After Parquet CRS validation (#17578), those files fail equality checks even though both identifiers refer to the same lon/lat WGS84 reference. Add Types.crsEquivalent / crsHashKey so GeometryType and GeographyType equals and hashCode treat the pair as interchangeable (case-insensitive), while leaving other schemes such as srid:4326 distinct. Closes #18680. Generated-by: Cursor Signed-off-by: Vincent <1729065730@qq.com> --- .../java/org/apache/iceberg/types/Types.java | 49 +++++++++++++++---- .../org/apache/iceberg/types/TestTypes.java | 21 ++++++++ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index f082915920ea..36649d4be7c5 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -570,6 +570,33 @@ public int hashCode() { } } + /** + * Returns true when two CRS identifiers should be treated as the same geospatial reference for + * type equality. Matching is case-insensitive. In addition, {@code OGC:CRS84} and {@code + * EPSG:4326} are treated as equivalent: Parquet writers that emit the WGS84 lon/lat default as + * {@code OGC:CRS84} must be compatible with Iceberg tables declared as {@code EPSG:4326}. + * + *

{@code srid:4326} is not included - that is a different identifier scheme. + */ + static boolean crsEquivalent(String left, String right) { + if (left.equalsIgnoreCase(right)) { + return true; + } + return isWgs84LonLatCrs(left) && isWgs84LonLatCrs(right); + } + + private static boolean isWgs84LonLatCrs(String crs) { + return "OGC:CRS84".equalsIgnoreCase(crs) || "EPSG:4326".equalsIgnoreCase(crs); + } + + /** Canonical upper-case key for CRS hashing, aligned with {@link #crsEquivalent}. */ + static String crsHashKey(String crs) { + if (isWgs84LonLatCrs(crs)) { + return "OGC:CRS84"; + } + return crs.toUpperCase(Locale.ROOT); + } + /** A geometry type, optionally parameterized by a CRS. The default CRS is {@code OGC:CRS84}. */ public static class GeometryType extends PrimitiveType { public static final String DEFAULT_CRS = "OGC:CRS84"; @@ -606,8 +633,9 @@ public String crs() { } /** - * Two geometry types are equal when their CRS match case-insensitively, so {@code OGC:CRS84} - * and {@code ogc:crs84} are equal. + * Two geometry types are equal when their CRS are equivalent: case-insensitive match, or the + * WGS84 lon/lat pair {@code OGC:CRS84} / {@code EPSG:4326} (see {@link + * Types#crsEquivalent(String, String)}). */ @Override public boolean equals(Object o) { @@ -618,13 +646,13 @@ public boolean equals(Object o) { } GeometryType that = (GeometryType) o; - return crs.equalsIgnoreCase(that.crs); + return crsEquivalent(crs, that.crs); } @Override public int hashCode() { - // hash the upper-cased CRS so it stays consistent with the case-insensitive equals - return Objects.hash(GeometryType.class, crs.toUpperCase(Locale.ROOT)); + // hash via crsHashKey so it stays consistent with crsEquivalent + return Objects.hash(GeometryType.class, crsHashKey(crs)); } @Override @@ -683,8 +711,9 @@ public EdgeAlgorithm algorithm() { } /** - * Two geography types are equal when their edge algorithms are equal and their CRS match - * case-insensitively, so {@code OGC:CRS84} and {@code ogc:crs84} are equal. + * Two geography types are equal when their edge algorithms are equal and their CRS are + * equivalent: case-insensitive match, or the WGS84 lon/lat pair {@code OGC:CRS84} / {@code + * EPSG:4326} (see {@link Types#crsEquivalent(String, String)}). */ @Override public boolean equals(Object o) { @@ -695,13 +724,13 @@ public boolean equals(Object o) { } GeographyType that = (GeographyType) o; - return crs.equalsIgnoreCase(that.crs) && Objects.equals(algorithm, that.algorithm); + return crsEquivalent(crs, that.crs) && Objects.equals(algorithm, that.algorithm); } @Override public int hashCode() { - // hash the upper-cased CRS so it stays consistent with the case-insensitive equals - return Objects.hash(GeographyType.class, crs.toUpperCase(Locale.ROOT), algorithm); + // hash via crsHashKey so it stays consistent with crsEquivalent + return Objects.hash(GeographyType.class, crsHashKey(crs), algorithm); } @Override diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index 2fb224aefb15..b6ac09c3b02b 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -213,6 +213,27 @@ public void testGeospatialTypeDefaultNormalization() { assertThat(Types.GeographyType.of("ogc:crs84")) .isEqualTo(Types.GeographyType.crs84()) .hasSameHashCodeAs(Types.GeographyType.crs84()); + + // OGC:CRS84 and EPSG:4326 are interchangeable WGS84 lon/lat identifiers (see #18680). + // Parquet writers that emit the default as OGC:CRS84 must match Iceberg EPSG:4326 tables. + assertThat(Types.GeometryType.of("EPSG:4326")) + .isEqualTo(Types.GeometryType.crs84()) + .hasSameHashCodeAs(Types.GeometryType.crs84()); + assertThat(Types.GeometryType.of("epsg:4326")) + .isEqualTo(Types.GeometryType.of("OGC:CRS84")) + .hasSameHashCodeAs(Types.GeometryType.of("ogc:crs84")); + assertThat(Types.GeographyType.of("EPSG:4326")) + .isEqualTo(Types.GeographyType.crs84()) + .hasSameHashCodeAs(Types.GeographyType.crs84()); + assertThat(Types.GeographyType.of("EPSG:4326", EdgeAlgorithm.KARNEY)) + .isEqualTo(Types.GeographyType.of("OGC:CRS84", EdgeAlgorithm.KARNEY)) + .hasSameHashCodeAs(Types.GeographyType.of("ogc:crs84", EdgeAlgorithm.KARNEY)); + // Different algorithm still differs even when CRS identifiers are equivalent + assertThat(Types.GeographyType.of("EPSG:4326", EdgeAlgorithm.KARNEY)) + .isNotEqualTo(Types.GeographyType.crs84()); + // srid:4326 is a different identifier scheme and must not be equated + assertThat(Types.GeometryType.of("srid:4326")).isNotEqualTo(Types.GeometryType.crs84()); + assertThat(Types.GeometryType.of("srid:4326")).isNotEqualTo(Types.GeometryType.of("EPSG:4326")); } @Test From 2bf6486548d4030db96d04e37ef6a4cfa3466ef3 Mon Sep 17 00:00:00 2001 From: ump45nose <52391318+ump45nose@users.noreply.github.com> Date: Fri, 6 Nov 2026 22:04:51 +0800 Subject: [PATCH 2/2] Parquet: Use a non-WGS84 CRS in the geography mismatch test EPSG:4326 and OGC:CRS84 are now equivalent, so the mismatch case uses EPSG:4269. Also cover pruning a file whose CRS is the equivalent identifier of the projected type. --- .../iceberg/parquet/TestPruneColumns.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java index 8d2b3210add3..a2453a3d95e0 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestPruneColumns.java @@ -865,13 +865,36 @@ public void rejectsMapValueGeometryCrsMismatchWithoutIds() { } @Test - public void rejectsGeographyCrsMismatch() { + public void acceptsEquivalentWgs84CrsIdentifiers() { MessageType fileSchema = Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as(LogicalTypeAnnotation.geometryType("OGC:CRS84")) + .id(1) + .named("geom") .optional(PrimitiveTypeName.BINARY) .as( LogicalTypeAnnotation.geographyType( "EPSG:4326", EdgeInterpolationAlgorithm.SPHERICAL)) + .id(2) + .named("geog") + .named("table"); + Schema projection = + new Schema( + NestedField.optional(1, "geom", GeometryType.of("EPSG:4326")), + NestedField.optional(2, "geog", GeographyType.crs84())); + + assertThat(ParquetSchemaUtil.pruneColumns(fileSchema, projection)).isEqualTo(fileSchema); + } + + @Test + public void rejectsGeographyCrsMismatch() { + MessageType fileSchema = + Types.buildMessage() + .optional(PrimitiveTypeName.BINARY) + .as( + LogicalTypeAnnotation.geographyType( + "EPSG:4269", EdgeInterpolationAlgorithm.SPHERICAL)) .id(1) .named("geog") .named("table");