Skip to content

Commit bd28a56

Browse files
committed
implement roundtrip read test
1 parent 868f6cd commit bd28a56

4 files changed

Lines changed: 115 additions & 24 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1838,6 +1838,16 @@ def _task_to_record_batches(
18381838
format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8}
18391839
if dictionary_columns and task.file.file_format == FileFormat.PARQUET:
18401840
format_kwargs["dictionary_columns"] = dictionary_columns
1841+
if (
1842+
task.file.file_format == FileFormat.PARQUET
1843+
and hasattr(ds.ParquetFragmentScanOptions, "arrow_extensions_enabled")
1844+
and any(
1845+
isinstance(table_schema.find_type(field_id), (GeometryType, GeographyType))
1846+
for field_id in projected_field_ids & table_schema.field_ids
1847+
)
1848+
):
1849+
# Dataset scans disable logical-type extension decoding by default.
1850+
format_kwargs["arrow_extensions_enabled"] = True
18411851
arrow_format = _create_arrow_read_format(task.file.file_format, task.file.key_metadata, **format_kwargs)
18421852
with io.new_input(task.file.file_path).open() as fin:
18431853
fragment = arrow_format.make_fragment(fin)
@@ -2017,6 +2027,8 @@ def __init__(
20172027
self._table_metadata = table_metadata
20182028
self._io = io
20192029
self._projected_schema = projected_schema
2030+
# Register extension types before inspecting any data file schemas.
2031+
self._arrow_schema = schema_to_pyarrow(projected_schema, include_field_ids=False)
20202032
self._bound_row_filter = bind(table_metadata.schema(), row_filter, case_sensitive=case_sensitive)
20212033
self._case_sensitive = case_sensitive
20222034
self._limit = limit
@@ -2055,14 +2067,12 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
20552067
ResolveError: When a required field cannot be found in the file
20562068
ValueError: When a field type in the file cannot be projected to the schema type
20572069
"""
2058-
arrow_schema = schema_to_pyarrow(self._projected_schema, include_field_ids=False)
2059-
20602070
batches = self.to_record_batches(tasks)
20612071
try:
20622072
first_batch = next(batches)
20632073
except StopIteration:
20642074
# Empty
2065-
return arrow_schema.empty_table()
2075+
return self._arrow_schema.empty_table()
20662076

20672077
# Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
20682078
# as different batches can use different schema's (due to large_ types)

‎pyiceberg/types.py‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,11 +66,12 @@
6666
DEFAULT_GEOGRAPHY_CRS = "OGC:CRS84"
6767
DEFAULT_GEOGRAPHY_ALGORITHM = "spherical"
6868

69-
# Regex patterns for parsing geometry and geography type strings
70-
# Matches: geometry, geometry('CRS'), geometry('crs'), geometry("CRS")
71-
GEOMETRY_REGEX = re.compile(r"geometry(?:\(\s*['\"]([^'\"]+)['\"]\s*\))?$")
72-
# Matches: geography, geography('CRS'), geography('crs', 'algo')
73-
GEOGRAPHY_REGEX = re.compile(r"geography(?:\(\s*['\"]([^'\"]+)['\"](?:\s*,\s*['\"]([^'\"]+)['\"])?\s*\))?$")
69+
# Accept Java's unquoted parameters and the existing quoted forms.
70+
_GEOSPATIAL_PARAMETER = r"""('[^']+'|"[^"]+"|[^'",()\s][^'",()]*?)"""
71+
GEOMETRY_REGEX = re.compile(rf"geometry\s*(?:\(\s*{_GEOSPATIAL_PARAMETER}\s*\))?", re.IGNORECASE)
72+
GEOGRAPHY_REGEX = re.compile(
73+
rf"geography\s*(?:\(\s*{_GEOSPATIAL_PARAMETER}\s*(?:,\s*{_GEOSPATIAL_PARAMETER}\s*)?\))?", re.IGNORECASE
74+
)
7475

7576

7677
def transform_dict_value_to_str(d: dict[str, Any]) -> dict[str, str]:
@@ -105,6 +106,12 @@ def _parse_fixed_type(fixed: Any) -> int:
105106
return fixed
106107

107108

109+
def _parse_geospatial_parameter(value: str | None, default: str) -> str:
110+
if value is None:
111+
return default
112+
return value[1:-1] if value.startswith(("'", '"')) else value.strip()
113+
114+
108115
def _parse_geometry_type(geometry: Any) -> str:
109116
"""Parse geometry type string and return CRS.
110117
@@ -115,10 +122,9 @@ def _parse_geometry_type(geometry: Any) -> str:
115122
The CRS string (defaults to DEFAULT_GEOMETRY_CRS if not specified).
116123
"""
117124
if isinstance(geometry, str):
118-
match = GEOMETRY_REGEX.match(geometry)
125+
match = GEOMETRY_REGEX.fullmatch(geometry.strip())
119126
if match:
120-
crs = match.group(1)
121-
return crs if crs else DEFAULT_GEOMETRY_CRS
127+
return _parse_geospatial_parameter(match.group(1), DEFAULT_GEOMETRY_CRS)
122128
else:
123129
raise ValidationError(f"Could not parse {geometry} into a GeometryType")
124130
elif isinstance(geometry, dict):
@@ -137,10 +143,10 @@ def _parse_geography_type(geography: Any) -> tuple[str, str]:
137143
Tuple of (CRS, algorithm) with defaults applied where not specified.
138144
"""
139145
if isinstance(geography, str):
140-
match = GEOGRAPHY_REGEX.match(geography)
146+
match = GEOGRAPHY_REGEX.fullmatch(geography.strip())
141147
if match:
142-
crs = match.group(1) if match.group(1) else DEFAULT_GEOGRAPHY_CRS
143-
algorithm = match.group(2) if match.group(2) else DEFAULT_GEOGRAPHY_ALGORITHM
148+
crs = _parse_geospatial_parameter(match.group(1), DEFAULT_GEOGRAPHY_CRS)
149+
algorithm = _parse_geospatial_parameter(match.group(2), DEFAULT_GEOGRAPHY_ALGORITHM).lower()
144150
return crs, algorithm
145151
else:
146152
raise ValidationError(f"Could not parse {geography} into a GeographyType")
@@ -188,9 +194,9 @@ def handle_primitive_type(cls, v: Any, handler: ValidatorFunctionWrapHandler) ->
188194
# a CRS string (or CRS, algorithm tuple). If we try to parse those as type
189195
# strings here, we'd re-enter this validator (or raise) instead of letting
190196
# pydantic validate the raw root values.
191-
if cls.__name__ == "GeometryType" and not v.startswith("geometry"):
197+
if cls.__name__ == "GeometryType" and not v.lstrip().lower().startswith("geometry"):
192198
return handler(v)
193-
if cls.__name__ == "GeographyType" and not v.startswith("geography"):
199+
if cls.__name__ == "GeographyType" and not v.lstrip().lower().startswith("geography"):
194200
return handler(v)
195201

196202
if v == "boolean":
@@ -228,10 +234,10 @@ def handle_primitive_type(cls, v: Any, handler: ValidatorFunctionWrapHandler) ->
228234
if v.startswith("decimal"):
229235
precision, scale = _parse_decimal_type(v)
230236
return DecimalType(precision, scale)
231-
if v.startswith("geometry"):
237+
if v.lstrip().lower().startswith("geometry"):
232238
crs = _parse_geometry_type(v)
233239
return GeometryType(crs)
234-
if v.startswith("geography"):
240+
if v.lstrip().lower().startswith("geography"):
235241
crs, algorithm = _parse_geography_type(v)
236242
return GeographyType(crs, algorithm)
237243
else:

‎tests/integration/test_reads.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,8 @@
5858
from pyiceberg.types import (
5959
BinaryType,
6060
BooleanType,
61+
GeographyType,
62+
GeometryType,
6163
IntegerType,
6264
LongType,
6365
NestedField,
@@ -1252,6 +1254,49 @@ def test_initial_default(catalog: Catalog, spark: SparkSession) -> None:
12521254
assert len(result_table) == 10
12531255

12541256

1257+
@pytest.mark.integration
1258+
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
1259+
def test_read_geospatial_types_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:
1260+
pytest.importorskip("geoarrow.types")
1261+
identifier = "default.test_read_geospatial_types_written_by_spark"
1262+
geom_wkb = "01010000000000000000003e400000000000002440" # POINT (30 10)
1263+
geog_wkb = "01010000000000000000c051c00000000000004540" # POINT (-71 42)
1264+
geospatial_enabled = spark.conf.get("spark.sql.geospatial.enabled", "false")
1265+
spark.conf.set("spark.sql.geospatial.enabled", "true")
1266+
try:
1267+
spark.sql("CREATE NAMESPACE IF NOT EXISTS default")
1268+
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
1269+
spark.sql(
1270+
f"CREATE TABLE {identifier} (id BIGINT, geom GEOMETRY(4326), geog GEOGRAPHY(4326)) "
1271+
"USING iceberg TBLPROPERTIES ('format-version'='3')"
1272+
)
1273+
spark.sql(
1274+
f"INSERT INTO {identifier} VALUES "
1275+
f"(1, st_setsrid(st_geomfromwkb(X'{geom_wkb}'), 4326), st_setsrid(st_geogfromwkb(X'{geog_wkb}'), 4326)), "
1276+
"(2, NULL, NULL)"
1277+
)
1278+
1279+
table = catalog.load_table(identifier)
1280+
assert table.schema().find_type("geom") == GeometryType()
1281+
assert table.schema().find_type("geog") == GeographyType()
1282+
1283+
result = table.scan().to_arrow()
1284+
assert sorted(result.to_pylist(), key=lambda row: row["id"]) == [
1285+
{"id": 1, "geom": bytes.fromhex(geom_wkb), "geog": bytes.fromhex(geog_wkb)},
1286+
{"id": 2, "geom": None, "geog": None},
1287+
]
1288+
for name in ("geom", "geog"):
1289+
arrow_type = result.schema.field(name).type
1290+
assert isinstance(arrow_type, pa.ExtensionType)
1291+
assert arrow_type.extension_name == "geoarrow.wkb"
1292+
assert pyarrow_to_schema(result.schema, name_mapping=table.schema().name_mapping, format_version=3) == table.schema()
1293+
finally:
1294+
try:
1295+
spark.sql(f"DROP TABLE IF EXISTS {identifier}")
1296+
finally:
1297+
spark.conf.set("spark.sql.geospatial.enabled", geospatial_enabled)
1298+
1299+
12551300
@pytest.mark.integration
12561301
@pytest.mark.parametrize("catalog", [lf("session_catalog")])
12571302
def test_read_first_row_ids_written_by_spark(catalog: Catalog, spark: SparkSession) -> None:

‎tests/test_types.py‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717
# pylint: disable=W0123,W0613
18+
import json
1819
import pickle
1920

2021
import pydantic_core
@@ -781,13 +782,19 @@ def test_geometry_type_deserialization() -> None:
781782
"""Test GeometryType JSON deserialization."""
782783
assert GeometryType.model_validate_json('"geometry"') == GeometryType()
783784
assert GeometryType.model_validate_json("\"geometry('EPSG:4326')\"") == GeometryType("EPSG:4326")
785+
for value in ("geometry(OGC:CRS84)", " GEOMETRY ( OGC:CRS84 ) ", "Geometry", 'geometry("OGC:CRS84")'):
786+
assert GeometryType.model_validate_json(json.dumps(value)) == GeometryType()
787+
assert IcebergType.model_validate(value) == GeometryType()
788+
assert NestedField.model_validate({"id": 1, "name": "geom", "type": value}).field_type == GeometryType()
789+
for crs in ("EPSG:3857", "projjson:custom_crs", "srid:0", "custom"):
790+
assert GeometryType.model_validate_json(json.dumps(f"geometry({crs})")) == GeometryType(crs)
784791

785792

786793
def test_geometry_type_deserialization_failure() -> None:
787794
"""Test GeometryType deserialization with invalid input."""
788-
with pytest.raises(ValidationError) as exc_info:
789-
GeometryType.model_validate_json('"geometry(invalid)"')
790-
assert "Could not parse geometry(invalid) into a GeometryType" in str(exc_info.value)
795+
for value in ("geometry()", "geometry( )", "geometry(a,b)", "geometry('EPSG:4326\")", "geometry(a) trailing"):
796+
with pytest.raises(ValidationError, match="Could not parse"):
797+
GeometryType.model_validate_json(json.dumps(value))
791798

792799

793800
def test_geometry_type_singleton() -> None:
@@ -857,13 +864,36 @@ def test_geography_type_deserialization() -> None:
857864
assert GeographyType.model_validate_json('"geography"') == GeographyType()
858865
assert GeographyType.model_validate_json("\"geography('EPSG:4326')\"") == GeographyType("EPSG:4326")
859866
assert GeographyType.model_validate_json("\"geography('EPSG:4326', 'planar')\"") == GeographyType("EPSG:4326", "planar")
867+
for value in (
868+
"geography(OGC:CRS84, spherical)",
869+
" GEOGRAPHY ( OGC:CRS84 , SPHERICAL ) ",
870+
"Geography",
871+
"geography(OGC:CRS84)",
872+
'geography("OGC:CRS84", "spherical")',
873+
):
874+
assert GeographyType.model_validate_json(json.dumps(value)) == GeographyType()
875+
assert IcebergType.model_validate(value) == GeographyType()
876+
assert NestedField.model_validate({"id": 1, "name": "geog", "type": value}).field_type == GeographyType()
877+
for algorithm in ("spherical", "vincenty", "thomas", "andoyer", "karney"):
878+
assert GeographyType.model_validate_json(json.dumps(f"geography(EPSG:4326, {algorithm})")) == GeographyType(
879+
"EPSG:4326", algorithm
880+
)
860881

861882

862883
def test_geography_type_deserialization_failure() -> None:
863884
"""Test GeographyType deserialization with invalid input."""
864-
with pytest.raises(ValidationError) as exc_info:
865-
GeographyType.model_validate_json('"geography(invalid)"')
866-
assert "Could not parse geography(invalid) into a GeographyType" in str(exc_info.value)
885+
for value in (
886+
"geography()",
887+
"geography( )",
888+
"geography(,spherical)",
889+
"geography(OGC:CRS84,)",
890+
"geography(a,b,c)",
891+
"geography('OGC:CRS84\", spherical)",
892+
"geography(OGC:CRS84, 'spherical\")",
893+
"geography(a) trailing",
894+
):
895+
with pytest.raises(ValidationError, match="Could not parse"):
896+
GeographyType.model_validate_json(json.dumps(value))
867897

868898

869899
def test_geography_type_singleton() -> None:

0 commit comments

Comments
 (0)