Skip to content

Commit 1cd1dbf

Browse files
committed
Tests: Enumerate Arrow and Iceberg primitive type conversions
Add table-driven tests that cover every supported primitive Arrow type, every unsupported type, and the round trip for every primitive Iceberg type. This makes the conversion matrix explicit instead of relying on per-type tests that are easy to miss when a type is added.
1 parent 07216ed commit 1cd1dbf

1 file changed

Lines changed: 86 additions & 0 deletions

File tree

‎tests/io/test_pyarrow_visitor.py‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,11 @@
6565
StructType,
6666
TimestampNanoType,
6767
TimestampType,
68+
TimestamptzNanoType,
6869
TimestamptzType,
6970
TimeType,
71+
UnknownType,
72+
UUIDType,
7073
)
7174

7275

@@ -333,6 +336,89 @@ def test_pyarrow_dictionary_encoded_type_to_iceberg(value_type: pa.DataType, exp
333336
assert visit_pyarrow(pyarrow_dict, _ConvertToIceberg()) == expected_result
334337

335338

339+
def test_every_supported_pyarrow_primitive_type_maps_to_iceberg() -> None:
340+
"""Every primitive Arrow type supported by the converter maps to the expected Iceberg type."""
341+
cases: list[tuple[pa.DataType, IcebergType]] = [
342+
(pa.bool_(), BooleanType()),
343+
(pa.int8(), IntegerType()),
344+
(pa.int16(), IntegerType()),
345+
(pa.int32(), IntegerType()),
346+
(pa.int64(), LongType()),
347+
(pa.uint8(), IntegerType()),
348+
(pa.uint16(), IntegerType()),
349+
(pa.uint32(), IntegerType()),
350+
(pa.uint64(), LongType()),
351+
(pa.float16(), FloatType()),
352+
(pa.float32(), FloatType()),
353+
(pa.float64(), DoubleType()),
354+
(pa.decimal128(38, 9), DecimalType(38, 9)),
355+
(pa.string(), StringType()),
356+
(pa.large_string(), StringType()),
357+
(pa.string_view(), StringType()),
358+
(pa.binary(), BinaryType()),
359+
(pa.large_binary(), BinaryType()),
360+
(pa.binary_view(), BinaryType()),
361+
(pa.binary(16), FixedType(16)),
362+
(pa.date32(), DateType()),
363+
(pa.time64("us"), TimeType()),
364+
(pa.timestamp("s"), TimestampType()),
365+
(pa.timestamp("ms"), TimestampType()),
366+
(pa.timestamp("us"), TimestampType()),
367+
(pa.timestamp("ns"), TimestampNanoType()),
368+
(pa.timestamp("us", tz="UTC"), TimestamptzType()),
369+
(pa.timestamp("ns", tz="UTC"), TimestamptzNanoType()),
370+
(pa.null(), UnknownType()),
371+
(pa.uuid(), UUIDType()),
372+
]
373+
374+
for pyarrow_type, expected in cases:
375+
assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == expected, pyarrow_type
376+
377+
378+
def test_every_unsupported_pyarrow_primitive_type_is_rejected() -> None:
379+
"""Primitive Arrow types without an Iceberg equivalent raise instead of being coerced."""
380+
unsupported: list[pa.DataType] = [
381+
pa.decimal256(76, 9),
382+
pa.date64(),
383+
pa.time32("s"),
384+
pa.time32("ms"),
385+
pa.time64("ns"),
386+
pa.duration("us"),
387+
]
388+
389+
for pyarrow_type in unsupported:
390+
with pytest.raises(TypeError):
391+
visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3))
392+
393+
394+
def test_every_iceberg_primitive_type_round_trips_through_pyarrow() -> None:
395+
"""Every primitive Iceberg type maps to the expected Arrow type and back."""
396+
cases: list[tuple[IcebergType, pa.DataType]] = [
397+
(BooleanType(), pa.bool_()),
398+
(IntegerType(), pa.int32()),
399+
(LongType(), pa.int64()),
400+
(FloatType(), pa.float32()),
401+
(DoubleType(), pa.float64()),
402+
(DecimalType(38, 9), pa.decimal128(38, 9)),
403+
(DateType(), pa.date32()),
404+
(TimeType(), pa.time64("us")),
405+
(TimestampType(), pa.timestamp("us")),
406+
(TimestampNanoType(), pa.timestamp("ns")),
407+
(TimestamptzType(), pa.timestamp("us", tz="UTC")),
408+
(TimestamptzNanoType(), pa.timestamp("ns", tz="UTC")),
409+
(StringType(), pa.large_string()),
410+
(UUIDType(), pa.uuid()),
411+
(UnknownType(), pa.null()),
412+
(BinaryType(), pa.large_binary()),
413+
(FixedType(16), pa.binary(16)),
414+
]
415+
416+
for iceberg_type, expected in cases:
417+
pyarrow_type = visit(iceberg_type, _ConvertToArrowSchema())
418+
assert pyarrow_type == expected, iceberg_type
419+
assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == iceberg_type, iceberg_type
420+
421+
336422
def test_round_schema_conversion_simple(table_schema_simple: Schema) -> None:
337423
actual = str(pyarrow_to_schema(schema_to_pyarrow(table_schema_simple)))
338424
expected = """table {

0 commit comments

Comments
 (0)