From 1cd1dbfbcd8eaf4247fadafa17b8b4fbcb36c808 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Fri, 20 Nov 2026 22:38:48 +0530 Subject: [PATCH] Tests: Enumerate Arrow and Iceberg primitive type conversions Add table-driven tests that cover every supported primitive Arrow type, every unsupported type, and the round trip for every primitive Iceberg type. This makes the conversion matrix explicit instead of relying on per-type tests that are easy to miss when a type is added. --- tests/io/test_pyarrow_visitor.py | 86 ++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..a2b460e1f8 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -65,8 +65,11 @@ StructType, TimestampNanoType, TimestampType, + TimestamptzNanoType, TimestamptzType, TimeType, + UnknownType, + UUIDType, ) @@ -333,6 +336,89 @@ def test_pyarrow_dictionary_encoded_type_to_iceberg(value_type: pa.DataType, exp assert visit_pyarrow(pyarrow_dict, _ConvertToIceberg()) == expected_result +def test_every_supported_pyarrow_primitive_type_maps_to_iceberg() -> None: + """Every primitive Arrow type supported by the converter maps to the expected Iceberg type.""" + cases: list[tuple[pa.DataType, IcebergType]] = [ + (pa.bool_(), BooleanType()), + (pa.int8(), IntegerType()), + (pa.int16(), IntegerType()), + (pa.int32(), IntegerType()), + (pa.int64(), LongType()), + (pa.uint8(), IntegerType()), + (pa.uint16(), IntegerType()), + (pa.uint32(), IntegerType()), + (pa.uint64(), LongType()), + (pa.float16(), FloatType()), + (pa.float32(), FloatType()), + (pa.float64(), DoubleType()), + (pa.decimal128(38, 9), DecimalType(38, 9)), + (pa.string(), StringType()), + (pa.large_string(), StringType()), + (pa.string_view(), StringType()), + (pa.binary(), BinaryType()), + (pa.large_binary(), BinaryType()), + (pa.binary_view(), BinaryType()), + (pa.binary(16), FixedType(16)), + (pa.date32(), DateType()), + (pa.time64("us"), TimeType()), + (pa.timestamp("s"), TimestampType()), + (pa.timestamp("ms"), TimestampType()), + (pa.timestamp("us"), TimestampType()), + (pa.timestamp("ns"), TimestampNanoType()), + (pa.timestamp("us", tz="UTC"), TimestamptzType()), + (pa.timestamp("ns", tz="UTC"), TimestamptzNanoType()), + (pa.null(), UnknownType()), + (pa.uuid(), UUIDType()), + ] + + for pyarrow_type, expected in cases: + assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == expected, pyarrow_type + + +def test_every_unsupported_pyarrow_primitive_type_is_rejected() -> None: + """Primitive Arrow types without an Iceberg equivalent raise instead of being coerced.""" + unsupported: list[pa.DataType] = [ + pa.decimal256(76, 9), + pa.date64(), + pa.time32("s"), + pa.time32("ms"), + pa.time64("ns"), + pa.duration("us"), + ] + + for pyarrow_type in unsupported: + with pytest.raises(TypeError): + visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) + + +def test_every_iceberg_primitive_type_round_trips_through_pyarrow() -> None: + """Every primitive Iceberg type maps to the expected Arrow type and back.""" + cases: list[tuple[IcebergType, pa.DataType]] = [ + (BooleanType(), pa.bool_()), + (IntegerType(), pa.int32()), + (LongType(), pa.int64()), + (FloatType(), pa.float32()), + (DoubleType(), pa.float64()), + (DecimalType(38, 9), pa.decimal128(38, 9)), + (DateType(), pa.date32()), + (TimeType(), pa.time64("us")), + (TimestampType(), pa.timestamp("us")), + (TimestampNanoType(), pa.timestamp("ns")), + (TimestamptzType(), pa.timestamp("us", tz="UTC")), + (TimestamptzNanoType(), pa.timestamp("ns", tz="UTC")), + (StringType(), pa.large_string()), + (UUIDType(), pa.uuid()), + (UnknownType(), pa.null()), + (BinaryType(), pa.large_binary()), + (FixedType(16), pa.binary(16)), + ] + + for iceberg_type, expected in cases: + pyarrow_type = visit(iceberg_type, _ConvertToArrowSchema()) + assert pyarrow_type == expected, iceberg_type + assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == iceberg_type, iceberg_type + + def test_round_schema_conversion_simple(table_schema_simple: Schema) -> None: actual = str(pyarrow_to_schema(schema_to_pyarrow(table_schema_simple))) expected = """table {