|
65 | 65 | StructType, |
66 | 66 | TimestampNanoType, |
67 | 67 | TimestampType, |
| 68 | + TimestamptzNanoType, |
68 | 69 | TimestamptzType, |
69 | 70 | TimeType, |
| 71 | + UnknownType, |
| 72 | + UUIDType, |
70 | 73 | ) |
71 | 74 |
|
72 | 75 |
|
@@ -333,6 +336,89 @@ def test_pyarrow_dictionary_encoded_type_to_iceberg(value_type: pa.DataType, exp |
333 | 336 | assert visit_pyarrow(pyarrow_dict, _ConvertToIceberg()) == expected_result |
334 | 337 |
|
335 | 338 |
|
| 339 | +def test_every_supported_pyarrow_primitive_type_maps_to_iceberg() -> None: |
| 340 | + """Every primitive Arrow type supported by the converter maps to the expected Iceberg type.""" |
| 341 | + cases: list[tuple[pa.DataType, IcebergType]] = [ |
| 342 | + (pa.bool_(), BooleanType()), |
| 343 | + (pa.int8(), IntegerType()), |
| 344 | + (pa.int16(), IntegerType()), |
| 345 | + (pa.int32(), IntegerType()), |
| 346 | + (pa.int64(), LongType()), |
| 347 | + (pa.uint8(), IntegerType()), |
| 348 | + (pa.uint16(), IntegerType()), |
| 349 | + (pa.uint32(), IntegerType()), |
| 350 | + (pa.uint64(), LongType()), |
| 351 | + (pa.float16(), FloatType()), |
| 352 | + (pa.float32(), FloatType()), |
| 353 | + (pa.float64(), DoubleType()), |
| 354 | + (pa.decimal128(38, 9), DecimalType(38, 9)), |
| 355 | + (pa.string(), StringType()), |
| 356 | + (pa.large_string(), StringType()), |
| 357 | + (pa.string_view(), StringType()), |
| 358 | + (pa.binary(), BinaryType()), |
| 359 | + (pa.large_binary(), BinaryType()), |
| 360 | + (pa.binary_view(), BinaryType()), |
| 361 | + (pa.binary(16), FixedType(16)), |
| 362 | + (pa.date32(), DateType()), |
| 363 | + (pa.time64("us"), TimeType()), |
| 364 | + (pa.timestamp("s"), TimestampType()), |
| 365 | + (pa.timestamp("ms"), TimestampType()), |
| 366 | + (pa.timestamp("us"), TimestampType()), |
| 367 | + (pa.timestamp("ns"), TimestampNanoType()), |
| 368 | + (pa.timestamp("us", tz="UTC"), TimestamptzType()), |
| 369 | + (pa.timestamp("ns", tz="UTC"), TimestamptzNanoType()), |
| 370 | + (pa.null(), UnknownType()), |
| 371 | + (pa.uuid(), UUIDType()), |
| 372 | + ] |
| 373 | + |
| 374 | + for pyarrow_type, expected in cases: |
| 375 | + assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == expected, pyarrow_type |
| 376 | + |
| 377 | + |
| 378 | +def test_every_unsupported_pyarrow_primitive_type_is_rejected() -> None: |
| 379 | + """Primitive Arrow types without an Iceberg equivalent raise instead of being coerced.""" |
| 380 | + unsupported: list[pa.DataType] = [ |
| 381 | + pa.decimal256(76, 9), |
| 382 | + pa.date64(), |
| 383 | + pa.time32("s"), |
| 384 | + pa.time32("ms"), |
| 385 | + pa.time64("ns"), |
| 386 | + pa.duration("us"), |
| 387 | + ] |
| 388 | + |
| 389 | + for pyarrow_type in unsupported: |
| 390 | + with pytest.raises(TypeError): |
| 391 | + visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) |
| 392 | + |
| 393 | + |
| 394 | +def test_every_iceberg_primitive_type_round_trips_through_pyarrow() -> None: |
| 395 | + """Every primitive Iceberg type maps to the expected Arrow type and back.""" |
| 396 | + cases: list[tuple[IcebergType, pa.DataType]] = [ |
| 397 | + (BooleanType(), pa.bool_()), |
| 398 | + (IntegerType(), pa.int32()), |
| 399 | + (LongType(), pa.int64()), |
| 400 | + (FloatType(), pa.float32()), |
| 401 | + (DoubleType(), pa.float64()), |
| 402 | + (DecimalType(38, 9), pa.decimal128(38, 9)), |
| 403 | + (DateType(), pa.date32()), |
| 404 | + (TimeType(), pa.time64("us")), |
| 405 | + (TimestampType(), pa.timestamp("us")), |
| 406 | + (TimestampNanoType(), pa.timestamp("ns")), |
| 407 | + (TimestamptzType(), pa.timestamp("us", tz="UTC")), |
| 408 | + (TimestamptzNanoType(), pa.timestamp("ns", tz="UTC")), |
| 409 | + (StringType(), pa.large_string()), |
| 410 | + (UUIDType(), pa.uuid()), |
| 411 | + (UnknownType(), pa.null()), |
| 412 | + (BinaryType(), pa.large_binary()), |
| 413 | + (FixedType(16), pa.binary(16)), |
| 414 | + ] |
| 415 | + |
| 416 | + for iceberg_type, expected in cases: |
| 417 | + pyarrow_type = visit(iceberg_type, _ConvertToArrowSchema()) |
| 418 | + assert pyarrow_type == expected, iceberg_type |
| 419 | + assert visit_pyarrow(pyarrow_type, _ConvertToIceberg(format_version=3)) == iceberg_type, iceberg_type |
| 420 | + |
| 421 | + |
336 | 422 | def test_round_schema_conversion_simple(table_schema_simple: Schema) -> None: |
337 | 423 | actual = str(pyarrow_to_schema(schema_to_pyarrow(table_schema_simple))) |
338 | 424 | expected = """table { |
|
0 commit comments