Skip to content

Commit e694079

Browse files
ump45nosekevinjqliuCopilot
authored
Support reading Avro timestamp-nanos logical type (#4136)
* Support reading Avro timestamp-nanos logical type * Use a loop instead of parametrize in timestamp-nanos conversion test Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: ump45nose <ump45nose@users.noreply.github.com> Co-authored-by: Kevin Liu <kevinjqliu@users.noreply.github.com> Co-authored-by: Kevin Liu <kevin.jq.liu@gmail.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 46669fa commit e694079

3 files changed

Lines changed: 61 additions & 2 deletions

File tree

‎pyiceberg/utils/schema_conversion.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@
4848
PrimitiveType,
4949
StringType,
5050
StructType,
51+
TimestampNanoType,
5152
TimestampType,
53+
TimestamptzNanoType,
5254
TimestamptzType,
5355
TimeType,
5456
UnknownType,
@@ -381,6 +383,11 @@ def _convert_logical_type(self, avro_logical_type: dict[str, Any]) -> IcebergTyp
381383
return TimestamptzType()
382384
else:
383385
return TimestampType()
386+
elif logical_type == "timestamp-nanos":
387+
if avro_logical_type.get("adjust-to-utc", False) is True:
388+
return TimestamptzNanoType()
389+
else:
390+
return TimestampNanoType()
384391
elif (logical_type, physical_type) in LOGICAL_FIELD_TYPE_MAPPING:
385392
return LOGICAL_FIELD_TYPE_MAPPING[(logical_type, physical_type)]
386393
else:
@@ -618,13 +625,13 @@ def visit_time(self, time_type: TimeType) -> AvroType:
618625
def visit_timestamp(self, timestamp_type: TimestampType) -> AvroType:
619626
return {"type": "long", "logicalType": "timestamp-micros", "adjust-to-utc": False}
620627

621-
def visit_timestamp_ns(self, timestamp_type: TimestampType) -> AvroType:
628+
def visit_timestamp_ns(self, timestamp_ns_type: TimestampNanoType) -> AvroType:
622629
return {"type": "long", "logicalType": "timestamp-nanos", "adjust-to-utc": False}
623630

624631
def visit_timestamptz(self, timestamptz_type: TimestamptzType) -> AvroType:
625632
return {"type": "long", "logicalType": "timestamp-micros", "adjust-to-utc": True}
626633

627-
def visit_timestamptz_ns(self, timestamptz_type: TimestamptzType) -> AvroType:
634+
def visit_timestamptz_ns(self, timestamptz_ns_type: TimestamptzNanoType) -> AvroType:
628635
return {"type": "long", "logicalType": "timestamp-nanos", "adjust-to-utc": True}
629636

630637
def visit_string(self, string_type: StringType) -> AvroType:

‎tests/avro/test_file.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,9 @@
5151
LongType,
5252
NestedField,
5353
StringType,
54+
TimestampNanoType,
5455
TimestampType,
56+
TimestamptzNanoType,
5557
TimestamptzType,
5658
TimeType,
5759
UUIDType,
@@ -455,6 +457,31 @@ def field_uuid(self) -> UUID:
455457
assert record[idx] == avro_entry_read_with_fastavro[idx], f"Invalid {field} read with fastavro"
456458

457459

460+
def test_timestamp_nanos_round_trip() -> None:
461+
schema = Schema(
462+
NestedField(field_id=1, name="ts_ns", field_type=TimestampNanoType(), required=True),
463+
NestedField(field_id=2, name="tstz_ns", field_type=TimestamptzNanoType(), required=False),
464+
schema_id=1,
465+
)
466+
records = [
467+
Record(1677629965123456789, 1677629965123456789),
468+
Record(-1, None),
469+
]
470+
471+
with TemporaryDirectory() as tmpdir:
472+
tmp_avro_file = tmpdir + "/timestamp_nanos.avro"
473+
with avro.AvroOutputFile[Record](PyArrowFileIO().new_output(tmp_avro_file), schema, "timestamp_nanos") as out:
474+
out.write_block(records)
475+
476+
with avro.AvroFile[Record](
477+
PyArrowFileIO().new_input(tmp_avro_file),
478+
schema,
479+
{-1: Record},
480+
) as avro_reader:
481+
assert avro_reader.schema == schema
482+
assert list(avro_reader) == records
483+
484+
458485
def manifest_entry(index: int) -> ManifestEntry:
459486
return ManifestEntry.from_args(
460487
status=ManifestEntryStatus.ADDED,

‎tests/utils/test_schema_conversion.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@
3333
NestedField,
3434
StringType,
3535
StructType,
36+
TimestampNanoType,
3637
TimestampType,
38+
TimestamptzNanoType,
3739
UnknownType,
3840
UUIDType,
3941
)
@@ -347,6 +349,29 @@ def test_convert_timestamp_micros_type() -> None:
347349
assert actual == TimestampType()
348350

349351

352+
def test_convert_timestamp_nanos_type() -> None:
353+
for adjust_to_utc, expected in [
354+
(None, TimestampNanoType()),
355+
(False, TimestampNanoType()),
356+
(True, TimestamptzNanoType()),
357+
]:
358+
avro_logical_type: dict[str, Any] = {"type": "long", "logicalType": "timestamp-nanos"}
359+
if adjust_to_utc is not None:
360+
avro_logical_type["adjust-to-utc"] = adjust_to_utc
361+
actual = AvroSchemaConversion()._convert_logical_type(avro_logical_type)
362+
assert actual == expected, f"adjust-to-utc={adjust_to_utc}"
363+
364+
365+
def test_timestamp_nanos_avro_schema_round_trip() -> None:
366+
schema = Schema(
367+
NestedField(field_id=1, name="ts_ns", field_type=TimestampNanoType(), required=True),
368+
NestedField(field_id=2, name="tstz_ns", field_type=TimestamptzNanoType(), required=False),
369+
)
370+
avro_schema = AvroSchemaConversion().iceberg_to_avro(schema)
371+
assert isinstance(avro_schema, dict)
372+
assert AvroSchemaConversion().avro_to_iceberg(avro_schema) == schema
373+
374+
350375
def test_unknown_logical_type() -> None:
351376
"""Test raising a ValueError when converting an unknown logical type as part of an Avro schema conversion"""
352377
avro_logical_type = {"type": "bytes", "logicalType": "date"}

0 commit comments

Comments
 (0)