Skip to content

Commit 2a1439e

Browse files
rahulsmahadevIsaac
andauthored
Fix append to a table partitioned by bucket on a time column (#4343)
* Fix append to a table partitioned by bucket on a time column The bucket transform returns an int, but the time partition representation only accepted datetime.time, so every append raised AttributeError. Accept the int, as the date, timestamp and uuid handlers already do. Co-authored-by: Isaac <no-reply@databricks.com> * Retrigger CI Co-authored-by: Isaac <no-reply@databricks.com> * Retrigger CI Co-authored-by: Isaac <no-reply@databricks.com> * Retrigger CI Co-authored-by: Isaac <no-reply@databricks.com> --------- Co-authored-by: Isaac <no-reply@databricks.com>
1 parent aeaddb1 commit 2a1439e

3 files changed

Lines changed: 41 additions & 4 deletions

File tree

‎pyiceberg/partitioning.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -549,8 +549,15 @@ def _(type: IcebergType, value: int | date | None) -> int | None:
549549

550550

551551
@_to_partition_representation.register(TimeType)
552-
def _(type: IcebergType, value: time | None) -> int | None:
553-
return time_to_micros(value) if value is not None else None
552+
def _(type: IcebergType, value: int | time | None) -> int | None:
553+
if value is None:
554+
return None
555+
elif isinstance(value, int):
556+
return value
557+
elif isinstance(value, time):
558+
return time_to_micros(value)
559+
else:
560+
raise ValueError(f"Type not recognized: {value}")
554561

555562

556563
@_to_partition_representation.register(UUIDType)

‎tests/io/test_pyarrow.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
import zlib
2727
from collections.abc import Callable, Iterator
2828
from dataclasses import replace
29-
from datetime import UTC, date, datetime
29+
from datetime import UTC, date, datetime, time
3030
from decimal import Decimal
3131
from pathlib import Path
3232
from typing import Any, Literal
@@ -3524,6 +3524,27 @@ def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> Non
35243524
assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == arrow_table.to_pylist()
35253525

35263526

3527+
def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None:
3528+
schema = Schema(
3529+
NestedField(id=1, name="id", field_type=LongType(), required=False),
3530+
NestedField(id=2, name="event_time", field_type=TimeType(), required=False),
3531+
)
3532+
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=BucketTransform(4), name="event_time_bucket"))
3533+
catalog.create_namespace("default")
3534+
table = catalog.create_table("default.test_append_bucketed_by_time", schema=schema, partition_spec=spec)
3535+
3536+
arrow_table = pa.table(
3537+
{
3538+
"id": pa.array([1, 2, 3], type=pa.int64()),
3539+
"event_time": pa.array([time(9, 30), time(22, 31, 8), None], type=pa.time64("us")),
3540+
}
3541+
)
3542+
table.append(arrow_table)
3543+
3544+
assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == arrow_table.to_pylist()
3545+
assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2]
3546+
3547+
35273548
def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None:
35283549
schema = Schema(
35293550
NestedField(id=1, name="foo", field_type=StringType(), required=True),

‎tests/test_transforms.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
# under the License.
1818
# pylint: disable=eval-used,protected-access,redefined-outer-name
1919
from collections.abc import Callable
20-
from datetime import date, datetime
20+
from datetime import date, datetime, time
2121
from decimal import Decimal
2222
from typing import Annotated, Any
2323
from uuid import UUID
@@ -1661,11 +1661,20 @@ def test_to_partition_representation_timestamps(source_type: PrimitiveType, valu
16611661
assert _to_partition_representation(source_type, value) == expected
16621662

16631663

1664+
def test_to_partition_representation_time() -> None:
1665+
assert _to_partition_representation(TimeType(), None) is None
1666+
assert _to_partition_representation(TimeType(), time(22, 31, 8)) == 81068000000
1667+
# The bucket transform returns an int, which is already the partition representation
1668+
assert _to_partition_representation(TimeType(), 3) == 3
1669+
1670+
16641671
def test_to_partition_representation_unrecognized_type_raises() -> None:
16651672
with pytest.raises(ValueError, match="Type not recognized"):
16661673
_to_partition_representation(TimestampType(), "not-a-datetime-or-int")
16671674
with pytest.raises(ValueError, match="Type not recognized"):
16681675
_to_partition_representation(TimestampNanoType(), "not-a-datetime-or-int")
1676+
with pytest.raises(ValueError, match="Type not recognized"):
1677+
_to_partition_representation(TimeType(), "not-a-time-or-int")
16691678

16701679

16711680
@pytest.mark.parametrize(

0 commit comments

Comments
 (0)