Skip to content

Commit f7c3fec

Browse files
rahulsmahadevIsaac
andcommitted
Decode 4-byte bounds of date columns promoted to timestamp
Bounds written before a date to timestamp or timestamp_ns promotion keep their 4-byte day count, but from_bytes used the 8-byte decoder for these types, so planning and inspect raised struct.error. Scale the day count to micros or nanos and return the exact int, so metrics comparisons stay exact (metadata tables can still raise OverflowError for bounds outside the Arrow range). Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 3c68d69 commit f7c3fec

3 files changed

Lines changed: 82 additions & 2 deletions

File tree

‎pyiceberg/conversions.py‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,10 @@
9090
_FLOAT_STRUCT = Struct("<f")
9191
_DOUBLE_STRUCT = Struct("<d")
9292

93+
# Scaled bounds are not range checked: a bound can be wider than the data, and Python ints keep comparisons exact
94+
_MICROS_PER_DAY = 86_400 * 1_000_000
95+
_NANOS_PER_DAY = 86_400 * 1_000_000_000
96+
9397

9498
def handle_none(func: Callable) -> Callable: # type: ignore
9599
"""Handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__".
@@ -352,14 +356,28 @@ def _(_: PrimitiveType, b: bytes) -> int:
352356

353357

354358
@from_bytes.register(TimeType)
355-
@from_bytes.register(TimestampType)
356359
@from_bytes.register(TimestamptzType)
357-
@from_bytes.register(TimestampNanoType)
358360
@from_bytes.register(TimestamptzNanoType)
359361
def _(_: PrimitiveType, b: bytes) -> int:
360362
return _LONG_STRUCT.unpack(b)[0]
361363

362364

365+
@from_bytes.register(TimestampType)
366+
def _(_: TimestampType, b: bytes) -> int:
367+
if len(b) < 8:
368+
# If the length is 4 bytes, it is a date promoted to timestamp, stored as days from the epoch
369+
return _INT_STRUCT.unpack(b)[0] * _MICROS_PER_DAY
370+
return _LONG_STRUCT.unpack(b)[0]
371+
372+
373+
@from_bytes.register(TimestampNanoType)
374+
def _(_: TimestampNanoType, b: bytes) -> int:
375+
if len(b) < 8:
376+
# If the length is 4 bytes, it is a date promoted to timestamp_ns, stored as days from the epoch
377+
return _INT_STRUCT.unpack(b)[0] * _NANOS_PER_DAY
378+
return _LONG_STRUCT.unpack(b)[0]
379+
380+
363381
@from_bytes.register(LongType)
364382
def _(_: PrimitiveType, b: bytes) -> int:
365383
if len(b) < 8:

‎tests/expressions/test_evaluator.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,14 @@
1717
# pylint:disable=redefined-outer-name
1818
from collections.abc import Iterator, Mapping
1919
from concurrent.futures import ThreadPoolExecutor
20+
from datetime import date, datetime
2021
from threading import Event
2122
from typing import Any
2223

24+
import pyarrow as pa
2325
import pytest
2426

27+
from pyiceberg.catalog import Catalog
2528
from pyiceberg.conversions import to_bytes
2629
from pyiceberg.expressions import (
2730
And,
@@ -56,6 +59,7 @@
5659
from pyiceberg.schema import Schema
5760
from pyiceberg.typedef import Record
5861
from pyiceberg.types import (
62+
DateType,
5963
DoubleType,
6064
FloatType,
6165
IcebergType,
@@ -64,6 +68,7 @@
6468
NestedField,
6569
PrimitiveType,
6670
StringType,
71+
TimestampType,
6772
)
6873

6974
INT_MIN_VALUE = 30
@@ -1831,6 +1836,33 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file
18311836
assert not should_read, "Should not match: no_nulls field does not have bounds"
18321837

18331838

1839+
def test_plan_and_inspect_after_date_to_timestamp_promotion(catalog: Catalog) -> None:
1840+
# PyIceberg cannot write v3 metadata yet (#1551), so the promotion is simulated on v2 metadata. The bound
1841+
# decoder does not depend on the format version, and the data files keep their 4-byte day counts either way.
1842+
catalog.create_namespace("default")
1843+
tbl = catalog.create_table("default.date_promotion", Schema(NestedField(1, "d", DateType(), required=False)))
1844+
tbl.append(pa.table({"d": pa.array([date(2020, 1, 1), date(2021, 6, 15)], pa.date32())}))
1845+
tbl.append(pa.table({"d": pa.array([date(2023, 3, 1), date(2024, 2, 29)], pa.date32())}))
1846+
paths = {row["readable_metrics"]["d"]["lower_bound"]: row["file_path"] for row in tbl.inspect.files().to_pylist()}
1847+
file_a, file_b = paths[date(2020, 1, 1)], paths[date(2023, 3, 1)]
1848+
1849+
# date -> timestamp is a v3 promotion that promote() does not implement, so the incompatible change is allowed
1850+
with tbl.update_schema(allow_incompatible_changes=True) as update:
1851+
update.update_column("d", field_type=TimestampType())
1852+
1853+
for row_filter, expected_files in [
1854+
# file A's latest date is 2021-06-15, file B's earliest date is 2023-03-01
1855+
(GreaterThan("d", "2022-01-01T00:00:00"), [file_b]),
1856+
(LessThan("d", "2021-01-01T00:00:00"), [file_a]),
1857+
]:
1858+
planned = tbl.scan(row_filter=row_filter).plan_files()
1859+
assert [task.file.file_path for task in planned] == expected_files
1860+
1861+
bounds = {row["file_path"]: row["readable_metrics"]["d"] for row in tbl.inspect.files().to_pylist()}
1862+
assert (bounds[file_a]["lower_bound"], bounds[file_a]["upper_bound"]) == (datetime(2020, 1, 1), datetime(2021, 6, 15))
1863+
assert (bounds[file_b]["lower_bound"], bounds[file_b]["upper_bound"]) == (datetime(2023, 3, 1), datetime(2024, 2, 29))
1864+
1865+
18341866
@pytest.mark.parametrize(
18351867
"file_type, evolved_type, lower_bound, upper_bound, op, lit, expected",
18361868
[
@@ -1840,6 +1872,9 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file
18401872
# Float -> Double
18411873
(FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_CANNOT_MATCH),
18421874
(FloatType(), DoubleType(), 30.0, 79.0, LessThan, 50.0, ROWS_MIGHT_MATCH),
1875+
# Date -> Timestamp, including a day count before the epoch
1876+
(DateType(), TimestampType(), 30, 79, GreaterThan, "2022-01-01T00:00:00", ROWS_CANNOT_MATCH),
1877+
(DateType(), TimestampType(), -10, 79, LessThan, "1969-12-31T00:00:00", ROWS_MIGHT_MATCH),
18431878
],
18441879
)
18451880
def test_inclusive_metrics_eval_bounds_after_promotion(
@@ -1880,6 +1915,8 @@ def test_inclusive_metrics_eval_bounds_after_promotion(
18801915
(FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_MIGHT_NOT_MATCH),
18811916
(FloatType(), DoubleType(), 30.0, 79.0, LessThan, 100.0, ROWS_MUST_MATCH),
18821917
(FloatType(), DoubleType(), 30.0, 79.0, LessThan, 20.0, ROWS_MIGHT_NOT_MATCH),
1918+
# Date -> Timestamp
1919+
(DateType(), TimestampType(), 30, 79, GreaterThan, "1970-01-15T00:00:00", ROWS_MUST_MATCH),
18831920
],
18841921
)
18851922
def test_strict_metrics_eval_bounds_after_promotion(

‎tests/test_conversions.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,31 @@ def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> Non
299299
assert conversions.from_bytes(primitive_type, b) == result
300300

301301

302+
def test_from_bytes_date_bound_promoted_to_timestamp() -> None:
303+
# A bound written before a date -> timestamp promotion keeps its 4-byte day count
304+
cases: list[tuple[PrimitiveType, bytes, int]] = [
305+
(TimestampType(), struct.pack("<i", 18793), 1623715200000000), # 2021-06-15
306+
(TimestampType(), struct.pack("<i", -3653), -315619200000000), # 1960-01-01
307+
(TimestampNanoType(), struct.pack("<i", 18793), 1623715200000000000),
308+
(TimestampNanoType(), struct.pack("<i", -3653), -315619200000000000),
309+
# The 8-byte encoding is unchanged
310+
(TimestampType(), struct.pack("<q", 1623715200000000), 1623715200000000),
311+
(TimestampNanoType(), struct.pack("<q", 1623715200000000000), 1623715200000000000),
312+
]
313+
for primitive_type, b, expected in cases:
314+
assert conversions.from_bytes(primitive_type, b) == expected, primitive_type
315+
316+
317+
def test_from_bytes_date_bound_promoted_to_timestamp_beyond_int64() -> None:
318+
# A bound can be wider than the data, so the scaled value stays exact outside the int64 range
319+
cases: list[tuple[PrimitiveType, int, int]] = [
320+
(TimestampNanoType(), 106752, 9223372800000000000),
321+
(TimestampNanoType(), -106752, -9223372800000000000),
322+
]
323+
for primitive_type, days, expected in cases:
324+
assert conversions.from_bytes(primitive_type, struct.pack("<i", days)) == expected
325+
326+
302327
@pytest.mark.parametrize(
303328
"primitive_type, b, result",
304329
[

0 commit comments

Comments
 (0)