From f7c3fec7c7fb8ade166deb268158f9c77db77833 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Tue, 17 Nov 2026 17:14:07 +0000 Subject: [PATCH] Decode 4-byte bounds of date columns promoted to timestamp Bounds written before a date to timestamp or timestamp_ns promotion keep their 4-byte day count, but from_bytes used the 8-byte decoder for these types, so planning and inspect raised struct.error. Scale the day count to micros or nanos and return the exact int, so metrics comparisons stay exact (metadata tables can still raise OverflowError for bounds outside the Arrow range). Co-authored-by: Isaac --- pyiceberg/conversions.py | 22 +++++++++++++++-- tests/expressions/test_evaluator.py | 37 +++++++++++++++++++++++++++++ tests/test_conversions.py | 25 +++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) diff --git a/pyiceberg/conversions.py b/pyiceberg/conversions.py index fde1885443..410551e035 100644 --- a/pyiceberg/conversions.py +++ b/pyiceberg/conversions.py @@ -90,6 +90,10 @@ _FLOAT_STRUCT = Struct(" Callable: # type: ignore """Handle cases where partition values are `None` or "__HIVE_DEFAULT_PARTITION__". @@ -352,14 +356,28 @@ def _(_: PrimitiveType, b: bytes) -> int: @from_bytes.register(TimeType) -@from_bytes.register(TimestampType) @from_bytes.register(TimestamptzType) -@from_bytes.register(TimestampNanoType) @from_bytes.register(TimestamptzNanoType) def _(_: PrimitiveType, b: bytes) -> int: return _LONG_STRUCT.unpack(b)[0] +@from_bytes.register(TimestampType) +def _(_: TimestampType, b: bytes) -> int: + if len(b) < 8: + # If the length is 4 bytes, it is a date promoted to timestamp, stored as days from the epoch + return _INT_STRUCT.unpack(b)[0] * _MICROS_PER_DAY + return _LONG_STRUCT.unpack(b)[0] + + +@from_bytes.register(TimestampNanoType) +def _(_: TimestampNanoType, b: bytes) -> int: + if len(b) < 8: + # If the length is 4 bytes, it is a date promoted to timestamp_ns, stored as days from the epoch + return _INT_STRUCT.unpack(b)[0] * _NANOS_PER_DAY + return _LONG_STRUCT.unpack(b)[0] + + @from_bytes.register(LongType) def _(_: PrimitiveType, b: bytes) -> int: if len(b) < 8: diff --git a/tests/expressions/test_evaluator.py b/tests/expressions/test_evaluator.py index bba4156e99..ee7265d7b3 100644 --- a/tests/expressions/test_evaluator.py +++ b/tests/expressions/test_evaluator.py @@ -17,11 +17,14 @@ # pylint:disable=redefined-outer-name from collections.abc import Iterator, Mapping from concurrent.futures import ThreadPoolExecutor +from datetime import date, datetime from threading import Event from typing import Any +import pyarrow as pa import pytest +from pyiceberg.catalog import Catalog from pyiceberg.conversions import to_bytes from pyiceberg.expressions import ( And, @@ -56,6 +59,7 @@ from pyiceberg.schema import Schema from pyiceberg.typedef import Record from pyiceberg.types import ( + DateType, DoubleType, FloatType, IcebergType, @@ -64,6 +68,7 @@ NestedField, PrimitiveType, StringType, + TimestampType, ) INT_MIN_VALUE = 30 @@ -1831,6 +1836,33 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file assert not should_read, "Should not match: no_nulls field does not have bounds" +def test_plan_and_inspect_after_date_to_timestamp_promotion(catalog: Catalog) -> None: + # PyIceberg cannot write v3 metadata yet (#1551), so the promotion is simulated on v2 metadata. The bound + # decoder does not depend on the format version, and the data files keep their 4-byte day counts either way. + catalog.create_namespace("default") + tbl = catalog.create_table("default.date_promotion", Schema(NestedField(1, "d", DateType(), required=False))) + tbl.append(pa.table({"d": pa.array([date(2020, 1, 1), date(2021, 6, 15)], pa.date32())})) + tbl.append(pa.table({"d": pa.array([date(2023, 3, 1), date(2024, 2, 29)], pa.date32())})) + paths = {row["readable_metrics"]["d"]["lower_bound"]: row["file_path"] for row in tbl.inspect.files().to_pylist()} + file_a, file_b = paths[date(2020, 1, 1)], paths[date(2023, 3, 1)] + + # date -> timestamp is a v3 promotion that promote() does not implement, so the incompatible change is allowed + with tbl.update_schema(allow_incompatible_changes=True) as update: + update.update_column("d", field_type=TimestampType()) + + for row_filter, expected_files in [ + # file A's latest date is 2021-06-15, file B's earliest date is 2023-03-01 + (GreaterThan("d", "2022-01-01T00:00:00"), [file_b]), + (LessThan("d", "2021-01-01T00:00:00"), [file_a]), + ]: + planned = tbl.scan(row_filter=row_filter).plan_files() + assert [task.file.file_path for task in planned] == expected_files + + bounds = {row["file_path"]: row["readable_metrics"]["d"] for row in tbl.inspect.files().to_pylist()} + assert (bounds[file_a]["lower_bound"], bounds[file_a]["upper_bound"]) == (datetime(2020, 1, 1), datetime(2021, 6, 15)) + assert (bounds[file_b]["lower_bound"], bounds[file_b]["upper_bound"]) == (datetime(2023, 3, 1), datetime(2024, 2, 29)) + + @pytest.mark.parametrize( "file_type, evolved_type, lower_bound, upper_bound, op, lit, expected", [ @@ -1840,6 +1872,9 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file # Float -> Double (FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_CANNOT_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 50.0, ROWS_MIGHT_MATCH), + # Date -> Timestamp, including a day count before the epoch + (DateType(), TimestampType(), 30, 79, GreaterThan, "2022-01-01T00:00:00", ROWS_CANNOT_MATCH), + (DateType(), TimestampType(), -10, 79, LessThan, "1969-12-31T00:00:00", ROWS_MIGHT_MATCH), ], ) def test_inclusive_metrics_eval_bounds_after_promotion( @@ -1880,6 +1915,8 @@ def test_inclusive_metrics_eval_bounds_after_promotion( (FloatType(), DoubleType(), 30.0, 79.0, GreaterThan, 100.0, ROWS_MIGHT_NOT_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 100.0, ROWS_MUST_MATCH), (FloatType(), DoubleType(), 30.0, 79.0, LessThan, 20.0, ROWS_MIGHT_NOT_MATCH), + # Date -> Timestamp + (DateType(), TimestampType(), 30, 79, GreaterThan, "1970-01-15T00:00:00", ROWS_MUST_MATCH), ], ) def test_strict_metrics_eval_bounds_after_promotion( diff --git a/tests/test_conversions.py b/tests/test_conversions.py index ec14e87d52..1651d2ae3d 100644 --- a/tests/test_conversions.py +++ b/tests/test_conversions.py @@ -299,6 +299,31 @@ def test_from_bytes(primitive_type: PrimitiveType, b: bytes, result: Any) -> Non assert conversions.from_bytes(primitive_type, b) == result +def test_from_bytes_date_bound_promoted_to_timestamp() -> None: + # A bound written before a date -> timestamp promotion keeps its 4-byte day count + cases: list[tuple[PrimitiveType, bytes, int]] = [ + (TimestampType(), struct.pack(" None: + # A bound can be wider than the data, so the scaled value stays exact outside the int64 range + cases: list[tuple[PrimitiveType, int, int]] = [ + (TimestampNanoType(), 106752, 9223372800000000000), + (TimestampNanoType(), -106752, -9223372800000000000), + ] + for primitive_type, days, expected in cases: + assert conversions.from_bytes(primitive_type, struct.pack("