1717# pylint:disable=redefined-outer-name
1818from collections .abc import Iterator , Mapping
1919from concurrent .futures import ThreadPoolExecutor
20+ from datetime import date , datetime
2021from threading import Event
2122from typing import Any
2223
24+ import pyarrow as pa
2325import pytest
2426
27+ from pyiceberg .catalog import Catalog
2528from pyiceberg .conversions import to_bytes
2629from pyiceberg .expressions import (
2730 And ,
5659from pyiceberg .schema import Schema
5760from pyiceberg .typedef import Record
5861from pyiceberg .types import (
62+ DateType ,
5963 DoubleType ,
6064 FloatType ,
6165 IcebergType ,
6468 NestedField ,
6569 PrimitiveType ,
6670 StringType ,
71+ TimestampType ,
6772)
6873
6974INT_MIN_VALUE = 30
@@ -1831,6 +1836,33 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file
18311836 assert not should_read , "Should not match: no_nulls field does not have bounds"
18321837
18331838
1839+ def test_plan_and_inspect_after_date_to_timestamp_promotion (catalog : Catalog ) -> None :
1840+ # PyIceberg cannot write v3 metadata yet (#1551), so the promotion is simulated on v2 metadata. The bound
1841+ # decoder does not depend on the format version, and the data files keep their 4-byte day counts either way.
1842+ catalog .create_namespace ("default" )
1843+ tbl = catalog .create_table ("default.date_promotion" , Schema (NestedField (1 , "d" , DateType (), required = False )))
1844+ tbl .append (pa .table ({"d" : pa .array ([date (2020 , 1 , 1 ), date (2021 , 6 , 15 )], pa .date32 ())}))
1845+ tbl .append (pa .table ({"d" : pa .array ([date (2023 , 3 , 1 ), date (2024 , 2 , 29 )], pa .date32 ())}))
1846+ paths = {row ["readable_metrics" ]["d" ]["lower_bound" ]: row ["file_path" ] for row in tbl .inspect .files ().to_pylist ()}
1847+ file_a , file_b = paths [date (2020 , 1 , 1 )], paths [date (2023 , 3 , 1 )]
1848+
1849+ # date -> timestamp is a v3 promotion that promote() does not implement, so the incompatible change is allowed
1850+ with tbl .update_schema (allow_incompatible_changes = True ) as update :
1851+ update .update_column ("d" , field_type = TimestampType ())
1852+
1853+ for row_filter , expected_files in [
1854+ # file A's latest date is 2021-06-15, file B's earliest date is 2023-03-01
1855+ (GreaterThan ("d" , "2022-01-01T00:00:00" ), [file_b ]),
1856+ (LessThan ("d" , "2021-01-01T00:00:00" ), [file_a ]),
1857+ ]:
1858+ planned = tbl .scan (row_filter = row_filter ).plan_files ()
1859+ assert [task .file .file_path for task in planned ] == expected_files
1860+
1861+ bounds = {row ["file_path" ]: row ["readable_metrics" ]["d" ] for row in tbl .inspect .files ().to_pylist ()}
1862+ assert (bounds [file_a ]["lower_bound" ], bounds [file_a ]["upper_bound" ]) == (datetime (2020 , 1 , 1 ), datetime (2021 , 6 , 15 ))
1863+ assert (bounds [file_b ]["lower_bound" ], bounds [file_b ]["upper_bound" ]) == (datetime (2023 , 3 , 1 ), datetime (2024 , 2 , 29 ))
1864+
1865+
18341866@pytest .mark .parametrize (
18351867 "file_type, evolved_type, lower_bound, upper_bound, op, lit, expected" ,
18361868 [
@@ -1840,6 +1872,9 @@ def test_strict_integer_not_in(strict_data_file_schema: Schema, strict_data_file
18401872 # Float -> Double
18411873 (FloatType (), DoubleType (), 30.0 , 79.0 , GreaterThan , 100.0 , ROWS_CANNOT_MATCH ),
18421874 (FloatType (), DoubleType (), 30.0 , 79.0 , LessThan , 50.0 , ROWS_MIGHT_MATCH ),
1875+ # Date -> Timestamp, including a day count before the epoch
1876+ (DateType (), TimestampType (), 30 , 79 , GreaterThan , "2022-01-01T00:00:00" , ROWS_CANNOT_MATCH ),
1877+ (DateType (), TimestampType (), - 10 , 79 , LessThan , "1969-12-31T00:00:00" , ROWS_MIGHT_MATCH ),
18431878 ],
18441879)
18451880def test_inclusive_metrics_eval_bounds_after_promotion (
@@ -1880,6 +1915,8 @@ def test_inclusive_metrics_eval_bounds_after_promotion(
18801915 (FloatType (), DoubleType (), 30.0 , 79.0 , GreaterThan , 100.0 , ROWS_MIGHT_NOT_MATCH ),
18811916 (FloatType (), DoubleType (), 30.0 , 79.0 , LessThan , 100.0 , ROWS_MUST_MATCH ),
18821917 (FloatType (), DoubleType (), 30.0 , 79.0 , LessThan , 20.0 , ROWS_MIGHT_NOT_MATCH ),
1918+ # Date -> Timestamp
1919+ (DateType (), TimestampType (), 30 , 79 , GreaterThan , "1970-01-15T00:00:00" , ROWS_MUST_MATCH ),
18831920 ],
18841921)
18851922def test_strict_metrics_eval_bounds_after_promotion (
0 commit comments