From 8d4683d49389e95ad614e5d5607847df27c66c6b Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 18:03:44 +0530 Subject: [PATCH 1/4] Fix scan filters at the edges of the date and float ranges Year and month partitions raised OverflowError when a filter projected past 9999-12-31 or before 0001-01-01, because the transform converted the day count to a datetime.date. Compute years and months from the day count with integer arithmetic, so projections just past the last representable date produce year 10000 and match no partition instead of raising. A Python float became a DoubleLiteral, and converting that to a float column turned inf into FloatAboveMax or FloatBelowMin, which bound the predicate to AlwaysFalse or AlwaysTrue. inf is a valid float value, so convert it to a FloatLiteral like any other float that fits. --- pyiceberg/expressions/literals.py | 4 +++- pyiceberg/utils/datetime.py | 34 ++++++++++++++++++++-------- tests/expressions/test_literals.py | 6 +++++ tests/expressions/test_projection.py | 15 ++++++++++++ tests/io/test_pyarrow.py | 29 +++++++++++++++++++++++- tests/utils/test_datetime.py | 32 ++++++++++++++++++++++++++ 6 files changed, 109 insertions(+), 11 deletions(-) diff --git a/pyiceberg/expressions/literals.py b/pyiceberg/expressions/literals.py index 9952fd82d4..377f6c2b4a 100644 --- a/pyiceberg/expressions/literals.py +++ b/pyiceberg/expressions/literals.py @@ -26,7 +26,7 @@ from datetime import date, datetime, time from decimal import ROUND_HALF_UP, Decimal from functools import singledispatchmethod -from math import isfinite, isnan +from math import isfinite, isinf, isnan from typing import Any, Generic from uuid import UUID @@ -433,6 +433,8 @@ def _(self, _: DoubleType) -> Literal[float]: @to.register(FloatType) def _(self, _: FloatType) -> Literal[float]: + if isinf(self.value): + return FloatLiteral(self.value) if FloatType.max < self.value: return FloatAboveMax() elif FloatType.min > self.value: diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index 70ffb1f303..fdefc534ce 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -229,22 +229,39 @@ def micros_to_hours(micros: int) -> int: return micros // 3_600_000_000 +def _civil_from_days(days: int) -> tuple[int, int]: + """Return the year and month of the number of days from 1970-01-01. + + The conversion uses integer arithmetic, so values outside the range of ``datetime.date``, such + as the day after 9999-12-31, are converted as well. Based on + https://howardhinnant.github.io/date_algorithms.html#civil_from_days. + """ + z = days + 719468 + era = (z if z >= 0 else z - 146096) // 146097 + doe = z - era * 146097 + yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365 + doy = doe - (365 * yoe + yoe // 4 - yoe // 100) + mp = (5 * doy + 2) // 153 + month = mp + 3 if mp < 10 else mp - 9 + year = yoe + era * 400 + (1 if month <= 2 else 0) + return year, month + + def days_to_months(days: int) -> int: - d = days_to_date(days) - return (d.year - EPOCH_DATE.year) * 12 + (d.month - EPOCH_DATE.month) + year, month = _civil_from_days(days) + return (year - EPOCH_DATE.year) * 12 + (month - EPOCH_DATE.month) def micros_to_months(micros: int) -> int: - dt = micros_to_timestamp(micros) - return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) + return days_to_months(micros // 86_400_000_000) def days_to_years(days: int) -> int: - return days_to_date(days).year - EPOCH_DATE.year + return _civil_from_days(days)[0] - EPOCH_DATE.year def micros_to_years(micros: int) -> int: - return micros_to_timestamp(micros).year - EPOCH_TIMESTAMP.year + return days_to_years(micros // 86_400_000_000) def nanos_to_timestamp(nanos: int) -> datetime: @@ -254,12 +271,11 @@ def nanos_to_timestamp(nanos: int) -> datetime: def nanos_to_years(nanos: int) -> int: - return nanos_to_timestamp(nanos).year - EPOCH_TIMESTAMP.year + return days_to_years(nanos // 86_400_000_000_000) def nanos_to_months(nanos: int) -> int: - dt = nanos_to_timestamp(nanos) - return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month) + return days_to_months(nanos // 86_400_000_000_000) def nanos_to_days(nanos: int) -> int: diff --git a/tests/expressions/test_literals.py b/tests/expressions/test_literals.py index 7e29f8d25d..4b18daa868 100644 --- a/tests/expressions/test_literals.py +++ b/tests/expressions/test_literals.py @@ -17,6 +17,7 @@ # pylint:disable=eval-used import datetime +import math import uuid from decimal import Decimal from typing import ( @@ -204,6 +205,11 @@ def test_long_to_float_outside_bound() -> None: assert below_min_lit == FloatBelowMin() +def test_double_to_float_infinity() -> None: + assert literal(math.inf).to(FloatType()) == FloatLiteral(math.inf) + assert literal(-math.inf).to(FloatType()) == FloatLiteral(-math.inf) + + def test_long_to_double_conversion() -> None: lit = literal(34).to(LongType()) dbl_lit = lit.to(DoubleType()) diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 4d0c2c1346..5d787fe5c6 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -16,6 +16,8 @@ # under the License. # pylint:disable=redefined-outer-name +from datetime import date + import pytest from pyiceberg.expressions import ( @@ -43,6 +45,7 @@ HourTransform, IdentityTransform, TruncateTransform, + YearTransform, ) from pyiceberg.types import ( DateType, @@ -237,6 +240,18 @@ def test_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: assert expected[index] == expr, predicate +def test_year_projection_at_date_limits(schema: Schema) -> None: + year_spec = PartitionSpec(PartitionField(3, 1000, YearTransform(), "event_year")) + max_date = date(9999, 12, 31) + min_date = date(1, 1, 1) + + project = inclusive_projection(schema, year_spec) + assert project(LessThanOrEqual("event_date", max_date)) == LessThanOrEqual("event_year", 8029) + assert project(GreaterThan("event_date", max_date)) == GreaterThanOrEqual("event_year", 8030) + assert project(LessThan("event_date", min_date)) == LessThanOrEqual("event_year", -1970) + assert project(GreaterThanOrEqual("event_date", min_date)) == GreaterThanOrEqual("event_year", -1969) + + def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: predicates = [ NotNull("event_date"), diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..7f72b3da2d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -17,6 +17,7 @@ # pylint: disable=protected-access,unused-argument,redefined-outer-name import json import logging +import math import os import struct import sys @@ -70,8 +71,11 @@ BoundStartsWith, EqualTo, GreaterThan, + In, IsNull, + LessThanOrEqual, Not, + NotEqualTo, Or, ) from pyiceberg.expressions.literals import literal @@ -115,7 +119,7 @@ from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID from pyiceberg.table.name_mapping import create_mapping_from_schema from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile -from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform +from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform, YearTransform from pyiceberg.typedef import UTF8, Properties, Record, TableVersion from pyiceberg.types import ( BinaryType, @@ -3713,6 +3717,29 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_scan_filters_on_infinity(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="x", field_type=FloatType(), required=False)) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_infinity", schema=schema) + table.append(pa.table({"x": pa.array([1.5, math.inf, -math.inf], type=pa.float32())})) + + assert table.scan(row_filter=EqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [math.inf] + assert table.scan(row_filter=NotEqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [1.5, -math.inf] + assert sorted(table.scan(row_filter=In("x", {math.inf, 1.5})).to_arrow()["x"].to_pylist()) == [1.5, math.inf] + + +def test_scan_filters_at_the_end_of_the_date_range(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="x", field_type=DateType(), required=False)) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=YearTransform(), name="x_year")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_date_range", schema=schema, partition_spec=spec) + last_day = date(9999, 12, 31) + table.append(pa.table({"x": pa.array([date(2024, 5, 1), last_day], type=pa.date32())})) + + assert table.scan(row_filter=LessThanOrEqual("x", last_day)).count() == 2 + assert table.scan(row_filter=GreaterThan("x", last_day)).count() == 0 + + def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="foo", field_type=StringType(), required=True), diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index 6cbbf9c1c4..6258772c75 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -24,10 +24,16 @@ datetime_to_millis, datetime_to_nanos, days_to_date, + days_to_months, + days_to_years, micros_to_days, + micros_to_months, + micros_to_years, millis_to_datetime, nanos_to_hours, nanos_to_micros, + nanos_to_months, + nanos_to_years, time_str_to_nanos, time_to_nanos, timestamp_to_micros, @@ -290,3 +296,29 @@ def test_to_human_timestamptz() -> None: (1_740_342_104_375_612, "2025-02-23T20:21:44.375612+00:00"), ]: assert to_human_timestamptz(micros) == expected + + +@pytest.mark.parametrize( + "days, years, months", + [ + (0, 0, 0), + (-1, -1, -1), + (2932896, 8029, 96359), + (2932897, 8030, 96360), + (-719162, -1969, -23628), + (-719163, -1970, -23629), + ], +) +def test_days_to_years_and_months(days: int, years: int, months: int) -> None: + assert days_to_years(days) == years + assert days_to_months(days) == months + + +def test_years_and_months_beyond_the_date_range() -> None: + day_micros = 86_400_000_000 + day_nanos = day_micros * 1000 + + assert micros_to_years(2932897 * day_micros) == 8030 + assert micros_to_months(2932897 * day_micros) == 96360 + assert nanos_to_years(2932897 * day_nanos) == 8030 + assert nanos_to_months(2932897 * day_nanos) == 96360 From 16903cf24bbda381481e69dc775f977a5c9a8203 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 16:25:28 +0530 Subject: [PATCH 2/4] fix(datetime): floor-divide the era in the civil date conversion The C++ algorithm truncates toward zero and compensates with z - 146096, but Python's // already floors. The correction made the era one too small for days before 0000-03-01, so year and month partitions before year 1 projected to the wrong partition. --- pyiceberg/utils/datetime.py | 2 +- tests/utils/test_datetime.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyiceberg/utils/datetime.py b/pyiceberg/utils/datetime.py index fdefc534ce..b187d8cac5 100644 --- a/pyiceberg/utils/datetime.py +++ b/pyiceberg/utils/datetime.py @@ -237,7 +237,7 @@ def _civil_from_days(days: int) -> tuple[int, int]: https://howardhinnant.github.io/date_algorithms.html#civil_from_days. """ z = days + 719468 - era = (z if z >= 0 else z - 146096) // 146097 + era = z // 146097 doe = z - era * 146097 yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365 doy = doe - (365 * yoe + yoe // 4 - yoe // 100) diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index 6258772c75..db01c8176c 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -307,6 +307,7 @@ def test_to_human_timestamptz() -> None: (2932897, 8030, 96360), (-719162, -1969, -23628), (-719163, -1970, -23629), + (-719529, -1971, -23641), ], ) def test_days_to_years_and_months(days: int, years: int, months: int) -> None: From 60759e1d705a358160a77df2656111397d582af9 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 16:25:34 +0530 Subject: [PATCH 3/4] test(datetime): drop nanos asserts beyond the timestamp_ns range A timestamp_ns cannot hold 2932897 days, so the asserts covered an input no table can contain. --- tests/utils/test_datetime.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/utils/test_datetime.py b/tests/utils/test_datetime.py index db01c8176c..01b6a330bd 100644 --- a/tests/utils/test_datetime.py +++ b/tests/utils/test_datetime.py @@ -32,8 +32,6 @@ millis_to_datetime, nanos_to_hours, nanos_to_micros, - nanos_to_months, - nanos_to_years, time_str_to_nanos, time_to_nanos, timestamp_to_micros, @@ -317,9 +315,6 @@ def test_days_to_years_and_months(days: int, years: int, months: int) -> None: def test_years_and_months_beyond_the_date_range() -> None: day_micros = 86_400_000_000 - day_nanos = day_micros * 1000 assert micros_to_years(2932897 * day_micros) == 8030 assert micros_to_months(2932897 * day_micros) == 96360 - assert nanos_to_years(2932897 * day_nanos) == 8030 - assert nanos_to_months(2932897 * day_nanos) == 96360 From eb30c12726ff73f4d99ffe6cfcbe386e8e3567e5 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 16:25:35 +0530 Subject: [PATCH 4/4] test(io): cover infinity range filters and deletes A double infinity on a float column now binds as a float literal, so <, <=, > and >= match rows instead of binding to always true or always false. --- tests/io/test_pyarrow.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 7f72b3da2d..20815436f8 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -71,8 +71,10 @@ BoundStartsWith, EqualTo, GreaterThan, + GreaterThanOrEqual, In, IsNull, + LessThan, LessThanOrEqual, Not, NotEqualTo, @@ -3728,6 +3730,32 @@ def test_scan_filters_on_infinity(catalog: InMemoryCatalog) -> None: assert sorted(table.scan(row_filter=In("x", {math.inf, 1.5})).to_arrow()["x"].to_pylist()) == [1.5, math.inf] +def test_scan_and_delete_filters_on_infinity(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=IntegerType(), required=False), + NestedField(id=2, name="x", field_type=FloatType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_and_delete_infinity", schema=schema) + table.append( + pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int32()), + "x": pa.array([1.5, math.inf, -math.inf, math.nan, None], type=pa.float32()), + } + ) + ) + + assert table.scan(row_filter=LessThan("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 3] + assert table.scan(row_filter=LessThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 2, 3] + assert table.scan(row_filter=GreaterThan("x", -math.inf)).to_arrow()["id"].to_pylist() == [1, 2] + assert table.scan(row_filter=GreaterThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [2] + + table.delete(LessThan("x", math.inf)) + + assert table.scan().to_arrow()["id"].to_pylist() == [2, 4, 5] + + def test_scan_filters_at_the_end_of_the_date_range(catalog: InMemoryCatalog) -> None: schema = Schema(NestedField(id=1, name="x", field_type=DateType(), required=False)) spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=YearTransform(), name="x_year"))