Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pyiceberg/expressions/literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
from datetime import date, datetime, time
from decimal import ROUND_HALF_UP, Decimal
from functools import singledispatchmethod
from math import isfinite, isnan
from math import isfinite, isinf, isnan
from typing import Any, Generic
from uuid import UUID

Expand Down Expand Up @@ -433,6 +433,8 @@ def _(self, _: DoubleType) -> Literal[float]:

@to.register(FloatType)
def _(self, _: FloatType) -> Literal[float]:
if isinf(self.value):
return FloatLiteral(self.value)
if FloatType.max < self.value:
return FloatAboveMax()
elif FloatType.min > self.value:
Expand Down
34 changes: 25 additions & 9 deletions pyiceberg/utils/datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,22 +229,39 @@ def micros_to_hours(micros: int) -> int:
return micros // 3_600_000_000


def _civil_from_days(days: int) -> tuple[int, int]:
"""Return the year and month of the number of days from 1970-01-01.

The conversion uses integer arithmetic, so values outside the range of ``datetime.date``, such
as the day after 9999-12-31, are converted as well. Based on
https://howardhinnant.github.io/date_algorithms.html#civil_from_days.
"""
z = days + 719468
era = z // 146097
doe = z - era * 146097
yoe = (doe - doe // 1460 + doe // 36524 - doe // 146096) // 365
doy = doe - (365 * yoe + yoe // 4 - yoe // 100)
mp = (5 * doy + 2) // 153
month = mp + 3 if mp < 10 else mp - 9
year = yoe + era * 400 + (1 if month <= 2 else 0)
return year, month


def days_to_months(days: int) -> int:
d = days_to_date(days)
return (d.year - EPOCH_DATE.year) * 12 + (d.month - EPOCH_DATE.month)
year, month = _civil_from_days(days)
return (year - EPOCH_DATE.year) * 12 + (month - EPOCH_DATE.month)


def micros_to_months(micros: int) -> int:
dt = micros_to_timestamp(micros)
return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month)
return days_to_months(micros // 86_400_000_000)


def days_to_years(days: int) -> int:
return days_to_date(days).year - EPOCH_DATE.year
return _civil_from_days(days)[0] - EPOCH_DATE.year


def micros_to_years(micros: int) -> int:
return micros_to_timestamp(micros).year - EPOCH_TIMESTAMP.year
return days_to_years(micros // 86_400_000_000)


def nanos_to_timestamp(nanos: int) -> datetime:
Expand All @@ -254,12 +271,11 @@ def nanos_to_timestamp(nanos: int) -> datetime:


def nanos_to_years(nanos: int) -> int:
return nanos_to_timestamp(nanos).year - EPOCH_TIMESTAMP.year
return days_to_years(nanos // 86_400_000_000_000)


def nanos_to_months(nanos: int) -> int:
dt = nanos_to_timestamp(nanos)
return (dt.year - EPOCH_TIMESTAMP.year) * 12 + (dt.month - EPOCH_TIMESTAMP.month)
return days_to_months(nanos // 86_400_000_000_000)


def nanos_to_days(nanos: int) -> int:
Expand Down
6 changes: 6 additions & 0 deletions tests/expressions/test_literals.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# pylint:disable=eval-used

import datetime
import math
import uuid
from decimal import Decimal
from typing import (
Expand Down Expand Up @@ -204,6 +205,11 @@ def test_long_to_float_outside_bound() -> None:
assert below_min_lit == FloatBelowMin()


def test_double_to_float_infinity() -> None:
assert literal(math.inf).to(FloatType()) == FloatLiteral(math.inf)
assert literal(-math.inf).to(FloatType()) == FloatLiteral(-math.inf)


def test_long_to_double_conversion() -> None:
lit = literal(34).to(LongType())
dbl_lit = lit.to(DoubleType())
Expand Down
15 changes: 15 additions & 0 deletions tests/expressions/test_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
# under the License.
# pylint:disable=redefined-outer-name

from datetime import date

import pytest

from pyiceberg.expressions import (
Expand Down Expand Up @@ -43,6 +45,7 @@
HourTransform,
IdentityTransform,
TruncateTransform,
YearTransform,
)
from pyiceberg.types import (
DateType,
Expand Down Expand Up @@ -237,6 +240,18 @@ def test_day_projection(schema: Schema, day_spec: PartitionSpec) -> None:
assert expected[index] == expr, predicate


def test_year_projection_at_date_limits(schema: Schema) -> None:
year_spec = PartitionSpec(PartitionField(3, 1000, YearTransform(), "event_year"))
max_date = date(9999, 12, 31)
min_date = date(1, 1, 1)

project = inclusive_projection(schema, year_spec)
assert project(LessThanOrEqual("event_date", max_date)) == LessThanOrEqual("event_year", 8029)
assert project(GreaterThan("event_date", max_date)) == GreaterThanOrEqual("event_year", 8030)
assert project(LessThan("event_date", min_date)) == LessThanOrEqual("event_year", -1970)
assert project(GreaterThanOrEqual("event_date", min_date)) == GreaterThanOrEqual("event_year", -1969)


def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None:
predicates = [
NotNull("event_date"),
Expand Down
57 changes: 56 additions & 1 deletion tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
# pylint: disable=protected-access,unused-argument,redefined-outer-name
import json
import logging
import math
import os
import struct
import sys
Expand Down Expand Up @@ -70,8 +71,13 @@
BoundStartsWith,
EqualTo,
GreaterThan,
GreaterThanOrEqual,
In,
IsNull,
LessThan,
LessThanOrEqual,
Not,
NotEqualTo,
Or,
)
from pyiceberg.expressions.literals import literal
Expand Down Expand Up @@ -115,7 +121,7 @@
from pyiceberg.table.metadata_columns import LAST_UPDATED_SEQUENCE_NUMBER, ROW_ID
from pyiceberg.table.name_mapping import create_mapping_from_schema
from pyiceberg.table.puffin import MAGIC_BYTES, PuffinFile
from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform
from pyiceberg.transforms import BucketTransform, HourTransform, IdentityTransform, Transform, TruncateTransform, YearTransform
from pyiceberg.typedef import UTF8, Properties, Record, TableVersion
from pyiceberg.types import (
BinaryType,
Expand Down Expand Up @@ -3713,6 +3719,55 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None:
assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2]


def test_scan_filters_on_infinity(catalog: InMemoryCatalog) -> None:
schema = Schema(NestedField(id=1, name="x", field_type=FloatType(), required=False))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_infinity", schema=schema)
table.append(pa.table({"x": pa.array([1.5, math.inf, -math.inf], type=pa.float32())}))

assert table.scan(row_filter=EqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [math.inf]
assert table.scan(row_filter=NotEqualTo("x", math.inf)).to_arrow()["x"].to_pylist() == [1.5, -math.inf]
assert sorted(table.scan(row_filter=In("x", {math.inf, 1.5})).to_arrow()["x"].to_pylist()) == [1.5, math.inf]


def test_scan_and_delete_filters_on_infinity(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=IntegerType(), required=False),
NestedField(id=2, name="x", field_type=FloatType(), required=False),
)
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_and_delete_infinity", schema=schema)
table.append(
pa.table(
{
"id": pa.array([1, 2, 3, 4, 5], type=pa.int32()),
"x": pa.array([1.5, math.inf, -math.inf, math.nan, None], type=pa.float32()),
}
)
)

assert table.scan(row_filter=LessThan("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 3]
assert table.scan(row_filter=LessThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [1, 2, 3]
assert table.scan(row_filter=GreaterThan("x", -math.inf)).to_arrow()["id"].to_pylist() == [1, 2]
assert table.scan(row_filter=GreaterThanOrEqual("x", math.inf)).to_arrow()["id"].to_pylist() == [2]

table.delete(LessThan("x", math.inf))

assert table.scan().to_arrow()["id"].to_pylist() == [2, 4, 5]


def test_scan_filters_at_the_end_of_the_date_range(catalog: InMemoryCatalog) -> None:
schema = Schema(NestedField(id=1, name="x", field_type=DateType(), required=False))
spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=YearTransform(), name="x_year"))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_date_range", schema=schema, partition_spec=spec)
last_day = date(9999, 12, 31)
table.append(pa.table({"x": pa.array([date(2024, 5, 1), last_day], type=pa.date32())}))

assert table.scan(row_filter=LessThanOrEqual("x", last_day)).count() == 2
assert table.scan(row_filter=GreaterThan("x", last_day)).count() == 0


def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="f", field_type=FloatType(), required=False),
Expand Down
28 changes: 28 additions & 0 deletions tests/utils/test_datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@
datetime_to_millis,
datetime_to_nanos,
days_to_date,
days_to_months,
days_to_years,
micros_to_days,
micros_to_months,
micros_to_years,
millis_to_datetime,
nanos_to_hours,
nanos_to_micros,
Expand Down Expand Up @@ -290,3 +294,27 @@ def test_to_human_timestamptz() -> None:
(1_740_342_104_375_612, "2025-02-23T20:21:44.375612+00:00"),
]:
assert to_human_timestamptz(micros) == expected


@pytest.mark.parametrize(
"days, years, months",
[
(0, 0, 0),
(-1, -1, -1),
(2932896, 8029, 96359),
(2932897, 8030, 96360),
(-719162, -1969, -23628),
(-719163, -1970, -23629),
(-719529, -1971, -23641),
],
)
def test_days_to_years_and_months(days: int, years: int, months: int) -> None:
assert days_to_years(days) == years
assert days_to_months(days) == months


def test_years_and_months_beyond_the_date_range() -> None:
day_micros = 86_400_000_000

assert micros_to_years(2932897 * day_micros) == 8030
assert micros_to_months(2932897 * day_micros) == 96360
Loading