Skip to content
Open
8 changes: 5 additions & 3 deletions pyiceberg/expressions/visitors.py
Original file line number Diff line number Diff line change
Expand Up @@ -1974,13 +1974,15 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
if isinstance(self._eval_projection(part.transform.strict_project(part.name, predicate)), AlwaysTrue):
return AlwaysTrue()

if isinstance(self._eval_projection(part.transform.project(part.name, predicate)), AlwaysFalse):
inclusive_projection = part.transform.project(part.name, predicate)
if isinstance(self._eval_projection(inclusive_projection), AlwaysFalse):
return AlwaysFalse()

return predicate

def _eval_projection(self, projection: UnboundPredicate | None) -> BooleanExpression | None:
if projection is None:
def _eval_projection(self, projection: BooleanExpression | None) -> BooleanExpression | None:
if not isinstance(projection, UnboundPredicate):
# only a single predicate can be evaluated against the partition value
return None

bound = projection.bind(self.partition_schema, case_sensitive=self.case_sensitive)
Expand Down
101 changes: 98 additions & 3 deletions pyiceberg/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from pydantic import Field, PositiveInt, PrivateAttr

from pyiceberg.expressions import (
BooleanExpression,
BoundEqualTo,
BoundGreaterThan,
BoundGreaterThanOrEqual,
Expand All @@ -52,6 +53,7 @@
LessThanOrEqual,
NotEqualTo,
NotStartsWith,
Or,
Reference,
StartsWith,
UnboundPredicate,
Expand Down Expand Up @@ -173,7 +175,7 @@ def result_type(self, source: IcebergType) -> IcebergType:
...

@abstractmethod
def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ...
def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None: ...

@abstractmethod
def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ...
Expand Down Expand Up @@ -788,7 +790,7 @@ def preserves_order(self) -> bool:
def source_type(self) -> IcebergType:
return self._source_type

def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None:
def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None:
field_type = pred.term.ref().field.field_type

if isinstance(pred.term, BoundTransform):
Expand All @@ -800,6 +802,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None:
return _set_apply_transform(name, pred, self.transform(field_type))
elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore
if isinstance(pred, BoundLiteralPredicate):
if isinstance(field_type, (IntegerType, LongType)):
return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width)
return _truncate_number(name, pred, self.transform(field_type))
elif isinstance(field_type, (BinaryType, StringType)):
if isinstance(pred, BoundLiteralPredicate):
Expand All @@ -825,6 +829,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate |

if isinstance(field_type, (IntegerType, LongType, DecimalType)):
if isinstance(pred, BoundLiteralPredicate):
if isinstance(field_type, (IntegerType, LongType)):
return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width)
return _truncate_number_strict(name, pred, self.transform(field_type))
elif isinstance(pred, BoundNotIn):
return _set_apply_transform(name, pred, self.transform(field_type))
Expand Down Expand Up @@ -864,7 +870,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]:
if isinstance(source, (IntegerType, LongType)):

def truncate_func(v: Any) -> Any:
return v - v % self._width
return _wrap_integer(v - v % self._width, source)

elif isinstance(source, (StringType, BinaryType)):

Expand Down Expand Up @@ -1030,6 +1036,95 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr
return lambda arr: pa.nulls(len(arr), type=arr.type)


def _integer_bounds(source: IcebergType) -> tuple[int, int]:
"""Return the minimum value and the size of the range of an integer type."""
if isinstance(source, IntegerType):
return -(2**31), 2**32
return -(2**63), 2**64


def _wrap_integer(value: int, source: IcebergType) -> int:
"""Wrap an integer into the range of the source type, like Java's two's complement arithmetic."""
minimum, modulus = _integer_bounds(source)
return (value - minimum) % modulus + minimum


def _truncate_integer_projection(
name: str,
pred: BoundLiteralPredicate,
transform: Callable[[Any | None], Any | None],
source: IcebergType,
width: int,
) -> BooleanExpression | None:
"""Project a predicate on a truncate partition of an integer column.

The values from the minimum of the source type truncate to a partition value at the top of the
type's range, which does not sort with the other partition values. A range that can match those
values therefore also matches that partition.
"""
minimum, _ = _integer_bounds(source)
floor_mod = minimum % width
if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
return _truncate_number(name, pred, transform)

wrapped_minimum = _wrap_integer(minimum - floor_mod, source)
# The first partition value after the block that holds the minimum of the source type
next_block_start = minimum - floor_mod + width
boundary = pred.literal.value

if isinstance(pred, BoundGreaterThan):
if boundary + 1 < next_block_start:
# The wrapped value sorts above the first block after the minimum, so it is included
return GreaterThanOrEqual(Reference(name), next_block_start)
return _truncate_number(name, pred, transform)
elif isinstance(pred, BoundGreaterThanOrEqual):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No test covers this branch: with if boundary < next_block_start: replaced by if False:, every test still passes, although x >= MIN then skips all ordinary partitions. An assert next to the GreaterThan one in test_int_truncate_projection_near_minimum would cover it:

assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == Or(
    GreaterThanOrEqual("id_trunc", -9223372036854775800), EqualTo("id_trunc", wrapped_minimum)
)

That test projects the LongType id column, so an IntegerType case would help too; only the scan test covers int.

if boundary < next_block_start:
return GreaterThanOrEqual(Reference(name), next_block_start)
return _truncate_number(name, pred, transform)

projected = _truncate_number(name, pred, transform)
if isinstance(pred, BoundLessThan) and projected is not None:
if boundary <= next_block_start:
return EqualTo(Reference(name), wrapped_minimum)
return Or(projected, EqualTo(Reference(name), wrapped_minimum))
elif isinstance(pred, BoundLessThanOrEqual) and projected is not None:
if boundary < next_block_start:
return EqualTo(Reference(name), wrapped_minimum)
return Or(projected, EqualTo(Reference(name), wrapped_minimum))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: when the boundary falls in the block that holds the minimum (for example x < -2**31 + 100 with truncate[1000]), _truncate_number projects to LessThanOrEqual(name, wrapped_minimum), which keeps almost every partition. Only the wrapped partition can match there, so this could return EqualTo(Reference(name), wrapped_minimum) alone. That is the case test_count_truncate_partitions_near_the_minimum exercises.

return projected


def _truncate_integer_strict(
name: str,
pred: BoundLiteralPredicate,
transform: Callable[[Any | None], Any | None],
source: IcebergType,
width: int,
) -> UnboundPredicate | None:
"""Project a predicate strictly on a truncate partition of an integer column.

Values from the minimum of the source type truncate to a partition value at the top of the
type's range, which does not sort with the other partition values. No single predicate on the
partition value can account for that, so those ranges have no strict projection.
"""
minimum, _ = _integer_bounds(source)
floor_mod = minimum % width
if floor_mod == 0 or not isinstance(pred.literal, LongLiteral):
return _truncate_number_strict(name, pred, transform)

first_block_end = minimum - floor_mod + width
if isinstance(pred, BoundLessThan):
if pred.literal.value < first_block_end:
return None
elif isinstance(pred, BoundLessThanOrEqual):
if pred.literal.increment().value < first_block_end:
return None
elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)):
return None

return _truncate_number_strict(name, pred, transform)


def _truncate_number(
name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None]
) -> UnboundPredicate | None:
Expand Down
31 changes: 29 additions & 2 deletions tests/expressions/test_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
)
from pyiceberg.types import (
DateType,
IntegerType,
LongType,
NestedField,
StringType,
Expand Down Expand Up @@ -270,6 +271,32 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None:
assert expected[index] == expr, predicate


def test_int_truncate_projection_near_minimum(schema: Schema) -> None:
spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc"))
project = inclusive_projection(schema, spec)
wrapped_minimum = 9223372036854775806

assert project(EqualTo("id", -(2**63))) == EqualTo("id_trunc", wrapped_minimum)
assert project(LessThan("id", -(2**63) + 5)) == EqualTo("id_trunc", wrapped_minimum)
assert project(LessThanOrEqual("id", -(2**63) + 5)) == EqualTo("id_trunc", wrapped_minimum)
assert project(LessThan("id", 0)) == Or(LessThanOrEqual("id_trunc", -10), EqualTo("id_trunc", wrapped_minimum))
assert project(GreaterThan("id", -(2**63))) == GreaterThanOrEqual("id_trunc", -9223372036854775800)
assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == GreaterThanOrEqual("id_trunc", -9223372036854775800)


def test_integer_truncate_projection_near_minimum() -> None:
schema = Schema(NestedField(1, "x", IntegerType(), required=False))
spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "x_trunc"))
project = inclusive_projection(schema, spec)
wrapped_minimum = 2147483646

assert project(EqualTo("x", -(2**31))) == EqualTo("x_trunc", wrapped_minimum)
assert project(LessThan("x", -(2**31) + 5)) == EqualTo("x_trunc", wrapped_minimum)
assert project(LessThanOrEqual("x", -(2**31) + 5)) == EqualTo("x_trunc", wrapped_minimum)
assert project(GreaterThan("x", -(2**31))) == GreaterThanOrEqual("x_trunc", -2147483640)
assert project(GreaterThanOrEqual("x", -(2**31) + 1)) == GreaterThanOrEqual("x_trunc", -2147483640)


def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None:
predicates = [
NotNull("data"),
Expand Down Expand Up @@ -320,8 +347,8 @@ def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpe
expected = [
NotNull("id_trunc"),
IsNull("id_trunc"),
LessThanOrEqual("id_trunc", 0),
LessThanOrEqual("id_trunc", 10),
Or(LessThanOrEqual("id_trunc", 0), EqualTo("id_trunc", 9223372036854775806)),
Or(LessThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 9223372036854775806)),
GreaterThanOrEqual("id_trunc", 10),
GreaterThanOrEqual("id_trunc", 10),
EqualTo("id_trunc", 10),
Expand Down
44 changes: 44 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
EqualTo,
GreaterThan,
IsNull,
LessThan,
Not,
Or,
)
Expand Down Expand Up @@ -3663,6 +3664,49 @@ def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None:
assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter


def test_scan_filters_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=True),
NestedField(id=2, name="x", field_type=IntegerType(), required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc"))
catalog.create_namespace("default")
table = catalog.create_table("default.test_scan_truncate_minimum", schema=schema, partition_spec=spec)
table.append(
pa.Table.from_pylist(
[{"id": 0, "x": -(2**31)}, {"id": 1, "x": -(2**31) + 100}, {"id": 2, "x": 5}], schema=schema.as_arrow()
)
)

assert table.scan(row_filter=EqualTo("x", -(2**31))).to_arrow()["id"].to_pylist() == [0]
assert sorted(table.scan(row_filter=LessThan("x", 0)).to_arrow()["id"].to_pylist()) == [0, 1]
assert sorted(table.scan(row_filter=GreaterThan("x", -(2**31))).to_arrow()["id"].to_pylist()) == [1, 2]


def test_count_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=True),
NestedField(id=2, name="x", field_type=IntegerType(), required=False),
)
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc"))
catalog.create_namespace("default")
table = catalog.create_table(
"default.test_count_truncate_minimum",
schema=schema,
partition_spec=spec,
properties={"write.metadata.metrics.default": "counts"},
)
table.append(pa.Table.from_pylist([{"id": 0, "x": -(2**31)}, {"id": 1, "x": 5}], schema=schema.as_arrow()))

less_than = table.scan(row_filter=LessThan("x", -(2**31) + 100))
assert less_than.to_arrow()["id"].to_pylist() == [0]
assert less_than.count() == 1

greater_than = table.scan(row_filter=GreaterThan("x", 0))
assert greater_than.to_arrow()["id"].to_pylist() == [1]
assert greater_than.count() == 1


def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None:
schema = Schema(
NestedField(id=1, name="id", field_type=LongType(), required=False),
Expand Down
Loading
Loading