diff --git a/pyiceberg/expressions/visitors.py b/pyiceberg/expressions/visitors.py index 7504548507..622bf4e88c 100644 --- a/pyiceberg/expressions/visitors.py +++ b/pyiceberg/expressions/visitors.py @@ -1974,13 +1974,15 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression: if isinstance(self._eval_projection(part.transform.strict_project(part.name, predicate)), AlwaysTrue): return AlwaysTrue() - if isinstance(self._eval_projection(part.transform.project(part.name, predicate)), AlwaysFalse): + inclusive_projection = part.transform.project(part.name, predicate) + if isinstance(self._eval_projection(inclusive_projection), AlwaysFalse): return AlwaysFalse() return predicate - def _eval_projection(self, projection: UnboundPredicate | None) -> BooleanExpression | None: - if projection is None: + def _eval_projection(self, projection: BooleanExpression | None) -> BooleanExpression | None: + if not isinstance(projection, UnboundPredicate): + # only a single predicate can be evaluated against the partition value return None bound = projection.bind(self.partition_schema, case_sensitive=self.case_sensitive) diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py index 45c8ffd196..5fc759c2a7 100644 --- a/pyiceberg/transforms.py +++ b/pyiceberg/transforms.py @@ -30,6 +30,7 @@ from pydantic import Field, PositiveInt, PrivateAttr from pyiceberg.expressions import ( + BooleanExpression, BoundEqualTo, BoundGreaterThan, BoundGreaterThanOrEqual, @@ -52,6 +53,7 @@ LessThanOrEqual, NotEqualTo, NotStartsWith, + Or, Reference, StartsWith, UnboundPredicate, @@ -173,7 +175,7 @@ def result_type(self, source: IcebergType) -> IcebergType: ... @abstractmethod - def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ... + def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None: ... @abstractmethod def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ... @@ -788,7 +790,7 @@ def preserves_order(self) -> bool: def source_type(self) -> IcebergType: return self._source_type - def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: + def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None: field_type = pred.term.ref().field.field_type if isinstance(pred.term, BoundTransform): @@ -800,6 +802,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: return _set_apply_transform(name, pred, self.transform(field_type)) elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore if isinstance(pred, BoundLiteralPredicate): + if isinstance(field_type, (IntegerType, LongType)): + return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width) return _truncate_number(name, pred, self.transform(field_type)) elif isinstance(field_type, (BinaryType, StringType)): if isinstance(pred, BoundLiteralPredicate): @@ -825,6 +829,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | if isinstance(field_type, (IntegerType, LongType, DecimalType)): if isinstance(pred, BoundLiteralPredicate): + if isinstance(field_type, (IntegerType, LongType)): + return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width) return _truncate_number_strict(name, pred, self.transform(field_type)) elif isinstance(pred, BoundNotIn): return _set_apply_transform(name, pred, self.transform(field_type)) @@ -864,7 +870,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: if isinstance(source, (IntegerType, LongType)): def truncate_func(v: Any) -> Any: - return v - v % self._width + return _wrap_integer(v - v % self._width, source) elif isinstance(source, (StringType, BinaryType)): @@ -1030,6 +1036,95 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr return lambda arr: pa.nulls(len(arr), type=arr.type) +def _integer_bounds(source: IcebergType) -> tuple[int, int]: + """Return the minimum value and the size of the range of an integer type.""" + if isinstance(source, IntegerType): + return -(2**31), 2**32 + return -(2**63), 2**64 + + +def _wrap_integer(value: int, source: IcebergType) -> int: + """Wrap an integer into the range of the source type, like Java's two's complement arithmetic.""" + minimum, modulus = _integer_bounds(source) + return (value - minimum) % modulus + minimum + + +def _truncate_integer_projection( + name: str, + pred: BoundLiteralPredicate, + transform: Callable[[Any | None], Any | None], + source: IcebergType, + width: int, +) -> BooleanExpression | None: + """Project a predicate on a truncate partition of an integer column. + + The values from the minimum of the source type truncate to a partition value at the top of the + type's range, which does not sort with the other partition values. A range that can match those + values therefore also matches that partition. + """ + minimum, _ = _integer_bounds(source) + floor_mod = minimum % width + if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): + return _truncate_number(name, pred, transform) + + wrapped_minimum = _wrap_integer(minimum - floor_mod, source) + # The first partition value after the block that holds the minimum of the source type + next_block_start = minimum - floor_mod + width + boundary = pred.literal.value + + if isinstance(pred, BoundGreaterThan): + if boundary + 1 < next_block_start: + # The wrapped value sorts above the first block after the minimum, so it is included + return GreaterThanOrEqual(Reference(name), next_block_start) + return _truncate_number(name, pred, transform) + elif isinstance(pred, BoundGreaterThanOrEqual): + if boundary < next_block_start: + return GreaterThanOrEqual(Reference(name), next_block_start) + return _truncate_number(name, pred, transform) + + projected = _truncate_number(name, pred, transform) + if isinstance(pred, BoundLessThan) and projected is not None: + if boundary <= next_block_start: + return EqualTo(Reference(name), wrapped_minimum) + return Or(projected, EqualTo(Reference(name), wrapped_minimum)) + elif isinstance(pred, BoundLessThanOrEqual) and projected is not None: + if boundary < next_block_start: + return EqualTo(Reference(name), wrapped_minimum) + return Or(projected, EqualTo(Reference(name), wrapped_minimum)) + return projected + + +def _truncate_integer_strict( + name: str, + pred: BoundLiteralPredicate, + transform: Callable[[Any | None], Any | None], + source: IcebergType, + width: int, +) -> UnboundPredicate | None: + """Project a predicate strictly on a truncate partition of an integer column. + + Values from the minimum of the source type truncate to a partition value at the top of the + type's range, which does not sort with the other partition values. No single predicate on the + partition value can account for that, so those ranges have no strict projection. + """ + minimum, _ = _integer_bounds(source) + floor_mod = minimum % width + if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): + return _truncate_number_strict(name, pred, transform) + + first_block_end = minimum - floor_mod + width + if isinstance(pred, BoundLessThan): + if pred.literal.value < first_block_end: + return None + elif isinstance(pred, BoundLessThanOrEqual): + if pred.literal.increment().value < first_block_end: + return None + elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): + return None + + return _truncate_number_strict(name, pred, transform) + + def _truncate_number( name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None] ) -> UnboundPredicate | None: diff --git a/tests/expressions/test_projection.py b/tests/expressions/test_projection.py index 4d0c2c1346..e0f244a386 100644 --- a/tests/expressions/test_projection.py +++ b/tests/expressions/test_projection.py @@ -46,6 +46,7 @@ ) from pyiceberg.types import ( DateType, + IntegerType, LongType, NestedField, StringType, @@ -270,6 +271,32 @@ def test_date_day_projection(schema: Schema, day_spec: PartitionSpec) -> None: assert expected[index] == expr, predicate +def test_int_truncate_projection_near_minimum(schema: Schema) -> None: + spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc")) + project = inclusive_projection(schema, spec) + wrapped_minimum = 9223372036854775806 + + assert project(EqualTo("id", -(2**63))) == EqualTo("id_trunc", wrapped_minimum) + assert project(LessThan("id", -(2**63) + 5)) == EqualTo("id_trunc", wrapped_minimum) + assert project(LessThanOrEqual("id", -(2**63) + 5)) == EqualTo("id_trunc", wrapped_minimum) + assert project(LessThan("id", 0)) == Or(LessThanOrEqual("id_trunc", -10), EqualTo("id_trunc", wrapped_minimum)) + assert project(GreaterThan("id", -(2**63))) == GreaterThanOrEqual("id_trunc", -9223372036854775800) + assert project(GreaterThanOrEqual("id", -(2**63) + 1)) == GreaterThanOrEqual("id_trunc", -9223372036854775800) + + +def test_integer_truncate_projection_near_minimum() -> None: + schema = Schema(NestedField(1, "x", IntegerType(), required=False)) + spec = PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "x_trunc")) + project = inclusive_projection(schema, spec) + wrapped_minimum = 2147483646 + + assert project(EqualTo("x", -(2**31))) == EqualTo("x_trunc", wrapped_minimum) + assert project(LessThan("x", -(2**31) + 5)) == EqualTo("x_trunc", wrapped_minimum) + assert project(LessThanOrEqual("x", -(2**31) + 5)) == EqualTo("x_trunc", wrapped_minimum) + assert project(GreaterThan("x", -(2**31))) == GreaterThanOrEqual("x_trunc", -2147483640) + assert project(GreaterThanOrEqual("x", -(2**31) + 1)) == GreaterThanOrEqual("x_trunc", -2147483640) + + def test_string_truncate_projection(schema: Schema, truncate_str_spec: PartitionSpec) -> None: predicates = [ NotNull("data"), @@ -320,8 +347,8 @@ def test_int_truncate_projection(schema: Schema, truncate_int_spec: PartitionSpe expected = [ NotNull("id_trunc"), IsNull("id_trunc"), - LessThanOrEqual("id_trunc", 0), - LessThanOrEqual("id_trunc", 10), + Or(LessThanOrEqual("id_trunc", 0), EqualTo("id_trunc", 9223372036854775806)), + Or(LessThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 9223372036854775806)), GreaterThanOrEqual("id_trunc", 10), GreaterThanOrEqual("id_trunc", 10), EqualTo("id_trunc", 10), diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..cdac621e35 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -71,6 +71,7 @@ EqualTo, GreaterThan, IsNull, + LessThan, Not, Or, ) @@ -3663,6 +3664,49 @@ def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None: assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter +def test_scan_filters_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_truncate_minimum", schema=schema, partition_spec=spec) + table.append( + pa.Table.from_pylist( + [{"id": 0, "x": -(2**31)}, {"id": 1, "x": -(2**31) + 100}, {"id": 2, "x": 5}], schema=schema.as_arrow() + ) + ) + + assert table.scan(row_filter=EqualTo("x", -(2**31))).to_arrow()["id"].to_pylist() == [0] + assert sorted(table.scan(row_filter=LessThan("x", 0)).to_arrow()["id"].to_pylist()) == [0, 1] + assert sorted(table.scan(row_filter=GreaterThan("x", -(2**31))).to_arrow()["id"].to_pylist()) == [1, 2] + + +def test_count_truncate_partitions_near_the_minimum(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="id", field_type=LongType(), required=True), + NestedField(id=2, name="x", field_type=IntegerType(), required=False), + ) + spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=TruncateTransform(1000), name="x_trunc")) + catalog.create_namespace("default") + table = catalog.create_table( + "default.test_count_truncate_minimum", + schema=schema, + partition_spec=spec, + properties={"write.metadata.metrics.default": "counts"}, + ) + table.append(pa.Table.from_pylist([{"id": 0, "x": -(2**31)}, {"id": 1, "x": 5}], schema=schema.as_arrow())) + + less_than = table.scan(row_filter=LessThan("x", -(2**31) + 100)) + assert less_than.to_arrow()["id"].to_pylist() == [0] + assert less_than.count() == 1 + + greater_than = table.scan(row_filter=GreaterThan("x", 0)) + assert greater_than.to_arrow()["id"].to_pylist() == [1] + assert greater_than.count() == 1 + + def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="id", field_type=LongType(), required=False), diff --git a/tests/test_transforms.py b/tests/test_transforms.py index 6ab7b52c4c..e8129ad882 100644 --- a/tests/test_transforms.py +++ b/tests/test_transforms.py @@ -465,6 +465,11 @@ def test_truncate_integer(type_var: PrimitiveType, input_var: int, expected: int assert trunc.transform(type_var)(input_var) == expected +def test_truncate_integer_minimum_wraps() -> None: + assert TruncateTransform(10).transform(IntegerType())(-(2**31)) == 2147483646 + assert TruncateTransform(10).transform(LongType())(-(2**63)) == 9223372036854775806 + + @pytest.mark.parametrize( "input_var,expected", [ @@ -1504,8 +1509,8 @@ def test_truncate_strict_integer_lower_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1522,8 +1527,8 @@ def test_truncate_strict_integer_upper_bound(bound_reference_int: BoundReference _assert_projection_strict(BoundLessThan(term=bound_reference_int, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_int, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_int, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_int, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_int, literal=value), transform, AlwaysFalse) @@ -1538,8 +1543,8 @@ def test_truncate_strict_long_lower_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "100") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "100") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "100") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1556,8 +1561,8 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundLessThan(term=bound_reference_long, literal=value), transform, LessThan, "90") _assert_projection_strict(BoundLessThanOrEqual(term=bound_reference_long, literal=value), transform, LessThan, "100") - _assert_projection_strict(BoundGreaterThan(term=bound_reference_long, literal=value), transform, GreaterThan, "90") - _assert_projection_strict(BoundGreaterThanOrEqual(term=bound_reference_long, literal=value), transform, GreaterThan, "90") + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=value)) is None + assert transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=value)) is None _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, NotIn, "90") _assert_projection_strict(BoundNotEqualTo(term=bound_reference_long, literal=value), transform, AlwaysFalse) value_dec = value.decrement() @@ -1568,6 +1573,50 @@ def test_truncate_strict_long_upper_bound(bound_reference_long: BoundReference) _assert_projection_strict(BoundIn(term=bound_reference_long, literals={value_dec, value, value_inc}), transform, NotIn) +def test_truncate_strict_integer_minimum(bound_reference_int: BoundReference) -> None: + minimum = -(2**31) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_int, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_int, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_int, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_int, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_int, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + +def test_truncate_strict_long_minimum(bound_reference_long: BoundReference) -> None: + minimum = -(2**63) + first_block_end = minimum - minimum % 1000 + 1000 + transform = TruncateTransform(1000) + + assert transform.strict_project(name="name", pred=BoundLessThan(term=bound_reference_long, literal=literal(minimum))) is None + assert ( + transform.strict_project(name="name", pred=BoundLessThanOrEqual(term=bound_reference_long, literal=literal(minimum + 1))) + is None + ) + assert transform.strict_project(name="name", pred=BoundGreaterThan(term=bound_reference_long, literal=literal(0))) is None + assert ( + transform.strict_project(name="name", pred=BoundGreaterThanOrEqual(term=bound_reference_long, literal=literal(0))) is None + ) + _assert_projection_strict( + BoundLessThan(term=bound_reference_long, literal=literal(first_block_end)), + transform, + LessThan, + str(first_block_end), + ) + + def test_truncate_strict_decimal_lower_bound(bound_reference_decimal: BoundReference) -> None: dec = DecimalType(9, 2) value = literal("100.00").to(dec)