Repository navigation
Keep rows in truncate partitions near the minimum int or long #4407
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
77612bf
af385e0
4fef874
fda11b8
53a6d3c
7f0ebc5
67e3316
3601078
472173e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,6 +30,7 @@ | |
| from pydantic import Field, PositiveInt, PrivateAttr | ||
|
|
||
| from pyiceberg.expressions import ( | ||
| BooleanExpression, | ||
| BoundEqualTo, | ||
| BoundGreaterThan, | ||
| BoundGreaterThanOrEqual, | ||
|
|
@@ -52,6 +53,7 @@ | |
| LessThanOrEqual, | ||
| NotEqualTo, | ||
| NotStartsWith, | ||
| Or, | ||
| Reference, | ||
| StartsWith, | ||
| UnboundPredicate, | ||
|
|
@@ -173,7 +175,7 @@ def result_type(self, source: IcebergType) -> IcebergType: | |
| ... | ||
|
|
||
| @abstractmethod | ||
| def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ... | ||
| def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None: ... | ||
|
|
||
| @abstractmethod | ||
| def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: ... | ||
|
|
@@ -788,7 +790,7 @@ def preserves_order(self) -> bool: | |
| def source_type(self) -> IcebergType: | ||
| return self._source_type | ||
|
|
||
| def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: | ||
| def project(self, name: str, pred: BoundPredicate) -> BooleanExpression | None: | ||
| field_type = pred.term.ref().field.field_type | ||
|
|
||
| if isinstance(pred.term, BoundTransform): | ||
|
|
@@ -800,6 +802,8 @@ def project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None: | |
| return _set_apply_transform(name, pred, self.transform(field_type)) | ||
| elif isinstance(field_type, (IntegerType, LongType, DecimalType)): # type: ignore | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
| if isinstance(field_type, (IntegerType, LongType)): | ||
| return _truncate_integer_projection(name, pred, self.transform(field_type), field_type, self._width) | ||
| return _truncate_number(name, pred, self.transform(field_type)) | ||
| elif isinstance(field_type, (BinaryType, StringType)): | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
|
|
@@ -825,6 +829,8 @@ def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | | |
|
|
||
| if isinstance(field_type, (IntegerType, LongType, DecimalType)): | ||
| if isinstance(pred, BoundLiteralPredicate): | ||
| if isinstance(field_type, (IntegerType, LongType)): | ||
| return _truncate_integer_strict(name, pred, self.transform(field_type), field_type, self._width) | ||
| return _truncate_number_strict(name, pred, self.transform(field_type)) | ||
| elif isinstance(pred, BoundNotIn): | ||
| return _set_apply_transform(name, pred, self.transform(field_type)) | ||
|
|
@@ -864,7 +870,7 @@ def transform(self, source: IcebergType) -> Callable[[S | None], S | None]: | |
| if isinstance(source, (IntegerType, LongType)): | ||
|
|
||
| def truncate_func(v: Any) -> Any: | ||
| return v - v % self._width | ||
| return _wrap_integer(v - v % self._width, source) | ||
|
|
||
| elif isinstance(source, (StringType, BinaryType)): | ||
|
|
||
|
|
@@ -1030,6 +1036,95 @@ def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Arr | |
| return lambda arr: pa.nulls(len(arr), type=arr.type) | ||
|
|
||
|
|
||
| def _integer_bounds(source: IcebergType) -> tuple[int, int]: | ||
| """Return the minimum value and the size of the range of an integer type.""" | ||
| if isinstance(source, IntegerType): | ||
| return -(2**31), 2**32 | ||
| return -(2**63), 2**64 | ||
|
|
||
|
|
||
| def _wrap_integer(value: int, source: IcebergType) -> int: | ||
| """Wrap an integer into the range of the source type, like Java's two's complement arithmetic.""" | ||
| minimum, modulus = _integer_bounds(source) | ||
| return (value - minimum) % modulus + minimum | ||
|
|
||
|
|
||
| def _truncate_integer_projection( | ||
| name: str, | ||
| pred: BoundLiteralPredicate, | ||
| transform: Callable[[Any | None], Any | None], | ||
| source: IcebergType, | ||
| width: int, | ||
| ) -> BooleanExpression | None: | ||
| """Project a predicate on a truncate partition of an integer column. | ||
|
|
||
| The values from the minimum of the source type truncate to a partition value at the top of the | ||
| type's range, which does not sort with the other partition values. A range that can match those | ||
| values therefore also matches that partition. | ||
| """ | ||
| minimum, _ = _integer_bounds(source) | ||
| floor_mod = minimum % width | ||
| if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): | ||
| return _truncate_number(name, pred, transform) | ||
|
|
||
| wrapped_minimum = _wrap_integer(minimum - floor_mod, source) | ||
| # The first partition value after the block that holds the minimum of the source type | ||
| next_block_start = minimum - floor_mod + width | ||
| boundary = pred.literal.value | ||
|
|
||
| if isinstance(pred, BoundGreaterThan): | ||
| if boundary + 1 < next_block_start: | ||
| # The wrapped value sorts above the first block after the minimum, so it is included | ||
| return GreaterThanOrEqual(Reference(name), next_block_start) | ||
| return _truncate_number(name, pred, transform) | ||
| elif isinstance(pred, BoundGreaterThanOrEqual): | ||
| if boundary < next_block_start: | ||
| return GreaterThanOrEqual(Reference(name), next_block_start) | ||
| return _truncate_number(name, pred, transform) | ||
|
|
||
| projected = _truncate_number(name, pred, transform) | ||
| if isinstance(pred, BoundLessThan) and projected is not None: | ||
| if boundary <= next_block_start: | ||
| return EqualTo(Reference(name), wrapped_minimum) | ||
| return Or(projected, EqualTo(Reference(name), wrapped_minimum)) | ||
| elif isinstance(pred, BoundLessThanOrEqual) and projected is not None: | ||
| if boundary < next_block_start: | ||
| return EqualTo(Reference(name), wrapped_minimum) | ||
| return Or(projected, EqualTo(Reference(name), wrapped_minimum)) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: when the boundary falls in the block that holds the minimum (for example |
||
| return projected | ||
|
|
||
|
|
||
| def _truncate_integer_strict( | ||
| name: str, | ||
| pred: BoundLiteralPredicate, | ||
| transform: Callable[[Any | None], Any | None], | ||
| source: IcebergType, | ||
| width: int, | ||
| ) -> UnboundPredicate | None: | ||
| """Project a predicate strictly on a truncate partition of an integer column. | ||
|
|
||
| Values from the minimum of the source type truncate to a partition value at the top of the | ||
| type's range, which does not sort with the other partition values. No single predicate on the | ||
| partition value can account for that, so those ranges have no strict projection. | ||
| """ | ||
| minimum, _ = _integer_bounds(source) | ||
| floor_mod = minimum % width | ||
| if floor_mod == 0 or not isinstance(pred.literal, LongLiteral): | ||
| return _truncate_number_strict(name, pred, transform) | ||
|
|
||
| first_block_end = minimum - floor_mod + width | ||
| if isinstance(pred, BoundLessThan): | ||
| if pred.literal.value < first_block_end: | ||
| return None | ||
| elif isinstance(pred, BoundLessThanOrEqual): | ||
| if pred.literal.increment().value < first_block_end: | ||
| return None | ||
| elif isinstance(pred, (BoundGreaterThan, BoundGreaterThanOrEqual)): | ||
| return None | ||
|
|
||
| return _truncate_number_strict(name, pred, transform) | ||
|
|
||
|
|
||
| def _truncate_number( | ||
| name: str, pred: BoundLiteralPredicate, transform: Callable[[Any | None], Any | None] | ||
| ) -> UnboundPredicate | None: | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No test covers this branch: with
if boundary < next_block_start:replaced byif False:, every test still passes, althoughx >= MINthen skips all ordinary partitions. An assert next to theGreaterThanone intest_int_truncate_projection_near_minimumwould cover it:That test projects the
LongTypeidcolumn, so anIntegerTypecase would help too; only the scan test covers int.