Skip to content

Commit 5d2c642

Browse files
committed
Preserve residual visitor compatibility
1 parent 2a02771 commit 5d2c642

3 files changed

Lines changed: 82 additions & 229 deletions

File tree

‎pyiceberg/expressions/visitors.py‎

Lines changed: 20 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1821,28 +1821,37 @@ def _can_contain_nans(self, field_id: int) -> bool:
18211821
return (nan_count := self.nan_counts.get(field_id)) is not None and nan_count > 0
18221822

18231823

1824-
class _ResidualEvaluationVisitor(BoundBooleanExpressionVisitor[BooleanExpression]):
1825-
"""Evaluate a residual expression for one partition."""
1824+
class ResidualVisitor(BoundBooleanExpressionVisitor[BooleanExpression], ABC):
1825+
"""Find residuals for an expression using partition values.
1826+
1827+
A residual expression is made by partially evaluating an expression using partition values.
1828+
For example, if a table is partitioned by day(utc_timestamp) and is read with a filter expression
1829+
utc_timestamp > a and utc_timestamp < b, then there are 4 possible residual expressions
1830+
for the partition data, d:
1831+
1832+
1. If d > day(a) and d < day(b), the residual is always true
1833+
2. If d == day(a) and d != day(b), the residual is utc_timestamp > a
1834+
3. If d == day(b) and d != day(a), the residual is utc_timestamp < b
1835+
4. If d == day(a) == day(b), the residual is utc_timestamp > a and utc_timestamp < b
1836+
"""
18261837

18271838
schema: Schema
18281839
spec: PartitionSpec
18291840
case_sensitive: bool
1841+
expr: BooleanExpression
18301842
partition_schema: Schema
18311843
struct: Record
18321844

1833-
def __init__(
1834-
self,
1835-
schema: Schema,
1836-
spec: PartitionSpec,
1837-
case_sensitive: bool,
1838-
partition_schema: Schema,
1839-
partition_data: Record,
1840-
) -> None:
1845+
def __init__(self, schema: Schema, spec: PartitionSpec, case_sensitive: bool, expr: BooleanExpression) -> None:
18411846
self.schema = schema
18421847
self.spec = spec
18431848
self.case_sensitive = case_sensitive
1844-
self.partition_schema = partition_schema
1849+
self.expr = expr
1850+
self.partition_schema = Schema(*spec.partition_type(schema).fields)
1851+
1852+
def eval(self, partition_data: Record) -> BooleanExpression:
18451853
self.struct = partition_data
1854+
return visit(self.expr, visitor=self)
18461855

18471856
def visit_true(self) -> BooleanExpression:
18481857
return AlwaysTrue()
@@ -2006,46 +2015,6 @@ def visit_unbound_predicate(self, predicate: UnboundPredicate) -> BooleanExpress
20062015
return bound
20072016

20082017

2009-
class ResidualVisitor:
2010-
"""Find residuals for an expression using partition values.
2011-
2012-
A residual expression is made by partially evaluating an expression using partition values.
2013-
For example, if a table is partitioned by day(utc_timestamp) and is read with a filter expression
2014-
utc_timestamp > a and utc_timestamp < b, then there are 4 possible residual expressions
2015-
for the partition data, d:
2016-
2017-
1. If d > day(a) and d < day(b), the residual is always true
2018-
2. If d == day(a) and d != day(b), the residual is utc_timestamp > a
2019-
3. If d == day(b) and d != day(a), the residual is utc_timestamp < b
2020-
4. If d == day(a) == day(b), the residual is utc_timestamp > a and utc_timestamp < b
2021-
"""
2022-
2023-
schema: Schema
2024-
spec: PartitionSpec
2025-
case_sensitive: bool
2026-
expr: BooleanExpression
2027-
partition_schema: Schema
2028-
2029-
def __init__(self, schema: Schema, spec: PartitionSpec, case_sensitive: bool, expr: BooleanExpression) -> None:
2030-
self.schema = schema
2031-
self.spec = spec
2032-
self.case_sensitive = case_sensitive
2033-
self.expr = expr
2034-
self.partition_schema = Schema(*spec.partition_type(schema).fields)
2035-
2036-
def eval(self, partition_data: Record) -> BooleanExpression:
2037-
return visit(
2038-
self.expr,
2039-
visitor=_ResidualEvaluationVisitor(
2040-
schema=self.schema,
2041-
spec=self.spec,
2042-
case_sensitive=self.case_sensitive,
2043-
partition_schema=self.partition_schema,
2044-
partition_data=partition_data,
2045-
),
2046-
)
2047-
2048-
20492018
class ResidualEvaluator(ResidualVisitor):
20502019
def residual_for(self, partition_data: Record) -> BooleanExpression:
20512020
return self.eval(partition_data)

‎tests/expressions/test_residual_evaluator.py‎

Lines changed: 10 additions & 90 deletions
Original file line numberDiff line numberDiff line change
@@ -15,16 +15,13 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717
# pylint:disable=redefined-outer-name
18-
from concurrent.futures import ThreadPoolExecutor
19-
from threading import Event
20-
from typing import Any
21-
2218
import pytest
2319

2420
from pyiceberg.expressions import (
2521
AlwaysFalse,
2622
AlwaysTrue,
2723
And,
24+
BooleanExpression,
2825
EqualTo,
2926
GreaterThan,
3027
GreaterThanOrEqual,
@@ -45,7 +42,7 @@
4542
from pyiceberg.schema import Schema
4643
from pyiceberg.transforms import DayTransform, IdentityTransform
4744
from pyiceberg.typedef import Record
48-
from pyiceberg.types import DoubleType, FloatType, IntegerType, NestedField, StringType, StructType, TimestampType
45+
from pyiceberg.types import DoubleType, FloatType, IntegerType, NestedField, StringType, TimestampType
4946

5047

5148
def test_identity_transform_residual() -> None:
@@ -92,102 +89,25 @@ def test_identity_transform_residual() -> None:
9289
assert residual == AlwaysFalse()
9390

9491

95-
def test_residual_evaluator_does_not_mutate_prepared_state() -> None:
96-
schema = Schema(NestedField(1, "a", IntegerType()), NestedField(2, "b", IntegerType()))
97-
spec = PartitionSpec(
98-
PartitionField(1, 1001, IdentityTransform(), "a_part"),
99-
PartitionField(2, 1002, IdentityTransform(), "b_part"),
100-
)
101-
evaluator = residual_evaluator_of(
102-
spec=spec,
103-
expr=And(EqualTo("a", 1), EqualTo("b", 1)),
104-
case_sensitive=True,
105-
schema=schema,
106-
)
107-
initial_state = vars(evaluator).copy()
108-
109-
assert evaluator.residual_for(Record(1, 1)) == AlwaysTrue()
110-
assert evaluator.residual_for(Record(0, 0)) == AlwaysFalse()
111-
assert evaluator.residual_for(Record(1, 1)) == AlwaysTrue()
112-
113-
assert isinstance(evaluator, ResidualVisitor)
114-
assert vars(evaluator) == initial_state
115-
116-
11792
def test_residual_visitor_preserves_public_eval_api() -> None:
11893
schema = Schema(NestedField(1, "a", IntegerType()))
11994
spec = PartitionSpec(PartitionField(1, 1001, IdentityTransform(), "a_part"))
12095
visitor = ResidualVisitor(schema=schema, spec=spec, case_sensitive=True, expr=EqualTo("a", 1))
121-
initial_state = vars(visitor).copy()
12296

12397
assert visitor.eval(Record(1)) == AlwaysTrue()
12498
assert visitor.eval(Record(0)) == AlwaysFalse()
125-
assert vars(visitor) == initial_state
126-
127-
128-
def test_residual_evaluator_concurrent_calls_do_not_share_partitions() -> None:
129-
class BlockingRecord(Record):
130-
def __init__(self, first_read: Event, release_first_read: Event, *values: Any) -> None:
131-
super().__init__(*values)
132-
self.first_read = first_read
133-
self.release_first_read = release_first_read
134-
135-
def __getitem__(self, pos: int) -> Any:
136-
value = super().__getitem__(pos)
137-
if pos == 0:
138-
self.first_read.set()
139-
if not self.release_first_read.wait(timeout=5):
140-
raise TimeoutError("Timed out waiting to interleave residual evaluations")
141-
return value
142-
143-
schema = Schema(NestedField(1, "a", IntegerType()), NestedField(2, "b", IntegerType()))
144-
spec = PartitionSpec(
145-
PartitionField(1, 1001, IdentityTransform(), "a_part"),
146-
PartitionField(2, 1002, IdentityTransform(), "b_part"),
147-
)
148-
evaluator = residual_evaluator_of(
149-
spec=spec,
150-
expr=And(EqualTo("a", 1), EqualTo("b", 1)),
151-
case_sensitive=True,
152-
schema=schema,
153-
)
154-
first_read = Event()
155-
release_first_read = Event()
15699

157-
with ThreadPoolExecutor(max_workers=2) as executor:
158-
matching_result = executor.submit(
159-
evaluator.residual_for,
160-
BlockingRecord(first_read, release_first_read, 1, 1),
161-
)
162-
assert first_read.wait(timeout=5)
163100

164-
try:
165-
non_matching_result = executor.submit(evaluator.residual_for, Record(0, 0)).result(timeout=5)
166-
finally:
167-
release_first_read.set()
101+
def test_residual_visitor_subclass_can_customize_evaluation() -> None:
102+
class FalseForTrueResidualVisitor(ResidualVisitor):
103+
def visit_true(self) -> BooleanExpression:
104+
return AlwaysFalse()
168105

169-
assert matching_result.result(timeout=5) == AlwaysTrue()
170-
assert non_matching_result == AlwaysFalse()
171-
172-
173-
def test_partition_schema_reused_across_residuals(monkeypatch: pytest.MonkeyPatch) -> None:
174-
schema = Schema(NestedField(50, "dateint", IntegerType()))
175-
spec = PartitionSpec(PartitionField(50, 1050, IdentityTransform(), "dateint_part"))
176-
partition_type_calls = 0
177-
original_partition_type = PartitionSpec.partition_type
178-
179-
def counting_partition_type(self: PartitionSpec, schema: Schema) -> StructType:
180-
nonlocal partition_type_calls
181-
partition_type_calls += 1
182-
return original_partition_type(self, schema)
183-
184-
monkeypatch.setattr(PartitionSpec, "partition_type", counting_partition_type)
185-
186-
evaluator = residual_evaluator_of(spec=spec, expr=EqualTo("dateint", 20170815), case_sensitive=True, schema=schema)
106+
schema = Schema(NestedField(1, "a", IntegerType()))
107+
spec = PartitionSpec(PartitionField(1, 1001, IdentityTransform(), "a_part"))
108+
visitor = FalseForTrueResidualVisitor(schema=schema, spec=spec, case_sensitive=True, expr=AlwaysTrue())
187109

188-
assert evaluator.residual_for(Record(20170815)) == AlwaysTrue()
189-
assert evaluator.residual_for(Record(20170816)) == AlwaysFalse()
190-
assert partition_type_calls == 1
110+
assert visitor.eval(Record(1)) == AlwaysFalse()
191111

192112

193113
def test_case_insensitive_identity_transform_residuals() -> None:

0 commit comments

Comments
 (0)