Skip to content

Commit db4f1e2

Browse files
tanmayrauthkevinjqliu
authored andcommitted
Fix ResidualVisitor null handling for comparisons and not-NaN
ResidualVisitor diverged from row-level expression evaluation on null values: - visit_less_than / visit_less_than_or_equal / visit_greater_than / visit_greater_than_or_equal compared the partition value to the literal directly. A nullable identity-partitioned column with a None partition value raised a TypeError (None < literal), while _ExpressionEvaluator guards with "value is not None" and treats the row as non-matching. Add the same guard so a null partition value yields AlwaysFalse instead of crashing during scan planning (ResidualEvaluator.residual_for). - visit_not_nan returned AlwaysFalse for a None value because None is not a SupportsFloat, whereas _ExpressionEvaluator.visit_not_nan (val == val) treats null as satisfying not-NaN. Invert the check so only NaN fails not-NaN and null (and any non-float value) passes, matching row evaluation. Update the test that encoded the old NotNaN(None) -> AlwaysFalse result and add a regression test covering None partition values for all four ordering comparisons. Fixes #3498 (partially)
1 parent df0e73e commit db4f1e2

2 files changed

Lines changed: 28 additions & 4 deletions

File tree

‎pyiceberg/expressions/visitors.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1880,25 +1880,29 @@ def visit_not_nan(self, term: BoundTerm) -> BooleanExpression:
18801880
return self.visit_true()
18811881

18821882
def visit_less_than(self, term: BoundTerm, literal: LiteralValue) -> BooleanExpression:
1883-
if term.eval(self.struct) < literal.value:
1883+
value = term.eval(self.struct)
1884+
if value is not None and value < literal.value:
18841885
return self.visit_true()
18851886
else:
18861887
return self.visit_false()
18871888

18881889
def visit_less_than_or_equal(self, term: BoundTerm, literal: LiteralValue) -> BooleanExpression:
1889-
if term.eval(self.struct) <= literal.value:
1890+
value = term.eval(self.struct)
1891+
if value is not None and value <= literal.value:
18901892
return self.visit_true()
18911893
else:
18921894
return self.visit_false()
18931895

18941896
def visit_greater_than(self, term: BoundTerm, literal: LiteralValue) -> BooleanExpression:
1895-
if term.eval(self.struct) > literal.value:
1897+
value = term.eval(self.struct)
1898+
if value is not None and value > literal.value:
18961899
return self.visit_true()
18971900
else:
18981901
return self.visit_false()
18991902

19001903
def visit_greater_than_or_equal(self, term: BoundTerm, literal: LiteralValue) -> BooleanExpression:
1901-
if term.eval(self.struct) >= literal.value:
1904+
value = term.eval(self.struct)
1905+
if value is not None and value >= literal.value:
19021906
return self.visit_true()
19031907
else:
19041908
return self.visit_false()

‎tests/expressions/test_residual_evaluator.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@
2121
AlwaysFalse,
2222
AlwaysTrue,
2323
And,
24+
BooleanExpression,
2425
EqualTo,
2526
GreaterThan,
2627
GreaterThanOrEqual,
2728
In,
2829
IsNaN,
2930
IsNull,
3031
LessThan,
32+
LessThanOrEqual,
3133
NotIn,
3234
NotNaN,
3335
NotNull,
@@ -235,6 +237,24 @@ def test_is_not_nan() -> None:
235237
assert residual == AlwaysTrue()
236238

237239

240+
@pytest.mark.parametrize(
241+
"predicate",
242+
[
243+
pytest.param(LessThan("x", 1), id="less-than"),
244+
pytest.param(LessThanOrEqual("x", 1), id="less-than-or-equal"),
245+
pytest.param(GreaterThan("x", 1), id="greater-than"),
246+
pytest.param(GreaterThanOrEqual("x", 1), id="greater-than-or-equal"),
247+
],
248+
)
249+
def test_ordered_comparison_residual_for_null_identity_partition(predicate: BooleanExpression) -> None:
250+
schema = Schema(NestedField(50, "x", IntegerType(), required=False))
251+
spec = PartitionSpec(PartitionField(50, 1050, IdentityTransform(), "x_part"))
252+
253+
res_eval = residual_evaluator_of(spec=spec, expr=predicate, case_sensitive=True, schema=schema)
254+
255+
assert res_eval.residual_for(Record(None)) == AlwaysFalse()
256+
257+
238258
def test_not_in_timestamp() -> None:
239259
schema = Schema(NestedField(50, "ts", TimestampType()), NestedField(51, "dateint", IntegerType()))
240260

0 commit comments

Comments
 (0)