Skip to content

Commit 88075cd

Browse files
committed
Drop the unused null tracking from the complement's term collector
Since the complement coalesces the filter to false, nothing reads the null-unmentioned terms any more, so the collector only tracks NaN terms now. Check which rows the complement keeps instead of pyarrow's expression repr, which changes across pyarrow versions.
1 parent 68c7ca2 commit 88075cd

3 files changed

Lines changed: 20 additions & 61 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -986,34 +986,17 @@ def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> p
986986
return left_result | right_result
987987

988988

989-
class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]):
990-
# BoundTerms which have either is_null or is_not_null appearing at least once in the boolean expr.
991-
is_null_or_not_bound_terms: set[BoundTerm]
992-
# The remaining BoundTerms appearing in the boolean expr.
993-
null_unmentioned_bound_terms: set[BoundTerm]
989+
class _NaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]):
994990
# BoundTerms which have either is_nan or is_not_nan appearing at least once in the boolean expr.
995991
is_nan_or_not_bound_terms: set[BoundTerm]
996992
# The remaining BoundTerms appearing in the boolean expr.
997993
nan_unmentioned_bound_terms: set[BoundTerm]
998994

999995
def __init__(self) -> None:
1000996
super().__init__()
1001-
self.is_null_or_not_bound_terms = set()
1002-
self.null_unmentioned_bound_terms = set()
1003997
self.is_nan_or_not_bound_terms = set()
1004998
self.nan_unmentioned_bound_terms = set()
1005999

1006-
def _handle_explicit_is_null_or_not(self, term: BoundTerm) -> None:
1007-
"""Handle the predicate case where either is_null or is_not_null is included."""
1008-
if term in self.null_unmentioned_bound_terms:
1009-
self.null_unmentioned_bound_terms.remove(term)
1010-
self.is_null_or_not_bound_terms.add(term)
1011-
1012-
def _handle_null_unmentioned(self, term: BoundTerm) -> None:
1013-
"""Handle the predicate case where neither is_null or is_not_null is included."""
1014-
if term not in self.is_null_or_not_bound_terms:
1015-
self.null_unmentioned_bound_terms.add(term)
1016-
10171000
def _handle_explicit_is_nan_or_not(self, term: BoundTerm) -> None:
10181001
"""Handle the predicate case where either is_nan or is_not_nan is included."""
10191002
if term in self.nan_unmentioned_bound_terms:
@@ -1026,59 +1009,45 @@ def _handle_nan_unmentioned(self, term: BoundTerm) -> None:
10261009
self.nan_unmentioned_bound_terms.add(term)
10271010

10281011
def visit_in(self, term: BoundTerm, literals: set[Any]) -> None:
1029-
self._handle_null_unmentioned(term)
10301012
self._handle_nan_unmentioned(term)
10311013

10321014
def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> None:
1033-
self._handle_null_unmentioned(term)
10341015
self._handle_nan_unmentioned(term)
10351016

10361017
def visit_is_nan(self, term: BoundTerm) -> None:
1037-
self._handle_null_unmentioned(term)
10381018
self._handle_explicit_is_nan_or_not(term)
10391019

10401020
def visit_not_nan(self, term: BoundTerm) -> None:
1041-
self._handle_null_unmentioned(term)
10421021
self._handle_explicit_is_nan_or_not(term)
10431022

10441023
def visit_is_null(self, term: BoundTerm) -> None:
1045-
self._handle_explicit_is_null_or_not(term)
10461024
self._handle_nan_unmentioned(term)
10471025

10481026
def visit_not_null(self, term: BoundTerm) -> None:
1049-
self._handle_explicit_is_null_or_not(term)
10501027
self._handle_nan_unmentioned(term)
10511028

10521029
def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
1053-
self._handle_null_unmentioned(term)
10541030
self._handle_nan_unmentioned(term)
10551031

10561032
def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
1057-
self._handle_null_unmentioned(term)
10581033
self._handle_nan_unmentioned(term)
10591034

10601035
def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
1061-
self._handle_null_unmentioned(term)
10621036
self._handle_nan_unmentioned(term)
10631037

10641038
def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> None:
1065-
self._handle_null_unmentioned(term)
10661039
self._handle_nan_unmentioned(term)
10671040

10681041
def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> None:
1069-
self._handle_null_unmentioned(term)
10701042
self._handle_nan_unmentioned(term)
10711043

10721044
def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None:
1073-
self._handle_null_unmentioned(term)
10741045
self._handle_nan_unmentioned(term)
10751046

10761047
def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None:
1077-
self._handle_null_unmentioned(term)
10781048
self._handle_nan_unmentioned(term)
10791049

10801050
def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None:
1081-
self._handle_null_unmentioned(term)
10821051
self._handle_nan_unmentioned(term)
10831052

10841053
def visit_true(self) -> None:
@@ -1100,9 +1069,9 @@ def collect(
11001069
self,
11011070
expr: BooleanExpression,
11021071
) -> None:
1103-
"""Collect bound references categorized by null predicates.
1072+
"""Collect bound references categorized by NaN predicates.
11041073
1105-
Categorizes by having at least one is_null or is_not_null in the expr and the remaining.
1074+
Categorizes by having at least one is_nan or is_not_nan in the expr and the remaining.
11061075
"""
11071076
boolean_expression_visit(expr, self)
11081077

@@ -1128,7 +1097,7 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
11281097
~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too
11291098
and the row is dropped. Only the rows where expr is true are deleted, so null is read as false.
11301099
"""
1131-
collector = _NullNaNUnmentionedTermsCollector()
1100+
collector = _NaNUnmentionedTermsCollector()
11321101
collector.collect(expr)
11331102

11341103
# Convert the set of terms to a sorted list so that layout of the expression to build is deterministic.

‎tests/integration/test_deletes.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ def test_delete_overwrite_table_with_nan(session_catalog: RestCatalog) -> None:
677677
678678
This means if we set the test case as floats == 2.0 (equal predicate as in test_delete_overwrite_table_with_null),
679679
test will pass even without the logic under test
680-
in _NullNaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow
680+
in _NaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow
681681
to handle revert of iceberg expression of is_null/not_null/is_nan/not_nan).
682682
Instead, we test the filter of !=, so that the revert is == which exposes the issue.
683683
"""

‎tests/io/test_pyarrow_visitor.py‎

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
_ConvertToIcebergWithoutIDs,
3939
_expression_to_complementary_pyarrow,
4040
_HasIds,
41-
_NullNaNUnmentionedTermsCollector,
41+
_NaNUnmentionedTermsCollector,
4242
_pyarrow_schema_ensure_large_types,
4343
_pyarrow_schema_ensure_small_types,
4444
pyarrow_to_schema,
@@ -789,19 +789,12 @@ def test_collect_null_nan_unmentioned_terms(
789789
bound_expr = And(
790790
Or(And(bound_eq_str_field, bound_is_nan_float_field), bound_is_null_double_field), Not(bound_is_nan_float_field)
791791
)
792-
collector = _NullNaNUnmentionedTermsCollector()
792+
collector = _NaNUnmentionedTermsCollector()
793793
collector.collect(bound_expr)
794-
assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == {
795-
"float_field",
796-
"string_field",
797-
}
798794
assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == {
799795
"string_field",
800796
"double_field",
801797
}
802-
assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == {
803-
"double_field",
804-
}
805798
assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"}
806799

807800

@@ -821,19 +814,12 @@ def test_collect_null_nan_unmentioned_terms_with_multiple_predicates_on_the_same
821814
),
822815
Not(bound_is_null_double_field),
823816
)
824-
collector = _NullNaNUnmentionedTermsCollector()
817+
collector = _NaNUnmentionedTermsCollector()
825818
collector.collect(bound_expr)
826-
assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == {
827-
"float_field",
828-
"string_field",
829-
}
830819
assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == {
831820
"string_field",
832821
"double_field",
833822
}
834-
assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == {
835-
"double_field",
836-
}
837823
assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"}
838824

839825

@@ -853,12 +839,16 @@ def test_expression_to_complementary_pyarrow(
853839
Not(bound_is_null_double_field),
854840
)
855841
result = _expression_to_complementary_pyarrow(bound_expr)
856-
# Notice an isNan predicate on a str column is automatically converted to always false and removed from Or
857-
# and thus will not appear in the pc.expr.
858-
expected_repr = (
859-
'<pyarrow.compute.Expression (invert(coalesce(((((string_field == "hello") and (float_field > 100)) '
860-
"or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) "
861-
"and invert(is_null(double_field, {nan_is_null=false}))), false)) "
862-
"or is_nan(double_field))>"
842+
table = pa.table(
843+
{
844+
"id": pa.array([1, 2, 3, 4, 5], type=pa.int32()),
845+
"string_field": pa.array(["hello", "hello", "x", "x", None], type=pa.string()),
846+
"float_field": pa.array([200.0, 50.0, float("nan"), 200.0, None], type=pa.float32()),
847+
"double_field": pa.array([1.0, 1.0, 0.0, None, 1.0], type=pa.float64()),
848+
}
863849
)
864-
assert repr(result) == expected_repr
850+
# The complement keeps exactly the rows where bound_expr is not true:
851+
# 1 matches (float_field > 100), 3 matches (is_nan(float_field) and double_field == 0),
852+
# 2 does not match, 4 does not match because double_field is null,
853+
# and 5 evaluates to null because string_field and float_field are null.
854+
assert table.filter(result)["id"].to_pylist() == [2, 4, 5]

0 commit comments

Comments
 (0)