From 68c7ca2917048970356cd98bf25d95e7739d4e97 Mon Sep 17 00:00:00 2001 From: Rahul-cs50 Date: Sat, 31 Oct 2026 12:25:59 +0530 Subject: [PATCH 1/2] Keep rows where a delete filter evaluates to null delete() and overwrite() rewrite a data file with the rows the delete filter does not select. The complementary filter was Not(filter) OR'd with IS NULL for each column the filter does not null-check, so a null kept rows the filter matches (p == 1 OR v == 'c' with a null v) and dropped rows it does not match, where Not(filter) is null. Coalesce the filter to false before inverting it, so exactly the rows where the filter is true are deleted, the rows scan(row_filter=filter) returns. Closes #4184 --- pyiceberg/io/pyarrow.py | 19 +++++++++---------- tests/catalog/test_catalog_behaviors.py | 25 +++++++++++++++++++++++++ tests/io/test_pyarrow_visitor.py | 7 +++---- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..69f0ca9d06 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -69,7 +69,7 @@ from pyiceberg.conversions import to_bytes from pyiceberg.exceptions import ResolveError -from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or +from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, BooleanExpression, BoundIsNaN, BoundTerm, Or from pyiceberg.expressions.literals import Literal from pyiceberg.expressions.visitors import ( BoundBooleanExpressionVisitor, @@ -1125,25 +1125,24 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema """Complementary filter conversion function of expression_to_pyarrow. Could not use expression_to_pyarrow(Not(expr)) to achieve this complementary effect because - ~ in pyarrow.compute.Expression does not handle null. + ~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too + and the row is dropped. Only the rows where expr is true are deleted, so null is read as false. """ collector = _NullNaNUnmentionedTermsCollector() collector.collect(expr) # Convert the set of terms to a sorted list so that layout of the expression to build is deterministic. - null_unmentioned_bound_terms: list[BoundTerm] = sorted( - collector.null_unmentioned_bound_terms, key=lambda term: term.ref().field.name - ) nan_unmentioned_bound_terms: list[BoundTerm] = sorted( collector.nan_unmentioned_bound_terms, key=lambda term: term.ref().field.name ) - preserve_expr: BooleanExpression = Not(expr) - for term in null_unmentioned_bound_terms: - preserve_expr = Or(preserve_expr, BoundIsNull(term=term)) + preserve_expr = ~pc.coalesce(expression_to_pyarrow(expr, schema), pc.scalar(False)) + nan_expr: BooleanExpression = AlwaysFalse() for term in nan_unmentioned_bound_terms: - preserve_expr = Or(preserve_expr, BoundIsNaN(term=term)) - return expression_to_pyarrow(preserve_expr, schema) + nan_expr = Or(nan_expr, BoundIsNaN(term=term)) + if nan_expr == AlwaysFalse(): + return preserve_expr + return preserve_expr | expression_to_pyarrow(nan_expr, schema) @lru_cache diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1148ca8527..394c9efc15 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1295,6 +1295,31 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None: assert len(tbl.scan().to_arrow()) == total_rows +def test_delete_and_overwrite_filter_with_null(catalog: Catalog) -> None: + catalog.create_namespace("default") + pa_table = pa.table({"x": [1, 2, 3], "p": [1, 2, 2], "v": [None, None, "c"]}) + filters = [ + # A null v makes the second branch null for x=2, which matches neither branch + "(p == 1 AND v IS NULL) OR (p == 2 AND v == 'c')", + # A null v leaves the filter true for x=1 + "p == 1 OR v == 'c'", + "v != 'c'", + "NOT (v == 'c')", + ] + for i, row_filter in enumerate(filters): + tbl = catalog.create_table(identifier=f"default.delete_filter_with_null_{i}_{catalog.name}", schema=pa_table.schema) + tbl.append(pa_table) + matched = set(tbl.scan(row_filter=row_filter).to_arrow()["x"].to_pylist()) + expected = sorted({1, 2, 3} - matched) + + tbl.delete(row_filter) + assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter + + tbl.overwrite(pa_table, overwrite_filter="x > 0") + tbl.overwrite(pa_table.slice(0, 0), overwrite_filter=row_filter) + assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter + + def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}" diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..2183d5b6f9 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -856,10 +856,9 @@ def test_expression_to_complementary_pyarrow( # Notice an isNan predicate on a str column is automatically converted to always false and removed from Or # and thus will not appear in the pc.expr. expected_repr = ( - ' 100)) ' + ' 100)) ' "or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) " - "and invert(is_null(double_field, {nan_is_null=false})))) " - "or is_null(float_field, {nan_is_null=false})) " - "or is_null(string_field, {nan_is_null=false})) or is_nan(double_field))>" + "and invert(is_null(double_field, {nan_is_null=false}))), false)) " + "or is_nan(double_field))>" ) assert repr(result) == expected_repr From 88075cd8fb28cae1368156d342bc377d1402f9eb Mon Sep 17 00:00:00 2001 From: Rahul-cs50 Date: Tue, 17 Nov 2026 16:25:53 +0530 Subject: [PATCH 2/2] Drop the unused null tracking from the complement's term collector Since the complement coalesces the filter to false, nothing reads the null-unmentioned terms any more, so the collector only tracks NaN terms now. Check which rows the complement keeps instead of pyarrow's expression repr, which changes across pyarrow versions. --- pyiceberg/io/pyarrow.py | 39 ++++-------------------------- tests/integration/test_deletes.py | 2 +- tests/io/test_pyarrow_visitor.py | 40 ++++++++++++------------------- 3 files changed, 20 insertions(+), 61 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 69f0ca9d06..fd0ddb722f 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -986,11 +986,7 @@ def visit_or(self, left_result: pc.Expression, right_result: pc.Expression) -> p return left_result | right_result -class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): - # BoundTerms which have either is_null or is_not_null appearing at least once in the boolean expr. - is_null_or_not_bound_terms: set[BoundTerm] - # The remaining BoundTerms appearing in the boolean expr. - null_unmentioned_bound_terms: set[BoundTerm] +class _NaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): # BoundTerms which have either is_nan or is_not_nan appearing at least once in the boolean expr. is_nan_or_not_bound_terms: set[BoundTerm] # The remaining BoundTerms appearing in the boolean expr. @@ -998,22 +994,9 @@ class _NullNaNUnmentionedTermsCollector(BoundBooleanExpressionVisitor[None]): def __init__(self) -> None: super().__init__() - self.is_null_or_not_bound_terms = set() - self.null_unmentioned_bound_terms = set() self.is_nan_or_not_bound_terms = set() self.nan_unmentioned_bound_terms = set() - def _handle_explicit_is_null_or_not(self, term: BoundTerm) -> None: - """Handle the predicate case where either is_null or is_not_null is included.""" - if term in self.null_unmentioned_bound_terms: - self.null_unmentioned_bound_terms.remove(term) - self.is_null_or_not_bound_terms.add(term) - - def _handle_null_unmentioned(self, term: BoundTerm) -> None: - """Handle the predicate case where neither is_null or is_not_null is included.""" - if term not in self.is_null_or_not_bound_terms: - self.null_unmentioned_bound_terms.add(term) - def _handle_explicit_is_nan_or_not(self, term: BoundTerm) -> None: """Handle the predicate case where either is_nan or is_not_nan is included.""" if term in self.nan_unmentioned_bound_terms: @@ -1026,59 +1009,45 @@ def _handle_nan_unmentioned(self, term: BoundTerm) -> None: self.nan_unmentioned_bound_terms.add(term) def visit_in(self, term: BoundTerm, literals: set[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_is_nan(self, term: BoundTerm) -> None: - self._handle_null_unmentioned(term) self._handle_explicit_is_nan_or_not(term) def visit_not_nan(self, term: BoundTerm) -> None: - self._handle_null_unmentioned(term) self._handle_explicit_is_nan_or_not(term) def visit_is_null(self, term: BoundTerm) -> None: - self._handle_explicit_is_null_or_not(term) self._handle_nan_unmentioned(term) def visit_not_null(self, term: BoundTerm) -> None: - self._handle_explicit_is_null_or_not(term) self._handle_nan_unmentioned(term) def visit_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_greater_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_greater_than(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_less_than(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_less_than_or_equal(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_not_starts_with(self, term: BoundTerm, literal: Literal[Any]) -> None: - self._handle_null_unmentioned(term) self._handle_nan_unmentioned(term) def visit_true(self) -> None: @@ -1100,9 +1069,9 @@ def collect( self, expr: BooleanExpression, ) -> None: - """Collect bound references categorized by null predicates. + """Collect bound references categorized by NaN predicates. - Categorizes by having at least one is_null or is_not_null in the expr and the remaining. + Categorizes by having at least one is_nan or is_not_nan in the expr and the remaining. """ boolean_expression_visit(expr, self) @@ -1128,7 +1097,7 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema ~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too and the row is dropped. Only the rows where expr is true are deleted, so null is read as false. """ - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(expr) # Convert the set of terms to a sorted list so that layout of the expression to build is deterministic. diff --git a/tests/integration/test_deletes.py b/tests/integration/test_deletes.py index 20205c59fb..64b14c031a 100644 --- a/tests/integration/test_deletes.py +++ b/tests/integration/test_deletes.py @@ -677,7 +677,7 @@ def test_delete_overwrite_table_with_nan(session_catalog: RestCatalog) -> None: This means if we set the test case as floats == 2.0 (equal predicate as in test_delete_overwrite_table_with_null), test will pass even without the logic under test - in _NullNaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow + in _NaNUnmentionedTermsCollector (a helper of _expression_to_complementary_pyarrow to handle revert of iceberg expression of is_null/not_null/is_nan/not_nan). Instead, we test the filter of !=, so that the revert is == which exposes the issue. """ diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index 2183d5b6f9..00eb1f1caf 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -38,7 +38,7 @@ _ConvertToIcebergWithoutIDs, _expression_to_complementary_pyarrow, _HasIds, - _NullNaNUnmentionedTermsCollector, + _NaNUnmentionedTermsCollector, _pyarrow_schema_ensure_large_types, _pyarrow_schema_ensure_small_types, pyarrow_to_schema, @@ -789,19 +789,12 @@ def test_collect_null_nan_unmentioned_terms( bound_expr = And( Or(And(bound_eq_str_field, bound_is_nan_float_field), bound_is_null_double_field), Not(bound_is_nan_float_field) ) - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(bound_expr) - assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == { - "float_field", - "string_field", - } assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == { "string_field", "double_field", } - assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == { - "double_field", - } assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"} @@ -821,19 +814,12 @@ def test_collect_null_nan_unmentioned_terms_with_multiple_predicates_on_the_same ), Not(bound_is_null_double_field), ) - collector = _NullNaNUnmentionedTermsCollector() + collector = _NaNUnmentionedTermsCollector() collector.collect(bound_expr) - assert {t.ref().field.name for t in collector.null_unmentioned_bound_terms} == { - "float_field", - "string_field", - } assert {t.ref().field.name for t in collector.nan_unmentioned_bound_terms} == { "string_field", "double_field", } - assert {t.ref().field.name for t in collector.is_null_or_not_bound_terms} == { - "double_field", - } assert {t.ref().field.name for t in collector.is_nan_or_not_bound_terms} == {"float_field"} @@ -853,12 +839,16 @@ def test_expression_to_complementary_pyarrow( Not(bound_is_null_double_field), ) result = _expression_to_complementary_pyarrow(bound_expr) - # Notice an isNan predicate on a str column is automatically converted to always false and removed from Or - # and thus will not appear in the pc.expr. - expected_repr = ( - ' 100)) ' - "or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) " - "and invert(is_null(double_field, {nan_is_null=false}))), false)) " - "or is_nan(double_field))>" + table = pa.table( + { + "id": pa.array([1, 2, 3, 4, 5], type=pa.int32()), + "string_field": pa.array(["hello", "hello", "x", "x", None], type=pa.string()), + "float_field": pa.array([200.0, 50.0, float("nan"), 200.0, None], type=pa.float32()), + "double_field": pa.array([1.0, 1.0, 0.0, None, 1.0], type=pa.float64()), + } ) - assert repr(result) == expected_repr + # The complement keeps exactly the rows where bound_expr is not true: + # 1 matches (float_field > 100), 3 matches (is_nan(float_field) and double_field == 0), + # 2 does not match, 4 does not match because double_field is null, + # and 5 evaluates to null because string_field and float_field are null. + assert table.filter(result)["id"].to_pylist() == [2, 4, 5]