Skip to content

Commit 68c7ca2

Browse files
committed
Keep rows where a delete filter evaluates to null
delete() and overwrite() rewrite a data file with the rows the delete filter does not select. The complementary filter was Not(filter) OR'd with IS NULL for each column the filter does not null-check, so a null kept rows the filter matches (p == 1 OR v == 'c' with a null v) and dropped rows it does not match, where Not(filter) is null. Coalesce the filter to false before inverting it, so exactly the rows where the filter is true are deleted, the rows scan(row_filter=filter) returns. Closes #4184
1 parent 06a529d commit 68c7ca2

3 files changed

Lines changed: 37 additions & 14 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969

7070
from pyiceberg.conversions import to_bytes
7171
from pyiceberg.exceptions import ResolveError
72-
from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or
72+
from pyiceberg.expressions import AlwaysFalse, AlwaysTrue, BooleanExpression, BoundIsNaN, BoundTerm, Or
7373
from pyiceberg.expressions.literals import Literal
7474
from pyiceberg.expressions.visitors import (
7575
BoundBooleanExpressionVisitor,
@@ -1125,25 +1125,24 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
11251125
"""Complementary filter conversion function of expression_to_pyarrow.
11261126
11271127
Could not use expression_to_pyarrow(Not(expr)) to achieve this complementary effect because
1128-
~ in pyarrow.compute.Expression does not handle null.
1128+
~ in pyarrow.compute.Expression does not handle null: where expr evaluates to null, Not(expr) is null too
1129+
and the row is dropped. Only the rows where expr is true are deleted, so null is read as false.
11291130
"""
11301131
collector = _NullNaNUnmentionedTermsCollector()
11311132
collector.collect(expr)
11321133

11331134
# Convert the set of terms to a sorted list so that layout of the expression to build is deterministic.
1134-
null_unmentioned_bound_terms: list[BoundTerm] = sorted(
1135-
collector.null_unmentioned_bound_terms, key=lambda term: term.ref().field.name
1136-
)
11371135
nan_unmentioned_bound_terms: list[BoundTerm] = sorted(
11381136
collector.nan_unmentioned_bound_terms, key=lambda term: term.ref().field.name
11391137
)
11401138

1141-
preserve_expr: BooleanExpression = Not(expr)
1142-
for term in null_unmentioned_bound_terms:
1143-
preserve_expr = Or(preserve_expr, BoundIsNull(term=term))
1139+
preserve_expr = ~pc.coalesce(expression_to_pyarrow(expr, schema), pc.scalar(False))
1140+
nan_expr: BooleanExpression = AlwaysFalse()
11441141
for term in nan_unmentioned_bound_terms:
1145-
preserve_expr = Or(preserve_expr, BoundIsNaN(term=term))
1146-
return expression_to_pyarrow(preserve_expr, schema)
1142+
nan_expr = Or(nan_expr, BoundIsNaN(term=term))
1143+
if nan_expr == AlwaysFalse():
1144+
return preserve_expr
1145+
return preserve_expr | expression_to_pyarrow(nan_expr, schema)
11471146

11481147

11491148
@lru_cache

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1295,6 +1295,31 @@ def test_overwrite_record_batch_reader(catalog: Catalog) -> None:
12951295
assert len(tbl.scan().to_arrow()) == total_rows
12961296

12971297

1298+
def test_delete_and_overwrite_filter_with_null(catalog: Catalog) -> None:
1299+
catalog.create_namespace("default")
1300+
pa_table = pa.table({"x": [1, 2, 3], "p": [1, 2, 2], "v": [None, None, "c"]})
1301+
filters = [
1302+
# A null v makes the second branch null for x=2, which matches neither branch
1303+
"(p == 1 AND v IS NULL) OR (p == 2 AND v == 'c')",
1304+
# A null v leaves the filter true for x=1
1305+
"p == 1 OR v == 'c'",
1306+
"v != 'c'",
1307+
"NOT (v == 'c')",
1308+
]
1309+
for i, row_filter in enumerate(filters):
1310+
tbl = catalog.create_table(identifier=f"default.delete_filter_with_null_{i}_{catalog.name}", schema=pa_table.schema)
1311+
tbl.append(pa_table)
1312+
matched = set(tbl.scan(row_filter=row_filter).to_arrow()["x"].to_pylist())
1313+
expected = sorted({1, 2, 3} - matched)
1314+
1315+
tbl.delete(row_filter)
1316+
assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter
1317+
1318+
tbl.overwrite(pa_table, overwrite_filter="x > 0")
1319+
tbl.overwrite(pa_table.slice(0, 0), overwrite_filter=row_filter)
1320+
assert sorted(tbl.scan().to_arrow()["x"].to_pylist()) == expected, row_filter
1321+
1322+
12981323
def test_append_record_batch_reader_to_partitioned_table_raises(catalog: Catalog) -> None:
12991324
catalog.create_namespace("default")
13001325
identifier = f"default.append_record_batch_reader_partitioned_{catalog.name}"

‎tests/io/test_pyarrow_visitor.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -856,10 +856,9 @@ def test_expression_to_complementary_pyarrow(
856856
# Notice an isNan predicate on a str column is automatically converted to always false and removed from Or
857857
# and thus will not appear in the pc.expr.
858858
expected_repr = (
859-
'<pyarrow.compute.Expression (((invert(((((string_field == "hello") and (float_field > 100)) '
859+
'<pyarrow.compute.Expression (invert(coalesce(((((string_field == "hello") and (float_field > 100)) '
860860
"or ((is_nan(float_field) and (double_field == 0)) or (float_field > 100))) "
861-
"and invert(is_null(double_field, {nan_is_null=false})))) "
862-
"or is_null(float_field, {nan_is_null=false})) "
863-
"or is_null(string_field, {nan_is_null=false})) or is_nan(double_field))>"
861+
"and invert(is_null(double_field, {nan_is_null=false}))), false)) "
862+
"or is_nan(double_field))>"
864863
)
865864
assert repr(result) == expected_repr

0 commit comments

Comments
 (0)