Skip to content

Commit ebee787

Browse files
tomronclaude
andcommitted
Fix row filter literals narrowed to old file types after type promotion
The scan re-bound the row filter to each data file's schema, which converted literals to the file's older type. After promoting float to double, `f > 0.1` and `f != 0.1` dropped rows written as float, and a widened decimal raised ArrowInvalid. Keep the filter bound to the table schema and only resolve predicates on columns missing from the file. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 06a529d commit ebee787

4 files changed

Lines changed: 316 additions & 5 deletions

File tree

‎pyiceberg/expressions/visitors.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -930,6 +930,9 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
930930
else AlwaysFalse()
931931
)
932932

933+
return self._visit_present_column(predicate, file_column_name)
934+
935+
def _visit_present_column(self, predicate: BoundPredicate, file_column_name: str) -> BooleanExpression:
933936
if isinstance(predicate, BoundUnaryPredicate):
934937
return predicate.as_unbound(file_column_name)
935938
elif isinstance(predicate, BoundLiteralPredicate):
@@ -940,12 +943,29 @@ def visit_bound_predicate(self, predicate: BoundPredicate) -> BooleanExpression:
940943
raise ValueError(f"Unsupported predicate: {predicate}")
941944

942945

946+
class _MissingColumnResolver(_ColumnNameTranslator):
947+
"""Resolves predicates on columns missing from the file, keeping the other predicates bound to the table schema.
948+
949+
Literals stay in the table's types, so they are not narrowed to the types of an older file after type promotion.
950+
"""
951+
952+
def _visit_present_column(self, predicate: BoundPredicate, file_column_name: str) -> BooleanExpression:
953+
return predicate
954+
955+
943956
def translate_column_names(
944957
expr: BooleanExpression, file_schema: Schema, case_sensitive: bool = True, projected_field_values: dict[int, Any] = EMPTY_DICT
945958
) -> BooleanExpression:
946959
return visit(expr, _ColumnNameTranslator(file_schema, case_sensitive, projected_field_values))
947960

948961

962+
def resolve_missing_columns(
963+
expr: BooleanExpression, file_schema: Schema, case_sensitive: bool = True, projected_field_values: dict[int, Any] = EMPTY_DICT
964+
) -> BooleanExpression:
965+
"""Replace predicates on columns missing from the file with constants, leaving the others bound to the table schema."""
966+
return visit(expr, _MissingColumnResolver(file_schema, case_sensitive, projected_field_values))
967+
968+
949969
class _ExpressionFieldIDs(BooleanExpressionVisitor[set[int]]):
950970
"""Extracts the field IDs used in the BooleanExpression."""
951971

‎pyiceberg/io/pyarrow.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@
7575
BoundBooleanExpressionVisitor,
7676
bind,
7777
extract_field_ids,
78-
translate_column_names,
78+
resolve_missing_columns,
7979
)
8080
from pyiceberg.expressions.visitors import visit as boolean_expression_visit
8181
from pyiceberg.io import (
@@ -1737,11 +1737,12 @@ def _task_to_record_batches(
17371737

17381738
pyarrow_filter = None
17391739
if bound_row_filter is not AlwaysTrue():
1740-
translated_row_filter = translate_column_names(
1740+
# The filter stays bound to the table schema so literals keep the table's types, which may be
1741+
# wider than the file's after type promotion. Arrow compares the mixed types.
1742+
file_row_filter = resolve_missing_columns(
17411743
bound_row_filter, file_schema, case_sensitive=case_sensitive, projected_field_values=projected_missing_fields
17421744
)
1743-
bound_file_filter = bind(file_schema, translated_row_filter, case_sensitive=case_sensitive)
1744-
pyarrow_filter = expression_to_pyarrow(bound_file_filter, file_schema)
1745+
pyarrow_filter = expression_to_pyarrow(file_row_filter, file_schema)
17451746

17461747
file_project_schema = prune_columns(file_schema, projected_field_ids, select_full_types=False)
17471748

‎tests/expressions/test_visitors.py‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@
7474
_ManifestEvalVisitor,
7575
expression_evaluator,
7676
expression_to_plain_format,
77+
resolve_missing_columns,
7778
rewrite_not,
7879
rewrite_to_dnf,
7980
translate_column_names,
@@ -1982,3 +1983,31 @@ def test_translate_column_names_missing_column_projected_field_ignores_initial_d
19821983

19831984
# Should evaluate to AlwaysFalse since projected field value doesn't match the expression literal
19841985
assert translated_expr == AlwaysFalse()
1986+
1987+
1988+
def test_resolve_missing_columns_keeps_bound_predicates_in_table_types() -> None:
1989+
"""Predicates on columns in the file stay bound, and predicates on missing columns become constants."""
1990+
table_schema = Schema(
1991+
NestedField(field_id=1, name="existing_col", field_type=DoubleType(), required=False),
1992+
NestedField(field_id=2, name="missing_col", field_type=IntegerType(), required=False),
1993+
schema_id=1,
1994+
)
1995+
file_schema = Schema(
1996+
NestedField(field_id=1, name="existing_col", field_type=FloatType(), required=False),
1997+
schema_id=0,
1998+
)
1999+
2000+
bound_expr = visit(
2001+
And(GreaterThan("existing_col", 0.1), EqualTo("missing_col", 42)),
2002+
visitor=BindVisitor(schema=table_schema, case_sensitive=True),
2003+
)
2004+
assert resolve_missing_columns(bound_expr, file_schema) == AlwaysFalse()
2005+
2006+
bound_expr = visit(
2007+
And(GreaterThan("existing_col", 0.1), IsNull("missing_col")),
2008+
visitor=BindVisitor(schema=table_schema, case_sensitive=True),
2009+
)
2010+
resolved = resolve_missing_columns(bound_expr, file_schema)
2011+
assert isinstance(resolved, BoundGreaterThan)
2012+
assert resolved.literal.value == 0.1
2013+
assert isinstance(resolved.term.ref().field.field_type, DoubleType)

‎tests/io/test_pyarrow.py‎

Lines changed: 262 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import zlib
2727
from collections.abc import Iterator
2828
from datetime import UTC, date, datetime
29+
from decimal import Decimal
2930
from pathlib import Path
3031
from typing import Any, Literal
3132
from unittest.mock import MagicMock, patch
@@ -66,10 +67,13 @@
6667
EqualTo,
6768
GreaterThan,
6869
IsNull,
70+
LessThan,
6971
Not,
72+
NotEqualTo,
7073
Or,
7174
)
7275
from pyiceberg.expressions.literals import literal
76+
from pyiceberg.expressions.visitors import bind
7377
from pyiceberg.io import ADLS_ACCOUNT_NAME, S3_RETRY_STRATEGY_IMPL, S3_SSL_CA_CERT, InputStream, OutputStream, load_file_io
7478
from pyiceberg.io.pyarrow import (
7579
ICEBERG_SCHEMA,
@@ -98,7 +102,7 @@
98102
from pyiceberg.manifest import DataFile, DataFileContent, FileFormat, ManifestEntry, ManifestEntryStatus
99103
from pyiceberg.partitioning import PartitionField, PartitionSpec
100104
from pyiceberg.schema import Schema, make_compatible_name, visit
101-
from pyiceberg.table import FileScanTask, TableProperties, WriteTask
105+
from pyiceberg.table import FileScanTask, Table, TableProperties, WriteTask
102106
from pyiceberg.table.delete_file_index import DeleteFileIndex
103107
from pyiceberg.table.deletion_vector import (
104108
_DV_BLOB_MAGIC_NUMBER,
@@ -6102,3 +6106,260 @@ def test_dictionary_columns_produces_dict_encoded_output(tmpdir: str) -> None:
61026106

61036107
# Values must be identical
61046108
assert result_plain.column("label").to_pylist() == result_dict.column("label").to_pylist()
6109+
6110+
6111+
def test_row_filter_on_files_written_before_type_promotion(catalog: InMemoryCatalog) -> None:
6112+
"""Filters compare in the table's type, not the type of older data files (https://github.com/apache/iceberg-python/issues/4203)."""
6113+
catalog.create_namespace("default")
6114+
table = catalog.create_table(
6115+
"default.test_filter_type_promotion",
6116+
schema=Schema(
6117+
NestedField(1, "id", IntegerType(), required=False),
6118+
NestedField(2, "f", FloatType(), required=False),
6119+
NestedField(3, "d", DecimalType(5, 2), required=False),
6120+
),
6121+
)
6122+
table.append(
6123+
pa.table(
6124+
{
6125+
"id": pa.array([1, 2], pa.int32()),
6126+
"f": pa.array([0.1, 2.5], pa.float32()),
6127+
"d": pa.array([Decimal("1.00"), Decimal("999.99")], pa.decimal128(5, 2)),
6128+
}
6129+
)
6130+
)
6131+
6132+
with table.update_schema() as update:
6133+
update.update_column("f", DoubleType())
6134+
update.update_column("d", DecimalType(12, 2))
6135+
update.update_column("id", LongType())
6136+
table = catalog.load_table("default.test_filter_type_promotion")
6137+
6138+
# The stored float32 0.1 is promoted to 0.10000000149011612, which is greater than and not equal to the double 0.1
6139+
for row_filter, expected in [
6140+
("f > 0.1", [1, 2]),
6141+
("f != 0.1", [1, 2]),
6142+
("f = 0.1", []),
6143+
("f < 0.1", []),
6144+
("d < '1000.00'", [1, 2]),
6145+
("d > '999.99'", []),
6146+
("id < 3000000000", [1, 2]),
6147+
]:
6148+
assert table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist() == expected, row_filter
6149+
6150+
6151+
def test_row_filter_after_type_promotion_with_evolved_schemas(catalog: InMemoryCatalog) -> None:
6152+
"""Promoted-column filters hold for nested, renamed and added columns, mixed files, and nulls (issue 4203)."""
6153+
catalog.create_namespace("default")
6154+
6155+
def check(table: Table, cases: list[tuple[str, list[int]]]) -> None:
6156+
for row_filter, expected in cases:
6157+
ids = sorted(table.scan(row_filter=row_filter).to_arrow()["id"].to_pylist())
6158+
assert ids == expected, f"{table.name()}: {row_filter}"
6159+
6160+
# Nested field
6161+
struct_type = pa.struct([("x", pa.float32())])
6162+
table = catalog.create_table(
6163+
"default.promotion_nested",
6164+
schema=Schema(
6165+
NestedField(1, "id", LongType(), required=False),
6166+
NestedField(2, "s", StructType(NestedField(3, "x", FloatType(), required=False)), required=False),
6167+
),
6168+
)
6169+
table.append(
6170+
pa.table(
6171+
{"id": pa.array([1, 2], pa.int64()), "s": pa.array([{"x": 0.1}, {"x": 2.5}], struct_type)},
6172+
)
6173+
)
6174+
with table.update_schema() as update:
6175+
update.update_column("s.x", DoubleType())
6176+
check(
6177+
catalog.load_table("default.promotion_nested"),
6178+
[("s.x > 0.1", [1, 2]), ("s.x != 0.1", [1, 2]), ("s.x = 0.1", []), ("s.x < 1.0", [1])],
6179+
)
6180+
6181+
float_data = pa.table({"id": pa.array([1, 2], pa.int64()), "f": pa.array([0.1, 2.5], pa.float32())})
6182+
float_schema = Schema(
6183+
NestedField(1, "id", LongType(), required=False),
6184+
NestedField(2, "f", FloatType(), required=False),
6185+
)
6186+
6187+
# Renamed column: the file keeps the old name
6188+
table = catalog.create_table("default.promotion_renamed", schema=float_schema)
6189+
table.append(float_data)
6190+
with table.update_schema() as update:
6191+
update.update_column("f", DoubleType())
6192+
with table.update_schema() as update:
6193+
update.rename_column("f", "g")
6194+
check(
6195+
catalog.load_table("default.promotion_renamed"),
6196+
[("g > 0.1", [1, 2]), ("g != 0.1", [1, 2]), ("g = 0.1", [])],
6197+
)
6198+
6199+
# Column added after the file was written, combined with the promoted column
6200+
table = catalog.create_table("default.promotion_added", schema=float_schema)
6201+
table.append(float_data)
6202+
with table.update_schema() as update:
6203+
update.update_column("f", DoubleType())
6204+
update.add_column("n", StringType())
6205+
check(
6206+
catalog.load_table("default.promotion_added"),
6207+
[
6208+
("f > 0.1 AND n IS NULL", [1, 2]),
6209+
("f > 0.1 AND n IS NOT NULL", []),
6210+
("f > 0.1 OR n = 'a'", [1, 2]),
6211+
("NOT (f > 0.1 AND n = 'a')", [1, 2]),
6212+
("f = 0.1 OR n IS NULL", [1, 2]),
6213+
("f < 1.0 AND n IS NULL", [1]),
6214+
],
6215+
)
6216+
6217+
# Files written before and after the promotion in the same scan
6218+
table = catalog.create_table("default.promotion_mixed", schema=float_schema)
6219+
table.append(float_data)
6220+
with table.update_schema() as update:
6221+
update.update_column("f", DoubleType())
6222+
table = catalog.load_table("default.promotion_mixed")
6223+
table.append(pa.table({"id": pa.array([3, 4], pa.int64()), "f": pa.array([0.1, 2.5], pa.float64())}))
6224+
check(
6225+
table,
6226+
[("f > 0.1", [1, 2, 4]), ("f != 0.1", [1, 2, 4]), ("f = 0.1", [3]), ("f <= 0.1", [3])],
6227+
)
6228+
6229+
# Nulls in the promoted column
6230+
table = catalog.create_table("default.promotion_nulls", schema=float_schema)
6231+
table.append(pa.table({"id": pa.array([1, 2, 3], pa.int64()), "f": pa.array([0.1, None, 2.5], pa.float32())}))
6232+
with table.update_schema() as update:
6233+
update.update_column("f", DoubleType())
6234+
check(
6235+
catalog.load_table("default.promotion_nulls"),
6236+
[
6237+
("f != 0.1", [1, 3]),
6238+
("f > 0.1", [1, 3]),
6239+
("f IS NULL", [2]),
6240+
("f IS NOT NULL", [1, 3]),
6241+
("NOT (f > 0.1)", []),
6242+
("f NOT IN (2.5)", [1]),
6243+
("f IS NULL OR f > 0.1", [1, 2, 3]),
6244+
],
6245+
)
6246+
6247+
6248+
def _scan_ids_with_promoted_float(
6249+
tmpdir: str,
6250+
table_schema: Schema,
6251+
row_filter: BooleanExpression,
6252+
partition_spec: PartitionSpec | None = None,
6253+
partition: Record | None = None,
6254+
positional_deletes: list[pa.ChunkedArray] | None = None,
6255+
) -> list[int]:
6256+
"""Scan a file written with a float32 `f` column through a table schema where `f` is a double."""
6257+
arrow_schema = pa.schema(
6258+
[
6259+
pa.field("id", pa.int64(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),
6260+
pa.field("f", pa.float32(), metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}),
6261+
]
6262+
)
6263+
file_path = _write_table_to_file(
6264+
f"{tmpdir}/promoted_float.parquet",
6265+
arrow_schema,
6266+
pa.table({"id": [10, 20, 30, 40], "f": pa.array([0.1, 0.1, 2.5, 0.1], pa.float32())}, schema=arrow_schema),
6267+
)
6268+
data_file = DataFile.from_args(
6269+
content=DataFileContent.DATA,
6270+
file_path=file_path,
6271+
file_format=FileFormat.PARQUET,
6272+
partition=partition if partition is not None else Record(),
6273+
record_count=4,
6274+
file_size_in_bytes=os.path.getsize(file_path),
6275+
)
6276+
batches = _task_to_record_batches(
6277+
PyArrowFileIO(),
6278+
FileScanTask(data_file),
6279+
bound_row_filter=bind(table_schema, row_filter, case_sensitive=True),
6280+
projected_schema=table_schema,
6281+
table_schema=table_schema,
6282+
projected_field_ids={field.field_id for field in table_schema.fields},
6283+
positional_deletes=positional_deletes,
6284+
case_sensitive=True,
6285+
partition_spec=partition_spec,
6286+
)
6287+
return sorted(id_ for batch in batches for id_ in batch.column("id").to_pylist())
6288+
6289+
6290+
def test_row_filter_on_promoted_column_with_positional_deletes(tmpdir: str) -> None:
6291+
"""Positional deletes are applied before the table-typed filter on a promoted column (issue 4203)."""
6292+
table_schema = Schema(
6293+
NestedField(1, "id", LongType(), required=False),
6294+
NestedField(2, "f", DoubleType(), required=False),
6295+
)
6296+
# File positions 0-3 hold ids 10, 20, 30, 40; position 1 (id 20) is deleted
6297+
positional_deletes = [pa.chunked_array([pa.array([1], type=pa.int64())])]
6298+
6299+
for row_filter, expected in [
6300+
(GreaterThan("f", 0.1), [10, 30, 40]),
6301+
(NotEqualTo("f", 0.1), [10, 30, 40]),
6302+
(EqualTo("f", 0.1), []),
6303+
(LessThan("f", 1.0), [10, 40]),
6304+
]:
6305+
assert (
6306+
_scan_ids_with_promoted_float(tmpdir, table_schema, row_filter, positional_deletes=positional_deletes) == expected
6307+
), row_filter
6308+
6309+
6310+
def test_row_filter_on_promoted_column_with_projected_missing_column(tmpdir: str) -> None:
6311+
"""Predicates on columns missing from the file resolve to constants next to table-typed predicates (issue 4203)."""
6312+
# Value of an identity-partitioned column that is not stored in the file
6313+
partitioned_schema = Schema(
6314+
NestedField(1, "id", LongType(), required=False),
6315+
NestedField(2, "f", DoubleType(), required=False),
6316+
NestedField(3, "p", IntegerType(), required=False),
6317+
)
6318+
spec = PartitionSpec(PartitionField(3, 1000, IdentityTransform(), "p"))
6319+
for row_filter, expected in [
6320+
(And(GreaterThan("f", 0.1), EqualTo("p", 5)), [10, 20, 30, 40]),
6321+
(And(GreaterThan("f", 1.0), EqualTo("p", 5)), [30]),
6322+
(And(GreaterThan("f", 0.1), EqualTo("p", 6)), []),
6323+
(Or(EqualTo("f", 0.1), EqualTo("p", 6)), []),
6324+
(Or(EqualTo("f", 0.1), EqualTo("p", 5)), [10, 20, 30, 40]),
6325+
(Or(GreaterThan("f", 1.0), EqualTo("p", 6)), [30]),
6326+
]:
6327+
assert _scan_ids_with_promoted_float(tmpdir, partitioned_schema, row_filter, spec, Record(5)) == expected, row_filter
6328+
6329+
# initial-default of a column added after the file was written
6330+
default_schema = Schema(
6331+
NestedField(1, "id", LongType(), required=False),
6332+
NestedField(2, "f", DoubleType(), required=False),
6333+
NestedField(3, "d", IntegerType(), required=False, initial_default=7),
6334+
)
6335+
for row_filter, expected in [
6336+
(And(GreaterThan("f", 0.1), EqualTo("d", 7)), [10, 20, 30, 40]),
6337+
(And(GreaterThan("f", 1.0), EqualTo("d", 7)), [30]),
6338+
(And(GreaterThan("f", 0.1), EqualTo("d", 8)), []),
6339+
(Or(EqualTo("f", 0.1), EqualTo("d", 8)), []),
6340+
(Or(EqualTo("f", 0.1), EqualTo("d", 7)), [10, 20, 30, 40]),
6341+
(Or(GreaterThan("f", 1.0), EqualTo("d", 8)), [30]),
6342+
]:
6343+
assert _scan_ids_with_promoted_float(tmpdir, default_schema, row_filter) == expected, row_filter
6344+
6345+
6346+
def test_delete_matches_scan_after_type_promotion(catalog: InMemoryCatalog) -> None:
6347+
"""`delete` removes exactly the rows a scan with the same filter returns after a type promotion (issue 4203)."""
6348+
catalog.create_namespace("default")
6349+
table = catalog.create_table(
6350+
"default.promotion_delete",
6351+
schema=Schema(
6352+
NestedField(1, "id", LongType(), required=False),
6353+
NestedField(2, "f", FloatType(), required=False),
6354+
),
6355+
)
6356+
table.append(pa.table({"id": pa.array([1, 2, 3], pa.int64()), "f": pa.array([0.1, 2.5, 0.0], pa.float32())}))
6357+
with table.update_schema() as update:
6358+
update.update_column("f", DoubleType())
6359+
table = catalog.load_table("default.promotion_delete")
6360+
6361+
matched = sorted(table.scan(row_filter="f > 0.1").to_arrow()["id"].to_pylist())
6362+
assert matched == [1, 2]
6363+
6364+
table.delete("f > 0.1")
6365+
assert table.scan().to_arrow()["id"].to_pylist() == [3]

0 commit comments

Comments
 (0)