diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..5961da883c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1952,11 +1952,17 @@ def __init__( @property def _projected_field_ids(self) -> set[int]: """Set of field IDs that should be projected from the data files.""" + schema = self._table_metadata.schema() + filter_field_ids = extract_field_ids(self._bound_row_filter) + # A list or map cannot be projected by its own ID, so a filter on one reads its nested fields instead + nested_field_ids = prune_columns( + schema, {id for id in filter_field_ids if isinstance(schema.find_type(id), (MapType, ListType))} + ).field_ids return { id for id in self._projected_schema.field_ids if not isinstance(self._projected_schema.find_type(id), (MapType, ListType)) - }.union(extract_field_ids(self._bound_row_filter)) + }.union(id for id in filter_field_ids | nested_field_ids if not isinstance(schema.find_type(id), (MapType, ListType))) def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table: """Scan the Iceberg table and return a pa.Table. diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index cd817512b4..4b8fea113e 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1429,6 +1429,25 @@ def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None: tbl.scan(row_filter="x > 2.6").to_arrow() +def test_scan_null_filter_on_list_and_map_columns(catalog: Catalog) -> None: + catalog.create_namespace("default") + data = pa.table( + { + "id": pa.array([1, 2, 3], pa.int32()), + "tags": pa.array([["a"], [], None], pa.list_(pa.string())), + "props": pa.array([[("k", 1)], [], None], pa.map_(pa.string(), pa.int32())), + "points": pa.array([[{"x": 1}], [], None], pa.list_(pa.struct([("x", pa.int32())]))), + } + ) + tbl = catalog.create_table(f"default.scan_null_filter_list_map_{catalog.name}", schema=data.schema) + tbl.append(data) + + for column in ("tags", "props", "points"): + assert tbl.scan(row_filter=f"{column} is null").to_arrow()["id"].to_pylist() == [3], column + assert sorted(tbl.scan(row_filter=f"{column} is not null").to_arrow()["id"].to_pylist()) == [1, 2], column + assert tbl.scan(row_filter=f"{column} is null", selected_fields=("id",)).to_arrow().to_pydict() == {"id": [3]}, column + + def test_scan_negated_comparison_matches_file_pruning_for_nan(catalog: Catalog) -> None: catalog.create_namespace("default") identifier = f"default.scan_negated_comparison_nan_{catalog.name}"