Skip to content

Commit 46d7835

Browse files
Read a list or map column referenced by a null filter (#4400)
* Read a list or map column referenced by a null filter A scan filter that tests a list or map column for null added the column's own field ID to the projected IDs, and prune_columns rejects a list or map selected by its own ID. Project the column's nested fields instead. * Test null filters on list and map columns through a scan
1 parent 0ebc58b commit 46d7835

2 files changed

Lines changed: 26 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1952,11 +1952,17 @@ def __init__(
19521952
@property
19531953
def _projected_field_ids(self) -> set[int]:
19541954
"""Set of field IDs that should be projected from the data files."""
1955+
schema = self._table_metadata.schema()
1956+
filter_field_ids = extract_field_ids(self._bound_row_filter)
1957+
# A list or map cannot be projected by its own ID, so a filter on one reads its nested fields instead
1958+
nested_field_ids = prune_columns(
1959+
schema, {id for id in filter_field_ids if isinstance(schema.find_type(id), (MapType, ListType))}
1960+
).field_ids
19551961
return {
19561962
id
19571963
for id in self._projected_schema.field_ids
19581964
if not isinstance(self._projected_schema.find_type(id), (MapType, ListType))
1959-
}.union(extract_field_ids(self._bound_row_filter))
1965+
}.union(id for id in filter_field_ids | nested_field_ids if not isinstance(schema.find_type(id), (MapType, ListType)))
19601966

19611967
def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
19621968
"""Scan the Iceberg table and return a pa.Table.

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1429,6 +1429,25 @@ def test_scan_integer_column_with_decimal_literal(catalog: Catalog) -> None:
14291429
tbl.scan(row_filter="x > 2.6").to_arrow()
14301430

14311431

1432+
def test_scan_null_filter_on_list_and_map_columns(catalog: Catalog) -> None:
1433+
catalog.create_namespace("default")
1434+
data = pa.table(
1435+
{
1436+
"id": pa.array([1, 2, 3], pa.int32()),
1437+
"tags": pa.array([["a"], [], None], pa.list_(pa.string())),
1438+
"props": pa.array([[("k", 1)], [], None], pa.map_(pa.string(), pa.int32())),
1439+
"points": pa.array([[{"x": 1}], [], None], pa.list_(pa.struct([("x", pa.int32())]))),
1440+
}
1441+
)
1442+
tbl = catalog.create_table(f"default.scan_null_filter_list_map_{catalog.name}", schema=data.schema)
1443+
tbl.append(data)
1444+
1445+
for column in ("tags", "props", "points"):
1446+
assert tbl.scan(row_filter=f"{column} is null").to_arrow()["id"].to_pylist() == [3], column
1447+
assert sorted(tbl.scan(row_filter=f"{column} is not null").to_arrow()["id"].to_pylist()) == [1, 2], column
1448+
assert tbl.scan(row_filter=f"{column} is null", selected_fields=("id",)).to_arrow().to_pydict() == {"id": [3]}, column
1449+
1450+
14321451
def test_scan_negated_comparison_matches_file_pruning_for_nan(catalog: Catalog) -> None:
14331452
catalog.create_namespace("default")
14341453
identifier = f"default.scan_negated_comparison_nan_{catalog.name}"

0 commit comments

Comments
 (0)