Skip to content

Commit 69a5bae

Browse files
Fix row ID scan with a filter when a column is named __iceberg_file_position
When `_row_id` is projected together with a row filter, the reader adds a temporary position column under a fixed name. A data column with that name made the lookup fail with a duplicate field KeyError. Pick a free name for the temporary column instead, as `_determine_partitions` does.
1 parent b84df8d commit 69a5bae

2 files changed

Lines changed: 48 additions & 3 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1842,12 +1842,16 @@ def _task_to_record_batches(
18421842
# (https://github.com/apache/arrow/pull/46057): RecordBatch.filter raises
18431843
# IndexError on PyArrow <21 when the result is empty; Table.filter does not.
18441844
table = pa.Table.from_batches([current_batch])
1845+
# Avoid conflicts with the data columns
1846+
position_column = _POSITION_COLUMN
1847+
while position_column in table.column_names:
1848+
position_column = f"_{position_column}"
18451849
if positions is not None:
1846-
table = table.append_column(_POSITION_COLUMN, positions)
1850+
table = table.append_column(position_column, positions)
18471851
table = table.filter(pyarrow_filter)
18481852
if positions is not None:
1849-
positions = table.column(_POSITION_COLUMN).combine_chunks()
1850-
table = table.drop_columns([_POSITION_COLUMN])
1853+
positions = table.column(position_column).combine_chunks()
1854+
table = table.drop_columns([position_column])
18511855
if table.num_rows == 0:
18521856
current_batch = current_batch.slice(0, 0)
18531857
else:

‎tests/io/test_pyarrow.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4645,6 +4645,47 @@ def test_task_to_record_batches_row_ids(
46454645
assert list(zip(result["id"].to_pylist(), result[ROW_ID.name].to_pylist(), strict=True)) == expected
46464646

46474647

4648+
def test_task_to_record_batches_row_ids_with_position_column_name(tmpdir: str) -> None:
4649+
from pyiceberg.expressions.visitors import bind
4650+
4651+
arrow_schema = pa.schema(
4652+
[
4653+
pa.field("id", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "1"}),
4654+
pa.field("__iceberg_file_position", pa.int32(), nullable=True, metadata={PYARROW_PARQUET_FIELD_ID_KEY: "2"}),
4655+
]
4656+
)
4657+
arrow_table = pa.table(
4658+
[pa.array([1, 2, 3], type=pa.int32()), pa.array([10, 20, 30], type=pa.int32())],
4659+
schema=arrow_schema,
4660+
)
4661+
data_file = _write_table_to_data_file(f"{tmpdir}/row_ids_position_name.parquet", arrow_schema, arrow_table)
4662+
data_file.first_row_id = 100
4663+
4664+
table_schema = Schema(
4665+
NestedField(1, "id", IntegerType(), required=False),
4666+
NestedField(2, "__iceberg_file_position", IntegerType(), required=False),
4667+
)
4668+
projected_schema = Schema(*table_schema.fields, ROW_ID)
4669+
result_batches = list(
4670+
_task_to_record_batches(
4671+
PyArrowFileIO(),
4672+
FileScanTask(data_file),
4673+
bound_row_filter=bind(table_schema, GreaterThan("id", 1), case_sensitive=True),
4674+
projected_schema=projected_schema,
4675+
table_schema=table_schema,
4676+
projected_field_ids={1, 2, ROW_ID.field_id},
4677+
positional_deletes=None,
4678+
case_sensitive=True,
4679+
)
4680+
)
4681+
4682+
assert pa.Table.from_batches(result_batches).to_pydict() == {
4683+
"id": [2, 3],
4684+
"__iceberg_file_position": [20, 30],
4685+
ROW_ID.name: [101, 102],
4686+
}
4687+
4688+
46484689
def _read_last_updated_sequence_numbers(
46494690
tmpdir: str, stored_sequence_numbers: list[int | None] | None, first_row_id: int | None, data_sequence_number: int | None
46504691
) -> list[int | None]:

0 commit comments

Comments
 (0)