Skip to content

Commit f254cfc

Browse files
PyArrow: Keep the row count when a scan selects no columns
Closes #4137 Generated-by: AI assistant
1 parent 042165a commit f254cfc

2 files changed

Lines changed: 22 additions & 0 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1888,6 +1888,10 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
18881888
arrow_schema = schema_to_pyarrow(self._projected_schema, include_field_ids=False)
18891889

18901890
batches = self.to_record_batches(tasks)
1891+
if not self._projected_schema.fields:
1892+
# pa.concat_tables drops the rows of tables without columns
1893+
return pa.Table.from_batches(batches, schema=arrow_schema)
1894+
18911895
try:
18921896
first_batch = next(batches)
18931897
except StopIteration:
@@ -1990,6 +1994,10 @@ def _to_requested_schema(
19901994
allow_timestamp_tz_mismatch: bool = False,
19911995
format_model: FileFormatModel | None = None,
19921996
) -> pa.RecordBatch:
1997+
if not requested_schema.fields:
1998+
# A struct array without fields has length 0, so keep the rows of the batch instead
1999+
return batch.select([])
2000+
19932001
# We could reuse some of these visitors
19942002
struct_array = visit_with_partner(
19952003
requested_schema,

‎tests/io/test_pyarrow.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1336,6 +1336,20 @@ def test_projection_concat_files(schema_int: Schema, file_int: str) -> None:
13361336
assert repr(result_table.schema) == "id: int32"
13371337

13381338

1339+
def test_projection_no_columns_keeps_row_count(schema_int: Schema, file_int: str) -> None:
1340+
result_table = project(Schema(), [file_int, file_int], table_schema=schema_int)
1341+
1342+
assert result_table.num_columns == 0
1343+
assert result_table.num_rows == 6
1344+
1345+
1346+
def test_projection_no_columns_with_filter(schema_int: Schema, file_int: str) -> None:
1347+
result_table = project(Schema(), [file_int], expr=GreaterThan("id", 0), table_schema=schema_int)
1348+
1349+
assert result_table.num_columns == 0
1350+
assert result_table.num_rows == 2
1351+
1352+
13391353
@pytest.mark.parametrize("with_field_ids", [False, True], ids=["name-mapping", "field-ids"])
13401354
def test_identity_transform_column_projection(tmp_path: str, catalog: InMemoryCatalog, with_field_ids: bool) -> None:
13411355
# Test by adding a non-partitioned data file to a partitioned table, verifying partition value

0 commit comments

Comments
 (0)