Skip to content

Commit 5d043a2

Browse files
authored
Row lineage: Preserve first_row_id from REST scan planning (#4352)
Preserve first_row_id from REST scan planning The REST data file model parses first-row-id, but it was dropped when converting to a DataFile, so planned scans lost row lineage info. Part of #4080.
1 parent 2317c25 commit 5d043a2

2 files changed

Lines changed: 9 additions & 0 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2418,6 +2418,7 @@ def _rest_file_to_data_file(rest_file: RESTContentFile, default_referenced_data_
24182418
referenced_data_file=referenced_data_file,
24192419
content_offset=rest_file.content_offset if isinstance(rest_file, RESTPositionDeleteFile) else None,
24202420
content_size_in_bytes=rest_file.content_size_in_bytes if isinstance(rest_file, RESTPositionDeleteFile) else None,
2421+
first_row_id=rest_file.first_row_id if isinstance(rest_file, RESTDataFile) else None,
24212422
)
24222423
data_file.spec_id = rest_file.spec_id
24232424
return data_file

‎tests/catalog/test_scan_planning_models.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -564,6 +564,14 @@ def test_plan_scan_with_shared_dv_file(
564564
}
565565

566566

567+
@pytest.mark.parametrize("first_row_id", [1000, None])
568+
def test_rest_task_preserves_first_row_id(first_row_id: int | None) -> None:
569+
rest_data_file = {**_rest_data_file(), **({"first-row-id": first_row_id} if first_row_id is not None else {})}
570+
task = FileScanTask.from_rest_response(RESTFileScanTask.model_validate({"data-file": rest_data_file}), [])
571+
572+
assert task.file.first_row_id == first_row_id
573+
574+
567575
@pytest.mark.parametrize("missing_field", ["content-offset", "content-size-in-bytes"])
568576
@pytest.mark.parametrize("include_referenced_data_file", [True, False])
569577
def test_rest_task_preserves_partial_dv_reference(missing_field: str, include_referenced_data_file: bool) -> None:

0 commit comments

Comments
 (0)