Skip to content

Commit ff5f5d4

Browse files
authored
Add ManifestEntry.is_live() (#4351)
Mirror Java's ManifestEntry.isLive() and use it in place of raw status comparisons, as suggested in review of #3972.
1 parent 8a79207 commit ff5f5d4

3 files changed

Lines changed: 16 additions & 4 deletions

File tree

‎pyiceberg/manifest.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -675,6 +675,10 @@ def data_file(self) -> DataFile:
675675
def data_file(self, value: DataFile) -> None:
676676
self._data[4] = value
677677

678+
def is_live(self) -> bool:
679+
"""Return whether the entry is live (added or existing) in the snapshot."""
680+
return self.status in (ManifestEntryStatus.ADDED, ManifestEntryStatus.EXISTING)
681+
678682

679683
PARTITION_FIELD_SUMMARY_TYPE = StructType(
680684
NestedField(509, "contains_null", BooleanType(), required=True),
@@ -933,15 +937,15 @@ def fetch_manifest_entry(
933937
next_row_id = self.first_row_id if self.content == ManifestContent.DATA else None
934938

935939
for entry in reader:
936-
if discard_deleted and entry.status == ManifestEntryStatus.DELETED:
940+
if discard_deleted and not entry.is_live():
937941
continue
938942
_inherit_from_manifest(entry, self)
939943

940944
# Inherit first row IDs, which are only valid when a data manifest has a first_row_id
941945
data_file = entry.data_file
942946
if next_row_id is None:
943947
data_file.first_row_id = None
944-
elif entry.status != ManifestEntryStatus.DELETED and data_file.first_row_id is None:
948+
elif entry.is_live() and data_file.first_row_id is None:
945949
data_file.first_row_id = next_row_id
946950
next_row_id += data_file.record_count
947951

@@ -1254,7 +1258,7 @@ def add_entry(self, entry: ManifestEntry) -> ManifestWriter:
12541258
self._partitions.append(entry.data_file.partition)
12551259

12561260
if (
1257-
(entry.status == ManifestEntryStatus.ADDED or entry.status == ManifestEntryStatus.EXISTING)
1261+
entry.is_live()
12581262
and entry.sequence_number is not None
12591263
and (self._min_sequence_number is None or entry.sequence_number < self._min_sequence_number)
12601264
):

‎pyiceberg/table/update/snapshot.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -980,7 +980,7 @@ def _create_manifest(self, spec_id: int, manifest_bin: list[ManifestFile]) -> Ma
980980
elif entry.status == ManifestEntryStatus.ADDED and entry.snapshot_id == self._snapshot_producer.snapshot_id:
981981
# added entries from this snapshot are still added, otherwise they should be existing
982982
writer.add(entry)
983-
elif entry.status != ManifestEntryStatus.DELETED:
983+
elif entry.is_live():
984984
# add all non-deleted files from the old manifest as existing files
985985
writer.existing(entry)
986986

‎tests/utils/test_manifest.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -236,6 +236,14 @@ def test_fetch_manifest_entry_with_filter(generated_manifest_entry_file: str) ->
236236
assert len(no_match) == 0
237237

238238

239+
@pytest.mark.parametrize(
240+
"status, expected",
241+
[(ManifestEntryStatus.ADDED, True), (ManifestEntryStatus.EXISTING, True), (ManifestEntryStatus.DELETED, False)],
242+
)
243+
def test_manifest_entry_is_live(status: ManifestEntryStatus, expected: bool) -> None:
244+
assert ManifestEntry.from_args(status=status).is_live() is expected
245+
246+
239247
def test_fetch_manifest_entry_inherits_first_row_id(tmp_path: Path) -> None:
240248
"""Data files inherit first row IDs from data manifests in file order, and have them cleared otherwise."""
241249
io = PyArrowFileIO()

0 commit comments

Comments
 (0)