Skip to content

Commit 6c2b0fb

Browse files
committed
Keep branch scans on the current table schema
Preserve branch context when copying scans so schema changes do not make branch filters read old field IDs or reject renamed columns. Tags and explicit snapshot scans continue to use their historical schemas.
1 parent 98922db commit 6c2b0fb

2 files changed

Lines changed: 65 additions & 9 deletions

File tree

‎pyiceberg/table/__init__.py‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
from pyiceberg.table.maintenance import MaintenanceTable
6464
from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata
6565
from pyiceberg.table.name_mapping import NameMapping
66-
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef
66+
from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType
6767
from pyiceberg.table.snapshots import (
6868
Operation,
6969
Snapshot,
@@ -333,7 +333,7 @@ def _apply(
333333

334334
def _scan(self, row_filter: str | BooleanExpression = ALWAYS_TRUE, case_sensitive: bool = True) -> DataScan:
335335
"""Minimal data scan of the table with the current state of the transaction."""
336-
return _TransactionScan(
336+
return DataScan(
337337
table_metadata=self.table_metadata, io=self._table.io, row_filter=row_filter, case_sensitive=case_sensitive
338338
)
339339

@@ -2209,6 +2209,7 @@ class TableScan(BaseScan):
22092209
"""A base class for table scans targeting a single snapshot."""
22102210

22112211
snapshot_id: int | None
2212+
_branch_name: str | None
22122213
catalog: Catalog | None
22132214
table_identifier: Identifier | None
22142215
table_config: Properties
@@ -2227,6 +2228,8 @@ def __init__(
22272228
table_identifier: Identifier | None = None,
22282229
table_config: Properties = EMPTY_DICT,
22292230
encryption_manager: EncryptionManager | None = None,
2231+
*,
2232+
_branch_name: str | None = None,
22302233
):
22312234
super().__init__(
22322235
table_metadata=table_metadata,
@@ -2239,6 +2242,7 @@ def __init__(
22392242
encryption_manager=encryption_manager,
22402243
)
22412244
self.snapshot_id = snapshot_id
2245+
self._branch_name = _branch_name
22422246
self.catalog = catalog
22432247
self.table_identifier = table_identifier
22442248
self.table_config = table_config
@@ -2250,7 +2254,7 @@ def snapshot(self) -> Snapshot | None:
22502254

22512255
def _scan_schema(self) -> Schema:
22522256
current_schema = self.table_metadata.schema()
2253-
if self.snapshot_id is not None:
2257+
if self.snapshot_id is not None and self._branch_name is None:
22542258
snapshot = self.table_metadata.snapshot_by_id(self.snapshot_id)
22552259
if snapshot is not None:
22562260
if snapshot.schema_id is not None:
@@ -2273,10 +2277,15 @@ def projection(self) -> Schema:
22732277
return current_schema.select(*self.selected_fields, case_sensitive=self.case_sensitive)
22742278

22752279
def use_ref(self: S, name: str) -> S:
2280+
"""Select a branch using the table schema or a tag using its snapshot schema."""
22762281
if self.snapshot_id is not None:
22772282
raise ValueError(f"Cannot override ref, already set snapshot id={self.snapshot_id}")
22782283
if snapshot := self.table_metadata.snapshot_by_name(name):
2279-
return self.update(snapshot_id=snapshot.snapshot_id)
2284+
ref = self.table_metadata.refs[name]
2285+
return self.update(
2286+
snapshot_id=snapshot.snapshot_id,
2287+
_branch_name=name if ref.snapshot_ref_type == SnapshotRefType.BRANCH else None,
2288+
)
22802289

22812290
raise ValueError(f"Cannot scan unknown ref={name}")
22822291

@@ -2624,11 +2633,6 @@ def count(self) -> int:
26242633
return res
26252634

26262635

2627-
class _TransactionScan(DataScan):
2628-
def _scan_schema(self) -> Schema:
2629-
return self.table_metadata.schema()
2630-
2631-
26322636
IAS = TypeVar("IAS", bound="IncrementalAppendScan", covariant=True)
26332637

26342638

‎tests/table/test_init.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,58 @@ def test_time_travel_row_filter_after_drop_and_readd(catalog: Catalog, arrow_tab
415415
assert table.scan(row_filter="bar IS NULL").count() == 3
416416

417417

418+
def test_branch_row_filter_after_drop_and_readd(catalog: Catalog, arrow_table_simple: pa.Table) -> None:
419+
catalog.create_namespace("default")
420+
table = catalog.create_table("default.branch_column_replacement", schema=arrow_table_simple.schema)
421+
table.append(arrow_table_simple)
422+
snapshot = table.current_snapshot()
423+
assert snapshot is not None
424+
table.manage_snapshots().create_branch(snapshot.snapshot_id, "audit").create_tag(snapshot.snapshot_id, "before").commit()
425+
426+
with table.update_schema() as update:
427+
update.delete_column("bar")
428+
with table.update_schema() as update:
429+
update.add_column("bar", IntegerType())
430+
431+
expected = {"foo": ["a", "b", "c"], "baz": [True, False, True], "bar": [None, None, None]}
432+
for branch in (MAIN_BRANCH, "audit"):
433+
scan = table.scan().use_ref(branch)
434+
filtered = scan.with_case_sensitive(False).filter("BAR >= 2").select("foo")
435+
assert filtered.to_arrow().to_pydict() == {"foo": []}
436+
assert filtered.to_arrow_batch_reader().read_all().to_pydict() == {"foo": []}
437+
assert filtered.count() == 0
438+
assert scan.projection() == table.schema()
439+
assert scan.to_arrow().to_pydict() == expected
440+
assert scan.to_arrow_batch_reader().read_all().to_pydict() == expected
441+
assert scan.filter("bar IS NULL").count() == 3
442+
443+
for scan in (table.scan(snapshot_id=snapshot.snapshot_id), table.scan().use_ref("before")):
444+
assert scan.to_arrow().to_pydict() == arrow_table_simple.to_pydict()
445+
assert scan.filter("bar >= 2").select("foo").to_arrow().to_pydict() == {"foo": ["b", "c"]}
446+
447+
448+
def test_branch_row_filter_after_column_rename(catalog: Catalog, arrow_table_simple: pa.Table) -> None:
449+
catalog.create_namespace("default")
450+
table = catalog.create_table("default.branch_column_rename", schema=arrow_table_simple.schema)
451+
table.append(arrow_table_simple)
452+
snapshot = table.current_snapshot()
453+
assert snapshot is not None
454+
table.manage_snapshots().create_branch(snapshot.snapshot_id, "audit").commit()
455+
with table.update_schema() as update:
456+
update.rename_column("bar", "score")
457+
458+
for branch in (MAIN_BRANCH, "audit"):
459+
scan = table.scan().use_ref(branch)
460+
filtered = scan.with_case_sensitive(False).filter("SCORE >= 2").select("foo")
461+
assert filtered.to_arrow().to_pydict() == {"foo": ["b", "c"]}
462+
assert filtered.to_arrow_batch_reader().read_all().to_pydict() == {"foo": ["b", "c"]}
463+
assert filtered.count() == 2
464+
assert scan.projection() == table.schema()
465+
assert scan.to_arrow().to_pydict() == {"foo": ["a", "b", "c"], "score": [1, 2, 3], "baz": [True, False, True]}
466+
with pytest.raises(ValueError, match="Could not find field with name bar"):
467+
scan.filter("bar >= 2").to_arrow()
468+
469+
418470
def test_time_travel_row_filter_after_partition_column_rename(catalog: Catalog, arrow_table_simple: pa.Table) -> None:
419471
catalog.create_namespace("default")
420472
table = catalog.create_table("default.renamed_partition_column", schema=arrow_table_simple.schema)

0 commit comments

Comments
 (0)