Skip to content

Commit 7aef85a

Browse files
committed
Distinguish unknown transforms by name and compare every source id in compatible_with
1 parent 7e728ee commit 7aef85a

5 files changed

Lines changed: 56 additions & 2 deletions

File tree

‎pyiceberg/partitioning.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def compatible_with(self, other: PartitionSpec) -> bool:
191191
if len(self.fields) != len(other.fields):
192192
return False
193193
return all(
194-
this_field.source_id == that_field.source_id
194+
this_field.transform_arguments == that_field.transform_arguments
195195
and this_field.transform == that_field.transform
196196
and this_field.name == that_field.name
197197
for this_field, that_field in zip(self.fields, other.fields, strict=True)

‎pyiceberg/transforms.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1003,6 +1003,16 @@ def __repr__(self) -> str:
10031003
"""Return the string representation of the UnknownTransform class."""
10041004
return f"UnknownTransform(transform={repr(self._transform)})"
10051005

1006+
def __eq__(self, other: Any) -> bool:
1007+
"""Compare the preserved transform name, since every unknown transform shares one root value."""
1008+
if isinstance(other, UnknownTransform):
1009+
return self._transform == other._transform
1010+
return False
1011+
1012+
def __hash__(self) -> int:
1013+
"""Hash the preserved transform name so distinct unknown transforms do not collide."""
1014+
return hash((self.root, self._transform))
1015+
10061016
def pyarrow_transform(self, source: IcebergType) -> "Callable[[pa.Array], pa.Array]":
10071017
raise NotImplementedError()
10081018

‎pyiceberg/utils/parsing.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ class ParseNumberFromBrackets:
2828

2929
def __init__(self, prefix: str):
3030
self.prefix = prefix
31-
self.regex = re.compile(rf"{prefix}\[(\d+)\]")
31+
# anchored: a name such as truncate[8]v2 is a different transform, not truncate[8]
32+
self.regex = re.compile(rf"^{prefix}\[(\d+)\]$")
3233

3334
def match(self, str_repr: str) -> int:
3435
matches = self.regex.search(str_repr)

‎tests/table/test_partitioning.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,31 @@ def test_partition_compatible_with() -> None:
105105
assert not lhs.compatible_with(rhs)
106106

107107

108+
def test_partition_compatible_with_compares_every_source_id() -> None:
109+
# multi-argument fields differ in their later source ids, which the first one cannot show
110+
lhs = PartitionSpec.model_validate(
111+
{"spec-id": 0, "fields": [{"field-id": 1000, "name": "p", "source-ids": [1, 2], "transform": "zorder(a,b)"}]}
112+
)
113+
rhs = PartitionSpec.model_validate(
114+
{"spec-id": 0, "fields": [{"field-id": 1000, "name": "p", "source-ids": [1, 3], "transform": "zorder(a,c)"}]}
115+
)
116+
117+
assert not lhs.compatible_with(rhs)
118+
assert lhs.compatible_with(lhs)
119+
120+
121+
def test_partition_compatible_with_compares_unknown_transform_names() -> None:
122+
# same source ids, different transform: the names are all that separates them
123+
lhs = PartitionSpec.model_validate(
124+
{"spec-id": 0, "fields": [{"field-id": 1000, "name": "p", "source-ids": [1, 2], "transform": "zorder(a,b)"}]}
125+
)
126+
rhs = PartitionSpec.model_validate(
127+
{"spec-id": 0, "fields": [{"field-id": 1000, "name": "p", "source-ids": [1, 2], "transform": "somethingelse(a,b)"}]}
128+
)
129+
130+
assert not lhs.compatible_with(rhs)
131+
132+
108133
def test_unpartitioned() -> None:
109134
assert len(UNPARTITIONED_PARTITION_SPEC.fields) == 0
110135
assert UNPARTITIONED_PARTITION_SPEC.is_unpartitioned()

‎tests/test_transforms.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,24 @@ def test_parse_transform_unknown_with_known_prefix() -> None:
591591
assert str(transform) == name
592592

593593

594+
def test_parse_transform_rejects_trailing_text_after_brackets() -> None:
595+
# a name that merely starts with a known transform is a different transform, so it has to be
596+
# preserved rather than silently rewritten to the known one
597+
from pyiceberg.transforms import parse_transform
598+
599+
for name in ("bucket[4]garbage", "truncate[8]v2"):
600+
transform = parse_transform(name)
601+
assert isinstance(transform, UnknownTransform), name
602+
assert str(transform) == name
603+
604+
605+
def test_unknown_transforms_compare_by_name() -> None:
606+
# every unknown transform shares the same root value, so equality has to use the preserved name
607+
assert UnknownTransform("zorder(x,y)") == UnknownTransform("zorder(x,y)")
608+
assert UnknownTransform("zorder(x,y)") != UnknownTransform("bucketv2[4]")
609+
assert len({UnknownTransform("zorder(x,y)"), UnknownTransform("bucketv2[4]")}) == 2
610+
611+
594612
def test_unknown_transform_repr() -> None:
595613
assert repr(UnknownTransform("unknown")) == "UnknownTransform(transform='unknown')"
596614

0 commit comments

Comments
 (0)