Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions pyiceberg/partitioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@
PARTITION_FIELD_ID_START: int = 1000


def _url_encode(value: str) -> str:
# Unlike quote_plus, java.net.URLEncoder leaves "*" as is and escapes "~"
return quote_plus(value, safe="*").replace("~", "%7E")


class PartitionField(IcebergBaseModel):
"""PartitionField represents how one partition value is derived from the source column via transformation.

Expand Down Expand Up @@ -248,8 +253,8 @@ def partition_to_path(self, data: Record, schema: Schema) -> str:
partition_field = self.fields[pos]
value_str = partition_field.transform.to_human_string(field_types[pos].field_type, value=data[pos])

value_strs.append(quote_plus(value_str, safe=""))
field_strs.append(quote_plus(partition_field.name, safe=""))
value_strs.append(_url_encode(value_str))
field_strs.append(_url_encode(partition_field.name))

path = "/".join([field_str + "=" + value_str for field_str, value_str in zip(field_strs, value_strs, strict=True)])
return path
Expand Down
8 changes: 8 additions & 0 deletions tests/table/test_partitioning.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,14 @@ def test_partition_spec_to_path() -> None:
assert spec.partition_to_path(record, schema) == "my%23str%25bucket=my%2Bstr/other+str%2Bbucket=%28+%29/my%21int%3Abucket=10"


def test_partition_spec_to_path_asterisk_and_tilde() -> None:
schema = Schema(NestedField(field_id=1, name="str", field_type=StringType(), required=False))
spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="my*str~field"))

# java.net.URLEncoder leaves "*" as is and escapes "~"
assert spec.partition_to_path(Record("a*b~c-d_e.f"), schema) == "my*str%7Efield=a*b%7Ec-d_e.f"


def test_partition_spec_to_path_dropped_source_id() -> None:
schema = Schema(
NestedField(field_id=1, name="str", field_type=StringType(), required=False),
Expand Down
Loading