Skip to content

Commit eeef9cd

Browse files
committed
Fix sort_indices usage for pyarrow 25
1 parent c93a427 commit eeef9cd

1 file changed

Lines changed: 16 additions & 4 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@
5050
TypeVar,
5151
cast,
5252
)
53+
from typing import (
54+
Literal as LiteralType,
55+
)
5356
from urllib.parse import urlparse
5457

5558
import pyarrow as pa
@@ -3059,11 +3062,13 @@ def _dataframe_to_data_files(
30593062

30603063
def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tuple[pa.Table, int | None]:
30613064
"""Apply a supported Iceberg sort order and return its file sort-order id."""
3065+
from packaging import version
3066+
30623067
sort_order = table_metadata.sort_order()
30633068
if sort_order.is_unsorted:
30643069
return table, None
30653070

3066-
sort_keys: list[tuple[str, str]] = []
3071+
sort_keys: list[tuple[str, LiteralType["ascending", "descending"]]] = []
30673072
null_orders = set()
30683073
for field in sort_order.fields:
30693074
if not isinstance(field.transform, IdentityTransform):
@@ -3079,15 +3084,22 @@ def _sort_table_for_write(table_metadata: TableMetadata, table: pa.Table) -> tup
30793084
stacklevel=2,
30803085
)
30813086
return table, None
3082-
direction = "descending" if field.direction == SortDirection.DESC else "ascending"
3087+
direction: LiteralType["ascending", "descending"] = "descending" if field.direction == SortDirection.DESC else "ascending"
30833088
sort_keys.append((name, direction))
30843089
null_orders.add(field.null_order)
30853090

30863091
if len(null_orders) != 1:
30873092
warnings.warn("Mixed null ordering is not supported; data files are marked unsorted", stacklevel=2)
30883093
return table, None
3089-
null_placement = "at_start" if null_orders == {NullOrder.NULLS_FIRST} else "at_end"
3090-
indices = pc.sort_indices(table, sort_keys=sort_keys, null_placement=null_placement)
3094+
null_placement: LiteralType["at_start", "at_end"] = "at_start" if null_orders == {NullOrder.NULLS_FIRST} else "at_end"
3095+
if version.parse(pyarrow.__version__) < version.parse("25.0.0"):
3096+
indices = pc.sort_indices(table, sort_keys=sort_keys, null_placement=null_placement)
3097+
else:
3098+
# Per-key null placement replaced the SortOptions-level argument in Arrow 25
3099+
indices = pc.sort_indices(
3100+
table,
3101+
sort_keys=cast(Any, [(name, order, null_placement) for name, order in sort_keys]),
3102+
)
30913103
return table.take(indices), sort_order.order_id
30923104

30933105

0 commit comments

Comments
 (0)