Skip to content

Commit 80877d8

Browse files
authored
Merge branch 'main' into fix/retry-commit-window-per-producer
2 parents 60d0a7e + 639a3bf commit 80877d8

22 files changed

Lines changed: 822 additions & 36 deletions

‎mkdocs/docs/configuration.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,9 @@ as a single request timeout (floored to whole seconds). Retries are applied to i
449449
only (`GET`, `HEAD`, `OPTIONS`, `PUT`, `DELETE`) and to the transient HTTP status codes `429`, `500`,
450450
`502`, `503`, `504`. Other failures are not retried.
451451

452+
When the server advertises `idempotency-key-lifetime` in its config response, mutating requests carry an
453+
`Idempotency-Key` header and `POST` is retried as well, reusing the same key for each retry.
454+
452455
#### Headers in REST Catalog
453456

454457
To configure custom headers in REST Catalog, include them in the catalog properties with `header.<Header-Name>`. This

‎pyiceberg/avro/file.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,11 +88,10 @@ def compression_codec(self) -> type[Codec] | None:
8888
"""Get the file's compression codec algorithm from the file's metadata.
8989
9090
In the case of a null codec, we return a None indicating that we
91-
don't need to compress/decompress.
91+
don't need to compress/decompress. A file without the avro.codec
92+
entry is uncompressed: the Avro specification makes "null" the default.
9293
"""
93-
from pyiceberg.table import TableProperties
94-
95-
codec_name = self.meta.get(AVRO_CODEC_KEY, TableProperties.WRITE_AVRO_COMPRESSION_DEFAULT)
94+
codec_name = self.meta.get(AVRO_CODEC_KEY, "null")
9695
if codec_name not in KNOWN_CODECS:
9796
raise ValueError(f"Unsupported codec: {codec_name}")
9897

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717
from __future__ import annotations
1818

1919
import logging
20+
import os
21+
import sys
2022
import time
23+
import uuid
2124
from collections import deque
2225
from collections.abc import Mapping
2326
from enum import Enum, StrEnum
@@ -88,6 +91,7 @@
8891
TableIdentifier,
8992
TableProperties,
9093
)
94+
from pyiceberg.table.labels import Labels
9195
from pyiceberg.table.metadata import TableMetadata
9296
from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder, assign_fresh_sort_order_ids
9397
from pyiceberg.table.update import (
@@ -298,6 +302,7 @@ class ScanPlanningMode(Enum):
298302
# (e.g. setting raise_on_status=False would swallow 4xx errors silently).
299303
_CONNECTION_RETRY_STATUS_FORCELIST = (429, 500, 502, 503, 504)
300304
_CONNECTION_RETRY_ALLOWED_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE"})
305+
IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"
301306
EMPTY_BODY_SHA256: str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
302307
OAUTH2_SERVER_URI = "oauth2-server-uri"
303308
SNAPSHOT_LOADING_MODE = "snapshot-loading-mode"
@@ -339,6 +344,17 @@ def _parse_scan_planning_mode(properties: Properties, *, strict: bool = True) ->
339344
return None
340345

341346

347+
if sys.version_info >= (3, 14):
348+
from uuid import uuid7 as _uuid7
349+
else:
350+
351+
def _uuid7() -> uuid.UUID:
352+
value = (time.time_ns() // 1_000_000) << 80 | int.from_bytes(os.urandom(10))
353+
value = (value & ~(0xF << 76)) | (0x7 << 76)
354+
value = (value & ~(0x3 << 62)) | (0x2 << 62)
355+
return uuid.UUID(int=value)
356+
357+
342358
def _retry_hook(retry_state: RetryCallState) -> None:
343359
rest_catalog: RestCatalog = retry_state.args[0]
344360
rest_catalog._refresh_token() # pylint: disable=protected-access
@@ -357,6 +373,7 @@ class TableResponse(IcebergBaseModel):
357373
metadata: TableMetadata
358374
config: Properties = Field(default_factory=dict)
359375
storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list)
376+
labels: Labels = Field(default_factory=Labels)
360377

361378

362379
class LoadCredentialsResponse(IcebergBaseModel):
@@ -367,6 +384,7 @@ class ViewResponse(IcebergBaseModel):
367384
metadata_location: str = Field(..., alias="metadata-location")
368385
metadata: ViewMetadata
369386
config: Properties = Field(default_factory=dict)
387+
labels: Labels = Field(default_factory=Labels)
370388

371389

372390
class CreateTableRequest(IcebergBaseModel):
@@ -421,6 +439,7 @@ class ConfigResponse(IcebergBaseModel):
421439
defaults: Properties | None = Field(default_factory=dict)
422440
overrides: Properties | None = Field(default_factory=dict)
423441
endpoints: set[Endpoint] | None = Field(default=None)
442+
idempotency_key_lifetime: str | None = Field(alias="idempotency-key-lifetime", default=None)
424443

425444
@field_validator("endpoints", mode="before")
426445
@classmethod
@@ -502,11 +521,12 @@ def send(
502521
return super().send(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
503522

504523

505-
def _create_connection_adapter(properties: Properties) -> _RetryTimeoutHTTPAdapter | None:
524+
def _create_connection_adapter(properties: Properties, retry_post: bool = False) -> _RetryTimeoutHTTPAdapter | None:
506525
"""Build a connection adapter from the optional `rest.client.*` properties.
507526
508527
Returns None when no connection properties are supplied, leaving the default
509-
Session behavior unchanged. Raises ValueError on invalid input.
528+
Session behavior unchanged. POST requests are retried only when `retry_post` is set.
529+
Raises ValueError on invalid input.
510530
"""
511531
connection_timeout_ms = property_as_int(properties, REST_CLIENT_CONNECTION_TIMEOUT_MS)
512532
if connection_timeout_ms is not None and connection_timeout_ms <= 0:
@@ -541,7 +561,7 @@ def _create_connection_adapter(properties: Properties) -> _RetryTimeoutHTTPAdapt
541561
total=retries if retries is not None else DEFAULT_RETRIES,
542562
backoff_factor=backoff_factor if backoff_factor is not None else 0.0,
543563
status_forcelist=list(_CONNECTION_RETRY_STATUS_FORCELIST),
544-
allowed_methods=_CONNECTION_RETRY_ALLOWED_METHODS,
564+
allowed_methods=_CONNECTION_RETRY_ALLOWED_METHODS | {"POST"} if retry_post else _CONNECTION_RETRY_ALLOWED_METHODS,
545565
# Return the final response on retry exhaustion (instead of raising MaxRetryError)
546566
# so `_handle_non_200_response` can map the 5xx status to a typed exception
547567
# (ServiceUnavailableError, etc.). 4xx codes are not in status_forcelist and are
@@ -557,6 +577,7 @@ class RestCatalog(Catalog):
557577
_auth_manager: AuthManager | None
558578
_supported_endpoints: set[Endpoint]
559579
_namespace_separator: str
580+
_idempotency_enabled: bool = False
560581

561582
def __init__(self, name: str, **properties: str):
562583
"""Rest Catalog.
@@ -579,7 +600,7 @@ def _create_session(self) -> Session:
579600

580601
# Mount the retry/timeout adapter when `connection.*` properties are set.
581602
# SigV4's adapter mounted below at `self.uri` is a longer prefix and still wins for that host.
582-
if (connection_adapter := _create_connection_adapter(self.properties)) is not None:
603+
if (connection_adapter := _create_connection_adapter(self.properties, self._idempotency_enabled)) is not None:
583604
session.mount("http://", connection_adapter)
584605
session.mount("https://", connection_adapter)
585606

@@ -1052,6 +1073,8 @@ def _fetch_config(self) -> None:
10521073
# Update URI based on overrides
10531074
self.uri = config[URI]
10541075

1076+
self._idempotency_enabled = config_response.idempotency_key_lifetime is not None
1077+
10551078
# Determine supported endpoints
10561079
endpoints = config_response.endpoints
10571080
if endpoints:
@@ -1068,6 +1091,9 @@ def _fetch_config(self) -> None:
10681091
raise ValueError("Namespace separator cannot be an empty string")
10691092
self._namespace_separator = unquote(separator_from_properties)
10701093

1094+
def _mutation_headers(self) -> dict[str, str]:
1095+
return {IDEMPOTENCY_KEY_HEADER: str(_uuid7())} if self._idempotency_enabled else {}
1096+
10711097
def _identifier_to_validated_tuple(self, identifier: str | Identifier) -> Identifier:
10721098
identifier_tuple = self.identifier_to_tuple(identifier)
10731099
if len(identifier_tuple) <= 1:
@@ -1244,6 +1270,7 @@ def _create_table(
12441270
response = self._session.post(
12451271
self.url(Endpoints.create_table, namespace=namespace_and_table["namespace"]),
12461272
data=serialized_json,
1273+
headers=self._mutation_headers(),
12471274
)
12481275
try:
12491276
response.raise_for_status()
@@ -1326,6 +1353,7 @@ def create_view(
13261353
response = self._session.post(
13271354
self.url(Endpoints.create_view, namespace=namespace_and_view["namespace"]),
13281355
data=serialized_json,
1356+
headers=self._mutation_headers(),
13291357
)
13301358

13311359
try:
@@ -1363,6 +1391,7 @@ def register_table(self, identifier: str | Identifier, metadata_location: str, o
13631391
response = self._session.post(
13641392
self.url(Endpoints.register_table, namespace=namespace_and_table["namespace"]),
13651393
data=serialized_json,
1394+
headers=self._mutation_headers(),
13661395
)
13671396
try:
13681397
response.raise_for_status()
@@ -1389,6 +1418,7 @@ def unregister_table(self, identifier: str | Identifier) -> tuple[str, TableMeta
13891418
namespace_and_table = self._split_identifier_for_path(identifier)
13901419
response = self._session.post(
13911420
self.url(Endpoints.unregister_table, prefixed=True, **namespace_and_table),
1421+
headers=self._mutation_headers(),
13921422
)
13931423
try:
13941424
response.raise_for_status()
@@ -1489,6 +1519,7 @@ def drop_table(self, identifier: str | Identifier, purge_requested: bool = False
14891519
response = self._session.delete(
14901520
self.url(Endpoints.drop_table, prefixed=True, **self._split_identifier_for_path(identifier)),
14911521
params={"purgeRequested": purge_requested},
1522+
headers=self._mutation_headers(),
14921523
)
14931524
try:
14941525
response.raise_for_status()
@@ -1518,7 +1549,7 @@ def rename_table(self, from_identifier: str | Identifier, to_identifier: str | I
15181549
if not self.namespace_exists(destination_namespace):
15191550
raise NoSuchNamespaceError(f"Destination namespace does not exist: {destination_namespace}")
15201551

1521-
response = self._session.post(self.url(Endpoints.rename_table), json=payload)
1552+
response = self._session.post(self.url(Endpoints.rename_table), json=payload, headers=self._mutation_headers())
15221553
try:
15231554
response.raise_for_status()
15241555
except HTTPError as exc:
@@ -1622,7 +1653,7 @@ def commit_table(
16221653
response = self._session.post(
16231654
self.url(Endpoints.update_table, prefixed=True, **self._split_identifier_for_path(table_request.identifier)),
16241655
data=table_request.model_dump_json().encode(UTF8),
1625-
headers=headers,
1656+
headers={**headers, **self._mutation_headers()},
16261657
)
16271658
try:
16281659
response.raise_for_status()
@@ -1644,7 +1675,7 @@ def create_namespace(self, namespace: str | Identifier, properties: Properties =
16441675
self._check_endpoint(Capability.V1_CREATE_NAMESPACE)
16451676
namespace_tuple = self._check_valid_namespace_identifier(namespace)
16461677
payload = {"namespace": namespace_tuple, "properties": properties}
1647-
response = self._session.post(self.url(Endpoints.create_namespace), json=payload)
1678+
response = self._session.post(self.url(Endpoints.create_namespace), json=payload, headers=self._mutation_headers())
16481679
try:
16491680
response.raise_for_status()
16501681
except HTTPError as exc:
@@ -1656,7 +1687,7 @@ def drop_namespace(self, namespace: str | Identifier) -> None:
16561687
self._check_endpoint(Capability.V1_DELETE_NAMESPACE)
16571688
namespace_tuple = self._check_valid_namespace_identifier(namespace)
16581689
namespace = self._encode_namespace_path(namespace_tuple)
1659-
response = self._session.delete(self.url(Endpoints.drop_namespace, namespace=namespace))
1690+
response = self._session.delete(self.url(Endpoints.drop_namespace, namespace=namespace), headers=self._mutation_headers())
16601691
try:
16611692
response.raise_for_status()
16621693
except HTTPError as exc:
@@ -1723,7 +1754,9 @@ def update_namespace_properties(
17231754
namespace_tuple = self._check_valid_namespace_identifier(namespace)
17241755
namespace = self._encode_namespace_path(namespace_tuple)
17251756
payload = {"removals": list(removals or []), "updates": updates}
1726-
response = self._session.post(self.url(Endpoints.update_namespace_properties, namespace=namespace), json=payload)
1757+
response = self._session.post(
1758+
self.url(Endpoints.update_namespace_properties, namespace=namespace), json=payload, headers=self._mutation_headers()
1759+
)
17271760
try:
17281761
response.raise_for_status()
17291762
except HTTPError as exc:
@@ -1838,6 +1871,7 @@ def register_view(self, identifier: str | Identifier, metadata_location: str) ->
18381871
response = self._session.post(
18391872
self.url(Endpoints.register_view, namespace=namespace_and_view["namespace"]),
18401873
data=serialized_json,
1874+
headers=self._mutation_headers(),
18411875
)
18421876
try:
18431877
response.raise_for_status()
@@ -1853,6 +1887,7 @@ def drop_view(self, identifier: str | Identifier) -> None:
18531887
self._check_endpoint(Capability.V1_DELETE_VIEW)
18541888
response = self._session.delete(
18551889
self.url(Endpoints.drop_view, prefixed=True, **self._split_identifier_for_path(identifier, IdentifierKind.VIEW)),
1890+
headers=self._mutation_headers(),
18561891
)
18571892
try:
18581893
response.raise_for_status()
@@ -1876,7 +1911,7 @@ def rename_view(self, from_identifier: str | Identifier, to_identifier: str | Id
18761911
raise NoSuchNamespaceError(f"Destination namespace does not exist: {destination_namespace}")
18771912

18781913
payload = {"source": source, "destination": destination}
1879-
response = self._session.post(self.url(Endpoints.rename_view), json=payload)
1914+
response = self._session.post(self.url(Endpoints.rename_view), json=payload, headers=self._mutation_headers())
18801915
try:
18811916
response.raise_for_status()
18821917
except HTTPError as exc:

‎pyiceberg/io/pyarrow.py‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
from typing_extensions import override
6969

7070
from pyiceberg.conversions import to_bytes
71+
from pyiceberg.encryption.key_metadata import StandardKeyMetadata
7172
from pyiceberg.exceptions import ResolveError
7273
from pyiceberg.expressions import AlwaysTrue, BooleanExpression, BoundIsNaN, BoundIsNull, BoundTerm, Not, Or
7374
from pyiceberg.expressions.literals import Literal
@@ -1146,9 +1147,10 @@ def _expression_to_complementary_pyarrow(expr: BooleanExpression, schema: Schema
11461147
return expression_to_pyarrow(preserve_expr, schema)
11471148

11481149

1149-
@lru_cache
1150-
def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.FileFormat:
1150+
def _create_arrow_read_format(file_format: FileFormat, key_metadata: bytes | None, **kwargs: Any) -> ds.FileFormat:
11511151
if file_format == FileFormat.PARQUET:
1152+
if key_metadata is not None:
1153+
kwargs["decryption_properties"] = _parquet_decryption_properties(key_metadata)
11521154
return ds.ParquetFileFormat(**kwargs)
11531155
elif file_format == FileFormat.ORC:
11541156
# ORC doesn't support pre_buffer and buffer_size parameters
@@ -1158,11 +1160,33 @@ def _get_file_format(file_format: FileFormat, **kwargs: dict[str, Any]) -> ds.Fi
11581160
raise ValueError(f"Unsupported file format: {file_format}")
11591161

11601162

1163+
def _check_encryption_supported(data_file: DataFile) -> None:
1164+
if data_file.key_metadata is not None and data_file.file_format not in (FileFormat.PARQUET, FileFormat.PUFFIN):
1165+
raise ValueError(
1166+
f"Cannot read encrypted {data_file.file_format} file {data_file.file_path}: only Parquet and Puffin are supported"
1167+
)
1168+
1169+
1170+
def _parquet_decryption_properties(key_metadata: bytes) -> pq.FileDecryptionProperties:
1171+
try:
1172+
from pyarrow.parquet.encryption import create_decryption_properties
1173+
except ImportError as e:
1174+
raise ImportError(f"pyarrow >= 25.0.0 required to read encrypted Parquet files, found {pyarrow.__version__}") from e
1175+
1176+
metadata = StandardKeyMetadata.from_bytes(key_metadata)
1177+
return create_decryption_properties(metadata.encryption_key, aad_prefix=metadata.aad_prefix)
1178+
1179+
11611180
def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]:
1181+
_check_encryption_supported(data_file)
11621182
if data_file.file_format == FileFormat.PARQUET:
11631183
with io.new_input(data_file.file_path).open() as fi:
1164-
delete_fragment = _get_file_format(
1165-
data_file.file_format, dictionary_columns=("file_path",), pre_buffer=True, buffer_size=ONE_MEGABYTE
1184+
delete_fragment = _create_arrow_read_format(
1185+
data_file.file_format,
1186+
data_file.key_metadata,
1187+
dictionary_columns=("file_path",),
1188+
pre_buffer=True,
1189+
buffer_size=ONE_MEGABYTE,
11661190
).make_fragment(fi)
11671191
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
11681192
table = table.unify_dictionaries()
@@ -1172,7 +1196,7 @@ def _read_deletes(io: FileIO, data_file: DataFile) -> dict[str, pa.ChunkedArray]
11721196
}
11731197
elif data_file.file_format == FileFormat.ORC:
11741198
with io.new_input(data_file.file_path).open() as fi:
1175-
delete_fragment = _get_file_format(data_file.file_format).make_fragment(fi)
1199+
delete_fragment = _create_arrow_read_format(data_file.file_format, data_file.key_metadata).make_fragment(fi)
11761200
table = ds.Scanner.from_fragment(fragment=delete_fragment).to_table()
11771201
# For ORC, file_path columns are not dictionary-encoded, so we use unique() directly
11781202
return {
@@ -1712,10 +1736,11 @@ def _task_to_record_batches(
17121736
downcast_ns_timestamp_to_us: bool | None = None,
17131737
dictionary_columns: tuple[str, ...] = (),
17141738
) -> Iterator[pa.RecordBatch]:
1739+
_check_encryption_supported(task.file)
17151740
format_kwargs: dict[str, Any] = {"pre_buffer": True, "buffer_size": ONE_MEGABYTE * 8}
17161741
if dictionary_columns and task.file.file_format == FileFormat.PARQUET:
17171742
format_kwargs["dictionary_columns"] = dictionary_columns
1718-
arrow_format = _get_file_format(task.file.file_format, **format_kwargs)
1743+
arrow_format = _create_arrow_read_format(task.file.file_format, task.file.key_metadata, **format_kwargs)
17191744
with io.new_input(task.file.file_path).open() as fin:
17201745
fragment = arrow_format.make_fragment(fin)
17211746
physical_schema = fragment.physical_schema
@@ -2651,6 +2676,7 @@ def data_file_statistics_from_parquet_metadata(
26512676
col_aggs = {}
26522677

26532678
invalidate_col: set[int] = set()
2679+
without_bounds: set[int] = set()
26542680
for r in range(parquet_metadata.num_row_groups):
26552681
# References:
26562682
# https://github.com/apache/iceberg/blob/fc381a81a1fdb8f51a0637ca27cd30673bd7aad3/parquet/src/main/java/org/apache/iceberg/parquet/ParquetUtil.java#L232
@@ -2690,6 +2716,11 @@ def data_file_statistics_from_parquet_metadata(
26902716
if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS):
26912717
continue
26922718

2719+
if not statistics.has_min_max:
2720+
if not (statistics.has_null_count and statistics.null_count == column.num_values):
2721+
without_bounds.add(field_id)
2722+
continue
2723+
26932724
if field_id not in col_aggs:
26942725
try:
26952726
col_aggs[field_id] = StatsAggregator(
@@ -2727,6 +2758,9 @@ def data_file_statistics_from_parquet_metadata(
27272758
col_aggs.pop(field_id, None)
27282759
null_value_counts.pop(field_id, None)
27292760

2761+
for field_id in without_bounds:
2762+
col_aggs.pop(field_id, None)
2763+
27302764
return DataFileStatistics(
27312765
record_count=parquet_metadata.num_rows,
27322766
column_sizes=column_sizes,

0 commit comments

Comments
 (0)