From 74e3edac56ba445b3bc83bbfa8f6a64a6b8c8f2c Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Wed, 11 Nov 2026 21:46:58 +0530 Subject: [PATCH 1/2] REST: Read an explicit null config, storage-credentials, defaults or overrides as empty A REST server that serializes unset fields as null made loading a table or view, or constructing the catalog, fail. Java reads a null config as empty. --- pyiceberg/catalog/rest/__init__.py | 20 ++++++++++++++++++++ tests/catalog/test_rest.py | 26 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index d284b06234..d95773b367 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -375,6 +375,16 @@ class TableResponse(IcebergBaseModel): storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list) labels: Labels = Field(default_factory=Labels) + @field_validator("config", mode="before") + @classmethod + def replace_none_config_with_dict(cls, v: Any) -> Properties: + return {} if v is None else v + + @field_validator("storage_credentials", mode="before") + @classmethod + def replace_none_storage_credentials_with_list(cls, v: Any) -> list[StorageCredential]: + return [] if v is None else v + class LoadCredentialsResponse(IcebergBaseModel): storage_credentials: list[StorageCredential] = Field(alias="storage-credentials") @@ -386,6 +396,11 @@ class ViewResponse(IcebergBaseModel): config: Properties = Field(default_factory=dict) labels: Labels = Field(default_factory=Labels) + @field_validator("config", mode="before") + @classmethod + def replace_none_config_with_dict(cls, v: Any) -> Properties: + return {} if v is None else v + class CreateTableRequest(IcebergBaseModel): name: str = Field() @@ -441,6 +456,11 @@ class ConfigResponse(IcebergBaseModel): endpoints: set[Endpoint] | None = Field(default=None) idempotency_key_lifetime: str | None = Field(alias="idempotency-key-lifetime", default=None) + @field_validator("defaults", "overrides", mode="before") + @classmethod + def replace_none_with_dict(cls, v: Any) -> Properties: + return {} if v is None else v + @field_validator("endpoints", mode="before") @classmethod def _parse_endpoints(cls, v: list[str] | None) -> set[Endpoint] | None: diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 1160d2889a..18dd3c7d6d 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -50,6 +50,7 @@ SIGV4_MAX_RETRIES_DEFAULT, SNAPSHOT_LOADING_MODE, Capability, + ConfigResponse, Endpoint, HttpMethod, RestCatalog, @@ -3838,6 +3839,31 @@ def test_load_responses_parse_labels( assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels() +def test_load_responses_with_null_config_and_storage_credentials( + example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], example_view_metadata_rest_json: dict[str, Any] +) -> None: + # Like Java, an explicit null config reads as an empty config + table_response = TableResponse.model_validate( + {**example_table_metadata_with_snapshot_v1_rest_json, "config": None, "storage-credentials": None} + ) + assert table_response.config == {} + assert table_response.storage_credentials == [] + + view_response = ViewResponse.model_validate({**example_view_metadata_rest_json, "config": None}) + assert view_response.config == {} + + +def test_config_response_with_null_defaults_and_overrides() -> None: + config_response = ConfigResponse.model_validate({"defaults": None, "overrides": None}) + assert config_response.defaults == {} + assert config_response.overrides == {} + + +def test_catalog_with_null_config_defaults_and_overrides(rest_mock: Mocker) -> None: + rest_mock.get(f"{TEST_URI}v1/config", json={"defaults": None, "overrides": None}, status_code=200) + assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).uri == TEST_URI + + def test_table_labels_from_rest_responses( rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], table_schema_simple: Schema ) -> None: From c76e032839743b66318e121159067df1ca242c78 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:09:47 +0530 Subject: [PATCH 2/2] Use annotated types for null-tolerant REST fields --- pyiceberg/catalog/rest/__init__.py | 48 +++++++++++++----------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index d95773b367..6fbe44ef70 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -22,15 +22,17 @@ import time import uuid from collections import deque -from collections.abc import Mapping +from collections.abc import Callable, Mapping from enum import Enum, StrEnum from typing import ( TYPE_CHECKING, + Annotated, Any, + TypeVar, ) from urllib.parse import quote, unquote -from pydantic import ConfigDict, Field, TypeAdapter, field_validator +from pydantic import BeforeValidator, ConfigDict, Field, TypeAdapter, field_validator from requests import HTTPError, PreparedRequest, Response, Session from requests.adapters import DEFAULT_RETRIES, HTTPAdapter from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt @@ -368,23 +370,25 @@ def _retry_hook(retry_state: RetryCallState) -> None: } +T = TypeVar("T") + + +def _empty_if_none(factory: Callable[[], Any]) -> BeforeValidator: + return BeforeValidator(lambda v: factory() if v is None else v) + + +# A JSON null is read as an empty list / map, like Java's `hasNonNull` +NullableList = Annotated[list[T], _empty_if_none(list)] +NullableProperties = Annotated[Properties, _empty_if_none(dict)] + + class TableResponse(IcebergBaseModel): metadata_location: str | None = Field(alias="metadata-location", default=None) metadata: TableMetadata - config: Properties = Field(default_factory=dict) - storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list) + config: NullableProperties = Field(default_factory=dict) + storage_credentials: NullableList[StorageCredential] = Field(alias="storage-credentials", default_factory=list) labels: Labels = Field(default_factory=Labels) - @field_validator("config", mode="before") - @classmethod - def replace_none_config_with_dict(cls, v: Any) -> Properties: - return {} if v is None else v - - @field_validator("storage_credentials", mode="before") - @classmethod - def replace_none_storage_credentials_with_list(cls, v: Any) -> list[StorageCredential]: - return [] if v is None else v - class LoadCredentialsResponse(IcebergBaseModel): storage_credentials: list[StorageCredential] = Field(alias="storage-credentials") @@ -393,14 +397,9 @@ class LoadCredentialsResponse(IcebergBaseModel): class ViewResponse(IcebergBaseModel): metadata_location: str = Field(..., alias="metadata-location") metadata: ViewMetadata - config: Properties = Field(default_factory=dict) + config: NullableProperties = Field(default_factory=dict) labels: Labels = Field(default_factory=Labels) - @field_validator("config", mode="before") - @classmethod - def replace_none_config_with_dict(cls, v: Any) -> Properties: - return {} if v is None else v - class CreateTableRequest(IcebergBaseModel): name: str = Field() @@ -451,16 +450,11 @@ class RegisterViewRequest(IcebergBaseModel): class ConfigResponse(IcebergBaseModel): - defaults: Properties | None = Field(default_factory=dict) - overrides: Properties | None = Field(default_factory=dict) + defaults: NullableProperties = Field(default_factory=dict) + overrides: NullableProperties = Field(default_factory=dict) endpoints: set[Endpoint] | None = Field(default=None) idempotency_key_lifetime: str | None = Field(alias="idempotency-key-lifetime", default=None) - @field_validator("defaults", "overrides", mode="before") - @classmethod - def replace_none_with_dict(cls, v: Any) -> Properties: - return {} if v is None else v - @field_validator("endpoints", mode="before") @classmethod def _parse_endpoints(cls, v: list[str] | None) -> set[Endpoint] | None: