Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions pyiceberg/catalog/rest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,17 @@
import time
import uuid
from collections import deque
from collections.abc import Mapping
from collections.abc import Callable, Mapping
from enum import Enum, StrEnum
from typing import (
TYPE_CHECKING,
Annotated,
Any,
TypeVar,
)
from urllib.parse import quote, unquote

from pydantic import ConfigDict, Field, TypeAdapter, field_validator
from pydantic import BeforeValidator, ConfigDict, Field, TypeAdapter, field_validator
from requests import HTTPError, PreparedRequest, Response, Session
from requests.adapters import DEFAULT_RETRIES, HTTPAdapter
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt
Expand Down Expand Up @@ -368,11 +370,23 @@ def _retry_hook(retry_state: RetryCallState) -> None:
}


T = TypeVar("T")


def _empty_if_none(factory: Callable[[], Any]) -> BeforeValidator:
return BeforeValidator(lambda v: factory() if v is None else v)


# A JSON null is read as an empty list / map, like Java's `hasNonNull`
NullableList = Annotated[list[T], _empty_if_none(list)]
NullableProperties = Annotated[Properties, _empty_if_none(dict)]


class TableResponse(IcebergBaseModel):
metadata_location: str | None = Field(alias="metadata-location", default=None)
metadata: TableMetadata
config: Properties = Field(default_factory=dict)
storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list)
config: NullableProperties = Field(default_factory=dict)
storage_credentials: NullableList[StorageCredential] = Field(alias="storage-credentials", default_factory=list)
labels: Labels = Field(default_factory=Labels)


Expand All @@ -383,7 +397,7 @@ class LoadCredentialsResponse(IcebergBaseModel):
class ViewResponse(IcebergBaseModel):
metadata_location: str = Field(..., alias="metadata-location")
metadata: ViewMetadata
config: Properties = Field(default_factory=dict)
config: NullableProperties = Field(default_factory=dict)
labels: Labels = Field(default_factory=Labels)


Expand Down Expand Up @@ -436,8 +450,8 @@ class RegisterViewRequest(IcebergBaseModel):


class ConfigResponse(IcebergBaseModel):
defaults: Properties | None = Field(default_factory=dict)
overrides: Properties | None = Field(default_factory=dict)
defaults: NullableProperties = Field(default_factory=dict)
overrides: NullableProperties = Field(default_factory=dict)
endpoints: set[Endpoint] | None = Field(default=None)
idempotency_key_lifetime: str | None = Field(alias="idempotency-key-lifetime", default=None)

Expand Down
26 changes: 26 additions & 0 deletions tests/catalog/test_rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
SIGV4_MAX_RETRIES_DEFAULT,
SNAPSHOT_LOADING_MODE,
Capability,
ConfigResponse,
Endpoint,
HttpMethod,
RestCatalog,
Expand Down Expand Up @@ -3838,6 +3839,31 @@ def test_load_responses_parse_labels(
assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels()


def test_load_responses_with_null_config_and_storage_credentials(
example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], example_view_metadata_rest_json: dict[str, Any]
) -> None:
# Like Java, an explicit null config reads as an empty config
table_response = TableResponse.model_validate(
{**example_table_metadata_with_snapshot_v1_rest_json, "config": None, "storage-credentials": None}
)
assert table_response.config == {}
assert table_response.storage_credentials == []

view_response = ViewResponse.model_validate({**example_view_metadata_rest_json, "config": None})
assert view_response.config == {}


def test_config_response_with_null_defaults_and_overrides() -> None:
config_response = ConfigResponse.model_validate({"defaults": None, "overrides": None})
assert config_response.defaults == {}
assert config_response.overrides == {}


def test_catalog_with_null_config_defaults_and_overrides(rest_mock: Mocker) -> None:
rest_mock.get(f"{TEST_URI}v1/config", json={"defaults": None, "overrides": None}, status_code=200)
assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).uri == TEST_URI


def test_table_labels_from_rest_responses(
rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], table_schema_simple: Schema
) -> None:
Expand Down
Loading