From 17f179ce7d4ad25b1fc936086b40b8f8cfe2d22a Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Fri, 20 Nov 2026 23:51:36 +0530 Subject: [PATCH 1/2] Core: Accept REST auth configuration from environment variables Configuration from environment variables arrives as flat strings, so a JSON encoded auth value was treated as an object and failed with AttributeError. Decode JSON strings, merge flattened auth.* keys, and normalize dashed option names before the auth manager is created. --- pyiceberg/catalog/rest/__init__.py | 43 +++++++++++++++- tests/catalog/test_rest.py | 78 ++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 2ad3684c9a..0dabb67bb2 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import json import logging import os import platform @@ -357,6 +358,45 @@ def _uuid7() -> uuid.UUID: return uuid.UUID(int=value) +def _normalize_auth_config(config: dict[str, Any]) -> dict[str, Any]: + """Normalize option names so dashed keys from configuration files and environment variables work.""" + return { + key.replace("-", "_"): _normalize_auth_config(value) if isinstance(value, dict) else value + for key, value in config.items() + } + + +def _parse_auth_config(properties: Properties) -> dict[str, Any]: + """Build the auth configuration from an object, a JSON string, or flattened dot-separated keys.""" + auth: Any = properties.get(AUTH) + if auth is None: + auth = {} + elif isinstance(auth, str): + try: + auth = json.loads(auth) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid auth configuration, expected a JSON object: {e}") from e + + if not isinstance(auth, dict): + raise ValueError("Invalid auth configuration, expected an object") + + auth = dict(auth) + prefix = f"{AUTH}." + for key, value in properties.items(): + if not key.startswith(prefix): + continue + *parents, leaf = key[len(prefix) :].replace("-", "_").split(".") + target = auth + for parent in parents: + child = target.setdefault(parent, {}) + if not isinstance(child, dict): + raise ValueError(f"Invalid auth configuration, {parent} is not an object") + target = child + target[leaf] = value + + return _normalize_auth_config(auth) + + def _retry_hook(retry_state: RetryCallState) -> None: rest_catalog: RestCatalog = retry_state.args[0] rest_catalog._refresh_token() # pylint: disable=protected-access @@ -624,7 +664,8 @@ def _create_session(self) -> Session: elif ssl_client_cert := ssl_client.get(CERT): session.cert = ssl_client_cert - if auth_config := self.properties.get(AUTH): + auth_config = _parse_auth_config(self.properties) + if auth_config: auth_type = auth_config.get("type") if auth_type is None: raise ValueError("auth.type must be defined") diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 3d2b64da92..90d6ba6ae3 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -18,6 +18,7 @@ from __future__ import annotations import base64 +import json import os import pickle import platform @@ -2754,6 +2755,83 @@ def test_rest_catalog_with_basic_auth_type(rest_mock: Mocker) -> None: assert rest_mock.last_request.headers["Authorization"] == expected_auth_header +def test_rest_catalog_with_json_string_auth(rest_mock: Mocker) -> None: + # Given + rest_mock.get( + f"{TEST_URI}v1/config", + json={"defaults": {}, "overrides": {}}, + status_code=200, + ) + catalog_properties = { + "uri": TEST_URI, + "auth": json.dumps( + { + "type": "basic", + "basic": { + "username": "one", + "password": "two", + }, + } + ), + } + + catalog = RestCatalog("rest", **catalog_properties) # type: ignore + + assert catalog.uri == TEST_URI + encoded_user_pass = base64.b64encode(b"one:two").decode() + assert rest_mock.last_request.headers["Authorization"] == f"Basic {encoded_user_pass}" + + +def test_rest_catalog_with_flattened_auth_properties(rest_mock: Mocker) -> None: + # Given + rest_mock.get( + f"{TEST_URI}v1/config", + json={"defaults": {}, "overrides": {}}, + status_code=200, + ) + catalog_properties = { + "uri": TEST_URI, + "auth.type": "basic", + "auth.basic.username": "one", + "auth.basic.password": "two", + } + + catalog = RestCatalog("rest", **catalog_properties) # type: ignore + + assert catalog.uri == TEST_URI + encoded_user_pass = base64.b64encode(b"one:two").decode() + assert rest_mock.last_request.headers["Authorization"] == f"Basic {encoded_user_pass}" + + +def test_parse_auth_config_normalizes_dashed_keys() -> None: + from pyiceberg.catalog.rest import _parse_auth_config + + assert _parse_auth_config( + { + "auth.type": "oauth2", + "auth.oauth2.client-id": "id", + "auth.oauth2.token-url": "https://auth.example/token", + } + ) == { + "type": "oauth2", + "oauth2": { + "client_id": "id", + "token_url": "https://auth.example/token", + }, + } + + +def test_rest_catalog_with_invalid_json_auth() -> None: + # Given + catalog_properties = { + "uri": TEST_URI, + "auth": "{not valid json}", + } + + with pytest.raises(ValueError, match="Invalid auth configuration"): + RestCatalog("rest", **catalog_properties) # type: ignore + + def test_rest_catalog_with_custom_auth_type() -> None: # Given catalog_properties = { From e3674de6b76ce31af4d93a380c7cc2c11f07f029 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 03:05:09 +0530 Subject: [PATCH 2/2] Remove unused type ignores from the auth tests --- tests/catalog/test_rest.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 90d6ba6ae3..6ce6963470 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -2775,7 +2775,7 @@ def test_rest_catalog_with_json_string_auth(rest_mock: Mocker) -> None: ), } - catalog = RestCatalog("rest", **catalog_properties) # type: ignore + catalog = RestCatalog("rest", **catalog_properties) assert catalog.uri == TEST_URI encoded_user_pass = base64.b64encode(b"one:two").decode() @@ -2796,7 +2796,7 @@ def test_rest_catalog_with_flattened_auth_properties(rest_mock: Mocker) -> None: "auth.basic.password": "two", } - catalog = RestCatalog("rest", **catalog_properties) # type: ignore + catalog = RestCatalog("rest", **catalog_properties) assert catalog.uri == TEST_URI encoded_user_pass = base64.b64encode(b"one:two").decode() @@ -2829,7 +2829,7 @@ def test_rest_catalog_with_invalid_json_auth() -> None: } with pytest.raises(ValueError, match="Invalid auth configuration"): - RestCatalog("rest", **catalog_properties) # type: ignore + RestCatalog("rest", **catalog_properties) def test_rest_catalog_with_custom_auth_type() -> None: