Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 42 additions & 1 deletion pyiceberg/catalog/rest/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.
from __future__ import annotations

import json
import logging
import os
import platform
Expand Down Expand Up @@ -357,6 +358,45 @@ def _uuid7() -> uuid.UUID:
return uuid.UUID(int=value)


def _normalize_auth_config(config: dict[str, Any]) -> dict[str, Any]:
"""Normalize option names so dashed keys from configuration files and environment variables work."""
return {
key.replace("-", "_"): _normalize_auth_config(value) if isinstance(value, dict) else value
for key, value in config.items()
}


def _parse_auth_config(properties: Properties) -> dict[str, Any]:
"""Build the auth configuration from an object, a JSON string, or flattened dot-separated keys."""
auth: Any = properties.get(AUTH)
if auth is None:
auth = {}
elif isinstance(auth, str):
try:
auth = json.loads(auth)
except json.JSONDecodeError as e:
raise ValueError(f"Invalid auth configuration, expected a JSON object: {e}") from e

if not isinstance(auth, dict):
raise ValueError("Invalid auth configuration, expected an object")

auth = dict(auth)
prefix = f"{AUTH}."
for key, value in properties.items():
if not key.startswith(prefix):
continue
*parents, leaf = key[len(prefix) :].replace("-", "_").split(".")
target = auth
for parent in parents:
child = target.setdefault(parent, {})
if not isinstance(child, dict):
raise ValueError(f"Invalid auth configuration, {parent} is not an object")
target = child
target[leaf] = value

return _normalize_auth_config(auth)


def _retry_hook(retry_state: RetryCallState) -> None:
rest_catalog: RestCatalog = retry_state.args[0]
rest_catalog._refresh_token() # pylint: disable=protected-access
Expand Down Expand Up @@ -624,7 +664,8 @@ def _create_session(self) -> Session:
elif ssl_client_cert := ssl_client.get(CERT):
session.cert = ssl_client_cert

if auth_config := self.properties.get(AUTH):
auth_config = _parse_auth_config(self.properties)
if auth_config:
auth_type = auth_config.get("type")
if auth_type is None:
raise ValueError("auth.type must be defined")
Expand Down
78 changes: 78 additions & 0 deletions tests/catalog/test_rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from __future__ import annotations

import base64
import json
import os
import pickle
import platform
Expand Down Expand Up @@ -2754,6 +2755,83 @@ def test_rest_catalog_with_basic_auth_type(rest_mock: Mocker) -> None:
assert rest_mock.last_request.headers["Authorization"] == expected_auth_header


def test_rest_catalog_with_json_string_auth(rest_mock: Mocker) -> None:
# Given
rest_mock.get(
f"{TEST_URI}v1/config",
json={"defaults": {}, "overrides": {}},
status_code=200,
)
catalog_properties = {
"uri": TEST_URI,
"auth": json.dumps(
{
"type": "basic",
"basic": {
"username": "one",
"password": "two",
},
}
),
}

catalog = RestCatalog("rest", **catalog_properties)

assert catalog.uri == TEST_URI
encoded_user_pass = base64.b64encode(b"one:two").decode()
assert rest_mock.last_request.headers["Authorization"] == f"Basic {encoded_user_pass}"


def test_rest_catalog_with_flattened_auth_properties(rest_mock: Mocker) -> None:
# Given
rest_mock.get(
f"{TEST_URI}v1/config",
json={"defaults": {}, "overrides": {}},
status_code=200,
)
catalog_properties = {
"uri": TEST_URI,
"auth.type": "basic",
"auth.basic.username": "one",
"auth.basic.password": "two",
}

catalog = RestCatalog("rest", **catalog_properties)

assert catalog.uri == TEST_URI
encoded_user_pass = base64.b64encode(b"one:two").decode()
assert rest_mock.last_request.headers["Authorization"] == f"Basic {encoded_user_pass}"


def test_parse_auth_config_normalizes_dashed_keys() -> None:
from pyiceberg.catalog.rest import _parse_auth_config

assert _parse_auth_config(
{
"auth.type": "oauth2",
"auth.oauth2.client-id": "id",
"auth.oauth2.token-url": "https://auth.example/token",
}
) == {
"type": "oauth2",
"oauth2": {
"client_id": "id",
"token_url": "https://auth.example/token",
},
}


def test_rest_catalog_with_invalid_json_auth() -> None:
# Given
catalog_properties = {
"uri": TEST_URI,
"auth": "{not valid json}",
}

with pytest.raises(ValueError, match="Invalid auth configuration"):
RestCatalog("rest", **catalog_properties)


def test_rest_catalog_with_custom_auth_type() -> None:
# Given
catalog_properties = {
Expand Down
Loading