Skip to content

Commit 9ba780e

Browse files
committed
Merge remote-tracking branch 'origin/main' into fix/v3-partition-field-id-reuse
2 parents 32a3b4e + eb30d8a commit 9ba780e

6 files changed

Lines changed: 268 additions & 134 deletions

File tree

‎pyiceberg/catalog/rest/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1676,7 +1676,8 @@ def list_namespaces(self, namespace: str | Identifier = ()) -> list[Identifier]:
16761676

16771677
while True:
16781678
if namespace_tuple:
1679-
params["parent"] = self._encode_namespace_path(namespace_tuple)
1679+
# Query parameters are URL-encoded by requests, so the parts must not be quoted here as well
1680+
params["parent"] = self._namespace_separator.join(namespace_tuple)
16801681
if page_token:
16811682
params["pageToken"] = page_token
16821683
response = self._session.get(self.url(Endpoints.list_namespaces), params=params)

‎pyiceberg/io/pyarrow.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2120,7 +2120,18 @@ def map(
21202120
)
21212121
if isinstance(value_result, pa.StructArray):
21222122
# Arrow does not allow reordering of fields, therefore we have to copy the array :(
2123-
return pa.MapArray.from_arrays(map_array.offsets, key_result, value_result, arrow_field)
2123+
entries = pa.StructArray.from_arrays(
2124+
[key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field]
2125+
)
2126+
# Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps
2127+
return pa.Array.from_buffers(
2128+
arrow_field,
2129+
len(map_array),
2130+
map_array.buffers()[:2],
2131+
map_array.null_count,
2132+
map_array.offset,
2133+
[entries],
2134+
)
21242135
else:
21252136
return map_array.cast(arrow_field)
21262137
else:

‎pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,8 @@ markers = [
243243
# Turns a warning into an error
244244
filterwarnings = [
245245
"error",
246-
# Ignore Python version deprecation warning from google.api_core while we still support 3.10
247-
"ignore:You are using a Python version.*which Google will stop supporting:FutureWarning:google.api_core",
246+
# Ignore Python version deprecation, end-of-life and unsupported warnings from google.api_core while we still support 3.10
247+
"ignore:You are using a (non-supported )?Python version:FutureWarning:google.api_core",
248248
# Python 3.13 sqlite3 module ResourceWarnings for unclosed database connections
249249
"ignore:unclosed database in <sqlite3.Connection object*:ResourceWarning",
250250
# Ignore Ray subprocess cleanup warnings

‎tests/catalog/test_rest.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from contextlib import contextmanager
2424
from typing import Any, cast
2525
from unittest import mock
26+
from urllib.parse import parse_qs, urlsplit
2627

2728
import pytest
2829
from requests import Request
@@ -972,6 +973,42 @@ def test_list_namespace_with_parent_200(rest_mock: Mocker) -> None:
972973
]
973974

974975

976+
@pytest.mark.parametrize(
977+
"parent",
978+
[
979+
("accounting",),
980+
("accounting", "tax"),
981+
("my namespace",),
982+
("dept/finance", "tax"),
983+
("100%",),
984+
("\u65e5\u672c",),
985+
],
986+
)
987+
def test_list_namespace_with_parent_is_encoded_once(rest_mock: Mocker, parent: tuple[str, ...]) -> None:
988+
rest_mock.get(
989+
f"{TEST_URI}v1/namespaces",
990+
json={"namespaces": [[*parent, "child"]]},
991+
status_code=200,
992+
request_headers=TEST_HEADERS,
993+
)
994+
assert RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).list_namespaces(parent) == [(*parent, "child")]
995+
query = parse_qs(urlsplit(rest_mock.last_request.url).query)
996+
assert query["parent"] == ["\x1f".join(parent)]
997+
998+
999+
def test_list_namespace_with_parent_custom_separator(rest_mock: Mocker) -> None:
1000+
rest_mock.get(
1001+
f"{TEST_URI}v1/namespaces",
1002+
json={"namespaces": [["my namespace", "tax", "child"]]},
1003+
status_code=200,
1004+
request_headers=TEST_HEADERS,
1005+
)
1006+
catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN, **{"namespace-separator": "."})
1007+
assert catalog.list_namespaces(("my namespace", "tax")) == [("my namespace", "tax", "child")]
1008+
query = parse_qs(urlsplit(rest_mock.last_request.url).query)
1009+
assert query["parent"] == ["my namespace.tax"]
1010+
1011+
9751012
def test_list_namespaces_paginated_200(rest_mock: Mocker) -> None:
9761013
# First page with next-page-token
9771014
rest_mock.get(

‎tests/io/test_pyarrow.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3211,6 +3211,48 @@ def test__to_requested_schema_timestamps_without_downcast_raises_exception(
32113211
assert "Unsupported schema projection from timestamp[ns] to timestamp[us]" in str(exc_info.value)
32123212

32133213

3214+
def test__to_requested_schema_null_map_of_structs() -> None:
3215+
requested_schema = Schema(
3216+
NestedField(
3217+
1,
3218+
"col_map_with_struct",
3219+
MapType(
3220+
11,
3221+
StringType(),
3222+
12,
3223+
StructType(NestedField(121, "source", StringType(), required=False)),
3224+
value_required=False,
3225+
),
3226+
required=False,
3227+
),
3228+
NestedField(2, "col_map", MapType(21, StringType(), 22, IntegerType(), value_required=False), required=False),
3229+
)
3230+
arrow_schema = pa.schema(
3231+
[
3232+
pa.field("col_map_with_struct", pa.map_(pa.string(), pa.struct([pa.field("source", pa.string())]))),
3233+
pa.field("col_map", pa.map_(pa.string(), pa.int32())),
3234+
]
3235+
)
3236+
batch = pa.RecordBatch.from_arrays(
3237+
[
3238+
pa.array(
3239+
[[("a", {"source": "x"})], [], None, [("b", {"source": "y"}), ("c", None)]], type=arrow_schema.field(0).type
3240+
),
3241+
pa.array([[("a", 1)], [], None, [("b", 2), ("c", None)]], type=arrow_schema.field(1).type),
3242+
],
3243+
schema=arrow_schema,
3244+
)
3245+
3246+
result = _to_requested_schema(requested_schema, requested_schema, batch)
3247+
assert result.column(0).to_pylist() == [[("a", {"source": "x"})], [], None, [("b", {"source": "y"}), ("c", None)]]
3248+
assert result.column(1).to_pylist() == [[("a", 1)], [], None, [("b", 2), ("c", None)]]
3249+
3250+
# A sliced batch has to keep the nulls as well
3251+
result = _to_requested_schema(requested_schema, requested_schema, batch.slice(1, 3))
3252+
assert result.column(0).to_pylist() == [[], None, [("b", {"source": "y"}), ("c", None)]]
3253+
assert result.column(1).to_pylist() == [[], None, [("b", 2), ("c", None)]]
3254+
3255+
32143256
@pytest.mark.parametrize(
32153257
"arrow_type,iceberg_type,expected_arrow_type",
32163258
[

0 commit comments

Comments
 (0)