Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions pyiceberg/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,15 @@ def _cast_if_needed(arr: "ArrayLike") -> "ArrayLike":
else:
return arr

def _decode_if_dictionary(arr: "ArrayLike") -> "ArrayLike":
return arr.dictionary_decode() if pa.types.is_dictionary(arr.type) else arr

if isinstance(array, pa.Array):
return _cast_if_needed(transform_func(array, *args))
return _cast_if_needed(transform_func(_decode_if_dictionary(array), *args))
elif isinstance(array, pa.ChunkedArray):
result_chunks = []
for arr in array.iterchunks():
result_chunks.append(_cast_if_needed(transform_func(arr, *args)))
result_chunks.append(_cast_if_needed(transform_func(_decode_if_dictionary(arr), *args)))
return pa.chunked_array(result_chunks)
else:
raise ValueError(f"PyArrow array can only be of type pa.Array or pa.ChunkedArray, but found {type(array)}")
Expand Down
18 changes: 18 additions & 0 deletions tests/catalog/test_catalog_behaviors.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,24 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie
table.append(pyarrow_table)


def test_write_dictionary_encoded_column_with_bucket_partition(catalog: Catalog) -> None:
catalog.create_namespace("default")
schema = pa.schema([pa.field("id", pa.int64()), pa.field("s", pa.string())])
table = catalog.create_table("default.dictionary_bucket", schema=schema)
with table.update_spec() as update:
update.add_field("s", "bucket[8]")

plain = pa.table({"id": [1, 2], "s": ["a", "b"]})
encoded = plain.cast(pa.schema([pa.field("id", pa.int64()), pa.field("s", pa.dictionary(pa.int32(), pa.string()))]))
table.append(encoded)

assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == [
{"id": 1, "s": "a"},
{"id": 2, "s": "b"},
]
assert len(table.inspect.partitions()) == 2


@pytest.mark.parametrize(
"schema,expected",
[
Expand Down
18 changes: 18 additions & 0 deletions tests/test_transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -1701,6 +1701,24 @@ def test_bucket_pyarrow_transforms(
assert expected == transform.pyarrow_transform(source_type)(input_arr)


def test_pyarrow_transform_decodes_dictionary_arrays() -> None:
transform: Transform[Any, Any] = BucketTransform(num_buckets=8)
plain = pa.array(["a", "b", None])
encoded = plain.dictionary_encode()

assert (
transform.pyarrow_transform(StringType())(encoded).to_pylist()
== transform.pyarrow_transform(StringType())(plain).to_pylist()
)

chunked_encoded = pa.chunked_array([encoded, pa.array(["c"]).dictionary_encode()])
chunked_plain = pa.chunked_array([plain, pa.array(["c"])])
assert (
transform.pyarrow_transform(StringType())(chunked_encoded).to_pylist()
== transform.pyarrow_transform(StringType())(chunked_plain).to_pylist()
)


def test_bucket_pyarrow_void_transform() -> None:
input_arr = pa.chunked_array([pa.array([1, 2], type=pa.int32()), pa.array([3, 4], type=pa.int32())])
output_arr = pa.array([None, None, None, None], type=pa.int32())
Expand Down
Loading