Skip to content

Commit da4024c

Browse files
committed
Decode dictionary columns when concatenating scan batches
1 parent aafad6e commit da4024c

2 files changed

Lines changed: 44 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1848,6 +1848,32 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st
18481848
return deletes_per_file
18491849

18501850

1851+
def _decode_unrequested_dictionary_columns(batch: pa.RecordBatch, dictionary_columns: set[str]) -> pa.RecordBatch:
1852+
"""Decode dictionary-encoded columns that were not requested as dictionary encoded.
1853+
1854+
Files with dictionary-encoded columns and files with plain columns produce different batch
1855+
schemas, which cannot be concatenated. Decoding the columns that were not requested as
1856+
dictionaries gives the batches one logical schema, while requested dictionary columns stay as
1857+
they are.
1858+
"""
1859+
to_decode = [field for field in batch.schema if pa.types.is_dictionary(field.type) and field.name not in dictionary_columns]
1860+
if not to_decode:
1861+
return batch
1862+
1863+
decode_names = {field.name for field in to_decode}
1864+
schema = pa.schema(
1865+
[
1866+
pa.field(field.name, field.type.value_type, field.nullable, field.metadata) if field.name in decode_names else field
1867+
for field in batch.schema
1868+
]
1869+
)
1870+
arrays = [
1871+
batch.column(i).dictionary_decode() if field.name in decode_names else batch.column(i)
1872+
for i, field in enumerate(batch.schema)
1873+
]
1874+
return pa.RecordBatch.from_arrays(arrays, schema=schema)
1875+
1876+
18511877
class ArrowScan:
18521878
_table_metadata: TableMetadata
18531879
_io: FileIO
@@ -1927,8 +1953,13 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
19271953

19281954
# Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
19291955
# as different batches can use different schema's (due to large_ types)
1956+
dictionary_columns = set(self._dictionary_columns)
19301957
result = pa.concat_tables(
1931-
(pa.Table.from_batches([batch]) for batch in itertools.chain([first_batch], batches)), promote_options="permissive"
1958+
(
1959+
pa.Table.from_batches([_decode_unrequested_dictionary_columns(batch, dictionary_columns)])
1960+
for batch in itertools.chain([first_batch], batches)
1961+
),
1962+
promote_options="permissive",
19321963
)
19331964

19341965
return result

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,18 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie
338338
table.append(pyarrow_table)
339339

340340

341+
def test_scan_mixed_dictionary_and_plain_string_batches(catalog: Catalog) -> None:
342+
catalog.create_namespace("default")
343+
table = catalog.create_table("default.mixed_dictionary", schema=pa.schema([pa.field("s", pa.string())]))
344+
345+
table.append(pa.table({"s": pa.array(["a"]).dictionary_encode()}))
346+
table.append(pa.table({"s": pa.array(["b"])}))
347+
348+
scanned = table.scan().to_arrow()
349+
assert scanned.schema.field("s").type == pa.string()
350+
assert sorted(scanned["s"].to_pylist()) == ["a", "b"]
351+
352+
341353
@pytest.mark.parametrize(
342354
"schema,expected",
343355
[

0 commit comments

Comments
 (0)