@@ -1848,6 +1848,32 @@ def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[st
18481848 return deletes_per_file
18491849
18501850
1851+ def _decode_unrequested_dictionary_columns (batch : pa .RecordBatch , dictionary_columns : set [str ]) -> pa .RecordBatch :
1852+ """Decode dictionary-encoded columns that were not requested as dictionary encoded.
1853+
1854+ Files with dictionary-encoded columns and files with plain columns produce different batch
1855+ schemas, which cannot be concatenated. Decoding the columns that were not requested as
1856+ dictionaries gives the batches one logical schema, while requested dictionary columns stay as
1857+ they are.
1858+ """
1859+ to_decode = [field for field in batch .schema if pa .types .is_dictionary (field .type ) and field .name not in dictionary_columns ]
1860+ if not to_decode :
1861+ return batch
1862+
1863+ decode_names = {field .name for field in to_decode }
1864+ schema = pa .schema (
1865+ [
1866+ pa .field (field .name , field .type .value_type , field .nullable , field .metadata ) if field .name in decode_names else field
1867+ for field in batch .schema
1868+ ]
1869+ )
1870+ arrays = [
1871+ batch .column (i ).dictionary_decode () if field .name in decode_names else batch .column (i )
1872+ for i , field in enumerate (batch .schema )
1873+ ]
1874+ return pa .RecordBatch .from_arrays (arrays , schema = schema )
1875+
1876+
18511877class ArrowScan :
18521878 _table_metadata : TableMetadata
18531879 _io : FileIO
@@ -1927,8 +1953,13 @@ def to_table(self, tasks: Iterable[FileScanTask]) -> pa.Table:
19271953
19281954 # Note: cannot use pa.Table.from_batches(itertools.chain([first_batch], batches)))
19291955 # as different batches can use different schema's (due to large_ types)
1956+ dictionary_columns = set (self ._dictionary_columns )
19301957 result = pa .concat_tables (
1931- (pa .Table .from_batches ([batch ]) for batch in itertools .chain ([first_batch ], batches )), promote_options = "permissive"
1958+ (
1959+ pa .Table .from_batches ([_decode_unrequested_dictionary_columns (batch , dictionary_columns )])
1960+ for batch in itertools .chain ([first_batch ], batches )
1961+ ),
1962+ promote_options = "permissive" ,
19321963 )
19331964
19341965 return result
0 commit comments