From 6135ddb03e37fe55f8f766c9ef62bb178c5b6d50 Mon Sep 17 00:00:00 2001 From: Shubhaang Kataruka <164889601+S-h-u-b-h-1@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:43:13 +0530 Subject: [PATCH] GH-55490: [Python] Preserve timezone in dictionary timestamp pandas conversion AI assisted with implementation and tests. Prepared as a draft pending contributor review. --- python/pyarrow/array.pxi | 5 +- python/pyarrow/pandas_compat.py | 15 ++++-- .../src/arrow/python/arrow_to_pandas.cc | 13 +++++ python/pyarrow/tests/test_pandas.py | 53 ++++++++++++++++--- 4 files changed, 74 insertions(+), 12 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index e090b1d0b3db..7db33b74965b 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2553,9 +2553,8 @@ cdef wrap_array_output(PyObject* output): cdef object obj = PyObject_to_object(output) if isinstance(obj, dict): - return _pandas_api.categorical_type.from_codes( - obj['indices'], categories=obj['dictionary'], ordered=obj['ordered'] - ) + from pyarrow.pandas_compat import _reconstruct_categorical + return _reconstruct_categorical(obj, obj['indices']) else: return obj diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index 495df5645f3f..bdb8e0657331 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -722,6 +722,17 @@ def get_datetimetz_type(values, dtype, type_): # Converting pyarrow.Table efficiently to pandas.DataFrame +def _reconstruct_categorical(item, indices): + categories = item['dictionary'] + if 'dictionary_timezone' in item and categories.dtype.kind == 'M': + unit, _ = np.datetime_data(categories.dtype) + dtype = make_datetimetz(unit, item['dictionary_timezone']) + categories = _pandas_api.pd.array( + categories.view('int64'), dtype=dtype, copy=False) + return _pandas_api.categorical_type.from_codes( + indices, categories=categories, ordered=item['ordered']) + + def _reconstruct_block(item, columns=None, extension_columns=None, return_block=True): """ Construct a pandas Block from the `item` dictionary coming from pyarrow's @@ -755,9 +766,7 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block= block_arr = item.get('block', None) placement = item['placement'] if 'dictionary' in item: - arr = _pandas_api.categorical_type.from_codes( - block_arr, categories=item['dictionary'], - ordered=item['ordered']) + arr = _reconstruct_categorical(item, block_arr) elif 'timezone' in item: unit, _ = np.datetime_data(block_arr.dtype) dtype = make_datetimetz(unit, item['timezone']) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index eb60d9d54f85..0fe457022589 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1850,6 +1850,10 @@ class CategoricalWriter PyObject* pydict; RETURN_NOT_OK(ConvertArrayToPandas(this->options_, dict, nullptr, &pydict)); dictionary_.reset(pydict); + if (dict_type.value_type()->id() == Type::TIMESTAMP) { + dictionary_timezone_ = + checked_cast(*dict_type.value_type()).timezone(); + } ordered_ = dict_type.ordered(); return Status::OK(); } @@ -1880,6 +1884,14 @@ class CategoricalWriter protected: Status AddResultMetadata(PyObject* result) override { PyDict_SetItemString(result, "dictionary", dictionary_.obj()); + if (!dictionary_timezone_.empty()) { + OwnedRef timezone(PyUnicode_FromStringAndSize( + dictionary_timezone_.data(), + static_cast(dictionary_timezone_.size()))); + RETURN_IF_PYERROR(); + PyDict_SetItemString(result, "dictionary_timezone", timezone.obj()); + RETURN_IF_PYERROR(); + } PyObject* py_ordered = ordered_ ? Py_True : Py_False; Py_INCREF(py_ordered); PyDict_SetItemString(result, "ordered", py_ordered); @@ -1996,6 +2008,7 @@ class CategoricalWriter } OwnedRefNoGIL dictionary_; + std::string dictionary_timezone_; bool ordered_; bool needs_copy_; }; diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 2391cb260009..572b78832976 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -3352,12 +3352,8 @@ def _fully_loaded_dataframe_example(): def test_roundtrip_fully_loaded_dataframe_example(): df = _fully_loaded_dataframe_example() - expected = df.copy() - expected["col3"] = df["col3"].cat.rename_categories( - df["col3"].cat.categories.tz_convert(None)) - _check_pandas_roundtrip(df, preserve_index=None, expected=expected) - _check_pandas_roundtrip(df, preserve_index=None, - expected=expected, use_threads=True) + _check_pandas_roundtrip(df, preserve_index=None) + _check_pandas_roundtrip(df, preserve_index=None, use_threads=True) @pytest.mark.parametrize('columns', ([b'foo'], ['foo'])) @@ -4203,6 +4199,51 @@ def random_strings(n, item_size, pct_null=0, dictionary=None): return result +@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns']) +@pytest.mark.parametrize('tz', ['UTC', 'Asia/Kolkata', None]) +@pytest.mark.parametrize('values', [[0, 0, None, 1000], [], [None, None]]) +def test_dictionary_timestamp_to_pandas(unit, tz, values): + # GH-55490: dictionary encoding must not discard the category timezone. + plain = pa.array(values, type=pa.timestamp(unit, tz=tz)) + encoded = plain.dictionary_encode() + expected = pd.Series(pd.Categorical(plain.to_pandas()), name='t') + chunked = pa.chunked_array([ + plain[:1].dictionary_encode(), plain[1:].dictionary_encode()]) + for array in [encoded, chunked]: + result = array.to_pandas().rename('t') + tm.assert_series_equal(result, expected) + table_result = pa.table({'t': array}).to_pandas()['t'] + tm.assert_series_equal(table_result, expected) + + +@pytest.mark.parametrize('options', [ + {'timestamp_as_object': True}, {'coerce_temporal_nanoseconds': True}, {}]) +def test_dictionary_timestamp_to_pandas_options(options): + dictionary = pa.array([0, 1000], type=pa.timestamp('ms', tz='Europe/Berlin')) + encoded = pa.DictionaryArray.from_arrays( + pa.array([0, None, 1, 0]), dictionary, ordered=True) + categories = dictionary.to_pandas(**options) + if options.get('timestamp_as_object'): + categories = categories.to_numpy() + expected = pd.Series(pd.Categorical.from_codes( + [0, -1, 1, 0], categories=categories, ordered=True), + name='t') + tm.assert_series_equal(encoded.to_pandas(**options).rename('t'), expected) + tm.assert_series_equal( + pa.table({'t': encoded}).to_pandas(**options)['t'], expected) + + +@pytest.mark.parametrize('tz', ['UTC', 'Asia/Kolkata']) +def test_dictionary_timestamp_to_pandas_no_chunks(tz): + dtype = pa.dictionary(pa.int32(), pa.timestamp('ns', tz=tz)) + array = pa.chunked_array([], type=dtype) + expected = pd.Series(pd.Categorical.from_codes( + [], categories=pd.DatetimeIndex([], dtype=f'datetime64[ns, {tz}]')), + name='t') + tm.assert_series_equal(array.to_pandas().rename('t'), expected) + tm.assert_series_equal(pa.table({'t': array}).to_pandas()['t'], expected) + + def test_variable_dictionary_to_pandas(): np.random.seed(12345)