Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions python/pyarrow/array.pxi
Original file line number Diff line number Diff line change
Expand Up @@ -2553,9 +2553,8 @@ cdef wrap_array_output(PyObject* output):
cdef object obj = PyObject_to_object(output)

if isinstance(obj, dict):
return _pandas_api.categorical_type.from_codes(
obj['indices'], categories=obj['dictionary'], ordered=obj['ordered']
)
from pyarrow.pandas_compat import _reconstruct_categorical
return _reconstruct_categorical(obj, obj['indices'])
else:
return obj

Expand Down
15 changes: 12 additions & 3 deletions python/pyarrow/pandas_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,17 @@ def get_datetimetz_type(values, dtype, type_):
# Converting pyarrow.Table efficiently to pandas.DataFrame


def _reconstruct_categorical(item, indices):
categories = item['dictionary']
if 'dictionary_timezone' in item and categories.dtype.kind == 'M':
unit, _ = np.datetime_data(categories.dtype)
dtype = make_datetimetz(unit, item['dictionary_timezone'])
categories = _pandas_api.pd.array(
categories.view('int64'), dtype=dtype, copy=False)
return _pandas_api.categorical_type.from_codes(
indices, categories=categories, ordered=item['ordered'])


def _reconstruct_block(item, columns=None, extension_columns=None, return_block=True):
"""
Construct a pandas Block from the `item` dictionary coming from pyarrow's
Expand Down Expand Up @@ -755,9 +766,7 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block=
block_arr = item.get('block', None)
placement = item['placement']
if 'dictionary' in item:
arr = _pandas_api.categorical_type.from_codes(
block_arr, categories=item['dictionary'],
ordered=item['ordered'])
arr = _reconstruct_categorical(item, block_arr)
elif 'timezone' in item:
unit, _ = np.datetime_data(block_arr.dtype)
dtype = make_datetimetz(unit, item['timezone'])
Expand Down
13 changes: 13 additions & 0 deletions python/pyarrow/src/arrow/python/arrow_to_pandas.cc
Original file line number Diff line number Diff line change
Expand Up @@ -1850,6 +1850,10 @@ class CategoricalWriter
PyObject* pydict;
RETURN_NOT_OK(ConvertArrayToPandas(this->options_, dict, nullptr, &pydict));
dictionary_.reset(pydict);
if (dict_type.value_type()->id() == Type::TIMESTAMP) {
dictionary_timezone_ =
checked_cast<const TimestampType&>(*dict_type.value_type()).timezone();
}
ordered_ = dict_type.ordered();
return Status::OK();
}
Expand Down Expand Up @@ -1880,6 +1884,14 @@ class CategoricalWriter
protected:
Status AddResultMetadata(PyObject* result) override {
PyDict_SetItemString(result, "dictionary", dictionary_.obj());
if (!dictionary_timezone_.empty()) {
OwnedRef timezone(PyUnicode_FromStringAndSize(
dictionary_timezone_.data(),
static_cast<Py_ssize_t>(dictionary_timezone_.size())));
RETURN_IF_PYERROR();
PyDict_SetItemString(result, "dictionary_timezone", timezone.obj());
RETURN_IF_PYERROR();
}
PyObject* py_ordered = ordered_ ? Py_True : Py_False;
Py_INCREF(py_ordered);
PyDict_SetItemString(result, "ordered", py_ordered);
Expand Down Expand Up @@ -1996,6 +2008,7 @@ class CategoricalWriter
}

OwnedRefNoGIL dictionary_;
std::string dictionary_timezone_;
bool ordered_;
bool needs_copy_;
};
Expand Down
53 changes: 47 additions & 6 deletions python/pyarrow/tests/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -3352,12 +3352,8 @@ def _fully_loaded_dataframe_example():

def test_roundtrip_fully_loaded_dataframe_example():
df = _fully_loaded_dataframe_example()
expected = df.copy()
expected["col3"] = df["col3"].cat.rename_categories(
df["col3"].cat.categories.tz_convert(None))
_check_pandas_roundtrip(df, preserve_index=None, expected=expected)
_check_pandas_roundtrip(df, preserve_index=None,
expected=expected, use_threads=True)
_check_pandas_roundtrip(df, preserve_index=None)
_check_pandas_roundtrip(df, preserve_index=None, use_threads=True)


@pytest.mark.parametrize('columns', ([b'foo'], ['foo']))
Expand Down Expand Up @@ -4203,6 +4199,51 @@ def random_strings(n, item_size, pct_null=0, dictionary=None):
return result


@pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns'])
@pytest.mark.parametrize('tz', ['UTC', 'Asia/Kolkata', None])
@pytest.mark.parametrize('values', [[0, 0, None, 1000], [], [None, None]])
def test_dictionary_timestamp_to_pandas(unit, tz, values):
# GH-55490: dictionary encoding must not discard the category timezone.
plain = pa.array(values, type=pa.timestamp(unit, tz=tz))
encoded = plain.dictionary_encode()
expected = pd.Series(pd.Categorical(plain.to_pandas()), name='t')
chunked = pa.chunked_array([
plain[:1].dictionary_encode(), plain[1:].dictionary_encode()])
for array in [encoded, chunked]:
result = array.to_pandas().rename('t')
tm.assert_series_equal(result, expected)
table_result = pa.table({'t': array}).to_pandas()['t']
tm.assert_series_equal(table_result, expected)


@pytest.mark.parametrize('options', [
{'timestamp_as_object': True}, {'coerce_temporal_nanoseconds': True}, {}])
def test_dictionary_timestamp_to_pandas_options(options):
dictionary = pa.array([0, 1000], type=pa.timestamp('ms', tz='Europe/Berlin'))
encoded = pa.DictionaryArray.from_arrays(
pa.array([0, None, 1, 0]), dictionary, ordered=True)
categories = dictionary.to_pandas(**options)
if options.get('timestamp_as_object'):
categories = categories.to_numpy()
expected = pd.Series(pd.Categorical.from_codes(
[0, -1, 1, 0], categories=categories, ordered=True),
name='t')
tm.assert_series_equal(encoded.to_pandas(**options).rename('t'), expected)
tm.assert_series_equal(
pa.table({'t': encoded}).to_pandas(**options)['t'], expected)


@pytest.mark.parametrize('tz', ['UTC', 'Asia/Kolkata'])
def test_dictionary_timestamp_to_pandas_no_chunks(tz):
dtype = pa.dictionary(pa.int32(), pa.timestamp('ns', tz=tz))
array = pa.chunked_array([], type=dtype)
expected = pd.Series(pd.Categorical.from_codes(
[], categories=pd.DatetimeIndex([], dtype=f'datetime64[ns, {tz}]')),
name='t')
tm.assert_series_equal(array.to_pandas().rename('t'), expected)
tm.assert_series_equal(pa.table({'t': array}).to_pandas()['t'], expected)


def test_variable_dictionary_to_pandas():
np.random.seed(12345)

Expand Down
Loading