diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index b648fbf66980..469f0f2674f1 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -415,7 +415,7 @@ cdef class NativeFile(_Weakrefable): # Allocate empty write space obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) - cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + cdef uint8_t* buf = cp.PyBytes_AsString( obj) with nogil: bytes_read = GetResultValue(handle.get().Read(c_nbytes, buf)) @@ -488,7 +488,7 @@ cdef class NativeFile(_Weakrefable): # Allocate empty write space obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) - cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + cdef uint8_t* buf = cp.PyBytes_AsString( obj) with nogil: bytes_read = GetResultValue(handle.get(). ReadAt(c_offset, c_nbytes, buf)) @@ -1612,7 +1612,7 @@ cdef class Buffer(_Weakrefable): if buffer.buf == NULL: # ARROW-16048: Ensure we don't export a NULL address. assert buffer.len == 0 - buffer.buf = cp.PyBytes_AS_STRING(b"") + buffer.buf = cp.PyBytes_AsString(b"") buffer.format = 'b' buffer.internal = NULL buffer.itemsize = 1 @@ -2641,7 +2641,7 @@ cdef class Codec(_Weakrefable): if asbytes: pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size) - output_buffer = cp.PyBytes_AS_STRING( pyobj) + output_buffer = cp.PyBytes_AsString( pyobj) else: out_buf = allocate_buffer( max_output_size, memory_pool=memory_pool, resizable=True @@ -2703,7 +2703,7 @@ cdef class Codec(_Weakrefable): if asbytes: pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size) - output_buffer = cp.PyBytes_AS_STRING(pybuf) + output_buffer = cp.PyBytes_AsString(pybuf) else: out_buf = allocate_buffer(output_size, memory_pool=memory_pool) output_buffer = out_buf.buffer.get().mutable_data() diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index f163266f3b87..6818f59408ce 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1016,8 +1016,8 @@ Status ConvertMap(PandasOptions options, const ChunkedArray& data, return CheckPyError(); }, [&list_item](int64_t idx, OwnedRef& key_value, OwnedRef& item_value) { - PyList_SET_ITEM(list_item.obj(), idx, - PyTuple_Pack(2, key_value.obj(), item_value.obj())); + PyList_SetItem(list_item.obj(), idx, + PyTuple_Pack(2, key_value.obj(), item_value.obj())); return CheckPyError(); }, [&list_item] { return list_item.detach(); }, data, py_keys, py_items, item_arrays, diff --git a/python/pyarrow/src/arrow/python/benchmark.cc b/python/pyarrow/src/arrow/python/benchmark.cc index 6dcc959ed221..2d66f9f9d3f7 100644 --- a/python/pyarrow/src/arrow/python/benchmark.cc +++ b/python/pyarrow/src/arrow/python/benchmark.cc @@ -29,7 +29,7 @@ void Benchmark_PandasObjectIsNull(PyObject* list) { } Py_ssize_t i, n = PyList_GET_SIZE(list); for (i = 0; i < n; i++) { - internal::PandasObjectIsNull(PyList_GET_ITEM(list, i)); + internal::PandasObjectIsNull(PyList_GetItem(list, i)); } } diff --git a/python/pyarrow/src/arrow/python/common.h b/python/pyarrow/src/arrow/python/common.h index a81782330b5e..52b18a9e56d9 100644 --- a/python/pyarrow/src/arrow/python/common.h +++ b/python/pyarrow/src/arrow/python/common.h @@ -23,6 +23,7 @@ #include #include "arrow/buffer.h" +#include "arrow/python/helpers.h" #include "arrow/python/pyarrow.h" #include "arrow/python/visibility.h" #include "arrow/result.h" @@ -398,12 +399,16 @@ struct PyBytesView { // View the given Python object as binary-like, i.e. bytes Status ParseBinary(PyObject* obj) { if (PyBytes_Check(obj)) { - bytes = PyBytes_AS_STRING(obj); - size = PyBytes_GET_SIZE(obj); + bytes = PyBytes_AsString(obj); + RETURN_IF_PYERROR(); + size = PyBytes_Size(obj); + RETURN_IF_PYERROR(); is_utf8 = false; } else if (PyByteArray_Check(obj)) { - bytes = PyByteArray_AS_STRING(obj); - size = PyByteArray_GET_SIZE(obj); + bytes = PyByteArray_AsString(obj); + RETURN_IF_PYERROR(); + size = PyByteArray_Size(obj); + RETURN_IF_PYERROR(); is_utf8 = false; } else if (PyMemoryView_Check(obj)) { PyObject* ref = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C'); @@ -413,8 +418,8 @@ struct PyBytesView { size = buffer->len; is_utf8 = false; } else { - return Status::TypeError("Expected bytes, got a '", Py_TYPE(obj)->tp_name, - "' object"); + return Status::TypeError("Expected bytes, got a '", + internal::PyObject_StdStringTypeName(obj), "' object"); } return Status::OK(); } @@ -425,10 +430,13 @@ struct PyBytesView { RETURN_IF_PYERROR(); if (!PyBytes_Check(ref.obj())) { return Status::TypeError("Expected uuid.UUID.bytes to return bytes, got '", - Py_TYPE(ref.obj())->tp_name, "' object"); + internal::PyObject_StdStringTypeName(ref.obj()), + "' object"); } - bytes = PyBytes_AS_STRING(ref.obj()); - size = PyBytes_GET_SIZE(ref.obj()); + bytes = PyBytes_AsString(ref.obj()); + RETURN_IF_PYERROR(); + size = PyBytes_Size(ref.obj()); + RETURN_IF_PYERROR(); is_utf8 = false; return Status::OK(); } diff --git a/python/pyarrow/src/arrow/python/datetime.cc b/python/pyarrow/src/arrow/python/datetime.cc index 6a835c2d37ce..2311a563d4ee 100644 --- a/python/pyarrow/src/arrow/python/datetime.cc +++ b/python/pyarrow/src/arrow/python/datetime.cc @@ -58,7 +58,7 @@ constexpr char* NonConst(const char* st) { return const_cast(st); } -static PyTypeObject MonthDayNanoTupleType = {}; +static PyTypeObject* MonthDayNanoTupleType = nullptr; static PyStructSequence_Field MonthDayNanoField[] = { {NonConst("months"), NonConst("The number of months in the interval")}, @@ -274,13 +274,14 @@ static inline Status PyDate_convert_int(int64_t val, const DateUnit unit, int64_ } PyObject* NewMonthDayNanoTupleType() { - if (MonthDayNanoTupleType.tp_name == nullptr) { - if (PyStructSequence_InitType2(&MonthDayNanoTupleType, &MonthDayNanoTupleDesc) != 0) { + if (MonthDayNanoTupleType == nullptr) { + MonthDayNanoTupleType = PyStructSequence_NewType(&MonthDayNanoTupleDesc); + if (MonthDayNanoTupleType == nullptr) { Py_FatalError("Could not initialize MonthDayNanoTuple"); } } - Py_INCREF(&MonthDayNanoTupleType); - return (PyObject*)&MonthDayNanoTupleType; + Py_INCREF(MonthDayNanoTupleType); + return (PyObject*)MonthDayNanoTupleType; } Status PyTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out) { @@ -596,7 +597,7 @@ Result TzinfoToString(PyObject* tzinfo) { PyObject* MonthDayNanoIntervalToNamedTuple( const MonthDayNanoIntervalType::MonthDayNanos& interval) { - OwnedRef tuple(PyStructSequence_New(&MonthDayNanoTupleType)); + OwnedRef tuple(PyStructSequence_New(MonthDayNanoTupleType)); if (ARROW_PREDICT_FALSE(tuple.obj() == nullptr)) { return nullptr; } diff --git a/python/pyarrow/src/arrow/python/decimal.cc b/python/pyarrow/src/arrow/python/decimal.cc index 1ba8d32192af..298820cfe2de 100644 --- a/python/pyarrow/src/arrow/python/decimal.cc +++ b/python/pyarrow/src/arrow/python/decimal.cc @@ -159,7 +159,7 @@ Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, return InternalDecimalFromPythonDecimal(obj, arrow_type, out); } else { return Status::TypeError("int or Decimal object expected, got ", - Py_TYPE(obj)->tp_name); + internal::PyObject_StdStringTypeName(obj)); } } diff --git a/python/pyarrow/src/arrow/python/extension_type.cc b/python/pyarrow/src/arrow/python/extension_type.cc index 8c3f3f1d8d0f..6904be0976f5 100644 --- a/python/pyarrow/src/arrow/python/extension_type.cc +++ b/python/pyarrow/src/arrow/python/extension_type.cc @@ -78,8 +78,8 @@ std::string PyExtensionType::ToString(bool show_metadata) const { std::stringstream ss; OwnedRef instance(GetInstance()); - ss << "extension<" << this->extension_name() << "<" << Py_TYPE(instance.obj())->tp_name - << ">>"; + ss << "extension<" << this->extension_name() << "<" + << internal::PyObject_StdStringTypeName(instance.obj()) << ">>"; return ss.str(); } diff --git a/python/pyarrow/src/arrow/python/helpers.cc b/python/pyarrow/src/arrow/python/helpers.cc index dc42c0ab770f..98a8a87ad27a 100644 --- a/python/pyarrow/src/arrow/python/helpers.cc +++ b/python/pyarrow/src/arrow/python/helpers.cc @@ -89,7 +89,8 @@ Result PyFloat_AsHalf(PyObject* obj) { return PyArrayScalar_VAL(obj, Half); } else { return Status::TypeError("conversion to float16 expects a `float` or ", - "`np.float16` object, got ", Py_TYPE(obj)->tp_name); + "`np.float16` object, got ", + internal::PyObject_StdStringTypeName(obj)); } } @@ -97,7 +98,7 @@ namespace internal { std::string PyBytes_AsStdString(PyObject* obj) { ARROW_DCHECK(PyBytes_Check(obj)); - return std::string(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + return std::string(PyBytes_AsString(obj), PyBytes_Size(obj)); } Status PyUnicode_AsStdString(PyObject* obj, std::string* out) { @@ -121,12 +122,30 @@ std::string PyObject_StdStringRepr(PyObject* obj) { if (!bytes_ref) { PyErr_Clear(); std::stringstream ss; - ss << "tp_name << "' repr() failed>"; + ss << ""; return ss.str(); } return PyBytes_AsStdString(bytes_ref.obj()); } +std::string PyObject_StdStringTypeName(PyObject* obj) { + // Once Python 3.10 is dropped, this can use PyType_GetName(Py_TYPE(obj)) ( + // added in 3.11). + OwnedRef name_ref( + PyObject_GetAttrString(reinterpret_cast(Py_TYPE(obj)), "__name__")); + if (!name_ref) { + PyErr_Clear(); + return "?"; + } + Py_ssize_t size; + const char* data = PyUnicode_AsUTF8AndSize(name_ref.obj(), &size); + if (data == nullptr) { + PyErr_Clear(); + return "?"; + } + return std::string(data, size); +} + Status PyObject_StdStringStr(PyObject* obj, std::string* out) { OwnedRef string_ref(PyObject_Str(obj)); RETURN_IF_PYERROR(); @@ -176,9 +195,10 @@ Result PyObjectToPyInt(PyObject* obj) { return std::move(ref); } PyErr_Clear(); - const auto nb = Py_TYPE(obj)->tp_as_number; - if (nb && nb->nb_int) { - ref.reset(nb->nb_int(obj)); + const auto nb_int = + reinterpret_cast(PyType_GetSlot(Py_TYPE(obj), Py_nb_int)); + if (nb_int) { + ref.reset(nb_int(obj)); if (!ref) { RETURN_IF_PYERROR(); } @@ -286,7 +306,7 @@ inline bool MayHaveNaN(PyObject* obj) { Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_BYTES_SUBCLASS | Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS; - return !PyType_HasFeature(Py_TYPE(obj), non_nan_tpflags); + return (PyType_GetFlags(Py_TYPE(obj)) & non_nan_tpflags) == 0; } bool PyFloat_IsNaN(PyObject* obj) { @@ -433,13 +453,13 @@ PyObject* BorrowPandasDataOffsetType() { return pandas_DateOffset; } Status InvalidValue(PyObject* obj, const std::string& why) { auto obj_as_str = PyObject_StdStringRepr(obj); return Status::Invalid("Could not convert ", std::move(obj_as_str), " with type ", - Py_TYPE(obj)->tp_name, ": ", why); + PyObject_StdStringTypeName(obj), ": ", why); } Status InvalidType(PyObject* obj, const std::string& why) { auto obj_as_str = PyObject_StdStringRepr(obj); return Status::TypeError("Could not convert ", std::move(obj_as_str), " with type ", - Py_TYPE(obj)->tp_name, ": ", why); + PyObject_StdStringTypeName(obj), ": ", why); } Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) { diff --git a/python/pyarrow/src/arrow/python/helpers.h b/python/pyarrow/src/arrow/python/helpers.h index 7393784f484b..790020642b31 100644 --- a/python/pyarrow/src/arrow/python/helpers.h +++ b/python/pyarrow/src/arrow/python/helpers.h @@ -123,6 +123,10 @@ Status PyObject_StdStringStr(PyObject* obj, std::string* out); ARROW_PYTHON_EXPORT std::string PyObject_StdStringRepr(PyObject* obj); +// \brief Return the type name of the given object as a std::string +ARROW_PYTHON_EXPORT +std::string PyObject_StdStringTypeName(PyObject* obj); + // \brief Cast the given size to int32_t, with error checking inline Status CastSize(Py_ssize_t size, int32_t* out, const char* error_msg = "Maximum size exceeded (2GB)") { diff --git a/python/pyarrow/src/arrow/python/inference.cc b/python/pyarrow/src/arrow/python/inference.cc index 291cc4214957..427535fc2218 100644 --- a/python/pyarrow/src/arrow/python/inference.cc +++ b/python/pyarrow/src/arrow/python/inference.cc @@ -708,7 +708,7 @@ class TypeInferrer { key = internal::PyBytes_AsStdString(key_obj); } else { return Status::TypeError("Expected dict key of type str or bytes, got '", - Py_TYPE(key_obj)->tp_name, "'"); + internal::PyObject_StdStringTypeName(key_obj), "'"); } // Get or create visitor for this key TypeInferrer* visitor; diff --git a/python/pyarrow/src/arrow/python/io.cc b/python/pyarrow/src/arrow/python/io.cc index 7a4d18ab9fa2..a4690e1f2f44 100644 --- a/python/pyarrow/src/arrow/python/io.cc +++ b/python/pyarrow/src/arrow/python/io.cc @@ -233,7 +233,8 @@ Result PyReadableFile::Read(int64_t nbytes, void* out) { return Status::TypeError( "Python file read() should have returned a bytes object or an object " "supporting the buffer protocol, got '", - Py_TYPE(bytes_obj)->tp_name, "' (did you open the file in binary mode?)"); + internal::PyObject_StdStringTypeName(bytes_obj), + "' (did you open the file in binary mode?)"); } }); } diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 5647e895d0f7..05842f4a024e 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -690,9 +690,11 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder return Status::Invalid("failed converting UTF32 to UTF8"); } - const int32_t length = static_cast(PyBytes_GET_SIZE(utf8_obj.obj())); - return builder->Append( - reinterpret_cast(PyBytes_AS_STRING(utf8_obj.obj())), length); + const int32_t length = static_cast(PyBytes_Size(utf8_obj.obj())); + RETURN_IF_PYERROR(); + const char* utf8_data = PyBytes_AsString(utf8_obj.obj()); + RETURN_IF_PYERROR(); + return builder->Append(reinterpret_cast(utf8_data), length); } } // namespace @@ -836,9 +838,12 @@ Status NumPyConverter::Visit(const StructType& type) { return Status::Invalid("Missing field '", field->name(), "' in struct array"); } PyArray_Descr* sub_dtype = - reinterpret_cast(PyTuple_GET_ITEM(tup, 0)); + reinterpret_cast(PyTuple_GetItem(tup, 0)); + RETURN_IF_PYERROR(); ARROW_DCHECK(PyObject_TypeCheck(sub_dtype, &PyArrayDescr_Type)); - int offset = static_cast(PyLong_AsLong(PyTuple_GET_ITEM(tup, 1))); + PyObject* offset_obj = PyTuple_GetItem(tup, 1); + RETURN_IF_PYERROR(); + int offset = static_cast(PyLong_AsLong(offset_obj)); RETURN_IF_PYERROR(); Py_INCREF(sub_dtype); /* PyArray_GetField() steals ref */ PyObject* sub_array = PyArray_GetField(arr_, sub_dtype, offset); diff --git a/python/pyarrow/src/arrow/python/pyarrow.cc b/python/pyarrow/src/arrow/python/pyarrow.cc index 4f675921708e..c5b1c929ffb7 100644 --- a/python/pyarrow/src/arrow/python/pyarrow.cc +++ b/python/pyarrow/src/arrow/python/pyarrow.cc @@ -37,7 +37,8 @@ namespace py { static Status UnwrapError(PyObject* obj, const char* expected_type) { return Status::TypeError("Could not unwrap ", expected_type, - " from Python object of type '", Py_TYPE(obj)->tp_name, "'"); + " from Python object of type '", + internal::PyObject_StdStringTypeName(obj), "'"); } int import_pyarrow() { diff --git a/python/pyarrow/src/arrow/python/python_to_arrow.cc b/python/pyarrow/src/arrow/python/python_to_arrow.cc index 47290a7e7f10..da79896d2b4b 100644 --- a/python/pyarrow/src/arrow/python/python_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/python_to_arrow.cc @@ -255,7 +255,7 @@ class PyValue { static Result Convert(const DoubleType*, const O&, I obj) { double value; if (PyFloat_Check(obj)) { - value = PyFloat_AS_DOUBLE(obj); + value = PyFloat_AsDouble(obj); } else if (internal::PyFloatScalar_Check(obj)) { // Other kinds of float-y things value = PyFloat_AsDouble(obj); @@ -445,12 +445,11 @@ class PyValue { return output; } if (PyTuple_Check(obj) && PyTuple_Size(obj) == 3) { - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 0), &output.months, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 0), &output.months, "Months (tuple item #0) too large")); - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 1), &output.days, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 1), &output.days, "Days (tuple item #1) too large")); - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 2), - &output.nanoseconds, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 2), &output.nanoseconds, "Nanoseconds (tuple item #2) too large")); return output; } @@ -1050,8 +1049,8 @@ class PyStructConverter : public StructConverter PyObject* unicode = PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size()); RETURN_IF_PYERROR(); - PyList_SET_ITEM(bytes_field_names_.obj(), i, bytes); - PyList_SET_ITEM(unicode_field_names_.obj(), i, unicode); + PyList_SetItem(bytes_field_names_.obj(), i, bytes); + PyList_SetItem(unicode_field_names_.obj(), i, unicode); } return Status::OK(); } @@ -1111,7 +1110,7 @@ class PyStructConverter : public StructConverter return Status::Invalid("Tuple size must be equal to number of struct fields"); } for (int i = 0; i < num_fields_; i++) { - PyObject* value = PyTuple_GET_ITEM(tuple, i); + PyObject* value = PyTuple_GetItem(tuple, i); RETURN_NOT_OK(this->children_[i]->Append(value)); } return Status::OK(); @@ -1266,7 +1265,7 @@ Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* siz RETURN_IF_PYERROR(); break; } - PyList_SET_ITEM(lst, i, item); + PyList_SetItem(lst, i, item); } // Shrink list if len(iterator) < size if (i < n && PyList_SetSlice(lst, i, n, NULL)) { diff --git a/python/pyarrow/src/arrow/python/udf.cc b/python/pyarrow/src/arrow/python/udf.cc index 940e403ef693..13309bb8d5eb 100644 --- a/python/pyarrow/src/arrow/python/udf.cc +++ b/python/pyarrow/src/arrow/python/udf.cc @@ -210,8 +210,9 @@ struct PythonUdfScalarAggregatorImpl : public ScalarUdfAggregator { out->value = std::move(val); return Status::OK(); } - return Status::TypeError("Unexpected output type: ", - Py_TYPE(result->obj())->tp_name, " (expected Scalar)"); + return Status::TypeError( + "Unexpected output type: ", internal::PyObject_StdStringTypeName(result->obj()), + " (expected Scalar)"); })); return Status::OK(); } @@ -352,7 +353,8 @@ struct PythonUdfHashAggregatorImpl : public HashUdfAggregator { ARROW_RETURN_NOT_OK(builder->AppendScalar(std::move(*val))); } else { return Status::TypeError("Unexpected output type: ", - Py_TYPE(result->obj())->tp_name, " (expected Scalar)"); + internal::PyObject_StdStringTypeName(result->obj()), + " (expected Scalar)"); } } ARROW_ASSIGN_OR_RAISE(auto result, builder->Finish()); @@ -435,8 +437,9 @@ struct PythonUdf : public PythonUdfKernelState { out->value = std::move(val->data()); return Status::OK(); } else { - return Status::TypeError("Unexpected output type: ", Py_TYPE(result.obj())->tp_name, - " (expected Array)"); + return Status::TypeError( + "Unexpected output type: ", internal::PyObject_StdStringTypeName(result.obj()), + " (expected Array)"); } return Status::OK(); }