GH-50398 [Python] Use more CPython Limited APIs in python/pyarrow

python/pyarrow/io.pxi

@@ -415,7 +415,7 @@ cdef class NativeFile(_Weakrefable):
         # Allocate empty write space
         obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes)
 
-        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
+        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AsString(<object> obj)
         with nogil:
             bytes_read = GetResultValue(handle.get().Read(c_nbytes, buf))
 
@@ -488,7 +488,7 @@ cdef class NativeFile(_Weakrefable):
         # Allocate empty write space
         obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes)
 
-        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
+        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AsString(<object> obj)
         with nogil:
             bytes_read = GetResultValue(handle.get().
                                         ReadAt(c_offset, c_nbytes, buf))
@@ -1612,7 +1612,7 @@ cdef class Buffer(_Weakrefable):
         if buffer.buf == NULL:
             # ARROW-16048: Ensure we don't export a NULL address.
             assert buffer.len == 0
-            buffer.buf = cp.PyBytes_AS_STRING(b"")
+            buffer.buf = cp.PyBytes_AsString(b"")
         buffer.format = 'b'
         buffer.internal = NULL
         buffer.itemsize = 1
@@ -2641,7 +2641,7 @@ cdef class Codec(_Weakrefable):
 
         if asbytes:
             pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size)
-            output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(<object> pyobj)
+            output_buffer = <uint8_t*> cp.PyBytes_AsString(<object> pyobj)
         else:
             out_buf = allocate_buffer(
                 max_output_size, memory_pool=memory_pool, resizable=True
@@ -2703,7 +2703,7 @@ cdef class Codec(_Weakrefable):
 
         if asbytes:
             pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size)
-            output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(pybuf)
+            output_buffer = <uint8_t*> cp.PyBytes_AsString(pybuf)
         else:
             out_buf = allocate_buffer(output_size, memory_pool=memory_pool)
             output_buffer = out_buf.buffer.get().mutable_data()

python/pyarrow/src/arrow/python/arrow_to_pandas.cc

@@ -1016,8 +1016,8 @@ Status ConvertMap(PandasOptions options, const ChunkedArray& data,
           return CheckPyError();
         },
         [&list_item](int64_t idx, OwnedRef& key_value, OwnedRef& item_value) {
-          PyList_SET_ITEM(list_item.obj(), idx,
-                          PyTuple_Pack(2, key_value.obj(), item_value.obj()));
+          PyList_SetItem(list_item.obj(), idx,
+                         PyTuple_Pack(2, key_value.obj(), item_value.obj()));
           return CheckPyError();
         },
         [&list_item] { return list_item.detach(); }, data, py_keys, py_items, item_arrays,

python/pyarrow/src/arrow/python/benchmark.cc

@@ -29,7 +29,7 @@ void Benchmark_PandasObjectIsNull(PyObject* list) {
   }
   Py_ssize_t i, n = PyList_GET_SIZE(list);
   for (i = 0; i < n; i++) {
-    internal::PandasObjectIsNull(PyList_GET_ITEM(list, i));
+    internal::PandasObjectIsNull(PyList_GetItem(list, i));
   }
 }
 

python/pyarrow/src/arrow/python/common.h

@@ -23,6 +23,7 @@
 #include <utility>
 
 #include "arrow/buffer.h"
+#include "arrow/python/helpers.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/visibility.h"
 #include "arrow/result.h"
@@ -398,12 +399,16 @@ struct PyBytesView {
   // View the given Python object as binary-like, i.e. bytes
   Status ParseBinary(PyObject* obj) {
     if (PyBytes_Check(obj)) {
-      bytes = PyBytes_AS_STRING(obj);
-      size = PyBytes_GET_SIZE(obj);
+      bytes = PyBytes_AsString(obj);
+      RETURN_IF_PYERROR();
+      size = PyBytes_Size(obj);
+      RETURN_IF_PYERROR();
       is_utf8 = false;
     } else if (PyByteArray_Check(obj)) {
-      bytes = PyByteArray_AS_STRING(obj);
-      size = PyByteArray_GET_SIZE(obj);
+      bytes = PyByteArray_AsString(obj);
+      RETURN_IF_PYERROR();
+      size = PyByteArray_Size(obj);
+      RETURN_IF_PYERROR();
       is_utf8 = false;
     } else if (PyMemoryView_Check(obj)) {
       PyObject* ref = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C');
@@ -413,8 +418,8 @@ struct PyBytesView {
       size = buffer->len;
       is_utf8 = false;
     } else {
-      return Status::TypeError("Expected bytes, got a '", Py_TYPE(obj)->tp_name,
-                               "' object");
+      return Status::TypeError("Expected bytes, got a '",
+                               internal::PyObject_StdStringTypeName(obj), "' object");
     }
     return Status::OK();
   }
@@ -425,10 +430,13 @@ struct PyBytesView {
     RETURN_IF_PYERROR();
     if (!PyBytes_Check(ref.obj())) {
       return Status::TypeError("Expected uuid.UUID.bytes to return bytes, got '",
-                               Py_TYPE(ref.obj())->tp_name, "' object");
+                               internal::PyObject_StdStringTypeName(ref.obj()),
+                               "' object");
     }
-    bytes = PyBytes_AS_STRING(ref.obj());
-    size = PyBytes_GET_SIZE(ref.obj());
+    bytes = PyBytes_AsString(ref.obj());
+    RETURN_IF_PYERROR();
+    size = PyBytes_Size(ref.obj());
+    RETURN_IF_PYERROR();
     is_utf8 = false;
     return Status::OK();
   }

python/pyarrow/src/arrow/python/datetime.cc

@@ -58,7 +58,7 @@ constexpr char* NonConst(const char* st) {
   return const_cast<char*>(st);
 }
 
-static PyTypeObject MonthDayNanoTupleType = {};
+static PyTypeObject* MonthDayNanoTupleType = nullptr;
 
 static PyStructSequence_Field MonthDayNanoField[] = {
     {NonConst("months"), NonConst("The number of months in the interval")},
@@ -274,13 +274,14 @@ static inline Status PyDate_convert_int(int64_t val, const DateUnit unit, int64_
 }
 
 PyObject* NewMonthDayNanoTupleType() {
-  if (MonthDayNanoTupleType.tp_name == nullptr) {
-    if (PyStructSequence_InitType2(&MonthDayNanoTupleType, &MonthDayNanoTupleDesc) != 0) {
+  if (MonthDayNanoTupleType == nullptr) {
+    MonthDayNanoTupleType = PyStructSequence_NewType(&MonthDayNanoTupleDesc);
+    if (MonthDayNanoTupleType == nullptr) {
       Py_FatalError("Could not initialize MonthDayNanoTuple");
     }
   }
-  Py_INCREF(&MonthDayNanoTupleType);
-  return (PyObject*)&MonthDayNanoTupleType;
+  Py_INCREF(MonthDayNanoTupleType);
+  return (PyObject*)MonthDayNanoTupleType;
 }
 
 Status PyTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out) {
@@ -596,7 +597,7 @@ Result<std::string> TzinfoToString(PyObject* tzinfo) {
 
 PyObject* MonthDayNanoIntervalToNamedTuple(
     const MonthDayNanoIntervalType::MonthDayNanos& interval) {
-  OwnedRef tuple(PyStructSequence_New(&MonthDayNanoTupleType));
+  OwnedRef tuple(PyStructSequence_New(MonthDayNanoTupleType));
   if (ARROW_PREDICT_FALSE(tuple.obj() == nullptr)) {
     return nullptr;
   }

python/pyarrow/src/arrow/python/decimal.cc

@@ -159,7 +159,7 @@ Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
     return InternalDecimalFromPythonDecimal<ArrowDecimal>(obj, arrow_type, out);
   } else {
     return Status::TypeError("int or Decimal object expected, got ",
-                             Py_TYPE(obj)->tp_name);
+                             internal::PyObject_StdStringTypeName(obj));
   }
 }
 

python/pyarrow/src/arrow/python/extension_type.cc

@@ -78,8 +78,8 @@ std::string PyExtensionType::ToString(bool show_metadata) const {
 
   std::stringstream ss;
   OwnedRef instance(GetInstance());
-  ss << "extension<" << this->extension_name() << "<" << Py_TYPE(instance.obj())->tp_name
-     << ">>";
+  ss << "extension<" << this->extension_name() << "<"
+     << internal::PyObject_StdStringTypeName(instance.obj()) << ">>";
   return ss.str();
 }
 

python/pyarrow/src/arrow/python/helpers.cc

@@ -89,15 +89,16 @@ Result<uint16_t> PyFloat_AsHalf(PyObject* obj) {
     return PyArrayScalar_VAL(obj, Half);
   } else {
     return Status::TypeError("conversion to float16 expects a `float` or ",
-                             "`np.float16` object, got ", Py_TYPE(obj)->tp_name);
+                             "`np.float16` object, got ",
+                             internal::PyObject_StdStringTypeName(obj));
   }
 }
 
 namespace internal {
 
 std::string PyBytes_AsStdString(PyObject* obj) {
   ARROW_DCHECK(PyBytes_Check(obj));
-  return std::string(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj));
+  return std::string(PyBytes_AsString(obj), PyBytes_Size(obj));
 }
 
 Status PyUnicode_AsStdString(PyObject* obj, std::string* out) {
@@ -121,12 +122,30 @@ std::string PyObject_StdStringRepr(PyObject* obj) {
   if (!bytes_ref) {
     PyErr_Clear();
     std::stringstream ss;
-    ss << "<object of type '" << Py_TYPE(obj)->tp_name << "' repr() failed>";
+    ss << "<object of type '" << PyObject_StdStringTypeName(obj) << "' repr() failed>";
     return ss.str();
   }
   return PyBytes_AsStdString(bytes_ref.obj());
 }
 
+std::string PyObject_StdStringTypeName(PyObject* obj) {
+  // Once Python 3.10 is dropped, this can use PyType_GetName(Py_TYPE(obj)) (
+  // added in 3.11).
+  OwnedRef name_ref(
+      PyObject_GetAttrString(reinterpret_cast<PyObject*>(Py_TYPE(obj)), "__name__"));
+  if (!name_ref) {
+    PyErr_Clear();
+    return "?";
+  }
+  Py_ssize_t size;
+  const char* data = PyUnicode_AsUTF8AndSize(name_ref.obj(), &size);
+  if (data == nullptr) {
+    PyErr_Clear();
+    return "?";
+  }
+  return std::string(data, size);
+}
+
 Status PyObject_StdStringStr(PyObject* obj, std::string* out) {
   OwnedRef string_ref(PyObject_Str(obj));
   RETURN_IF_PYERROR();
@@ -176,9 +195,10 @@ Result<OwnedRef> PyObjectToPyInt(PyObject* obj) {
     return std::move(ref);
   }
   PyErr_Clear();
-  const auto nb = Py_TYPE(obj)->tp_as_number;
-  if (nb && nb->nb_int) {
-    ref.reset(nb->nb_int(obj));
+  const auto nb_int =
+      reinterpret_cast<unaryfunc>(PyType_GetSlot(Py_TYPE(obj), Py_nb_int));
+  if (nb_int) {
+    ref.reset(nb_int(obj));
     if (!ref) {
       RETURN_IF_PYERROR();
     }
@@ -286,7 +306,7 @@ inline bool MayHaveNaN(PyObject* obj) {
                                   Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_BYTES_SUBCLASS |
                                   Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS |
                                   Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
-  return !PyType_HasFeature(Py_TYPE(obj), non_nan_tpflags);
+  return (PyType_GetFlags(Py_TYPE(obj)) & non_nan_tpflags) == 0;
 }
 
 bool PyFloat_IsNaN(PyObject* obj) {
@@ -433,13 +453,13 @@ PyObject* BorrowPandasDataOffsetType() { return pandas_DateOffset; }
 Status InvalidValue(PyObject* obj, const std::string& why) {
   auto obj_as_str = PyObject_StdStringRepr(obj);
   return Status::Invalid("Could not convert ", std::move(obj_as_str), " with type ",
-                         Py_TYPE(obj)->tp_name, ": ", why);
+                         PyObject_StdStringTypeName(obj), ": ", why);
 }
 
 Status InvalidType(PyObject* obj, const std::string& why) {
   auto obj_as_str = PyObject_StdStringRepr(obj);
   return Status::TypeError("Could not convert ", std::move(obj_as_str), " with type ",
-                           Py_TYPE(obj)->tp_name, ": ", why);
+                           PyObject_StdStringTypeName(obj), ": ", why);
 }
 
 Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {

python/pyarrow/src/arrow/python/helpers.h

@@ -123,6 +123,10 @@ Status PyObject_StdStringStr(PyObject* obj, std::string* out);
 ARROW_PYTHON_EXPORT
 std::string PyObject_StdStringRepr(PyObject* obj);
 
+// \brief Return the type name of the given object as a std::string
+ARROW_PYTHON_EXPORT
+std::string PyObject_StdStringTypeName(PyObject* obj);
+
 // \brief Cast the given size to int32_t, with error checking
 inline Status CastSize(Py_ssize_t size, int32_t* out,
                        const char* error_msg = "Maximum size exceeded (2GB)") {

python/pyarrow/src/arrow/python/inference.cc

@@ -708,7 +708,7 @@ class TypeInferrer {
         key = internal::PyBytes_AsStdString(key_obj);
       } else {
         return Status::TypeError("Expected dict key of type str or bytes, got '",
-                                 Py_TYPE(key_obj)->tp_name, "'");
+                                 internal::PyObject_StdStringTypeName(key_obj), "'");
       }
       // Get or create visitor for this key
       TypeInferrer* visitor;

python/pyarrow/src/arrow/python/io.cc

@@ -233,7 +233,8 @@ Result<int64_t> PyReadableFile::Read(int64_t nbytes, void* out) {
       return Status::TypeError(
           "Python file read() should have returned a bytes object or an object "
           "supporting the buffer protocol, got '",
-          Py_TYPE(bytes_obj)->tp_name, "' (did you open the file in binary mode?)");
+          internal::PyObject_StdStringTypeName(bytes_obj),
+          "' (did you open the file in binary mode?)");
     }
   });
 }

python/pyarrow/src/arrow/python/numpy_to_arrow.cc

@@ -690,9 +690,11 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder
     return Status::Invalid("failed converting UTF32 to UTF8");
   }
 
-  const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(utf8_obj.obj()));
-  return builder->Append(
-      reinterpret_cast<const uint8_t*>(PyBytes_AS_STRING(utf8_obj.obj())), length);
+  const int32_t length = static_cast<int32_t>(PyBytes_Size(utf8_obj.obj()));
+  RETURN_IF_PYERROR();
+  const char* utf8_data = PyBytes_AsString(utf8_obj.obj());
+  RETURN_IF_PYERROR();
+  return builder->Append(reinterpret_cast<const uint8_t*>(utf8_data), length);
 }
 
 }  // namespace
@@ -836,9 +838,12 @@ Status NumPyConverter::Visit(const StructType& type) {
         return Status::Invalid("Missing field '", field->name(), "' in struct array");
       }
       PyArray_Descr* sub_dtype =
-          reinterpret_cast<PyArray_Descr*>(PyTuple_GET_ITEM(tup, 0));
+          reinterpret_cast<PyArray_Descr*>(PyTuple_GetItem(tup, 0));
+      RETURN_IF_PYERROR();
       ARROW_DCHECK(PyObject_TypeCheck(sub_dtype, &PyArrayDescr_Type));
-      int offset = static_cast<int>(PyLong_AsLong(PyTuple_GET_ITEM(tup, 1)));
+      PyObject* offset_obj = PyTuple_GetItem(tup, 1);
+      RETURN_IF_PYERROR();
+      int offset = static_cast<int>(PyLong_AsLong(offset_obj));
       RETURN_IF_PYERROR();
       Py_INCREF(sub_dtype); /* PyArray_GetField() steals ref */
       PyObject* sub_array = PyArray_GetField(arr_, sub_dtype, offset);

python/pyarrow/src/arrow/python/pyarrow.cc

@@ -37,7 +37,8 @@ namespace py {
 
 static Status UnwrapError(PyObject* obj, const char* expected_type) {
   return Status::TypeError("Could not unwrap ", expected_type,
-                           " from Python object of type '", Py_TYPE(obj)->tp_name, "'");
+                           " from Python object of type '",
+                           internal::PyObject_StdStringTypeName(obj), "'");
 }
 
 int import_pyarrow() {

python/pyarrow/src/arrow/python/python_to_arrow.cc

@@ -255,7 +255,7 @@ class PyValue {
   static Result<double> Convert(const DoubleType*, const O&, I obj) {
     double value;
     if (PyFloat_Check(obj)) {
-      value = PyFloat_AS_DOUBLE(obj);
+      value = PyFloat_AsDouble(obj);
     } else if (internal::PyFloatScalar_Check(obj)) {
       // Other kinds of float-y things
       value = PyFloat_AsDouble(obj);
@@ -445,12 +445,11 @@ class PyValue {
       return output;
     }
     if (PyTuple_Check(obj) && PyTuple_Size(obj) == 3) {
-      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 0), &output.months,
+      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 0), &output.months,
                                              "Months (tuple item #0) too large"));
-      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 1), &output.days,
+      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 1), &output.days,
                                              "Days (tuple item #1) too large"));
-      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 2),
-                                             &output.nanoseconds,
+      RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 2), &output.nanoseconds,
                                              "Nanoseconds (tuple item #2) too large"));
       return output;
     }
@@ -1050,8 +1049,8 @@ class PyStructConverter : public StructConverter<PyConverter, PyConverterTrait>
       PyObject* unicode =
           PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size());
       RETURN_IF_PYERROR();
-      PyList_SET_ITEM(bytes_field_names_.obj(), i, bytes);
-      PyList_SET_ITEM(unicode_field_names_.obj(), i, unicode);
+      PyList_SetItem(bytes_field_names_.obj(), i, bytes);
+      PyList_SetItem(unicode_field_names_.obj(), i, unicode);
     }
     return Status::OK();
   }
@@ -1111,7 +1110,7 @@ class PyStructConverter : public StructConverter<PyConverter, PyConverterTrait>
       return Status::Invalid("Tuple size must be equal to number of struct fields");
     }
     for (int i = 0; i < num_fields_; i++) {
-      PyObject* value = PyTuple_GET_ITEM(tuple, i);
+      PyObject* value = PyTuple_GetItem(tuple, i);
       RETURN_NOT_OK(this->children_[i]->Append(value));
     }
     return Status::OK();
@@ -1266,7 +1265,7 @@ Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* siz
         RETURN_IF_PYERROR();
         break;
       }
-      PyList_SET_ITEM(lst, i, item);
+      PyList_SetItem(lst, i, item);
     }
     // Shrink list if len(iterator) < size
     if (i < n && PyList_SetSlice(lst, i, n, NULL)) {