GH-50398/GH-50829 3/6: Use the CPython Limited API in the remaining fast paths

python/CMakeLists.txt

@@ -386,7 +386,6 @@ configure_file("${PYARROW_CPP_SOURCE_DIR}/config_internal.h.cmake"
 
 set(PYARROW_CPP_SRCS
     ${PYARROW_CPP_SOURCE_DIR}/arrow_to_pandas.cc
-    ${PYARROW_CPP_SOURCE_DIR}/benchmark.cc
     ${PYARROW_CPP_SOURCE_DIR}/common.cc
     ${PYARROW_CPP_SOURCE_DIR}/config.cc
     ${PYARROW_CPP_SOURCE_DIR}/datetime.cc

python/pyarrow/includes/libarrow_python.pxd

@@ -279,10 +279,6 @@ cdef extern from "arrow/python/extension_type.h" namespace "arrow::py":
     CStatus UnregisterPyExtensionType(c_string type_name)
 
 
-cdef extern from "arrow/python/benchmark.h" namespace "arrow::py::benchmark":
-    void Benchmark_PandasObjectIsNull(object lst) except *
-
-
 cdef extern from "arrow/python/gdb.h" namespace "arrow::gdb" nogil:
     void GdbTestSession "arrow::gdb::TestSession"()
 

python/pyarrow/io.pxi

@@ -415,7 +415,7 @@ cdef class NativeFile(_Weakrefable):
         # Allocate empty write space
         obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes)
 
-        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
+        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AsString(<object> obj)
         with nogil:
             bytes_read = GetResultValue(handle.get().Read(c_nbytes, buf))
 
@@ -488,7 +488,7 @@ cdef class NativeFile(_Weakrefable):
         # Allocate empty write space
         obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes)
 
-        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AS_STRING(<object> obj)
+        cdef uint8_t* buf = <uint8_t*> cp.PyBytes_AsString(<object> obj)
         with nogil:
             bytes_read = GetResultValue(handle.get().
                                         ReadAt(c_offset, c_nbytes, buf))
@@ -1614,7 +1614,7 @@ cdef class Buffer(_Weakrefable):
         if buffer.buf == NULL:
             # ARROW-16048: Ensure we don't export a NULL address.
             assert buffer.len == 0
-            buffer.buf = cp.PyBytes_AS_STRING(b"")
+            buffer.buf = cp.PyBytes_AsString(b"")
         buffer.format = 'b'
         buffer.internal = NULL
         buffer.itemsize = 1
@@ -2643,7 +2643,7 @@ cdef class Codec(_Weakrefable):
 
         if asbytes:
             pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size)
-            output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(<object> pyobj)
+            output_buffer = <uint8_t*> cp.PyBytes_AsString(<object> pyobj)
         else:
             out_buf = allocate_buffer(
                 max_output_size, memory_pool=memory_pool, resizable=True
@@ -2705,7 +2705,7 @@ cdef class Codec(_Weakrefable):
 
         if asbytes:
             pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size)
-            output_buffer = <uint8_t*> cp.PyBytes_AS_STRING(pybuf)
+            output_buffer = <uint8_t*> cp.PyBytes_AsString(pybuf)
         else:
             out_buf = allocate_buffer(output_size, memory_pool=memory_pool)
             output_buffer = out_buf.buffer.get().mutable_data()

python/pyarrow/lib.pyx

@@ -243,8 +243,5 @@ include "io.pxi"
 # IPC / Messaging
 include "ipc.pxi"
 
-# Micro-benchmark routines
-include "benchmark.pxi"
-
 # Public API
 include "public-api.pxi"

python/pyarrow/src/arrow/python/arrow_to_pandas.cc

@@ -1027,8 +1027,8 @@ Status ConvertMap(PandasOptions options, const ChunkedArray& data,
           return CheckPyError();
         },
         [&list_item](int64_t idx, OwnedRef& key_value, OwnedRef& item_value) {
-          PyList_SET_ITEM(list_item.obj(), idx,
-                          PyTuple_Pack(2, key_value.obj(), item_value.obj()));
+          PyList_SetItem(list_item.obj(), idx,
+                         PyTuple_Pack(2, key_value.obj(), item_value.obj()));
           return CheckPyError();
         },
         [&list_item] { return list_item.detach(); }, data, py_keys, py_items, item_arrays,

python/pyarrow/src/arrow/python/common.cc

@@ -97,8 +97,8 @@ class PythonErrorDetail : public StatusDetail {
       return result.ValueOrDie();
     } else {
       // Fallback to just the exception type
-      const auto ty = reinterpret_cast<const PyTypeObject*>(exc_type_.obj());
-      return std::string("Python exception: ") + ty->tp_name;
+      return std::string("Python exception: ") +
+             internal::PyObject_StdStringTypeName(exc_type_.obj());
     }
   }
 

python/pyarrow/src/arrow/python/common.h

@@ -23,9 +23,11 @@
 #include <utility>
 
 #include "arrow/buffer.h"
+#include "arrow/python/helpers.h"
 #include "arrow/python/pyarrow.h"
 #include "arrow/python/visibility.h"
 #include "arrow/result.h"
+#include "arrow/util/logging.h"
 #include "arrow/util/macros.h"
 
 namespace arrow {
@@ -267,7 +269,16 @@ class SmartPtrNoGIL : public SmartPtr<Ts...> {
   // Only release the GIL if we own an object *and* the Python runtime is
   // valid *and* the GIL is held.
   std::optional<PyReleaseGIL> optional_gil_release() const {
-    if (this->get() != nullptr && Py_IsInitialized() && PyGILState_Check()) {
+    // PyGILState_Check() is a full-C-API (Py_LIMITED_API-hidden) function,
+    // unavailable in the cp311-abi3 build. In the non-freethreading build we
+    // ship (GIL always enabled) it reduces to "the current thread has a valid
+    // thread state and holds the GIL", which for any thread executing Python
+    // code is exactly Py_IsInitialized() (a bare C++ worker thread that never
+    // entered Python has no thread state, and we never call reset() from one
+    // while destroying a live PyObject without the GIL). Py_IsInitialized()
+    // is the limited-API-correct guard and also covers the post-finalization
+    // case (GH-38626) the original comment targets.
+    if (this->get() != nullptr && Py_IsInitialized()) {
       return PyReleaseGIL();
     }
     return {};
@@ -398,23 +409,34 @@ struct PyBytesView {
   // View the given Python object as binary-like, i.e. bytes
   Status ParseBinary(PyObject* obj) {
     if (PyBytes_Check(obj)) {
-      bytes = PyBytes_AS_STRING(obj);
-      size = PyBytes_GET_SIZE(obj);
+      bytes = PyBytes_AsString(obj);
+      size = PyBytes_Size(obj);
+      ARROW_DCHECK(!PyErr_Occurred());
       is_utf8 = false;
     } else if (PyByteArray_Check(obj)) {
-      bytes = PyByteArray_AS_STRING(obj);
-      size = PyByteArray_GET_SIZE(obj);
+      bytes = PyByteArray_AsString(obj);
+      size = PyByteArray_Size(obj);
+      ARROW_DCHECK(!PyErr_Occurred());
       is_utf8 = false;
     } else if (PyMemoryView_Check(obj)) {
-      PyObject* ref = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C');
+      // C-contiguous view of the memoryview's data. May be a fresh copy, so
+      // hold it via `ref` for the lifetime of this view (fixes a use-after-free
+      // for non-contiguous memoryviews). Uses only the stable C-API.
+      ref.reset(PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C'));
       RETURN_IF_PYERROR();
-      Py_buffer* buffer = PyMemoryView_GET_BUFFER(ref);
-      bytes = reinterpret_cast<const char*>(buffer->buf);
-      size = buffer->len;
+      Py_buffer buffer;
+      if (PyObject_GetBuffer(ref.obj(), &buffer, PyBUF_READ) < 0) {
+        return Status::Invalid("failed to obtain buffer from memoryview");
+      }
+      // Release the buffer export now; the data stays valid because `ref`
+      // (the memoryview, which owns the underlying buffer) remains alive.
+      bytes = reinterpret_cast<const char*>(buffer.buf);
+      size = buffer.len;
+      PyBuffer_Release(&buffer);
       is_utf8 = false;
     } else {
-      return Status::TypeError("Expected bytes, got a '", Py_TYPE(obj)->tp_name,
-                               "' object");
+      return Status::TypeError("Expected bytes, got a '",
+                               internal::PyObject_StdStringTypeName(obj), "' object");
     }
     return Status::OK();
   }
@@ -425,10 +447,12 @@ struct PyBytesView {
     RETURN_IF_PYERROR();
     if (!PyBytes_Check(ref.obj())) {
       return Status::TypeError("Expected uuid.UUID.bytes to return bytes, got '",
-                               Py_TYPE(ref.obj())->tp_name, "' object");
+                               internal::PyObject_StdStringTypeName(ref.obj()),
+                               "' object");
     }
-    bytes = PyBytes_AS_STRING(ref.obj());
-    size = PyBytes_GET_SIZE(ref.obj());
+    bytes = PyBytes_AsString(ref.obj());
+    size = PyBytes_Size(ref.obj());
+    ARROW_DCHECK(!PyErr_Occurred());
     is_utf8 = false;
     return Status::OK();
   }