diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 8eae91e0ed19..88860710d8ff 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -386,7 +386,6 @@ configure_file("${PYARROW_CPP_SOURCE_DIR}/config_internal.h.cmake" set(PYARROW_CPP_SRCS ${PYARROW_CPP_SOURCE_DIR}/arrow_to_pandas.cc - ${PYARROW_CPP_SOURCE_DIR}/benchmark.cc ${PYARROW_CPP_SOURCE_DIR}/common.cc ${PYARROW_CPP_SOURCE_DIR}/config.cc ${PYARROW_CPP_SOURCE_DIR}/datetime.cc diff --git a/python/benchmarks/microbenchmarks.py b/python/benchmarks/microbenchmarks.py deleted file mode 100644 index f8ba383c70b1..000000000000 --- a/python/benchmarks/microbenchmarks.py +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import pyarrow.benchmark as pb - -from . import common - - -class PandasObjectIsNull(object): - size = 10 ** 5 - types = ('int', 'float', 'object', 'decimal') - - param_names = ['type'] - params = [types] - - def setup(self, type_name): - gen = common.BuiltinsGenerator() - if type_name == 'int': - lst = gen.generate_int_list(self.size) - elif type_name == 'float': - lst = gen.generate_float_list(self.size, use_nan=True) - elif type_name == 'object': - lst = gen.generate_object_list(self.size) - elif type_name == 'decimal': - lst = gen.generate_decimal_list(self.size) - else: - assert 0 - self.lst = lst - - def time_PandasObjectIsNull(self, *args): - pb.benchmark_PandasObjectIsNull(self.lst) diff --git a/python/pyarrow/benchmark.pxi b/python/pyarrow/benchmark.pxi deleted file mode 100644 index ab251017db78..000000000000 --- a/python/pyarrow/benchmark.pxi +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -def benchmark_PandasObjectIsNull(list obj): - Benchmark_PandasObjectIsNull(obj) diff --git a/python/pyarrow/benchmark.py b/python/pyarrow/benchmark.py deleted file mode 100644 index 25ee1141f08d..000000000000 --- a/python/pyarrow/benchmark.py +++ /dev/null @@ -1,21 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# flake8: noqa - - -from pyarrow.lib import benchmark_PandasObjectIsNull diff --git a/python/pyarrow/includes/libarrow_python.pxd b/python/pyarrow/includes/libarrow_python.pxd index 385a2924d1da..6bafdd87a384 100644 --- a/python/pyarrow/includes/libarrow_python.pxd +++ b/python/pyarrow/includes/libarrow_python.pxd @@ -279,10 +279,6 @@ cdef extern from "arrow/python/extension_type.h" namespace "arrow::py": CStatus UnregisterPyExtensionType(c_string type_name) -cdef extern from "arrow/python/benchmark.h" namespace "arrow::py::benchmark": - void Benchmark_PandasObjectIsNull(object lst) except * - - cdef extern from "arrow/python/gdb.h" namespace "arrow::gdb" nogil: void GdbTestSession "arrow::gdb::TestSession"() diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 32f7bc26b1ff..bd06f31f717e 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -415,7 +415,7 @@ cdef class NativeFile(_Weakrefable): # Allocate empty write space obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) - cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + cdef uint8_t* buf = cp.PyBytes_AsString( obj) with nogil: bytes_read = GetResultValue(handle.get().Read(c_nbytes, buf)) @@ -488,7 +488,7 @@ cdef class NativeFile(_Weakrefable): # Allocate empty write space obj = PyBytes_FromStringAndSizeNative(NULL, c_nbytes) - cdef uint8_t* buf = cp.PyBytes_AS_STRING( obj) + cdef uint8_t* buf = cp.PyBytes_AsString( obj) with nogil: bytes_read = GetResultValue(handle.get(). ReadAt(c_offset, c_nbytes, buf)) @@ -1614,7 +1614,7 @@ cdef class Buffer(_Weakrefable): if buffer.buf == NULL: # ARROW-16048: Ensure we don't export a NULL address. assert buffer.len == 0 - buffer.buf = cp.PyBytes_AS_STRING(b"") + buffer.buf = cp.PyBytes_AsString(b"") buffer.format = 'b' buffer.internal = NULL buffer.itemsize = 1 @@ -2643,7 +2643,7 @@ cdef class Codec(_Weakrefable): if asbytes: pyobj = PyBytes_FromStringAndSizeNative(NULL, max_output_size) - output_buffer = cp.PyBytes_AS_STRING( pyobj) + output_buffer = cp.PyBytes_AsString( pyobj) else: out_buf = allocate_buffer( max_output_size, memory_pool=memory_pool, resizable=True @@ -2705,7 +2705,7 @@ cdef class Codec(_Weakrefable): if asbytes: pybuf = cp.PyBytes_FromStringAndSize(NULL, output_size) - output_buffer = cp.PyBytes_AS_STRING(pybuf) + output_buffer = cp.PyBytes_AsString(pybuf) else: out_buf = allocate_buffer(output_size, memory_pool=memory_pool) output_buffer = out_buf.buffer.get().mutable_data() diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx index d949720ca8e5..df469d89d3a2 100644 --- a/python/pyarrow/lib.pyx +++ b/python/pyarrow/lib.pyx @@ -243,8 +243,5 @@ include "io.pxi" # IPC / Messaging include "ipc.pxi" -# Micro-benchmark routines -include "benchmark.pxi" - # Public API include "public-api.pxi" diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index eb60d9d54f85..61b464c6e6f1 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1027,8 +1027,8 @@ Status ConvertMap(PandasOptions options, const ChunkedArray& data, return CheckPyError(); }, [&list_item](int64_t idx, OwnedRef& key_value, OwnedRef& item_value) { - PyList_SET_ITEM(list_item.obj(), idx, - PyTuple_Pack(2, key_value.obj(), item_value.obj())); + PyList_SetItem(list_item.obj(), idx, + PyTuple_Pack(2, key_value.obj(), item_value.obj())); return CheckPyError(); }, [&list_item] { return list_item.detach(); }, data, py_keys, py_items, item_arrays, diff --git a/python/pyarrow/src/arrow/python/benchmark.cc b/python/pyarrow/src/arrow/python/benchmark.cc deleted file mode 100644 index 6dcc959ed221..000000000000 --- a/python/pyarrow/src/arrow/python/benchmark.cc +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "arrow/python/benchmark.h" -#include "arrow/python/helpers.h" - -namespace arrow { -namespace py { -namespace benchmark { - -void Benchmark_PandasObjectIsNull(PyObject* list) { - if (!PyList_CheckExact(list)) { - PyErr_SetString(PyExc_TypeError, "expected a list"); - return; - } - Py_ssize_t i, n = PyList_GET_SIZE(list); - for (i = 0; i < n; i++) { - internal::PandasObjectIsNull(PyList_GET_ITEM(list, i)); - } -} - -} // namespace benchmark -} // namespace py -} // namespace arrow diff --git a/python/pyarrow/src/arrow/python/benchmark.h b/python/pyarrow/src/arrow/python/benchmark.h deleted file mode 100644 index 8060dd33722a..000000000000 --- a/python/pyarrow/src/arrow/python/benchmark.h +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "arrow/python/platform.h" - -#include "arrow/python/visibility.h" - -namespace arrow { -namespace py { -namespace benchmark { - -// Micro-benchmark routines for use from ASV - -// Run PandasObjectIsNull() once over every object in *list* -ARROW_PYTHON_EXPORT -void Benchmark_PandasObjectIsNull(PyObject* list); - -} // namespace benchmark -} // namespace py -} // namespace arrow diff --git a/python/pyarrow/src/arrow/python/common.cc b/python/pyarrow/src/arrow/python/common.cc index 2009f0925906..ffd844ae659f 100644 --- a/python/pyarrow/src/arrow/python/common.cc +++ b/python/pyarrow/src/arrow/python/common.cc @@ -97,8 +97,8 @@ class PythonErrorDetail : public StatusDetail { return result.ValueOrDie(); } else { // Fallback to just the exception type - const auto ty = reinterpret_cast(exc_type_.obj()); - return std::string("Python exception: ") + ty->tp_name; + return std::string("Python exception: ") + + internal::PyObject_StdStringTypeName(exc_type_.obj()); } } diff --git a/python/pyarrow/src/arrow/python/common.h b/python/pyarrow/src/arrow/python/common.h index a81782330b5e..f4815d72478d 100644 --- a/python/pyarrow/src/arrow/python/common.h +++ b/python/pyarrow/src/arrow/python/common.h @@ -23,9 +23,11 @@ #include #include "arrow/buffer.h" +#include "arrow/python/helpers.h" #include "arrow/python/pyarrow.h" #include "arrow/python/visibility.h" #include "arrow/result.h" +#include "arrow/util/logging.h" #include "arrow/util/macros.h" namespace arrow { @@ -267,7 +269,16 @@ class SmartPtrNoGIL : public SmartPtr { // Only release the GIL if we own an object *and* the Python runtime is // valid *and* the GIL is held. std::optional optional_gil_release() const { - if (this->get() != nullptr && Py_IsInitialized() && PyGILState_Check()) { + // PyGILState_Check() is a full-C-API (Py_LIMITED_API-hidden) function, + // unavailable in the cp311-abi3 build. In the non-freethreading build we + // ship (GIL always enabled) it reduces to "the current thread has a valid + // thread state and holds the GIL", which for any thread executing Python + // code is exactly Py_IsInitialized() (a bare C++ worker thread that never + // entered Python has no thread state, and we never call reset() from one + // while destroying a live PyObject without the GIL). Py_IsInitialized() + // is the limited-API-correct guard and also covers the post-finalization + // case (GH-38626) the original comment targets. + if (this->get() != nullptr && Py_IsInitialized()) { return PyReleaseGIL(); } return {}; @@ -398,23 +409,34 @@ struct PyBytesView { // View the given Python object as binary-like, i.e. bytes Status ParseBinary(PyObject* obj) { if (PyBytes_Check(obj)) { - bytes = PyBytes_AS_STRING(obj); - size = PyBytes_GET_SIZE(obj); + bytes = PyBytes_AsString(obj); + size = PyBytes_Size(obj); + ARROW_DCHECK(!PyErr_Occurred()); is_utf8 = false; } else if (PyByteArray_Check(obj)) { - bytes = PyByteArray_AS_STRING(obj); - size = PyByteArray_GET_SIZE(obj); + bytes = PyByteArray_AsString(obj); + size = PyByteArray_Size(obj); + ARROW_DCHECK(!PyErr_Occurred()); is_utf8 = false; } else if (PyMemoryView_Check(obj)) { - PyObject* ref = PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C'); + // C-contiguous view of the memoryview's data. May be a fresh copy, so + // hold it via `ref` for the lifetime of this view (fixes a use-after-free + // for non-contiguous memoryviews). Uses only the stable C-API. + ref.reset(PyMemoryView_GetContiguous(obj, PyBUF_READ, 'C')); RETURN_IF_PYERROR(); - Py_buffer* buffer = PyMemoryView_GET_BUFFER(ref); - bytes = reinterpret_cast(buffer->buf); - size = buffer->len; + Py_buffer buffer; + if (PyObject_GetBuffer(ref.obj(), &buffer, PyBUF_READ) < 0) { + return Status::Invalid("failed to obtain buffer from memoryview"); + } + // Release the buffer export now; the data stays valid because `ref` + // (the memoryview, which owns the underlying buffer) remains alive. + bytes = reinterpret_cast(buffer.buf); + size = buffer.len; + PyBuffer_Release(&buffer); is_utf8 = false; } else { - return Status::TypeError("Expected bytes, got a '", Py_TYPE(obj)->tp_name, - "' object"); + return Status::TypeError("Expected bytes, got a '", + internal::PyObject_StdStringTypeName(obj), "' object"); } return Status::OK(); } @@ -425,10 +447,12 @@ struct PyBytesView { RETURN_IF_PYERROR(); if (!PyBytes_Check(ref.obj())) { return Status::TypeError("Expected uuid.UUID.bytes to return bytes, got '", - Py_TYPE(ref.obj())->tp_name, "' object"); + internal::PyObject_StdStringTypeName(ref.obj()), + "' object"); } - bytes = PyBytes_AS_STRING(ref.obj()); - size = PyBytes_GET_SIZE(ref.obj()); + bytes = PyBytes_AsString(ref.obj()); + size = PyBytes_Size(ref.obj()); + ARROW_DCHECK(!PyErr_Occurred()); is_utf8 = false; return Status::OK(); } diff --git a/python/pyarrow/src/arrow/python/datetime.cc b/python/pyarrow/src/arrow/python/datetime.cc index 6a835c2d37ce..fcd7cc2b3b3b 100644 --- a/python/pyarrow/src/arrow/python/datetime.cc +++ b/python/pyarrow/src/arrow/python/datetime.cc @@ -58,7 +58,7 @@ constexpr char* NonConst(const char* st) { return const_cast(st); } -static PyTypeObject MonthDayNanoTupleType = {}; +static PyTypeObject* MonthDayNanoTupleType = nullptr; static PyStructSequence_Field MonthDayNanoField[] = { {NonConst("months"), NonConst("The number of months in the interval")}, @@ -67,8 +67,13 @@ static PyStructSequence_Field MonthDayNanoField[] = { {nullptr, nullptr}}; static PyStructSequence_Desc MonthDayNanoTupleDesc = { - NonConst("MonthDayNano"), - NonConst("A calendar interval consisting of months, days and nanoseconds."), + NonConst("pyarrow.lib.MonthDayNano"), + NonConst("A calendar interval consisting of months, days and nanoseconds.\n" + "\n" + "Parameters\n" + "----------\n" + "iterable : sequence\n" + " The months, days and nanoseconds fields of the interval.\n"), MonthDayNanoField, /*n_in_sequence=*/3}; @@ -274,13 +279,14 @@ static inline Status PyDate_convert_int(int64_t val, const DateUnit unit, int64_ } PyObject* NewMonthDayNanoTupleType() { - if (MonthDayNanoTupleType.tp_name == nullptr) { - if (PyStructSequence_InitType2(&MonthDayNanoTupleType, &MonthDayNanoTupleDesc) != 0) { + if (MonthDayNanoTupleType == nullptr) { + MonthDayNanoTupleType = PyStructSequence_NewType(&MonthDayNanoTupleDesc); + if (MonthDayNanoTupleType == nullptr) { Py_FatalError("Could not initialize MonthDayNanoTuple"); } } - Py_INCREF(&MonthDayNanoTupleType); - return (PyObject*)&MonthDayNanoTupleType; + Py_INCREF(MonthDayNanoTupleType); + return (PyObject*)MonthDayNanoTupleType; } Status PyTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** out) { @@ -313,9 +319,12 @@ Status PyDateTime_from_int(int64_t val, const TimeUnit::type unit, PyObject** ou return Status::OK(); } -int64_t PyDate_to_days(PyDateTime_Date* pydate) { - return get_days_from_date(PyDateTime_GET_YEAR(pydate), PyDateTime_GET_MONTH(pydate), - PyDateTime_GET_DAY(pydate)); +int64_t PyDate_to_days(PyObject* pydate) { + // datetime struct-field accessors (PyDateTime_GET_YEAR etc.) are hidden under + // Py_LIMITED_API, so read the calendar fields through the stable attribute API. + return get_days_from_date(PyDatetimeField(pydate, "year"), + PyDatetimeField(pydate, "month"), + PyDatetimeField(pydate, "day")); } Result PyDateTime_utcoffset_s(PyObject* obj) { @@ -324,8 +333,7 @@ Result PyDateTime_utcoffset_s(PyObject* obj) { OwnedRef pyoffset(PyObject_CallMethod(obj, "utcoffset", NULL)); RETURN_IF_PYERROR(); if (pyoffset.obj() != nullptr && pyoffset.obj() != Py_None) { - auto delta = reinterpret_cast(pyoffset.obj()); - return internal::PyDelta_to_s(delta); + return internal::PyDelta_to_s(pyoffset.obj()); } else { return 0; } @@ -341,10 +349,9 @@ Result PyTZInfo_utcoffset_hhmm(PyObject* pytzinfo) { "Object returned by tzinfo.utcoffset(None) is not an instance of " "datetime.timedelta"); } - auto pydelta = reinterpret_cast(pydelta_object.obj()); // retrieve the offset as seconds - auto total_seconds = internal::PyDelta_to_s(pydelta); + auto total_seconds = internal::PyDelta_to_s(pydelta_object.obj()); // determine whether the offset is positive or negative auto sign = (total_seconds < 0) ? "-" : "+"; @@ -596,7 +603,7 @@ Result TzinfoToString(PyObject* tzinfo) { PyObject* MonthDayNanoIntervalToNamedTuple( const MonthDayNanoIntervalType::MonthDayNanos& interval) { - OwnedRef tuple(PyStructSequence_New(&MonthDayNanoTupleType)); + OwnedRef tuple(PyStructSequence_New(MonthDayNanoTupleType)); if (ARROW_PREDICT_FALSE(tuple.obj() == nullptr)) { return nullptr; } diff --git a/python/pyarrow/src/arrow/python/datetime.h b/python/pyarrow/src/arrow/python/datetime.h index 84f46fe2d19c..dd7b2fec5b85 100644 --- a/python/pyarrow/src/arrow/python/datetime.h +++ b/python/pyarrow/src/arrow/python/datetime.h @@ -20,6 +20,7 @@ #include #include +#include "arrow/python/common.h" #include "arrow/python/platform.h" #include "arrow/python/visibility.h" #include "arrow/result.h" @@ -38,7 +39,79 @@ # include "datetime.h" #else # define PyDateTimeAPI ::arrow::py::internal::datetime_api -#endif + +// Under Py_LIMITED_API (the cp311-abi3 build) CPython's is +// entirely absent (it lives behind #ifndef Py_LIMITED_API), so neither the +// PyDateTime_CAPI struct nor the PyDate_Check / PyDate_FromDate macros are +// provided. The datetime C-API is still exposed at runtime through the +// "datetime.datetime_CAPI" capsule (which InitDatetime() imports and stores in +// datetime_api); the struct layout is a documented, stable part of that +// capsule contract. Define the struct and the check/constructor macros we use +// so the same code compiles in the limited-API build. The layout must stay in +// sync with CPython's Modules/_datetimemodule.c datetime_capsule. +# ifdef Py_LIMITED_API +# ifndef DATETIME_H +# define DATETIME_H +# endif +# ifndef PyDateTime_CAPSULE_NAME +# define PyDateTime_CAPSULE_NAME "datetime.datetime_CAPI" +# endif +typedef struct { + /* type objects */ + PyTypeObject* DateType; + PyTypeObject* DateTimeType; + PyTypeObject* TimeType; + PyTypeObject* DeltaType; + PyTypeObject* TZInfoType; + + /* singletons */ + PyObject* TimeZone_UTC; + + /* constructors */ + PyObject* (*Date_FromDate)(int, int, int, PyTypeObject*); + PyObject* (*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, + PyObject*, PyTypeObject*); + PyObject* (*Time_FromTime)(int, int, int, int, PyObject*, PyTypeObject*); + PyObject* (*Delta_FromDelta)(int, int, int, int, PyTypeObject*); + PyObject* (*TimeZone_FromTimeZone)(PyObject* offset, PyObject* name); + + /* constructors for the DB API */ + PyObject* (*DateTime_FromTimestamp)(PyObject*, PyObject*, PyObject*); + PyObject* (*Date_FromTimestamp)(PyObject*, PyObject*); + + /* PEP 495 constructors */ + PyObject* (*DateTime_FromDateAndTimeAndFold)(int, int, int, int, int, int, + int, PyObject*, int, + PyTypeObject*); + PyObject* (*Time_FromTimeAndFold)(int, int, int, int, PyObject*, int, + PyTypeObject*); +} PyDateTime_CAPI; + +# define PyDate_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DateType) +# define PyDate_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DateType) +# define PyDateTime_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DateTimeType) +# define PyDateTime_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DateTimeType) +# define PyTime_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->TimeType) +# define PyTime_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->TimeType) +# define PyDelta_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->DeltaType) +# define PyDelta_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->DeltaType) +# define PyTZInfo_Check(op) PyObject_TypeCheck(op, PyDateTimeAPI->TZInfoType) +# define PyTZInfo_CheckExact(op) Py_IS_TYPE(op, PyDateTimeAPI->TZInfoType) +# define PyDateTime_TimeZone_UTC PyDateTimeAPI->TimeZone_UTC +# define PyDate_FromDate(year, month, day) \ + PyDateTimeAPI->Date_FromDate(year, month, day, PyDateTimeAPI->DateType) +# define PyDateTime_FromDateAndTime(year, month, day, hour, min, sec, usec) \ + PyDateTimeAPI->DateTime_FromDateAndTime(year, month, day, hour, min, sec, \ + usec, Py_None, \ + PyDateTimeAPI->DateTimeType) +# define PyTime_FromTime(hour, minute, second, usecond) \ + PyDateTimeAPI->Time_FromTime(hour, minute, second, usecond, Py_None, \ + PyDateTimeAPI->TimeType) +# define PyDelta_FromDSU(days, seconds, useconds) \ + PyDateTimeAPI->Delta_FromDelta(days, seconds, useconds, 1, \ + PyDateTimeAPI->DeltaType) +# endif // Py_LIMITED_API +#endif // !PYPY_VERSION namespace arrow { using internal::AddWithOverflow; @@ -57,12 +130,31 @@ void InitDatetime(); ARROW_PYTHON_EXPORT PyObject* NewMonthDayNanoTupleType(); +// Reads an integer field ("year", "hour", "days", ...) off a Python +// date/time/datetime/timedelta object through the stable C-API. The fast +// datetime struct-field accessors (PyDateTime_TIME_GET_HOUR etc.) are hidden +// under Py_LIMITED_API (abi3), so attribute access is the portable path. +// Callers pass a type-checked object, so the field is guaranteed present and +// integral; a failure is a genuine invariant violation. +ARROW_PYTHON_EXPORT +inline int64_t PyDatetimeField(PyObject* obj, const char* name) { + OwnedRef field(PyObject_GetAttrString(obj, name)); + if (ARROW_PREDICT_FALSE(field.obj() == nullptr)) { + Py_FatalError("arrow: failed to read datetime field"); + } + long long v = PyLong_AsLongLong(field.obj()); + if (ARROW_PREDICT_FALSE(v == -1 && PyErr_Occurred())) { + Py_FatalError("arrow: datetime field is not an integer"); + } + return v; +} + ARROW_PYTHON_EXPORT inline int64_t PyTime_to_us(PyObject* pytime) { - return (PyDateTime_TIME_GET_HOUR(pytime) * 3600000000LL + - PyDateTime_TIME_GET_MINUTE(pytime) * 60000000LL + - PyDateTime_TIME_GET_SECOND(pytime) * 1000000LL + - PyDateTime_TIME_GET_MICROSECOND(pytime)); + return (PyDatetimeField(pytime, "hour") * 3600000000LL + + PyDatetimeField(pytime, "minute") * 60000000LL + + PyDatetimeField(pytime, "second") * 1000000LL + + PyDatetimeField(pytime, "microsecond")); } ARROW_PYTHON_EXPORT @@ -89,45 +181,42 @@ using TimePoint = std::chrono::time_point; ARROW_PYTHON_EXPORT -int64_t PyDate_to_days(PyDateTime_Date* pydate); +int64_t PyDate_to_days(PyObject* pydate); ARROW_PYTHON_EXPORT -inline int64_t PyDate_to_s(PyDateTime_Date* pydate) { - return PyDate_to_days(pydate) * 86400LL; -} +inline int64_t PyDate_to_s(PyObject* pydate) { return PyDate_to_days(pydate) * 86400LL; } ARROW_PYTHON_EXPORT -inline int64_t PyDate_to_ms(PyDateTime_Date* pydate) { +inline int64_t PyDate_to_ms(PyObject* pydate) { return PyDate_to_days(pydate) * 86400000LL; } ARROW_PYTHON_EXPORT -inline int64_t PyDateTime_to_s(PyDateTime_DateTime* pydatetime) { - return (PyDate_to_s(reinterpret_cast(pydatetime)) + - PyDateTime_DATE_GET_HOUR(pydatetime) * 3600LL + - PyDateTime_DATE_GET_MINUTE(pydatetime) * 60LL + - PyDateTime_DATE_GET_SECOND(pydatetime)); +inline int64_t PyDateTime_to_s(PyObject* pydatetime) { + return (PyDate_to_s(pydatetime) + PyDatetimeField(pydatetime, "hour") * 3600LL + + PyDatetimeField(pydatetime, "minute") * 60LL + + PyDatetimeField(pydatetime, "second")); } ARROW_PYTHON_EXPORT -inline int64_t PyDateTime_to_ms(PyDateTime_DateTime* pydatetime) { +inline int64_t PyDateTime_to_ms(PyObject* pydatetime) { return (PyDateTime_to_s(pydatetime) * 1000LL + - PyDateTime_DATE_GET_MICROSECOND(pydatetime) / 1000); + PyDatetimeField(pydatetime, "microsecond") / 1000); } ARROW_PYTHON_EXPORT -inline int64_t PyDateTime_to_us(PyDateTime_DateTime* pydatetime) { +inline int64_t PyDateTime_to_us(PyObject* pydatetime) { return (PyDateTime_to_s(pydatetime) * 1000000LL + - PyDateTime_DATE_GET_MICROSECOND(pydatetime)); + PyDatetimeField(pydatetime, "microsecond")); } ARROW_PYTHON_EXPORT -inline int64_t PyDateTime_to_ns(PyDateTime_DateTime* pydatetime) { +inline int64_t PyDateTime_to_ns(PyObject* pydatetime) { return PyDateTime_to_us(pydatetime) * 1000LL; } ARROW_PYTHON_EXPORT -inline TimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime) { +inline TimePoint PyDateTime_to_TimePoint(PyObject* pydatetime) { return TimePoint(TimePoint::duration(PyDateTime_to_ns(pydatetime))); } @@ -145,31 +234,31 @@ inline TimePoint TimePoint_from_ns(int64_t val) { } ARROW_PYTHON_EXPORT -inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) { - return (PyDateTime_DELTA_GET_DAYS(pytimedelta) * 86400LL + - PyDateTime_DELTA_GET_SECONDS(pytimedelta)); +inline int64_t PyDelta_to_s(PyObject* pytimedelta) { + return (PyDatetimeField(pytimedelta, "days") * 86400LL + + PyDatetimeField(pytimedelta, "seconds")); } ARROW_PYTHON_EXPORT -inline int64_t PyDelta_to_ms(PyDateTime_Delta* pytimedelta) { +inline int64_t PyDelta_to_ms(PyObject* pytimedelta) { return (PyDelta_to_s(pytimedelta) * 1000LL + - PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000); + PyDatetimeField(pytimedelta, "microseconds") / 1000); } ARROW_PYTHON_EXPORT -inline Result PyDelta_to_us(PyDateTime_Delta* pytimedelta) { +inline Result PyDelta_to_us(PyObject* pytimedelta) { int64_t result = PyDelta_to_s(pytimedelta); if (MultiplyWithOverflow(result, 1000000LL, &result)) { return Status::Invalid("Timedelta too large to fit in 64-bit integer"); } - if (AddWithOverflow(result, PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta), &result)) { + if (AddWithOverflow(result, PyDatetimeField(pytimedelta, "microseconds"), &result)) { return Status::Invalid("Timedelta too large to fit in 64-bit integer"); } return result; } ARROW_PYTHON_EXPORT -inline Result PyDelta_to_ns(PyDateTime_Delta* pytimedelta) { +inline Result PyDelta_to_ns(PyObject* pytimedelta) { ARROW_ASSIGN_OR_RAISE(int64_t result, PyDelta_to_us(pytimedelta)); if (MultiplyWithOverflow(result, 1000LL, &result)) { return Status::Invalid("Timedelta too large to fit in 64-bit integer"); diff --git a/python/pyarrow/src/arrow/python/decimal.cc b/python/pyarrow/src/arrow/python/decimal.cc index 1ba8d32192af..298820cfe2de 100644 --- a/python/pyarrow/src/arrow/python/decimal.cc +++ b/python/pyarrow/src/arrow/python/decimal.cc @@ -159,7 +159,7 @@ Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, return InternalDecimalFromPythonDecimal(obj, arrow_type, out); } else { return Status::TypeError("int or Decimal object expected, got ", - Py_TYPE(obj)->tp_name); + internal::PyObject_StdStringTypeName(obj)); } } diff --git a/python/pyarrow/src/arrow/python/extension_type.cc b/python/pyarrow/src/arrow/python/extension_type.cc index 8c3f3f1d8d0f..659744cb5de5 100644 --- a/python/pyarrow/src/arrow/python/extension_type.cc +++ b/python/pyarrow/src/arrow/python/extension_type.cc @@ -78,8 +78,8 @@ std::string PyExtensionType::ToString(bool show_metadata) const { std::stringstream ss; OwnedRef instance(GetInstance()); - ss << "extension<" << this->extension_name() << "<" << Py_TYPE(instance.obj())->tp_name - << ">>"; + ss << "extension<" << this->extension_name() << "<" + << internal::PyObject_StdStringTypeName(instance.obj()) << ">>"; return ss.str(); } @@ -164,20 +164,18 @@ PyObject* PyExtensionType::GetInstance() const { PyErr_SetString(PyExc_TypeError, "Not an instance"); return nullptr; } - ARROW_DCHECK(PyWeakref_CheckRef(type_instance_.obj())); - PyObject* inst = NULL; - int result = PyWeakref_GetRef(type_instance_.obj(), &inst); - if (result == 1) { + // PyWeakref_GetRef is a full-C-API (3.12+) function, unavailable in the + // cp311-abi3 build. PyWeakref_GetObject is the stable-API equivalent: it + // returns the referent as a new strong reference, or Py_None if dead (and + // raises for a non-weakref, so no separate type check is needed). + PyObject* inst = PyWeakref_GetObject(type_instance_.obj()); + if (inst != Py_None) { // Alive: inst is a new strong reference return inst; - } else if (result == 0) { - // Weakref is dead, must reconstruct from serialized form - // XXX cache again? - return DeserializeExtInstance(type_class_.obj(), storage_type_, serialized_); - } else { - // -1 = exception - return nullptr; } + // Weakref is dead, must reconstruct from serialized form + // XXX cache again? + return DeserializeExtInstance(type_class_.obj(), storage_type_, serialized_); } Status PyExtensionType::SetInstance(PyObject* inst) const { diff --git a/python/pyarrow/src/arrow/python/helpers.cc b/python/pyarrow/src/arrow/python/helpers.cc index 3b895001f928..c54f1fbb112b 100644 --- a/python/pyarrow/src/arrow/python/helpers.cc +++ b/python/pyarrow/src/arrow/python/helpers.cc @@ -86,11 +86,25 @@ Result PyFloat_AsHalf(PyObject* obj) { arrow::util::Float16 half_val = arrow::util::Float16::FromDouble(PyFloat_AsDouble(obj)); return half_val.bits(); - } else if (has_numpy() && PyArray_IsScalar(obj, Half)) { - return PyArrayScalar_VAL(obj, Half); + } else if (has_numpy()) { + // The numpy C-API scalar accessors (PyArray_IsScalar / PyArrayScalar_VAL, + // and the `Half` scalar type) are full-C-API and hidden under + // Py_LIMITED_API. np.float16 supports the Python float protocol, so route + // through __float__ -- stable-API and version-agnostic. + OwnedRef pyfloat(PyObject_CallMethod(obj, "__float__", NULL)); + RETURN_IF_PYERROR(); + if (pyfloat.obj() == nullptr) { + return Status::TypeError( + "conversion to float16 expects a `float` or `np.float16` object, got ", + internal::PyObject_StdStringTypeName(obj)); + } + arrow::util::Float16 half_val = + arrow::util::Float16::FromDouble(PyFloat_AsDouble(pyfloat.obj())); + return half_val.bits(); } else { return Status::TypeError("conversion to float16 expects a `float` or ", - "`np.float16` object, got ", Py_TYPE(obj)->tp_name); + "`np.float16` object, got ", + internal::PyObject_StdStringTypeName(obj)); } } @@ -98,7 +112,18 @@ namespace internal { std::string PyBytes_AsStdString(PyObject* obj) { ARROW_DCHECK(PyBytes_Check(obj)); - return std::string(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); + char* buffer = nullptr; + Py_ssize_t length = 0; + PyBytes_AsStringAndSize(obj, &buffer, &length); + return std::string(buffer, length); +} + +std::string_view PyBytes_AsStdStringView(PyObject* obj) { + ARROW_DCHECK(PyBytes_Check(obj)); + char* buffer = nullptr; + Py_ssize_t length = 0; + PyBytes_AsStringAndSize(obj, &buffer, &length); + return std::string_view(buffer, length); } Status PyUnicode_AsStdString(PyObject* obj, std::string* out) { @@ -122,12 +147,26 @@ std::string PyObject_StdStringRepr(PyObject* obj) { if (!bytes_ref) { PyErr_Clear(); std::stringstream ss; - ss << "tp_name << "' repr() failed>"; + ss << ""; return ss.str(); } return PyBytes_AsStdString(bytes_ref.obj()); } +std::string PyObject_StdStringTypeName(PyObject* obj) { + OwnedRef name_ref(PyType_GetName(Py_TYPE(obj))); + if (!name_ref) { + PyErr_Clear(); + return "?"; + } + std::string result; + if (!PyUnicode_AsStdString(name_ref.obj(), &result).ok()) { + PyErr_Clear(); + return "?"; + } + return result; +} + Status PyObject_StdStringStr(PyObject* obj, std::string* out) { OwnedRef string_ref(PyObject_Str(obj)); RETURN_IF_PYERROR(); @@ -177,9 +216,10 @@ Result PyObjectToPyInt(PyObject* obj) { return std::move(ref); } PyErr_Clear(); - const auto nb = Py_TYPE(obj)->tp_as_number; - if (nb && nb->nb_int) { - ref.reset(nb->nb_int(obj)); + const auto nb_int = + reinterpret_cast(PyType_GetSlot(Py_TYPE(obj), Py_nb_int)); + if (nb_int) { + ref.reset(nb_int(obj)); if (!ref) { RETURN_IF_PYERROR(); } @@ -287,7 +327,7 @@ inline bool MayHaveNaN(PyObject* obj) { Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_BYTES_SUBCLASS | Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS; - return !PyType_HasFeature(Py_TYPE(obj), non_nan_tpflags); + return (PyType_GetFlags(Py_TYPE(obj)) & non_nan_tpflags) == 0; } bool PyFloat_IsNaN(PyObject* obj) { @@ -373,8 +413,12 @@ Result UuidFromBytes(std::string_view bytes, PyObject* kwargs) { if (PyDict_SetItemString(kwargs, "bytes", py_bytes.obj()) < 0) { RETURN_IF_PYERROR(); } - PyObject* empty_args = Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE); - PyObject* result = PyObject_Call(uuid_class, empty_args, kwargs); + // Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_TUPLE) is a full-C-API (3.12+) + // function, unavailable in the cp311-abi3 build. A fresh empty tuple is + // equivalent here and owned by this statement. + OwnedRef empty_args(PyTuple_New(0)); + RETURN_IF_PYERROR(); + PyObject* result = PyObject_Call(uuid_class, empty_args.obj(), kwargs); RETURN_IF_PYERROR(); return result; } @@ -456,13 +500,13 @@ PyObject* BorrowPandasDataOffsetType() { return pandas_DateOffset; } Status InvalidValue(PyObject* obj, const std::string& why) { auto obj_as_str = PyObject_StdStringRepr(obj); return Status::Invalid("Could not convert ", std::move(obj_as_str), " with type ", - Py_TYPE(obj)->tp_name, ": ", why); + PyObject_StdStringTypeName(obj), ": ", why); } Status InvalidType(PyObject* obj, const std::string& why) { auto obj_as_str = PyObject_StdStringRepr(obj); return Status::TypeError("Could not convert ", std::move(obj_as_str), " with type ", - Py_TYPE(obj)->tp_name, ": ", why); + PyObject_StdStringTypeName(obj), ": ", why); } Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) { diff --git a/python/pyarrow/src/arrow/python/helpers.h b/python/pyarrow/src/arrow/python/helpers.h index e2d41d7ff6a6..daad7ff0a48a 100644 --- a/python/pyarrow/src/arrow/python/helpers.h +++ b/python/pyarrow/src/arrow/python/helpers.h @@ -120,6 +120,11 @@ Status PyUnicode_AsStdString(PyObject* obj, std::string* out); ARROW_PYTHON_EXPORT std::string PyBytes_AsStdString(PyObject* obj); +// \brief Return a std::string_view over a Python bytes object's buffer +// \note The view is only valid while a reference to \c obj is held +ARROW_PYTHON_EXPORT +std::string_view PyBytes_AsStdStringView(PyObject* obj); + // \brief Call str() on the given object and return the result as a std::string ARROW_PYTHON_EXPORT Status PyObject_StdStringStr(PyObject* obj, std::string* out); @@ -128,6 +133,10 @@ Status PyObject_StdStringStr(PyObject* obj, std::string* out); ARROW_PYTHON_EXPORT std::string PyObject_StdStringRepr(PyObject* obj); +// \brief Return the type name of the given object as a std::string +ARROW_PYTHON_EXPORT +std::string PyObject_StdStringTypeName(PyObject* obj); + // \brief Cast the given size to int32_t, with error checking inline Status CastSize(Py_ssize_t size, int32_t* out, const char* error_msg = "Maximum size exceeded (2GB)") { diff --git a/python/pyarrow/src/arrow/python/inference.cc b/python/pyarrow/src/arrow/python/inference.cc index 291cc4214957..427535fc2218 100644 --- a/python/pyarrow/src/arrow/python/inference.cc +++ b/python/pyarrow/src/arrow/python/inference.cc @@ -708,7 +708,7 @@ class TypeInferrer { key = internal::PyBytes_AsStdString(key_obj); } else { return Status::TypeError("Expected dict key of type str or bytes, got '", - Py_TYPE(key_obj)->tp_name, "'"); + internal::PyObject_StdStringTypeName(key_obj), "'"); } // Get or create visitor for this key TypeInferrer* visitor; diff --git a/python/pyarrow/src/arrow/python/io.cc b/python/pyarrow/src/arrow/python/io.cc index 7a4d18ab9fa2..a4690e1f2f44 100644 --- a/python/pyarrow/src/arrow/python/io.cc +++ b/python/pyarrow/src/arrow/python/io.cc @@ -233,7 +233,8 @@ Result PyReadableFile::Read(int64_t nbytes, void* out) { return Status::TypeError( "Python file read() should have returned a bytes object or an object " "supporting the buffer protocol, got '", - Py_TYPE(bytes_obj)->tp_name, "' (did you open the file in binary mode?)"); + internal::PyObject_StdStringTypeName(bytes_obj), + "' (did you open the file in binary mode?)"); } }); } diff --git a/python/pyarrow/src/arrow/python/iterators.h b/python/pyarrow/src/arrow/python/iterators.h index dd467f6ac407..94838ccbcab1 100644 --- a/python/pyarrow/src/arrow/python/iterators.h +++ b/python/pyarrow/src/arrow/python/iterators.h @@ -72,18 +72,20 @@ inline Status VisitSequenceGeneric(PyObject* obj, int64_t offset, VisitorFunc&& #else if (PyList_Check(obj) || PyTuple_Check(obj)) { #endif - // Use fast item access - const Py_ssize_t size = PySequence_Fast_GET_SIZE(obj); + // Fast item access via the limited-API PySequence_Fast + PyList_GetItem + const OwnedRef seq_ref(PySequence_Fast(obj, "index sequence")); + RETURN_IF_PYERROR(); + const Py_ssize_t size = PyList_Size(seq_ref.obj()); for (Py_ssize_t i = offset; keep_going && i < size; ++i) { - PyObject* value = PySequence_Fast_GET_ITEM(obj, i); - RETURN_NOT_OK(func(value, static_cast(i), &keep_going)); + const OwnedRef item_ref(PyList_GetItem(seq_ref.obj(), i)); + RETURN_NOT_OK(func(item_ref.obj(), static_cast(i), &keep_going)); } } else { // Regular sequence: avoid making a potentially large copy const Py_ssize_t size = PySequence_Size(obj); RETURN_IF_PYERROR(); for (Py_ssize_t i = offset; keep_going && i < size; ++i) { - OwnedRef value_ref(PySequence_ITEM(obj, i)); + OwnedRef value_ref(PySequence_GetItem(obj, i)); RETURN_IF_PYERROR(); RETURN_NOT_OK(func(value_ref.obj(), static_cast(i), &keep_going)); } @@ -157,7 +159,8 @@ inline Status VisitSequenceMasked(PyObject* obj, PyObject* mo, int64_t offset, return VisitSequenceGeneric( obj, offset, [&func, &mo](PyObject* value, int64_t i, bool* keep_going) { - OwnedRef value_ref(PySequence_ITEM(mo, i)); + OwnedRef value_ref(PySequence_GetItem(mo, i)); + RETURN_IF_PYERROR(); if (!PyBool_Check(value_ref.obj())) return Status::TypeError("Mask must be a sequence of booleans"); return func(value, value_ref.obj() == Py_True, keep_going); diff --git a/python/pyarrow/src/arrow/python/numpy_convert.cc b/python/pyarrow/src/arrow/python/numpy_convert.cc index 6e59835286d9..fc7cd1873d28 100644 --- a/python/pyarrow/src/arrow/python/numpy_convert.cc +++ b/python/pyarrow/src/arrow/python/numpy_convert.cc @@ -488,14 +488,11 @@ Status NdarraysToSparseCSFTensor(MemoryPool* pool, PyObject* data_ao, PyObject* std::vector> indptr(ndim - 1); std::vector> indices(ndim); + const OwnedRef indptr_seq_ref(PySequence_Fast(indptr_ao, "indptr")); + RETURN_IF_PYERROR(); for (int i = 0; i < ndim - 1; ++i) { -#ifdef Py_GIL_DISABLED - PyObject* item = PySequence_ITEM(indptr_ao, i); - RETURN_IF_PYERROR(); - OwnedRef item_ref(item); -#else - PyObject* item = PySequence_Fast_GET_ITEM(indptr_ao, i); -#endif + const OwnedRef item_ref(PyList_GetItem(indptr_seq_ref.obj(), i)); + PyObject* item = item_ref.obj(); if (!PyArray_Check(item)) { return Status::TypeError("Did not pass ndarray object for indptr"); } @@ -503,14 +500,11 @@ Status NdarraysToSparseCSFTensor(MemoryPool* pool, PyObject* data_ao, PyObject* ARROW_CHECK_EQ(indptr[i]->type_id(), Type::INT64); // Should be ensured by caller } + const OwnedRef indices_seq_ref(PySequence_Fast(indices_ao, "indices")); + RETURN_IF_PYERROR(); for (int i = 0; i < ndim; ++i) { -#ifdef Py_GIL_DISABLED - PyObject* item = PySequence_ITEM(indices_ao, i); - RETURN_IF_PYERROR(); - OwnedRef item_ref(item); -#else - PyObject* item = PySequence_Fast_GET_ITEM(indices_ao, i); -#endif + const OwnedRef item_ref(PyList_GetItem(indices_seq_ref.obj(), i)); + PyObject* item = item_ref.obj(); if (!PyArray_Check(item)) { return Status::TypeError("Did not pass ndarray object for indices"); } diff --git a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc index 5647e895d0f7..83f336f16ef1 100644 --- a/python/pyarrow/src/arrow/python/numpy_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/numpy_to_arrow.cc @@ -158,7 +158,8 @@ class NumPyNullsConverter { int64_t MaskToBitmap(PyArrayObject* mask, int64_t length, uint8_t* bitmap) { int64_t null_count = 0; - if (!PyArray_Check(mask)) return -1; + // numpy types are opaque under Py_LIMITED_API; reinterpret for the Python C-API. + if (!PyArray_Check(reinterpret_cast(mask))) return -1; Ndarray1DIndexer mask_values(mask); for (int i = 0; i < length; ++i) { @@ -690,9 +691,9 @@ Status AppendUTF32(const char* data, int64_t itemsize, int byteorder, T* builder return Status::Invalid("failed converting UTF32 to UTF8"); } - const int32_t length = static_cast(PyBytes_GET_SIZE(utf8_obj.obj())); - return builder->Append( - reinterpret_cast(PyBytes_AS_STRING(utf8_obj.obj())), length); + const std::string_view utf8_view = internal::PyBytes_AsStdStringView(utf8_obj.obj()); + return builder->Append(reinterpret_cast(utf8_view.data()), + static_cast(utf8_view.size())); } } // namespace @@ -828,19 +829,25 @@ Status NumPyConverter::Visit(const StructType& type) { } for (auto field : type.fields()) { - PyObject* tup; - PyDict_GetItemStringRef(PyDataType_FIELDS(dtype_), field->name().c_str(), &tup); - RETURN_IF_PYERROR(); + // PyDict_GetItemStringRef is full-C-API (3.13+); PyDict_GetItemString is the + // stable-API equivalent (returns NULL for a missing key, no exception). + PyObject* borrowed = + PyDict_GetItemString(PyDataType_FIELDS(dtype_), field->name().c_str()); + PyObject* tup = borrowed ? (Py_INCREF(borrowed), borrowed) : nullptr; OwnedRef tupref(tup); if (tup == NULL) { return Status::Invalid("Missing field '", field->name(), "' in struct array"); } PyArray_Descr* sub_dtype = - reinterpret_cast(PyTuple_GET_ITEM(tup, 0)); - ARROW_DCHECK(PyObject_TypeCheck(sub_dtype, &PyArrayDescr_Type)); - int offset = static_cast(PyLong_AsLong(PyTuple_GET_ITEM(tup, 1))); + reinterpret_cast(PyTuple_GetItem(tup, 0)); + RETURN_IF_PYERROR(); + ARROW_DCHECK(PyObject_TypeCheck(reinterpret_cast(sub_dtype), + &PyArrayDescr_Type)); + PyObject* offset_obj = PyTuple_GetItem(tup, 1); + RETURN_IF_PYERROR(); + int offset = static_cast(PyLong_AsLong(offset_obj)); RETURN_IF_PYERROR(); - Py_INCREF(sub_dtype); /* PyArray_GetField() steals ref */ + Py_INCREF(reinterpret_cast(sub_dtype)); /* steals ref */ PyObject* sub_array = PyArray_GetField(arr_, sub_dtype, offset); RETURN_IF_PYERROR(); sub_arrays.emplace_back(sub_array); diff --git a/python/pyarrow/src/arrow/python/pyarrow.cc b/python/pyarrow/src/arrow/python/pyarrow.cc index 4f675921708e..c5b1c929ffb7 100644 --- a/python/pyarrow/src/arrow/python/pyarrow.cc +++ b/python/pyarrow/src/arrow/python/pyarrow.cc @@ -37,7 +37,8 @@ namespace py { static Status UnwrapError(PyObject* obj, const char* expected_type) { return Status::TypeError("Could not unwrap ", expected_type, - " from Python object of type '", Py_TYPE(obj)->tp_name, "'"); + " from Python object of type '", + internal::PyObject_StdStringTypeName(obj), "'"); } int import_pyarrow() { diff --git a/python/pyarrow/src/arrow/python/python_to_arrow.cc b/python/pyarrow/src/arrow/python/python_to_arrow.cc index 5b83875813a1..8025880e1aef 100644 --- a/python/pyarrow/src/arrow/python/python_to_arrow.cc +++ b/python/pyarrow/src/arrow/python/python_to_arrow.cc @@ -255,7 +255,7 @@ class PyValue { static Result Convert(const DoubleType*, const O&, I obj) { double value; if (PyFloat_Check(obj)) { - value = PyFloat_AS_DOUBLE(obj); + value = PyFloat_AsDouble(obj); } else if (internal::PyFloatScalar_Check(obj)) { // Other kinds of float-y things value = PyFloat_AsDouble(obj); @@ -295,8 +295,7 @@ class PyValue { static Result Convert(const Date32Type*, const O&, I obj) { int32_t value; if (PyDate_Check(obj)) { - auto pydate = reinterpret_cast(obj); - value = static_cast(internal::PyDate_to_days(pydate)); + value = static_cast(internal::PyDate_to_days(obj)); } else { RETURN_NOT_OK( internal::CIntFromPython(obj, &value, "Integer too large for date32")); @@ -307,14 +306,12 @@ class PyValue { static Result Convert(const Date64Type*, const O&, I obj) { int64_t value; if (PyDateTime_Check(obj)) { - auto pydate = reinterpret_cast(obj); - value = internal::PyDateTime_to_ms(pydate); + value = internal::PyDateTime_to_ms(obj); // Truncate any intraday milliseconds // TODO: introduce an option for this value -= value % 86400000LL; } else if (PyDate_Check(obj)) { - auto pydate = reinterpret_cast(obj); - value = internal::PyDate_to_ms(pydate); + value = internal::PyDate_to_ms(obj); } else { RETURN_NOT_OK( internal::CIntFromPython(obj, &value, "Integer too large for date64")); @@ -368,7 +365,7 @@ class PyValue { } else { ARROW_ASSIGN_OR_RAISE(offset, internal::PyDateTime_utcoffset_s(obj)); } - auto dt = reinterpret_cast(obj); + PyObject* dt = obj; switch (type->unit()) { case TimeUnit::SECOND: value = internal::PyDateTime_to_s(dt) - offset; @@ -445,12 +442,11 @@ class PyValue { return output; } if (PyTuple_Check(obj) && PyTuple_Size(obj) == 3) { - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 0), &output.months, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 0), &output.months, "Months (tuple item #0) too large")); - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 1), &output.days, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 1), &output.days, "Days (tuple item #1) too large")); - RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GET_ITEM(obj, 2), - &output.nanoseconds, + RETURN_NOT_OK(internal::CIntFromPython(PyTuple_GetItem(obj, 2), &output.nanoseconds, "Nanoseconds (tuple item #2) too large")); return output; } @@ -460,7 +456,7 @@ class PyValue { static Result Convert(const DurationType* type, const O&, I obj) { int64_t value; if (PyDelta_Check(obj)) { - auto dt = reinterpret_cast(obj); + PyObject* dt = obj; switch (type->unit()) { case TimeUnit::SECOND: value = internal::PyDelta_to_s(dt); @@ -1050,8 +1046,8 @@ class PyStructConverter : public StructConverter PyObject* unicode = PyUnicode_FromStringAndSize(field_name.c_str(), field_name.size()); RETURN_IF_PYERROR(); - PyList_SET_ITEM(bytes_field_names_.obj(), i, bytes); - PyList_SET_ITEM(unicode_field_names_.obj(), i, unicode); + PyList_SetItem(bytes_field_names_.obj(), i, bytes); + PyList_SetItem(unicode_field_names_.obj(), i, unicode); } return Status::OK(); } @@ -1107,11 +1103,13 @@ class PyStructConverter : public StructConverter if (!PyTuple_Check(tuple)) { return internal::InvalidType(tuple, "was expecting a tuple"); } - if (PyTuple_GET_SIZE(tuple) != num_fields_) { + const Py_ssize_t tuple_size = PyTuple_Size(tuple); + RETURN_IF_PYERROR(); + if (tuple_size != num_fields_) { return Status::Invalid("Tuple size must be equal to number of struct fields"); } for (int i = 0; i < num_fields_; i++) { - PyObject* value = PyTuple_GET_ITEM(tuple, i); + PyObject* value = PyTuple_GetItem(tuple, i); RETURN_NOT_OK(this->children_[i]->Append(value)); } return Status::OK(); @@ -1161,12 +1159,14 @@ class PyStructConverter : public StructConverter Status AppendDict(PyObject* dict, PyObject* field_names) { // NOTE we're ignoring any extraneous dict items for (int i = 0; i < num_fields_; i++) { - PyObject* name = PyList_GetItemRef(field_names, i); + // PyList_GetItemRef is a full-C-API (3.13+) function; PyList_GetItem is + // the stable-API equivalent, but returns a borrowed reference. + PyObject* name = PyList_GetItem(field_names, i); RETURN_IF_PYERROR(); OwnedRef nameref(name); - PyObject* value; - PyDict_GetItemRef(dict, name, &value); - RETURN_IF_PYERROR(); + // PyDict_GetItemRef is full-C-API (3.13+); PyDict_GetItemWithError is the + // stable-API equivalent (returns NULL for a missing key without raising). + PyObject* value = PyDict_GetItemWithError(dict, name); OwnedRef valueref(value); RETURN_NOT_OK(this->children_[i]->Append(value ? value : Py_None)); } @@ -1197,7 +1197,9 @@ class PyStructConverter : public StructConverter ARROW_ASSIGN_OR_RAISE(auto pair, GetKeyValuePair(items, i)); // validate that the key and the field name are equal - PyObject* name = PyList_GetItemRef(field_names, i); + // PyList_GetItemRef is full-C-API (3.13+); PyList_GetItem is stable and + // returns a borrowed reference. + PyObject* name = PyList_GetItem(field_names, i); RETURN_IF_PYERROR(); OwnedRef nameref(name); bool are_equal = PyObject_RichCompareBool(pair.first, name, Py_EQ); @@ -1250,7 +1252,7 @@ Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* siz // unknown size, exhaust iterator *seq = PySequence_List(obj); RETURN_IF_PYERROR(); - *size = static_cast(PyList_GET_SIZE(*seq)); + *size = static_cast(PyList_Size(*seq)); } else { // size is known but iterator could be infinite Py_ssize_t i, n = *size; @@ -1267,7 +1269,7 @@ Status ConvertToSequenceAndInferSize(PyObject* obj, PyObject** seq, int64_t* siz RETURN_IF_PYERROR(); break; } - PyList_SET_ITEM(lst, i, item); + PyList_SetItem(lst, i, item); } // Shrink list if len(iterator) < size if (i < n && PyList_SetSlice(lst, i, n, NULL)) { diff --git a/python/pyarrow/src/arrow/python/udf.cc b/python/pyarrow/src/arrow/python/udf.cc index 940e403ef693..13309bb8d5eb 100644 --- a/python/pyarrow/src/arrow/python/udf.cc +++ b/python/pyarrow/src/arrow/python/udf.cc @@ -210,8 +210,9 @@ struct PythonUdfScalarAggregatorImpl : public ScalarUdfAggregator { out->value = std::move(val); return Status::OK(); } - return Status::TypeError("Unexpected output type: ", - Py_TYPE(result->obj())->tp_name, " (expected Scalar)"); + return Status::TypeError( + "Unexpected output type: ", internal::PyObject_StdStringTypeName(result->obj()), + " (expected Scalar)"); })); return Status::OK(); } @@ -352,7 +353,8 @@ struct PythonUdfHashAggregatorImpl : public HashUdfAggregator { ARROW_RETURN_NOT_OK(builder->AppendScalar(std::move(*val))); } else { return Status::TypeError("Unexpected output type: ", - Py_TYPE(result->obj())->tp_name, " (expected Scalar)"); + internal::PyObject_StdStringTypeName(result->obj()), + " (expected Scalar)"); } } ARROW_ASSIGN_OR_RAISE(auto result, builder->Finish()); @@ -435,8 +437,9 @@ struct PythonUdf : public PythonUdfKernelState { out->value = std::move(val->data()); return Status::OK(); } else { - return Status::TypeError("Unexpected output type: ", Py_TYPE(result.obj())->tp_name, - " (expected Array)"); + return Status::TypeError( + "Unexpected output type: ", internal::PyObject_StdStringTypeName(result.obj()), + " (expected Array)"); } return Status::OK(); } diff --git a/python/pyarrow/src/arrow/python/vendored/pythoncapi_compat.h b/python/pyarrow/src/arrow/python/vendored/pythoncapi_compat.h index 4baa7b34a935..964834d06fa8 100644 --- a/python/pyarrow/src/arrow/python/vendored/pythoncapi_compat.h +++ b/python/pyarrow/src/arrow/python/vendored/pythoncapi_compat.h @@ -141,6 +141,11 @@ static inline void _Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) #endif +// The PyFrame* / PyThreadState_GetFrame* shims reference PyFrameObject, which +// is only defined when frameobject.h is included -- not under Py_LIMITED_API. +// Guard the whole frame block so a limited-API (abi3) build compiles. (Local +// arrow patch; the pyarrow sources never call the PyFrame* API.) +#ifndef Py_LIMITED_API // bpo-40421 added PyFrame_GetCode() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 || defined(PYPY_VERSION) static inline PyCodeObject* PyFrame_GetCode(PyFrameObject *frame) @@ -283,6 +288,7 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) return value; } #endif +#endif // !Py_LIMITED_API // bpo-39947 added PyThreadState_GetInterpreter() to Python 3.9.0a5 @@ -296,6 +302,7 @@ PyThreadState_GetInterpreter(PyThreadState *tstate) #endif +#ifndef Py_LIMITED_API // bpo-40429 added PyThreadState_GetFrame() to Python 3.9.0b1 #if PY_VERSION_HEX < 0x030900B1 && !defined(PYPY_VERSION) static inline PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) @@ -314,6 +321,7 @@ _PyThreadState_GetFrameBorrow(PyThreadState *tstate) return frame; } #endif +#endif // !Py_LIMITED_API // bpo-39947 added PyInterpreterState_Get() to Python 3.9.0a5 diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index f9530a34362f..5427f8458d48 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -4403,7 +4403,7 @@ def month_day_nano_interval(): Create a scalar with month_day_nano_interval type: >>> pa.scalar((1, 15, -30), type=pa.month_day_nano_interval()) - + """ return primitive_type(_Type_INTERVAL_MONTH_DAY_NANO)