From f1871d032579692da75c667606349540b4a5306d Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 18 Aug 2026 08:29:47 +0200 Subject: [PATCH 01/21] Add ImportTensorVersioned --- cpp/src/arrow/c/dlpack.cc | 223 +++++++++++++++++++++ cpp/src/arrow/c/dlpack.h | 29 +++ cpp/src/arrow/c/dlpack_test.cc | 349 +++++++++++++++++++++++++++++++++ cpp/src/arrow/tensor.cc | 36 +++- cpp/src/arrow/tensor.h | 9 + 5 files changed, 641 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 4e25d50bb56f..521248b06c11 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -18,22 +18,32 @@ #include "arrow/c/dlpack.h" #include +#include #include +#include #include #include #include "arrow/array/array_base.h" +#include "arrow/array/util.h" #include "arrow/buffer.h" #include "arrow/c/dlpack_abi.h" #include "arrow/device.h" #include "arrow/tensor.h" #include "arrow/type.h" #include "arrow/type_traits.h" +#include "arrow/util/checked_cast.h" +#include "arrow/util/logging_internal.h" +#include "arrow/util/small_vector.h" namespace arrow::dlpack { namespace { +/*************** + * Producers * + ***************/ + Result GetDLDataType(const DataType& type) { auto dtype = DLDataType{}; dtype.lanes = 1; @@ -248,4 +258,217 @@ Result ExportDevice(const std::shared_ptr& t) { return ExportDeviceImpl(t); } +/*************** + * Consumers * + ***************/ + +namespace { + +class CppDLTensor { + public: + using value_type = DLManagedTensorVersioned; + using pointer_type = value_type*; + + static Result TakeOwnership(pointer_type ptr) { + if (ptr == nullptr) { + return Status::Invalid("Received null pointer."); + } + // Create the wrapper before checking the version as the spec mandates that the + // deleter MUST be called on version major mismatch. + auto out = CppDLTensor(ptr); + if (out.ptr_->version.major != DLPACK_MAJOR_VERSION) { + return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, + ", expected ", DLPACK_MAJOR_VERSION); + } + return out; + } + + const DLTensor& tensor() const { return ptr_->dl_tensor; } + + int64_t ndim() const { + DCHECK_GE(tensor().ndim, 0); + return tensor().ndim; + } + + template + T* data_as() { + return static_cast(tensor().data); + } + + std::span shape() const { + return {tensor().shape, static_cast(ndim())}; + } + + std::span strides() const { + return {tensor().strides, static_cast(ndim())}; + } + + bool flag_is_set(uint8_t bits) const { return (ptr_->flags & bits) == bits; } + + bool is_readonly() const { return flag_is_set(DLPACK_FLAG_BITMASK_READ_ONLY); } + + int32_t byte_width() const { return tensor().dtype.bits / 8; } + + private: + struct Deleter { + void operator()(pointer_type ptr) { + // Null is valid in DLPack spec + if (auto del = ptr->deleter) { + del(ptr); + } + } + }; + + /// Make a safe wrapper that will delete the resource in case of exception. + std::unique_ptr ptr_; + + explicit CppDLTensor(pointer_type ptr) : ptr_(ptr) {} +}; + +Result> DataTypeFromDLPack(DLDataType dtype) { + if (dtype.lanes != 1) { + return Status::TypeError("Only type with one lane are supported."); + } + + auto constexpr as_fw = [](auto dt) { + return std::static_pointer_cast(std::move(dt)); + }; + + switch (dtype.code) { + case kDLInt: { + switch (dtype.bits) { + case 8: + return as_fw(int8()); + case 16: + return as_fw(int16()); + case 32: + return as_fw(int32()); + case 64: + return as_fw(int64()); + default: + return Status::Invalid("unsupported integer bit width ", + static_cast(dtype.bits)); + } + } + case kDLUInt: { + switch (dtype.bits) { + case 8: + return as_fw(uint8()); + case 16: + return as_fw(uint16()); + case 32: + return as_fw(uint32()); + case 64: + return as_fw(uint64()); + default: + return Status::Invalid("unsupported unsigned integer bit width ", + static_cast(dtype.bits)); + } + } + case kDLFloat: { + switch (dtype.bits) { + case 16: + return as_fw(float16()); + case 32: + return as_fw(float32()); + case 64: + return as_fw(float64()); + default: + return Status::Invalid("unsupported float bit width ", + static_cast(dtype.bits)); + } + } + default: { + return Status::Invalid("unsupported DLPack type ", static_cast(dtype.code)); + } + } +} + +inline std::vector StridesInBytes(std::span strides, + int64_t byte_width) { + std::vector out{}; + out.reserve(strides.size()); + for (const auto& s : strides) { + out.push_back(s * byte_width); + } + return out; +} + +Result> ImportBuffer(CppDLTensor&& dl, bool copy) { + // DLPack strides are in number of elements, so is the size we compute from them. + ARROW_ASSIGN_OR_RAISE(const auto nelements, + internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); + const auto nbytes = nelements * dl.byte_width(); + // DLPack mandates a null data pointer when the tensor holds no element, so there is + // neither anything to share nor to copy. + uint8_t* data = + (nbytes == 0) ? nullptr : dl.data_as() + dl.tensor().byte_offset; + + std::shared_ptr buffer = nullptr; + if (nbytes == 0) { + // DLPack data pointer may be null on empty tensors + buffer = std::make_shared(data, nbytes); + } else if (copy) { + ARROW_ASSIGN_OR_RAISE(buffer, MutableBuffer::CopyNonOwned( + {data, nbytes}, default_cpu_memory_manager())); + } else { + const bool readonly = dl.is_readonly(); + // Trick to keep DLPack data alive taken from `Buffer::FromVector`. + auto deleter = [dl = std::move(dl)](auto* buffer) { delete buffer; }; + if (readonly) { + buffer = {new Buffer{data, nbytes}, std::move(deleter)}; + } else { + buffer = std::shared_ptr{ + new MutableBuffer{data, nbytes}, + std::move(deleter), + }; + } + } + + return buffer; +} + +} // namespace + +Result> ImportArrayVersioned(DLManagedTensorVersioned* unmanaged, + bool copy) { + ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); + + if (dl.tensor().device.device_type != kDLCPU) { + return Status::NotImplemented( + "DLPack support is implemented only for buffers on CPU device."); + } + + if (dl.ndim() != 1 || dl.strides().front() != 1) { + return Status::NotImplemented( + "Only contiguous one dimensional tensor can be imported as arrays." + " Try importing to Tensor first."); + } + + ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); + const auto nelements = dl.shape().front(); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + auto data = ArrayData::Make(type, nelements, {nullptr, std::move(buffer)}); + return MakeArray(std::move(data)); +} + +Result> ImportTensorVersioned(DLManagedTensorVersioned* unmanaged, + bool copy) { + ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); + + if (dl.tensor().device.device_type != kDLCPU) { + return Status::NotImplemented( + "DLPack support is implemented only for buffers on CPU device."); + } + + ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); + auto shape = std::vector(dl.shape().begin(), dl.shape().end()); + auto strides = std::vector(dl.strides().begin(), dl.strides().end()); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + const auto byte_width = type->byte_width(); + + return Tensor::Make(std::move(type), std::move(buffer), std::move(shape), + StridesInBytes(std::move(strides), byte_width)); +} + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 8a9084f36c73..152dd8dbbad7 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -105,4 +105,33 @@ Result ExportDevice(const std::shared_ptr& arr); ARROW_EXPORT Result ExportDevice(const std::shared_ptr& t); +/// \brief Import a DLPack tensor as an Arrow Array. +/// +/// Same restrictions on data types as `ExportArrayVersioned`, and only row-major +/// tensors are supported. Dimensions beyond the first are imported as nested +/// fixed size lists. +/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// +/// \param[in] raw DLPack tensor +/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack +/// producer. +/// \return An Arrow Array +ARROW_EXPORT +Result> ImportArrayVersioned(DLManagedTensorVersioned* raw, + bool copy); + +/// \brief Import a DLPack tensor as an Arrow Tensor. +/// +/// Same restrictions on data types as `ExportTensorVersioned`. +/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// If the DLPack input is marked as readonly, this will produce an immutable tensor. +/// +/// \param[in] raw Arrow array +/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack +/// producer. +/// \return An Arrow Tensor +ARROW_EXPORT +Result> ImportTensorVersioned(DLManagedTensorVersioned* raw, + bool copy); + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 05de22237a9f..6f320e24b25e 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -329,4 +329,353 @@ TYPED_TEST(TestExportTensor, TestTensorStrided) { f_dlpack_strides); } +/*************** + * Consumers * + ***************/ + +/// A DLPack tensor as a foreign library would produce it. +struct ForeignTensor { + DLDataType dtype = {.code = kDLFloat, .bits = 32, .lanes = 1}; + std::vector shape = {}; + /// In number of elements, as mandated by DLPack. + std::vector strides = {}; + std::vector data = {}; + DLDevice device = {.device_type = kDLCPU, .device_id = 0}; + uint64_t byte_offset = 0; + uint64_t flags = 0; + /// Incremented when the consumer releases the tensor. + std::shared_ptr deleted = std::make_shared(0); + + DLManagedTensorVersioned managed = {}; +}; + +template +std::vector ToBytes(const std::vector& values) { + std::vector bytes(values.size() * sizeof(T)); + std::memcpy(bytes.data(), values.data(), bytes.size()); + return bytes; +} + +/// Hand out a DLPack tensor owning ``foreign``, releasing it through its deleter. +DLManagedTensorVersioned* Produce(ForeignTensor foreign) { + auto owned = std::make_unique(std::move(foreign)); + owned->managed = { + .version = {.major = DLPACK_MAJOR_VERSION, .minor = DLPACK_MINOR_VERSION}, + .manager_ctx = owned.get(), + .deleter = + [](DLManagedTensorVersioned* self) { + auto* ctx = static_cast(self->manager_ctx); + ++(*ctx->deleted); + delete ctx; + }, + .flags = owned->flags, + .dl_tensor = + { + .data = owned->data.data(), + .device = owned->device, + .ndim = static_cast(owned->shape.size()), + .dtype = owned->dtype, + .shape = owned->shape.data(), + .strides = owned->strides.data(), + .byte_offset = owned->byte_offset, + }, + }; + return &owned.release()->managed; +} + +template +struct TensorConsumer { + using Imported = std::shared_ptr; + static constexpr bool copy = kCopy; + static constexpr const char* name = copy ? "TensorCopied" : "TensorShared"; + + static Result Import(DLManagedTensorVersioned* raw) { + return ImportTensorVersioned(raw, copy); + } + static std::shared_ptr ValueType(const Imported& t) { return t->type(); } + static const uint8_t* RawData(const Imported& t) { return t->raw_data(); } + static bool IsMutable(const Imported& t) { return t->is_mutable(); } + static int64_t Size(const Imported& t) { return t->size(); } +}; + +template +struct ArrayConsumer { + using Imported = std::shared_ptr; + static constexpr bool copy = kCopy; + static constexpr const char* name = copy ? "ArrayCopied" : "ArrayShared"; + + static Result Import(DLManagedTensorVersioned* raw) { + return ImportArrayVersioned(raw, copy); + } + static std::shared_ptr ValueType(const Imported& arr) { return arr->type(); } + static const uint8_t* RawData(const Imported& arr) { + return arr->data()->buffers[1]->data() + arr->offset() * arr->type()->byte_width(); + } + static bool IsMutable(const Imported& arr) { + return arr->data()->buffers[1]->is_mutable(); + } + static int64_t Size(const Imported& arr) { return arr->length(); } +}; + +struct ConsumerNames { + template + static std::string GetName(int) { + return Consumer::name; + } +}; + +using ConsumerTypes = ::testing::Types, TensorConsumer, + ArrayConsumer, ArrayConsumer>; +using TensorConsumerTypes = ::testing::Types, TensorConsumer>; +using ArrayConsumerTypes = ::testing::Types, ArrayConsumer>; + +/// Tests sharing the same expectations for Arrow Tensor and Array imports. +template +class TestImport : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImport, ConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImport, Basic) { + auto foreign = ForeignTensor{ + .shape = {6}, + .strides = {1}, + .data = ToBytes(std::vector{0, 0, 1, 2, 3, 4, 5, 6}), + .byte_offset = 2 * sizeof(float), + .flags = DLPACK_FLAG_BITMASK_READ_ONLY, + }; + const auto deleted = foreign.deleted; + const auto expected = std::vector{1, 2, 3, 4, 5, 6}; + const auto* values = foreign.data.data() + foreign.byte_offset; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + + AssertTypeEqual(*float32(), *TypeParam::ValueType(imported)); + ASSERT_EQ(6, TypeParam::Size(imported)); + // A copy is ours to mutate, whatever the producer flagged + ASSERT_EQ(TypeParam::copy, TypeParam::IsMutable(imported)); + ASSERT_EQ(0, std::memcmp(TypeParam::RawData(imported), expected.data(), + expected.size() * sizeof(float))); + + if constexpr (TypeParam::copy) { + // The producer tensor is released as soon as its data has been copied + ASSERT_EQ(1, *deleted); + } else { + ASSERT_EQ(TypeParam::RawData(imported), values); + // The producer tensor is kept alive by the imported data + ASSERT_EQ(0, *deleted); + imported.reset(); + ASSERT_EQ(1, *deleted); + } +} + +TYPED_TEST(TestImport, Mutable) { + auto foreign = ForeignTensor{ + .shape = {4}, + .strides = {1}, + .data = ToBytes(std::vector{1, 2, 3, 4}), + .flags = 0, + }; + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_TRUE(TypeParam::IsMutable(imported)); +} + +TYPED_TEST(TestImport, NullDeleter) { + // The DLPack spec allows producers not to set a deleter + auto* managed = + Produce({.shape = {2}, .strides = {1}, .data = std::vector(8)}); + auto* foreign = static_cast(managed->manager_ctx); + managed->deleter = nullptr; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + imported.reset(); + delete foreign; +} + +TYPED_TEST(TestImport, DataTypes) { + const std::vector>> cases = { + {{kDLInt, 8, 1}, int8()}, {{kDLInt, 16, 1}, int16()}, + {{kDLInt, 32, 1}, int32()}, {{kDLInt, 64, 1}, int64()}, + {{kDLUInt, 8, 1}, uint8()}, {{kDLUInt, 16, 1}, uint16()}, + {{kDLUInt, 32, 1}, uint32()}, {{kDLUInt, 64, 1}, uint64()}, + {{kDLFloat, 16, 1}, float16()}, {{kDLFloat, 32, 1}, float32()}, + {{kDLFloat, 64, 1}, float64()}}; + + for (const auto& [dtype, expected] : cases) { + ARROW_SCOPED_TRACE("dtype ", expected->ToString()); + ASSERT_OK_AND_ASSIGN( + auto imported, TypeParam::Import(Produce( + {.dtype = dtype, + .shape = {3}, + .strides = {1}, + .data = std::vector(3 * expected->byte_width())}))); + AssertTypeEqual(*expected, *TypeParam::ValueType(imported)); + } +} + +TYPED_TEST(TestImport, Empty) { + // DLPack mandates a null data pointer when the tensor holds no element + auto* managed = Produce({.shape = {0}, .strides = {1}}); + managed->dl_tensor.data = nullptr; + + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + ASSERT_EQ(0, TypeParam::Size(imported)); +} + +TYPED_TEST(TestImport, Errors) { + auto check = [](ForeignTensor foreign, const std::string& message) { + const auto deleted = foreign.deleted; + const auto status = TypeParam::Import(Produce(std::move(foreign))).status(); + EXPECT_EQ(message, status.ToStringWithoutContextLines()); + // Ownership is taken even when the import fails + EXPECT_EQ(1, *deleted); + }; + + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Received null pointer.", + TypeParam::Import(nullptr)); + check({.shape = {2}, + .strides = {1}, + .data = std::vector(8), + .device = {.device_type = kDLCUDA, .device_id = 0}}, + "NotImplemented: DLPack support is implemented only for buffers on CPU device."); + check({.dtype = {kDLFloat, 32, 2}, .shape = {2}, .strides = {1}}, + "Type error: Only type with one lane are supported."); + check({.dtype = {kDLInt, 4, 1}, .shape = {2}, .strides = {1}}, + "Invalid: unsupported integer bit width 4"); + check({.dtype = {kDLBool, 8, 1}, .shape = {2}, .strides = {1}}, + "Invalid: unsupported DLPack type " + std::to_string(kDLBool)); +} + +TYPED_TEST(TestImport, UnsupportedVersion) { + auto* managed = + Produce({.shape = {2}, .strides = {1}, .data = std::vector(8)}); + const auto deleted = static_cast(managed->manager_ctx)->deleted; + const auto major = DLPACK_MAJOR_VERSION + 1; + managed->version.major = major; + + ASSERT_RAISES_WITH_MESSAGE(Invalid, + "Invalid: Unsupported DLPack major version " + + std::to_string(major) + ", expected " + + std::to_string(DLPACK_MAJOR_VERSION), + TypeParam::Import(managed)); + // The spec mandates the deleter to be called on major version mismatch + ASSERT_EQ(1, *deleted); +} + +template +class TestImportTensor : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImportTensor, TensorConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImportTensor, ShapeAndStrides) { + auto foreign = ForeignTensor{ + .shape = {2, 3}, + .strides = {3, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), + }; + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(foreign)))); + + ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(2, 3)); + // Arrow strides are in bytes, DLPack strides in elements + ASSERT_THAT(tensor->strides(), + ::testing::ElementsAre(3 * sizeof(float), sizeof(float))); +} + +TYPED_TEST(TestImportTensor, Empty) { + auto* managed = Produce({.shape = {0, 3}, .strides = {3, 1}}); + managed->dl_tensor.data = nullptr; + + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(0, 3)); +} + +TYPED_TEST(TestImportTensor, Strided) { + auto column_major = ForeignTensor{ + .shape = {2, 3}, + .strides = {1, 2}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), + }; + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(column_major)))); + ASSERT_TRUE(tensor->is_column_major()); + + // A 2x2 window over every other row of a 4x2 buffer + auto non_contiguous = ForeignTensor{ + .shape = {2, 2}, + .strides = {4, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6, 7, 8}), + }; + ASSERT_OK_AND_ASSIGN(tensor, TypeParam::Import(Produce(std::move(non_contiguous)))); + ASSERT_FALSE(tensor->is_contiguous()); + ASSERT_EQ(6, tensor->template Value({1, 1})); +} + +TYPED_TEST(TestImportTensor, NegativeStrides) { + auto foreign = ForeignTensor{ + .shape = {2, 2}, + .strides = {-2, 1}, + .data = std::vector(16), + }; + ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: negative strides not supported", + TypeParam::Import(Produce(std::move(foreign)))); +} + +TYPED_TEST(TestImportTensor, RoundTrip) { + const auto original = TensorFromJSON(float64(), "[1, 2, 3, 4, 5, 6]", {3, 2}); + + ASSERT_OK_AND_ASSIGN(auto* managed, ExportTensorVersioned(original, /*copy=*/false)); + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + + ASSERT_TRUE(tensor->Equals(*original)); + if constexpr (!TypeParam::copy) { + ASSERT_EQ(original->raw_data(), tensor->raw_data()); + } +} + +template +class TestImportArray : public ::testing::Test {}; + +TYPED_TEST_SUITE(TestImportArray, ArrayConsumerTypes, ConsumerNames); + +TYPED_TEST(TestImportArray, OneDimension) { + auto foreign = ForeignTensor{ + .dtype = {.code = kDLInt, .bits = 32, .lanes = 1}, + .shape = {4}, + .strides = {1}, + .data = ToBytes(std::vector{1, 2, 3, 4}), + }; + ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(Produce(std::move(foreign)))); + AssertArraysEqual(*ArrayFromJSON(int32(), "[1, 2, 3, 4]"), *array); +} + +TYPED_TEST(TestImportArray, Unsupported) { + auto check = [](ForeignTensor foreign) { + ASSERT_RAISES_WITH_MESSAGE( + NotImplemented, + "NotImplemented: Only contiguous one dimensional tensor can be imported as" + " arrays. Try importing to Tensor first.", + TypeParam::Import(Produce(std::move(foreign)))); + }; + + // Only a Tensor can hold more than one dimension + check({.shape = {2, 3}, + .strides = {3, 1}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6})}); + check({.shape = {0, 3}, .strides = {1, 1}}); + // Array values are contiguous, whatever the dimension count + check( + {.shape = {3}, .strides = {2}, .data = ToBytes(std::vector{1, 2, 3, 4, 5})}); + check({.shape = {2, 2}, .strides = {-2, 1}, .data = std::vector(16)}); +} + +TYPED_TEST(TestImportArray, RoundTrip) { + const auto original = ArrayFromJSON(float64(), "[1, 2, 3, 4, 5, 6]"); + + ASSERT_OK_AND_ASSIGN(auto* managed, ExportArrayVersioned(original, /*copy=*/false)); + ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(managed)); + + AssertArraysEqual(*original, *array); + if constexpr (!TypeParam::copy) { + ASSERT_EQ(TypeParam::RawData(original), TypeParam::RawData(array)); + } +} + } // namespace arrow::dlpack diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index f2ff11a4f66e..d4b8ffef3611 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -109,9 +109,35 @@ Status ComputeColumnMajorStrides(const FixedWidthType& type, return Status::OK(); } -} // namespace internal +Result ComputeTensorSize(std::span shape, + std::span strides, int64_t elem_size) { + // Check the largest offset can be computed without overflow + const size_t ndim = shape.size(); + int64_t largest_offset = elem_size; + for (size_t i = 0; i < ndim; ++i) { + if (shape[i] == 0) continue; + if (strides[i] < 0) { + // TODO(mrkn): Support negative strides for sharing views + return Status::Invalid("negative strides not supported"); + } -namespace { + int64_t dim_offset = 0; + if (!internal::MultiplyWithOverflow(shape[i] - 1, strides[i], &dim_offset)) { + if (!internal::AddWithOverflow(largest_offset, dim_offset, &largest_offset)) { + continue; + } + } + + return Status::Invalid( + "offsets computed from shape and strides would not fit in 64-bit integer"); + } + + // A dimension with no element means empty for which the preceding does not apply. + if (std::find(shape.begin(), shape.end(), 0) != shape.end()) { + return 0; + } + return largest_offset; +} inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, const std::vector& shape, @@ -194,7 +220,7 @@ Status CheckTensorStridesValidity(const std::shared_ptr& data, return Status::OK(); } -} // namespace +} // namespace internal namespace internal { @@ -532,11 +558,11 @@ bool Tensor::is_contiguous() const { } bool Tensor::is_row_major() const { - return IsTensorStridesRowMajor(type_, shape_, strides_); + return internal::IsTensorStridesRowMajor(type_, shape_, strides_); } bool Tensor::is_column_major() const { - return IsTensorStridesColumnMajor(type_, shape_, strides_); + return internal::IsTensorStridesColumnMajor(type_, shape_, strides_); } Type::type Tensor::type_id() const { return type_->id(); } diff --git a/cpp/src/arrow/tensor.h b/cpp/src/arrow/tensor.h index f3270313434e..2917905049ce 100644 --- a/cpp/src/arrow/tensor.h +++ b/cpp/src/arrow/tensor.h @@ -71,6 +71,15 @@ bool IsTensorStridesContiguous(const std::shared_ptr& type, const std::vector& shape, const std::vector& strides); +/// Compute the size needed to store the tensor with the given strides and shape. +/// +/// If the strides are in number of element, pass `elem_size=1` to compute the buffer size +/// in the number of elements. If the strides are in bytes, pass the element size in byte +/// to `elem_size` and get the result in bytes. +ARROW_EXPORT +Result ComputeTensorSize(std::span shape, + std::span strides, int64_t elem_size); + ARROW_EXPORT Status ValidateTensorParameters(const std::shared_ptr& type, const std::shared_ptr& data, From 04d94002e1d0cd3bc118d8f459038b9f3964add5 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 1 Sep 2026 15:59:01 +0200 Subject: [PATCH 02/21] Bind FixedShapeTensorArray.from_tensor --- python/pyarrow/array.pxi | 26 ++++++++++++++++++++++++++ python/pyarrow/includes/libarrow.pxd | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 691623b87f0e..ea67b90ca28d 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -4969,6 +4969,32 @@ cdef class FixedShapeTensorArray(ExtensionArray): return self.to_tensor().to_numpy() + @staticmethod + def from_tensor(Tensor tensor not None): + """ + Convert a pyarrow.Tensor to a fixed shape tensor extension array. + + The first dimension of the tensor becomes the length of the fixed shape + tensor array and the remaining dimensions the shape of the individual + tensors. If the tensor provides strides, they are used to determine the + dimension permutation, otherwise row-major layout is assumed. + + Parameters + ---------- + tensor : pyarrow.Tensor + + Returns + ------- + FixedShapeTensorArray + """ + cdef shared_ptr[CFixedShapeTensorArray] c_array + + with nogil: + c_array = GetResultValue( + CFixedShapeTensorArray.FromTensor(tensor.sp_tensor)) + + return pyarrow_wrap_array( c_array) + @staticmethod def from_numpy_ndarray(obj, dim_names=None): """ diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index ffc02ffd79ae..5eac72ace8a9 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -3090,6 +3090,13 @@ cdef extern from "arrow/extension/fixed_shape_tensor.h" namespace "arrow::extens const vector[int64_t] permutation() const vector[c_string] dim_names() + cdef cppclass CFixedShapeTensorArray \ + " arrow::extension::FixedShapeTensorArray"(CExtensionArray): + + @staticmethod + CResult[shared_ptr[CFixedShapeTensorArray]] FromTensor( + const shared_ptr[CTensor]& tensor) + cdef extern from "arrow/extension/opaque.h" namespace "arrow::extension" nogil: cdef cppclass COpaqueType \ From bdf64b4668a39fa17bbb7a73b35cd9e11f26f404 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 1 Sep 2026 16:55:58 +0200 Subject: [PATCH 03/21] Fix linkage issue --- cpp/src/arrow/tensor.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cpp/src/arrow/tensor.cc b/cpp/src/arrow/tensor.cc index d4b8ffef3611..8b3137d66a35 100644 --- a/cpp/src/arrow/tensor.cc +++ b/cpp/src/arrow/tensor.cc @@ -139,9 +139,10 @@ Result ComputeTensorSize(std::span shape, return largest_offset; } -inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, - const std::vector& shape, - const std::vector& strides) { +namespace { +bool IsTensorStridesRowMajor(const std::shared_ptr& type, + const std::vector& shape, + const std::vector& strides) { std::vector c_strides; const auto& fw_type = checked_cast(*type); if (internal::ComputeRowMajorStrides(fw_type, shape, &c_strides).ok()) { @@ -151,9 +152,9 @@ inline bool IsTensorStridesRowMajor(const std::shared_ptr& type, } } -inline bool IsTensorStridesColumnMajor(const std::shared_ptr& type, - const std::vector& shape, - const std::vector& strides) { +bool IsTensorStridesColumnMajor(const std::shared_ptr& type, + const std::vector& shape, + const std::vector& strides) { std::vector f_strides; const auto& fw_type = checked_cast(*type); if (internal::ComputeColumnMajorStrides(fw_type, shape, &f_strides).ok()) { @@ -163,9 +164,9 @@ inline bool IsTensorStridesColumnMajor(const std::shared_ptr& type, } } -inline Status CheckTensorValidity(const std::shared_ptr& type, - const std::shared_ptr& data, - const std::vector& shape) { +Status CheckTensorValidity(const std::shared_ptr& type, + const std::shared_ptr& data, + const std::vector& shape) { if (!type) { return Status::Invalid("Null type is supplied"); } @@ -219,7 +220,7 @@ Status CheckTensorStridesValidity(const std::shared_ptr& data, } return Status::OK(); } - +} // namespace } // namespace internal namespace internal { From 8fd632de7d88485c8dfa7ce731beea8cde0563a5 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 3 Sep 2026 17:28:58 +0200 Subject: [PATCH 04/21] Add from_dlpack --- cpp/src/arrow/c/dlpack.cc | 11 ++-- cpp/src/arrow/c/dlpack.h | 3 ++ python/pyarrow/array.pxi | 49 +++++++++++++++++- python/pyarrow/includes/libarrow.pxd | 13 +++++ python/pyarrow/tensor.pxi | 47 ++++++++++++++++- python/pyarrow/tests/test_dlpack.py | 77 ++++++++++++++++++++++++++++ 6 files changed, 195 insertions(+), 5 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 521248b06c11..a0a45c09ed5a 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -38,6 +38,11 @@ namespace arrow::dlpack { +extern const DLPackVersion VERSION = { + .major = DLPACK_MAJOR_VERSION, + .minor = DLPACK_MINOR_VERSION, +}; + namespace { /*************** @@ -118,7 +123,7 @@ DT* ExportBuffer(ExportBufferParams&& p) { // Strides must be non-null when ndim > 0 ctx->tensor.dl_tensor.strides = ctx->strides.data(); if constexpr (std::is_same_v) { - ctx->tensor.version = {.major = DLPACK_MAJOR_VERSION, .minor = DLPACK_MINOR_VERSION}; + ctx->tensor.version = VERSION; ctx->tensor.flags = p.flags; } @@ -276,9 +281,9 @@ class CppDLTensor { // Create the wrapper before checking the version as the spec mandates that the // deleter MUST be called on version major mismatch. auto out = CppDLTensor(ptr); - if (out.ptr_->version.major != DLPACK_MAJOR_VERSION) { + if (out.ptr_->version.major != VERSION.major) { return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, - ", expected ", DLPACK_MAJOR_VERSION); + ", expected ", VERSION.major); } return out; } diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 152dd8dbbad7..93b5534c0cc0 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -25,6 +25,9 @@ namespace arrow::dlpack { +/// The DLPack version used during compilation. +extern const DLPackVersion VERSION; + /// \brief Export Arrow array as DLPack tensor. /// /// DLMangedTensor is produced as defined by the DLPack protocol, diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index ea67b90ca28d..5b1bba4ac112 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -15,7 +15,12 @@ # specific language governing permissions and limitations # under the License. -from cpython.pycapsule cimport PyCapsule_CheckExact, PyCapsule_GetPointer, PyCapsule_New +from cpython.pycapsule cimport ( + PyCapsule_CheckExact, + PyCapsule_GetPointer, + PyCapsule_New, + PyCapsule_SetName, +) from collections.abc import Sequence import os @@ -2269,6 +2274,48 @@ cdef class Array(_PandasConvertible): return pyarrow_wrap_array(array) + def from_dlpack(x, /, *, device=None, copy=None): + """ + Construct an Array from an object implementing the DLPack protocol. + + Parameters + ---------- + x : object + The input object containing array data, following the DLPack + protocol (has a ``__dlpack__`` method). + device : tuple[enum.Enum, int], optional + Designates where the resulting Array should reside, in the + format returned by :meth:`Array.__dlpack_device__`. When None, + the output Array occupies the same device as the source. + Default: None. + copy : bool, optional + Controls duplication behavior. True mandates copying; False + prohibits copying and raises ``BufferError`` if unavoidable; + None duplicates only when necessary. Default: None. + + Returns + ------- + Array + An Array housing the data from the input object, potentially + as a copy or view. + """ + version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) + pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( + pycapsule, "dltensor_versioned") + if ptr == NULL: + raise ValueError( + 'DLPack producer did not produce a "dltensor_versioned" PyCapsule') + # Mark the capsule as consumed so its destructor does not also invoke the deleter. + # ImportArrayVersionedFromDLPack will take ownership even if it errors (calling + # the deleter in that case). + PyCapsule_SetName(pycapsule, "used_dltensor_versioned") + with nogil: + # Copy handled on producer side + result = ImportArrayVersionedFromDLPack(ptr, False) + carray = GetResultValue(result) + return pyarrow_wrap_array(carray) + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): """ Export a primitive array as a DLPack capsule. diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 5eac72ace8a9..5b857b33c08d 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1460,6 +1460,10 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: cdef extern from "arrow/c/dlpack_abi.h" nogil: + ctypedef struct DLPackVersion: + uint32_t major + uint32_t minor + ctypedef enum DLDeviceType: kDLCPU = 1 @@ -1475,6 +1479,8 @@ cdef extern from "arrow/c/dlpack_abi.h" nogil: cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: + const DLPackVersion DLPACK_VERSION" arrow::dlpack::VERSION" + CResult[DLManagedTensor*] ExportArrayToDLPack" arrow::dlpack::ExportArray"( const shared_ptr[CArray]& arr) CResult[DLManagedTensor*] ExportTensorToDLPack" arrow::dlpack::ExportTensor"( @@ -1490,6 +1496,13 @@ cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: CResult[DLDevice] ExportDevice(const shared_ptr[CArray]& arr) CResult[DLDevice] ExportDevice(const shared_ptr[CTensor]& tensor) + CResult[shared_ptr[CArray]] \ + ImportArrayVersionedFromDLPack" arrow::dlpack::ImportArrayVersioned"( + DLManagedTensorVersioned* raw, c_bool copy) + CResult[shared_ptr[CTensor]] \ + ImportTensorVersionedFromDLPack" arrow::dlpack::ImportTensorVersioned"( + DLManagedTensorVersioned* raw, c_bool copy) + cdef extern from "arrow/builder.h" namespace "arrow" nogil: diff --git a/python/pyarrow/tensor.pxi b/python/pyarrow/tensor.pxi index 521ee0c3f44f..656da9f260a9 100644 --- a/python/pyarrow/tensor.pxi +++ b/python/pyarrow/tensor.pxi @@ -18,6 +18,8 @@ # Avoid name clash with `pa.struct` function import struct as _struct +from cpython.pycapsule cimport PyCapsule_SetName, PyCapsule_GetPointer + cdef class Tensor(_Weakrefable): """ @@ -300,7 +302,50 @@ strides: {self.strides}""" buffer.strides = cp.PyBytes_AsString(self._ssize_t_strides) buffer.suboffsets = NULL - def __dlpack__(self, stream=None, max_version=None, dl_device=None, copy=None): + def from_dlpack(x, /, *, device=None, copy=None): + """ + Construct a Tensor from an object implementing the DLPack protocol. + + Parameters + ---------- + x : object + The input object containing array data, following the DLPack + protocol (has a ``__dlpack__`` method) or the array API's + ``__array_namespace__`` protocol. + device : tuple[enum.Enum, int], optional + Designates where the resulting Tensor should reside, in the + format returned by :meth:`Tensor.__dlpack_device__`. When None, + the output Tensor occupies the same device as the source. + Default: None. + copy : bool, optional + Controls duplication behavior. True mandates copying; False + prohibits copying and raises ``BufferError`` if unavoidable; + None duplicates only when necessary. Default: None. + + Returns + ------- + Tensor + A Tensor housing the data from the input object, potentially + as a copy or view. + """ + version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) + pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( + pycapsule, "dltensor_versioned") + if ptr == NULL: + raise ValueError( + 'DLPack producer did not produce a "dltensor_versioned" PyCapsule') + # Mark the capsule as consumed so its destructor does not also invoke the deleter. + # ImportTensorVersionedFromDLPack will take ownership even if it errors (calling + # the deleter in that case). + PyCapsule_SetName(pycapsule, "used_dltensor_versioned") + with nogil: + # Copy handled on producer side + result = ImportTensorVersionedFromDLPack(ptr, False) + ctensor = GetResultValue(result) + return pyarrow_wrap_tensor(ctensor) + + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): """ Export a Tensor as a DLPack capsule. diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index e3cc2fd3e9ee..3ba734e1fbed 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -374,3 +374,80 @@ def test_dlpack_cuda_not_supported(): with pytest.raises(NotImplementedError, match="DLPack support is implemented " "only for buffers on CPU device."): carr.__dlpack_device__() + + +@check_bytes_allocated +@pytest.mark.parametrize('np_type', + [np.uint8, np.uint16, np.uint32, np.uint64, + np.int8, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64]) +def test_tensor_from_dlpack(np_type): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + # Non-contiguous, strided slice: DLPack carries explicit strides, so this + # should not need a copy on export. + base = np.arange(24, dtype=np_type).reshape((4, 6)) + expected = base[::2, 1::2] + assert not expected.flags['C_CONTIGUOUS'] + tensor = pa.Tensor.from_dlpack(expected) + assert isinstance(tensor, pa.Tensor) + np.testing.assert_array_equal(tensor.to_numpy(), expected, strict=True) + + +@check_bytes_allocated +@pytest.mark.parametrize('np_type', + [np.uint8, np.uint16, np.uint32, np.uint64, + np.int8, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64]) +def test_array_from_dlpack(np_type): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3, 4, 5], dtype=np_type) + arr = pa.Array.from_dlpack(expected) + assert isinstance(arr, pa.Array) + np.testing.assert_array_equal(arr.to_numpy(), expected, strict=True) + + +@check_bytes_allocated +def test_from_dlpack_zero_copy(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3], dtype=np.int64) + tensor = pa.Tensor.from_dlpack(expected) + result = tensor.to_numpy() + expected[0] = 100 + # Zero-copy import: mutating the source is visible through the tensor. + assert result[0] == 100 + + +@check_bytes_allocated +def test_from_dlpack_explicit_copy(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.array([1, 2, 3], dtype=np.int64) + tensor = pa.Tensor.from_dlpack(expected, copy=True) + result = tensor.to_numpy() + expected[0] = 100 + # The data was copied, so mutating the source is not visible. + assert result[0] == 1 + + +def test_from_dlpack_no_dlpack_method(): + with pytest.raises(AttributeError): + pa.Tensor.from_dlpack(object()) + + +@check_bytes_allocated +def test_array_from_dlpack_multi_dim_not_supported(): + if Version(np.__version__) < Version("2.1.0"): + pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + + expected = np.arange(6, dtype=np.int32).reshape((2, 3)) + with pytest.raises(pa.ArrowNotImplementedError, + match="Only contiguous one dimensional tensor can be " + "imported as arrays"): + pa.Array.from_dlpack(expected) From 1acc550188c5cc45adb1eabcc02e29d6b53f7ff0 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Thu, 3 Sep 2026 18:15:53 +0200 Subject: [PATCH 05/21] Add missing export --- cpp/src/arrow/c/dlpack.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 93b5534c0cc0..6b552cf818f4 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -26,7 +26,7 @@ namespace arrow::dlpack { /// The DLPack version used during compilation. -extern const DLPackVersion VERSION; +ARROW_EXPORT extern const DLPackVersion VERSION; /// \brief Export Arrow array as DLPack tensor. /// From e937256099c42bedcbff4ddadf2dbfa13c9b986b Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Fri, 4 Sep 2026 09:53:04 +0200 Subject: [PATCH 06/21] Apply review comments --- cpp/src/arrow/c/dlpack.cc | 43 +++++++++++++++++++++-------- cpp/src/arrow/c/dlpack.h | 9 +++--- cpp/src/arrow/c/dlpack_test.cc | 4 +-- python/pyarrow/array.pxi | 3 ++ python/pyarrow/tensor.pxi | 12 ++++++-- python/pyarrow/tests/test_dlpack.py | 7 +++-- 6 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index a0a45c09ed5a..1a990ebd9d5a 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -18,9 +18,7 @@ #include "arrow/c/dlpack.h" #include -#include #include -#include #include #include @@ -32,9 +30,9 @@ #include "arrow/tensor.h" #include "arrow/type.h" #include "arrow/type_traits.h" -#include "arrow/util/checked_cast.h" +#include "arrow/util/int_util_overflow.h" #include "arrow/util/logging_internal.h" -#include "arrow/util/small_vector.h" +#include "arrow/util/macros.h" namespace arrow::dlpack { @@ -275,16 +273,27 @@ class CppDLTensor { using pointer_type = value_type*; static Result TakeOwnership(pointer_type ptr) { - if (ptr == nullptr) { + if (ARROW_PREDICT_FALSE(ptr == nullptr)) { return Status::Invalid("Received null pointer."); } // Create the wrapper before checking the version as the spec mandates that the // deleter MUST be called on version major mismatch. auto out = CppDLTensor(ptr); - if (out.ptr_->version.major != VERSION.major) { + if (ARROW_PREDICT_FALSE(out.ptr_->version.major != VERSION.major)) { return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, ", expected ", VERSION.major); } + if (ARROW_PREDICT_FALSE(out.tensor().ndim < 0)) { + return Status::Invalid("Invalid DLPack tensor: ndim must be >= 0"); + } + if (ARROW_PREDICT_FALSE(out.tensor().ndim != 0 && out.tensor().shape == nullptr)) { + return Status::Invalid( + "Invalid DLPack tensor: shape must be non-null when ndim != 0"); + } + if (ARROW_PREDICT_FALSE(out.tensor().ndim != 0 && out.tensor().strides == nullptr)) { + return Status::Invalid( + "Invalid DLPack tensor: strides must be non-null when ndim != 0"); + } return out; } @@ -389,12 +398,17 @@ Result> DataTypeFromDLPack(DLDataType dtype) { } } -inline std::vector StridesInBytes(std::span strides, - int64_t byte_width) { +Result> StridesInBytes(std::span strides, + int64_t byte_width) { std::vector out{}; out.reserve(strides.size()); for (const auto& s : strides) { - out.push_back(s * byte_width); + int64_t stride_bytes = 0; + if (ARROW_PREDICT_FALSE( + internal::MultiplyWithOverflow(s, byte_width, &stride_bytes))) { + return Status::Invalid("Overflow computing DLPack tensor stride in bytes."); + } + out.push_back(stride_bytes); } return out; } @@ -403,7 +417,11 @@ Result> ImportBuffer(CppDLTensor&& dl, bool copy) { // DLPack strides are in number of elements, so is the size we compute from them. ARROW_ASSIGN_OR_RAISE(const auto nelements, internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); - const auto nbytes = nelements * dl.byte_width(); + int64_t nbytes = 0; + if (ARROW_PREDICT_FALSE(internal::MultiplyWithOverflow( + nelements, static_cast(dl.byte_width()), &nbytes))) { + return Status::Invalid("Overflow computing DLPack tensor size in bytes."); + } // DLPack mandates a null data pointer when the tensor holds no element, so there is // neither anything to share nor to copy. uint8_t* data = @@ -445,7 +463,7 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* un } if (dl.ndim() != 1 || dl.strides().front() != 1) { - return Status::NotImplemented( + return Status::Invalid( "Only contiguous one dimensional tensor can be imported as arrays." " Try importing to Tensor first."); } @@ -471,9 +489,10 @@ Result> ImportTensorVersioned(DLManagedTensorVersioned* auto strides = std::vector(dl.strides().begin(), dl.strides().end()); ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); const auto byte_width = type->byte_width(); + ARROW_ASSIGN_OR_RAISE(auto strides_bytes, StridesInBytes(strides, byte_width)); return Tensor::Make(std::move(type), std::move(buffer), std::move(shape), - StridesInBytes(std::move(strides), byte_width)); + std::move(strides_bytes)); } } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 6b552cf818f4..4c7f7a41ab2c 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -110,10 +110,9 @@ Result ExportDevice(const std::shared_ptr& t); /// \brief Import a DLPack tensor as an Arrow Array. /// -/// Same restrictions on data types as `ExportArrayVersioned`, and only row-major -/// tensors are supported. Dimensions beyond the first are imported as nested -/// fixed size lists. -/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// Same restrictions on data types as `ExportArrayVersioned`, only row-major +/// tensors are supported. Takes ownership of the `DLManagedTensorVersioned` in +/// an error-safe fashion. /// /// \param[in] raw DLPack tensor /// \param[in] copy Whether to copy the data instead of sharing it with the DLPack @@ -126,7 +125,7 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* ra /// \brief Import a DLPack tensor as an Arrow Tensor. /// /// Same restrictions on data types as `ExportTensorVersioned`. -/// Takes ownership of the `DLManagedTensorVersioned` though it may point to shared data. +/// Takes ownership of the `DLManagedTensorVersioned` in an error-safe fashion. /// If the DLPack input is marked as readonly, this will produce an immutable tensor. /// /// \param[in] raw Arrow array diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 6f320e24b25e..4f22c5c527a1 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -649,8 +649,8 @@ TYPED_TEST(TestImportArray, OneDimension) { TYPED_TEST(TestImportArray, Unsupported) { auto check = [](ForeignTensor foreign) { ASSERT_RAISES_WITH_MESSAGE( - NotImplemented, - "NotImplemented: Only contiguous one dimensional tensor can be imported as" + Invalid, + "Invalid: Only contiguous one dimensional tensor can be imported as" " arrays. Try importing to Tensor first.", TypeParam::Import(Produce(std::move(foreign)))); }; diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5b1bba4ac112..bed4c5a644db 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2274,6 +2274,7 @@ cdef class Array(_PandasConvertible): return pyarrow_wrap_array(array) + @staticmethod def from_dlpack(x, /, *, device=None, copy=None): """ Construct an Array from an object implementing the DLPack protocol. @@ -2301,6 +2302,8 @@ cdef class Array(_PandasConvertible): """ version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + if not PyCapsule_CheckExact(pycapsule): + raise TypeError("DLPack producer did not return a PyCapsule") cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( pycapsule, "dltensor_versioned") if ptr == NULL: diff --git a/python/pyarrow/tensor.pxi b/python/pyarrow/tensor.pxi index 656da9f260a9..a40449866995 100644 --- a/python/pyarrow/tensor.pxi +++ b/python/pyarrow/tensor.pxi @@ -18,7 +18,11 @@ # Avoid name clash with `pa.struct` function import struct as _struct -from cpython.pycapsule cimport PyCapsule_SetName, PyCapsule_GetPointer +from cpython.pycapsule cimport ( + PyCapsule_CheckExact, + PyCapsule_GetPointer, + PyCapsule_SetName, +) cdef class Tensor(_Weakrefable): @@ -302,6 +306,7 @@ strides: {self.strides}""" buffer.strides = cp.PyBytes_AsString(self._ssize_t_strides) buffer.suboffsets = NULL + @staticmethod def from_dlpack(x, /, *, device=None, copy=None): """ Construct a Tensor from an object implementing the DLPack protocol. @@ -310,8 +315,7 @@ strides: {self.strides}""" ---------- x : object The input object containing array data, following the DLPack - protocol (has a ``__dlpack__`` method) or the array API's - ``__array_namespace__`` protocol. + protocol (has a ``__dlpack__`` method). device : tuple[enum.Enum, int], optional Designates where the resulting Tensor should reside, in the format returned by :meth:`Tensor.__dlpack_device__`. When None, @@ -330,6 +334,8 @@ strides: {self.strides}""" """ version = (DLPACK_VERSION.major, DLPACK_VERSION.minor) pycapsule = x.__dlpack__(max_version=version, dl_device=device, copy=copy) + if not PyCapsule_CheckExact(pycapsule): + raise TypeError("DLPack producer did not return a PyCapsule") cdef DLManagedTensorVersioned* ptr = PyCapsule_GetPointer( pycapsule, "dltensor_versioned") if ptr == NULL: diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 3ba734e1fbed..6236b540af19 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -447,7 +447,8 @@ def test_array_from_dlpack_multi_dim_not_supported(): pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") expected = np.arange(6, dtype=np.int32).reshape((2, 3)) - with pytest.raises(pa.ArrowNotImplementedError, - match="Only contiguous one dimensional tensor can be " - "imported as arrays"): + with pytest.raises( + pa.ArrowInvalid, + match="Only contiguous one dimensional tensor can be imported as arrays", + ): pa.Array.from_dlpack(expected) From 1a0d286041a717d57015752b169dcd35cc830ccf Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 7 Sep 2026 14:33:35 +0200 Subject: [PATCH 07/21] Rename version constant --- cpp/src/arrow/c/dlpack.cc | 8 ++++---- cpp/src/arrow/c/dlpack.h | 2 +- python/pyarrow/includes/libarrow.pxd | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 1a990ebd9d5a..98e7a48977a6 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -36,7 +36,7 @@ namespace arrow::dlpack { -extern const DLPackVersion VERSION = { +extern const DLPackVersion kVersion = { .major = DLPACK_MAJOR_VERSION, .minor = DLPACK_MINOR_VERSION, }; @@ -121,7 +121,7 @@ DT* ExportBuffer(ExportBufferParams&& p) { // Strides must be non-null when ndim > 0 ctx->tensor.dl_tensor.strides = ctx->strides.data(); if constexpr (std::is_same_v) { - ctx->tensor.version = VERSION; + ctx->tensor.version = kVersion; ctx->tensor.flags = p.flags; } @@ -279,9 +279,9 @@ class CppDLTensor { // Create the wrapper before checking the version as the spec mandates that the // deleter MUST be called on version major mismatch. auto out = CppDLTensor(ptr); - if (ARROW_PREDICT_FALSE(out.ptr_->version.major != VERSION.major)) { + if (ARROW_PREDICT_FALSE(out.ptr_->version.major != kVersion.major)) { return Status::Invalid("Unsupported DLPack major version ", out.ptr_->version.major, - ", expected ", VERSION.major); + ", expected ", kVersion.major); } if (ARROW_PREDICT_FALSE(out.tensor().ndim < 0)) { return Status::Invalid("Invalid DLPack tensor: ndim must be >= 0"); diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 4c7f7a41ab2c..7fe1153ec2e2 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -26,7 +26,7 @@ namespace arrow::dlpack { /// The DLPack version used during compilation. -ARROW_EXPORT extern const DLPackVersion VERSION; +ARROW_EXPORT extern const DLPackVersion kVersion; /// \brief Export Arrow array as DLPack tensor. /// diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index 5b857b33c08d..ee4fc4832f47 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1479,7 +1479,7 @@ cdef extern from "arrow/c/dlpack_abi.h" nogil: cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: - const DLPackVersion DLPACK_VERSION" arrow::dlpack::VERSION" + const DLPackVersion DLPACK_VERSION" arrow::dlpack::kVersion" CResult[DLManagedTensor*] ExportArrayToDLPack" arrow::dlpack::ExportArray"( const shared_ptr[CArray]& arr) From 3ab9addd6c3aee7ef750c2d484139db7cc12d368 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 7 Sep 2026 15:33:50 +0200 Subject: [PATCH 08/21] Add Buffer::TakeOwnership --- cpp/src/arrow/buffer.cc | 19 ++++-------- cpp/src/arrow/buffer.h | 62 ++++++++++++++++++++++++++++++++++----- cpp/src/arrow/c/dlpack.cc | 22 +++++++------- 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/cpp/src/arrow/buffer.cc b/cpp/src/arrow/buffer.cc index 17e745204640..9abe41b4c3a3 100644 --- a/cpp/src/arrow/buffer.cc +++ b/cpp/src/arrow/buffer.cc @@ -148,22 +148,13 @@ Result> Buffer::ViewOrCopy( return MemoryManager::CopyBuffer(source, to); } -class StlStringBuffer : public Buffer { - public: - explicit StlStringBuffer(std::string data) : input_(std::move(data)) { - if (!input_.empty()) { - data_ = reinterpret_cast(input_.c_str()); - size_ = static_cast(input_.size()); - capacity_ = size_; - } +std::shared_ptr Buffer::FromString(std::string data) { + if (data.empty()) { + return std::shared_ptr{new Buffer()}; } - private: - std::string input_; -}; - -std::shared_ptr Buffer::FromString(std::string data) { - return std::make_shared(std::move(data)); + auto size_in_bytes = static_cast(data.size()); + return TakeOwnership(std::move(data), size_in_bytes); } std::shared_ptr SliceMutableBuffer(std::shared_ptr buffer, diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 07f9931eba73..f1724fc182c1 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -146,6 +147,58 @@ class ARROW_EXPORT Buffer { } } + /// \brief Default data accessor used by TakeOwnership. + struct DefaultGetData { + template + auto* operator()(T& container) const { + return container.data(); + } + }; + + /// \brief Construct an immutable buffer that takes ownership of a container. + /// + /// This operation does not make a copy. If the underlying container is mutable (as + /// detected by the return type of `get_data`) then returned buffer will be mutable. + /// + /// \param[in] container The container to own. The container mus own data as a + /// contiguous slice. This buffer does not need to remain stable across a + /// container move. + /// \param[in] nbytes The size of the data, which must not exceed the number of bytes + /// readable from the pointer returned by \p get_data + /// \param[in] get_data Callable returning the address of the container's data. This + /// function enable the function to get the data *after* the container has + /// been moved to a stable to work with types such as `std::string`. + /// \return a new Buffer instance + template + static auto TakeOwnership(T container, int64_t nbytes, Func&& get_data = {}) { + using DataPtr = decltype(std::forward(get_data)(container)); + constexpr bool is_mutable = !std::is_const_v>; + using BufferType = std::conditional_t; + using Byte = std::conditional_t; + + // Hold the container and the Buffer in a single allocation. Declaration order + // matters: the container is constructed first and destroyed last, so the Buffer + // never outlives the memory it points into. + struct ControlBlock { + T container; + BufferType buffer; + + ControlBlock(T container, int64_t nbytes, Func&& get_data) + : container(std::move(container)), + // Read the data pointer only once the container has reached its final + // address, since moving it may invalidate the pointer (e.g. in a small + // string optimization). + buffer(reinterpret_cast(std::forward(get_data)(this->container)), + nbytes) {} + }; + + auto owner = std::make_shared(std::move(container), nbytes, + std::forward(get_data)); + // Aliasing constructor + auto* buffer = &owner->buffer; + return std::shared_ptr{std::move(owner), buffer}; + } + /// \brief Construct an immutable buffer that takes ownership of the contents /// of an std::string (without copying it). /// @@ -168,15 +221,8 @@ class ARROW_EXPORT Buffer { return std::shared_ptr{new Buffer()}; } - auto* data = reinterpret_cast(vec.data()); auto size_in_bytes = static_cast(vec.size() * sizeof(T)); - return std::shared_ptr{ - new Buffer{data, size_in_bytes}, - // Keep the vector's buffer alive inside the shared_ptr's destructor until after - // we have deleted the Buffer. Note we can't use this trick in FromString since - // std::string's data is inline for short strings so moving invalidates pointers - // into the string's buffer. - [vec = std::move(vec)](Buffer* buffer) { delete buffer; }}; + return TakeOwnership(std::move(vec), size_in_bytes); } /// \brief Create buffer referencing typed memory with some length without diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 98e7a48977a6..8ed23bd67b56 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -434,18 +434,18 @@ Result> ImportBuffer(CppDLTensor&& dl, bool copy) { } else if (copy) { ARROW_ASSIGN_OR_RAISE(buffer, MutableBuffer::CopyNonOwned( {data, nbytes}, default_cpu_memory_manager())); + } else if (const auto byte_offset = dl.tensor().byte_offset; dl.is_readonly()) { + auto get_data = [byte_offset](auto& d) { + return d.template data_as() + byte_offset; + }; + buffer = Buffer::TakeOwnership(std::move(dl), nbytes, get_data); + ARROW_DCHECK(!buffer->is_mutable()); } else { - const bool readonly = dl.is_readonly(); - // Trick to keep DLPack data alive taken from `Buffer::FromVector`. - auto deleter = [dl = std::move(dl)](auto* buffer) { delete buffer; }; - if (readonly) { - buffer = {new Buffer{data, nbytes}, std::move(deleter)}; - } else { - buffer = std::shared_ptr{ - new MutableBuffer{data, nbytes}, - std::move(deleter), - }; - } + auto get_data = [byte_offset](auto& d) { + return d.template data_as() + byte_offset; + }; + buffer = Buffer::TakeOwnership(std::move(dl), nbytes, get_data); + ARROW_DCHECK(buffer->is_mutable()); } return buffer; From 4d20306184e7b93332b49d9af74b60100c4b6745 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 7 Sep 2026 17:08:14 +0200 Subject: [PATCH 09/21] Improve tests --- cpp/src/arrow/c/dlpack_test.cc | 57 ++++++++++++++++++++--------- python/pyarrow/tests/test_dlpack.py | 20 +++++++--- 2 files changed, 54 insertions(+), 23 deletions(-) diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 4f22c5c527a1..70295046078a 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -396,6 +396,14 @@ struct TensorConsumer { static const uint8_t* RawData(const Imported& t) { return t->raw_data(); } static bool IsMutable(const Imported& t) { return t->is_mutable(); } static int64_t Size(const Imported& t) { return t->size(); } + static Status Validate(const Imported& t) { return t->Validate(); } + + /// Import and additionally check the result is a well-formed Arrow object. + static Result ImportAndValidate(DLManagedTensorVersioned* raw) { + ARROW_ASSIGN_OR_RAISE(auto imported, Import(raw)); + RETURN_NOT_OK(Validate(imported)); + return imported; + } }; template @@ -415,6 +423,14 @@ struct ArrayConsumer { return arr->data()->buffers[1]->is_mutable(); } static int64_t Size(const Imported& arr) { return arr->length(); } + static Status Validate(const Imported& t) { return t->ValidateFull(); } + + /// Import and additionally check the result is a well-formed Arrow object. + static Result ImportAndValidate(DLManagedTensorVersioned* raw) { + ARROW_ASSIGN_OR_RAISE(auto imported, Import(raw)); + RETURN_NOT_OK(Validate(imported)); + return imported; + } }; struct ConsumerNames { @@ -447,7 +463,8 @@ TYPED_TEST(TestImport, Basic) { const auto expected = std::vector{1, 2, 3, 4, 5, 6}; const auto* values = foreign.data.data() + foreign.byte_offset; - ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_OK_AND_ASSIGN(auto imported, + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); AssertTypeEqual(*float32(), *TypeParam::ValueType(imported)); ASSERT_EQ(6, TypeParam::Size(imported)); @@ -475,7 +492,8 @@ TYPED_TEST(TestImport, Mutable) { .data = ToBytes(std::vector{1, 2, 3, 4}), .flags = 0, }; - ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_OK_AND_ASSIGN(auto imported, + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); ASSERT_TRUE(TypeParam::IsMutable(imported)); } @@ -486,7 +504,7 @@ TYPED_TEST(TestImport, NullDeleter) { auto* foreign = static_cast(managed->manager_ctx); managed->deleter = nullptr; - ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::ImportAndValidate(managed)); imported.reset(); delete foreign; } @@ -503,7 +521,7 @@ TYPED_TEST(TestImport, DataTypes) { for (const auto& [dtype, expected] : cases) { ARROW_SCOPED_TRACE("dtype ", expected->ToString()); ASSERT_OK_AND_ASSIGN( - auto imported, TypeParam::Import(Produce( + auto imported, TypeParam::ImportAndValidate(Produce( {.dtype = dtype, .shape = {3}, .strides = {1}, @@ -517,21 +535,22 @@ TYPED_TEST(TestImport, Empty) { auto* managed = Produce({.shape = {0}, .strides = {1}}); managed->dl_tensor.data = nullptr; - ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::Import(managed)); + ASSERT_OK_AND_ASSIGN(auto imported, TypeParam::ImportAndValidate(managed)); ASSERT_EQ(0, TypeParam::Size(imported)); } TYPED_TEST(TestImport, Errors) { auto check = [](ForeignTensor foreign, const std::string& message) { const auto deleted = foreign.deleted; - const auto status = TypeParam::Import(Produce(std::move(foreign))).status(); + const auto status = + TypeParam::ImportAndValidate(Produce(std::move(foreign))).status(); EXPECT_EQ(message, status.ToStringWithoutContextLines()); // Ownership is taken even when the import fails EXPECT_EQ(1, *deleted); }; ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: Received null pointer.", - TypeParam::Import(nullptr)); + TypeParam::ImportAndValidate(nullptr)); check({.shape = {2}, .strides = {1}, .data = std::vector(8), @@ -556,7 +575,7 @@ TYPED_TEST(TestImport, UnsupportedVersion) { "Invalid: Unsupported DLPack major version " + std::to_string(major) + ", expected " + std::to_string(DLPACK_MAJOR_VERSION), - TypeParam::Import(managed)); + TypeParam::ImportAndValidate(managed)); // The spec mandates the deleter to be called on major version mismatch ASSERT_EQ(1, *deleted); } @@ -572,7 +591,8 @@ TYPED_TEST(TestImportTensor, ShapeAndStrides) { .strides = {3, 1}, .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), }; - ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_OK_AND_ASSIGN(auto tensor, + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(2, 3)); // Arrow strides are in bytes, DLPack strides in elements @@ -584,7 +604,7 @@ TYPED_TEST(TestImportTensor, Empty) { auto* managed = Produce({.shape = {0, 3}, .strides = {3, 1}}); managed->dl_tensor.data = nullptr; - ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::ImportAndValidate(managed)); ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(0, 3)); } @@ -594,7 +614,8 @@ TYPED_TEST(TestImportTensor, Strided) { .strides = {1, 2}, .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), }; - ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(Produce(std::move(column_major)))); + ASSERT_OK_AND_ASSIGN(auto tensor, + TypeParam::ImportAndValidate(Produce(std::move(column_major)))); ASSERT_TRUE(tensor->is_column_major()); // A 2x2 window over every other row of a 4x2 buffer @@ -603,7 +624,8 @@ TYPED_TEST(TestImportTensor, Strided) { .strides = {4, 1}, .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6, 7, 8}), }; - ASSERT_OK_AND_ASSIGN(tensor, TypeParam::Import(Produce(std::move(non_contiguous)))); + ASSERT_OK_AND_ASSIGN(tensor, + TypeParam::ImportAndValidate(Produce(std::move(non_contiguous)))); ASSERT_FALSE(tensor->is_contiguous()); ASSERT_EQ(6, tensor->template Value({1, 1})); } @@ -615,14 +637,14 @@ TYPED_TEST(TestImportTensor, NegativeStrides) { .data = std::vector(16), }; ASSERT_RAISES_WITH_MESSAGE(Invalid, "Invalid: negative strides not supported", - TypeParam::Import(Produce(std::move(foreign)))); + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); } TYPED_TEST(TestImportTensor, RoundTrip) { const auto original = TensorFromJSON(float64(), "[1, 2, 3, 4, 5, 6]", {3, 2}); ASSERT_OK_AND_ASSIGN(auto* managed, ExportTensorVersioned(original, /*copy=*/false)); - ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::Import(managed)); + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::ImportAndValidate(managed)); ASSERT_TRUE(tensor->Equals(*original)); if constexpr (!TypeParam::copy) { @@ -642,7 +664,8 @@ TYPED_TEST(TestImportArray, OneDimension) { .strides = {1}, .data = ToBytes(std::vector{1, 2, 3, 4}), }; - ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(Produce(std::move(foreign)))); + ASSERT_OK_AND_ASSIGN(auto array, + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); AssertArraysEqual(*ArrayFromJSON(int32(), "[1, 2, 3, 4]"), *array); } @@ -652,7 +675,7 @@ TYPED_TEST(TestImportArray, Unsupported) { Invalid, "Invalid: Only contiguous one dimensional tensor can be imported as" " arrays. Try importing to Tensor first.", - TypeParam::Import(Produce(std::move(foreign)))); + TypeParam::ImportAndValidate(Produce(std::move(foreign)))); }; // Only a Tensor can hold more than one dimension @@ -670,7 +693,7 @@ TYPED_TEST(TestImportArray, RoundTrip) { const auto original = ArrayFromJSON(float64(), "[1, 2, 3, 4, 5, 6]"); ASSERT_OK_AND_ASSIGN(auto* managed, ExportArrayVersioned(original, /*copy=*/false)); - ASSERT_OK_AND_ASSIGN(auto array, TypeParam::Import(managed)); + ASSERT_OK_AND_ASSIGN(auto array, TypeParam::ImportAndValidate(managed)); AssertArraysEqual(*original, *array); if constexpr (!TypeParam::copy) { diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 6236b540af19..9c457767c5e0 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -385,14 +385,18 @@ def test_tensor_from_dlpack(np_type): if Version(np.__version__) < Version("2.1.0"): pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") + def make_array(): + base = np.arange(24, dtype=np_type).reshape((4, 6)) + array = base[::2, 1::2] + assert not array.flags['C_CONTIGUOUS'] + return array + # Non-contiguous, strided slice: DLPack carries explicit strides, so this # should not need a copy on export. - base = np.arange(24, dtype=np_type).reshape((4, 6)) - expected = base[::2, 1::2] - assert not expected.flags['C_CONTIGUOUS'] - tensor = pa.Tensor.from_dlpack(expected) + tensor = pa.Tensor.from_dlpack(make_array()) assert isinstance(tensor, pa.Tensor) - np.testing.assert_array_equal(tensor.to_numpy(), expected, strict=True) + gc.collect() # Attempts to free input array memory + np.testing.assert_array_equal(tensor.to_numpy(), make_array(), strict=True) @check_bytes_allocated @@ -406,6 +410,7 @@ def test_array_from_dlpack(np_type): expected = np.array([1, 2, 3, 4, 5], dtype=np_type) arr = pa.Array.from_dlpack(expected) + arr.validate(full=True) assert isinstance(arr, pa.Array) np.testing.assert_array_equal(arr.to_numpy(), expected, strict=True) @@ -418,9 +423,12 @@ def test_from_dlpack_zero_copy(): expected = np.array([1, 2, 3], dtype=np.int64) tensor = pa.Tensor.from_dlpack(expected) result = tensor.to_numpy() - expected[0] = 100 # Zero-copy import: mutating the source is visible through the tensor. + expected[0] = 100 assert result[0] == 100 + # Same for mutating the result + result[1] = 42 + assert expected[1] == 42 @check_bytes_allocated From e343b9e171986fa398c01c235c5b2c89b5c3d08c Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Mon, 7 Sep 2026 17:40:29 +0200 Subject: [PATCH 10/21] Remove consumer copy param --- cpp/src/arrow/c/dlpack.cc | 16 ++++------ cpp/src/arrow/c/dlpack.h | 10 ++----- cpp/src/arrow/c/dlpack_test.cc | 45 ++++++++++------------------ python/pyarrow/array.pxi | 2 +- python/pyarrow/includes/libarrow.pxd | 4 +-- python/pyarrow/tensor.pxi | 2 +- 6 files changed, 27 insertions(+), 52 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 8ed23bd67b56..0be180e7070a 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -413,7 +413,7 @@ Result> StridesInBytes(std::span strides, return out; } -Result> ImportBuffer(CppDLTensor&& dl, bool copy) { +Result> ImportBuffer(CppDLTensor&& dl) { // DLPack strides are in number of elements, so is the size we compute from them. ARROW_ASSIGN_OR_RAISE(const auto nelements, internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); @@ -431,9 +431,6 @@ Result> ImportBuffer(CppDLTensor&& dl, bool copy) { if (nbytes == 0) { // DLPack data pointer may be null on empty tensors buffer = std::make_shared(data, nbytes); - } else if (copy) { - ARROW_ASSIGN_OR_RAISE(buffer, MutableBuffer::CopyNonOwned( - {data, nbytes}, default_cpu_memory_manager())); } else if (const auto byte_offset = dl.tensor().byte_offset; dl.is_readonly()) { auto get_data = [byte_offset](auto& d) { return d.template data_as() + byte_offset; @@ -453,8 +450,7 @@ Result> ImportBuffer(CppDLTensor&& dl, bool copy) { } // namespace -Result> ImportArrayVersioned(DLManagedTensorVersioned* unmanaged, - bool copy) { +Result> ImportArrayVersioned(DLManagedTensorVersioned* unmanaged) { ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); if (dl.tensor().device.device_type != kDLCPU) { @@ -470,13 +466,13 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* un ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); const auto nelements = dl.shape().front(); - ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl))); auto data = ArrayData::Make(type, nelements, {nullptr, std::move(buffer)}); return MakeArray(std::move(data)); } -Result> ImportTensorVersioned(DLManagedTensorVersioned* unmanaged, - bool copy) { +Result> ImportTensorVersioned( + DLManagedTensorVersioned* unmanaged) { ARROW_ASSIGN_OR_RAISE(auto dl, CppDLTensor::TakeOwnership(unmanaged)); if (dl.tensor().device.device_type != kDLCPU) { @@ -487,7 +483,7 @@ Result> ImportTensorVersioned(DLManagedTensorVersioned* ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); auto shape = std::vector(dl.shape().begin(), dl.shape().end()); auto strides = std::vector(dl.strides().begin(), dl.strides().end()); - ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl), copy)); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl))); const auto byte_width = type->byte_width(); ARROW_ASSIGN_OR_RAISE(auto strides_bytes, StridesInBytes(strides, byte_width)); diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 7fe1153ec2e2..765b0fa3e09c 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -115,12 +115,9 @@ Result ExportDevice(const std::shared_ptr& t); /// an error-safe fashion. /// /// \param[in] raw DLPack tensor -/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack -/// producer. /// \return An Arrow Array ARROW_EXPORT -Result> ImportArrayVersioned(DLManagedTensorVersioned* raw, - bool copy); +Result> ImportArrayVersioned(DLManagedTensorVersioned* raw); /// \brief Import a DLPack tensor as an Arrow Tensor. /// @@ -129,11 +126,8 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* ra /// If the DLPack input is marked as readonly, this will produce an immutable tensor. /// /// \param[in] raw Arrow array -/// \param[in] copy Whether to copy the data instead of sharing it with the DLPack -/// producer. /// \return An Arrow Tensor ARROW_EXPORT -Result> ImportTensorVersioned(DLManagedTensorVersioned* raw, - bool copy); +Result> ImportTensorVersioned(DLManagedTensorVersioned* raw); } // namespace arrow::dlpack diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 70295046078a..5965ecbca008 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -383,14 +383,12 @@ DLManagedTensorVersioned* Produce(ForeignTensor foreign) { return &owned.release()->managed; } -template struct TensorConsumer { using Imported = std::shared_ptr; - static constexpr bool copy = kCopy; - static constexpr const char* name = copy ? "TensorCopied" : "TensorShared"; + static constexpr const char* name = "Tensor"; static Result Import(DLManagedTensorVersioned* raw) { - return ImportTensorVersioned(raw, copy); + return ImportTensorVersioned(raw); } static std::shared_ptr ValueType(const Imported& t) { return t->type(); } static const uint8_t* RawData(const Imported& t) { return t->raw_data(); } @@ -406,14 +404,12 @@ struct TensorConsumer { } }; -template struct ArrayConsumer { using Imported = std::shared_ptr; - static constexpr bool copy = kCopy; - static constexpr const char* name = copy ? "ArrayCopied" : "ArrayShared"; + static constexpr const char* name = "Array"; static Result Import(DLManagedTensorVersioned* raw) { - return ImportArrayVersioned(raw, copy); + return ImportArrayVersioned(raw); } static std::shared_ptr ValueType(const Imported& arr) { return arr->type(); } static const uint8_t* RawData(const Imported& arr) { @@ -440,10 +436,9 @@ struct ConsumerNames { } }; -using ConsumerTypes = ::testing::Types, TensorConsumer, - ArrayConsumer, ArrayConsumer>; -using TensorConsumerTypes = ::testing::Types, TensorConsumer>; -using ArrayConsumerTypes = ::testing::Types, ArrayConsumer>; +using ConsumerTypes = ::testing::Types; +using TensorConsumerTypes = ::testing::Types; +using ArrayConsumerTypes = ::testing::Types; /// Tests sharing the same expectations for Arrow Tensor and Array imports. template @@ -468,21 +463,15 @@ TYPED_TEST(TestImport, Basic) { AssertTypeEqual(*float32(), *TypeParam::ValueType(imported)); ASSERT_EQ(6, TypeParam::Size(imported)); - // A copy is ours to mutate, whatever the producer flagged - ASSERT_EQ(TypeParam::copy, TypeParam::IsMutable(imported)); + ASSERT_FALSE(TypeParam::IsMutable(imported)); ASSERT_EQ(0, std::memcmp(TypeParam::RawData(imported), expected.data(), expected.size() * sizeof(float))); - if constexpr (TypeParam::copy) { - // The producer tensor is released as soon as its data has been copied - ASSERT_EQ(1, *deleted); - } else { - ASSERT_EQ(TypeParam::RawData(imported), values); - // The producer tensor is kept alive by the imported data - ASSERT_EQ(0, *deleted); - imported.reset(); - ASSERT_EQ(1, *deleted); - } + ASSERT_EQ(TypeParam::RawData(imported), values); + // The producer tensor is kept alive by the imported data + ASSERT_EQ(0, *deleted); + imported.reset(); + ASSERT_EQ(1, *deleted); } TYPED_TEST(TestImport, Mutable) { @@ -647,9 +636,7 @@ TYPED_TEST(TestImportTensor, RoundTrip) { ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::ImportAndValidate(managed)); ASSERT_TRUE(tensor->Equals(*original)); - if constexpr (!TypeParam::copy) { - ASSERT_EQ(original->raw_data(), tensor->raw_data()); - } + ASSERT_EQ(original->raw_data(), tensor->raw_data()); } template @@ -696,9 +683,7 @@ TYPED_TEST(TestImportArray, RoundTrip) { ASSERT_OK_AND_ASSIGN(auto array, TypeParam::ImportAndValidate(managed)); AssertArraysEqual(*original, *array); - if constexpr (!TypeParam::copy) { - ASSERT_EQ(TypeParam::RawData(original), TypeParam::RawData(array)); - } + ASSERT_EQ(TypeParam::RawData(original), TypeParam::RawData(array)); } } // namespace arrow::dlpack diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index bed4c5a644db..989870a48916 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2315,7 +2315,7 @@ cdef class Array(_PandasConvertible): PyCapsule_SetName(pycapsule, "used_dltensor_versioned") with nogil: # Copy handled on producer side - result = ImportArrayVersionedFromDLPack(ptr, False) + result = ImportArrayVersionedFromDLPack(ptr) carray = GetResultValue(result) return pyarrow_wrap_array(carray) diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index ee4fc4832f47..5264e7492793 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -1498,10 +1498,10 @@ cdef extern from "arrow/c/dlpack.h" namespace "arrow::dlpack" nogil: CResult[shared_ptr[CArray]] \ ImportArrayVersionedFromDLPack" arrow::dlpack::ImportArrayVersioned"( - DLManagedTensorVersioned* raw, c_bool copy) + DLManagedTensorVersioned* raw) CResult[shared_ptr[CTensor]] \ ImportTensorVersionedFromDLPack" arrow::dlpack::ImportTensorVersioned"( - DLManagedTensorVersioned* raw, c_bool copy) + DLManagedTensorVersioned* raw) cdef extern from "arrow/builder.h" namespace "arrow" nogil: diff --git a/python/pyarrow/tensor.pxi b/python/pyarrow/tensor.pxi index a40449866995..7f006f294175 100644 --- a/python/pyarrow/tensor.pxi +++ b/python/pyarrow/tensor.pxi @@ -347,7 +347,7 @@ strides: {self.strides}""" PyCapsule_SetName(pycapsule, "used_dltensor_versioned") with nogil: # Copy handled on producer side - result = ImportTensorVersionedFromDLPack(ptr, False) + result = ImportTensorVersionedFromDLPack(ptr) ctensor = GetResultValue(result) return pyarrow_wrap_tensor(ctensor) From 6d0aa95a10d9ce25b2fad2ea6af2489ca5b5dd32 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 09:14:27 +0200 Subject: [PATCH 11/21] Document mutable buffers in Buffer::FromString / Vec --- cpp/src/arrow/buffer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index f1724fc182c1..8c98c375c0f4 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -199,14 +199,14 @@ class ARROW_EXPORT Buffer { return std::shared_ptr{std::move(owner), buffer}; } - /// \brief Construct an immutable buffer that takes ownership of the contents + /// \brief Construct an mutable buffer that takes ownership of the contents /// of an std::string (without copying it). /// /// \param[in] data a string to own /// \return a new Buffer instance static std::shared_ptr FromString(std::string data); - /// \brief Construct an immutable buffer that takes ownership of the contents + /// \brief Construct an mutable buffer that takes ownership of the contents /// of an std::vector (without copying it). Only vectors of TrivialType objects /// (integers, floating point numbers, ...) can be wrapped by this function. /// From 9a56ffea6f2d2e332e72c149f789380f45a72a06 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 09:16:39 +0200 Subject: [PATCH 12/21] Fix doc --- cpp/src/arrow/c/dlpack.h | 2 +- python/pyarrow/array.pxi | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/c/dlpack.h b/cpp/src/arrow/c/dlpack.h index 765b0fa3e09c..29738dff6d16 100644 --- a/cpp/src/arrow/c/dlpack.h +++ b/cpp/src/arrow/c/dlpack.h @@ -125,7 +125,7 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* ra /// Takes ownership of the `DLManagedTensorVersioned` in an error-safe fashion. /// If the DLPack input is marked as readonly, this will produce an immutable tensor. /// -/// \param[in] raw Arrow array +/// \param[in] raw DLPack tensor /// \return An Arrow Tensor ARROW_EXPORT Result> ImportTensorVersioned(DLManagedTensorVersioned* raw); diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 989870a48916..2c38f7e4e1f7 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2278,6 +2278,9 @@ cdef class Array(_PandasConvertible): def from_dlpack(x, /, *, device=None, copy=None): """ Construct an Array from an object implementing the DLPack protocol. + Only 1 dimensional contiguous tensor are accepted as input. + For multi-dimensional tensor, first use `Tensor.from_dpack`, then + convert to an array using `FixedShapeTensorArray.from_tensor()`. Parameters ---------- From 55ade0955f83e73986159e9f8605253eb4bf3016 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 09:25:42 +0200 Subject: [PATCH 13/21] Add FixedShapeTensorArray.from_tensor test --- python/pyarrow/tests/test_extension_type.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index bdd898767b06..2cc258e2d2b5 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -1708,6 +1708,26 @@ def test_tensor_class_methods(np_type_str): assert result.to_tensor().strides == (12 * bw, 1 * bw, 3 * bw, 6 * bw) +@pytest.mark.numpy +@pytest.mark.parametrize( + ("transpose", "permutation"), + [(False, [0, 1]), (True, [1, 0])] +) +def test_tensor_array_from_tensor(transpose, permutation): + arr = np.arange(24, dtype=np.int32).reshape(2, 3, 4) + arr = arr.transpose(0, 2, 1) if transpose else arr + + result = pa.FixedShapeTensorArray.from_tensor(pa.Tensor.from_numpy(arr)) + + assert isinstance(result.type, pa.FixedShapeTensorType) + assert result.type.value_type == pa.int32() + # Shape is in physical order (unpermuted) + assert result.type.shape == [3, 4] + assert result.type.permutation == permutation + assert len(result) == 2 + np.testing.assert_array_equal(result.to_numpy_ndarray(), arr) + + @pytest.mark.numpy @pytest.mark.parametrize("np_type_str", ("int8", "int64", "float32")) def test_tensor_array_from_numpy(np_type_str): From 164676be5bd0e47d242de22b908134b5a1e5d942 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 09:32:55 +0200 Subject: [PATCH 14/21] Fix mutability test --- cpp/src/arrow/buffer.h | 14 +++++++------- cpp/src/arrow/buffer_test.cc | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 8c98c375c0f4..207313bccb00 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -160,14 +160,14 @@ class ARROW_EXPORT Buffer { /// This operation does not make a copy. If the underlying container is mutable (as /// detected by the return type of `get_data`) then returned buffer will be mutable. /// - /// \param[in] container The container to own. The container mus own data as a - /// contiguous slice. This buffer does not need to remain stable across a - /// container move. + /// \param[in] container The container to own. The container must own its data as a + /// contiguous slice. That data does not need to remain at a stable address + /// across a container move. /// \param[in] nbytes The size of the data, which must not exceed the number of bytes /// readable from the pointer returned by \p get_data /// \param[in] get_data Callable returning the address of the container's data. This - /// function enable the function to get the data *after* the container has - /// been moved to a stable to work with types such as `std::string`. + /// callable is invoked *after* the container has been moved to its final + /// address, to work with types such as `std::string`. /// \return a new Buffer instance template static auto TakeOwnership(T container, int64_t nbytes, Func&& get_data = {}) { @@ -199,14 +199,14 @@ class ARROW_EXPORT Buffer { return std::shared_ptr{std::move(owner), buffer}; } - /// \brief Construct an mutable buffer that takes ownership of the contents + /// \brief Construct a mutable buffer that takes ownership of the contents /// of an std::string (without copying it). /// /// \param[in] data a string to own /// \return a new Buffer instance static std::shared_ptr FromString(std::string data); - /// \brief Construct an mutable buffer that takes ownership of the contents + /// \brief Construct a mutable buffer that takes ownership of the contents /// of an std::vector (without copying it). Only vectors of TrivialType objects /// (integers, floating point numbers, ...) can be wrapped by this function. /// diff --git a/cpp/src/arrow/buffer_test.cc b/cpp/src/arrow/buffer_test.cc index 4dd210076ed1..bdda0b4a7476 100644 --- a/cpp/src/arrow/buffer_test.cc +++ b/cpp/src/arrow/buffer_test.cc @@ -586,7 +586,7 @@ TEST(TestBuffer, FromStringRvalue) { AssertIsCPUBuffer(*buffer); } - ASSERT_FALSE(buffer->is_mutable()); + ASSERT_TRUE(buffer->is_mutable()); ASSERT_EQ(0, memcmp(buffer->data(), expected.c_str(), expected.size())); ASSERT_EQ(static_cast(expected.size()), buffer->size()); From b2095a160403884fd66291cf8a9eabc8dfaaf80f Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 09:43:56 +0200 Subject: [PATCH 15/21] Typos --- cpp/src/arrow/buffer.h | 8 ++++---- python/pyarrow/array.pxi | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index 207313bccb00..c623076e0054 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -155,7 +155,7 @@ class ARROW_EXPORT Buffer { } }; - /// \brief Construct an immutable buffer that takes ownership of a container. + /// \brief Construct a buffer that takes ownership of a container. /// /// This operation does not make a copy. If the underlying container is mutable (as /// detected by the return type of `get_data`) then returned buffer will be mutable. @@ -172,9 +172,9 @@ class ARROW_EXPORT Buffer { template static auto TakeOwnership(T container, int64_t nbytes, Func&& get_data = {}) { using DataPtr = decltype(std::forward(get_data)(container)); - constexpr bool is_mutable = !std::is_const_v>; - using BufferType = std::conditional_t; - using Byte = std::conditional_t; + constexpr bool kIsMutable = !std::is_const_v>; + using BufferType = std::conditional_t; + using Byte = std::conditional_t; // Hold the container and the Buffer in a single allocation. Declaration order // matters: the container is constructed first and destroyed last, so the Buffer diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 2c38f7e4e1f7..e4e20c170f7d 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2278,8 +2278,8 @@ cdef class Array(_PandasConvertible): def from_dlpack(x, /, *, device=None, copy=None): """ Construct an Array from an object implementing the DLPack protocol. - Only 1 dimensional contiguous tensor are accepted as input. - For multi-dimensional tensor, first use `Tensor.from_dpack`, then + Only 1-dimensional contiguous tensors are accepted as input. + For multi-dimensional tensors, first use `Tensor.from_dlpack`, then convert to an array using `FixedShapeTensorArray.from_tensor()`. Parameters From c52f747e4ac88c6167c5c9a54535296c507ac09b Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 10:47:33 +0200 Subject: [PATCH 16/21] Update DLPack documentation --- docs/source/python/dlpack.rst | 124 ++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 13 deletions(-) diff --git a/docs/source/python/dlpack.rst b/docs/source/python/dlpack.rst index 6e74cd5c82c1..4e102ce8b0a5 100644 --- a/docs/source/python/dlpack.rst +++ b/docs/source/python/dlpack.rst @@ -41,31 +41,51 @@ and more about DLPack in the Implementation of DLPack in PyArrow ----------------------------------- -The producing side of the DLPack Protocol is implemented for ``pa.Array`` -and can be used to interchange data between PyArrow and other tensor -libraries. Supported data types are integer, unsigned integer and float. The -protocol has no missing data support meaning PyArrow arrays with -missing values cannot be transferred through the DLPack -protocol. Currently, the Arrow implementation of the protocol only supports +The protocol is implemented for ``pa.Array`` and ``pa.Tensor`` with different behaviors. +``pa.Tensor`` can produce and consume all shapes and strides of a generic DLPack tensor. +``pa.Array`` on the other hand is purposely limited to produce and consume 1-dimensional +contiguous tensors (where the only dimension is the array's length). +The only exception is ``pa.FixedShapeTensorArray``, which is designed to represent +tensors and supports more generic shapes and strides. +It can produce and consume DLPack tensors whose outermost dimension has the largest +stride, that dimension being mapped to the array's length. + +For both ``pa.Tensor`` and ``pa.Array``, only numeric data types are supported: integer, +unsigned integer and float. + +Some array types can be understood as some form of tensor. +For instance, a nested fixed size list of a numeric data type has the same memory +representation as a row major tensor. +It is possible to get a (zero-copy) tensor from such an array using +``array.to_tensor()``, and then use DLPack on the resulting tensor. + +The DLPack protocol fails on arrays with nulls, though these can be ignored with +an explicit conversion to a tensor using ``array.to_tensor(allow_nulls=True)``. +In that case, the null entries hold an unspecified value. +This is free, compared to ``pa.compute.fill_null`` which explicitly modifies the +array data to replace the null values. + +Currently, the Arrow implementation of the protocol only supports data on a CPU device. Data interchange syntax of the protocol includes -1. ``from_dlpack(x)``: consuming an array object that implements a - ``__dlpack__`` method and creating a new array while sharing the +1. ``from_dlpack(x, /, *, device=None, copy=None)``: consuming an array object that + implements a ``__dlpack__`` method and creating a new array while sharing the memory. -2. ``__dlpack__(self, stream=None)`` and ``__dlpack_device__``: +2. ``__dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None)`` + and ``__dlpack_device__``: producing a PyCapsule with the DLPack struct which is called from within ``from_dlpack(x)``. - -PyArrow implements the second part of the protocol -(``__dlpack__(self, stream=None)`` and ``__dlpack_device__``) and can -thus be consumed by libraries implementing ``from_dlpack``. + This method is intended for library authors. Examples -------- +Producing +~~~~~~~~~ + Convert a PyArrow CPU array into a NumPy array: .. code-block:: python @@ -101,3 +121,81 @@ Convert a PyArrow CPU array into a JAX array: Array([2, 0, 2, 4], dtype=int32) >>> jax.dlpack.from_dlpack(array) # doctest: +SKIP Array([2, 0, 2, 4], dtype=int32) + +Arrays with a tensor memory layout, such as fixed size lists of a numeric type, need an +explicit conversion to a ``pa.Tensor``, which exports its full multi-dimensional shape: + +.. code-block:: python + + >>> list_array = pa.array([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]], + ... pa.list_(pa.float64(), 2)) + >>> list_array.to_tensor() + + type: double + shape: (3, 2) + strides: (16, 8) + >>> np.from_dlpack(list_array.to_tensor()) + array([[1., 2.], + [3., 4.], + [5., 6.]]) + +A ``pa.FixedShapeTensorArray`` exports directly, the array length becoming the outermost +dimension, followed by the shape of the element tensors: + +.. code-block:: python + + >>> nested = pa.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], + ... pa.list_(pa.list_(pa.int32(), 2), 2)) + >>> tensor_array = pa.FixedShapeTensorArray.from_tensor(nested.to_tensor()) + >>> tensor_array.type + FixedShapeTensorType(extension) + >>> np.from_dlpack(tensor_array).shape + (2, 2, 2) + +Arrays with nulls are rejected, unless the conversion to a tensor explicitly allows +them, in which case the null entries hold an unspecified value: + +.. code-block:: python + + >>> array_with_nulls = pa.array([2, None, 4], pa.int32()) + >>> np.from_dlpack(array_with_nulls) + Traceback (most recent call last): + ... + pyarrow.lib.ArrowTypeError: Can only use DLPack on arrays with no nulls. + >>> np.from_dlpack(array_with_nulls.to_tensor(allow_nulls=True)) + array([2, ..., 4], dtype=int32) + +Consuming +~~~~~~~~~ + +Any object implementing the DLPack protocol can be imported, without copying the data: + +.. code-block:: python + + >>> pa.Array.from_dlpack(np.array([2, 0, 2, 4])) + + [ + 2, + 0, + 2, + 4 + ] + >>> pa.Tensor.from_dlpack(np.array([[2, 0], [2, 4]], np.int32)) + + type: int32 + shape: (2, 2) + strides: (8, 4) + +``pa.Array.from_dlpack`` only accepts 1-dimensional contiguous tensors. +Multi-dimensional data can be imported as a ``pa.FixedShapeTensorArray``, the outermost +dimension becoming the length of the array: + +.. code-block:: python + + >>> array = pa.FixedShapeTensorArray.from_dlpack( + ... np.arange(12, dtype=np.int32).reshape(3, 2, 2) + ... ) + >>> array.type + FixedShapeTensorType(extension) + >>> len(array) + 3 From 98d41987b5f7fc25a6d9f2c3056a168d22beae92 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 11:08:17 +0200 Subject: [PATCH 17/21] Add FixedShapeTensorArray.from_dlpack --- python/pyarrow/array.pxi | 39 +++++++++++++++++++++++++-- python/pyarrow/tests/test_dlpack.py | 42 ++++++++++++++++++----------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index e4e20c170f7d..3060c533255b 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2279,8 +2279,8 @@ cdef class Array(_PandasConvertible): """ Construct an Array from an object implementing the DLPack protocol. Only 1-dimensional contiguous tensors are accepted as input. - For multi-dimensional tensors, first use `Tensor.from_dlpack`, then - convert to an array using `FixedShapeTensorArray.from_tensor()`. + For multi-dimensional tensors, use `Tensor.from_dlpack` or + `FixedShapeTensorArray.from_dlpack`. Parameters ---------- @@ -5124,6 +5124,41 @@ cdef class FixedShapeTensorArray(ExtensionArray): FixedSizeListArray.from_arrays(values, shape[1:].prod()) ) + @staticmethod + def from_dlpack(x, /, *, device=None, copy=None): + """ + Construct a FixedShapeTensorArray from an object implementing the DLPack + protocol. + + The outermost dimension of the input becomes the length of the tensor + array, and the remaining dimensions the shape of the individual tensors. + The outermost dimension must have the largest stride. + + Parameters + ---------- + x : object + The input object containing array data, following the DLPack + protocol (has a ``__dlpack__`` method). + device : tuple[enum.Enum, int], optional + Designates where the resulting array should reside, in the + format returned by :meth:`Array.__dlpack_device__`. When None, + the output array occupies the same device as the source. + Default: None. + copy : bool, optional + Controls duplication behavior. True mandates copying; False + prohibits copying and raises ``BufferError`` if unavoidable; + None duplicates only when necessary. Default: None. + + Returns + ------- + FixedShapeTensorArray + An array housing the data from the input object, potentially + as a copy or view. + + """ + return FixedShapeTensorArray.from_tensor( + Tensor.from_dlpack(x, device=device, copy=copy)) + def __dlpack__(self, *, stream=None, max_version=None, dl_device=None, copy=None): """ Export a tensor array as a DLPack capsule. diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 9c457767c5e0..877ce7f702f4 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -376,15 +376,13 @@ def test_dlpack_cuda_not_supported(): carr.__dlpack_device__() +@requires_numpy_version("2.1.0") @check_bytes_allocated @pytest.mark.parametrize('np_type', [np.uint8, np.uint16, np.uint32, np.uint64, np.int8, np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]) def test_tensor_from_dlpack(np_type): - if Version(np.__version__) < Version("2.1.0"): - pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") - def make_array(): base = np.arange(24, dtype=np_type).reshape((4, 6)) array = base[::2, 1::2] @@ -399,15 +397,13 @@ def make_array(): np.testing.assert_array_equal(tensor.to_numpy(), make_array(), strict=True) +@requires_numpy_version("2.1.0") @check_bytes_allocated @pytest.mark.parametrize('np_type', [np.uint8, np.uint16, np.uint32, np.uint64, np.int8, np.int16, np.int32, np.int64, np.float16, np.float32, np.float64]) def test_array_from_dlpack(np_type): - if Version(np.__version__) < Version("2.1.0"): - pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") - expected = np.array([1, 2, 3, 4, 5], dtype=np_type) arr = pa.Array.from_dlpack(expected) arr.validate(full=True) @@ -415,11 +411,31 @@ def test_array_from_dlpack(np_type): np.testing.assert_array_equal(arr.to_numpy(), expected, strict=True) +@requires_numpy_version("2.1.0") @check_bytes_allocated -def test_from_dlpack_zero_copy(): - if Version(np.__version__) < Version("2.1.0"): - pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") +@pytest.mark.parametrize('np_type', + [np.uint8, np.uint16, np.uint32, np.uint64, + np.int8, np.int16, np.int32, np.int64, + np.float16, np.float32, np.float64]) +def test_fixed_shape_tensor_array_from_dlpack(np_type): + source = np.arange(12, dtype=np_type).reshape((3, 2, 2)) + arr = pa.FixedShapeTensorArray.from_dlpack(source) + arr.validate(full=True) + assert arr.type == pa.fixed_shape_tensor(pa.from_numpy_dtype(np_type), [2, 2]) + assert arr.to_pylist() == [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]] + # Zero-copy import: mutating the source is visible through the array. + source[0, 0, 0] = 100 + assert arr.to_pylist()[0] == [100, 1, 2, 3] + + copied = pa.FixedShapeTensorArray.from_dlpack(source, copy=True) + source[0, 0, 0] = 0 + assert copied.to_pylist()[0] == [100, 1, 2, 3] + + +@requires_numpy_version("2.1.0") +@check_bytes_allocated +def test_from_dlpack_zero_copy(): expected = np.array([1, 2, 3], dtype=np.int64) tensor = pa.Tensor.from_dlpack(expected) result = tensor.to_numpy() @@ -431,11 +447,9 @@ def test_from_dlpack_zero_copy(): assert expected[1] == 42 +@requires_numpy_version("2.1.0") @check_bytes_allocated def test_from_dlpack_explicit_copy(): - if Version(np.__version__) < Version("2.1.0"): - pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") - expected = np.array([1, 2, 3], dtype=np.int64) tensor = pa.Tensor.from_dlpack(expected, copy=True) result = tensor.to_numpy() @@ -449,11 +463,9 @@ def test_from_dlpack_no_dlpack_method(): pa.Tensor.from_dlpack(object()) +@requires_numpy_version("2.1.0") @check_bytes_allocated def test_array_from_dlpack_multi_dim_not_supported(): - if Version(np.__version__) < Version("2.1.0"): - pytest.skip("Versioned DLPack capsules require numpy 2.1.0 or later") - expected = np.arange(6, dtype=np.int32).reshape((2, 3)) with pytest.raises( pa.ArrowInvalid, From d8ea00c576e4403384d1f304b1991d686354a7bc Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Tue, 8 Sep 2026 17:29:08 +0200 Subject: [PATCH 18/21] Add missing tests --- cpp/src/arrow/buffer.h | 17 +++++------ python/pyarrow/tests/test_dlpack.py | 32 +++++++++++++++++++++ python/pyarrow/tests/test_extension_type.py | 11 +++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/cpp/src/arrow/buffer.h b/cpp/src/arrow/buffer.h index c623076e0054..388829eba05a 100644 --- a/cpp/src/arrow/buffer.h +++ b/cpp/src/arrow/buffer.h @@ -51,6 +51,15 @@ namespace arrow { /// /// The following invariant is always true: Size <= Capacity class ARROW_EXPORT Buffer { + private: + /// \brief Default data accessor used by TakeOwnership. + struct DefaultGetData { + template + auto* operator()(T& container) const { + return container.data(); + } + }; + public: ARROW_DISALLOW_COPY_AND_ASSIGN(Buffer); @@ -147,14 +156,6 @@ class ARROW_EXPORT Buffer { } } - /// \brief Default data accessor used by TakeOwnership. - struct DefaultGetData { - template - auto* operator()(T& container) const { - return container.data(); - } - }; - /// \brief Construct a buffer that takes ownership of a container. /// /// This operation does not make a copy. If the underlying container is mutable (as diff --git a/python/pyarrow/tests/test_dlpack.py b/python/pyarrow/tests/test_dlpack.py index 877ce7f702f4..0c3c082fd6ed 100644 --- a/python/pyarrow/tests/test_dlpack.py +++ b/python/pyarrow/tests/test_dlpack.py @@ -433,6 +433,38 @@ def test_fixed_shape_tensor_array_from_dlpack(np_type): assert copied.to_pylist()[0] == [100, 1, 2, 3] +@requires_numpy_version("2.1.0") +@check_bytes_allocated +@pytest.mark.parametrize('np_type', [np.uint8, np.int32, np.float64]) +def test_fixed_shape_tensor_array_from_dlpack_transposed(np_type): + source = np.arange(12, dtype=np_type).reshape((3, 2, 2)).transpose(0, 2, 1) + arr = pa.FixedShapeTensorArray.from_dlpack(source) + arr.validate(full=True) + assert arr.type == pa.fixed_shape_tensor( + pa.from_numpy_dtype(np_type), [2, 2], permutation=[1, 0]) + np.testing.assert_array_equal(arr.to_numpy_ndarray(), source) + + # Zero-copy import: mutating the source is visible through the array. + source[0, 0, 0] = 100 + np.testing.assert_array_equal(arr.to_numpy_ndarray(), source) + + copied = pa.FixedShapeTensorArray.from_dlpack(source, copy=True) + expected = source.copy() + source[0, 0, 0] = 0 + np.testing.assert_array_equal(copied.to_numpy_ndarray(), expected) + + +@requires_numpy_version("2.1.0") +@check_bytes_allocated +def test_fixed_shape_tensor_array_from_dlpack_not_first_major(): + # The outermost dimension indexes the tensor elements, so it must remain + # the major one. + source = np.arange(12, dtype=np.int32).reshape((3, 2, 2)).transpose(1, 0, 2) + with pytest.raises(pa.ArrowInvalid, + match="Only first-major tensors can be zero-copy"): + pa.FixedShapeTensorArray.from_dlpack(source) + + @requires_numpy_version("2.1.0") @check_bytes_allocated def test_from_dlpack_zero_copy(): diff --git a/python/pyarrow/tests/test_extension_type.py b/python/pyarrow/tests/test_extension_type.py index 2cc258e2d2b5..a9d15cfec10a 100644 --- a/python/pyarrow/tests/test_extension_type.py +++ b/python/pyarrow/tests/test_extension_type.py @@ -1718,6 +1718,7 @@ def test_tensor_array_from_tensor(transpose, permutation): arr = arr.transpose(0, 2, 1) if transpose else arr result = pa.FixedShapeTensorArray.from_tensor(pa.Tensor.from_numpy(arr)) + result.validate(full=True) assert isinstance(result.type, pa.FixedShapeTensorType) assert result.type.value_type == pa.int32() @@ -1728,6 +1729,16 @@ def test_tensor_array_from_tensor(transpose, permutation): np.testing.assert_array_equal(result.to_numpy_ndarray(), arr) +@pytest.mark.numpy +@pytest.mark.parametrize("permutation", [(1, 0, 2), (1, 2, 0), (2, 1, 0)]) +def test_tensor_array_from_tensor_not_first_major(permutation): + arr = np.arange(24, dtype=np.int32).reshape(2, 3, 4).transpose(*permutation) + + with pytest.raises(pa.ArrowInvalid, + match="Only first-major tensors can be zero-copy"): + pa.FixedShapeTensorArray.from_tensor(pa.Tensor.from_numpy(arr)) + + @pytest.mark.numpy @pytest.mark.parametrize("np_type_str", ("int8", "int64", "float32")) def test_tensor_array_from_numpy(np_type_str): From 381e5ea41800c67ae79f44c213d1f9f519f47d72 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 9 Sep 2026 10:23:43 +0200 Subject: [PATCH 19/21] Handle row major strides --- cpp/src/arrow/c/dlpack.cc | 65 +++++++++++++++++++++++++--------- cpp/src/arrow/c/dlpack_test.cc | 14 ++++++++ 2 files changed, 63 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 0be180e7070a..be3e15a1f986 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -290,10 +291,7 @@ class CppDLTensor { return Status::Invalid( "Invalid DLPack tensor: shape must be non-null when ndim != 0"); } - if (ARROW_PREDICT_FALSE(out.tensor().ndim != 0 && out.tensor().strides == nullptr)) { - return Status::Invalid( - "Invalid DLPack tensor: strides must be non-null when ndim != 0"); - } + // Null strides are handled as row major return out; } @@ -313,8 +311,12 @@ class CppDLTensor { return {tensor().shape, static_cast(ndim())}; } + /// Strides or empty span for old DLPack row-major convention. std::span strides() const { - return {tensor().strides, static_cast(ndim())}; + if (auto strides = tensor().strides; strides != nullptr) { + return {strides, static_cast(ndim())}; + } + return {}; } bool flag_is_set(uint8_t bits) const { return (ptr_->flags & bits) == bits; } @@ -323,6 +325,34 @@ class CppDLTensor { int32_t byte_width() const { return tensor().dtype.bits / 8; } + /// Number of element in this tensor's buffer. + /// + /// Possibly more elements than represented in the tensor for non-contiguous tensors. + Result ComputeNumElements() const { + if (ndim() == 0) { + return 0; + } + const auto strides = this->strides(); + const auto shape = this->shape(); + if (strides.size() > 0) { + // DLPack strides are in number of elements, so is the size we compute from them. + return internal::ComputeTensorSize(shape, strides, 1); + } + // DLPack <1.3 my set strides == nullptr for row major + return std::reduce(shape.begin(), shape.end(), 1, std::multiplies{}); + } + + /// Number of bytes needed to store this tensor data. + Result ComputeNumBytes() const { + ARROW_ASSIGN_OR_RAISE(const auto nelements, ComputeNumElements()); + int64_t nbytes = 0; + if (ARROW_PREDICT_FALSE(internal::MultiplyWithOverflow( + nelements, static_cast(byte_width()), &nbytes))) { + return Status::Invalid("Overflow computing DLPack tensor size in bytes."); + } + return nbytes; + } + private: struct Deleter { void operator()(pointer_type ptr) { @@ -414,14 +444,8 @@ Result> StridesInBytes(std::span strides, } Result> ImportBuffer(CppDLTensor&& dl) { - // DLPack strides are in number of elements, so is the size we compute from them. - ARROW_ASSIGN_OR_RAISE(const auto nelements, - internal::ComputeTensorSize(dl.shape(), dl.strides(), 1)); - int64_t nbytes = 0; - if (ARROW_PREDICT_FALSE(internal::MultiplyWithOverflow( - nelements, static_cast(dl.byte_width()), &nbytes))) { - return Status::Invalid("Overflow computing DLPack tensor size in bytes."); - } + ARROW_ASSIGN_OR_RAISE(const int64_t nbytes, dl.ComputeNumBytes()); + // DLPack mandates a null data pointer when the tensor holds no element, so there is // neither anything to share nor to copy. uint8_t* data = @@ -458,7 +482,8 @@ Result> ImportArrayVersioned(DLManagedTensorVersioned* un "DLPack support is implemented only for buffers on CPU device."); } - if (dl.ndim() != 1 || dl.strides().front() != 1) { + const auto strides = dl.strides(); + if (dl.ndim() != 1 || (!strides.empty() && strides.front() != 1)) { return Status::Invalid( "Only contiguous one dimensional tensor can be imported as arrays." " Try importing to Tensor first."); @@ -482,11 +507,19 @@ Result> ImportTensorVersioned( ARROW_ASSIGN_OR_RAISE(auto type, DataTypeFromDLPack(dl.tensor().dtype)); auto shape = std::vector(dl.shape().begin(), dl.shape().end()); - auto strides = std::vector(dl.strides().begin(), dl.strides().end()); + const auto strides = dl.strides(); + ARROW_ASSIGN_OR_RAISE(auto buffer, ImportBuffer(std::move(dl))); const auto byte_width = type->byte_width(); - ARROW_ASSIGN_OR_RAISE(auto strides_bytes, StridesInBytes(strides, byte_width)); + // In older DLPack null strides means row major, same as in Arrow. + if (strides.empty()) { + return Tensor::Make(std::move(type), std::move(buffer), std::move(shape)); + } + + ARROW_ASSIGN_OR_RAISE( + auto strides_bytes, + StridesInBytes(std::vector(strides.begin(), strides.end()), byte_width)); return Tensor::Make(std::move(type), std::move(buffer), std::move(shape), std::move(strides_bytes)); } diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 5965ecbca008..0eb3221a7893 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -619,6 +619,20 @@ TYPED_TEST(TestImportTensor, Strided) { ASSERT_EQ(6, tensor->template Value({1, 1})); } +TYPED_TEST(TestImportTensor, NullStrides) { + // DLPack < 1.3 uses null strides to mean row major + auto* managed = Produce({ + .shape = {2, 3}, + .data = ToBytes(std::vector{1, 2, 3, 4, 5, 6}), + }); + managed->dl_tensor.strides = nullptr; + + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::ImportAndValidate(managed)); + ASSERT_THAT(tensor->shape(), ::testing::ElementsAre(2, 3)); + ASSERT_TRUE(tensor->is_row_major()); + ASSERT_EQ(6, tensor->template Value({1, 2})); +} + TYPED_TEST(TestImportTensor, NegativeStrides) { auto foreign = ForeignTensor{ .shape = {2, 2}, From ac31c1ef410b8f1b3549fbd0e8d740ce9eb95f07 Mon Sep 17 00:00:00 2001 From: AntoinePrv Date: Wed, 9 Sep 2026 11:26:18 +0200 Subject: [PATCH 20/21] Fix 0-D tensor --- cpp/src/arrow/c/dlpack.cc | 7 +++---- cpp/src/arrow/c/dlpack_test.cc | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index be3e15a1f986..462a54e522bf 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -328,10 +328,9 @@ class CppDLTensor { /// Number of element in this tensor's buffer. /// /// Possibly more elements than represented in the tensor for non-contiguous tensors. + /// + /// A zero dimensional tensor is a scalar, it holds a single element. Result ComputeNumElements() const { - if (ndim() == 0) { - return 0; - } const auto strides = this->strides(); const auto shape = this->shape(); if (strides.size() > 0) { @@ -339,7 +338,7 @@ class CppDLTensor { return internal::ComputeTensorSize(shape, strides, 1); } // DLPack <1.3 my set strides == nullptr for row major - return std::reduce(shape.begin(), shape.end(), 1, std::multiplies{}); + return std::reduce(shape.begin(), shape.end(), int64_t{1}, std::multiplies{}); } /// Number of bytes needed to store this tensor data. diff --git a/cpp/src/arrow/c/dlpack_test.cc b/cpp/src/arrow/c/dlpack_test.cc index 0eb3221a7893..8cbff0692da3 100644 --- a/cpp/src/arrow/c/dlpack_test.cc +++ b/cpp/src/arrow/c/dlpack_test.cc @@ -619,6 +619,21 @@ TYPED_TEST(TestImportTensor, Strided) { ASSERT_EQ(6, tensor->template Value({1, 1})); } +TYPED_TEST(TestImportTensor, ZeroDimensionIsScalar) { + auto* managed = Produce({ + .shape = {}, + .data = ToBytes(std::vector{42}), + }); + managed->dl_tensor.shape = nullptr; + managed->dl_tensor.strides = nullptr; + + ASSERT_OK_AND_ASSIGN(auto tensor, TypeParam::ImportAndValidate(managed)); + ASSERT_THAT(tensor->shape(), ::testing::IsEmpty()); + ASSERT_EQ(1, tensor->size()); + ASSERT_EQ(sizeof(float), tensor->data()->size()); + ASSERT_EQ(42, tensor->template Value({})); +} + TYPED_TEST(TestImportTensor, NullStrides) { // DLPack < 1.3 uses null strides to mean row major auto* managed = Produce({ From 728079d9826458884456c465a8196dd0a6bca548 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 10 Sep 2026 15:07:24 +0200 Subject: [PATCH 21/21] Fix typo --- cpp/src/arrow/c/dlpack.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/c/dlpack.cc b/cpp/src/arrow/c/dlpack.cc index 462a54e522bf..1b58b5a503c0 100644 --- a/cpp/src/arrow/c/dlpack.cc +++ b/cpp/src/arrow/c/dlpack.cc @@ -337,7 +337,7 @@ class CppDLTensor { // DLPack strides are in number of elements, so is the size we compute from them. return internal::ComputeTensorSize(shape, strides, 1); } - // DLPack <1.3 my set strides == nullptr for row major + // DLPack <1.3 may set strides == nullptr for row major return std::reduce(shape.begin(), shape.end(), int64_t{1}, std::multiplies{}); }