From ca82231c579f1cf8d799a5de4d5324ea5d8472c8 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:17:59 +0200 Subject: [PATCH 01/15] Improve error reporting for method calls without a C++ object (#41) * [cpyrt] Improve error reporting for method calls without C++ object Co-Authored-By: Claude Fable 5 * [test] Add test for method calls on an instance without a C++ object Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Grigori Rybkine Co-authored-by: Claude Fable 5 --- src/cpyrt/CPPMethod.cxx | 3 ++- src/cpyrt/CPPOverload.cxx | 3 ++- test/test_fragile.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/cpyrt/CPPMethod.cxx b/src/cpyrt/CPPMethod.cxx index 761d0ed..3f0d0e5 100644 --- a/src/cpyrt/CPPMethod.cxx +++ b/src/cpyrt/CPPMethod.cxx @@ -1058,7 +1058,8 @@ PyObject* cpyrt::CPPMethod::Call(CPPInstance*& self, cpyrt_PyArgs_t args, // validity check that should not fail if (!object) { - PyErr_SetString(PyExc_ReferenceError, "attempt to access a null-pointer"); + PyErr_SetString(PyExc_ReferenceError, "no C++ object available"); + ctxt->fFlags |= CallContext::kCppException; return nullptr; } diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 8bda467..49778df 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -624,7 +624,8 @@ static PyObject* mp_vectorcall(CPPOverload* pymeth, PyObject* const* args, return HandleReturn(pymeth, im_self, result); // fall through: python is dynamic, and so, the hashing isn't infallible - ctxt.fFlags &= ~CallContext::kAllowImplicit; + ctxt.fFlags &= ~(CallContext::kAllowImplicit | CallContext::kPyException | + CallContext::kCppException); PyErr_Clear(); ResetCallState(pymeth->fSelf, im_self); } diff --git a/test/test_fragile.py b/test/test_fragile.py index 181a38b..e4cf3bc 100644 --- a/test/test_fragile.py +++ b/test/test_fragile.py @@ -761,6 +761,24 @@ def test31_template_with_class_enum(self): for ns, val in [(cppjit.gbl, 42), (cppjit.gbl.ClassEnumNS, 37)]: assert ns.EnumTemplate[ns.ClassEnumA.A]().foo() == val + def test32_overloaded_method_error_with_null_object(self): + """Check exception type and message when method invoked on instance without C++ object""" + + import cppjit + from cppjit import gbl + + cppjit.cppdef(r"""\ + using fragile::D; + D *something = new D; + D *nothing = nullptr; + """) + + assert gbl.something.check() == gbl.something.check(0, 1) + with raises(ReferenceError, match=r"^no C\+\+ object available$"): + gbl.nothing.check() # raises error + with raises(ReferenceError, match=r"^no C\+\+ object available$"): + gbl.nothing.check(0, 1) # raises error + class TestSIGNALS: def setup_class(cls): From 323503946e59904de07f4c4ede355a9bc514a8d2 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:18:35 +0200 Subject: [PATCH 02/15] Penalize void* arguments in overload priority as intended (#40) * [cpyrt] Penalize void* arguments in overload priority as intended * [test] Add regression test for void* overload priority --------- Co-authored-by: Emery Conrad Co-authored-by: Claude Fable 5 --- src/cpyrt/CPPMethod.cxx | 13 ++++++++----- test/test_overloads.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/src/cpyrt/CPPMethod.cxx b/src/cpyrt/CPPMethod.cxx index 3f0d0e5..3fe3e37 100644 --- a/src/cpyrt/CPPMethod.cxx +++ b/src/cpyrt/CPPMethod.cxx @@ -529,7 +529,14 @@ int cpyrt::CPPMethod::GetPriority() { // type: // interop::TCppType_t type = interop::GetMethodArgType(fMethod, iarg); - if (interop::IsBuiltin(aname)) { + // Not builtin and spelled "const void *", so match the compacted name. + std::string compact = aname; + compact.erase(std::remove(compact.begin(), compact.end(), ' '), + compact.end()); + + if (compact.find("void*") != std::string::npos) { + priority -= 1000; // void*/void** shouldn't be too greedy + } else if (interop::IsBuiltin(aname)) { // complex type (note: double penalty: for complex and the template type) if (strstr(aname.c_str(), "std::complex")) priority -= 10; // prefer double, float, etc. over conversion @@ -557,10 +564,6 @@ int cpyrt::CPPMethod::GetPriority() { else if (strstr(aname.c_str(), "char") && aname[aname.size() - 1] != '*') priority += -60; // prefer (const) char* over char - // oddball - else if (strstr(aname.c_str(), "void*")) - priority -= 1000; // void*/void** shouldn't be too greedy - } else { // This is a user-defined type (class, struct, enum, etc.). diff --git a/test/test_overloads.py b/test/test_overloads.py index 24b8d0a..f736c8a 100644 --- a/test/test_overloads.py +++ b/test/test_overloads.py @@ -411,3 +411,32 @@ def test15_disallow_mutable_pointer_references(self): ptr = cppjit.gbl.MyClass() raises(TypeError, cppjit.gbl.changePtr, ptr) + + def test16_voidp_does_not_outrank_conversion(self): + """Verify that a const void* overload does not shadow a converting one.""" + + import cppjit + + cppjit.cppdef(""" + namespace VoidPPriority { + struct Handle { + void* data; + Handle() : data(nullptr) {} + Handle(void* p) : data(p) {} + }; + struct ConstHandle { + const void* data; + ConstHandle() : data(nullptr) {} + ConstHandle(const void* p) : data(p) {} // declared first on purpose + ConstHandle(Handle h) : data(h.data) {} + }; + Handle make_handle() { return Handle((void*)0xABCD1234); } + bool kept_value(ConstHandle c) { return c.data == (const void*)0xABCD1234; } + }""") + + ns = cppjit.gbl.VoidPPriority + + # taking ConstHandle(const void*) would pass the proxy's address instead + h = ns.make_handle() + assert ns.kept_value(h) + assert ns.kept_value(ns.make_handle()) From 46cde5532b02c824b02b1d624504f6b009573dfa Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:18:07 +0200 Subject: [PATCH 03/15] Unify installed layout under cppjit, drop backend (#43) --- CMakeLists.txt | 22 +++++++++++----------- pyproject.toml | 2 +- python/cppjit/__init__.py | 4 ++-- python/cppjit/_cpython_cppjit.py | 6 +++--- python/cppjit_backend/__init__.py | 1 - python/cppjit_backend/_version.py | 1 - 6 files changed, 17 insertions(+), 19 deletions(-) delete mode 100644 python/cppjit_backend/__init__.py delete mode 100644 python/cppjit_backend/_version.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 7158a15..50403dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -101,7 +101,7 @@ if(_python_platlib) else() set(CPPINTEROP_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") endif() -set(CPPINTEROP_INSTALL_DIR "${CPPINTEROP_INSTALL_PREFIX}/cppjit_backend") +set(CPPINTEROP_INSTALL_DIR "${CPPINTEROP_INSTALL_PREFIX}/cppjit/interop") # Include cmake for CppInterOp config and build using ExternalProject. include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddCppInterOp.cmake) @@ -121,11 +121,11 @@ add_dependencies(cppjit CppInterOp) # falling back to the install prefix (see cppinterop_paths()); the clang # major names the versioned compiler probed for the runtime resource dir. target_compile_definitions(cppjit PRIVATE - CPPINTEROP_INSTALL_PREFIX="${CPPINTEROP_INSTALL_PREFIX}" - CPPINTEROP_LIBRARY="cppjit_backend/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" - CPPINTEROP_INCLUDE_DIR="cppjit_backend/include" + CPPINTEROP_INSTALL_PREFIX="${CPPINTEROP_INSTALL_PREFIX}/cppjit" + CPPINTEROP_LIBRARY="interop/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" + CPPINTEROP_INCLUDE_DIR="interop/include" CPPJIT_CLANG_MAJOR="${LLVM_VERSION_MAJOR}" - CPPJIT_CLANG_INCLUDE_DIR="cppjit_backend/lib/clang/${LLVM_VERSION_MAJOR}" + CPPJIT_CLANG_INCLUDE_DIR="interop/lib/clang/${LLVM_VERSION_MAJOR}" ) target_include_directories(cppjit PRIVATE @@ -159,21 +159,21 @@ set_target_properties(cppjit PROPERTIES PREFIX "lib" ) -# libcppjit.so is installed at the site-packages root (import libcppjit) +# the extension lives inside the package (import cppjit.libcppjit) install(TARGETS cppjit - LIBRARY DESTINATION . + LIBRARY DESTINATION cppjit ) # install CppInterOp libraries and headers install(CODE " file(GLOB _interop_libs \"${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp*\") foreach(_lib \${_interop_libs}) - file(INSTALL \${_lib} DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit_backend/lib) + file(INSTALL \${_lib} DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit/interop/lib) endforeach() ") install(CODE " - file(INSTALL \"${CPPINTEROP_INSTALL_DIR}/include/\" DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit_backend/include) + file(INSTALL \"${CPPINTEROP_INSTALL_DIR}/include/\" DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit/interop/include) ") # ship the builtin headers of the build clang, laid out as a headers-only @@ -185,7 +185,7 @@ if(NOT EXISTS "${_clang_resource_dir}/include") "${LLVM_DIR} carries no clang resource directory") endif() install(DIRECTORY "${_clang_resource_dir}/include/" - DESTINATION "cppjit_backend/lib/clang/${LLVM_VERSION_MAJOR}/include" + DESTINATION "cppjit/interop/lib/clang/${LLVM_VERSION_MAJOR}/include" ) # the public cpyrt API headers keep their installed cpyrt/ prefix @@ -195,5 +195,5 @@ install(FILES src/cpyrt/DispatchPtr.h src/cpyrt/PyException.h src/cpyrt/Reflex.h - DESTINATION cppjit_backend/include/cpyrt + DESTINATION cppjit/interop/include/cpyrt ) diff --git a/pyproject.toml b/pyproject.toml index 9878475..5308b63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ maintainers = [ [tool.scikit-build] wheel.install-dir = "." -wheel.packages = ["python/cppjit", "python/cppjit_backend"] +wheel.packages = ["python/cppjit"] cmake.build-type = "Release" [[tool.dynamic-metadata]] diff --git a/python/cppjit/__init__.py b/python/cppjit/__init__.py index ae87217..6280d6e 100644 --- a/python/cppjit/__init__.py +++ b/python/cppjit/__init__.py @@ -348,10 +348,10 @@ def _setup_include_paths(): if os.path.basename(apipath_extra) == "cpyrt": apipath_extra = os.path.dirname(apipath_extra) else: - spec = importlib.util.find_spec("libcppjit") + spec = importlib.util.find_spec("cppjit.libcppjit") if spec is not None and spec.origin: apipath_extra = os.path.join( - os.path.dirname(spec.origin), "cppjit_backend", "include" + os.path.dirname(spec.origin), "interop", "include" ) if apipath_extra and apipath_extra.lower() != "none": diff --git a/python/cppjit/_cpython_cppjit.py b/python/cppjit/_cpython_cppjit.py index 0b11ec3..50a32f3 100644 --- a/python/cppjit/_cpython_cppjit.py +++ b/python/cppjit/_cpython_cppjit.py @@ -21,9 +21,9 @@ def _preload_backend_library(): # preload the merged extension with ctypes and run LoadCppInterOp() first, # so the interpreter is ready before the extension module initializes - spec = importlib.util.find_spec("libcppjit") + spec = importlib.util.find_spec("cppjit.libcppjit") if spec is None or not spec.origin: - raise ImportError("cannot locate the libcppjit extension module") + raise ImportError("cannot locate the cppjit.libcppjit extension module") lib = ctypes.CDLL(spec.origin, ctypes.RTLD_GLOBAL) if not lib.LoadCppInterOp(): raise RuntimeError("failed to load CppInterOp (LoadCppInterOp returned 0)") @@ -32,7 +32,7 @@ def _preload_backend_library(): _w = _preload_backend_library() -import libcppjit as _backend # noqa: E402 +from . import libcppjit as _backend # noqa: E402 ### template support --------------------------------------------------------- diff --git a/python/cppjit_backend/__init__.py b/python/cppjit_backend/__init__.py deleted file mode 100644 index aab79a8..0000000 --- a/python/cppjit_backend/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from ._version import __version__ as __version__ diff --git a/python/cppjit_backend/_version.py b/python/cppjit_backend/_version.py deleted file mode 100644 index 3dc1f76..0000000 --- a/python/cppjit_backend/_version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.1.0" From 821257bd68848e39ae44d7659bddbd710344bd83 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:24:55 +0200 Subject: [PATCH 04/15] Add single header for interop API/types used by cpyrt (#42) Fixes the long-standing header duplication (previously `Cppyy.h` and `cpp_cppyy.h`) between cpyrt and interop, so that there is only a single source of definitions for the `cppjit::interop` API and types. --- src/cpyrt/CallContext.h | 36 +- src/cpyrt/cppjit_interop.h | 467 ------------------ src/interop/callcontext.h | 21 +- .../{cpp_cppjit.h => cppjit_interop.h} | 15 +- src/interop/interop_wrapper.cxx | 12 +- 5 files changed, 30 insertions(+), 521 deletions(-) delete mode 100644 src/cpyrt/cppjit_interop.h rename src/interop/{cpp_cppjit.h => cppjit_interop.h} (96%) diff --git a/src/cpyrt/CallContext.h b/src/cpyrt/CallContext.h index cf614f1..b52915b 100644 --- a/src/cpyrt/CallContext.h +++ b/src/cpyrt/CallContext.h @@ -12,40 +12,8 @@ namespace cppjit::cpyrt { -// small number that allows use of stack for argument passing -const int SMALL_ARGS_N = 8; - -// convention to pass flag for direct calls (similar to Python's vector calls) -#define DIRECT_CALL ((size_t)1 << (8 * sizeof(size_t) - 1)) - -#ifndef CPYRT_PARAMETER -#define CPYRT_PARAMETER -// general place holder for function parameters -struct Parameter { - union Value { - bool fBool; - int8_t fInt8; - uint8_t fUInt8; - short fShort; - unsigned short fUShort; - int fInt; - unsigned int fUInt; - long fLong; - intptr_t fIntPtr; - unsigned long fULong; - long long fLLong; - unsigned long long fULLong; - int64_t fInt64; - uint64_t fUInt64; - float fFloat; - double fDouble; - long double fLDouble; - void* fVoidp; - } fValue; - void* fRef; - char fTypeCode; -}; -#endif // CPYRT_PARAMETER +// Parameter and the call-ABI constants (SMALL_ARGS_N, DIRECT_CALL) come +// from the interop callcontext.h via cppjit_interop.h // extra call information struct CallContext { diff --git a/src/cpyrt/cppjit_interop.h b/src/cpyrt/cppjit_interop.h deleted file mode 100644 index f2cdb69..0000000 --- a/src/cpyrt/cppjit_interop.h +++ /dev/null @@ -1,467 +0,0 @@ -#ifndef CPYRT_CPPJIT_H -#define CPYRT_CPPJIT_H - -// Standard -#include -#include -#include -#include -#include - -// import/export (after precommondefs.h from PyPy) -#ifdef _MSC_VER -#define CPPJIT_IMPORT extern __declspec(dllimport) -#else -#define CPPJIT_IMPORT extern -#endif - -// some more types; assumes cppjit_interop.h follows Python.h -#ifndef PY_LONG_LONG -#ifdef _WIN32 -typedef __int64 PY_LONG_LONG; -#else -typedef long long PY_LONG_LONG; -#endif -#endif - -#ifndef PY_ULONG_LONG -#ifdef _WIN32 -typedef unsigned __int64 PY_ULONG_LONG; -#else -typedef unsigned long long PY_ULONG_LONG; -#endif -#endif - -#ifndef PY_LONG_DOUBLE -typedef long double PY_LONG_DOUBLE; -#endif - -// FIXME: We should not duplicate these definitions here and in CppInterOp.h -// The current setup relies on finding an identical symbol definition in -// libcppjitbackend.so which is fragile and requires updating both locations -// when changing. Ideally we should have the ability to set/get the template arg -// info provided through some factory methods in CppInterOp API, so the clients -// can rely completely on opaque pointers like we do for the rest of the -// argument types. -struct TemplateArgInfo { - void* m_Type; - const char* m_IntegralValue; - TemplateArgInfo(void* type, const char* integral_value = nullptr) - : m_Type(type), m_IntegralValue(integral_value) {} -}; - -namespace Cpp { -using TemplateArgInfo = ::TemplateArgInfo; - -struct DeclRef { - void* data; - DeclRef() : data(nullptr) {} - DeclRef(void* P) : data(P) {} - DeclRef(decltype(nullptr)) : data(nullptr) {} - explicit operator bool() const { return data != nullptr; } - friend bool operator==(DeclRef a, DeclRef b) { return a.data == b.data; } - friend bool operator!=(DeclRef a, DeclRef b) { return !(a == b); } -}; - -struct TypeRef { - void* data; - TypeRef() : data(nullptr) {} - TypeRef(void* P) : data(P) {} - TypeRef(decltype(nullptr)) : data(nullptr) {} - explicit operator bool() const { return data != nullptr; } - friend bool operator==(TypeRef a, TypeRef b) { return a.data == b.data; } - friend bool operator!=(TypeRef a, TypeRef b) { return !(a == b); } -}; - -struct FuncRef { - void* data; - FuncRef() : data(nullptr) {} - FuncRef(void* P) : data(P) {} - FuncRef(decltype(nullptr)) : data(nullptr) {} - explicit operator bool() const { return data != nullptr; } - friend bool operator==(FuncRef a, FuncRef b) { return a.data == b.data; } - friend bool operator!=(FuncRef a, FuncRef b) { return !(a == b); } -}; - -struct ObjectRef { - void* data; - ObjectRef() : data(nullptr) {} - ObjectRef(void* P) : data(P) {} - ObjectRef(decltype(nullptr)) : data(nullptr) {} - explicit operator bool() const { return data != nullptr; } - friend bool operator==(ObjectRef a, ObjectRef b) { return a.data == b.data; } - friend bool operator!=(ObjectRef a, ObjectRef b) { return !(a == b); } -}; -} // namespace Cpp - -template <> struct std::hash { - std::size_t operator()(const Cpp::DeclRef& obj) const { - return std::hash{}(obj.data); - } -}; -template <> struct std::hash { - std::size_t operator()(const Cpp::TypeRef& obj) const { - return std::hash{}(obj.data); - } -}; -template <> struct std::hash { - std::size_t operator()(const Cpp::FuncRef& obj) const { - return std::hash{}(obj.data); - } -}; -template <> struct std::hash { - std::size_t operator()(const Cpp::ObjectRef& obj) const { - return std::hash{}(obj.data); - } -}; - -namespace cppjit::interop { -typedef Cpp::DeclRef TCppScope_t; -typedef Cpp::TypeRef TCppType_t; -typedef Cpp::ObjectRef TCppObject_t; -typedef Cpp::FuncRef TCppMethod_t; -typedef size_t TCppIndex_t; -typedef void* TCppFuncAddr_t; - -// direct interpreter access ------------------------------------------------- -CPPJIT_IMPORT -bool Compile(const std::string& code, bool silent = false); -CPPJIT_IMPORT -std::string ToString(TCppScope_t klass, TCppObject_t obj); - -// name to opaque C++ scope representation ----------------------------------- -CPPJIT_IMPORT -std::string ResolveName(const std::string& cppitem_name); -CPPJIT_IMPORT -TCppType_t ResolveType(TCppType_t cppitem_name); -CPPJIT_IMPORT -TCppType_t ResolveEnumReferenceType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t ResolveEnumPointerType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t GetRealType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t GetPointerType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t GetReferencedType(TCppType_t type, bool rvalue = false); -CPPJIT_IMPORT -std::string ResolveEnum(TCppScope_t enum_scope); -CPPJIT_IMPORT -bool IsLValueReferenceType(TCppType_t type); -CPPJIT_IMPORT -bool IsRValueReferenceType(TCppType_t type); -CPPJIT_IMPORT -bool IsClassType(TCppType_t type); -CPPJIT_IMPORT -bool IsIntegerType(TCppType_t type, bool* is_signed = nullptr); -CPPJIT_IMPORT -bool IsPointerType(TCppType_t type); -CPPJIT_IMPORT -bool IsFunctionPointerType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t GetType(const std::string& name, bool enable_slow_lookup = false); -CPPJIT_IMPORT -bool AppendTypesSlow(const std::string& name, - std::vector& types, - interop::TCppScope_t parent = nullptr); -CPPJIT_IMPORT -TCppType_t GetComplexType(const std::string& element_type); -CPPJIT_IMPORT -TCppScope_t GetScope(const std::string& scope_name, - TCppScope_t parent_scope = TCppScope_t{}); -CPPJIT_IMPORT -TCppScope_t GetUnderlyingScope(TCppScope_t scope); -CPPJIT_IMPORT -TCppScope_t GetFullScope(const std::string& scope_name); -CPPJIT_IMPORT -TCppScope_t GetTypeScope(TCppScope_t klass); -CPPJIT_IMPORT -TCppScope_t GetNamed(const std::string& scope_name, - TCppScope_t parent_scope = TCppScope_t{}); -CPPJIT_IMPORT -TCppScope_t GetParentScope(TCppScope_t scope); -CPPJIT_IMPORT -TCppScope_t GetScopeFromType(TCppType_t type); -CPPJIT_IMPORT -TCppType_t GetTypeFromScope(TCppScope_t klass); -CPPJIT_IMPORT -TCppScope_t GetGlobalScope(); -CPPJIT_IMPORT -TCppScope_t GetActualClass(TCppScope_t klass, TCppObject_t obj); -CPPJIT_IMPORT -size_t SizeOf(TCppScope_t klass); -CPPJIT_IMPORT -size_t SizeOfType(TCppType_t type); - -CPPJIT_IMPORT -bool IsBuiltin(const std::string& type_name); - -CPPJIT_IMPORT -bool IsBuiltin(TCppType_t type); - -CPPJIT_IMPORT -bool IsComplete(TCppScope_t type); - -// memory management --------------------------------------------------------- -CPPJIT_IMPORT -TCppObject_t Allocate(TCppScope_t scope); -CPPJIT_IMPORT -void Deallocate(TCppScope_t scope, TCppObject_t instance); -CPPJIT_IMPORT -TCppObject_t Construct(TCppScope_t scope, void* arena = nullptr); -CPPJIT_IMPORT -void Destruct(TCppScope_t scope, TCppObject_t instance); - -// method/function dispatching ----------------------------------------------- -CPPJIT_IMPORT -void CallV(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -unsigned char CallB(TCppMethod_t method, TCppObject_t self, size_t nargs, - void* args); -CPPJIT_IMPORT -char CallC(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -short CallH(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -int CallI(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -long CallL(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -PY_LONG_LONG CallLL(TCppMethod_t method, TCppObject_t self, size_t nargs, - void* args); -CPPJIT_IMPORT -float CallF(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -double CallD(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -PY_LONG_DOUBLE CallLD(TCppMethod_t method, TCppObject_t self, size_t nargs, - void* args); - -CPPJIT_IMPORT -void* CallR(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args); -CPPJIT_IMPORT -char* CallS(TCppMethod_t method, TCppObject_t self, size_t nargs, void* args, - size_t* length); -CPPJIT_IMPORT -TCppObject_t CallConstructor(TCppMethod_t method, TCppScope_t klass, - size_t nargs, void* args); -CPPJIT_IMPORT -void CallDestructor(TCppScope_t type, TCppObject_t self); -CPPJIT_IMPORT -TCppObject_t CallO(TCppMethod_t method, TCppObject_t self, size_t nargs, - void* args, TCppType_t result_type); - -CPPJIT_IMPORT -TCppFuncAddr_t GetFunctionAddress(TCppMethod_t method, - bool check_enabled = true); - -// handling of function argument buffer -------------------------------------- -CPPJIT_IMPORT -void* AllocateFunctionArgs(size_t nargs); -CPPJIT_IMPORT -void DeallocateFunctionArgs(void* args); -CPPJIT_IMPORT -size_t GetFunctionArgSizeof(); -CPPJIT_IMPORT -size_t GetFunctionArgTypeoffset(); - -// scope reflection information ---------------------------------------------- -CPPJIT_IMPORT -bool IsNamespace(TCppScope_t scope); -CPPJIT_IMPORT -bool IsClass(TCppScope_t scope); -CPPJIT_IMPORT -bool IsTemplate(TCppScope_t scope); -CPPJIT_IMPORT -bool IsTemplateInstantiation(TCppScope_t scope); -CPPJIT_IMPORT -bool IsTypedefed(TCppScope_t scope); -CPPJIT_IMPORT -bool IsAbstract(TCppScope_t scope); -CPPJIT_IMPORT -bool IsEnumScope(TCppScope_t scope); -CPPJIT_IMPORT -bool IsEnumConstant(TCppScope_t scope); -CPPJIT_IMPORT -bool IsEnumType(TCppType_t type); -CPPJIT_IMPORT -bool IsAggregate(TCppScope_t type); -CPPJIT_IMPORT -bool IsDefaultConstructable(TCppScope_t scope); -CPPJIT_IMPORT -bool IsVariable(TCppScope_t scope); - -CPPJIT_IMPORT -void GetAllCppNames(TCppScope_t scope, std::set& cppnames); - -// namespace reflection information ------------------------------------------ -CPPJIT_IMPORT -std::vector GetUsingNamespaces(TCppScope_t); - -// class reflection information ---------------------------------------------- -CPPJIT_IMPORT -std::string GetFinalName(TCppScope_t type); -CPPJIT_IMPORT -std::string GetScopedFinalName(TCppScope_t type); -CPPJIT_IMPORT -bool HasVirtualDestructor(TCppScope_t type); -CPPJIT_IMPORT -TCppIndex_t GetNumBases(TCppScope_t klass); -CPPJIT_IMPORT -TCppIndex_t GetNumBasesLongestBranch(TCppScope_t klass); -CPPJIT_IMPORT -std::string GetBaseName(TCppScope_t klass, TCppIndex_t ibase); -CPPJIT_IMPORT -TCppScope_t GetBaseScope(TCppScope_t klass, TCppIndex_t ibase); -CPPJIT_IMPORT -bool IsSubclass(TCppScope_t derived, TCppScope_t base); -CPPJIT_IMPORT -bool IsSmartPtr(TCppScope_t klass); -CPPJIT_IMPORT -bool GetSmartPtrInfo(const std::string&, TCppScope_t* raw, TCppMethod_t* deref); -// calculate offsets between declared and actual type, up-cast: direction > 0; -// down-cast: direction < 0 -CPPJIT_IMPORT -ptrdiff_t GetBaseOffset(TCppScope_t derived, TCppScope_t base, - TCppObject_t address, int direction, - bool rerror = false); - -// method/function reflection information ------------------------------------ -CPPJIT_IMPORT -void GetClassMethods(TCppScope_t scope, std::vector& methods); -CPPJIT_IMPORT -std::vector GetMethodsFromName(TCppScope_t scope, - const std::string& name); -CPPJIT_IMPORT -std::string GetName(TCppScope_t); -CPPJIT_IMPORT -std::string GetFullName(TCppScope_t); -CPPJIT_IMPORT -TCppType_t GetMethodReturnType(TCppMethod_t); -CPPJIT_IMPORT -std::string GetMethodReturnTypeAsString(TCppMethod_t); -CPPJIT_IMPORT -TCppIndex_t GetMethodNumArgs(TCppMethod_t); -CPPJIT_IMPORT -TCppIndex_t GetMethodReqArgs(TCppMethod_t); -CPPJIT_IMPORT -std::string GetMethodArgName(TCppMethod_t, TCppIndex_t iarg); -CPPJIT_IMPORT -TCppType_t GetMethodArgType(TCppMethod_t, TCppIndex_t iarg); -CPPJIT_IMPORT -TCppIndex_t CompareMethodArgType(TCppMethod_t, TCppIndex_t iarg, - const std::string& req_type); -CPPJIT_IMPORT -std::string GetMethodArgTypeAsString(TCppMethod_t method, TCppIndex_t iarg); -CPPJIT_IMPORT -std::string GetMethodArgCanonTypeAsString(TCppMethod_t method, - TCppIndex_t iarg); -CPPJIT_IMPORT -std::string GetMethodArgDefault(TCppMethod_t, TCppIndex_t iarg); -CPPJIT_IMPORT -std::string GetMethodSignature(TCppMethod_t, bool show_formal_args, - TCppIndex_t max_args = (TCppIndex_t)-1); -// GetMethodPrototype is unused. -CPPJIT_IMPORT -std::string GetMethodPrototype(TCppMethod_t, bool show_formal_args); -CPPJIT_IMPORT -std::string GetDoxygenComment(TCppScope_t scope, bool strip_markers = true); -CPPJIT_IMPORT -bool IsConstMethod(TCppMethod_t); -// Templated method/function reflection information -// ------------------------------------ -CPPJIT_IMPORT -void GetTemplatedMethods(TCppScope_t scope, std::vector& methods); -CPPJIT_IMPORT -TCppIndex_t GetNumTemplatedMethods(TCppScope_t scope, - bool accept_namespace = false); -CPPJIT_IMPORT -std::string GetTemplatedMethodName(TCppScope_t scope, TCppIndex_t imeth); -CPPJIT_IMPORT -bool ExistsMethodTemplate(TCppScope_t scope, const std::string& name); -CPPJIT_IMPORT -bool IsTemplatedMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsStaticTemplate(TCppScope_t scope, const std::string& name); -CPPJIT_IMPORT -TCppMethod_t GetMethodTemplate(TCppScope_t scope, const std::string& name, - const std::string& proto); -CPPJIT_IMPORT -void GetClassOperators(interop::TCppScope_t klass, const std::string& opname, - std::vector& operators); -CPPJIT_IMPORT -TCppMethod_t GetGlobalOperator(TCppScope_t scope, const std::string& lc, - const std::string& rc, const std::string& op); - -// method properties --------------------------------------------------------- -CPPJIT_IMPORT -bool IsDeletedMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsPublicMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsProtectedMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsPrivateMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsConstructor(TCppMethod_t method); -CPPJIT_IMPORT -bool IsDestructor(TCppMethod_t method); -CPPJIT_IMPORT -bool IsStaticMethod(TCppMethod_t method); -CPPJIT_IMPORT -bool IsExplicit(TCppMethod_t method); - -// data member reflection information ---------------------------------------- -CPPJIT_IMPORT -void GetDatamembers(TCppScope_t scope, std::vector& datamembers); -CPPJIT_IMPORT -bool IsLambdaClass(TCppType_t type); -CPPJIT_IMPORT -TCppScope_t WrapLambdaFromVariable(TCppScope_t var); -CPPJIT_IMPORT -TCppMethod_t AdaptFunctionForLambdaReturn(TCppMethod_t fn); -CPPJIT_IMPORT -TCppType_t GetDatamemberType(TCppScope_t data); -CPPJIT_IMPORT -std::string GetDatamemberTypeAsString(TCppScope_t var); -CPPJIT_IMPORT -std::string GetTypeAsString(TCppType_t type); -CPPJIT_IMPORT -intptr_t GetDatamemberOffset(TCppScope_t var, TCppScope_t klass = nullptr); -CPPJIT_IMPORT -bool CheckDatamember(TCppScope_t scope, const std::string& name); - -// // data member properties -// ---------------------------------------------------- -CPPJIT_IMPORT -bool IsPublicData(TCppScope_t var); -CPPJIT_IMPORT -bool IsProtectedData(TCppScope_t var); -CPPJIT_IMPORT -bool IsPrivateData(TCppScope_t var); -CPPJIT_IMPORT -bool IsStaticDatamember(TCppScope_t var); -CPPJIT_IMPORT -bool IsConstVar(TCppScope_t var); -CPPJIT_IMPORT -TCppMethod_t ReduceReturnType(TCppMethod_t fn, TCppType_t reduce); -CPPJIT_IMPORT -std::vector GetDimensions(TCppType_t type); - -// enum properties ----------------------------------------------------------- -CPPJIT_IMPORT -std::vector GetEnumConstants(TCppScope_t scope); -CPPJIT_IMPORT -TCppType_t GetEnumConstantType(TCppScope_t scope); -CPPJIT_IMPORT -TCppIndex_t GetEnumDataValue(TCppScope_t scope); - -CPPJIT_IMPORT -TCppScope_t InstantiateTemplate(TCppScope_t tmpl, Cpp::TemplateArgInfo* args, - size_t args_size); - -CPPJIT_IMPORT -void DumpScope(TCppScope_t scope); -} // namespace cppjit::interop - -#endif // !CPYRT_CPPJIT_H diff --git a/src/interop/callcontext.h b/src/interop/callcontext.h index d0f04f9..5573dba 100644 --- a/src/interop/callcontext.h +++ b/src/interop/callcontext.h @@ -1,11 +1,23 @@ -#ifndef CPYRT_CALLCONTEXT_H -#define CPYRT_CALLCONTEXT_H +#ifndef CPPJIT_INTEROP_CALLCONTEXT_H +#define CPPJIT_INTEROP_CALLCONTEXT_H // Standard -#include +#include +#include + +// convention to pass flag for direct calls (similar to Python's vector calls) +#define DIRECT_CALL ((size_t)1 << (8 * sizeof(size_t) - 1)) namespace cppjit::cpyrt { +// small number that allows use of stack for argument passing +const int SMALL_ARGS_N = 8; + +// The shipped cpyrt/API.h carries an identical Parameter for JIT-side +// code, which cannot see this in-tree header; the shared CPYRT_PARAMETER +// guard keeps one definition per TU. Keep both copies identical. +#ifndef CPYRT_PARAMETER +#define CPYRT_PARAMETER // general place holder for function parameters struct Parameter { union Value { @@ -31,7 +43,8 @@ struct Parameter { void* fRef; char fTypeCode; }; +#endif // CPYRT_PARAMETER } // namespace cppjit::cpyrt -#endif // !CPYRT_CALLCONTEXT_H +#endif // !CPPJIT_INTEROP_CALLCONTEXT_H diff --git a/src/interop/cpp_cppjit.h b/src/interop/cppjit_interop.h similarity index 96% rename from src/interop/cpp_cppjit.h rename to src/interop/cppjit_interop.h index 5fb4be4..ca7c07e 100644 --- a/src/interop/cpp_cppjit.h +++ b/src/interop/cppjit_interop.h @@ -1,5 +1,5 @@ -#ifndef CPYRT_CPPJIT_H -#define CPYRT_CPPJIT_H +#ifndef CPPJIT_INTEROP_H +#define CPPJIT_INTEROP_H #include #include @@ -36,15 +36,6 @@ typedef unsigned long long PY_ULONG_LONG; typedef long double PY_LONG_DOUBLE; #endif -typedef cppjit::cpyrt::Parameter Parameter; - -// small number that allows use of stack for argument passing -const int SMALL_ARGS_N = 8; - -// convention to pass flag for direct calls (similar to Python's vector calls) -#define DIRECT_CALL ((size_t)1 << (8 * sizeof(size_t) - 1)) -static inline size_t CALL_NARGS(size_t nargs) { return nargs & ~DIRECT_CALL; } - namespace cppjit::interop { typedef Cpp::DeclRef TCppScope_t; typedef Cpp::TypeRef TCppType_t; @@ -402,4 +393,4 @@ RPY_EXPORTED void DumpScope(TCppScope_t scope); } // namespace cppjit::interop -#endif // !CPYRT_CPPJIT_H +#endif // !CPPJIT_INTEROP_H diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 0b71747..cce30fa 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -8,11 +8,15 @@ #include "precommondefs.h" // This defines several system feature macros and should be included before any system header. // Bindings -#include "cpp_cppjit.h" +#include "cppjit_interop.h" using namespace cppjit; #include "callcontext.h" +typedef cppjit::cpyrt::Parameter Parameter; + +static inline size_t CALL_NARGS(size_t nargs) { return nargs & ~DIRECT_CALL; } + #ifndef _WIN32 #include #endif @@ -111,7 +115,7 @@ static InterOpPaths cppinterop_paths() { // The one place libclangCppInterOp is dlopen'd. static bool loadDispatchAPI(const InterOpPaths& Paths) { if (!Cpp::LoadDispatchAPI(Paths.Library.c_str())) { - std::cerr << "[cppjit-backend] Failed to load CppInterOp" << std::endl; + std::cerr << "[cppjit] Failed to load CppInterOp" << std::endl; return false; } return true; @@ -859,8 +863,8 @@ static inline bool WrapperCall(interop::TCppMethod_t method, size_t nargs, InterOpMutex.unlock(); bool runRelease = false; // const auto& fgen = /* is_direct ? faceptr.fDirect : */ faceptr; - if (nargs <= SMALL_ARGS_N) { - void* smallbuf[SMALL_ARGS_N]; + if (nargs <= cpyrt::SMALL_ARGS_N) { + void* smallbuf[cpyrt::SMALL_ARGS_N]; if (nargs) runRelease = copy_args(args, nargs, smallbuf); // CLING_CATCH_UNCAUGHT_ From 133da28801fb843be99da42c8a27243f443175a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kerem=20=C5=9Eahin?= Date: Sun, 30 Aug 2026 22:20:26 +0300 Subject: [PATCH 05/15] [test] Fix failures with parallel pytest runs (#49) Changed definition of a function to the test it is actually used, and added a missing import --- test/test_doc_features.py | 12 ++++++++---- test/test_lowlevel.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/test/test_doc_features.py b/test/test_doc_features.py index 9822605..70fa02e 100644 --- a/test/test_doc_features.py +++ b/test/test_doc_features.py @@ -142,10 +142,6 @@ class Abstract2 { return f(i1, i2); } -template -C multiply(A a, B b) { - return static_cast(a * b); -} //----- namespace Namespace { @@ -714,6 +710,14 @@ def test09_templated_function(self): import cppjit + cppjit.cppdef(""" + +template +C multiply(A a, B b) { +return static_cast(a * b); +} + +""") mul = cppjit.gbl.multiply assert "multiply" in cppjit.gbl.__dict__ diff --git a/test/test_lowlevel.py b/test/test_lowlevel.py index 0208a5f..bcc7ac3 100644 --- a/test/test_lowlevel.py +++ b/test/test_lowlevel.py @@ -61,13 +61,14 @@ def test03_memory(self): """Memory allocation and free-ing""" import cppjit + from cppjit import ll # regular C malloc/free mem = cppjit.gbl.malloc(16) cppjit.gbl.free(mem) # typed styles - mem = cppjit.ll.malloc[int](self.N) + mem = ll.malloc[int](self.N) assert len(mem) == self.N assert not mem.__cpp_array__ for i in range(self.N): From 6d89803d34c768f622ca31d255c3b797baf330d3 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:36:42 +0200 Subject: [PATCH 06/15] [build] Lean install: ship single stripped libclangCppInterOp (#50) Drops wheel sizes by about half. Previously the entire install tree of CppInterOp was staged that included duplicate shared libs due to versioning. This is fixed by adding a stripped shared-lib option in CppInterOp, leveraged in this patch. --- CMakeLists.txt | 26 ++++++++++++-------------- cmake/AddCppInterOp.cmake | 10 ++++++++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50403dc..465d158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ include(GNUInstallDirs) # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests? option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF) set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "") -set(CPPINTEROP_GIT_TAG "8d624c621a4b95e36ff73ac708c85a768287478f" CACHE STRING "") +set(CPPINTEROP_GIT_TAG "9802d61921ad5688ae42e4e628d754fc1192244d" CACHE STRING "") set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") @@ -101,7 +101,10 @@ if(_python_platlib) else() set(CPPINTEROP_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") endif() -set(CPPINTEROP_INSTALL_DIR "${CPPINTEROP_INSTALL_PREFIX}/cppjit/interop") + +# CppInterOp installs here; cppjit's own rules ship a subset, so the wheel +# owns every installed file. +set(CPPINTEROP_STAGE_DIR "${CMAKE_BINARY_DIR}/cppinterop-stage") # Include cmake for CppInterOp config and build using ExternalProject. include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/AddCppInterOp.cmake) @@ -134,7 +137,7 @@ target_include_directories(cppjit PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ${CMAKE_CURRENT_SOURCE_DIR}/src/cpyrt ${CMAKE_CURRENT_SOURCE_DIR}/src/interop - ${CPPINTEROP_INSTALL_DIR}/include + ${CPPINTEROP_STAGE_DIR}/include ${Python_INCLUDE_DIRS} ) @@ -164,17 +167,12 @@ install(TARGETS cppjit LIBRARY DESTINATION cppjit ) -# install CppInterOp libraries and headers -install(CODE " - file(GLOB _interop_libs \"${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp*\") - foreach(_lib \${_interop_libs}) - file(INSTALL \${_lib} DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit/interop/lib) - endforeach() -") - -install(CODE " - file(INSTALL \"${CPPINTEROP_INSTALL_DIR}/include/\" DESTINATION \${CMAKE_INSTALL_PREFIX}/cppjit/interop/include) -") +install(DIRECTORY "${CPPINTEROP_STAGE_DIR}/lib/" + DESTINATION cppjit/interop/lib +) +install(DIRECTORY "${CPPINTEROP_STAGE_DIR}/include/" + DESTINATION cppjit/interop/include +) # ship the builtin headers of the build clang, laid out as a headers-only # resource dir: only include/ ships diff --git a/cmake/AddCppInterOp.cmake b/cmake/AddCppInterOp.cmake index 50c069e..be0fef2 100644 --- a/cmake/AddCppInterOp.cmake +++ b/cmake/AddCppInterOp.cmake @@ -22,7 +22,9 @@ function(cppjit_add_cppinterop) -DLLVM_DIR=${LLVM_DIR} -DCPPINTEROP_ENABLE_TESTING=${CPPJIT_ENABLE_CPPINTEROP_TESTS} -DBUILD_SHARED_LIBS=ON - -DCMAKE_INSTALL_PREFIX=${CPPINTEROP_INSTALL_DIR} + # The wheel ships a single unversioned library file. + -DCPPINTEROP_SHARED_LIBRARY_VERSIONING=OFF + -DCMAKE_INSTALL_PREFIX=${CPPINTEROP_STAGE_DIR} -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_CXX_STANDARD=17 @@ -85,12 +87,16 @@ function(cppjit_add_cppinterop) set(_log_args "") endif() + # Install only the library and headers, not CppInterOp's full install tree. ExternalProject_Add(CppInterOp ${_source_args} PREFIX "${CMAKE_BINARY_DIR}/CppInterOp" CMAKE_ARGS ${_args} + # -stripped keeps .dynsym, so the dlsym-based dispatch still resolves. + INSTALL_COMMAND ${CMAKE_COMMAND} --build + --target install-clangCppInterOp-stripped install-cppinterop-headers BUILD_BYPRODUCTS - "${CPPINTEROP_INSTALL_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" + "${CPPINTEROP_STAGE_DIR}/lib/libclangCppInterOp${CMAKE_SHARED_LIBRARY_SUFFIX}" ${_log_args} ) From ab31d05b25596a43713a2d7e8ecdba346ec73e5c Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Mon, 31 Aug 2026 18:23:31 +0200 Subject: [PATCH 07/15] [interop] Silence unused-parameter warnings. NFC (#53) --- src/interop/interop_wrapper.cxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index cce30fa..1b5b0eb 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -1254,8 +1254,8 @@ std::string interop::GetMethodArgDefault(TCppMethod_t method, } interop::TCppIndex_t -interop::CompareMethodArgType(TCppMethod_t /*method*/, TCppIndex_t iarg, - const std::string& req_type) { +interop::CompareMethodArgType(TCppMethod_t /*method*/, TCppIndex_t /*iarg*/, + const std::string& /*req_type*/) { // if (method) { // TFunction* f = m2f(method); // TMethodArg* arg = (TMethodArg From b48635e0a280f89080950031e6134928bf300093 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Mon, 31 Aug 2026 19:56:51 +0200 Subject: [PATCH 08/15] [cpyrt] Clear stale Python errors before C API calls (#52) [cpyrt] Clear the error indicator only where a call failed A debug build of CPython asserts when a C API call is made with the error indicator already set, so the __cpp_cross__ annotation, the meta_setattro fallthrough to tp_setattro, and the VectorData alias need it cleared. Clear it only on the failing path: AddToClass reports failure, and the two CPPScope sites can test the value they just produced, so an error raised elsewhere still propagates. meta_getattro returns a new reference; release it instead of leaking it. --- src/cpyrt/CPPScope.cxx | 19 ++++++++++++++----- src/cpyrt/Pythonize.cxx | 3 ++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/cpyrt/CPPScope.cxx b/src/cpyrt/CPPScope.cxx index fded0d5..b58469d 100644 --- a/src/cpyrt/CPPScope.cxx +++ b/src/cpyrt/CPPScope.cxx @@ -274,10 +274,14 @@ static PyObject* pt_new(PyTypeObject* subtype, PyObject* args, PyObject* kwds) { // also signals that this is a cross-inheritance class) PyObject* bname = cpyrt_PyText_FromString( interop::GetBaseName(result->fCppType, 0).c_str()); - if (PyObject_SetAttrString((PyObject*)result, "__cpp_cross__", - bname) == -1) + if (!bname) PyErr_Clear(); - Py_DECREF(bname); + else { + if (PyObject_SetAttrString((PyObject*)result, "__cpp_cross__", + bname) == -1) + PyErr_Clear(); + Py_DECREF(bname); + } } } else if (sz == (Py_ssize_t)-1) PyErr_Clear(); @@ -571,8 +575,13 @@ static int meta_setattro(PyObject* pyclass, PyObject* pyname, PyObject* pyval) { if (((CPPScope*)pyclass)->fFlags & CPPScope::kIsNamespace && !cpyrt::CPPDataMember_Check(pyval) && !cpyrt::CPPScope_Check(pyval)) { std::string name = cpyrt_PyText_AsString(pyname); - if (interop::GetNamed(name, ((CPPScope*)pyclass)->fCppType)) - meta_getattro(pyclass, pyname); // triggers creation + if (interop::GetNamed(name, ((CPPScope*)pyclass)->fCppType)) { + PyObject* attr = meta_getattro(pyclass, pyname); // triggers creation + if (!attr) + PyErr_Clear(); + else + Py_DECREF(attr); + } } return PyType_Type.tp_setattro(pyclass, pyname, pyval); diff --git a/src/cpyrt/Pythonize.cxx b/src/cpyrt/Pythonize.cxx index 3e6b874..e14067c 100644 --- a/src/cpyrt/Pythonize.cxx +++ b/src/cpyrt/Pythonize.cxx @@ -1898,7 +1898,8 @@ bool cpyrt::Pythonize(PyObject* pyclass, interop::TCppScope_t scope) { METH_VARARGS | METH_KEYWORDS); // data with size - Utility::AddToClass(pyclass, "__real_data", "data"); + if (!Utility::AddToClass(pyclass, "__real_data", "data")) + PyErr_Clear(); // no 'data' method to alias Utility::AddToClass(pyclass, "data", (PyCFunction)VectorData); // numpy array conversion From 94c2728411d644bafb93ce6ff59ce76aa5efc638 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:02:24 +0200 Subject: [PATCH 09/15] Support building wheels for manylinux and OSX (#22) * [test] Find eigen and boost under the Homebrew and MacPorts prefixes * [ci] Build and test manylinux and macOS arm64 wheels --- .github/wheel_smoke.py | 20 ++++++ .github/workflows/wheels.yml | 114 +++++++++++++++++++++++++++++++++++ pyproject.toml | 23 ++++++- test/test_boost.py | 30 +++++++-- test/test_eigen.py | 2 + 5 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 .github/wheel_smoke.py create mode 100644 .github/workflows/wheels.yml diff --git a/.github/wheel_smoke.py b/.github/wheel_smoke.py new file mode 100644 index 0000000..dd3267c --- /dev/null +++ b/.github/wheel_smoke.py @@ -0,0 +1,20 @@ +"""Wheel smoke test, run from a clean venv by cibuildwheel's test step: +libcppjit.so must locate libclangCppInterOp relative to its own path (the +build tree is gone by test time), and the template instantiation plus the +header check prove the shipped include tree.""" + +import os + +import cppjit + +cppjit.cppdef("int wheel_smoke(int x) { return x + 1; }") +assert cppjit.gbl.wheel_smoke(41) == 42 + +v = cppjit.gbl.std.vector["int"]() +v.push_back(7) +assert v[0] == 7 + +api = os.path.join( + os.path.dirname(cppjit.__file__), "interop", "include", "cpyrt", "API.h" +) +assert os.path.exists(api), api diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..a7b3cb4 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,114 @@ +name: Wheels + +# Build the wheels (cibuildwheel; config in pyproject.toml) and the sdist +# as artifacts. setup-recipe stages the llvm-wheel toolchain at /opt/llvm; +# linux mounts it into the build container, the same manylinux_2_28 image +# the toolchain was built on. + +on: + workflow_dispatch: + pull_request: + paths: + - '.github/workflows/wheels.yml' + - '.github/wheel_smoke.py' + - 'pyproject.toml' + - 'CMakeLists.txt' + - 'cmake/**' + - 'src/interop/**' + - 'python/cppjit/_cpython_cppjit.py' + push: + tags: ['v*'] + schedule: + - cron: '30 4 * * 1' + +permissions: + contents: read + +concurrency: + group: wheels-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + wheels: + name: wheels ${{ matrix.label }} + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, label: manylinux-x86_64, arch: x86_64 } + - { os: macos-26, label: macosx-arm64, arch: arm64 } + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v7 + + # ref pins the recipe content the cache key is computed from. + - uses: compiler-research/ci-workflows/actions/setup-recipe@main + id: llvm + with: + recipe: llvm-wheel + version: '21.1.8' + os: ${{ matrix.os }} + arch: ${{ matrix.arch }} + ref: b760e4c171961786b7b20e2cc514302df5373eef + + - name: Stage the toolchain at /opt/llvm + env: + RECIPE_PATH: ${{ steps.llvm.outputs.path }} + run: sudo mv "$RECIPE_PATH" /opt/llvm + + - uses: pypa/cibuildwheel@v4.2.0 + + - uses: actions/upload-artifact@v7 + with: + name: wheels-${{ matrix.label }} + path: wheelhouse/*.whl + + sdist: + name: sdist + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - run: pipx run build --sdist + + - uses: actions/upload-artifact@v7 + with: + name: sdist + path: dist/*.tar.gz + + # Run the full suite on a plain runner, outside the manylinux + # container the wheel was built in. + test-wheel: + name: test wheel (full suite) + needs: wheels + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - uses: actions/download-artifact@v8 + with: + name: wheels-manylinux-x86_64 + path: wheelhouse + + - name: Install the test suite's native deps + # test_eigen/test_boost need them; the CI cells install the same pair. + run: sudo apt-get -q update && sudo apt-get -y install libeigen3-dev libboost-dev + + - name: Install the wheel and the test requirements + run: python -m pip install wheelhouse/cppjit-*cp312*.whl -r requirements.txt + + - name: Smoke the wheel outside pytest + run: python -X faulthandler .github/wheel_smoke.py + + - name: Run the test suite against the installed wheel + env: + CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20 + run: | + cd test + make -j$(nproc) PYTHON=python + python -m pytest -ra diff --git a/pyproject.toml b/pyproject.toml index 5308b63..63d0fc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "scikit_build_core.build" name = "cppjit" dynamic = ["version"] description = "CppJIT: fast and automatic Python-C++ interoperability" -license = {text = "LBNL BSD"} +license = "BSD-3-Clause-LBNL" requires-python = ">=3.12" authors = [ {name = "Aaron Jomy"}, @@ -21,6 +21,7 @@ maintainers = [ ] [tool.scikit-build] +minimum-version = "build-system.requires" wheel.install-dir = "." wheel.packages = ["python/cppjit"] cmake.build-type = "Release" @@ -30,6 +31,26 @@ provider = "scikit_build_core.metadata.regex" field = "version" input = "python/cppjit/_version.py" +[tool.cibuildwheel] +build = ["cp312-*", "cp313-*", "cp314-*"] +skip = ["*-musllinux*"] +build-verbosity = 1 +test-sources = ["test", "requirements.txt", ".github/wheel_smoke.py"] +test-command = "python .github/wheel_smoke.py" + +[tool.cibuildwheel.linux] +archs = ["x86_64"] +manylinux-x86_64-image = "manylinux_2_28" +# /opt/llvm is staged on the runner by wheels.yml. +container-engine = { name = "docker", create-args = ["--volume=/opt/llvm:/opt/llvm"] } +environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/opt/llvm/lib/cmake/clang" } + +[tool.cibuildwheel.macos] +archs = ["arm64"] +before-test = "brew install eigen boost" +test-command = "python -m pip install -r requirements.txt && python .github/wheel_smoke.py && cd test && make -j$(sysctl -n hw.ncpu) PYTHON=python && CPPINTEROP_EXTRA_INTERPRETER_ARGS=-std=c++20 python -m pytest -ra" +environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/opt/llvm/lib/cmake/clang", MACOSX_DEPLOYMENT_TARGET = "14.0" } + [tool.pytest.ini_options] testpaths = ["test"] pythonpath = ["test"] diff --git a/test/test_boost.py b/test/test_boost.py index 3680641..5d51cae 100644 --- a/test/test_boost.py +++ b/test/test_boost.py @@ -3,12 +3,30 @@ from pytest import mark, raises, skip from support import IS_MAC_ARM, IS_MAC_X86 -noboost = False -if not ( +# /usr/include and /usr/local/include are on the compiler's default search +# path; the Homebrew (arm64) and MacPorts prefixes are not, so a hit there +# is remembered and added explicitly before the first include. +boost_extra_inc = None +noboost = not ( os.path.exists(os.path.join(os.path.sep, "usr", "include", "boost")) or os.path.exists(os.path.join(os.path.sep, "usr", "local", "include", "boost")) -): - noboost = True +) +if noboost: + for p in ( + os.path.join(os.path.sep, "opt", "homebrew", "include"), + os.path.join(os.path.sep, "opt", "local", "include"), + ): + if os.path.exists(os.path.join(p, "boost")): + boost_extra_inc = p + noboost = False + break + + +def add_boost_include_path(): + if boost_extra_inc is not None: + import cppjit + + cppjit.add_include_path(boost_extra_inc) @mark.skipif(noboost == True, reason="boost not found") @@ -16,6 +34,7 @@ class TestBOOSTANY: def setup_class(cls): import cppjit + add_boost_include_path() cppjit.include("boost/any.hpp") @mark.skipif((IS_MAC_ARM or IS_MAC_X86), reason="Fails to include boost on OS X") @@ -76,6 +95,7 @@ class TestBOOSTOPERATORS: def setup_class(cls): import cppjit + add_boost_include_path() cppjit.include("boost/operators.hpp") def test01_ordered(self): @@ -101,6 +121,7 @@ class TestBOOSTVARIANT: def setup_class(cls): import cppjit + add_boost_include_path() cppjit.include("boost/variant/variant.hpp") cppjit.include("boost/variant/get.hpp") @@ -147,6 +168,7 @@ class TestBOOSTERASURE: def setup_class(cls): import cppjit + add_boost_include_path() cppjit.include("boost/type_erasure/any.hpp") cppjit.include("boost/type_erasure/member.hpp") cppjit.include("boost/mpl/vector.hpp") diff --git a/test/test_eigen.py b/test/test_eigen.py index ab33c34..88a0772 100644 --- a/test/test_eigen.py +++ b/test/test_eigen.py @@ -5,6 +5,8 @@ inc_paths = [ os.path.join(os.path.sep, "usr", "include"), os.path.join(os.path.sep, "usr", "local", "include"), + os.path.join(os.path.sep, "opt", "homebrew", "include"), # Homebrew on arm64 + os.path.join(os.path.sep, "opt", "local", "include"), # MacPorts ] eigen_path = None From 6ac8260063c828ee7525cc054c5bd9b64b75600c Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:02:42 +0200 Subject: [PATCH 10/15] [cpyrt] Reuse InitializerListConverter element converters (#54) SetArg() created an element converter per call and appended it to fConverters, but Clear() frees only fBuffer, so the vector grew without bound across repeated std::initializer_list conversions. Create each element converter once, on first use of its index, and reuse it. --- src/cpyrt/Converters.cxx | 8 ++++---- test/test_leakcheck.py | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/cpyrt/Converters.cxx b/src/cpyrt/Converters.cxx index 31ab848..c647696 100644 --- a/src/cpyrt/Converters.cxx +++ b/src/cpyrt/Converters.cxx @@ -3186,7 +3186,9 @@ bool cpyrt::InitializerListConverter::SetArg(PyObject* pyobject, PyObject* item = PySequence_GetItem(pyobject, i); bool convert_ok = false; if (item) { - Converter* converter = CreateConverter(fValueTypeName); + if (i >= fConverters.size()) + fConverters.emplace_back(CreateConverter(fValueTypeName)); + Converter* converter = fConverters[i]; if (!converter) { if (CPPInstance_Check(item)) { // by convention, use byte copy @@ -3208,10 +3210,8 @@ bool cpyrt::InitializerListConverter::SetArg(PyObject* pyobject, .c_str()); entries += 1; } - if (memloc) { + if (memloc) convert_ok = converter->ToMemory(item, memloc); - } - fConverters.emplace_back(converter); } Py_DECREF(item); diff --git a/test/test_leakcheck.py b/test/test_leakcheck.py index ea21184..6813f29 100644 --- a/test/test_leakcheck.py +++ b/test/test_leakcheck.py @@ -282,3 +282,21 @@ def wrapped_list_by_value(): ns.leak_list = wrapped_list_by_value self.check_func(ns, "leak_list") + + def test09_initializer_list_argument(self): + """Leak check of passing a list as an std::initializer_list argument""" + + import cppjit + + cppjit.cppdef("""\ + namespace LeakCheck { + int sum_il(std::initializer_list l) { + int s = 0; + for (auto i : l) s += i; + return s; + } + }""") + + ns = cppjit.gbl.LeakCheck + + self.check_func(ns, "sum_il", [1, 2, 3]) From 8e1f14085df3d5a6e3ea0c10b95b944a8e2bed72 Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Tue, 1 Sep 2026 13:35:19 +0200 Subject: [PATCH 11/15] [test] Improve pytest infrastructure, markers and xdist support (#51) * Serialize and atomize test dictionary builds * Force loadfile scheduling for distributed test runs * Normalize the xfail marker keyword order * Correct the xfail markers and add missing reasons * Add the --run-crashing-xfails collection option * Enable strict xfail * Drop xfail markers that no longer fail on macOS and cling * Make the span tests include their own header --- .gitignore | 2 ++ pyproject.toml | 1 + test/Makefile | 7 ++-- test/conftest.py | 66 +++++++++++++++++++++++++++++++++++ test/support.py | 27 ++++++++++---- test/test_advancedcpp.py | 9 +++-- test/test_api.py | 4 +-- test/test_basic_api.py | 9 ++--- test/test_boost.py | 4 +-- test/test_concurrent.py | 5 ++- test/test_conversions.py | 2 +- test/test_cpp11features.py | 14 ++++---- test/test_crossinheritance.py | 42 +++++++++------------- test/test_datatypes.py | 9 ++--- test/test_doc_features.py | 29 ++++++--------- test/test_fragile.py | 21 +++++++++-- test/test_lowlevel.py | 4 +-- test/test_numba.py | 2 +- test/test_overloads.py | 7 +--- test/test_pythonization.py | 4 +-- test/test_regression.py | 22 +++++------- test/test_stltypes.py | 47 ++++++++++--------------- test/test_streams.py | 4 +-- test/test_templates.py | 15 ++++---- 24 files changed, 201 insertions(+), 155 deletions(-) create mode 100644 test/conftest.py diff --git a/.gitignore b/.gitignore index 2013091..65e307c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ __pycache__/ # Built test dictionaries and extension modules *.so +*.so.*.tmp +*Dict.lock # Packaging build/ diff --git a/pyproject.toml b/pyproject.toml index 63d0fc9..b9a2821 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/o [tool.pytest.ini_options] testpaths = ["test"] pythonpath = ["test"] +xfail_strict = true [tool.ruff] show-fixes = true diff --git a/test/Makefile b/test/Makefile index e07e775..7f1433e 100644 --- a/test/Makefile +++ b/test/Makefile @@ -29,8 +29,9 @@ ifeq ($(PLATFORM),Darwin) cppflags+=-dynamiclib -single_module -undefined dynamic_lookup -Wno-delete-non-virtual-dtor endif -cpp/%Dict.so: cpp/%.cxx - $(CXX) $(cppflags) -shared -o $@ $^ +# a worker can load the library while another rebuilds it, so publish it whole +cpp/%Dict.so: cpp/%.cxx cpp/%.h + $(CXX) $(cppflags) -shared -o $@.$$$$.tmp $< && mv -f $@.$$$$.tmp $@ # convenience: `make datatypesDict.so` builds cpp/datatypesDict.so %Dict.so: cpp/%Dict.so ; @@ -41,4 +42,4 @@ test: pytest test_*.py clean: - -rm -f $(dicts) + -rm -f $(dicts) cpp/*.tmp cpp/*.lock diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..525f071 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,66 @@ +"""Suite-wide pytest infrastructure. + +Tests within a file share interpreter state (cppdefs, loaded dictionaries, +pythonizations), so distributed runs must keep whole files on one worker. +""" + +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--run-crashing-xfails", + action="store_true", + default=False, + help="run xfail(run=False) crash-class tests; a pass is a strict xpass", + ) + + +def _applies_here(mark): + """Whether a mark's conditions hold; pytest evaluates string ones itself.""" + + conditions = list(mark.args[:1]) + if "condition" in mark.kwargs: + conditions.append(mark.kwargs["condition"]) + return all(True if isinstance(c, str) else bool(c) for c in conditions) + + +def pytest_collection_modifyitems(config, items): + if not config.getoption("--run-crashing-xfails"): + return + # Keep only the crash markers that claim this platform, and let them run: + # the marker stays, so one that stopped crashing reports as a strict + # xpass. The rest are deselected; they would only add state the real + # suite never has. + selected, deselected = [], [] + for item in items: + crashing = [ + m + for m in item.own_markers + if m.name == "xfail" and m.kwargs.get("run") is False and _applies_here(m) + ] + if not crashing: + deselected.append(item) + continue + item.own_markers = [ + pytest.mark.xfail(*m.args, **{**m.kwargs, "run": True}).mark + if m in crashing + else m + for m in item.own_markers + ] + selected.append(item) + if deselected: + config.hook.pytest_deselected(items=deselected) + items[:] = selected + + +def pytest_configure(config): + # -n implies --dist load; every mode finer than per-file is remapped + # ("each" and "no" already keep files whole). + if config.getoption("numprocesses", None) and config.getoption("dist", "no") in ( + "load", + "worksteal", + "loadscope", + "loadgroup", + ): + config.option.dist = "loadfile" diff --git a/test/support.py b/test/support.py index 5d1e74e..de2532d 100644 --- a/test/support.py +++ b/test/support.py @@ -6,6 +6,11 @@ import py +try: + import fcntl +except ImportError: # Windows: no concurrent make workflow to serialize + fcntl = None + currpath = py.path.local(__file__).dirpath() @@ -13,13 +18,21 @@ def setup_make(targetname): if os.getenv("CPPJIT_TEST_SKIP_MAKE", False): return - popen = subprocess.Popen( - ["make", targetname + "Dict.so"], - cwd=str(currpath), - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) - stdout, _ = popen.communicate() + # several files share a dictionary, so workers race make for it; the lock + # is per target to keep unrelated builds parallel + lockf = open(str(currpath.join("cpp", targetname + "Dict.lock")), "a") + try: + if fcntl is not None: + fcntl.flock(lockf, fcntl.LOCK_EX) + popen = subprocess.Popen( + ["make", targetname + "Dict.so"], + cwd=str(currpath), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + stdout, _ = popen.communicate() + finally: + lockf.close() if popen.returncode: raise OSError("'make' failed:\n%s" % (stdout,)) diff --git a/test/test_advancedcpp.py b/test/test_advancedcpp.py index 2d0f469..9d754e7 100644 --- a/test/test_advancedcpp.py +++ b/test/test_advancedcpp.py @@ -643,7 +643,7 @@ def test15_template_instantiation_with_vector_of_float(self): b.m_b.push_back(i) assert round(b.m_b[i], 5) == float(i) - @mark.xfail + @mark.xfail(reason="templated free function returns a string proxy, not str") def test16_template_global_functions(self): """Test template global function lookup and calls""" @@ -708,7 +708,6 @@ def test19_comparator(self): assert a.__eq__(a) == False assert b.__eq__(b) == False - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test20_overload_order_with_proper_return(self): """Test return type against proper overload w/ const and covariance""" @@ -717,7 +716,7 @@ def test20_overload_order_with_proper_return(self): assert cppjit.gbl.overload_one_way().gime() == 1 assert cppjit.gbl.overload_the_other_way().gime() == "aap" - @mark.xfail(run=not IS_VALGRIND) + @mark.xfail(condition=IS_VALGRIND, run=False, reason="hangs under valgrind") def test21_access_to_global_variables(self): """Access global_variables_and_pointers""" @@ -752,8 +751,8 @@ def test21_access_to_global_variables(self): assert len(cppjit.gbl.gtestv2) == 1 @mark.xfail( - run=False, condition=IS_MAC_ARM, + run=False, reason="Crashes with exception not being caught on Apple Silicon", ) def test22_exceptions(self): @@ -779,7 +778,7 @@ def test22_exceptions(self): caught = True assert caught == True - @mark.xfail + @mark.xfail(reason="using-declared overloads expose the base class signature") def test23_using(self): """Accessibility of using declarations""" diff --git a/test/test_api.py b/test/test_api.py index c52fc4c..f6f4926 100644 --- a/test/test_api.py +++ b/test/test_api.py @@ -67,7 +67,7 @@ class APICheck2 { m2 = API.Instance_FromVoidPtr(voidp, "APICheck2") assert m is m2 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test04_custom_converter(self): """Custom type converter""" @@ -146,7 +146,7 @@ class APICheck3Converter : public cppjit::cpyrt::Converter { assert type(gA3b) == cppjit.gbl.APICheck3 assert not gA3b.wasFromMemoryCalled() - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test05_custom_executor(self): """Custom type executor""" diff --git a/test/test_basic_api.py b/test/test_basic_api.py index 9e9fdbe..10e2e81 100644 --- a/test/test_basic_api.py +++ b/test/test_basic_api.py @@ -2,8 +2,8 @@ import tempfile import py -from pytest import mark, raises -from support import IS_MAC, setup_make +from pytest import raises +from support import setup_make # reuse the example01 currpath = py.path.local(__file__).dirpath() @@ -15,7 +15,6 @@ def setup_module(mod): class TestBASICAPI: - @mark.xfail(IS_MAC, reason="evaluate is broken on macos") def test01_evaluate(self): import cppjit @@ -34,10 +33,6 @@ def test01_evaluate(self): x = 42 assert cppjit.evaluate(str(x)) == x - @mark.xfail( - IS_MAC, - reason="unidentified IsDebugOutputEnabled issue on macos, also failing in test_fragile", - ) def test02_cppdef(self): import cppjit diff --git a/test/test_boost.py b/test/test_boost.py index 5d51cae..bba20a1 100644 --- a/test/test_boost.py +++ b/test/test_boost.py @@ -50,7 +50,7 @@ def test01_any_class(self): assert std.list[any] - @mark.xfail(run=False) + @mark.xfail(run=False, reason="boost::any casting crashes") def test02_any_usage(self): """boost::any assignment and casting""" @@ -125,7 +125,7 @@ def setup_class(cls): cppjit.include("boost/variant/variant.hpp") cppjit.include("boost/variant/get.hpp") - @mark.xfail(run=False) + @mark.xfail(run=False, reason="boost::variant access crashes") def test01_variant_usage(self): """boost::variant usage""" diff --git a/test/test_concurrent.py b/test/test_concurrent.py index 7c176d0..6adea08 100644 --- a/test/test_concurrent.py +++ b/test/test_concurrent.py @@ -1,5 +1,5 @@ from pytest import mark, skip -from support import IS_LINUX_ARM, IS_MAC_ARM, IS_MAC_X86 +from support import IS_LINUX_ARM, IS_MAC_ARM class TestCONCURRENT: @@ -91,7 +91,6 @@ def test03_timeout(self): if t.is_alive(): # was timed-out cppjit.gbl.test12_timeout.stopit[0] = True - @mark.xfail(condition=IS_MAC_X86, reason="Fails on OS X x86") def test04_cpp_threading_with_exceptions(self): """Threads and Python exceptions""" @@ -173,7 +172,7 @@ def process(self, c): assert "RuntimeError" in w.err_msg assert "all wrong" in w.err_msg - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test05_float2d_callback(self): """Passing of 2-dim float arguments""" diff --git a/test/test_conversions.py b/test/test_conversions.py index 0e980f0..8741534 100644 --- a/test/test_conversions.py +++ b/test/test_conversions.py @@ -98,7 +98,7 @@ def test03_error_handling(self): assert CC.s_count == 0 @mark.xfail( - run=IS_CLANG_REPL, condition=IS_MAC or IS_CLING, reason="Crashes on Cling" + condition=IS_MAC or IS_CLING, run=IS_CLANG_REPL, reason="Crashes on Cling" ) def test04_implicit_conversion_from_tuple(self): """Allow implicit conversions from tuples as arguments {}-like""" diff --git a/test/test_cpp11features.py b/test/test_cpp11features.py index d5fc75f..a221480 100644 --- a/test/test_cpp11features.py +++ b/test/test_cpp11features.py @@ -26,7 +26,7 @@ def setup_class(cls): cls.cpp11features = cppjit.load_reflection_info(cls.test_dct) - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test01_smart_ptr(self): """Usage and access of std::shared/unique_ptr<>""" @@ -60,8 +60,8 @@ def test01_smart_ptr(self): assert TestSmartPtr.s_counter == 0 @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Valgrind issues on ARM", ) def test02_smart_ptr_construction(self): @@ -92,7 +92,7 @@ class C(TestSmartPtr): gc.collect() assert TestSmartPtr.s_counter == 0 - @mark.xfail(run=False, condition=IS_LINUX and IS_VALGRIND, reason="Valgrind issue") + @mark.xfail(condition=IS_LINUX and IS_VALGRIND, run=False, reason="Valgrind issue") def test03_smart_ptr_memory_handling(self): """Test shared/unique pointer memory ownership""" @@ -124,7 +124,7 @@ class C(TestSmartPtr): gc.collect() assert TestSmartPtr.s_counter == 0 - @mark.xfail(run=False, condition=IS_VALGRIND, reason="Crashes on Valgrind") + @mark.xfail(condition=IS_VALGRIND, run=False, reason="Crashes on Valgrind") def test04_shared_ptr_passing(self): """Ability to pass shared_ptr through shared_ptr""" @@ -444,7 +444,7 @@ def test13_stdhash(self): assert hash(sw) == 17 assert hash(sw) == 17 - @mark.xfail + @mark.xfail(reason="plain pointer does not convert to a shared_ptr argument") def test14_shared_ptr_passing(self): """Ability to pass normal pointers through shared_ptr by value""" @@ -498,7 +498,7 @@ def test15_unique_ptr_template_deduction(self): with raises(ValueError): # not an RValue cppjit.gbl.UniqueTempl.returnptr[int](uptr_in) - @mark.xfail(IS_MAC, reason="Fails on Mac platforms") + @mark.xfail(condition=IS_MAC, reason="Fails on Mac platforms") def test16_unique_ptr_moves(self): """std::unique_ptr requires moves""" @@ -590,8 +590,8 @@ def test18_unique_ptr_identity(self): assert p1 is p2 @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Valgrind issues on ARM", ) def test19_smartptr_from_callback(self): diff --git a/test/test_crossinheritance.py b/test/test_crossinheritance.py index ab0ae9e..7923749 100644 --- a/test/test_crossinheritance.py +++ b/test/test_crossinheritance.py @@ -52,7 +52,7 @@ def get_value(self): assert Base1.call_get_value(Base1()) == 42 assert Base1.call_get_value(Derived()) == 13 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test02_constructor(self): """Test constructor usage for derived classes""" @@ -90,7 +90,7 @@ def get_value(self): assert d.get_value() == 29 assert Base1.call_get_value(d) == 29 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test03_override_function_abstract_base(self): """Test ability to override a simple function with an abstract base""" @@ -149,8 +149,8 @@ def get_value(self): assert CX.IBase2.call_get_value(c4) == 77 @mark.xfail( - run=False, condition=IS_MAC_ARM, + run=False, reason="Crashes with exception not being caught on Apple Silicon", ) def test04_arguments(self): @@ -193,7 +193,7 @@ def pass_value5(self, b): d2 = Derived2() assert Base1.sum_pass_value(d2) == 12 + 4 * d2.m_int - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test05_override_overloads(self): """Test ability to override overloaded functions""" @@ -215,7 +215,7 @@ def sum_all(self, *args): assert d.sum_all(-7, -5) == 1 assert Base1.call_sum_all(d, -7, -5) == 1 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test06_const_methods(self): """Declared const methods should keep that qualifier""" @@ -239,9 +239,7 @@ def __init__(self): assert CX.IBase4.call_get_value(c1) == 17 assert CX.IBase4.call_get_value(c2) == 27 - @mark.xfail( - run=False, condition=IS_LINUX_ARM, reason="Fails with ModuleNotFound error" - ) + @mark.xfail(condition=IS_LINUX_ARM, reason="Fails with ModuleNotFoundError") def test07_templated_base(self): """Derive from a base class that is instantiated from a template""" @@ -264,7 +262,7 @@ def get_value(self): p1 = TPyDerived1() assert p1.get_value() == 13 - @mark.xfail(run=not IS_MAC_ARM, condition=IS_MAC, reason="Fails on OS X") + @mark.xfail(condition=IS_MAC_ARM, run=False, reason="Fails on macOS arm") def test08_error_handling(self): """Python errors should propagate through wrapper""" @@ -310,8 +308,8 @@ def sum_value(self, val): assert os.path.basename(__file__) in res @mark.xfail( - run=not IS_MAC_ARM, condition=IS_MAC_ARM, + run=False, reason="Crashes with exception not being caught on Apple Silicon", ) def test09_interface_checking(self): @@ -380,7 +378,7 @@ def call(self): gc.collect() assert CB.s_count == 0 + start_count - @mark.xfail(run=False, condition=IS_CLING, reason="Crashes on Cling") + @mark.xfail(condition=IS_CLING, run=False, reason="Crashes on Cling") def test11_python_in_make_shared(self): """Usage of Python derived objects with std::make_shared""" @@ -447,7 +445,7 @@ def call(self): gc.collect() assert CB.s_count == 0 + start_count - @mark.xfail(run=False, condition=IS_VALGRIND, reason="Valgrind issue") + @mark.xfail(condition=IS_VALGRIND, run=False, reason="Valgrind issue") def test12_python_shared_ptr_memory(self): """Usage of Python derived objects with std::shared_ptr""" @@ -564,7 +562,7 @@ def __init__(self): assert m.get_data() == 42 assert m.get_data_v() == 42 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test15_object_returns(self): """Return of C++ objects from overridden functions""" @@ -632,7 +630,6 @@ def whoami(self): assert not not new_obj assert new_obj.whoami() == "PyDerived4" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test16_cctor_access_controlled(self): """Python derived class of C++ class with access controlled cctor""" @@ -675,7 +672,6 @@ def whoami(self): obj = PyDerived() assert ns.callit(obj) == "PyDerived" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test17_deep_hierarchy(self): """Test a deep Python hierarchy with pure virtual functions""" @@ -722,7 +718,6 @@ def whoami(self): assert obj.whoami() == "PyDerived4" assert ns.callit(obj) == "PyDerived4" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test18_abstract_hierarchy(self): """Hierarchy with abstract classes""" @@ -799,7 +794,7 @@ class Derived(ns.Base): def abstract1(self): return ns.Result(1) - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test20_basic_multiple_inheritance(self): """Basic multiple inheritance""" @@ -879,8 +874,8 @@ def z(self): assert a.m_3 == 67 @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Crashes with Valgrind on Linux ARM", ) def test21_multiple_inheritance_with_constructors(self): @@ -971,8 +966,8 @@ def z(self): assert a.m_3 == -11 @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Crashes with Valgrind on Linux ARM", ) def test22_multiple_inheritance_with_defaults(self): @@ -1095,7 +1090,6 @@ def return_const(self): assert a.return_const().m_value == "abcdef" assert ns.callit(a).m_value == "abcdef" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test24_non_copyable(self): """Inheriting from a non-copyable base class""" @@ -1350,8 +1344,8 @@ class D(B): assert inst.fun2() == inst.fun1() @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Crashes with Valgrind on Linux ARM", ) def test29_cross_deep_multi(self): @@ -1603,8 +1597,8 @@ def getValue(self): assert ns.Component.get_count() == 0 @mark.xfail( - run=False, condition=IS_LINUX_ARM and IS_VALGRIND, + run=False, reason="Crashes with Valgrind on Linux ARM", ) def test32_by_value_arguments(self): @@ -1681,7 +1675,7 @@ def func(self): c = C() assert c.func() == 3 - @mark.xfail + @mark.xfail(reason="deriving from a ctor-less base does not raise TypeError") def test34_no_ctors_in_base(self): """Base classes with no constructors""" @@ -1800,7 +1794,6 @@ def __del__(self): del o1 assert Derived.was_py_deleted == True - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test37_deep_tree(self): """Find overridable methods deep in the tree""" @@ -1873,7 +1866,6 @@ def f3(self): assert pysub.f3() == "Python: PySub::f3()" assert ns.call_fs(pysub) == pysub.f1() + pysub.f2() + pysub.f3() - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test38_protected_data(self): """Multiple cross inheritance with protected data""" diff --git a/test/test_datatypes.py b/test/test_datatypes.py index 89df961..2983d4d 100644 --- a/test/test_datatypes.py +++ b/test/test_datatypes.py @@ -687,7 +687,6 @@ def test07_type_conversions(self): c.__destruct__() - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test08_global_builtin_type(self): """Test access to a global builtin type""" @@ -1409,7 +1408,7 @@ def run(self, f, buf, total): run(self, cppjit.gbl.sum_uc_data, buf, total) run(self, cppjit.gbl.sum_byte_data, buf, total) - @mark.xfail(run=False, condition=IS_MAC, reason="Crashes on OSX") + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes on OSX") def test26_function_pointers(self): """Function pointer passing""" @@ -1474,7 +1473,7 @@ def sum_in_python(i1, i2, i3): ns = cppjit.gbl.FuncPtrReturn assert ns.foo()() == "Hello, World!" - @mark.xfail(run=False, condition=IS_MAC, reason="Crashes") + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes") def test27_callable_passing(self): """Passing callables through function pointers""" @@ -1553,7 +1552,7 @@ def pyd(arg0, arg1): gc.collect() raises(TypeError, c, 3, 3) # lambda gone out of scope - @mark.xfail(run=False, condition=IS_MAC, reason="Crashes on MacOS") + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes on MacOS") def test28_callable_through_function_passing(self): """Passing callables through std::function""" @@ -1632,7 +1631,6 @@ def pyd(arg0, arg1): gc.collect() raises(TypeError, c, 3, 3) # lambda gone out of scope - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test29_std_function_life_lines(self): """Life lines to std::function data members""" @@ -1914,7 +1912,6 @@ def test34_object_pointers(self): assert c.s_strp == "noot" assert sn == "noot" # set through pointer - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test35_restrict(self): """Strip __restrict keyword from use""" diff --git a/test/test_doc_features.py b/test/test_doc_features.py index 70fa02e..fd15cdc 100644 --- a/test/test_doc_features.py +++ b/test/test_doc_features.py @@ -268,7 +268,7 @@ def test_enums(self): pass - @mark.xfail(run=False, condition=IS_MAC, reason="Seg Fault") + @mark.xfail(condition=IS_MAC, run=False, reason="Seg Fault") def test_functions(self): from cppjit.gbl import Namespace, call_int_int_function, global_function @@ -434,9 +434,6 @@ def abstract_method(self): pc = PyConcrete4() assert call_abstract_method(pc) == "Hello, Python World! (4)" - @mark.xfail( - condition=((IS_MAC) and IS_CLANG_REPL), reason="Fails on OSX with Clang-REPL" - ) def test_multi_x_inheritance(self): """Multiple cross-inheritance""" @@ -455,8 +452,8 @@ def abstract_method2(self): assert cppjit.gbl.call_abstract_method2(pc) == "second message" @mark.xfail( - run=False, condition=IS_MAC_ARM, + run=False, reason="Crashes with exception not being caught on Apple Silicon", ) def test_exceptions(self): @@ -583,9 +580,7 @@ def test02_python_introspection(self): assert isinstance(i, Integer1) @mark.xfail( - run=(not IS_MAC and IS_CLANG_REPL), - condition=IS_MAC and IS_CLING, - reason="Crashes on OS X Cling", + condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OS X Cling" ) def test03_STL_containers(self): """Instantiate STL containers with new class""" @@ -674,7 +669,6 @@ def test07_run_zoo(self): assert Zoo.identify_animal(mouse) == "the animal is a mouse" assert Zoo.identify_animal(lion) == "the animal is a lion" - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test08_shared_ptr(self): """Shared pointer transparency""" @@ -893,9 +887,7 @@ def test03_use_of_ctypes_and_enum(self): cppjit.gbl.free(vp) @mark.xfail( - run=(not IS_MAC and IS_CLANG_REPL), - condition=IS_MAC and IS_CLING, - reason="Crashes on OS X Cling", + condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OS X Cling" ) def test04_ptr_ptr_python_owns(self): """Example of ptr-ptr use where python owns""" @@ -1049,7 +1041,7 @@ def test08_voidptr_array(self): assert len(n.p) == 3 @mark.xfail( - condition=(IS_CLANG_REPL and IS_MAC), + condition=IS_CLANG_REPL and IS_MAC, run=False, reason="Crashes with ClangRepl with 'toString not implemented'", ) @@ -1167,7 +1159,7 @@ def test_template_instantiation(self): assert len(v) == 10 assert [m.fData for m in v] == list(range(10)) - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test_cross_inheritance(self): """Cross-inheritance example""" @@ -1187,7 +1179,7 @@ def add(self, i): m = PyMyClass(1) assert CC.callb(m, 2) == 5 - @mark.xfail(run=not IS_MAC_ARM, condition=IS_MAC_ARM, reason="Crashes on OS X arm") + @mark.xfail(condition=IS_MAC_ARM, run=False, reason="Crashes on OS X arm") def test_cross_and_templates(self): """Template instantiation with cross-inheritance example""" @@ -1207,7 +1199,7 @@ def add(self, i): assert v.back().add(17) == 4 + 42 + 2 * 17 - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test_fallbacks(self): """Template instantation switches based on value sizes""" @@ -1226,7 +1218,7 @@ def test_fallbacks(self): assert CC.passT(2**64 - 1) == 2**64 - 1 assert "unsigned long long" in CC.passT.__doc__ - @mark.xfail(run=False, condition=IS_LINUX_ARM, reason="Crashes pytest on Linux ARM") + @mark.xfail(condition=IS_LINUX_ARM, run=False, reason="Crashes pytest on Linux ARM") def test_callbacks(self): """Function callback example""" @@ -1254,8 +1246,8 @@ def f(val): assert CC.callFun(lambda i: 6 * i, 4) == 24 @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM, + run=False, reason="Crashes on Valgrind-ARM", ) def test_templated_callback(self): @@ -1328,7 +1320,6 @@ class MyException : public std::exception { with raises(CC.MyException): CC.throw_error() - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test_unicode(self): """Unicode non-UTF-8 example""" diff --git a/test/test_fragile.py b/test/test_fragile.py index e4cf3bc..631cc7a 100644 --- a/test/test_fragile.py +++ b/test/test_fragile.py @@ -21,6 +21,19 @@ def setup_module(mod): setup_make("fragile") +def has_asan_interface(): + import cppjit + + return ( + cppjit.evaluate("""#if __has_include() + true + #else + false + #endif\n""") + == 1 + ) + + class TestFRAGILE: def setup_class(cls): cls.test_dct = test_dct @@ -500,7 +513,6 @@ def test19_gbl_contents(self): assert "ESysConstants" not in dd assert "kDoRed" not in dd - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test20_capture_output(self): """Capture cerr into a string""" @@ -592,7 +604,10 @@ def test23_set_debug(self): cppjit.set_debug(False) assert cppjit.gbl.Cpp.IsDebugOutputEnabled() == False - @mark.xfail(condition=IS_LINUX, reason="Fails on Ubuntu") + @mark.xfail( + condition=IS_LINUX and not has_asan_interface(), + reason="sanitizer/asan_interface.h not available", + ) def test24_asan(self): """Check availability of ASAN with gcc""" @@ -603,7 +618,7 @@ def test24_asan(self): cppjit.include("sanitizer/asan_interface.h") - @mark.xfail + @mark.xfail(reason="cppdef of invalid code does not raise SyntaxError") def test25_cppdef_error_reporting(self): """Check error reporting of cppjit.cppdef""" diff --git a/test/test_lowlevel.py b/test/test_lowlevel.py index bcc7ac3..7a4be36 100644 --- a/test/test_lowlevel.py +++ b/test/test_lowlevel.py @@ -172,8 +172,8 @@ def test05_array_as_ref(self): assert f[0] == -5.0 @mark.xfail( - run=False, condition=IS_VALGRIND or IS_CLING, + run=False, reason="Valgrind detects memory leak with invalid delete[] operator, crashes on Cling", ) def test06_ctypes_as_ref_and_ptr(self): @@ -502,7 +502,7 @@ def test09_numpy_bool_array(self): x = np.array([True], dtype=bool) assert cppjit.gbl.convert_bool(x) - @mark.xfail(run=False, condition=IS_MAC, reason="Crashes on OSX") + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes on OSX") def test10_array_of_const_char_star(self): """Test passting of const char*[]""" diff --git a/test/test_numba.py b/test/test_numba.py index ee89390..081e4ae 100644 --- a/test/test_numba.py +++ b/test/test_numba.py @@ -491,7 +491,7 @@ def inc_c(d, k): assert c.value == y + k @mark.xfail( - run=False, condition=IS_LINUX_ARM, reason="Crash in llvmlite on Linux ARM" + condition=IS_LINUX_ARM, run=False, reason="Crash in llvmlite on Linux ARM" ) def test12_std_vector_pass_by_ref(self): """Numba-JITing of a method that performs scalar addition to a std::vector initialised through pointers""" diff --git a/test/test_overloads.py b/test/test_overloads.py index f736c8a..c9a8cd7 100644 --- a/test/test_overloads.py +++ b/test/test_overloads.py @@ -73,7 +73,6 @@ def test02_class_based_overloads_explicit_resolution(self): nb = ns_a_overload.b_overload() raises(TypeError, nb.f, c_overload()) - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test03_fragile_class_based_overloads(self): """Test functions overloaded on void* and non-existing classes""" @@ -95,7 +94,6 @@ def test03_fragile_class_based_overloads(self): dd = cppjit.gbl.get_dd_ol() assert more_overloads().call(dd) == "dd_ol" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test04_fully_fragile_overloads(self): """Test that unknown* is preferred over unknown&""" @@ -127,7 +125,6 @@ def test05_array_overloads(self): assert c_overload().get_int(ah) == 25 assert d_overload().get_int(ah) == 25 - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test06_double_int_overloads(self): """Test overloads on int/doubles""" @@ -156,7 +153,6 @@ def test07_mean_overloads(self): a = array.array(l, numbers) assert round(cmean(len(a), a) - mean, 8) == 0 - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test08_const_non_const_overloads(self): """Check selectability of const/non-const overloads""" @@ -215,7 +211,7 @@ def test09_bool_int_overloads(self): with raises(ValueError): cpp.BoolInt4.fff(2) - @mark.xfail(run=not IS_MAC_ARM, condition=IS_MAC, reason="Seg Faults") + @mark.xfail(condition=IS_MAC, run=not IS_MAC_ARM, reason="Seg Faults") def test10_overload_and_exceptions(self): """Prioritize reporting C++ exceptions from callee""" @@ -270,7 +266,6 @@ class MyClass3 { with raises(TypeError): ns.MyClass3("some_file") - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test11_deep_inheritance(self): """Prioritize expected most derived class""" diff --git a/test/test_pythonization.py b/test/test_pythonization.py index 61b0cee..f823546 100644 --- a/test/test_pythonization.py +++ b/test/test_pythonization.py @@ -165,8 +165,8 @@ def test04_transparency(self): assert mine.say_hi() == "Hi!" @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM and IS_CLANG_REPL, + run=False, reason="Crashes on Valgind Clang-Repl-ARM", ) def test05_converters(self): @@ -195,8 +195,8 @@ def test05_converters(self): pz.renew_mine() @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM and IS_CLANG_REPL, + run=False, reason="Fails with Valgrind with Clang-Repl ARM", ) def test06_executors(self): diff --git a/test/test_regression.py b/test/test_regression.py index 638fc50..af96a3b 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -30,7 +30,7 @@ def stringpager(text, title="", cls=cls): pydoc.pager = stringpager - @mark.xfail + @mark.xfail(reason="pydoc rendering of KDcrawIface fails") def test01_kdcraw(self): """Doc strings for KDcrawIface (used to crash).""" @@ -220,7 +220,7 @@ def test07_class_refcounting(self): assert sys.getrefcount(x) == old_refcnt - @mark.xfail(run=False, condition=IS_MAC and IS_CLING, reason="Crahes on OSX-Cling") + @mark.xfail(condition=IS_MAC and IS_CLING, run=False, reason="Crahes on OSX-Cling") def test08_typedef_identity(self): """Nested typedefs should retain identity""" @@ -262,7 +262,7 @@ def test09_gil_not_released(self): cppjit.cppdef(code) cppjit.gbl.some_foo_calling_python() - @mark.xfail(run=False, condition=IS_CLING, reason="Crashes on Cling") + @mark.xfail(condition=IS_CLING, run=False, reason="Crashes on Cling") def test10_enum_in_global_space(self): """Enum declared in search.h did not appear in global space""" @@ -383,7 +383,6 @@ class Bar { f = sds.Foo() assert f.bar.x == 5 - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test15_vector_vs_initializer_list(self): """Prefer vector in template and initializer_list in formal arguments""" @@ -556,7 +555,6 @@ class SignedCharRefGetter { assert obj.getter() == "c" - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test21_temporaries_and_vector(self): """Extend a life line to references into a vector if needed""" @@ -569,7 +567,6 @@ def test21_temporaries_and_vector(self): l = [e for e in cppjit.gbl.get_some_temporary_vector()] assert l == ["x", "y", "z"] - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test22_initializer_list_and_temporary(self): """Conversion rules when selecting intializer_list v.s. temporary""" @@ -824,8 +821,8 @@ def test28_exception_as_shared_ptr(self): assert not null @mark.xfail( - run=False, condition=(IS_CLING and IS_MAC) or IS_MAC_ARM, + run=False, reason="Dispatcher fix #53 introduces canonical types with std:: namespace that introduces OS X exceptions similar to test_stltypes", ) def test29_callback_pointer_values(self): @@ -1055,9 +1052,7 @@ def test34_print_empty_collection(self): v = cppjit.gbl.std.vector[int]() str(v) - @mark.xfail( - run=IS_CLANG_REPL, condition=IS_MAC or IS_CLING, reason="Crashes on Cling" - ) + @mark.xfail(condition=IS_CLING, run=False, reason="Crashes on Cling") def test35_filesytem(self): """Static path object used to crash on destruction""" @@ -1132,7 +1127,7 @@ def test37_array_of_pointers_argument(self): assert cppjit.addressof(res) == cppjit.addressof(arr) @mark.xfail( - run=False, condition=(IS_MAC and IS_CLING), reason="Crashes on OS X Cling" + condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OS X Cling" ) def test38_char16_arrays(self): """Access to fixed-size char16 arrays as data members""" @@ -1194,7 +1189,6 @@ def test38_char16_arrays(self): assert ai.name[:5] == "hello" cppjit.ll.array_delete(aa) - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test39_vector_of_pointers_conversion(self): """vector's const T*& used to be T**, now T*""" @@ -1270,7 +1264,7 @@ def test39_vector_of_pointers_conversion(self): assert type(list(vec2)[0]) == Base2 assert len([d for d in vec3 if isinstance(d, Derived3)]) == 1 - @mark.xfail(run=False, condition=not IS_CLANG_REPL, reason="Crashes with Cling") + @mark.xfail(condition=not IS_CLANG_REPL, run=False, reason="Crashes with Cling") def test40_explicit_initializer_list(self): """Construct and pass an explicit initializer list""" @@ -1439,8 +1433,8 @@ def test45_typedef_resolution(self): assert cppjit.gbl.cppjit.interop.ResolveName("cmy_custom_type_t") == "const int" @mark.xfail( - run=False, condition=IS_MAC_ARM, + run=False, reason="Crashes with exception not being caught on Apple Silicon", ) def test46_exception_narrowing(self): diff --git a/test/test_stltypes.py b/test/test_stltypes.py index 4873977..40795b0 100644 --- a/test/test_stltypes.py +++ b/test/test_stltypes.py @@ -313,7 +313,7 @@ def test01_builtin_type_vector_types(self): assert v.size() == self.N assert len(v) == self.N - @mark.xfail(condition=IS_MAC, run=not IS_MAC, reason="Crashes on OSX") + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes on OSX") def test02_user_type_vector_type(self): """Test access to an std::vector""" @@ -450,9 +450,7 @@ def test06_vector_indexing(self): assert v2[-1] == v[-2] assert v2[self.N - 4] == v[-2] - @mark.xfail( - run=False, condition=(IS_MAC and IS_CLING), reason="Crashes on OSX Cling" - ) + @mark.xfail(condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OSX Cling") def test07_vector_bool(self): """Usability of std::vector which can be a specialization""" @@ -471,7 +469,7 @@ def test07_vector_bool(self): assert len(vb[4:8]) == 4 assert list(vb[4:8]) == [False] * 3 + [True] - @mark.xfail(run=False, condition=IS_MAC and IS_CLING, reason="Crashes on OSX-Cling") + @mark.xfail(condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OSX-Cling") def test08_vector_enum(self): """Usability of std::vector<> of some enums""" @@ -493,9 +491,7 @@ def test08_vector_enum(self): ve[0] = cppjit.gbl.VecTestEnumNS.EVal2 assert ve[0] == 42 - @mark.xfail( - run=not (IS_MAC_ARM or IS_MAC_X86), condition=IS_MAC, reason="Fails on OS X" - ) + @mark.xfail(condition=IS_MAC, run=False, reason="Fails on OS X") def test09_vector_of_string(self): """Adverse effect of implicit conversion on vector""" @@ -596,8 +592,8 @@ def test12_vector_lifeline(self): assert hasattr(val, "__lifeline") @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM and IS_CLANG_REPL, + run=False, reason="Fails with Valgrind with Clang-Repl ARM", ) def test13_vector_smartptr_iteration(self): @@ -633,11 +629,7 @@ def test13_vector_smartptr_iteration(self): i += 1 assert i == len(result) - @mark.xfail( - run=not (IS_MAC and IS_CLING), - condition=(IS_MAC and IS_CLING), - reason="Fails on OSX-Cling", - ) + @mark.xfail(condition=IS_MAC and IS_CLING, run=False, reason="Fails on OSX-Cling") def test14_vector_of_vector_of_(self): """Nested vectors""" @@ -776,7 +768,6 @@ class Point3D { assert cppsum == pysum - @mark.xfail(condition=IS_CLING, reason="Fails on Cling") def test20_vector_cstring(self): """Usage of a vector of const char*""" @@ -993,7 +984,6 @@ def test03_string_with_null_character(self): assert repr(std.string("ab\0c")) == repr(b"ab\0c") assert str(std.string("ab\0c")) == str("ab\0c") - @mark.xfail(condition=IS_MAC, run=False, reason="Fails on OS X") def test04_array_of_strings(self): """Access to global arrays of strings""" @@ -1074,9 +1064,7 @@ def test05_stlstring_and_unicode(self): assert str(uas.get_string_cr(bval)) == "ℕ" assert str(uas.get_string_cc(bval)) == "ℕ" - @mark.xfail( - run=not IS_CLING, condition=IS_MAC or IS_CLING, reason="Fails on OS X and Cling" - ) + @mark.xfail(condition=IS_CLING, run=False, reason="Fails on Cling") def test06_stlstring_bytes_and_text(self): """Mixing of bytes and str""" @@ -1326,7 +1314,7 @@ def test04_iter_of_iter(self): assert a == i i += 1 - @mark.xfail(run=False, condition=IS_MAC and IS_CLING, reason="Crashes on OSX-Cling") + @mark.xfail(condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OSX-Cling") def test05_list_cpp17_style(self): """C++17 style initialization of std::list""" @@ -1894,11 +1882,7 @@ def test02_string_view_from_unicode(self): assert "Lorem ipsum dolor sit amet" in str(text) - @mark.xfail( - run=not IS_MAC, - condition=IS_MAC or IS_CLING, - reason="Crashes on OSX, fails with cling", - ) + @mark.xfail(condition=IS_MAC, run=False, reason="Crashes on OSX") def test03_string_view_pythonize(self): """Pythonization of std::string_view""" @@ -1944,7 +1928,7 @@ def test01_deque_byvalue_regression(self): del x @mark.xfail( - run=False, condition=IS_MAC and IS_CLING, reason="Crashes on OS X Cling" + condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OS X Cling" ) def test02_deque_cpp17_style(self): """C++17 style initialization of std::deque""" @@ -2024,7 +2008,7 @@ def test03_initialize_from_set(self): s = cppjit.gbl.std.set[int](set(["aap", "noot", "mies"])) @mark.xfail( - run=False, condition=IS_MAC and IS_CLING, reason="Crashes with OSX-Cling" + condition=IS_MAC and IS_CLING, run=False, reason="Crashes with OSX-Cling" ) def test04_set_cpp17_style(self): """C++17 style initialization of std::set""" @@ -2259,7 +2243,7 @@ def raiseit(cls): except cppjit.gbl.YourError as e: assert e.what() == "Oops" - @mark.xfail(condition=(IS_MAC_ARM or IS_MAC_X86), reason="Fails with OS X") + @mark.xfail(condition=IS_MAC_ARM or IS_MAC_X86, reason="Fails with OS X") def test03_memory(self): """Memory handling of C++ c// helper for exception base class testing""" @@ -2308,7 +2292,7 @@ def run_raiseit(t1, t2): gc.collect() assert cppjit.gbl.GetMyErrorCount() == 0 - @mark.xfail(run=False, condition=IS_MAC_ARM, reason="Seg Faults on OSX-ARM") + @mark.xfail(condition=IS_MAC_ARM, run=False, reason="Seg Faults on OSX-ARM") def test04_from_cpp(self): """Catch C++ exceptiosn from C++""" @@ -2354,6 +2338,11 @@ def has_cpp_20(): class TestSTLSPAN: import cppjit + def setup_class(cls): + import cppjit + + cppjit.include("span") + def test01_span_iterators(self): """ Test that std::span::begin() and std::span::end() can be used. diff --git a/test/test_streams.py b/test/test_streams.py index 6fc6052..8045382 100644 --- a/test/test_streams.py +++ b/test/test_streams.py @@ -1,6 +1,5 @@ import py -from pytest import mark -from support import IS_MAC, setup_make +from support import setup_make currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("cpp/std_streamsDict")) @@ -34,7 +33,6 @@ def test02_std_cout(self): assert cppjit.gbl.std.cout is not None - @mark.xfail(condition=IS_MAC, reason="Fails on OS X") def test03_consistent_naming_if_char_traits(self): """Naming consistency if char_traits""" diff --git a/test/test_templates.py b/test/test_templates.py index 7f2ad13..127ec7f 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -295,7 +295,7 @@ class RTTest_SomeClassWithTCtor { assert round(RTTest2[int](1, 3.1).m_double - 4.1, 8) == 0.0 assert round(RTTest2[int]().m_double + 1.0, 8) == 0.0 - @mark.xfail(run=False, condition=IS_CLING, reason="Crashes on Cling") + @mark.xfail(condition=IS_CLING, run=False, reason="Crashes on Cling") def test12_template_aliases(self): """Access to templates made available with 'using'""" @@ -472,7 +472,6 @@ def get_tn(ns): b.b_T["int"](1, 1.0, "a") assert get_tn(ns).find("int(some_variadic::B::*)(int&&,double&&,std::") == 0 - @mark.xfail(condition=IS_MAC, reason="Fails on OSX") def test17_empty_body(self): """Use of templated function with empty body""" @@ -617,8 +616,8 @@ def test23_overloaded_setitem(self): v[0] = 1 # used to throw TypeError @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM and IS_CLING, + run=False, reason="Crashes on Valgind Cling-ARM", ) def test24_stdfunction_templated_arguments(self): @@ -648,8 +647,8 @@ def callback(x): assert cppjit.gbl.std.function["double(std::vector)"] @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM, + run=False, reason="Crashes on Valgrind-ARM", ) def test25_stdfunction_ref_and_ptr_args(self): @@ -838,8 +837,8 @@ def test28_enum_in_constructor(self): assert ns.FS("i", ns.ST.TI.I32, ns.FS.R.EQ, 10) @mark.xfail( - run=False, condition=IS_VALGRIND and IS_LINUX_ARM, + run=False, reason="Crashes on Valgrind-ARM", ) def test29_function_ptr_as_template_arg(self): @@ -952,7 +951,7 @@ class Templated: public NonTemplated { ns.Templated() # used to crash - @mark.xfail(run=False, condition=IS_CLING, reason="Crashed with Cling") + @mark.xfail(condition=IS_CLING, run=False, reason="Crashed with Cling") def test31_ltlt_in_template_name(self): """Verify lookup of template names with << in the name""" @@ -1198,7 +1197,7 @@ class TNaVU; getattr(run_n, t) @mark.xfail( - run=False, condition=IS_MAC and IS_CLING, reason="Crashes on OS X + Cling" + condition=IS_MAC and IS_CLING, run=False, reason="Crashes on OS X + Cling" ) def test33_using_template_argument(self): """`using` type as template argument""" @@ -1459,7 +1458,7 @@ def setup_class(cls): cls.templates = cppjit.load_reflection_info(cls.test_dct) - @mark.xfail + @mark.xfail(reason="using-typedef resolution drops non-type template args") def test01_using(self): """Test presence and validity of using typedefs""" From d1f5c8e8cdecc1be2522b87e8c2d01af75f2ebda Mon Sep 17 00:00:00 2001 From: Aaron Jomy <75925957+aaronj0@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:42:44 +0200 Subject: [PATCH 12/15] [ci] Harden the wheels workflow and cibuildwheel config (#59) --- .github/wheel_contents_check.py | 32 +++++++++++++++ .github/workflows/wheels.yml | 69 +++++++++++++++++++++++++++++++++ pyproject.toml | 12 ++++-- zizmor.yml | 9 +++++ 4 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 .github/wheel_contents_check.py create mode 100644 zizmor.yml diff --git a/.github/wheel_contents_check.py b/.github/wheel_contents_check.py new file mode 100644 index 0000000..fb39f0c --- /dev/null +++ b/.github/wheel_contents_check.py @@ -0,0 +1,32 @@ +"""Fail when a wheel holds a file outside the install-layout allowlist. + +Usage: python wheel_contents_check.py [ ...]""" + +import fnmatch +import sys +import zipfile + +# fnmatch's * crosses path separators, so one pattern covers a subtree. +ALLOWED = [ + "cppjit/*.py", + "cppjit/libcppjit.so", + "cppjit/interop/lib/libclangCppInterOp*", + "cppjit/interop/lib/clang/*", + "cppjit/interop/include/*", + "cppjit-*.dist-info/*", +] + + +def check(path): + # directory entries (trailing slash) carry no content + members = [m for m in zipfile.ZipFile(path).namelist() if not m.endswith("/")] + bad = [m for m in members if not any(fnmatch.fnmatch(m, p) for p in ALLOWED)] + for member in bad: + print(f"{path}: unexpected member {member}") + return not bad + + +if __name__ == "__main__": + if len(sys.argv) < 2: + sys.exit(__doc__) + sys.exit(0 if all([check(path) for path in sys.argv[1:]]) else 1) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index a7b3cb4..0ea475c 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -11,6 +11,7 @@ on: paths: - '.github/workflows/wheels.yml' - '.github/wheel_smoke.py' + - '.github/wheel_contents_check.py' - 'pyproject.toml' - 'CMakeLists.txt' - 'cmake/**' @@ -41,6 +42,8 @@ jobs: steps: - uses: actions/checkout@v7 + with: + persist-credentials: false # ref pins the recipe content the cache key is computed from. - uses: compiler-research/ci-workflows/actions/setup-recipe@main @@ -59,23 +62,36 @@ jobs: - uses: pypa/cibuildwheel@v4.2.0 + - name: Assert the build left the checkout clean + run: git diff --exit-code + + - name: Check the wheels against the content allowlist + run: python3 .github/wheel_contents_check.py wheelhouse/*.whl + - uses: actions/upload-artifact@v7 with: name: wheels-${{ matrix.label }} path: wheelhouse/*.whl + if-no-files-found: error sdist: name: sdist runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 + with: + persist-credentials: false - run: pipx run build --sdist + - name: Check the sdist metadata + run: pipx run twine check dist/*.tar.gz + - uses: actions/upload-artifact@v7 with: name: sdist path: dist/*.tar.gz + if-no-files-found: error # Run the full suite on a plain runner, outside the manylinux # container the wheel was built in. @@ -85,6 +101,8 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 + with: + persist-credentials: false - uses: actions/setup-python@v7 with: @@ -112,3 +130,54 @@ jobs: cd test make -j$(nproc) PYTHON=python python -m pytest -ra + + # Build from the sdist and run the full suite against the install. + test-sdist: + name: test sdist (build + full suite) + needs: sdist + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - uses: compiler-research/ci-workflows/actions/setup-recipe@main + id: llvm + with: + recipe: llvm-wheel + version: '21.1.8' + os: ubuntu-24.04 + arch: x86_64 + ref: b760e4c171961786b7b20e2cc514302df5373eef + + - uses: actions/download-artifact@v8 + with: + name: sdist + path: dist + + - name: Install the test suite's native deps + run: sudo apt-get -q update && sudo apt-get -y install libeigen3-dev libboost-dev + + - name: Build and install from the sdist with the test requirements + env: + RECIPE_PATH: ${{ steps.llvm.outputs.path }} + run: > + python -m pip install dist/cppjit-*.tar.gz -v + --config-settings=cmake.define.LLVM_DIR="$RECIPE_PATH/lib/cmake/llvm" + --config-settings=cmake.define.Clang_DIR="$RECIPE_PATH/lib/cmake/clang" + -r requirements.txt + + - name: Smoke the install outside pytest + run: python -X faulthandler .github/wheel_smoke.py + + - name: Run the test suite against the sdist install + env: + CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20 + run: | + cd test + make -j$(nproc) PYTHON=python + python -m pytest -ra diff --git a/pyproject.toml b/pyproject.toml index b9a2821..521cde6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ minimum-version = "build-system.requires" wheel.install-dir = "." wheel.packages = ["python/cppjit"] cmake.build-type = "Release" +sdist.exclude = [".github", ".gitignore", ".clang-format"] [[tool.dynamic-metadata]] provider = "scikit_build_core.metadata.regex" @@ -32,23 +33,28 @@ field = "version" input = "python/cppjit/_version.py" [tool.cibuildwheel] +# cp314t needs a free-threading audit first; cp315 joins at its release. build = ["cp312-*", "cp313-*", "cp314-*"] skip = ["*-musllinux*"] build-verbosity = 1 +audit-requires = ["twine"] +audit-command = "twine check {wheel}" test-sources = ["test", "requirements.txt", ".github/wheel_smoke.py"] test-command = "python .github/wheel_smoke.py" +# imports must resolve from the installed wheel, not the checkout +test-environment = { PYTHONSAFEPATH = "1" } [tool.cibuildwheel.linux] archs = ["x86_64"] manylinux-x86_64-image = "manylinux_2_28" # /opt/llvm is staged on the runner by wheels.yml. -container-engine = { name = "docker", create-args = ["--volume=/opt/llvm:/opt/llvm"] } +container-engine = { name = "docker", create-args = ["--volume=/opt/llvm:/opt/llvm"], disable-host-mount = true } environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/opt/llvm/lib/cmake/clang" } [tool.cibuildwheel.macos] archs = ["arm64"] -before-test = "brew install eigen boost" -test-command = "python -m pip install -r requirements.txt && python .github/wheel_smoke.py && cd test && make -j$(sysctl -n hw.ncpu) PYTHON=python && CPPINTEROP_EXTRA_INTERPRETER_ARGS=-std=c++20 python -m pytest -ra" +before-test = "brew install eigen boost && python -m pip install -r {project}/requirements.txt" +test-command = "python .github/wheel_smoke.py && cd test && make -j$(sysctl -n hw.ncpu) PYTHON=python && CPPINTEROP_EXTRA_INTERPRETER_ARGS=-std=c++20 python -m pytest -ra" environment = { CMAKE_ARGS = "-DLLVM_DIR=/opt/llvm/lib/cmake/llvm -DClang_DIR=/opt/llvm/lib/cmake/clang", MACOSX_DEPLOYMENT_TARGET = "14.0" } [tool.pytest.ini_options] diff --git a/zizmor.yml b/zizmor.yml new file mode 100644 index 0000000..7bb1574 --- /dev/null +++ b/zizmor.yml @@ -0,0 +1,9 @@ +# Version tags for the actions we consume; compiler-research/* rides @main. +rules: + unpinned-uses: + config: + policies: + "actions/*": ref-pin + "pypa/*": ref-pin + "compiler-research/*": ref-pin + "*": hash-pin From 8884e0e45f3fa68e92ad46f2866629d13b811b89 Mon Sep 17 00:00:00 2001 From: keremsahn Date: Sat, 22 Aug 2026 01:30:05 +0300 Subject: [PATCH 13/15] [memory-analysis] Added a field to CPPMethod to store memory-ownership information and this information effects kIsCreator flag of overload group , currently analyzer is not called, just attribute checker is called --- CMakeLists.txt | 4 +- src/cpyrt/CPPMethod.cxx | 8 ++ src/cpyrt/CPPMethod.h | 3 + src/cpyrt/CPPOverload.cxx | 6 ++ src/cpyrt/PyCallable.h | 3 + src/interop/cppjit_interop.h | 3 + src/interop/interop_wrapper.cxx | 5 + test/Makefile | 1 + test/cpp/MemoryOwnership/MemOwnrship.apinotes | 9 ++ .../MemoryOwnership/memory_analysis_redecl.h | 10 ++ test/cpp/MemoryOwnership/module.modulemap | 1 + test/cpp/memory_analysis.cxx | 24 +++++ test/cpp/memory_analysis.h | 36 +++++++ test/test_memoryanalysis.py | 102 ++++++++++++++++++ 14 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 test/cpp/MemoryOwnership/MemOwnrship.apinotes create mode 100644 test/cpp/MemoryOwnership/memory_analysis_redecl.h create mode 100644 test/cpp/MemoryOwnership/module.modulemap create mode 100644 test/cpp/memory_analysis.cxx create mode 100644 test/cpp/memory_analysis.h create mode 100644 test/test_memoryanalysis.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 465d158..96d4bc9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,8 +11,8 @@ include(GNUInstallDirs) # This option won't make a lot of sense since we only ship the shared library in site-packages # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests? option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF) -set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "") -set(CPPINTEROP_GIT_TAG "9802d61921ad5688ae42e4e628d754fc1192244d" CACHE STRING "") +set(CPPINTEROP_GIT_REPOSITORY "https://github.com/keremsahn/CppInterOp.git" CACHE STRING "") +set(CPPINTEROP_GIT_TAG "attr-design" CACHE STRING "") set(CPPINTEROP_SOURCE_DIR "" CACHE PATH "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source") diff --git a/src/cpyrt/CPPMethod.cxx b/src/cpyrt/CPPMethod.cxx index 3fe3e37..61b3c9a 100644 --- a/src/cpyrt/CPPMethod.cxx +++ b/src/cpyrt/CPPMethod.cxx @@ -752,6 +752,14 @@ PyObject* cpyrt::CPPMethod::GetArgDefault(int iarg, bool silent) { bool cpyrt::CPPMethod::IsConst() { return interop::IsConstMethod(GetMethod()); } +//---------------------------------------------------------------------------- +interop::AllocType cpyrt::CPPMethod::GetAllocBehaviour() { + if (fAllocType.has_value()) + return *fAllocType; + interop::AllocType attrResult = interop::IsAllocator(GetMethod()); + fAllocType = attrResult; + return attrResult; +} //---------------------------------------------------------------------------- PyObject* cpyrt::CPPMethod::GetScopeProxy() { // Get or build the scope of this method. diff --git a/src/cpyrt/CPPMethod.h b/src/cpyrt/CPPMethod.h index 54429a2..09ed617 100644 --- a/src/cpyrt/CPPMethod.h +++ b/src/cpyrt/CPPMethod.h @@ -5,6 +5,7 @@ #include "PyCallable.h" // Standard +#include #include #include #include @@ -62,6 +63,7 @@ class CPPMethod : public PyCallable { PyObject* GetCoVarNames() override; PyObject* GetArgDefault(int iarg, bool silent = true) override; bool IsConst() override; + cppjit::interop::AllocType GetAllocBehaviour() override; PyObject* GetScopeProxy() override; interop::TCppFuncAddr_t GetFunctionAddress() override; @@ -116,6 +118,7 @@ class CPPMethod : public PyCallable { protected: // cached value that doubles as initialized flag (uninitialized if -1) int fArgsRequired; + std::optional fAllocType; }; } // namespace cppjit::cpyrt diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 49778df..8a5d729 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -155,6 +155,12 @@ static inline PyObject* HandleReturn(CPPOverload* pymeth, CPPInstance* im_self, CPPInstance* cppres = (CPPInstance*)(CPPInstance_Check(result) ? result : nullptr); + interop::AllocType AT = + pymeth->fMethodInfo->fMethods[0]->GetAllocBehaviour(); + if (AT != interop::AllocType::None && AT != interop::AllocType::Null && + AT != interop::AllocType::Unknown) + pymeth->fMethodInfo->fFlags |= CallContext::kIsCreator; + // if this method creates new objects, always take ownership if (IsCreator(pymeth->fMethodInfo->fFlags)) { diff --git a/src/cpyrt/PyCallable.h b/src/cpyrt/PyCallable.h index 4b79ab0..e1238ab 100644 --- a/src/cpyrt/PyCallable.h +++ b/src/cpyrt/PyCallable.h @@ -37,6 +37,9 @@ class PyCallable { virtual PyObject* GetCoVarNames() = 0; virtual PyObject* GetArgDefault(int /* iarg */, bool silent = true) = 0; virtual bool IsConst() { return false; } + virtual cppjit::interop::AllocType GetAllocBehaviour() { + return cppjit::interop::AllocType::None; + } virtual PyObject* GetScopeProxy() = 0; virtual interop::TCppFuncAddr_t GetFunctionAddress() = 0; diff --git a/src/interop/cppjit_interop.h b/src/interop/cppjit_interop.h index ca7c07e..d38af5e 100644 --- a/src/interop/cppjit_interop.h +++ b/src/interop/cppjit_interop.h @@ -44,6 +44,7 @@ typedef Cpp::FuncRef TCppMethod_t; typedef Cpp::InterpRef TInterp_t; typedef size_t TCppIndex_t; typedef void* TCppFuncAddr_t; +typedef Cpp::AllocType AllocType; // direct interpreter access ------------------------------------------------- RPY_EXPORTED @@ -297,6 +298,8 @@ RPY_EXPORTED std::string GetDoxygenComment(TCppScope_t scope, bool strip_markers = true); RPY_EXPORTED bool IsConstMethod(TCppMethod_t); +RPY_EXPORTED +AllocType IsAllocator(TCppMethod_t); // Templated method/function reflection information // ------------------------------------ RPY_EXPORTED diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 1b5b0eb..71128b5 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -1200,6 +1200,11 @@ interop::TCppType_t interop::GetMethodReturnType(TCppMethod_t method) { return Cpp::GetFunctionReturnType(method); } +interop::AllocType interop::IsAllocator(TCppMethod_t method) { + std::lock_guard Lock(InterOpMutex); + return Cpp::IsAllocator(method); +} + std::string interop::GetMethodReturnTypeAsString(TCppMethod_t method) { std::lock_guard Lock(InterOpMutex); return Cpp::GetTypeAsString( diff --git a/test/Makefile b/test/Makefile index 7f1433e..a9d79c3 100644 --- a/test/Makefile +++ b/test/Makefile @@ -10,6 +10,7 @@ dictnames = advancedcpp \ doc_helper \ example01 \ fragile \ + memory_analysis \ operators \ overloads \ pythonizables \ diff --git a/test/cpp/MemoryOwnership/MemOwnrship.apinotes b/test/cpp/MemoryOwnership/MemOwnrship.apinotes new file mode 100644 index 0000000..2ebd2d0 --- /dev/null +++ b/test/cpp/MemoryOwnership/MemOwnrship.apinotes @@ -0,0 +1,9 @@ +Name: MemOwnrship +Functions: + - Name: memOwnAllocGlobal + SwiftReturnOwnership: cppAllocNew +Tags: + - Name: memOwn + Methods: + - Name: memOwnAllocator + SwiftReturnOwnership: cppAllocNew diff --git a/test/cpp/MemoryOwnership/memory_analysis_redecl.h b/test/cpp/MemoryOwnership/memory_analysis_redecl.h new file mode 100644 index 0000000..d8e9e63 --- /dev/null +++ b/test/cpp/MemoryOwnership/memory_analysis_redecl.h @@ -0,0 +1,10 @@ +#ifndef MEMORY_ANALYSIS_REDECL_H +#define MEMORY_ANALYSIS_REDECL_H +#include "../memory_analysis.h" + +namespace memory { +[[clang::annotate("cppAllocNew")]] +memOwn* allocDefaultMemOwn(); +} + +#endif diff --git a/test/cpp/MemoryOwnership/module.modulemap b/test/cpp/MemoryOwnership/module.modulemap new file mode 100644 index 0000000..123a620 --- /dev/null +++ b/test/cpp/MemoryOwnership/module.modulemap @@ -0,0 +1 @@ +module MemOwnrship { header "../memory_analysis.h" } diff --git a/test/cpp/memory_analysis.cxx b/test/cpp/memory_analysis.cxx new file mode 100644 index 0000000..2dc7f05 --- /dev/null +++ b/test/cpp/memory_analysis.cxx @@ -0,0 +1,24 @@ +#include "memory_analysis.h" + +namespace memory { + +__attribute__((malloc)) memAnalysisKlass* mallocAttr() { + return new memAnalysisKlass; +} + +__attribute__((ownership_returns(malloc))) memAnalysisKlass* +ownershipReturnsAttr() { + return new memAnalysisKlass; +} + +// Expected to not return ownership when analysis is off, and there is just +// attr-check +memAnalysisKlass* noAttr() { return new memAnalysisKlass; } + +memOwn* memOwnAllocGlobal() { return (memOwn*)malloc(sizeof(memOwn)); } + +memOwn* allocDefaultMemOwn() { return new memOwn; } + +memOwn* noAttrAlloc() { return new memOwn; } + +} // namespace memory diff --git a/test/cpp/memory_analysis.h b/test/cpp/memory_analysis.h new file mode 100644 index 0000000..fad0258 --- /dev/null +++ b/test/cpp/memory_analysis.h @@ -0,0 +1,36 @@ +#ifndef MEMORY_ANALYSIS_H +#define MEMORY_ANALYSIS_H + +#include +#include +namespace memory { + +class memAnalysisKlass { +public: + int val; +}; +__attribute__((malloc)) memAnalysisKlass* mallocAttr(); +__attribute__((ownership_returns(malloc))) memAnalysisKlass* +ownershipReturnsAttr(); +memAnalysisKlass* noAttr(); + +struct memOwn { + int val; + memOwn(int value) : val(value) {} + memOwn() { val = 0; } + // Attribute injected by APINotes + static memOwn* memOwnAllocator(int x) { return new memOwn(x); } +}; + +// Attribute injected by APINotes +memOwn* memOwnAllocGlobal(); + +// Attribute injected by redeclaration +memOwn* allocDefaultMemOwn(); + +// No ownership attribute anywhere +memOwn* noAttrAlloc(); + +} // namespace memory + +#endif // MEMORY_ANALYSIS_H diff --git a/test/test_memoryanalysis.py b/test/test_memoryanalysis.py new file mode 100644 index 0000000..646e708 --- /dev/null +++ b/test/test_memoryanalysis.py @@ -0,0 +1,102 @@ +import os +import subprocess +import sys + +import py +from pytest import mark +from support import IS_CLING, setup_make + +currpath = py.path.local(__file__).dirpath() +test_dct = str(currpath.join("cpp/memory_analysisDict")) + +FLAGS = "-fmodules -fimplicit-module-maps -fapinotes-modules" +IN_CHILD = "-fapinotes-modules" in os.getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") + + +def setup_module(mod): + setup_make("memory_analysis") + + +@mark.skipif( + IN_CHILD or IS_CLING, + reason="Cling asserts in collectModuleMaps when built with " + FLAGS, +) +def test00_driver(): + env = os.environ.copy() + env["CPPINTEROP_EXTRA_INTERPRETER_ARGS"] = ( + env.get("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") + " " + FLAGS + ) + subprocess.check_call([sys.executable, "-m", "pytest", __file__], env=env) + + +class TestMEMORYANALYSIS: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cppjit.include("memory_analysis_redecl.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def test01_malloc_attr(self): + import cppjit + + obj = cppjit.gbl.memory.mallocAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert obj.__python_owns__ + + def test02_ownership_returns_attr(self): + import cppjit + + obj = cppjit.gbl.memory.ownershipReturnsAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert obj.__python_owns__ + + def test03_no_attr(self): + import cppjit + + obj = cppjit.gbl.memory.noAttr() + assert type(obj) == cppjit.gbl.memory.memAnalysisKlass + assert not obj.__python_owns__ + obj.__python_owns__ = True + + def test04_redecl_attr(self): + import cppjit + + obj = cppjit.gbl.memory.allocDefaultMemOwn() + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + + def test05_redecl_no_attr(self): + import cppjit + + obj = cppjit.gbl.memory.noAttrAlloc() + assert type(obj) == cppjit.gbl.memory.memOwn + assert not obj.__python_owns__ + obj.__python_owns__ = True + + +@mark.skipif(not IN_CHILD, reason="needs " + FLAGS) +class TestMEMORYANALYSIS_APINOTES: + def setup_class(cls): + cls.test_dct = test_dct + import cppjit + + cppjit.add_include_path(str(currpath.join("cpp", "MemoryOwnership"))) + cppjit.include("../memory_analysis.h") + cls.memory_analysis = cppjit.load_library(cls.test_dct + ".so") + + def test01_apinotes_attr_method(self): + import cppjit + + obj = cppjit.gbl.memory.memOwn.memOwnAllocator(5) + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + + def test02_apinotes_attr_func(self): + import cppjit + + obj = cppjit.gbl.memory.memOwnAllocGlobal() + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ From b2e1b1b281ba50dbcec9d2f705e08eb97856b70a Mon Sep 17 00:00:00 2001 From: keremsahn Date: Sun, 30 Aug 2026 21:10:42 +0300 Subject: [PATCH 14/15] Added usage for memory allocation analyzer in the case there is no memory-related attributes in the FunctionDecl, analyzer is disabled by default and set by cppjit.use_alloc_analyzer(True/False) --- python/cppjit/__init__.py | 6 ++++++ src/cpyrt/CPPMethod.cxx | 6 ++++++ src/cpyrt/cpyrtModule.cxx | 17 +++++++++++++++++ src/interop/cppjit_interop.h | 2 ++ src/interop/interop_wrapper.cxx | 5 +++++ test/cpp/memory_analysis.h | 2 ++ test/test_memoryanalysis.py | 25 +++++++++++++++++++++++++ 7 files changed, 63 insertions(+) diff --git a/python/cppjit/__init__.py b/python/cppjit/__init__.py index 6280d6e..b024415 100644 --- a/python/cppjit/__init__.py +++ b/python/cppjit/__init__.py @@ -48,6 +48,7 @@ "add_library_path", # add a path to search for libraries "add_autoload_map", # explicitly include an autoload map "set_debug", # enable/disable debug output + "use_alloc_analyzer", # enable/disable memory ownership analyzer ] import ctypes @@ -397,6 +398,11 @@ def set_debug(enable=True): gbl.Cpp.EnableDebugOutput(enable) +def use_alloc_analyzer(enable=True): + """Enable/disable memory ownership analyzer""" + _backend.UseAllocAnalyzer(enable) + + def _get_name(tt): if isinstance(tt, str): return tt diff --git a/src/cpyrt/CPPMethod.cxx b/src/cpyrt/CPPMethod.cxx index 61b3c9a..1d52adf 100644 --- a/src/cpyrt/CPPMethod.cxx +++ b/src/cpyrt/CPPMethod.cxx @@ -34,6 +34,7 @@ extern PyObject* gBusException; extern PyObject* gSegvException; extern PyObject* gIllException; extern PyObject* gAbrtException; +extern bool gUseAllocAnalyzer; } // namespace cppjit::cpyrt //- public helper ------------------------------------------------------------ @@ -757,6 +758,11 @@ interop::AllocType cpyrt::CPPMethod::GetAllocBehaviour() { if (fAllocType.has_value()) return *fAllocType; interop::AllocType attrResult = interop::IsAllocator(GetMethod()); + if (attrResult == interop::AllocType::Unknown && gUseAllocAnalyzer) { + interop::AllocType analyzeResult = interop::GetAllocType(GetMethod()); + fAllocType = analyzeResult; + return analyzeResult; + } fAllocType = attrResult; return attrResult; } diff --git a/src/cpyrt/cpyrtModule.cxx b/src/cpyrt/cpyrtModule.cxx index b2c0733..4b16ffe 100644 --- a/src/cpyrt/cpyrtModule.cxx +++ b/src/cpyrt/cpyrtModule.cxx @@ -279,6 +279,7 @@ PyObject* gAbrtException = nullptr; std::unordered_set gPinnedTypes; std::ostringstream gCapturedError; std::streambuf* gOldErrorBuffer = nullptr; +bool gUseAllocAnalyzer = false; std::unordered_map>& pythonizations() { static std::unordered_map> pyzMap; @@ -1012,6 +1013,20 @@ static PyObject* EndCaptureStderr(PyObject*, PyObject*) { return Py_BuildValue("s", capturedError.c_str()); } + +//---------------------------------------------------------------------------- +static PyObject* UseAllocAnalyzer(PyObject*, PyObject* args) { + // Set allocation-analyzer policy, disabled by default + // Usage: enabling ->SetUseAllocAnalyzer(True) / SetUseAllocAnalyzer(1) + // disabling ->SetUseAllocAnalyzer(False) / SetUseAllocAnalyzer(0) + int enable = 0; + if (!PyArg_ParseTuple(args, const_cast("p"), &enable)) + return nullptr; + + gUseAllocAnalyzer = enable; + + Py_RETURN_NONE; +} } // unnamed namespace //- data ----------------------------------------------------------------------- @@ -1061,6 +1076,8 @@ static PyMethodDef gcpyrtMethods[] = { METH_NOARGS, (char*)"Begin capturing stderr to a in memory buffer."}, {(char*)"_end_capture_stderr", (PyCFunction)EndCaptureStderr, METH_NOARGS, (char*)"End capturing stderr and returns the captured buffer."}, + {(char*)"UseAllocAnalyzer", (PyCFunction)UseAllocAnalyzer, METH_VARARGS, + (char*)"Enable/disable memory-allocation analyzer."}, {nullptr, nullptr, 0, nullptr}}; struct module_state { diff --git a/src/interop/cppjit_interop.h b/src/interop/cppjit_interop.h index d38af5e..15da55f 100644 --- a/src/interop/cppjit_interop.h +++ b/src/interop/cppjit_interop.h @@ -300,6 +300,8 @@ RPY_EXPORTED bool IsConstMethod(TCppMethod_t); RPY_EXPORTED AllocType IsAllocator(TCppMethod_t); +RPY_EXPORTED +AllocType GetAllocType(TCppMethod_t); // Templated method/function reflection information // ------------------------------------ RPY_EXPORTED diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 71128b5..3d951ae 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -1205,6 +1205,11 @@ interop::AllocType interop::IsAllocator(TCppMethod_t method) { return Cpp::IsAllocator(method); } +interop::AllocType interop::GetAllocType(TCppMethod_t method) { + std::lock_guard Lock(InterOpMutex); + return Cpp::GetAllocType(method); +} + std::string interop::GetMethodReturnTypeAsString(TCppMethod_t method) { std::lock_guard Lock(InterOpMutex); return Cpp::GetTypeAsString( diff --git a/test/cpp/memory_analysis.h b/test/cpp/memory_analysis.h index fad0258..77aaf87 100644 --- a/test/cpp/memory_analysis.h +++ b/test/cpp/memory_analysis.h @@ -31,6 +31,8 @@ memOwn* allocDefaultMemOwn(); // No ownership attribute anywhere memOwn* noAttrAlloc(); +inline memAnalysisKlass* allocAnalyzerOn() { return new memAnalysisKlass; } +inline memAnalysisKlass* allocAnalyzerOff() { return new memAnalysisKlass; } } // namespace memory #endif // MEMORY_ANALYSIS_H diff --git a/test/test_memoryanalysis.py b/test/test_memoryanalysis.py index 646e708..4805eab 100644 --- a/test/test_memoryanalysis.py +++ b/test/test_memoryanalysis.py @@ -76,6 +76,31 @@ def test05_redecl_no_attr(self): assert not obj.__python_owns__ obj.__python_owns__ = True + def test06_analyzer_on(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + obj = cppjit.gbl.memory.allocAnalyzerOn() + assert obj.__python_owns__ + + def test07_analyzer_off(self): + import cppjit + + cppjit.use_alloc_analyzer(False) + obj = cppjit.gbl.memory.allocAnalyzerOff() + assert not (obj.__python_owns__) + + def test08_analyzer_off_but_cache(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + obj = cppjit.gbl.memory.allocAnalyzerOn() + assert obj.__python_owns__ + + cppjit.use_alloc_analyzer(False) + obj2 = cppjit.gbl.memory.allocAnalyzerOn() + assert obj2.__python_owns__ + @mark.skipif(not IN_CHILD, reason="needs " + FLAGS) class TestMEMORYANALYSIS_APINOTES: From eba937ecfd3b72c5cabb3691aa40ff467df8051d Mon Sep 17 00:00:00 2001 From: keremsahn Date: Tue, 1 Sep 2026 14:01:10 +0300 Subject: [PATCH 15/15] [memory-ownership] Run the deallocator matching AllocType, not always delete AllocType was collapsed into kIsCreator in HandleReturn and never reached op_dealloc_nofree, so malloc'd, new[]'d and raw operator-new'd memory were all freed with a scalar delete. Carry it on the instance as three flags saying what to do: kIsArrayAlloc, kIsNoConstruct, kIsMalloc. HandleReturn sets them when taking ownership; op_dealloc_nofree branches on them to delete[], free, ::operator delete or ::operator delete[]. No destructor runs for the raw cases. All flags clear keeps the old delete path, so constructors and explicit __python_owns__ are unaffected. delete[] goes through CppInterOp's dtor wrapper, so interop::Destruct just stops pinning its count to 0; the rest are plain host-side calls. Fixed two fixtures that claimed malloc via attributes but called new, added coverage for the new paths plus a destructor counter, and exposed read-only __is_array_alloc__ / __is_no_construct__ / __is_malloc__ for the tests. Verified under valgrind: no mismatched free or memory leak. Also manual ownership setting is added to test07. --- src/cpyrt/CPPInstance.cxx | 33 +++- src/cpyrt/CPPInstance.h | 34 ++-- src/cpyrt/CPPOverload.cxx | 22 ++- src/interop/cppjit_interop.h | 2 +- src/interop/interop_wrapper.cxx | 5 +- test/cpp/MemoryOwnership/MemOwnrship.apinotes | 4 +- test/cpp/memory_analysis.cxx | 27 ++- test/cpp/memory_analysis.h | 29 +++- test/support.py | 8 + test/test_memoryanalysis.py | 164 +++++++++++++++++- 10 files changed, 295 insertions(+), 33 deletions(-) diff --git a/src/cpyrt/CPPInstance.cxx b/src/cpyrt/CPPInstance.cxx index 59cde8a..a6784da 100644 --- a/src/cpyrt/CPPInstance.cxx +++ b/src/cpyrt/CPPInstance.cxx @@ -16,6 +16,7 @@ using namespace cppjit; // Standard #include +#include #include //- data _____________________________________________________________________ @@ -226,7 +227,17 @@ void cpyrt::op_dealloc_nofree(CPPInstance* pyobj) { if (pyobj->fFlags & CPPInstance::kIsValue) { interop::CallDestructor(klass, cppobj); interop::Deallocate(klass, cppobj); - } else + } else if (pyobj->fFlags & CPPInstance::kIsMalloc) + std::free(cppobj); + else if (pyobj->fFlags & CPPInstance::kIsNoConstruct) { + if (pyobj->fFlags & CPPInstance::kIsArrayAlloc) + ::operator delete[](cppobj); + else + ::operator delete(cppobj); + } else if (pyobj->fFlags & CPPInstance::kIsArrayAlloc) + interop::Destruct(klass, cppobj, 1); + // Default case: just kIsOwner set in all of memory-ownership flags + else interop::Destruct(klass, cppobj); } cppobj = nullptr; @@ -954,10 +965,30 @@ static int op_setownership(CPPInstance* pyobj, PyObject* value, void*) { return 0; } +// Added for testing purposes +//----------------------------------------------------------------------------- +static PyObject* op_get_array_alloc(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsArrayAlloc)); +} +//----------------------------------------------------------------------------- +static PyObject* op_get_no_construct(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsNoConstruct)); +} +//----------------------------------------------------------------------------- +static PyObject* op_get_malloc(CPPInstance* pyobj, void*) { + return PyBool_FromLong((long)(pyobj->fFlags & CPPInstance::kIsMalloc)); +} //----------------------------------------------------------------------------- static PyGetSetDef op_getset[] = { {(char*)"__python_owns__", (getter)op_getownership, (setter)op_setownership, (char*)"If true, python manages the life time of this object", nullptr}, + {(char*)"__is_array_alloc__", (getter)op_get_array_alloc, nullptr, + (char*)"If true, the object was allocated with new[]/operator new[]", + nullptr}, + {(char*)"__is_no_construct__", (getter)op_get_no_construct, nullptr, + (char*)"If true, the memory is raw: no constructor was run", nullptr}, + {(char*)"__is_malloc__", (getter)op_get_malloc, nullptr, + (char*)"If true, the object was allocated with malloc", nullptr}, {(char*)nullptr, nullptr, nullptr, nullptr, nullptr}}; //= cpyrt type number stubs to allow dynamic overrides ===================== diff --git a/src/cpyrt/CPPInstance.h b/src/cpyrt/CPPInstance.h index 27febdc..4a2de1b 100644 --- a/src/cpyrt/CPPInstance.h +++ b/src/cpyrt/CPPInstance.h @@ -27,21 +27,25 @@ typedef std::vector> CI_DatamemberCache_t; class CPPInstance { public: enum EFlags { - kDefault = 0x0000, - kNoWrapConv = 0x0001, // use type as-is (eg. no smart ptr wrap) - kIsOwner = 0x0002, // Python instance owns C++ object/memory - kIsExtended = 0x0004, // has extended data - kIsValue = 0x0008, // was created from a by-value return - kIsReference = 0x0010, // represents one indirection - kIsArray = 0x0020, // represents an array of objects - kIsSmartPtr = 0x0040, // is or embeds a smart pointer - kIsPtrPtr = 0x0080, // represents two indirections - kIsRValue = 0x0100, // can be used as an r-value - kIsLValue = 0x0200, // can be used as an l-value - kNoMemReg = 0x0400, // do not register with memory regulator - kIsRegulated = 0x0800, // is registered with memory regulator - kIsActual = 0x1000, // has been downcasted to actual type - kHasLifeLine = 0x2000, // has a life line set + kDefault = 0x00000, + kNoWrapConv = 0x00001, // use type as-is (eg. no smart ptr wrap) + kIsOwner = 0x00002, // Python instance owns C++ object/memory + kIsExtended = 0x00004, // has extended data + kIsValue = 0x00008, // was created from a by-value return + kIsReference = 0x00010, // represents one indirection + kIsArray = 0x00020, // represents an array of objects + kIsSmartPtr = 0x00040, // is or embeds a smart pointer + kIsPtrPtr = 0x00080, // represents two indirections + kIsRValue = 0x00100, // can be used as an r-value + kIsLValue = 0x00200, // can be used as an l-value + kNoMemReg = 0x00400, // do not register with memory regulator + kIsRegulated = 0x00800, // is registered with memory regulator + kIsActual = 0x01000, // has been downcasted to actual type + kHasLifeLine = 0x02000, // has a life line set + kIsArrayAlloc = 0x04000, // represents a heap allocated array of objects + kIsNoConstruct = + 0x08000, // represents constructor is not called in the allocation + kIsMalloc = 0x10000, // is allocated with malloc }; public: // public, as the python C-API works with C structs diff --git a/src/cpyrt/CPPOverload.cxx b/src/cpyrt/CPPOverload.cxx index 8a5d729..0fedf05 100644 --- a/src/cpyrt/CPPOverload.cxx +++ b/src/cpyrt/CPPOverload.cxx @@ -171,8 +171,28 @@ static inline PyObject* HandleReturn(CPPOverload* pymeth, CPPInstance* im_self, } // ... or be a regular method with an object proxy return value - else if (cppres) + else if (cppres) { cppres->PythonOwns(); + // After giving ownership, set proper flags to indicate allocation + // method/func + switch (AT) { + case interop::AllocType::Malloc: + cppres->fFlags |= CPPInstance::kIsMalloc; + break; + case interop::AllocType::NewArr: + cppres->fFlags |= CPPInstance::kIsArrayAlloc; + break; + case interop::AllocType::OperatorNew: + cppres->fFlags |= CPPInstance::kIsNoConstruct; + break; + case interop::AllocType::OperatorNewArr: + cppres->fFlags |= CPPInstance::kIsArrayAlloc; + cppres->fFlags |= CPPInstance::kIsNoConstruct; + break; + default: + break; + } + } } // if this new object falls inside self, make sure its lifetime is proper diff --git a/src/interop/cppjit_interop.h b/src/interop/cppjit_interop.h index 15da55f..6f996d0 100644 --- a/src/interop/cppjit_interop.h +++ b/src/interop/cppjit_interop.h @@ -133,7 +133,7 @@ void Deallocate(TCppScope_t scope, TCppObject_t instance); RPY_EXPORTED TCppObject_t Construct(TCppScope_t scope, void* arena = nullptr); RPY_EXPORTED -void Destruct(TCppScope_t scope, TCppObject_t instance); +void Destruct(TCppScope_t scope, TCppObject_t instance, size_t count = 0); // method/function dispatching ----------------------------------------------- RPY_EXPORTED diff --git a/src/interop/interop_wrapper.cxx b/src/interop/interop_wrapper.cxx index 3d951ae..b494304 100644 --- a/src/interop/interop_wrapper.cxx +++ b/src/interop/interop_wrapper.cxx @@ -817,10 +817,11 @@ interop::TCppObject_t interop::Construct(TCppScope_t scope, return Cpp::Construct(scope, arena, /*count=*/1); } -void interop::Destruct(TCppScope_t scope, TCppObject_t instance) { +void interop::Destruct(TCppScope_t scope, TCppObject_t instance, + size_t count /*=0*/) { std::lock_guard Lock( InterOpMutex); // TODO: this shouldn't locks the JIT call - Cpp::Destruct(instance, scope, true, /*count=*/0); + Cpp::Destruct(instance, scope, true, count); } static inline bool copy_args(Parameter* args, size_t nargs, void** vargs) { diff --git a/test/cpp/MemoryOwnership/MemOwnrship.apinotes b/test/cpp/MemoryOwnership/MemOwnrship.apinotes index 2ebd2d0..397d4f9 100644 --- a/test/cpp/MemoryOwnership/MemOwnrship.apinotes +++ b/test/cpp/MemoryOwnership/MemOwnrship.apinotes @@ -1,7 +1,7 @@ Name: MemOwnrship Functions: - - Name: memOwnAllocGlobal - SwiftReturnOwnership: cppAllocNew + - Name: memOwnOperatorNew + SwiftReturnOwnership: cppAllocOperatorNew Tags: - Name: memOwn Methods: diff --git a/test/cpp/memory_analysis.cxx b/test/cpp/memory_analysis.cxx index 2dc7f05..f0a5759 100644 --- a/test/cpp/memory_analysis.cxx +++ b/test/cpp/memory_analysis.cxx @@ -2,23 +2,44 @@ namespace memory { +int memOwn::dtorCount = 0; + +memOwn::memOwn(int value) : val(value) {} + +memOwn::memOwn() { val = 0; } + +memOwn::~memOwn() { ++dtorCount; } + __attribute__((malloc)) memAnalysisKlass* mallocAttr() { - return new memAnalysisKlass; + return (memAnalysisKlass*)malloc(sizeof(memAnalysisKlass)); } __attribute__((ownership_returns(malloc))) memAnalysisKlass* ownershipReturnsAttr() { - return new memAnalysisKlass; + return (memAnalysisKlass*)malloc(sizeof(memAnalysisKlass)); } // Expected to not return ownership when analysis is off, and there is just // attr-check memAnalysisKlass* noAttr() { return new memAnalysisKlass; } -memOwn* memOwnAllocGlobal() { return (memOwn*)malloc(sizeof(memOwn)); } +memOwn* memOwnOperatorNew() { return (memOwn*)::operator new(sizeof(memOwn)); } memOwn* allocDefaultMemOwn() { return new memOwn; } memOwn* noAttrAlloc() { return new memOwn; } +memOwn* allocOperatorNewArrAttr(size_t size) { + return (memOwn*)::operator new[](sizeof(memOwn) * size); +} + +memOwn* allocNewArrAttr(int count) { return new memOwn[count]; } + +memOwn* allocMallocAttr(size_t size) { + return (memOwn*)malloc(sizeof(memOwn) * size); +} + +memOwn* allocOperatorNewAttr() { + return (memOwn*)::operator new(sizeof(memOwn)); +} } // namespace memory diff --git a/test/cpp/memory_analysis.h b/test/cpp/memory_analysis.h index 77aaf87..a6626a6 100644 --- a/test/cpp/memory_analysis.h +++ b/test/cpp/memory_analysis.h @@ -16,14 +16,16 @@ memAnalysisKlass* noAttr(); struct memOwn { int val; - memOwn(int value) : val(value) {} - memOwn() { val = 0; } + static int dtorCount; + memOwn(int value); + memOwn(); // Attribute injected by APINotes static memOwn* memOwnAllocator(int x) { return new memOwn(x); } + ~memOwn(); }; // Attribute injected by APINotes -memOwn* memOwnAllocGlobal(); +memOwn* memOwnOperatorNew(); // Attribute injected by redeclaration memOwn* allocDefaultMemOwn(); @@ -33,6 +35,27 @@ memOwn* noAttrAlloc(); inline memAnalysisKlass* allocAnalyzerOn() { return new memAnalysisKlass; } inline memAnalysisKlass* allocAnalyzerOff() { return new memAnalysisKlass; } +inline memOwn* allocOperatorNewArr(size_t size) { + return (memOwn*)::operator new[](sizeof(memOwn) * size); +} +inline memOwn* allocNewArr(int count) { return new memOwn[count]; } +inline memOwn* allocMalloc(size_t size) { + return (memOwn*)malloc(sizeof(memOwn) * size); +} +inline memOwn* allocOperatorNew() { + return (memOwn*)::operator new(sizeof(memOwn)); +} +[[clang::annotate("cppAllocOperatorNewArr")]] +memOwn* allocOperatorNewArrAttr(size_t size); + +[[clang::annotate("cppAllocNewArr")]] +memOwn* allocNewArrAttr(int count); + +[[clang::annotate("cppAllocMalloc")]] +memOwn* allocMallocAttr(size_t size); + +[[clang::annotate("cppAllocOperatorNew")]] +memOwn* allocOperatorNewAttr(); } // namespace memory #endif // MEMORY_ANALYSIS_H diff --git a/test/support.py b/test/support.py index de2532d..922db67 100644 --- a/test/support.py +++ b/test/support.py @@ -115,4 +115,12 @@ def setup_make(targetname): #endif\n""") == 1 ) +IS_CLANG_LT_22 = ( + cppjit.evaluate("""#if __clang_major__ < 22 + true + #else + false + #endif\n""") + == 1 +) IS_VALGRIND = True if os.getenv("IS_VALGRIND") else False diff --git a/test/test_memoryanalysis.py b/test/test_memoryanalysis.py index 4805eab..251c111 100644 --- a/test/test_memoryanalysis.py +++ b/test/test_memoryanalysis.py @@ -4,7 +4,7 @@ import py from pytest import mark -from support import IS_CLING, setup_make +from support import IS_CLANG_LT_22, IS_CLING, setup_make currpath = py.path.local(__file__).dirpath() test_dct = str(currpath.join("cpp/memory_analysisDict")) @@ -12,6 +12,11 @@ FLAGS = "-fmodules -fimplicit-module-maps -fapinotes-modules" IN_CHILD = "-fapinotes-modules" in os.getenv("CPPINTEROP_EXTRA_INTERPRETER_ARGS", "") +skip_if_inline_from_module = mark.skipif( + IN_CHILD and IS_CLANG_LT_22, + reason="LLVM < 22 does not emit inline definitions that come from a module", +) + def setup_module(mod): setup_make("memory_analysis") @@ -45,6 +50,7 @@ def test01_malloc_attr(self): obj = cppjit.gbl.memory.mallocAttr() assert type(obj) == cppjit.gbl.memory.memAnalysisKlass assert obj.__python_owns__ + assert obj.__is_malloc__ def test02_ownership_returns_attr(self): import cppjit @@ -52,6 +58,7 @@ def test02_ownership_returns_attr(self): obj = cppjit.gbl.memory.ownershipReturnsAttr() assert type(obj) == cppjit.gbl.memory.memAnalysisKlass assert obj.__python_owns__ + assert obj.__is_malloc__ def test03_no_attr(self): import cppjit @@ -64,9 +71,18 @@ def test03_no_attr(self): def test04_redecl_attr(self): import cppjit - obj = cppjit.gbl.memory.allocDefaultMemOwn() - assert type(obj) == cppjit.gbl.memory.memOwn - assert obj.__python_owns__ + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocDefaultMemOwn() + assert type(obj) == cppjit.gbl.memory.memOwn + assert obj.__python_owns__ + assert not obj.__is_malloc__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 1 def test05_redecl_no_attr(self): import cppjit @@ -76,6 +92,12 @@ def test05_redecl_no_attr(self): assert not obj.__python_owns__ obj.__python_owns__ = True + # Setting only python_owns, intends object is allocated with new + assert not obj.__is_malloc__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + + @skip_if_inline_from_module def test06_analyzer_on(self): import cppjit @@ -83,13 +105,16 @@ def test06_analyzer_on(self): obj = cppjit.gbl.memory.allocAnalyzerOn() assert obj.__python_owns__ + @skip_if_inline_from_module def test07_analyzer_off(self): import cppjit cppjit.use_alloc_analyzer(False) obj = cppjit.gbl.memory.allocAnalyzerOff() assert not (obj.__python_owns__) + obj.__python_owns__ = True + @skip_if_inline_from_module def test08_analyzer_off_but_cache(self): import cppjit @@ -101,6 +126,134 @@ def test08_analyzer_off_but_cache(self): obj2 = cppjit.gbl.memory.allocAnalyzerOn() assert obj2.__python_owns__ + @skip_if_inline_from_module + def test09_allocwith_operator_newarr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocOperatorNewArr(5) + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + @skip_if_inline_from_module + def test10_allocwith_newarr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocNewArr(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 5 + + @skip_if_inline_from_module + def test11_allocwith_malloc(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocMalloc(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert obj.__is_malloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + @skip_if_inline_from_module + def test12_allocwith_operator_new(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocOperatorNew() + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert not obj.__is_malloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test13_allocwith_operator_newarr_attr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocOperatorNewArrAttr(5) + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test14_allocwith_newarr_attr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocNewArrAttr(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert obj.__is_array_alloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 5 + + def test15_allocwith_malloc_attr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocMallocAttr(5) + assert obj.__python_owns__ + assert not obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert obj.__is_malloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + + def test16_allocwith_operator_new_attr(self): + import cppjit + + cppjit.use_alloc_analyzer(True) + cppjit.gbl.memory.memOwn.dtorCount = 0 + + def alloc(): + obj = cppjit.gbl.memory.allocOperatorNewAttr() + assert obj.__python_owns__ + assert obj.__is_no_construct__ + assert not obj.__is_array_alloc__ + assert not obj.__is_malloc__ + + alloc() + assert cppjit.gbl.memory.memOwn.dtorCount == 0 + @mark.skipif(not IN_CHILD, reason="needs " + FLAGS) class TestMEMORYANALYSIS_APINOTES: @@ -122,6 +275,7 @@ def test01_apinotes_attr_method(self): def test02_apinotes_attr_func(self): import cppjit - obj = cppjit.gbl.memory.memOwnAllocGlobal() + obj = cppjit.gbl.memory.memOwnOperatorNew() assert type(obj) == cppjit.gbl.memory.memOwn assert obj.__python_owns__ + assert obj.__is_no_construct__