From 8c7bafadb0337bb55cc3e98b4237c814150411c3 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 12 Jun 2026 08:43:18 +0200 Subject: [PATCH 1/3] [cppyy] Auto-downcast objects returned through smart pointers Returning an object by raw pointer already triggers an automatic downcast to its actual (most derived) class, but returning it through a smart pointer (`std::unique_ptr`, `std::shared_ptr`) did not: the object was bound as the declared underlying type, so derived-class members were not accessible. This became more and more of a nuisance as smart pointers become more common in C++ interface. Therefore, this commit implements automatic downcasting also for returned smart pointers. Closes #16210. --- .../pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx | 11 +++++ .../pyroot/cppyy/CPyCppyy/src/CPPInstance.h | 1 + .../pyroot/cppyy/CPyCppyy/src/Converters.cxx | 7 ++- .../cppyy/CPyCppyy/src/ProxyWrappers.cxx | 26 +++++++++- .../pyroot/cppyy/cppyy/test/cpp11features.cxx | 12 +++++ .../pyroot/cppyy/cppyy/test/cpp11features.h | 42 +++++++++++++++++ .../cppyy/cppyy/test/test_cpp11features.py | 47 +++++++++++++++++++ 7 files changed, 144 insertions(+), 2 deletions(-) diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx index f5d8e59cf139e..ac702c25769f7 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.cxx @@ -187,6 +187,17 @@ Cppyy::TCppType_t CPyCppyy::CPPInstance::GetSmartIsA() const return SMART_TYPE(this); } +//---------------------------------------------------------------------------- +Cppyy::TCppType_t CPyCppyy::CPPInstance::GetSmartUnderlyingType() const +{ +// The declared underlying type of the embedded smart pointer (e.g. 'Base' for +// a std::unique_ptr). This is independent of any auto-down-cast applied +// to the dereferenced object, and so is what must be used to decide whether the +// smart pointer can be passed to a function expecting a particular smart type. + if (!IsSmart()) return (Cppyy::TCppType_t)0; + return SMART_CLS(this)->fUnderlyingType; +} + //---------------------------------------------------------------------------- CPyCppyy::CI_DatamemberCache_t& CPyCppyy::CPPInstance::GetDatamemberCache() { diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h index e657999327c11..d6b7adcbe350b 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPInstance.h @@ -77,6 +77,7 @@ class CPPInstance { void SetSmart(PyObject* smart_type); void* GetSmartObject() { return GetObjectRaw(); } Cppyy::TCppType_t GetSmartIsA() const; + Cppyy::TCppType_t GetSmartUnderlyingType() const; // cross-inheritance dispatch void SetDispatchPtr(void*); diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx index cf7d27adc84db..d3e472350afe2 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Converters.cxx @@ -3066,7 +3066,12 @@ bool CPyCppyy::SmartPtrConverter::SetArg( } // final option, try mapping pointer types held (TODO: do not allow for non-const ref) - if (pyobj->IsSmart() && Cppyy::IsSubtype(oisa, fUnderlyingType)) { +// Note: this must be decided on the smart pointer's *declared* underlying type, not +// on the (possibly auto-down-cast) type of the dereferenced object. A +// std::unique_ptr holding a Derived must not be accepted where a +// std::unique_ptr is expected: the held smart pointer is still a +// unique_ptr and does not convert to unique_ptr. + if (pyobj->IsSmart() && Cppyy::IsSubtype(pyobj->GetSmartUnderlyingType(), fUnderlyingType)) { para.fValue.fVoidp = ((CPPInstance*)pyobject)->GetSmartObject(); para.fTypeCode = 'V'; return true; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx index 35a76b0552763..6aad4bf43a22e 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/ProxyWrappers.cxx @@ -842,7 +842,31 @@ PyObject* CPyCppyy::BindCppObjectNoCast(Cppyy::TCppObject_t address, PyObject* smart_type = (!(flags & CPPInstance::kNoWrapConv) && \ (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart)) ? pyclass : nullptr; if (smart_type) { - pyclass = CreateScopeProxy(((CPPSmartClass*)smart_type)->fUnderlyingType); + Cppyy::TCppType_t underlying = ((CPPSmartClass*)smart_type)->fUnderlyingType; + + // Down-cast the underlying object to its actual (most derived) class, just + // as BindCppObject does for raw pointers. Two conditions must hold: + // * the reported actual class must really be a subtype of the declared one. + // Cppyy::GetActualClass() can return a *base* class (e.g. when the actual + // class has no dictionary of its own and inherits IsA() from a base with + // ClassDef, ROOT reports that base) -- such an up-cast must be rejected. + // * the cast must require no pointer adjustment, because the smart pointer's + // dereferencer always yields a pointer to the underlying (declared) class, + // so a non-zero offset can not be applied consistently on later member + // access (e.g. with multiple inheritance). + if (address && !isRef) { + void* deref = Cppyy::CallR( + ((CPPSmartClass*)smart_type)->fDereferencer, address, 0, nullptr); + if (deref) { + Cppyy::TCppType_t clActual = Cppyy::GetActualClass(underlying, deref); + if (clActual && clActual != underlying && + Cppyy::IsSubtype(clActual, underlying) && + Cppyy::GetBaseOffset(clActual, underlying, deref, -1 /* down-cast */) == 0) + underlying = clActual; + } + } + + pyclass = CreateScopeProxy(underlying); if (!pyclass) { // simply restore and expose as the actual smart pointer class pyclass = smart_type; diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx index 7319875c30e4e..edbe06978a7e8 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx @@ -40,6 +40,18 @@ TestSmartPtr create_TestSmartPtr_by_value() { return TestSmartPtr{}; } +std::shared_ptr create_shared_ptr_to_derived() { + return std::shared_ptr(new PubDerivedTestSmartPtr); +} + +std::unique_ptr create_unique_ptr_to_derived() { + return std::unique_ptr(new PubDerivedTestSmartPtr); +} + +std::unique_ptr create_unique_ptr_to_offset_derived() { + return std::unique_ptr(new MultiDerivedTestSmartPtr); +} + // for move ctors etc. int TestMoving1::s_move_counter = 0; diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h index 183ce7082e319..1ef186b60f01c 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h @@ -36,6 +36,48 @@ int move_unique_ptr_derived(std::unique_ptr&& p); TestSmartPtr create_TestSmartPtr_by_value(); +// for auto-downcast of objects returned through a smart pointer +class PubDerivedTestSmartPtr : public TestSmartPtr { +public: + int only_in_derived() { return 27; } +}; + +// second base so that the cross-cast to the most derived type needs a +// non-zero pointer adjustment, which the smart pointer's dereferencer can +// not apply consistently (so no down-cast should happen in that case) +class TestSmartPtrIface { +public: + virtual ~TestSmartPtrIface() {} + long m_pad = 0; + int only_in_iface() { return 37; } +}; + +class MultiDerivedTestSmartPtr : public PubDerivedTestSmartPtr, public TestSmartPtrIface { +}; + +std::shared_ptr create_shared_ptr_to_derived(); +std::unique_ptr create_unique_ptr_to_derived(); +std::unique_ptr create_unique_ptr_to_offset_derived(); + +// sinks expecting a smart pointer to the *derived* type; a base-class smart +// pointer (even when its object was auto-down-cast) must not be accepted here +int pass_unique_ptr_to_derived(std::unique_ptr p) { + return p->only_in_derived(); +} + +int pass_shared_ptr_to_derived(std::shared_ptr p) { + return p->only_in_derived(); +} + +// Overloaded function to check if automatic downcasting is consistently +// applied for regular proxy objects and smart pointer proxies. +std::string pass_ptr_overloaded(TestSmartPtr *) { return "TestSmartPtr"; } +std::string pass_ptr_overloaded(PubDerivedTestSmartPtr *) { return "PubDerivedTestSmartPtr"; } +std::string pass_ref_overloaded(TestSmartPtr &) { return "TestSmartPtr"; } +std::string pass_ref_overloaded(PubDerivedTestSmartPtr &) { return "PubDerivedTestSmartPtr"; } +std::string pass_val_overloaded(TestSmartPtr) { return "TestSmartPtr"; } +std::string pass_val_overloaded(PubDerivedTestSmartPtr) { return "PubDerivedTestSmartPtr"; } + //=========================================================================== class TestMoving1 { // for move ctors etc. diff --git a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py index 71c30153dfe02..f659a59437356 100644 --- a/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py +++ b/bindings/pyroot/cppyy/cppyy/test/test_cpp11features.py @@ -593,6 +593,53 @@ def test20_tuple_element(self): cppyy.gbl.std.tuple_element[1, ATuple].type + def test21_smart_ptr_downcast(self): + """Object returned through a smart pointer is auto-downcast""" + + import cppyy + + gbl = cppyy.gbl + + # unique_ptr holding a Derived comes back as Derived, with the + # derived-only method callable, just like a raw pointer return + for cf in [gbl.create_unique_ptr_to_derived, gbl.create_shared_ptr_to_derived]: + obj = cf() + assert type(obj) == gbl.PubDerivedTestSmartPtr + assert obj.only_in_derived() == 27 + assert obj.__smartptr__() # smart-pointer semantics preserved + + # an object that really is of the declared type stays that type + obj = gbl.create_unique_ptr_instance() + assert type(obj) == gbl.TestSmartPtr + + # the most derived type sits at a non-zero offset from the declared + # interface, which the dereferencer can not apply: stay the declared + # type and keep behaving correctly + obj = gbl.create_unique_ptr_to_offset_derived() + assert type(obj) == gbl.TestSmartPtrIface + assert obj.only_in_iface() == 37 + + # the auto-down-cast must not enable C++-invalid conversions: the proxy + # still embeds a smart pointer to the *base* type, which does not convert + # to a smart pointer to the derived type (no implicit down-conversion of + # smart pointers in C++), so passing it to such a sink must be rejected + raises(TypeError, gbl.pass_unique_ptr_to_derived, gbl.create_unique_ptr_to_derived()) + raises(TypeError, gbl.pass_shared_ptr_to_derived, gbl.create_shared_ptr_to_derived()) + + # passing it where the matching base smart pointer is expected still works + assert gbl.pass_shared_ptr(gbl.create_shared_ptr_to_derived()) == 17 + + # calling function with overloads for both the base class and the + # derived class should resolve to the downcasted type overload, + # no matter if the Python proxy is a regular proxy or wraps a smart pointer + # (should hold for pointer, reference, and value types) + assert gbl.pass_ptr_overloaded(gbl.PubDerivedTestSmartPtr()) == "PubDerivedTestSmartPtr" + assert gbl.pass_ptr_overloaded(gbl.create_unique_ptr_to_derived()) == "PubDerivedTestSmartPtr" + assert gbl.pass_ref_overloaded(gbl.PubDerivedTestSmartPtr()) == "PubDerivedTestSmartPtr" + assert gbl.pass_ref_overloaded(gbl.create_unique_ptr_to_derived()) == "PubDerivedTestSmartPtr" + assert gbl.pass_val_overloaded(gbl.PubDerivedTestSmartPtr()) == "PubDerivedTestSmartPtr" + assert gbl.pass_val_overloaded(gbl.create_unique_ptr_to_derived()) == "PubDerivedTestSmartPtr" + if __name__ == "__main__": exit(pytest.main(args=['-sv', '-ra', __file__])) From 6b7ae08034f6ccf8215152925a6597fb28d6f7d9 Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Fri, 12 Jun 2026 08:47:58 +0200 Subject: [PATCH 2/3] [RF] Remove unneeded pointer dereferencing in RooFit tutorial rf617 Now that also objects returned by smart pointer get automatically downcasted to the actual type, we don't need to dereference returned smart pointer objects to trigger the automatic downcasting. --- ...f617_simulation_based_inference_multidimensional.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tutorials/roofit/roofit/rf617_simulation_based_inference_multidimensional.py b/tutorials/roofit/roofit/rf617_simulation_based_inference_multidimensional.py index 46429c22912ae..fc34d2242e9e9 100644 --- a/tutorials/roofit/roofit/rf617_simulation_based_inference_multidimensional.py +++ b/tutorials/roofit/roofit/rf617_simulation_based_inference_multidimensional.py @@ -261,17 +261,9 @@ def learned_likelihood_ratio(*args): nllr_learned.plotOn(frame1, LineColor="kP6Blue", ShiftToZero=True, Name="learned") -# Declare a helper function in ROOT to dereference unique_ptr -ROOT.gInterpreter.Declare( - """ -RooAbsArg &my_deref(std::unique_ptr const& ptr) { return *ptr; } -""" -) - # Choose normalization set for lhr_calc to plot over norm_set = ROOT.RooArgSet(x_vars) -lhr_calc_final_ptr = ROOT.RooFit.Detail.compileForNormSet(lhr_calc, norm_set) -lhr_calc_final = ROOT.my_deref(lhr_calc_final_ptr) +lhr_calc_final = ROOT.RooFit.Detail.compileForNormSet(lhr_calc, norm_set) lhr_calc_final.recursiveRedirectServers(norm_set) # Plot the likelihood ratio functions From 3a4918e979b5c7298e8103c0189f918e3a0e230c Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Mon, 15 Jun 2026 11:34:58 +0200 Subject: [PATCH 3/3] [Python] Document auto-downcasting and restructure pythonizations.md This commit documents the auto-downcasting for return types of C++ functions, and also improves the structure of the pythonization docs as follows: * The term "pythonizations" is very overloaded, so expand the explanation in the beginning a bit * Cleanly separate into explanation of pythonization in ROOT, and later pythonization of use classes. Before, the section on user pythonization was just labeled "Pythonization example", which was confusing to me. Will this be an example of a pythonization in ROOT? * Move the explanation of the user pythonizations to the end, as this is an advanced feature. --- .../pythonizations/doc/pythonizations.md | 190 ++++++++++++------ 1 file changed, 129 insertions(+), 61 deletions(-) diff --git a/bindings/pyroot/pythonizations/doc/pythonizations.md b/bindings/pyroot/pythonizations/doc/pythonizations.md index 88e03eecf596d..6983ea0a7eaa8 100644 --- a/bindings/pyroot/pythonizations/doc/pythonizations.md +++ b/bindings/pyroot/pythonizations/doc/pythonizations.md @@ -2,9 +2,131 @@ \ingroup Python \brief Python-specific functionalities offered by ROOT -This page lists the so-called "pythonizations", that is those functionalities offered by ROOT for classes and functions which are specific to Python usage of the package and provide a more pythonic experience. +This page describes ROOT's *pythonizations*: functionality that makes ROOT's C++ classes and functions behave more naturally when used from Python. Rather than simply reproducing C++ behavior, a pythonization augments how you interact with a C++ type. For instance, letting you use NumPy arrays in place of C++ arrays in function calls, or automatic downcasting of C++ instances to their actual types. -### Pythonization example +Some pythonizations are general and applied automatically to most C++ types, while others apply only to specific ROOT types. Below we explain some of the automatic pythonizations in detail, then show how to implement custom pythonizations for your own C++ classes. + + +### Pretty-printing pythonization +This example illustrates the pretty printing feature of PyROOT, which reveals +the content of the object if a string representation is requested, e.g., by +Python's print statement. The printing behaves similar to the ROOT prompt +powered by the C++ interpreter cling. +Create an object with PyROOT + +~~~{.py} +obj = ROOT.std.vector("int")(3) +for i in range(obj.size()): + obj[i] = i +~~~ + +Print the object, which reveals the content. Note that `print` calls the special +method `__str__` of the object internally. + +~~~{.py} +print(obj) +~~~ + +The output can be retrieved as string by any function that triggers the `__str__` +special method of the object, e.g., `str` or `format`. + +~~~{.py} +print(str(obj)) +print("{}".format(obj)) +~~~ + +Note that the interactive Python prompt does not call `__str__`, it calls +`__repr__`, which implements a formal and unique string representation of +the object. + +~~~{.py} +print(repr(obj)) +obj +~~~ + +The print output behaves similar to the ROOT prompt, e.g., here for a ROOT histogram. + +~~~{.py} +hist = ROOT.TH1F("name", "title", 10, 0, 1) +print(hist) +~~~ + +If cling cannot produce any nice representation for the class, we fall back to a +"" format, which is what `__repr__` returns + +~~~{.py} +ROOT.gInterpreter.Declare('class MyClass {};') +m = ROOT.MyClass() +print(m) +print(str(m) == repr(m)) +~~~ + +### Automatic downcasting pythonization + +In C++, it is possible to use a base-class pointer to refer to an instance of a +derived type. +```cpp +class Base { +public: + virtual ~Base() = default; +}; +class Derived : public Base {}; + +Base *foo() { + static Derived obj; + return &obj; +} +``` +The same is also possible for smart pointers, like `std::unique_ptr` or +`std::shared_ptr`. For example: +```cpp +std::unique_ptr foo_unique() { + return std::unique_ptr{new Derived}; +} +``` +Using the `Derived` interface on the return value is not possible in C++ without +type casting (e.g. a `dynamic_cast`). Since explicit type casting is not natural +in Python, ROOT attempts to automatically downcast raw pointer or smart pointer +return values to their actual type. Demonstrating this with the types above: +```python +p1 = ROOT.foo() +p2 = ROOT.foo_unique() + +# if you absolutely need a base class proxy, there is a way: +p3 = ROOT.bind_object(p1, "Base") + +print(p1) +print(p2) +print(p3) +``` +will give you something like: +```txt + +> + +``` + +**Note 1**: keep in mind that the auto downcasting also affects overload +resolution. For example, consider these two overloads: +```cpp +void consume(Base *) {} // overload 1 +void consume(Derived *) {} // overload 2 +``` +In C++, `consume(foo())` will hit overload 1. In Python, +`ROOT.consume(ROOT.foo())` resolves to the second overload because the pointee +was automatically downcast. If you really need to hit the first `Base` overload, you'll have to explicitly cast back to the base class type with `ROOT.bind_object`, as shown before. + +**Note 2**: while the type of the pointee gets downcast, smart pointer types +remain unchanged. That's because `std::unique_ptr` and +`std::unique_ptr` are distinct, unrelated types. Template +instantiations don't inherit from one another even when their type arguments do. + +**Note 3**: automatic downcasting is not enabled +unconditionally. It is only available for polymorphic base +classes, which is why the `Base` class in the example has a virtual destructor. + + +### Custom pythonizations for C++ user classes by example This example shows how to use the `@pythonization` decorator to add extra behaviour to C++ user classes that are used from Python via PyROOT. @@ -24,17 +146,17 @@ class MyClass {}; ~~~ Next, we define a pythonizor function: the function that will be responsible -for injecting new behaviour in our C++ class `MyClass`. +for injecting new behaviour in our C++ class `MyClass`. To convert a given Python function into a pythonizor, we need to decorate it with the @pythonization decorator. Such decorator allows us to define which which class we want to pythonize by providing its class name and its namespace (if the latter is not specified, it defaults to the global -namespace, i.e. '::'). +namespace, i.e. '::'). The decorated function - the pythonizor - must accept either one or two parameters: 1. The class to be pythonized (proxy object where new behaviour can be injected) -2. The fully-qualified name of that class (optional). +2. The fully-qualified name of that class (optional). Let's see all this with a simple example. Suppose I would like to define how `MyClass` objects are represented as a string in Python (i.e. what would be shown when I print that object). For that purpose, I can define the following @@ -113,7 +235,7 @@ for o in o1, o2: In addition, @pythonization also accepts prefixes of classes in a certain namespace in order to match multiple classes in that namespace. To signal that what we provide to @pythonization is a prefix, we need to set the `is_prefix` -argument to `True` (default is `False`). +argument to `True` (default is `False`). A common case where matching prefixes is useful is when we have a templated class and we want to pythonize all possible instantiations of that template. For example, we can pythonize the `std::vector` (templated) class like so: @@ -199,7 +321,7 @@ first time in the application. However, it can also happen that our target class/es have already been accessed by the time we register a pythonization. In such a scenario, the pythonizor is applied immediately (at registration time) to the target -class/es +class/es Let's see an example of what was just explained. We will define a new class and immediately create an object of that class. We can check how the object still does not have a new attribute `pythonized` that we are going to inject @@ -230,57 +352,3 @@ Now our object does have the `pythonized` attribute: ~~~{.py} print(o.pythonized) # prints True ~~~ - -### Pythonization printing example -This example illustrates the pretty printing feature of PyROOT, which reveals -the content of the object if a string representation is requested, e.g., by -Python's print statement. The printing behaves similar to the ROOT prompt -powered by the C++ interpreter cling. -Create an object with PyROOT - -~~~{.py} -obj = ROOT.std.vector("int")(3) -for i in range(obj.size()): - obj[i] = i -~~~ - -Print the object, which reveals the content. Note that `print` calls the special -method `__str__` of the object internally. - -~~~{.py} -print(obj) -~~~ - -The output can be retrieved as string by any function that triggers the `__str__` -special method of the object, e.g., `str` or `format`. - -~~~{.py} -print(str(obj)) -print("{}".format(obj)) -~~~ - -Note that the interactive Python prompt does not call `__str__`, it calls -`__repr__`, which implements a formal and unique string representation of -the object. - -~~~{.py} -print(repr(obj)) -obj -~~~ - -The print output behaves similar to the ROOT prompt, e.g., here for a ROOT histogram. - -~~~{.py} -hist = ROOT.TH1F("name", "title", 10, 0, 1) -print(hist) -~~~ - -If cling cannot produce any nice representation for the class, we fall back to a -"" format, which is what `__repr__` returns - -~~~{.py} -ROOT.gInterpreter.Declare('class MyClass {};') -m = ROOT.MyClass() -print(m) -print(str(m) == repr(m)) -~~~