[RF][CMake] Allow building RooBatchCompute_CUDA standalone

README/ReleaseNotes/v642/index.md

@@ -217,6 +217,28 @@ The header X3DBuffer.h is no longer part of the installed ROOT headers.
 
 ## Build, Configuration and Testing
 
+### Building the CUDA backend of RooFit separately
+
+RooFit evaluates its models with backend libraries that `libRooBatchCompute`
+loads at runtime, one of which, `libRooBatchCompute_CUDA`, is the only part of
+ROOT that requires the CUDA toolkit. It is now possible to build that backend
+on its own against an already installed ROOT, so that distributions can ship a
+CUDA-free ROOT and provide the GPU backend as a separate package:
+
+```bash
+cmake -S <root-source-dir>/roofit/batchcompute -B build -DCMAKE_PREFIX_PATH=<root-install-prefix>
+cmake --build build
+cmake --install build
+```
+
+`roofit/batchcompute/CMakeLists.txt` doubles as the top-level `CMakeLists.txt`
+of that standalone project, so no CMake code has to be maintained downstream.
+By default the library is installed into the library directory of the ROOT
+installation it was configured against, which is where RooFit looks for it. See
+`roofit/batchcompute/README.md` for the details. Nothing changes for the regular
+ROOT build: `-Dcuda=ON` still builds the CUDA backend together with everything
+else.
+
 ## Versions of built-in packages
 
 The version of the following packages has been updated:

roofit/batchcompute/CMakeLists.txt

@@ -1,4 +1,96 @@
 # Library which powers fast batch computations in Roofit.
+#
+# This file is used in two different ways:
+#
+#   1. It is pulled in with add_subdirectory() by the regular ROOT build. It
+#      then defines libRooBatchCompute and all its backends: the CPU ones
+#      always, and the CUDA one if ROOT was configured with -Dcuda=ON.
+#
+#   2. It can be used directly as the top-level CMakeLists.txt of a standalone
+#      project that builds *only* the CUDA backend, libRooBatchCompute_CUDA,
+#      against an already installed ROOT. This makes it possible to distribute a
+#      CUDA-free ROOT and to ship the GPU backend as a separate package, without
+#      having to maintain a copy of this file downstream. See README.md in this
+#      directory for the exact invocation.
+#
+# The two modes share the same source lists and the same target configuration,
+# so there is only one place where the list of files is maintained.
+
+if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+
+  cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
+
+  project(RooBatchComputeCUDA LANGUAGES CXX)
+
+  set(RooBatchCompute_STANDALONE_CUDA TRUE)
+
+  # The standalone project exists to build the CUDA backend, nothing else.
+  set(cuda ON)
+
+  # Default to an optimized build, like the ROOT build does. Without this the
+  # kernels would be compiled without any optimization flags at all.
+  if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING
+        "Specifies the build type on single-configuration generators" FORCE)
+  endif()
+
+  find_package(ROOT REQUIRED CONFIG)
+
+  if(NOT TARGET ROOT::RooBatchCompute)
+    message(FATAL_ERROR
+        "The ROOT installation in ${ROOT_DIR} does not provide libRooBatchCompute. "
+        "It was most likely configured with -Droofit=OFF, in which case the CUDA "
+        "backend is of no use. Point CMAKE_PREFIX_PATH or ROOT_DIR at a ROOT "
+        "installation that includes RooFit.")
+  endif()
+
+  if(TARGET ROOT::RooBatchCompute_CUDA)
+    message(WARNING
+        "The ROOT installation in ${ROOT_DIR} was configured with -Dcuda=ON and "
+        "already provides libRooBatchCompute_CUDA. Installing this project with the "
+        "default RooBatchCompute_CUDA_INSTALL_LIBDIR will overwrite it.")
+  endif()
+
+  # Match the language standard of the ROOT installation we are extending. The
+  # C++ standard is also propagated by the imported ROOT targets, but the CUDA
+  # standard has to be set explicitly.
+  if(NOT DEFINED CMAKE_CXX_STANDARD)
+    set(CMAKE_CXX_STANDARD ${ROOT_CXX_STANDARD})
+  endif()
+  if(NOT DEFINED CMAKE_CUDA_STANDARD)
+    set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD})
+  endif()
+
+  enable_language(CUDA)
+
+  # libRooBatchCompute_CUDA is loaded at runtime by the RooBatchCompute
+  # dispatcher with gSystem->Load("libRooBatchCompute_CUDA"), so it has to end
+  # up in a directory that is part of ROOT's dynamic library search path. By
+  # default we therefore install next to the ROOT installation we were
+  # configured against. Relative paths are interpreted with respect to
+  # CMAKE_INSTALL_PREFIX, which is what package managers want to use.
+  # The cache entries are of type STRING and not PATH on purpose: CMake resolves
+  # relative values of PATH cache entries given on the command line against the
+  # current working directory, which is not what we want here.
+  set(RooBatchCompute_CUDA_INSTALL_LIBDIR "${ROOT_LIBRARY_DIR}" CACHE STRING
+      "Directory to install libRooBatchCompute_CUDA into. It must be in ROOT's dynamic library search path.")
+  set(RooBatchCompute_CUDA_INSTALL_BINDIR "${ROOT_BINDIR}" CACHE STRING
+      "Directory to install RooBatchCompute_CUDA.dll into (Windows only).")
+
+  # Make sure the library finds libRooBatchCompute and libCore next to itself,
+  # and also in the ROOT installation it was linked against.
+  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+  if(APPLE)
+    list(APPEND CMAKE_INSTALL_RPATH "@loader_path")
+  else()
+    list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN")
+  endif()
+
+  message(STATUS "Building libRooBatchCompute_CUDA standalone against ROOT ${ROOT_VERSION} in ${ROOT_LIBRARY_DIR}")
+
+endif()
+
+if(NOT RooBatchCompute_STANDALONE_CUDA)
 
 ROOT_LINKER_LIBRARY(RooBatchCompute
     src/Initialisation.cxx
@@ -64,9 +156,32 @@ if (ROOT_PLATFORM MATCHES "linux|macosx" AND CMAKE_SYSTEM_PROCESSOR MATCHES x86_
 
 endif() # vector versions of library
 
+endif() # NOT RooBatchCompute_STANDALONE_CUDA
+
 if (cuda)
   set(shared_object_sources_cu src/RooBatchCompute.cu src/ComputeFunctions.cu src/CudaInterface.cu)
-  ROOT_LINKER_LIBRARY(RooBatchCompute_CUDA  ${shared_object_sources_cu} TYPE SHARED DEPENDENCIES RooBatchCompute)
+
+  if(RooBatchCompute_STANDALONE_CUDA)
+    # No ROOT build tree around us, so ROOT_LINKER_LIBRARY() and the variables
+    # it relies on are not available. Create the library by hand, with the same
+    # name and layout, and link it against the installed libRooBatchCompute
+    # (which owns the RooBatchCompute::dispatchCUDA pointer that the static
+    # initializer of this library overwrites when it is loaded).
+    add_library(RooBatchCompute_CUDA SHARED ${shared_object_sources_cu})
+    # ROOT prefixes its libraries with "lib" on every platform, Windows
+    # included, and that is the name gSystem->Load() is called with.
+    set_target_properties(RooBatchCompute_CUDA PROPERTIES PREFIX lib IMPORT_PREFIX lib)
+    target_include_directories(RooBatchCompute_CUDA
+        PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/res ${CMAKE_CURRENT_SOURCE_DIR}/src)
+    target_link_libraries(RooBatchCompute_CUDA PUBLIC ROOT::RooBatchCompute)
+    install(TARGETS RooBatchCompute_CUDA
+            LIBRARY DESTINATION ${RooBatchCompute_CUDA_INSTALL_LIBDIR}
+            ARCHIVE DESTINATION ${RooBatchCompute_CUDA_INSTALL_LIBDIR}
+            RUNTIME DESTINATION ${RooBatchCompute_CUDA_INSTALL_BINDIR})
+  else()
+    ROOT_LINKER_LIBRARY(RooBatchCompute_CUDA  ${shared_object_sources_cu} TYPE SHARED DEPENDENCIES RooBatchCompute)
+  endif()
+
   target_compile_options(RooBatchCompute_CUDA  PRIVATE -lineinfo --expt-relaxed-constexpr)
   if(NOT CMAKE_VERSION VERSION_LESS "3.23.0")
     target_sources(

roofit/batchcompute/README.md

@@ -0,0 +1,78 @@
+# RooBatchCompute
+
+`libRooBatchCompute` provides the vectorized/parallelized computation kernels
+that RooFit uses to evaluate its models. The actual kernels do not live in
+`libRooBatchCompute` itself: they are compiled into one *backend* library per
+target architecture, and `libRooBatchCompute` loads the appropriate one at
+runtime with `gSystem->Load()`:
+
+| Backend library                | Loaded by                          |
+|--------------------------------|------------------------------------|
+| `libRooBatchCompute_GENERIC`   | `RooBatchCompute::initCPU()`       |
+| `libRooBatchCompute_SSE4.1`    | `RooBatchCompute::initCPU()`       |
+| `libRooBatchCompute_AVX`       | `RooBatchCompute::initCPU()`       |
+| `libRooBatchCompute_AVX2`      | `RooBatchCompute::initCPU()`       |
+| `libRooBatchCompute_AVX512`    | `RooBatchCompute::initCPU()`       |
+| `libRooBatchCompute_CUDA`      | `RooBatchCompute::initCUDA()`      |
+
+Because the backends are plugins that are looked up by name in ROOT's dynamic
+library search path, they do not have to be built together with the rest of
+ROOT. This is in particular useful for the CUDA backend: it is the only part of
+ROOT that needs the CUDA toolkit, and distributions may not want to make all of
+ROOT depend on it.
+
+## Building only the CUDA backend against an installed ROOT
+
+The `CMakeLists.txt` in this directory doubles as the top-level `CMakeLists.txt`
+of a standalone project that builds nothing but `libRooBatchCompute_CUDA`. Point
+CMake at this directory instead of at the top of the ROOT source tree:
+
+```bash
+cmake -S <root-source-dir>/roofit/batchcompute -B rbc_cuda_build \
+      -DCMAKE_PREFIX_PATH=<root-install-prefix> \
+      -DCMAKE_CUDA_ARCHITECTURES=<architectures>
+cmake --build rbc_cuda_build -j$(nproc)
+cmake --install rbc_cuda_build
+```
+
+Notes:
+
+* The ROOT sources have to be the ones the installed ROOT was built from. The
+  backend and `libRooBatchCompute` share the `RooBatchComputeInterface` ABI,
+  which is not stable across ROOT versions.
+* `CMAKE_PREFIX_PATH` can be omitted if `thisroot.sh` has been sourced. Instead
+  of the installation prefix, `ROOT_DIR` can be pointed directly at the
+  directory that contains `ROOTConfig.cmake` (`<root-install-prefix>/cmake` in
+  ROOT's default install layout, but distributions often move it). Note that a
+  `ROOT_DIR` that does not exist is silently ignored by `find_package()`, which
+  then falls back to any other ROOT it can find, for instance one that is in
+  `PATH`; check the `Building libRooBatchCompute_CUDA standalone against ROOT
+  ...` line that the configuration step prints.
+* `CMAKE_CUDA_ARCHITECTURES` can be omitted, in which case CMake compiles for
+  whatever architecture the CUDA compiler defaults to, just like the regular
+  ROOT build does.
+* If `CMAKE_BUILD_TYPE` is not given, it defaults to `Release`, as in the ROOT
+  build.
+* The build only needs a CUDA compiler, the ROOT headers and `libRooBatchCompute`
+  from the ROOT installation. No dictionaries are generated and no part of ROOT
+  is rebuilt.
+* By default the library is installed straight into the library directory of the
+  ROOT installation that was found (`${ROOT_LIBRARY_DIR}`), which is where
+  `RooBatchCompute::initCUDA()` will look for it. To stage it somewhere else,
+  for example when building a package, set
+  `-DRooBatchCompute_CUDA_INSTALL_LIBDIR=lib` — a relative path is interpreted
+  with respect to `CMAKE_INSTALL_PREFIX`, so the usual
+  `CMAKE_INSTALL_PREFIX`/`DESTDIR` mechanisms apply. Wherever the library ends
+  up, that directory has to be in ROOT's dynamic library search path (i.e. in
+  `$ROOTSYS/lib`, in `LD_LIBRARY_PATH`, or added to `Unix.*.Root.DynamicPath` in
+  `.rootrc`).
+
+The ROOT installation itself does not need to know anything about the CUDA
+backend: it is enough that `libRooBatchCompute_CUDA` is findable. RooFit only
+attempts to load it when a computation is requested on the CUDA backend, so a
+ROOT installation without the library keeps working as before.
+
+## Building everything at once
+
+Nothing changes for the regular ROOT build: configuring ROOT with `-Dcuda=ON`
+builds `libRooBatchCompute_CUDA` alongside the CPU backends, exactly as before.