diff --git a/README/ReleaseNotes/v642/index.md b/README/ReleaseNotes/v642/index.md index e83165d9f748d..a281ac5303589 100644 --- a/README/ReleaseNotes/v642/index.md +++ b/README/ReleaseNotes/v642/index.md @@ -217,6 +217,28 @@ The header X3DBuffer.h is no longer part of the installed ROOT headers. ## Build, Configuration and Testing +### Building the CUDA backend of RooFit separately + +RooFit evaluates its models with backend libraries that `libRooBatchCompute` +loads at runtime, one of which, `libRooBatchCompute_CUDA`, is the only part of +ROOT that requires the CUDA toolkit. It is now possible to build that backend +on its own against an already installed ROOT, so that distributions can ship a +CUDA-free ROOT and provide the GPU backend as a separate package: + +```bash +cmake -S /roofit/batchcompute -B build -DCMAKE_PREFIX_PATH= +cmake --build build +cmake --install build +``` + +`roofit/batchcompute/CMakeLists.txt` doubles as the top-level `CMakeLists.txt` +of that standalone project, so no CMake code has to be maintained downstream. +By default the library is installed into the library directory of the ROOT +installation it was configured against, which is where RooFit looks for it. See +`roofit/batchcompute/README.md` for the details. Nothing changes for the regular +ROOT build: `-Dcuda=ON` still builds the CUDA backend together with everything +else. + ## Versions of built-in packages The version of the following packages has been updated: diff --git a/roofit/batchcompute/CMakeLists.txt b/roofit/batchcompute/CMakeLists.txt index b9b45820987ef..9a56c07016dbe 100644 --- a/roofit/batchcompute/CMakeLists.txt +++ b/roofit/batchcompute/CMakeLists.txt @@ -1,4 +1,96 @@ # Library which powers fast batch computations in Roofit. +# +# This file is used in two different ways: +# +# 1. It is pulled in with add_subdirectory() by the regular ROOT build. It +# then defines libRooBatchCompute and all its backends: the CPU ones +# always, and the CUDA one if ROOT was configured with -Dcuda=ON. +# +# 2. It can be used directly as the top-level CMakeLists.txt of a standalone +# project that builds *only* the CUDA backend, libRooBatchCompute_CUDA, +# against an already installed ROOT. This makes it possible to distribute a +# CUDA-free ROOT and to ship the GPU backend as a separate package, without +# having to maintain a copy of this file downstream. See README.md in this +# directory for the exact invocation. +# +# The two modes share the same source lists and the same target configuration, +# so there is only one place where the list of files is maintained. + +if(CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + + cmake_minimum_required(VERSION 3.20 FATAL_ERROR) + + project(RooBatchComputeCUDA LANGUAGES CXX) + + set(RooBatchCompute_STANDALONE_CUDA TRUE) + + # The standalone project exists to build the CUDA backend, nothing else. + set(cuda ON) + + # Default to an optimized build, like the ROOT build does. Without this the + # kernels would be compiled without any optimization flags at all. + if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release CACHE STRING + "Specifies the build type on single-configuration generators" FORCE) + endif() + + find_package(ROOT REQUIRED CONFIG) + + if(NOT TARGET ROOT::RooBatchCompute) + message(FATAL_ERROR + "The ROOT installation in ${ROOT_DIR} does not provide libRooBatchCompute. " + "It was most likely configured with -Droofit=OFF, in which case the CUDA " + "backend is of no use. Point CMAKE_PREFIX_PATH or ROOT_DIR at a ROOT " + "installation that includes RooFit.") + endif() + + if(TARGET ROOT::RooBatchCompute_CUDA) + message(WARNING + "The ROOT installation in ${ROOT_DIR} was configured with -Dcuda=ON and " + "already provides libRooBatchCompute_CUDA. Installing this project with the " + "default RooBatchCompute_CUDA_INSTALL_LIBDIR will overwrite it.") + endif() + + # Match the language standard of the ROOT installation we are extending. The + # C++ standard is also propagated by the imported ROOT targets, but the CUDA + # standard has to be set explicitly. + if(NOT DEFINED CMAKE_CXX_STANDARD) + set(CMAKE_CXX_STANDARD ${ROOT_CXX_STANDARD}) + endif() + if(NOT DEFINED CMAKE_CUDA_STANDARD) + set(CMAKE_CUDA_STANDARD ${CMAKE_CXX_STANDARD}) + endif() + + enable_language(CUDA) + + # libRooBatchCompute_CUDA is loaded at runtime by the RooBatchCompute + # dispatcher with gSystem->Load("libRooBatchCompute_CUDA"), so it has to end + # up in a directory that is part of ROOT's dynamic library search path. By + # default we therefore install next to the ROOT installation we were + # configured against. Relative paths are interpreted with respect to + # CMAKE_INSTALL_PREFIX, which is what package managers want to use. + # The cache entries are of type STRING and not PATH on purpose: CMake resolves + # relative values of PATH cache entries given on the command line against the + # current working directory, which is not what we want here. + set(RooBatchCompute_CUDA_INSTALL_LIBDIR "${ROOT_LIBRARY_DIR}" CACHE STRING + "Directory to install libRooBatchCompute_CUDA into. It must be in ROOT's dynamic library search path.") + set(RooBatchCompute_CUDA_INSTALL_BINDIR "${ROOT_BINDIR}" CACHE STRING + "Directory to install RooBatchCompute_CUDA.dll into (Windows only).") + + # Make sure the library finds libRooBatchCompute and libCore next to itself, + # and also in the ROOT installation it was linked against. + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + if(APPLE) + list(APPEND CMAKE_INSTALL_RPATH "@loader_path") + else() + list(APPEND CMAKE_INSTALL_RPATH "$ORIGIN") + endif() + + message(STATUS "Building libRooBatchCompute_CUDA standalone against ROOT ${ROOT_VERSION} in ${ROOT_LIBRARY_DIR}") + +endif() + +if(NOT RooBatchCompute_STANDALONE_CUDA) ROOT_LINKER_LIBRARY(RooBatchCompute src/Initialisation.cxx @@ -64,9 +156,32 @@ if (ROOT_PLATFORM MATCHES "linux|macosx" AND CMAKE_SYSTEM_PROCESSOR MATCHES x86_ endif() # vector versions of library +endif() # NOT RooBatchCompute_STANDALONE_CUDA + if (cuda) set(shared_object_sources_cu src/RooBatchCompute.cu src/ComputeFunctions.cu src/CudaInterface.cu) - ROOT_LINKER_LIBRARY(RooBatchCompute_CUDA ${shared_object_sources_cu} TYPE SHARED DEPENDENCIES RooBatchCompute) + + if(RooBatchCompute_STANDALONE_CUDA) + # No ROOT build tree around us, so ROOT_LINKER_LIBRARY() and the variables + # it relies on are not available. Create the library by hand, with the same + # name and layout, and link it against the installed libRooBatchCompute + # (which owns the RooBatchCompute::dispatchCUDA pointer that the static + # initializer of this library overwrites when it is loaded). + add_library(RooBatchCompute_CUDA SHARED ${shared_object_sources_cu}) + # ROOT prefixes its libraries with "lib" on every platform, Windows + # included, and that is the name gSystem->Load() is called with. + set_target_properties(RooBatchCompute_CUDA PROPERTIES PREFIX lib IMPORT_PREFIX lib) + target_include_directories(RooBatchCompute_CUDA + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/res ${CMAKE_CURRENT_SOURCE_DIR}/src) + target_link_libraries(RooBatchCompute_CUDA PUBLIC ROOT::RooBatchCompute) + install(TARGETS RooBatchCompute_CUDA + LIBRARY DESTINATION ${RooBatchCompute_CUDA_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${RooBatchCompute_CUDA_INSTALL_LIBDIR} + RUNTIME DESTINATION ${RooBatchCompute_CUDA_INSTALL_BINDIR}) + else() + ROOT_LINKER_LIBRARY(RooBatchCompute_CUDA ${shared_object_sources_cu} TYPE SHARED DEPENDENCIES RooBatchCompute) + endif() + target_compile_options(RooBatchCompute_CUDA PRIVATE -lineinfo --expt-relaxed-constexpr) if(NOT CMAKE_VERSION VERSION_LESS "3.23.0") target_sources( diff --git a/roofit/batchcompute/README.md b/roofit/batchcompute/README.md new file mode 100644 index 0000000000000..67b353d2acd1b --- /dev/null +++ b/roofit/batchcompute/README.md @@ -0,0 +1,78 @@ +# RooBatchCompute + +`libRooBatchCompute` provides the vectorized/parallelized computation kernels +that RooFit uses to evaluate its models. The actual kernels do not live in +`libRooBatchCompute` itself: they are compiled into one *backend* library per +target architecture, and `libRooBatchCompute` loads the appropriate one at +runtime with `gSystem->Load()`: + +| Backend library | Loaded by | +|--------------------------------|------------------------------------| +| `libRooBatchCompute_GENERIC` | `RooBatchCompute::initCPU()` | +| `libRooBatchCompute_SSE4.1` | `RooBatchCompute::initCPU()` | +| `libRooBatchCompute_AVX` | `RooBatchCompute::initCPU()` | +| `libRooBatchCompute_AVX2` | `RooBatchCompute::initCPU()` | +| `libRooBatchCompute_AVX512` | `RooBatchCompute::initCPU()` | +| `libRooBatchCompute_CUDA` | `RooBatchCompute::initCUDA()` | + +Because the backends are plugins that are looked up by name in ROOT's dynamic +library search path, they do not have to be built together with the rest of +ROOT. This is in particular useful for the CUDA backend: it is the only part of +ROOT that needs the CUDA toolkit, and distributions may not want to make all of +ROOT depend on it. + +## Building only the CUDA backend against an installed ROOT + +The `CMakeLists.txt` in this directory doubles as the top-level `CMakeLists.txt` +of a standalone project that builds nothing but `libRooBatchCompute_CUDA`. Point +CMake at this directory instead of at the top of the ROOT source tree: + +```bash +cmake -S /roofit/batchcompute -B rbc_cuda_build \ + -DCMAKE_PREFIX_PATH= \ + -DCMAKE_CUDA_ARCHITECTURES= +cmake --build rbc_cuda_build -j$(nproc) +cmake --install rbc_cuda_build +``` + +Notes: + +* The ROOT sources have to be the ones the installed ROOT was built from. The + backend and `libRooBatchCompute` share the `RooBatchComputeInterface` ABI, + which is not stable across ROOT versions. +* `CMAKE_PREFIX_PATH` can be omitted if `thisroot.sh` has been sourced. Instead + of the installation prefix, `ROOT_DIR` can be pointed directly at the + directory that contains `ROOTConfig.cmake` (`/cmake` in + ROOT's default install layout, but distributions often move it). Note that a + `ROOT_DIR` that does not exist is silently ignored by `find_package()`, which + then falls back to any other ROOT it can find, for instance one that is in + `PATH`; check the `Building libRooBatchCompute_CUDA standalone against ROOT + ...` line that the configuration step prints. +* `CMAKE_CUDA_ARCHITECTURES` can be omitted, in which case CMake compiles for + whatever architecture the CUDA compiler defaults to, just like the regular + ROOT build does. +* If `CMAKE_BUILD_TYPE` is not given, it defaults to `Release`, as in the ROOT + build. +* The build only needs a CUDA compiler, the ROOT headers and `libRooBatchCompute` + from the ROOT installation. No dictionaries are generated and no part of ROOT + is rebuilt. +* By default the library is installed straight into the library directory of the + ROOT installation that was found (`${ROOT_LIBRARY_DIR}`), which is where + `RooBatchCompute::initCUDA()` will look for it. To stage it somewhere else, + for example when building a package, set + `-DRooBatchCompute_CUDA_INSTALL_LIBDIR=lib` — a relative path is interpreted + with respect to `CMAKE_INSTALL_PREFIX`, so the usual + `CMAKE_INSTALL_PREFIX`/`DESTDIR` mechanisms apply. Wherever the library ends + up, that directory has to be in ROOT's dynamic library search path (i.e. in + `$ROOTSYS/lib`, in `LD_LIBRARY_PATH`, or added to `Unix.*.Root.DynamicPath` in + `.rootrc`). + +The ROOT installation itself does not need to know anything about the CUDA +backend: it is enough that `libRooBatchCompute_CUDA` is findable. RooFit only +attempts to load it when a computation is requested on the CUDA backend, so a +ROOT installation without the library keeps working as before. + +## Building everything at once + +Nothing changes for the regular ROOT build: configuring ROOT with `-Dcuda=ON` +builds `libRooBatchCompute_CUDA` alongside the CPU backends, exactly as before.