cuD-PDLP

.github/workflows/multi_gpu_cpp_test.yaml

@@ -39,6 +39,13 @@ on:
       script:
         type: string
         default: "ci/test_cpp_multi_gpu.sh"
+      container-options:
+        description: |
+          Extra 'docker run' options. NCCL allocates its inter-GPU buffers in
+          /dev/shm; the container default of 64 MB is too small and makes NCCL
+          setup fail with "No space left on device", so bump it with --shm-size.
+        type: string
+        default: "--shm-size=1g"
       continue-on-error:
         description: "Treat failures as non-blocking (useful while the MG suite is maturing)."
         type: boolean
@@ -63,5 +70,6 @@ jobs:
       node_type: ${{ inputs.node_type }}
       arch: ${{ inputs.arch }}
       container_image: ${{ inputs.container_image }}
+      container-options: ${{ inputs.container-options }}
       script: ${{ inputs.script }}
       continue-on-error: ${{ inputs.continue-on-error }}

ci/test_cpp_multi_gpu.sh

@@ -64,6 +64,10 @@ rapids-print-env
 rapids-logger "Check GPU usage"
 nvidia-smi
 
+# Dump the GPU topology matrix
+rapids-logger "Check GPU topology"
+nvidia-smi topo -m
+
 # Multi-GPU tests are meaningless on a single device — fail loudly rather than
 # passing a run that never exercised NCCL.
 GPU_COUNT=$(nvidia-smi -L | wc -l)
@@ -74,6 +78,17 @@ if [ "${GPU_COUNT}" -lt 2 ]; then
   exit 1
 fi
 
+# Distributed PDLP parity tests use a small set of git-ignored MPS instances.
+# Download only those direct-MPS fixtures here: downloading the full PDLP suite
+# falls back to netlib conversion when S3 credentials are not available, and that
+# path needs gcc (not present in this test environment).
+rapids-logger "Download datasets"
+python benchmarks/linear_programming/utils/get_datasets.py -datasets graph40-40
+python benchmarks/linear_programming/utils/get_datasets.py -datasets ex10
+
+RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)"
+export RAPIDS_DATASET_ROOT_DIR
+
 # Locate the installed gtest binaries (same search order as ci/run_ctests.sh).
 installed_test_location="${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/gtests/libcuopt/"
 devcontainers_test_location="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/../cpp/build/latest/gtests/libcuopt/"
@@ -97,6 +112,11 @@ if [ "${#mg_tests[@]}" -eq 0 ]; then
   exit 0
 fi
 
+# PHB topology on the 2-GPU runner: disable direct GPU peer access (see topo above).
+# NCCL then uses the SHM transport, which needs a larger /dev/shm than the 64 MB
+# container default — the workflow launches the container with --shm-size for this.
+export NCCL_P2P_DISABLE=1
+
 EXITCODE=0
 for gt in "${mg_tests[@]}"; do
   test_name=$(basename "${gt}")

ci/validate_wheel.sh

@@ -22,11 +22,11 @@ PYDISTCHECK_ARGS=(
 if [[ "${package_dir}" == "python/libcuopt" ]]; then
     if [[ "${RAPIDS_CUDA_MAJOR}" == "12" ]]; then
         PYDISTCHECK_ARGS+=(
-            --max-allowed-size-compressed '690Mi'
+            --max-allowed-size-compressed '960Mi'
         )
     else
         PYDISTCHECK_ARGS+=(
-            --max-allowed-size-compressed '550Mi'
+            --max-allowed-size-compressed '800Mi'
         )
     fi
 elif [[ "${package_dir}" != "python/cuopt" ]] && \

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -42,6 +42,7 @@ dependencies:
 - msgpack-numpy==0.4.8
 - msgpack-python==1.2.1
 - myst-parser
+- nccl >=2.19
 - ninja
 - notebook
 - numba-cuda>=0.22.1

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -42,6 +42,7 @@ dependencies:
 - msgpack-numpy==0.4.8
 - msgpack-python==1.2.1
 - myst-parser
+- nccl >=2.19
 - ninja
 - notebook
 - numba-cuda>=0.22.1

conda/environments/all_cuda-133_arch-aarch64.yaml

@@ -42,6 +42,7 @@ dependencies:
 - msgpack-numpy==0.4.8
 - msgpack-python==1.2.1
 - myst-parser
+- nccl >=2.19
 - ninja
 - notebook
 - numba-cuda>=0.22.1

conda/environments/all_cuda-133_arch-x86_64.yaml

@@ -42,6 +42,7 @@ dependencies:
 - msgpack-numpy==0.4.8
 - msgpack-python==1.2.1
 - myst-parser
+- nccl >=2.19
 - ninja
 - notebook
 - numba-cuda>=0.22.1

conda/recipes/libcuopt/recipe.yaml

@@ -89,6 +89,7 @@ cache:
       - libcusparse-dev
       - libnvjitlink-dev
       - libboost-devel
+      - nccl >=2.19
       - tbb-devel
       - zlib
       - bzip2
@@ -127,6 +128,7 @@ outputs:
         - libcudss-dev >=0.7,<0.8
         - libcusparse-dev
         - libnvjitlink-dev
+        - nccl >=2.19
         - openssl
         - c-ares
         - libuuid
@@ -141,6 +143,7 @@ outputs:
         - librmm =${{ minor_version }}
         - cuda-nvrtc
         - libcudss >=0.7,<0.8
+        - nccl >=2.19
         - openssl
         - c-ares
         - libuuid
@@ -189,6 +192,7 @@ outputs:
         - libcublas
         - libcudss-dev >=0.7,<0.8
         - libcusparse-dev
+        - nccl >=2.19
         - openssl
         - c-ares
         - libgrpc >=1.78.0,<1.80.0a0

cpp/CMakeLists.txt

@@ -311,6 +311,44 @@ create_logger_macros(CUOPT "cuopt::default_logger()" include/cuopt)
 
 find_package(CUDSS REQUIRED)
 
+# ##################################################################################################
+# - NCCL (multi-GPU distributed PDLP) -------------------------------------------------------------
+# NCCL is shipped via the conda env; no canonical CMake config target, so look it
+# up by name in the standard lib paths (plus CONDA_PREFIX as a hint).
+set(NCCL_HINT_PREFIXES "")
+if (DEFINED ENV{CONDA_PREFIX} AND NOT "$ENV{CONDA_PREFIX}" STREQUAL "")
+    list(APPEND NCCL_HINT_PREFIXES "$ENV{CONDA_PREFIX}")
+endif ()
+find_path(NCCL_INCLUDE_DIR
+    NAMES nccl.h
+    HINTS ${NCCL_HINT_PREFIXES}
+    PATH_SUFFIXES include
+)
+find_library(NCCL_LIBRARY
+    NAMES nccl
+    HINTS ${NCCL_HINT_PREFIXES}
+    PATH_SUFFIXES lib lib64
+)
+if (NOT NCCL_INCLUDE_DIR OR NOT NCCL_LIBRARY)
+    message(FATAL_ERROR "NCCL not found. Looked in ${NCCL_HINT_PREFIXES}. Install nccl-dev / libnccl-dev in the active env.")
+endif ()
+add_library(nccl_external UNKNOWN IMPORTED GLOBAL)
+set_target_properties(nccl_external PROPERTIES
+    IMPORTED_LOCATION "${NCCL_LIBRARY}"
+    INTERFACE_INCLUDE_DIRECTORIES "${NCCL_INCLUDE_DIR}"
+)
+message(STATUS "Using NCCL: ${NCCL_LIBRARY}")
+
+# ##################################################################################################
+# - KaMinPar (multi-threaded partitioning for distributed PDLP) ------------------------------------
+# Brought in the RAPIDS way (rapids_cpm_find): uses an installed KaMinPar (deb/rpm/conda,
+# discovered via its CMake config) if present, otherwise builds the pinned source via CPM.
+include(cmake/thirdparty/get_kaminpar.cmake)
+if (NOT TARGET KaMinPar::KaMinPar)
+    message(FATAL_ERROR "KaMinPar::KaMinPar was not made available by get_kaminpar.cmake")
+endif ()
+message(STATUS "Using KaMinPar (distributed PDLP graph partitioner)")
+
 # ##################################################################################################
 # - gRPC and Protobuf setup -----------------------------------------------------------------------
 
@@ -555,6 +593,32 @@ target_include_directories(cuopt
 target_link_libraries(cuopt PRIVATE $<TARGET_FILE:PSLP>)
 add_dependencies(cuopt PSLP)
 
+# Link KaMinPar by file to avoid export dependency tracking (mirrors PSLP above).
+# KaMinPar is a from-source static library fully embedded into libcuopt.so; it is never
+# installed (INSTALL_KAMINPAR OFF) and consumers of cuopt::cuopt never use it, so it must
+# not leak into cuopt's exported link interface (otherwise rapids_export fails with
+# "target KaMinPar is not in any export set"). libKaMinPar.a is self-contained (the
+# kaminpar-common OBJECT lib is archived into it); we only need its public headers
+# (<kaminpar.h>, which pulls in stdlib + TBB) at compile time.
+target_include_directories(cuopt SYSTEM PRIVATE
+        $<TARGET_PROPERTY:KaMinPar::KaMinPar,INTERFACE_INCLUDE_DIRECTORIES>)
+# partitioner.cpp includes <kaminpar.h>. Because KaMinPar is linked by file, cuopt
+# does NOT inherit KaMinPar's PUBLIC compile definitions, so we must replicate the ones that
+# affect what <kaminpar.h> compiles to:
+#   * TBB_PREVIEW_GLOBAL_CONTROL  - <kaminpar.h> includes <tbb/global_control.h>, which older
+#                                   TBB gates behind this macro.
+#   * KAMINPAR_64BIT_EDGE_IDS     - selects EdgeID = uint64 in <kaminpar.h>. MUST stay in sync
+#                                   with the KAMINPAR_64BIT_* options in get_kaminpar.cmake;
+#                                   otherwise the public API types (e.g. copy_graph's span<>
+#                                   widths) mismatch the prebuilt archive => undefined refs.
+target_compile_definitions(cuopt PRIVATE TBB_PREVIEW_GLOBAL_CONTROL KAMINPAR_64BIT_EDGE_IDS)
+target_link_libraries(cuopt PRIVATE $<TARGET_FILE:KaMinPar::KaMinPar>)
+# Real (non-imported) target only exists when KaMinPar is built from source via CPM;
+# force it to build before cuopt since it is EXCLUDE_FROM_ALL.
+if (TARGET KaMinPar)
+        add_dependencies(cuopt KaMinPar)
+endif ()
+
 # ##################################################################################################
 # - link libraries --------------------------------------------------------------------------------
 
@@ -605,6 +669,7 @@ target_link_libraries(cuopt
         ${CUDSS_LIB_FILE}
         PRIVATE
         ${CUOPT_PRIVATE_CUDA_LIBS}
+        nccl_external
         $<$<BOOL:${CUOPT_ENABLE_GRPC}>:protobuf::libprotobuf>
         $<$<BOOL:${CUOPT_ENABLE_GRPC}>:gRPC::grpc++>
 )

cpp/cmake/thirdparty/get_kaminpar.cmake

@@ -0,0 +1,95 @@
+# cmake-format: off
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+# cmake-format: on
+
+# Multi-threaded graph partitioner for distributed PDLP.
+# Uses rapids_cpm_find so a system / conda / .deb install of KaMinPar (which ships a
+# CMake config package exporting KaMinPar::KaMinPar) is used when available, and
+# otherwise the pinned source is cloned and built via CPM. KaMinPar depends on TBB,
+# which cuOpt already requires (see find_package(TBB) for papilo).
+function(find_and_configure_kaminpar)
+    set(oneValueArgs VERSION PINNED_TAG)
+    cmake_parse_arguments(PKG "" "${oneValueArgs}" "" ${ARGN})
+
+    # Silence the (many) warnings from KaMinPar and its bundled third-party
+    # dependencies (KaHyPar, KaGen, growt, ...). They are compiled from pinned
+    # upstream source we do not maintain, so their diagnostics only drown out
+    # cuOpt's own build output. Appending -w here is function-scoped: the CPM
+    # add_subdirectory below inherits these flags, but cuOpt's targets (compiled
+    # in the parent scope with -Wall -Werror) are unaffected. A system/conda
+    # install of KaMinPar builds nothing, so the flag is simply unused there.
+    string(APPEND CMAKE_C_FLAGS " -w")
+    string(APPEND CMAKE_CXX_FLAGS " -w")
+
+    # NOTE: KaMinPar is intentionally NOT added to cuopt's BUILD/INSTALL export sets.
+    # It is a from-source static dependency that is fully embedded into libcuopt.so and
+    # never installed (INSTALL_KAMINPAR OFF below). Registering it in cuopt-exports would
+    # both break export generation ("target KaMinPar is not in any export set") and emit a
+    # bogus find_dependency(KaMinPar) into the installed cuopt config. It is linked by file
+    # in cpp/CMakeLists.txt (mirroring PSLP) so it stays out of cuopt's export interface.
+    rapids_cpm_find(KaMinPar ${PKG_VERSION}
+        GLOBAL_TARGETS KaMinPar::KaMinPar
+        CPM_ARGS
+        GIT_REPOSITORY https://github.com/KaHIP/KaMinPar.git
+        GIT_TAG ${PKG_PINNED_TAG}
+        EXCLUDE_FROM_ALL
+        OPTIONS
+            "KAMINPAR_BUILD_APPS OFF"
+            "KAMINPAR_BUILD_TOOLS OFF"
+            "KAMINPAR_BUILD_TESTS OFF"
+            "KAMINPAR_BUILD_BENCHMARKS OFF"
+            "KAMINPAR_BUILD_EXAMPLES OFF"
+            "KAMINPAR_BUILD_DISTRIBUTED OFF"
+            # KaMinPar defaults this to ON, which injects `-march=native -mtune=native`
+            # (and propagates the same flag to its bundled KaHyPar / KaGen via
+            # KAHYPAR_ENABLE_ARCH_COMPILE_OPTIMIZATIONS / KAGEN_USE_MARCH_NATIVE).
+            # That bakes in whatever ISA the build host happens to support
+            # (e.g. AVX-512 on Sapphire Rapids), which then SIGILLs on any test
+            # host with a leaner CPU. Force generic codegen instead.
+            "KAMINPAR_BUILD_WITH_MTUNE_NATIVE OFF"
+            # Timers use global state and force single-threaded use of the library
+            # interface; disable so cuOpt can call the partitioner freely.
+            "KAMINPAR_ENABLE_TIMERS OFF"
+            # Avoid an extra hard dependency on Google Sparsehash.
+            "KAMINPAR_BUILD_WITH_SPARSEHASH OFF"
+            # cuOpt's TBB is discovered via a legacy find that only exposes TBB::tbb
+            # (no TBB::tbbmalloc target); disable KaMinPar's optional tbbmalloc use.
+            "KAMINPAR_ENABLE_TBB_MALLOC OFF"
+            # Large LP constraint graphs can exceed 2^31 directed edges.
+            "KAMINPAR_64BIT_EDGE_IDS ON"
+            "INSTALL_KAMINPAR OFF"
+            # Build KaMinPar as a STATIC library that is embedded into libcuopt.so (linked
+            # by file in cpp/CMakeLists.txt). The wheel build configures with
+            # BUILD_SHARED_LIBS=ON; without this override KaMinPar would build a separate
+            # libKaMinPar.so that is neither embedded nor shipped in the wheel. Forcing PIC
+            # is required so the static objects can be linked into the shared libcuopt.so
+            # (KaMinPar's KaMinParCommon OBJECT lib otherwise lacks -fPIC).
+            "BUILD_SHARED_LIBS OFF"
+            "CMAKE_POSITION_INDEPENDENT_CODE ON"
+    )
+
+    if(KaMinPar_ADDED)
+        message(VERBOSE "CUOPT: Using KaMinPar located in ${KaMinPar_SOURCE_DIR}")
+        # KaMinPar's public header pulls in <tbb/global_control.h>. On older TBB releases
+        # that header is gated behind TBB_PREVIEW_GLOBAL_CONTROL (KaMinPar upstream assumes a
+        # newer oneTBB and never defines it). Define it on KaMinParCommon PUBLIC so it
+        # propagates to all KaMinPar translation units (KaMinPar links KaMinParCommon PUBLIC).
+        # Harmless on newer oneTBB where global_control is no longer a preview feature.
+        # Also force PIC on every KaMinPar target (the KaMinParCommon OBJECT library does not
+        # reliably inherit CMAKE_POSITION_INDEPENDENT_CODE) so the static archive can be
+        # embedded into the shared libcuopt.so.
+        foreach(_kaminpar_tgt KaMinParCommon KaMinPar KaMinParIO)
+            if(TARGET ${_kaminpar_tgt})
+                set_target_properties(${_kaminpar_tgt} PROPERTIES POSITION_INDEPENDENT_CODE ON)
+            endif()
+        endforeach()
+        if(TARGET KaMinParCommon)
+            target_compile_definitions(KaMinParCommon PUBLIC TBB_PREVIEW_GLOBAL_CONTROL)
+        endif()
+    else()
+        message(VERBOSE "CUOPT: Using KaMinPar located in ${KaMinPar_DIR}")
+    endif()
+endfunction()
+
+find_and_configure_kaminpar(VERSION 3.7.3 PINNED_TAG v3.7.3)