From e2572f7afbbdb682795feb045fed3906383df1d4 Mon Sep 17 00:00:00 2001 From: Ishika Roy Date: Fri, 21 Aug 2026 17:35:25 +0000 Subject: [PATCH 1/4] cache reuse for linear objective updates --- .../pdlp/solver_settings.hpp | 9 + .../utilities/barrier_cache.hpp | 109 ++ .../utilities/cython_solve.hpp | 4 +- .../utilities/cython_types.hpp | 4 + .../utilities/solver_cache_profiler.hpp | 152 +++ cpp/src/barrier/CMakeLists.txt | 1 + cpp/src/barrier/barrier.cu | 978 +++++++++++++++--- cpp/src/barrier/barrier.hpp | 20 +- .../barrier_factorization_sparsity_hash.cu | 25 + .../barrier_factorization_sparsity_hash.hpp | 126 +++ cpp/src/barrier/barrier_symbolic_cache.hpp | 86 ++ cpp/src/barrier/cusparse_view.cu | 10 + cpp/src/barrier/cusparse_view.hpp | 2 + cpp/src/barrier/device_sparse_matrix.cuh | 8 + cpp/src/barrier/sparse_cholesky.cuh | 172 ++- cpp/src/dual_simplex/solve.cpp | 169 ++- cpp/src/dual_simplex/solve.hpp | 23 +- cpp/src/pdlp/CMakeLists.txt | 1 + cpp/src/pdlp/solve.cu | 82 +- cpp/src/pdlp/utilities/barrier_cache.cu | 174 ++++ .../utilities/barrier_front_end_cache.hpp | 98 ++ cpp/src/pdlp/utilities/cython_solve.cu | 83 +- .../data_model/data_model.py | 17 + .../data_model/data_model_wrapper.pxd | 1 + .../data_model/data_model_wrapper.pyx | 45 + .../linear_programming/solver/solver.pxd | 8 + .../solver/solver_wrapper.pyx | 57 +- .../solver_settings/solver_settings.pxd | 11 + .../solver_settings/solver_settings.pyx | 11 + 29 files changed, 2260 insertions(+), 226 deletions(-) create mode 100644 cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp create mode 100644 cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp create mode 100644 cpp/src/barrier/barrier_factorization_sparsity_hash.cu create mode 100644 cpp/src/barrier/barrier_factorization_sparsity_hash.hpp create mode 100644 cpp/src/barrier/barrier_symbolic_cache.hpp create mode 100644 cpp/src/pdlp/utilities/barrier_cache.cu create mode 100644 cpp/src/pdlp/utilities/barrier_front_end_cache.hpp diff --git a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp index 0882f75e0f..4adeae9769 100644 --- a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp +++ b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp @@ -22,6 +22,11 @@ #include + +namespace cuopt::cython { +class barrier_cache_t; +} + namespace cuopt { namespace CUOPT_EXPORT mathematical_optimization { @@ -356,6 +361,10 @@ class pdlp_solver_settings_t { // Used to force batch PDLP to solve a subbatch of the problems at a time // The 0 default value will make the solver use its heuristic to determine the subbatch size i_t fixed_batch_size{0}; + /** When true, first GPU barrier/QCQP solve returns a ``barrier_cache_t`` capsule. */ + bool sequence_solve{false}; + /** Non-owning cache pointer set by ``call_solve`` for barrier symbolic reuse. */ + cuopt::cython::barrier_cache_t* barrier_cache{nullptr}; private: /** Initial primal solution */ diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp new file mode 100644 index 0000000000..d548aa3dbf --- /dev/null +++ b/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp @@ -0,0 +1,109 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +#include + +#include +#include + +namespace cuopt::mathematical_optimization::barrier { +template +class iteration_data_t; +template +struct barrier_symbolic_cache_t; + +template +void barrier_store_symbolic_cache_from_iteration_data(iteration_data_t& data, + barrier_symbolic_cache_t& cache); + +void destroy_iteration_data(iteration_data_t* data); + +void apply_barrier_linear_objective(iteration_data_t& data, + double const* barrier_c, + int n); +} // namespace cuopt::mathematical_optimization::barrier + +namespace cuopt { +namespace CUOPT_EXPORT cython { + +struct barrier_front_end_cache_t; + +/** + * @brief Lean GPU solve session: owns RAFT handle + stream, optional barrier symbolic cache, + * and optional barrier iteration_data_t (GPU IPM workspace) after an Optimal solve. + * + * Created on first solve when sequence_solve; reused on subsequent solves with the same capsule. + * Per-solve convert/presolve/scaling remain stack-local until the continue path (D); A keeps + * iteration_data_t, B keeps front-end maps + c_dirty. + */ +class barrier_cache_t { + public: + static std::unique_ptr create(unsigned stream_flags); + + barrier_cache_t(barrier_cache_t&&) noexcept; + barrier_cache_t& operator=(barrier_cache_t&&) noexcept; + ~barrier_cache_t(); + + [[nodiscard]] raft::handle_t* handle_ptr(); + [[nodiscard]] raft::handle_t const* handle_ptr() const; + [[nodiscard]] rmm::cuda_stream_view stream_view() const; + + /** + * @brief Returns cached symbolic state when valid and @p handle matches the stored handle. + */ + [[nodiscard]] mathematical_optimization::barrier::barrier_symbolic_cache_t* + symbolic_cache_for_reuse(raft::handle_t const* handle); + + void clear_symbolic_cache(); + + void store_symbolic_cache( + mathematical_optimization::barrier::iteration_data_t& data); + + /** + * @brief Take ownership of barrier iteration workspace. @p data may be null (clears). + */ + void store_iteration_data( + mathematical_optimization::barrier::iteration_data_t* data); + + /** + * @brief Release ownership of cached iteration workspace; caller must delete or wrap it. + */ + mathematical_optimization::barrier::iteration_data_t* release_iteration_data(); + + [[nodiscard]] mathematical_optimization::barrier::iteration_data_t* + iteration_data(); + + void clear_iteration_data(); + + void store_front_end_cache(std::unique_ptr cache); + [[nodiscard]] barrier_front_end_cache_t* front_end_cache(); + [[nodiscard]] barrier_front_end_cache_t const* front_end_cache() const; + void clear_front_end_cache(); + void set_c_dirty(bool dirty); + [[nodiscard]] bool c_dirty() const; + [[nodiscard]] bool has_front_end_cache() const; + + /** + * Crush user-space linear objective into cached iteration_data_t.c / d_c_ and set c_dirty. + * Requires a stored front-end cache and iteration_data from an Optimal solve. + */ + void update_linear_objective(double const* c, int n); + + private: + barrier_cache_t(std::unique_ptr stream, + std::unique_ptr handle); + + struct impl; + std::unique_ptr impl_; +}; + +} // namespace CUOPT_EXPORT cython +} // namespace cuopt diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp index f84119a8dc..fb6a3a21d9 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp +++ b/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp @@ -12,7 +12,6 @@ #include #include #include - #include #include #include @@ -56,7 +55,8 @@ std::unique_ptr call_solve( cuopt::mathematical_optimization::io::data_model_view_t*, mathematical_optimization::solver_settings_t*, unsigned int flags = cudaStreamNonBlocking, - bool is_batch_mode = false); + bool is_batch_mode = false, + barrier_cache_t* session_in = nullptr); std::pair>, double> solve_batch_remote( std::vector*>, diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp index 69d6f91604..fd92f93964 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp +++ b/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -86,6 +87,9 @@ struct linear_programming_ret_t { double solve_time_{}; mathematical_optimization::method_t solved_by_{}; + /** GPU barrier session (stream + handle + symbolic cache); moved to Python capsule when set. */ + std::unique_ptr barrier_cache; + bool is_gpu() const { return std::holds_alternative(solutions_); } }; diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp new file mode 100644 index 0000000000..0b72d45666 --- /dev/null +++ b/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp @@ -0,0 +1,152 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace cuopt::linear_programming::cache_profile { + +enum class cache_id : int { + C01 = 0, // raft::handle_t + stream + C02, // cuBLAS / cuSparse warmup (init_handler) + C03, // problem fingerprint (structural hash) + C04, // augmented vs ADAT choice + C05, // ADAT / augmented sparsity pattern + C06, // cuDSS handle + config + C07, // cuDSS symbolic factorization + C08, // dense-column / SOC layout metadata + C09, // device buffer allocation (iteration_data setup) + COUNT +}; + +inline constexpr int num_cache_ids = static_cast(cache_id::COUNT); + +inline const char* cache_id_label(cache_id id) +{ + switch (id) { + case cache_id::C01: return "C01 raft handle+stream"; + case cache_id::C02: return "C02 cuBLAS/cuSparse warmup"; + case cache_id::C03: return "C03 problem fingerprint"; + case cache_id::C04: return "C04 augmented vs ADAT choice"; + case cache_id::C05: return "C05 KKT sparsity pattern"; + case cache_id::C06: return "C06 cuDSS handle+config"; + case cache_id::C07: return "C07 cuDSS symbolic factorization"; + case cache_id::C08: return "C08 dense-column/SOC layout"; + case cache_id::C09: return "C09 device buffer allocation"; + default: return "C?? unknown"; + } +} + +class profiler_t { + public: + static profiler_t& instance() + { + static profiler_t prof; + return prof; + } + + bool enabled() const { return enabled_; } + + void reset() + { + times_.fill(0.0); + } + + void add(cache_id id, double seconds) + { + if (!enabled_) { return; } + times_[static_cast(id)] += seconds; + } + + double get(cache_id id) const { return times_[static_cast(id)]; } + + double total_measured() const + { + double sum = 0.0; + for (double t : times_) { + sum += t; + } + return sum; + } + + void log_summary() const + { + if (!enabled_) { return; } + auto emit = [](const char* fmt, ...) { + va_list args; + va_start(args, fmt); + char buf[512]; + vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + CUOPT_LOG_INFO("%s", buf); + fprintf(stderr, "%s\n", buf); + }; + emit("=== Solver cache profile (ms) ==="); + for (int i = 0; i < num_cache_ids; ++i) { + const double ms = times_[i] * 1000.0; + emit("Cache profile: %s %.3f", cache_id_label(static_cast(i)), ms); + } + emit("Cache profile: TOTAL measured %.3f", total_measured() * 1000.0); + emit("=== End solver cache profile ==="); + } + + private: + profiler_t() + { + const char* env = std::getenv("CUOPT_CACHE_PROFILE"); + enabled_ = env != nullptr && env[0] != '\0' && std::strcmp(env, "0") != 0; + } + + bool enabled_{false}; + std::array times_{}; +}; + +inline bool enabled() { return profiler_t::instance().enabled(); } + +inline void reset() { profiler_t::instance().reset(); } + +inline void add(cache_id id, double seconds) { profiler_t::instance().add(id, seconds); } + +inline void log_summary() { profiler_t::instance().log_summary(); } + +class scoped_timer_t { + public: + explicit scoped_timer_t(cache_id id) : id_(id), start_(clock_::now()), active_(enabled()) {} + + ~scoped_timer_t() + { + if (!active_) { return; } + const double elapsed = + std::chrono::duration(clock_::now() - start_).count(); + add(id_, elapsed); + } + + private: + using clock_ = std::chrono::steady_clock; + cache_id id_; + clock_::time_point start_; + bool active_; +}; + +} // namespace cuopt::linear_programming::cache_profile + +#define CUOPT_CACHE_PROFILE_SCOPE(id) \ + ::cuopt::linear_programming::cache_profile::scoped_timer_t CUOPT_CACHE_PROFILE_CONCAT( \ + _cuopt_cache_scope_, __LINE__)(id) + +#define CUOPT_CACHE_PROFILE_CONCAT(a, b) CUOPT_CACHE_PROFILE_CONCAT_IMPL(a, b) +#define CUOPT_CACHE_PROFILE_CONCAT_IMPL(a, b) a##b diff --git a/cpp/src/barrier/CMakeLists.txt b/cpp/src/barrier/CMakeLists.txt index 650bc733e9..2d5fb27dc7 100644 --- a/cpp/src/barrier/CMakeLists.txt +++ b/cpp/src/barrier/CMakeLists.txt @@ -6,6 +6,7 @@ set(BARRIER_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/cusparse_view.cu ${CMAKE_CURRENT_SOURCE_DIR}/barrier.cu + ${CMAKE_CURRENT_SOURCE_DIR}/barrier_factorization_sparsity_hash.cu ${CMAKE_CURRENT_SOURCE_DIR}/device_sparse_matrix.cu ${CMAKE_CURRENT_SOURCE_DIR}/pinned_host_allocator.cu ) diff --git a/cpp/src/barrier/barrier.cu b/cpp/src/barrier/barrier.cu index c164296a25..f824ff8b88 100644 --- a/cpp/src/barrier/barrier.cu +++ b/cpp/src/barrier/barrier.cu @@ -7,6 +7,8 @@ #include +#include +#include #include #include #include @@ -29,6 +31,13 @@ #include +#include + +#include + +#include +#include + #include #include @@ -38,6 +47,7 @@ #include #include +#include #include #include @@ -228,7 +238,8 @@ class iteration_data_t { i_t num_upper_bounds, const std::vector& direct_free_variables, const csc_matrix_t& Qin, - const simplex_solver_settings_t& settings) + const simplex_solver_settings_t& settings, + barrier_symbolic_cache_t* adopt_symbolic = nullptr) : upper_bounds(num_upper_bounds), c(lp.objective), b(lp.rhs), @@ -262,8 +273,8 @@ class iteration_data_t { A(lp.A), Q(Qin), cusparse_Q_view_(lp.handle_ptr, Q), - cusparse_view_(lp.handle_ptr, lp.A), - cusparse_info(lp.handle_ptr), + cusparse_view_(lp.handle_ptr, A), + cusparse_info_(nullptr), device_AD(lp.num_cols, lp.num_rows, 0, lp.handle_ptr->get_stream()), device_A(lp.num_cols, lp.num_rows, 0, lp.handle_ptr->get_stream()), device_ADAT(lp.num_rows, lp.num_rows, 0, lp.handle_ptr->get_stream()), @@ -350,12 +361,14 @@ class iteration_data_t { indefinite_Q(false), Q_diagonal(false), symbolic_status(0), + adopted_symbolic_(false), cone_combined_step_(false), cone_sigma_mu_(f_t(0)) { raft::common::nvtx::range fun_scope("Barrier: LP Data Creation"); { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: direct free linear"); // Setup tracking of direct free variables (linear columns only j < cone_start) n_direct_free_linear = direct_free_variables.size(); @@ -378,6 +391,7 @@ class iteration_data_t { bool has_Q = Q.x.size() > 0; indefinite_Q = false; { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: Q setup"); if (has_Q) { Qdiag.resize(lp.num_cols, 0.0); @@ -419,6 +433,7 @@ class iteration_data_t { } if (!lp.second_order_cone_dims.empty()) { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C08); raft::common::nvtx::range scope("Barrier: LP Data: SOC setup"); cone_var_start_ = lp.cone_var_start; i_t total_cone_dim = @@ -439,6 +454,7 @@ class iteration_data_t { } { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: complementarity buffers"); const i_t linear_xz_rhs_size = linear_xz_size(lp.num_cols); d_complementarity_xz_rhs_.resize(linear_xz_rhs_size, stream_view_); @@ -458,6 +474,7 @@ class iteration_data_t { } { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: upper bounds"); // Create the upper bounds vector n_upper_bounds = 0; @@ -471,6 +488,7 @@ class iteration_data_t { std::vector dense_columns_unordered; { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C04); raft::common::nvtx::range scope("Barrier: LP Data: dense columns and augmented"); // Decide if we are going to use the augmented system or not n_dense_columns = 0; @@ -548,6 +566,7 @@ class iteration_data_t { } { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: diag and inv_diag"); // D = I + EET diag.set_scalar(1.0); @@ -579,6 +598,7 @@ class iteration_data_t { if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: AD matrix setup"); // Copy A into AD AD = lp.A; @@ -626,6 +646,7 @@ class iteration_data_t { // device_AD / device_A / ADAT path is only used when forming ADAT (!use_augmented). if (!use_augmented) { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: device AD path"); device_AD.copy(AD, handle_ptr->get_stream()); d_original_A_values.resize(device_AD.x.size(), handle_ptr->get_stream()); @@ -644,37 +665,488 @@ class iteration_data_t { if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C06); raft::common::nvtx::range scope("Barrier: LP Data: Cholesky init"); i_t factorization_size = use_augmented ? augmented_system_size(lp.num_cols, lp.num_rows) : lp.num_rows; - chol = std::make_unique>( - handle_ptr, settings, factorization_size); - chol->set_positive_definite(false); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - { - raft::common::nvtx::range scope("Barrier: LP Data: symbolic analysis"); - // Perform symbolic analysis - symbolic_status = 0; - if (use_augmented) { - { - raft::common::nvtx::range form_scope("Barrier: LP Data: form augmented"); - // Build the sparsity pattern of the augmented system - form_augmented(true); + + auto adopt_augmented_symbolic = [&]() -> bool { + if (has_cones() || !use_augmented || adopt_symbolic == nullptr) { return false; } + if (adopt_symbolic->device_augmented.x.size() == 0) { return false; } + + const barrier_sparsity_hash_t host_hash = hash_augmented_kkt_sparsity(A, AT, Q); + const bool matched = + adopt_symbolic->matches_reuse(host_hash, true, handle_ptr); + if (!matched) { return false; } + + chol = adopt_symbolic->chol; + static_cast*>(chol.get())->rebind_settings(settings); + pinned_device_augmented_ = &adopt_symbolic->device_augmented; + d_augmented_diagonal_indices_.resize(adopt_symbolic->d_augmented_diagonal_indices_.size(), + stream_view_); + raft::copy(d_augmented_diagonal_indices_.data(), + adopt_symbolic->d_augmented_diagonal_indices_.data(), + adopt_symbolic->d_augmented_diagonal_indices_.size(), + stream_view_); + handle_ptr->sync_stream(); + static_cast*>(chol.get())->rebind_csr_matrix(aug_mat()); + adopted_symbolic_ = true; + symbolic_status = 0; + return true; + }; + + auto unpin_adat_workspace = [&]() { + pinned_device_ADAT_ = nullptr; + pinned_device_A_ = nullptr; + pinned_device_AD_ = nullptr; + pinned_d_original_A_values_ = nullptr; + pinned_device_A_x_values_ = nullptr; + pinned_cusparse_info_ = nullptr; + }; + + auto pin_adat_from_cache = [&](barrier_symbolic_cache_t& cache) { + pinned_device_ADAT_ = &cache.device_ADAT; + pinned_device_A_ = &cache.device_A; + pinned_device_AD_ = &cache.device_AD; + pinned_d_original_A_values_ = &cache.d_original_A_values; + pinned_device_A_x_values_ = &cache.device_A_x_values; + pinned_cusparse_info_ = cache.cusparse_info.get(); + }; + + auto adopt_adat_symbolic = [&]() -> bool { + if (has_cones() || use_augmented || adopt_symbolic == nullptr || n_dense_columns > 0) { + return false; + } + if (!adopt_symbolic->valid || adopt_symbolic->use_augmented) { return false; } + + // Gate on the *incoming* A sparsity before pinning SpGEMM workspace. + // Hashing ADAT after pin/form used the cached A and could false-match when + // only the new problem's pattern changed (same idea as augmented host gate). + // device_A already holds the current problem's CSR (uploaded above). + const barrier_sparsity_hash_t a_hash = + hash_device_csr_sparsity_pattern(device_A, stream_view_); + if (!adopt_symbolic->matches_reuse(a_hash, false, handle_ptr)) { + settings_.log.printf( + "Barrier: ADAT A-sparsity hash mismatch; rebuilding symbolic analysis\n"); + adopt_symbolic->clear(); + return false; + } + if (adopt_symbolic->device_A.x.size() == 0) { return false; } + + pin_adat_from_cache(*adopt_symbolic); + form_adat(true); + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + unpin_adat_workspace(); + return false; + } + + chol = adopt_symbolic->chol; + static_cast*>(chol.get())->rebind_settings(settings); + handle_ptr->sync_stream(); + static_cast*>(chol.get())->rebind_csr_matrix(adat_mat()); + adopted_symbolic_ = true; + symbolic_status = 0; + return true; + }; + + if (!adopt_augmented_symbolic() && !adopt_adat_symbolic()) { + if (use_augmented) { + { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C05); + raft::common::nvtx::range form_scope("Barrier: LP Data: form augmented"); + form_augmented(true); + } + } else { + { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C05); + raft::common::nvtx::range form_scope("Barrier: LP Data: form ADAT"); + form_adat(true); + } } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - symbolic_status = chol->analyze(device_augmented); - } else { + + chol = std::make_shared>( + handle_ptr, settings, factorization_size); + chol->set_positive_definite(false); + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } + symbolic_status = 0; { - raft::common::nvtx::range form_scope("Barrier: LP Data: form ADAT"); - form_adat(true); + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C07); + raft::common::nvtx::range analyze_scope("Barrier: LP Data: symbolic analysis"); + if (use_augmented) { + symbolic_status = chol->analyze(aug_mat()); + } else { + symbolic_status = chol->analyze(adat_mat()); + } } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - symbolic_status = chol->analyze(device_ADAT); } } } + [[nodiscard]] bool adopted_symbolic() const { return adopted_symbolic_; } + + device_csr_matrix_t& augmented_system() { return aug_mat(); } + const device_csr_matrix_t& augmented_system() const { return aug_mat(); } + + void store_symbolic_cache(barrier_symbolic_cache_t& cache) + { + if (symbolic_status != 0 || has_cones()) { return; } + auto* cudss_chol = dynamic_cast*>(chol.get()); + if (cudss_chol == nullptr) { return; } + + cache.chol = std::static_pointer_cast>(chol); + cache.handle_ptr = handle_ptr; + + if (use_augmented) { + cache.cusparse_info.reset(); + cache.use_augmented = true; + + if (pinned_device_augmented_ != nullptr) { + // Warm reuse: sparsity_hash unchanged since adopt (values-only refresh); unpin only. + pinned_device_augmented_ = nullptr; + } else { + cache.sparsity_hash = + hash_device_csr_sparsity_pattern(device_augmented, handle_ptr->get_stream()); + // Keep KKT buffers on iteration_data_t so the session can retain the workspace. + } + +#ifndef NDEBUG + const barrier_sparsity_hash_t host_hash = hash_augmented_kkt_sparsity(A, AT, Q); + cuopt_assert(cache.sparsity_hash == host_hash, + "store_symbolic_cache: device/host augmented sparsity hash mismatch"); +#endif + + cache.valid = true; + settings_.log.printf( + "Barrier: stored augmented symbolic cache hash=0x%016llx\n", + static_cast(cache.sparsity_hash)); + return; + } + + if (n_dense_columns > 0) { return; } + + cache.use_augmented = false; + if (pinned_device_ADAT_ != nullptr) { + // Warm reuse: sparsity_hash is the A-pattern hash from adopt; unpin only. + pinned_device_ADAT_ = nullptr; + pinned_device_A_ = nullptr; + pinned_device_AD_ = nullptr; + pinned_d_original_A_values_ = nullptr; + pinned_device_A_x_values_ = nullptr; + pinned_cusparse_info_ = nullptr; + } else { + // Store A sparsity (not ADAT): adopt compares the incoming A CSR before pin. + cache.sparsity_hash = + hash_device_csr_sparsity_pattern(device_A, handle_ptr->get_stream()); + // Keep ADAT/A buffers on iteration_data_t so the session can retain the workspace. + } + + cache.valid = true; + settings_.log.printf( + "Barrier: stored ADAT symbolic cache hash=0x%016llx\n", + static_cast(cache.sparsity_hash)); + } + + bool refresh_augmented_values() + { + i_t n = A.n; + i_t m = A.m; + i_t nnzA = A.col_start[n]; + i_t nnzQ = Q.n > 0 ? Q.col_start[n] : 0; + + i_t new_nnz = 2 * nnzA + n + m + nnzQ; + csr_matrix_t augmented_CSR(n + m, n + m, new_nnz); + i_t q = 0; + i_t off_diag_Qnz = 0; + + for (i_t i = 0; i < n; i++) { + augmented_CSR.row_start[i] = q; + if (nnzQ == 0) { + augmented_CSR.j[q] = i; + augmented_CSR.x[q++] = -diag[i] - dual_perturb; + } else { + const i_t q_col_beg = Q.col_start[i]; + const i_t q_col_end = Q.col_start[i + 1]; + bool has_diagonal = false; + for (i_t p = q_col_beg; p < q_col_end; ++p) { + augmented_CSR.j[q] = Q.i[p]; + if (Q.i[p] == i) { + has_diagonal = true; + augmented_CSR.x[q++] = -Q.x[p] - diag[i] - dual_perturb; + } else { + off_diag_Qnz++; + augmented_CSR.x[q++] = -Q.x[p]; + } + } + if (!has_diagonal) { + augmented_CSR.j[q] = i; + augmented_CSR.x[q++] = -diag[i] - dual_perturb; + } + } + const i_t col_beg = A.col_start[i]; + const i_t col_end = A.col_start[i + 1]; + for (i_t p = col_beg; p < col_end; ++p) { + augmented_CSR.j[q] = A.i[p] + n; + augmented_CSR.x[q++] = A.x[p]; + } + } + + for (i_t k = n; k < n + m; ++k) { + augmented_CSR.row_start[k] = q; + const i_t l = k - n; + const i_t col_beg = AT.col_start[l]; + const i_t col_end = AT.col_start[l + 1]; + for (i_t p = col_beg; p < col_end; ++p) { + augmented_CSR.j[q] = AT.i[p]; + augmented_CSR.x[q++] = AT.x[p]; + } + augmented_CSR.j[q] = k; + augmented_CSR.x[q++] = primal_perturb; + } + augmented_CSR.row_start[n + m] = q; + if (q != static_cast(aug_mat().x.size()) || q != 2 * nnzA + n + m + off_diag_Qnz) { + return false; + } + + augmented_CSR.j.resize(q); + augmented_CSR.x.resize(q); + raft::copy(aug_mat().x.data(), augmented_CSR.x.data(), q, handle_ptr->get_stream()); + RAFT_CHECK_CUDA(handle_ptr->get_stream()); + return true; + } + + bool rebuild_augmented_symbolic() + { + if (!use_augmented) { return false; } + + settings_.log.printf( + "Barrier: augmented nnz mismatch on cached symbolic; rebuilding symbolic analysis\n"); + + adopted_symbolic_ = false; + pinned_device_augmented_ = nullptr; + + const i_t factorization_size = A.n + A.m; + chol = std::make_shared>( + handle_ptr, settings_, factorization_size); + chol->set_positive_definite(false); + if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } + + form_augmented(true); + if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } + + symbolic_status = chol->analyze(aug_mat()); + if (symbolic_status != 0) { return false; } + + reset_for_new_solve(); + return true; + } + + bool refresh_adat_values() + { + if (use_augmented || n_dense_columns > 0) { return false; } + + const i_t expected_nnz = static_cast(adat_mat().x.size()); + form_adat(false); + handle_ptr->sync_stream(); + return static_cast(adat_mat().x.size()) == expected_nnz; + } + + bool rebuild_adat_symbolic() + { + if (use_augmented || n_dense_columns > 0) { return false; } + + settings_.log.printf( + "Barrier: ADAT nnz mismatch on cached symbolic; rebuilding symbolic analysis\n"); + + adopted_symbolic_ = false; + pinned_device_ADAT_ = nullptr; + pinned_device_A_ = nullptr; + pinned_device_AD_ = nullptr; + pinned_d_original_A_values_ = nullptr; + pinned_device_A_x_values_ = nullptr; + pinned_cusparse_info_ = nullptr; + cusparse_info_.reset(); + + const i_t factorization_size = A.m; + chol = std::make_shared>( + handle_ptr, settings_, factorization_size); + chol->set_positive_definite(false); + if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } + + form_adat(true); + if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } + + symbolic_status = chol->analyze(adat_mat()); + if (symbolic_status != 0) { return false; } + + reset_for_new_solve(); + return true; + } + + // Restore D = I + EE^T (+ Q_diag on the ADAT path) and the regularization used + // for the first factorization. The IPM loop overwrites both with the values of + // its last iterate, so a solve that reuses this workspace must reset them + // before an initial point is computed. + void reset_diagonal_scaling() + { + raft::common::nvtx::range fun_scope("Barrier: reset diagonal scaling"); + const bool has_Q = Q.n > 0; + + if (has_cones()) { + primal_perturb = 1e-8; + dual_perturb = 1e-8; + } else { + primal_perturb = 1e-6; + dual_perturb = 0; + } + + diag.set_scalar(1.0); + for (i_t k = 0; k < n_upper_bounds; k++) { + diag[upper_bounds[k]] = 2.0; + } + if (has_Q && !use_augmented) { + for (i_t j = 0; j < Q.n; j++) { + diag[j] += Qdiag[j]; + } + } + + inv_diag.set_scalar(1.0); + if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { diag.inverse(inv_diag); } + raft::copy(d_diag_.data(), diag.data(), diag.size(), stream_view_); + raft::copy(d_inv_diag.data(), inv_diag.data(), inv_diag.size(), stream_view_); + inv_sqrt_diag.set_scalar(1.0); + if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { inv_diag.sqrt(inv_sqrt_diag); } + } + + // Re-form the ADAT or augmented values from the current diag, keeping the + // cached symbolic factorization whenever the sparsity pattern is unchanged. + bool refresh_linear_system_values() + { + if (use_augmented) { + if (!refresh_augmented_values()) { return rebuild_augmented_symbolic(); } + return true; + } + if (!adopted_symbolic_) { return true; } + if (!refresh_adat_values()) { return rebuild_adat_symbolic(); } + handle_ptr->sync_stream(); + if (chol != nullptr) { chol->rebind_csr_matrix(adat_mat()); } + return true; + } + + bool refresh_lp_numerics(const lp_problem_t& lp) + { + raft::common::nvtx::range fun_scope("Barrier: refresh LP numerics"); + + c = lp.objective; + b = lp.rhs; + raft::copy(d_c_.data(), c.data(), c.size(), stream_view_); + raft::copy(d_b_.data(), b.data(), b.size(), stream_view_); + + AD = A; + if (!use_augmented && n_dense_columns > 0) { AD.remove_columns(cols_to_remove); } + AT.transpose(AD); + + const bool has_Q = Q.n > 0; + if (has_Q) { + for (i_t j = 0; j < Q.n; j++) { + Qdiag[j] = 0.0; + const i_t col_start = Q.col_start[j]; + const i_t col_end = Q.col_start[j + 1]; + for (i_t p = col_start; p < col_end; p++) { + const i_t row = Q.i[p]; + if (j == row) { + Qdiag[j] = Q.x[p]; + break; + } + } + } + if (d_Q_diag_.size() > 0) { + raft::copy(d_Q_diag_.data(), Qdiag.data(), Qdiag.size(), stream_view_); + } + } + + reset_diagonal_scaling(); + + if (!use_augmented) { + ad_mat().copy(AD, handle_ptr->get_stream()); + raft::copy(original_a_values().data(), + ad_mat().x.data(), + ad_mat().x.size(), + handle_ptr->get_stream()); + raft::copy(a_x_values().data(), ad_mat().x.data(), ad_mat().x.size(), handle_ptr->get_stream()); + ad_mat().to_compressed_row(a_mat(), handle_ptr->get_stream()); + RAFT_CHECK_CUDA(handle_ptr->get_stream()); + } + + if (!refresh_linear_system_values()) { return false; } + + cusparse_view_.update_matrix_values(A); + if (Q.n > 0) { cusparse_Q_view_.update_matrix_values(Q); } + + reset_for_new_solve(); + return true; + } + + bool rebind_from_lp(const lp_problem_t& lp, + const std::vector& direct_free_variables, + const csc_matrix_t& Qin, + const simplex_solver_settings_t& settings) + { + raft::common::nvtx::range fun_scope("Barrier: rebind iteration data"); + if (handle_ptr != lp.handle_ptr) { return false; } + if (chol == nullptr || symbolic_status != 0) { return false; } + if (A.m != lp.A.m || A.n != lp.A.n || A.nnz() != lp.A.nnz()) { return false; } + if (A.col_start != lp.A.col_start || A.i != lp.A.i) { return false; } + if (Q.n != Qin.n || Q.m != Qin.m || Q.nnz() != Qin.nnz()) { return false; } + if (Q.col_start != Qin.col_start || Q.i != Qin.i) { return false; } + if (static_cast(direct_free_variables.size()) != n_direct_free_linear) { return false; } + i_t new_n_upper = 0; + for (i_t j = 0; j < lp.num_cols; j++) { + if (lp.upper[j] < inf) { new_n_upper++; } + } + if (new_n_upper != n_upper_bounds) { return false; } + + A = lp.A; + Q = Qin; + settings_ = settings; + if (chol != nullptr) { + static_cast*>(chol.get())->rebind_settings(settings_); + } + adopted_symbolic_ = true; + return refresh_lp_numerics(lp); + } + + bool rebind_settings_for_continue(const simplex_solver_settings_t& settings) + { + if (chol == nullptr || symbolic_status != 0) { return false; } + settings_ = settings; + adopted_symbolic_ = true; + if (chol != nullptr) { + static_cast*>(chol.get())->rebind_settings(settings_); + } + // A and Q are unchanged, so only the iterate-dependent state has to be + // rewound: the initial point is computed from D and the linear system built + // from it, both of which the previous solve left at its final iterate. + reset_diagonal_scaling(); + if (!refresh_linear_system_values()) { return false; } + reset_for_new_solve(); + return true; + } + + void reset_for_new_solve() + { + has_factorization = false; + has_solve_info = false; + relative_primal_residual_save = inf; + relative_dual_residual_save = inf; + relative_complementarity_residual_save = inf; + primal_residual_norm_save = inf; + dual_residual_norm_save = inf; + complementarity_residual_norm_save = inf; + if (chol != nullptr) { chol->invalidate_numeric_factor(); } + handle_ptr->sync_stream(); + } + bool has_cones() const { return cones_.has_value(); } cone_data_t& cones() @@ -814,7 +1286,7 @@ class iteration_data_t { thrust::for_each_n(rmm::exec_policy(handle_ptr->get_stream()), thrust::make_counting_iterator(0), linear_n, - [span_x = cuopt::make_span(device_augmented.x), + [span_x = cuopt::make_span(aug_mat().x), span_diag_indices = cuopt::make_span(d_augmented_diagonal_indices_), span_q_diag = cuopt::make_span(d_Q_diag_), span_diag = cuopt::make_span(d_diag_), @@ -828,7 +1300,7 @@ class iteration_data_t { thrust::for_each_n(rmm::exec_policy(handle_ptr->get_stream()), thrust::make_counting_iterator(n), i_t(m), - [span_x = cuopt::make_span(device_augmented.x), + [span_x = cuopt::make_span(aug_mat().x), span_diag_indices = cuopt::make_span(d_augmented_diagonal_indices_), primal_perturb_value = primal_perturb] __device__(i_t j) { span_x[span_diag_indices[j]] = primal_perturb_value; @@ -876,9 +1348,9 @@ class iteration_data_t { { raft::common::nvtx::range scope("Barrier: Form ADAT: restore A"); - raft::copy(device_AD.x.data(), - d_original_A_values.data(), - d_original_A_values.size(), + raft::copy(ad_mat().x.data(), + original_a_values().data(), + original_a_values().size(), handle_ptr->get_stream()); } { @@ -910,20 +1382,23 @@ class iteration_data_t { raft::common::nvtx::range scope("Barrier: Form ADAT: scale AD"); thrust::for_each_n(rmm::exec_policy(stream_view_), thrust::make_counting_iterator(0), - i_t(device_AD.x.size()), - [span_x = cuopt::make_span(device_AD.x), + i_t(ad_mat().x.size()), + [span_x = cuopt::make_span(ad_mat().x), span_scale = cuopt::make_span(d_inv_diag_prime), - span_col_ind = cuopt::make_span(device_AD.col_index)] __device__(i_t i) { + span_col_ind = cuopt::make_span(ad_mat().col_index)] __device__(i_t i) { span_x[i] *= span_scale[span_col_ind[i]]; }); RAFT_CHECK_CUDA(stream_view_); } if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return; } - if (first_call) { + if (first_call && pinned_cusparse_info_ == nullptr) { raft::common::nvtx::range scope("Barrier: Form ADAT: cusparse init"); try { + if (!cusparse_info_) { + cusparse_info_ = std::make_unique>(handle_ptr); + } initialize_cusparse_data( - handle_ptr, device_A, device_AD, device_ADAT, cusparse_info); + handle_ptr, a_mat(), ad_mat(), adat_mat(), spgemm_info()); } catch (const raft::cuda_error& e) { settings_.log.printf("Error in initialize_cusparse_data: %s\n", e.what()); return; @@ -933,11 +1408,11 @@ class iteration_data_t { { raft::common::nvtx::range scope("Barrier: Form ADAT: ADAT multiply"); - multiply_kernels(handle_ptr, device_A, device_AD, device_ADAT, cusparse_info); + multiply_kernels(handle_ptr, a_mat(), ad_mat(), adat_mat(), spgemm_info()); handle_ptr->sync_stream(); } - auto adat_nnz = device_ADAT.row_start.element(device_ADAT.m, handle_ptr->get_stream()); + auto adat_nnz = adat_mat().row_start.element(adat_mat().m, handle_ptr->get_stream()); float64_t adat_time = toc(start_form_adat); if (num_factorizations == 0) { @@ -947,7 +1422,7 @@ class iteration_data_t { settings_.log.printf( "ADAT density : %.2f\n", static_cast(adat_nnz) / - (static_cast(device_ADAT.m) * static_cast(device_ADAT.m))); + (static_cast(adat_mat().m) * static_cast(adat_mat().m))); } } @@ -1372,7 +1847,7 @@ class iteration_data_t { solution.z = z_tilde; dense_vector_t dual_res = z_tilde; - dual_res.axpy(-1.0, lp.objective, 1.0); + dual_res.axpy(-1.0, c, 1.0); cusparse_view.transpose_spmv(1.0, solution.y, 1.0, dual_res); if (Q.n > 0) { matrix_vector_multiply(Q, -1.0, x, 1.0, dual_res); } f_t dual_residual_norm = vector_norm_inf(dual_res, stream_view_); @@ -1957,12 +2432,60 @@ class iteration_data_t { dense_matrix_t AD_dense; dense_matrix_t H; dense_matrix_t Hchol; - const csc_matrix_t& A; + csc_matrix_t A; - const csc_matrix_t& Q; + csc_matrix_t Q; std::vector Qdiag; bool Q_diagonal; rmm::device_uvector d_augmented_diagonal_indices_; + device_csr_matrix_t* pinned_device_augmented_{nullptr}; + device_csr_matrix_t* pinned_device_ADAT_{nullptr}; + device_csr_matrix_t* pinned_device_A_{nullptr}; + device_csc_matrix_t* pinned_device_AD_{nullptr}; + rmm::device_uvector* pinned_d_original_A_values_{nullptr}; + rmm::device_uvector* pinned_device_A_x_values_{nullptr}; + cusparse_info_t* pinned_cusparse_info_{nullptr}; + + device_csr_matrix_t& aug_mat() + { + return pinned_device_augmented_ != nullptr ? *pinned_device_augmented_ : device_augmented; + } + const device_csr_matrix_t& aug_mat() const + { + return pinned_device_augmented_ != nullptr ? *pinned_device_augmented_ : device_augmented; + } + device_csr_matrix_t& adat_mat() + { + return pinned_device_ADAT_ != nullptr ? *pinned_device_ADAT_ : device_ADAT; + } + const device_csr_matrix_t& adat_mat() const + { + return pinned_device_ADAT_ != nullptr ? *pinned_device_ADAT_ : device_ADAT; + } + device_csr_matrix_t& a_mat() + { + return pinned_device_A_ != nullptr ? *pinned_device_A_ : device_A; + } + device_csc_matrix_t& ad_mat() + { + return pinned_device_AD_ != nullptr ? *pinned_device_AD_ : device_AD; + } + rmm::device_uvector& original_a_values() + { + return pinned_d_original_A_values_ != nullptr ? *pinned_d_original_A_values_ + : d_original_A_values; + } + rmm::device_uvector& a_x_values() + { + return pinned_device_A_x_values_ != nullptr ? *pinned_device_A_x_values_ : device_A_x_values; + } + cusparse_info_t& spgemm_info() + { + cuopt_assert(pinned_cusparse_info_ != nullptr || cusparse_info_ != nullptr, + "spgemm_info: cusparse workspace unset"); + return pinned_cusparse_info_ != nullptr ? *pinned_cusparse_info_ : *cusparse_info_; + } + cone_kkt_data_t cone_kkt_data_; bool indefinite_Q; cusparse_view_t cusparse_Q_view_; @@ -1972,6 +2495,7 @@ class iteration_data_t { bool use_augmented; i_t symbolic_status; + bool adopted_symbolic_; i_t n_direct_free_linear{0}; rmm::device_uvector d_is_direct_free_linear_; // 1 if variable is free in the linear block, else 0 @@ -1980,13 +2504,13 @@ class iteration_data_t { f_t dual_perturb{1e-8}; f_t primal_perturb{1e-8}; - std::unique_ptr> chol; + std::shared_ptr> chol; bool has_factorization; bool has_solve_info; i_t num_factorizations; - cusparse_info_t cusparse_info; + std::unique_ptr> cusparse_info_; cusparse_view_t cusparse_view_; pdlp::cusparse_dn_vec_descr_wrapper_t cusparse_tmp4_; pdlp::cusparse_dn_vec_descr_wrapper_t cusparse_h_; @@ -2078,7 +2602,7 @@ class iteration_data_t { rmm::cuda_stream_view stream_view_; - const simplex_solver_settings_t& settings_; + simplex_solver_settings_t settings_; }; // Move the Cholesky debug logic to a reusable function. @@ -2183,8 +2707,8 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // LP block: e = 1, SOC block: e = (sqrt(2), 0, ..., 0) if (data.has_cones()) { const i_t cs = data.cone_start(); - const f_t norm_b = vector_norm_inf(lp.rhs); - const f_t norm_c = vector_norm_inf(lp.objective); + const f_t norm_b = vector_norm_inf(data.b); + const f_t norm_c = vector_norm_inf(data.c); const f_t mu = std::sqrt((1.0 + norm_b) * (1.0 + norm_c)); const f_t sqrt2 = std::sqrt(2.0); const f_t x_soc = mu * sqrt2; @@ -2225,13 +2749,13 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // Perform a numerical factorization i_t status; if (use_augmented) { - status = data.chol->factorize(data.device_augmented); + status = data.chol->factorize(data.aug_mat()); #ifdef CHOLESKY_DEBUG_CHECK cholesky_debug_check(data, lp, use_augmented); #endif } else { - status = data.chol->factorize(data.device_ADAT); + status = data.chol->factorize(data.adat_mat()); } if (status == CONCURRENT_HALT_RETURN) { return CONCURRENT_HALT_RETURN; } if (status != 0) { @@ -2242,7 +2766,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) data.has_solve_info = false; // rhs_x <- b - dense_vector_t rhs_x(lp.rhs); + dense_vector_t rhs_x(data.b); dense_vector_t Fu(lp.num_cols); data.gather_upper_bounds(lp.upper, Fu); @@ -2335,7 +2859,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // Verify A*x = b dense_vector_t init_primal_residual(lp.num_rows); - init_primal_residual = lp.rhs; + init_primal_residual = data.b; data.cusparse_view_.spmv(1.0, data.x, -1.0, init_primal_residual); data.handle_ptr->get_stream().synchronize(); #ifdef PRINT_INFO @@ -2363,7 +2887,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // y = 0 data.y.set_scalar(0.0); - f_t epsilon = 1.0 + vector_norm1(lp.objective); + f_t epsilon = 1.0 + vector_norm1(data.c); // A^T y + z - E^T v - Q x = c // when y = 0, z - E^T v = c + Q x @@ -2429,7 +2953,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // First compute rhs = A*Dinv*c dense_vector_t rhs(lp.num_rows); dense_vector_t Dinvc(lp.num_cols); - data.inv_diag.pairwise_product(lp.objective, Dinvc); + data.inv_diag.pairwise_product(data.c, Dinvc); // rhs = 1.0 * A * Dinv * c data.cusparse_view_.spmv(1.0, Dinvc, 0.0, rhs); @@ -2792,7 +3316,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_tfactorize(data.device_augmented); + status = data.chol->factorize(data.aug_mat()); } #ifdef CHOLESKY_DEBUG_CHECK @@ -2812,7 +3336,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_tfactorize(data.device_ADAT); + status = data.chol->factorize(data.adat_mat()); } } data.has_factorization = true; @@ -4096,57 +4620,98 @@ lp_status_t barrier_solver_t::check_for_suboptimal_solution( return lp_status_t::NUMERICAL_ISSUES; } + template -lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t& solution) +lp_status_t finish_barrier_session(cuopt::cython::barrier_cache_t* session, + std::unique_ptr>& owned_data, + lp_status_t status) +{ + if (session == nullptr) { return status; } + if (owned_data) { + if (status == lp_status_t::OPTIMAL) { + session->store_symbolic_cache(*owned_data); + session->store_iteration_data(owned_data.release()); + } else { + session->clear_symbolic_cache(); + } + } + return status; +} + +template +lp_status_t barrier_solver_t::barrier_solve_advanced( + f_t start_time, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session) { settings.log.printf("Barrier solver started at %.2f seconds\n", toc(start_time)); try { - raft::common::nvtx::range fun_scope("Barrier: solve"); + raft::common::nvtx::range fun_scope("Barrier: barrier_solve_advanced"); i_t n = lp.num_cols; i_t m = lp.num_rows; - solution.resize(m, n); settings.log.printf( "Barrier solver: %d constraints, %d variables, %ld nonzeros\n", m, n, lp.A.col_start[n]); - - settings.log.printf("\n"); - if (lp.Q.n > 0) { settings.log.printf("Quadratic objective matrix : %d nonzeros\n", lp.Q.row_start[lp.Q.n]); } - if (lp.second_order_cone_dims.size() > 0) { - settings.log.printf("Second-order cones : %d\n", - static_cast(lp.second_order_cone_dims.size())); - } - // Compute the number of free variables - i_t num_free_variables = presolve_info.free_variable_pairs.size() / 2; - if (num_free_variables > 0) { - settings.log.printf("Free variables : %d\n", num_free_variables); + std::unique_ptr> owned_data; + if (session != nullptr) { + if (auto* cached = session->release_iteration_data()) { owned_data.reset(cached); } } - - // Compute the number of upper bounds - i_t num_upper_bounds = 0; - for (i_t j = 0; j < n; j++) { - if (lp.upper[j] < inf) { num_upper_bounds++; } + if (!owned_data) { + settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + return lp_status_t::NUMERICAL_ISSUES; + } + try { + if (!owned_data->rebind_settings_for_continue(settings)) { + owned_data.reset(); + if (session != nullptr) { session->clear_symbolic_cache(); } + settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + return lp_status_t::NUMERICAL_ISSUES; + } + } catch (const raft::cuda_error&) { + owned_data.reset(); + if (session != nullptr) { session->clear_symbolic_cache(); } + settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + return lp_status_t::NUMERICAL_ISSUES; } - csc_matrix_t Q(lp.num_cols, 0, 0); - if (lp.Q.n > 0) { create_Q(lp, Q); } - - iteration_data_t data( - lp, num_upper_bounds, presolve_info.direct_free_variables, Q, settings); + iteration_data_t& data = *owned_data; if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return lp_status_t::CONCURRENT_LIMIT; + return finish_barrier_session(session, owned_data, lp_status_t::CONCURRENT_LIMIT); + } + if (data.indefinite_Q) { + return finish_barrier_session(session, owned_data, lp_status_t::NUMERICAL_ISSUES); } - if (data.indefinite_Q) { return lp_status_t::NUMERICAL_ISSUES; } if (data.symbolic_status != 0) { settings.log.printf("Error in symbolic analysis\n"); - return lp_status_t::NUMERICAL_ISSUES; + return finish_barrier_session(session, owned_data, lp_status_t::NUMERICAL_ISSUES); } + return run_ipm(start_time, solution, session, owned_data); + } catch (const raft::cuda_error& e) { + settings.log.printf("Error in barrier_solver_t: %s\n", e.what()); + return lp_status_t::NUMERICAL_ISSUES; + } catch (const std::bad_alloc& e) { + settings.log.printf("Out of memory in barrier_solver_t: %s\n", e.what()); + return lp_status_t::NUMERICAL_ISSUES; + } +} +template +lp_status_t barrier_solver_t::run_ipm( + f_t start_time, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session, + std::unique_ptr>& owned_data) +{ + auto finish_session = [&](lp_status_t status) -> lp_status_t { + return finish_barrier_session(session, owned_data, status); + }; + iteration_data_t& data = *owned_data; data.cusparse_dual_residual_ = data.cusparse_view_.create_vector(data.d_dual_residual_); data.cusparse_r1_ = data.cusparse_view_.create_vector(data.d_r1_); data.cusparse_tmp4_ = data.cusparse_view_.create_vector(data.d_tmp4_); @@ -4154,27 +4719,27 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return lp_status_t::TIME_LIMIT; + return finish_session(lp_status_t::TIME_LIMIT); } i_t initial_status = initial_point(data); if (toc(start_time) > settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return lp_status_t::TIME_LIMIT; + return finish_session(lp_status_t::TIME_LIMIT); } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return lp_status_t::CONCURRENT_LIMIT; + return finish_session(lp_status_t::CONCURRENT_LIMIT); } if (initial_status != 0) { settings.log.printf("Unable to compute initial point\n"); - return lp_status_t::NUMERICAL_ISSUES; + return finish_session(lp_status_t::NUMERICAL_ISSUES); } // Upload initial point to device and compute initial residuals/norms on GPU data.d_complementarity_wv_residual_.resize(data.n_upper_bounds, stream_view_); @@ -4279,11 +4844,11 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return lp_status_t::TIME_LIMIT; + return finish_session(lp_status_t::TIME_LIMIT); } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return lp_status_t::CONCURRENT_LIMIT; + return finish_session(lp_status_t::CONCURRENT_LIMIT); } // Compute the affine step. This is the call that (re)factorizes the @@ -4300,29 +4865,29 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return lp_status_t::TIME_LIMIT; + return finish_session(lp_status_t::TIME_LIMIT); } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return lp_status_t::CONCURRENT_LIMIT; + return finish_session(lp_status_t::CONCURRENT_LIMIT); } f_t mu_aff, sigma, new_mu; @@ -4341,30 +4906,30 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return lp_status_t::TIME_LIMIT; + return finish_session(lp_status_t::TIME_LIMIT); } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return lp_status_t::CONCURRENT_LIMIT; + return finish_session(lp_status_t::CONCURRENT_LIMIT); } compute_final_direction(data); @@ -4431,17 +4996,17 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t::solve(f_t start_time, lp_solution_t::solve(f_t start_time, lp_solution_t::solve(f_t start_time, lp_solution_t +lp_status_t barrier_solver_t::solve(f_t start_time, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session) +{ + settings.log.printf("Barrier solver started at %.2f seconds\n", toc(start_time)); + try { + raft::common::nvtx::range fun_scope("Barrier: solve"); + + i_t n = lp.num_cols; + i_t m = lp.num_rows; + + solution.resize(m, n); + settings.log.printf( + "Barrier solver: %d constraints, %d variables, %ld nonzeros\n", m, n, lp.A.col_start[n]); + + settings.log.printf("\n"); + + if (lp.Q.n > 0) { + settings.log.printf("Quadratic objective matrix : %d nonzeros\n", lp.Q.row_start[lp.Q.n]); + } + if (lp.second_order_cone_dims.size() > 0) { + settings.log.printf("Second-order cones : %d\n", + static_cast(lp.second_order_cone_dims.size())); + } + + i_t num_free_variables = presolve_info.free_variable_pairs.size() / 2; + if (num_free_variables > 0) { + settings.log.printf("Free variables : %d\n", num_free_variables); + } + + i_t num_upper_bounds = 0; + for (i_t j = 0; j < n; j++) { + if (lp.upper[j] < inf) { num_upper_bounds++; } + } + + csc_matrix_t Q(lp.num_cols, 0, 0); + std::unique_ptr> owned_data; + + auto finish_session = [&](lp_status_t status) -> lp_status_t { + return finish_barrier_session(session, owned_data, status); + }; + + if (lp.Q.n > 0) { create_Q(lp, Q); } + bool reused_iteration_data = false; + if (session != nullptr) { + if (auto* cached = session->release_iteration_data()) { owned_data.reset(cached); } + if (owned_data) { + try { + bool rebound = owned_data->rebind_from_lp( + lp, presolve_info.direct_free_variables, Q, settings); + if (rebound) { + reused_iteration_data = true; + } else { + owned_data.reset(); + session->clear_symbolic_cache(); + } + } catch (const raft::cuda_error&) { + owned_data.reset(); + session->clear_symbolic_cache(); + } + } + } + if (!owned_data) { + barrier_symbolic_cache_t* adopt_cache = nullptr; + if (session != nullptr) { adopt_cache = session->symbolic_cache_for_reuse(lp.handle_ptr); } + owned_data = std::make_unique>( + lp, num_upper_bounds, presolve_info.direct_free_variables, Q, settings, adopt_cache); + } + iteration_data_t& data = *owned_data; + + if (reused_iteration_data) { + settings.log.printf("Barrier: reusing cuDSS symbolic analysis (sparsity hash match)\n"); + } else if (data.adopted_symbolic()) { + try { + if (!data.refresh_lp_numerics(lp)) { + settings.log.printf( + "Barrier: hash match but numeric refresh and symbolic rebuild failed\n"); + if (session != nullptr) { session->clear_symbolic_cache(); } + return finish_session(lp_status_t::NUMERICAL_ISSUES); + } + if (data.adopted_symbolic()) { + settings.log.printf("Barrier: reusing cuDSS symbolic analysis (sparsity hash match)\n"); + } else { + settings.log.printf( + "Barrier: rebuilt cuDSS symbolic analysis (%s nnz mismatch)\n", + data.use_augmented ? "augmented" : "adat"); + } + } catch (const raft::cuda_error&) { + settings.log.printf( + "Barrier: hash match but numeric refresh failed (CUDA); clearing symbolic cache\n"); + if (session != nullptr) { session->clear_symbolic_cache(); } + return finish_session(lp_status_t::NUMERICAL_ISSUES); + } + } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_session(lp_status_t::CONCURRENT_LIMIT); + } + if (data.indefinite_Q) { return finish_session(lp_status_t::NUMERICAL_ISSUES); } + if (data.symbolic_status != 0) { + settings.log.printf("Error in symbolic analysis\n"); + return finish_session(lp_status_t::NUMERICAL_ISSUES); + } + + return run_ipm(start_time, solution, session, owned_data); } catch (const raft::cuda_error& e) { settings.log.printf("Error in barrier_solver_t: %s\n", e.what()); return lp_status_t::NUMERICAL_ISSUES; } catch (const std::bad_alloc& e) { - // Covers rmm::out_of_memory and any other allocation failure. The barrier sizes its normal - // equations from the problem, so a shape it cannot hold is a property of the input rather - // than a defect, and the solvers running concurrently with it are unaffected. settings.log.printf("Out of memory in barrier_solver_t: %s\n", e.what()); return lp_status_t::NUMERICAL_ISSUES; } } + +template +void barrier_store_symbolic_cache_from_iteration_data(iteration_data_t& data, + barrier_symbolic_cache_t& cache) +{ + data.store_symbolic_cache(cache); +} + #ifdef DUAL_SIMPLEX_INSTANTIATE_DOUBLE template bool validate_barrier_cone_layout( const lp_problem_t& problem, const simplex_solver_settings_t& settings); @@ -4549,6 +5227,24 @@ template class barrier_solver_t; template class sparse_cholesky_base_t; template class sparse_cholesky_cudss_t; template class iteration_data_t; + +template void barrier_store_symbolic_cache_from_iteration_data( + iteration_data_t& data, barrier_symbolic_cache_t& cache); #endif +void destroy_iteration_data(iteration_data_t* data) { delete data; } + +void apply_barrier_linear_objective(iteration_data_t& data, + double const* barrier_c, + int n) +{ + if (barrier_c == nullptr || static_cast(data.c.size()) != n || + static_cast(data.d_c_.size()) != n) { + throw std::invalid_argument( + "update_q: barrier linear objective size does not match cached iteration_data_t."); + } + std::copy(barrier_c, barrier_c + n, data.c.data()); + raft::copy(data.d_c_.data(), data.c.data(), static_cast(n), data.handle_ptr->get_stream()); +} + } // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier.hpp b/cpp/src/barrier/barrier.hpp index 46fe91dcd4..1bf31cb191 100644 --- a/cpp/src/barrier/barrier.hpp +++ b/cpp/src/barrier/barrier.hpp @@ -15,9 +15,16 @@ #include #include +#include + #include #include + +namespace cuopt::cython { +class barrier_cache_t; +} // namespace cuopt::cython + namespace cuopt::mathematical_optimization::barrier { /** Validates SOC layout on an simplex::lp_problem_t before barrier presolve/solve. */ @@ -34,9 +41,20 @@ class barrier_solver_t { barrier_solver_t(const simplex::lp_problem_t& lp, const simplex::presolve_info_t& presolve, const simplex::simplex_solver_settings_t& settings); - simplex::lp_status_t solve(f_t start_time, simplex::lp_solution_t& solution); + simplex::lp_status_t solve(f_t start_time, + simplex::lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session = nullptr); + // Continue path: cached iteration_data_t already has the updated linear objective. + // Rebind settings, compute a new initial point, run IPM. Same status/solution contract as solve(). + simplex::lp_status_t barrier_solve_advanced(f_t start_time, + simplex::lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session); private: + simplex::lp_status_t run_ipm(f_t start_time, + simplex::lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session, + std::unique_ptr>& owned_data); void my_pop_range(bool debug) const; void create_Q(const simplex::lp_problem_t& lp, csc_matrix_t& Q); int initial_point(iteration_data_t& data); diff --git a/cpp/src/barrier/barrier_factorization_sparsity_hash.cu b/cpp/src/barrier/barrier_factorization_sparsity_hash.cu new file mode 100644 index 0000000000..7ecd48bd03 --- /dev/null +++ b/cpp/src/barrier/barrier_factorization_sparsity_hash.cu @@ -0,0 +1,25 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#include + +namespace cuopt::mathematical_optimization::barrier { + +template +barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( + device_csr_matrix_t& mat, rmm::cuda_stream_view stream) +{ + const csr_matrix_t host = mat.to_host(stream); + return hash_host_csr_sparsity_pattern(host.m, host.row_start, host.j); +} + +template barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( + device_csr_matrix_t&, rmm::cuda_stream_view); +template barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( + device_csr_matrix_t&, rmm::cuda_stream_view); + +} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp b/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp new file mode 100644 index 0000000000..799b267ce1 --- /dev/null +++ b/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp @@ -0,0 +1,126 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +#include + +#include +#include + +namespace cuopt::mathematical_optimization::barrier { + +using barrier_sparsity_hash_t = std::uint64_t; + +/// FNV-1a style mix for incremental hashing. +inline barrier_sparsity_hash_t barrier_hash_combine(barrier_sparsity_hash_t h, std::uint64_t value) +{ + constexpr barrier_sparsity_hash_t kPrime = 1099511628211ULL; + h ^= value; + h *= kPrime; + return h; +} + +inline barrier_sparsity_hash_t barrier_hash_u64(std::uint64_t value) +{ + return barrier_hash_combine(1469598103934665603ULL, value); +} + +/** + * @brief Hash CSR sparsity (row_start + col indices); numeric values are ignored. + */ +template +barrier_sparsity_hash_t hash_host_csr_sparsity_pattern(i_t num_rows, + const std::vector& row_start, + const std::vector& col_indices) +{ + barrier_sparsity_hash_t h = barrier_hash_u64(static_cast(num_rows)); + h = barrier_hash_combine(h, static_cast(col_indices.size())); + for (i_t k = 0; k <= num_rows; ++k) { + h = barrier_hash_combine(h, static_cast(row_start[static_cast(k)])); + } + for (i_t col : col_indices) { + h = barrier_hash_combine(h, static_cast(col)); + } + return h; +} + +/** + * @brief Hash the sparsity pattern of the augmented KKT matrix passed to cuDSS (host CSR). + * + * Must match the index layout produced by iteration_data_t::form_augmented(true). + */ +template +barrier_sparsity_hash_t hash_augmented_kkt_sparsity(const csc_matrix_t& A, + const csc_matrix_t& AT, + const csc_matrix_t& Q) +{ + const i_t n = A.n; + const i_t m = A.m; + const i_t size = n + m; + + std::vector row_start(static_cast(size + 1), 0); + std::vector col_indices; + col_indices.reserve(static_cast(2) * static_cast(A.col_start[n]) + + static_cast(n + m) + + (Q.n > 0 ? static_cast(Q.col_start[n]) : 0)); + + i_t q = 0; + for (i_t i = 0; i < n; ++i) { + row_start[static_cast(i)] = q; + if (Q.n == 0) { + col_indices.push_back(i); + ++q; + } else { + const i_t q_col_beg = Q.col_start[i]; + const i_t q_col_end = Q.col_start[i + 1]; + bool has_diagonal = false; + for (i_t p = q_col_beg; p < q_col_end; ++p) { + col_indices.push_back(Q.i[p]); + ++q; + if (Q.i[p] == i) { has_diagonal = true; } + } + if (!has_diagonal) { + col_indices.push_back(i); + ++q; + } + } + const i_t col_beg = A.col_start[i]; + const i_t col_end = A.col_start[i + 1]; + for (i_t p = col_beg; p < col_end; ++p) { + col_indices.push_back(A.i[p] + n); + ++q; + } + } + + for (i_t k = n; k < n + m; ++k) { + row_start[static_cast(k)] = q; + const i_t l = k - n; + const i_t col_beg = AT.col_start[l]; + const i_t col_end = AT.col_start[l + 1]; + for (i_t p = col_beg; p < col_end; ++p) { + col_indices.push_back(AT.i[p]); + ++q; + } + col_indices.push_back(k); + ++q; + } + row_start[static_cast(size)] = q; + + return hash_host_csr_sparsity_pattern(size, row_start, col_indices); +} + +/** + * @brief Hash CSR sparsity from a device matrix (copies row/col indices to host). + */ +template +barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( + device_csr_matrix_t& mat, rmm::cuda_stream_view stream); + +} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier_symbolic_cache.hpp b/cpp/src/barrier/barrier_symbolic_cache.hpp new file mode 100644 index 0000000000..63826a2530 --- /dev/null +++ b/cpp/src/barrier/barrier_symbolic_cache.hpp @@ -0,0 +1,86 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include +#include +#include + +#include + +#include +#include + +namespace cuopt::mathematical_optimization::barrier { + +using barrier_sparsity_hash_t = std::uint64_t; + +template +class sparse_cholesky_cudss_t; + +/** + * @brief Cached cuDSS symbolic state and GPU buffers for hash-gated barrier reuse. + * + * Holds reordering + symbolic factorization in @p chol, a sparsity hash used to gate reuse, + * and path-specific GPU workspace (augmented KKT or ADAT + cuSPARSE). + * + * Hash meaning: augmented store uses device KKT CSR (adopt uses matching host synthetic); + * ADAT store/adopt use the constraint-matrix @c device_A CSR pattern (not ADAT), so adopt can + * reject before pinning SpGEMM workspace. + */ +template +struct barrier_symbolic_cache_t { + std::shared_ptr> chol; + barrier_sparsity_hash_t sparsity_hash{0}; + raft::handle_t const* handle_ptr{nullptr}; + bool use_augmented{false}; + bool valid{false}; + + // --- Augmented KKT (use_augmented == true) --- + device_csr_matrix_t device_augmented; + rmm::device_uvector d_augmented_diagonal_indices_; + + // --- ADAT (use_augmented == false) --- + device_csr_matrix_t device_ADAT; + device_csc_matrix_t device_AD; + device_csr_matrix_t device_A; + rmm::device_uvector d_original_A_values; + rmm::device_uvector device_A_x_values; + std::unique_ptr> cusparse_info; + + explicit barrier_symbolic_cache_t(rmm::cuda_stream_view stream) + : device_augmented(stream), + d_augmented_diagonal_indices_(0, stream), + device_ADAT(stream), + device_AD(stream), + device_A(stream), + d_original_A_values(0, stream), + device_A_x_values(0, stream) + { + } + + void clear() + { + chol.reset(); + sparsity_hash = 0; + handle_ptr = nullptr; + use_augmented = false; + valid = false; + cusparse_info.reset(); + } + + [[nodiscard]] bool matches_reuse(barrier_sparsity_hash_t hash, + bool augmented, + raft::handle_t const* handle) const + { + return valid && handle != nullptr && handle_ptr == handle && use_augmented == augmented && + sparsity_hash == hash; + } +}; + +} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/cusparse_view.cu b/cpp/src/barrier/cusparse_view.cu index f787bed8f2..5e5a54139e 100644 --- a/cpp/src/barrier/cusparse_view.cu +++ b/cpp/src/barrier/cusparse_view.cu @@ -245,6 +245,16 @@ cusparse_view_t::~cusparse_view_t() if (A_T_ != nullptr) { CUOPT_CUSPARSE_TRY_NO_THROW(cusparseDestroySpMat(A_T_)); } } +template +void cusparse_view_t::update_matrix_values(const csc_matrix_t& A) +{ + const auto stream = handle_ptr_->get_stream(); + raft::copy(A_T_data_.data(), A.x.data(), A.x.size(), stream); + csr_matrix_t A_csr(A.m, A.n, 1); + A.to_compressed_row(A_csr); + raft::copy(A_data_.data(), A_csr.x.data(), A_csr.x.size(), stream); +} + template pdlp::cusparse_dn_vec_descr_wrapper_t cusparse_view_t::create_vector( rmm::device_uvector const& vec) diff --git a/cpp/src/barrier/cusparse_view.hpp b/cpp/src/barrier/cusparse_view.hpp index ea6bf363b9..1815b8a9b7 100644 --- a/cpp/src/barrier/cusparse_view.hpp +++ b/cpp/src/barrier/cusparse_view.hpp @@ -57,6 +57,8 @@ class cusparse_view_t { f_t beta, pdlp::cusparse_dn_vec_descr_wrapper_t const& y); + void update_matrix_values(const csc_matrix_t& A); + raft::handle_t const* handle_ptr_{nullptr}; private: diff --git a/cpp/src/barrier/device_sparse_matrix.cuh b/cpp/src/barrier/device_sparse_matrix.cuh index 974e2b0f4a..7272b9efa0 100644 --- a/cpp/src/barrier/device_sparse_matrix.cuh +++ b/cpp/src/barrier/device_sparse_matrix.cuh @@ -179,6 +179,10 @@ class device_csc_matrix_t { { } + device_csc_matrix_t(device_csc_matrix_t&&) = default; + device_csc_matrix_t& operator=(device_csc_matrix_t&&) = default; + device_csc_matrix_t& operator=(const device_csc_matrix_t&) = delete; + device_csc_matrix_t(const csc_matrix_t& A, rmm::cuda_stream_view stream) : m(A.m), n(A.n), @@ -318,6 +322,10 @@ class device_csr_matrix_t { { } + device_csr_matrix_t(device_csr_matrix_t&&) = default; + device_csr_matrix_t& operator=(device_csr_matrix_t&&) = default; + device_csr_matrix_t& operator=(const device_csr_matrix_t&) = delete; + device_csr_matrix_t(const csr_matrix_t& A, rmm::cuda_stream_view stream) : m(A.m), n(A.n), diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 01045847d1..f0a5b07f1b 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -33,6 +33,9 @@ class sparse_cholesky_base_t { virtual i_t solve(const dense_vector_t& b, dense_vector_t& x) = 0; virtual i_t solve(rmm::device_uvector& b, rmm::device_uvector& x) = 0; virtual void set_positive_definite(bool positive_definite) = 0; + virtual void invalidate_numeric_factor() {} + virtual void rebind_csr_matrix(device_csr_matrix_t& Arow) {} + virtual void rebind_settings(const simplex::simplex_solver_settings_t& settings) {} }; #define CUDSS_EXAMPLE_FREE \ @@ -143,7 +146,9 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { first_factor(true), positive_definite(true), A_created(false), - settings_(settings), + settings_(&settings), + symbolic_done_(false), + numeric_factor_valid_(false), stream(handle_ptr->get_stream()) { int major, minor, patch; @@ -155,9 +160,10 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { cuda_error = cudaSuccess; status = CUDSS_STATUS_SUCCESS; - if (CUDART_VERSION >= 13000 && settings_.concurrent_halt != nullptr && - settings_.num_gpus == 1) { + if (CUDART_VERSION >= 13000 && settings_->concurrent_halt != nullptr && + settings_->num_gpus == 1) { cuGetErrorString_func = cuopt::get_driver_entry_point("cuGetErrorString"); + // 1. Set up the GPU resources CUdevResource initial_device_GPU_resources = {}; auto cuDeviceGetDevResource_func = cuopt::get_driver_entry_point("cuDeviceGetDevResource"); @@ -277,18 +283,18 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { "cudssConfigSet for device count"); #if CUDSS_VERSION_MAJOR >= 0 && CUDSS_VERSION_MINOR >= 7 - if (settings_.concurrent_halt != nullptr) { + if (settings_->concurrent_halt != nullptr) { CUDSS_CALL_AND_CHECK_EXIT(cudssDataSet(handle, solverData, CUDSS_DATA_USER_HOST_INTERRUPT, - (void*)settings_.concurrent_halt, + (void*)settings_->concurrent_halt, sizeof(int)), status, "cudssDataSet for interrupt"); } - if (settings_.cudss_deterministic) { - settings_.log.printf("cuDSS solve mode : deterministic\n"); + if (settings_->cudss_deterministic) { + settings_->log.printf("cuDSS solve mode : deterministic\n"); int32_t deterministic = 1; CUDSS_CALL_AND_CHECK_EXIT( cudssConfigSet( @@ -307,7 +313,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { #endif #if USE_MATCHING - settings_.log.printf("Using matching\n"); + settings_->log.printf("Using matching\n"); int32_t use_matching = 1; CUDSS_CALL_AND_CHECK_EXIT( cudssConfigSet(solverConfig, CUDSS_CONFIG_USE_MATCHING, &use_matching, sizeof(int32_t)), @@ -373,7 +379,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { CUDA_CALL_AND_CHECK_EXIT(cudaStreamSynchronize(stream), "cudaStreamSynchronize"); #if CUDART_VERSION >= 13000 - if (settings_.concurrent_halt != nullptr && settings_.num_gpus == 1) { + if (settings_.concurrent_halt != nullptr && settings_->num_gpus == 1) { auto cuStreamDestroy_func = cuopt::get_driver_entry_point("cuStreamDestroy"); CU_CHECK(reinterpret_cast(cuStreamDestroy_func)(stream), reinterpret_cast(cuGetErrorString_func)); @@ -396,9 +402,9 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { csc_matrix_t A_col(Arow_host.m, Arow_host.n, 1); Arow_host.to_compressed_col(A_col); FILE* fid = fopen("A_to_factorize.mtx", "w"); - settings_.log.printf("writing matrix matrix\n"); + settings_->log.printf("writing matrix matrix\n"); A_col.write_matrix_market(fid); - settings_.log.printf("finished\n"); + settings_->log.printf("finished\n"); fclose(fid); } #endif @@ -407,9 +413,9 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { const f_t density = static_cast(nnz) / (static_cast(n) * static_cast(n)); if (first_factor && - ((settings_.ordering == -1 && density >= 0.05 && nnz > n) || settings_.ordering == 1) && + ((settings_->ordering == -1 && density >= 0.05 && nnz > n) || settings_->ordering == 1) && n > 1) { - settings_.log.printf("Reordering algorithm : AMD\n"); + settings_->log.printf("Reordering algorithm : AMD\n"); // Tell cuDSS to use AMD #if CUDSS_VERSION_MAJOR > 0 || (CUDSS_VERSION_MAJOR == 0 && CUDSS_VERSION_MINOR >= 8) cudssReorderingAlg_t reorder_alg = CUDSS_REORDERING_ALG_AMD; @@ -482,27 +488,27 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { raft::common::nvtx::range fun_scope("Barrier: cuDSS Analyze : CUDSS_PHASE_ANALYSIS"); status = cudssExecute(handle, CUDSS_PHASE_REORDERING, solverConfig, solverData, A, cudss_x, cudss_b); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf( + settings_->log.printf( "FAILED: CUDSS call ended unsuccessfully with status = %d, details: cuDSSExecute for " "reordering\n", status); return -1; } f_t reordering_time = toc(start_symbolic); - settings_.log.printf("Reordering time : %.2fs\n", reordering_time); + settings_->log.printf("Reordering time : %.2fs\n", reordering_time); start_symbolic_factor = tic(); status = cudssExecute( handle, CUDSS_PHASE_SYMBOLIC_FACTORIZATION, solverConfig, solverData, A, cudss_x, cudss_b); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf( + settings_->log.printf( "FAILED: CUDSS call ended unsuccessfully with status = %d, details: cuDSSExecute for " "symbolic factorization\n", status); @@ -511,32 +517,42 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { } RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); f_t symbolic_factorization_time = toc(start_symbolic_factor); - settings_.log.printf("Symbolic factorization time : %.2fs\n", symbolic_factorization_time); + settings_->log.printf("Symbolic factorization time : %.2fs\n", symbolic_factorization_time); int64_t lu_nz = 0; size_t size_written = 0; CUDSS_CALL_AND_CHECK( cudssDataGet(handle, solverData, CUDSS_DATA_LU_NNZ, &lu_nz, sizeof(int64_t), &size_written), status, "cudssDataGet for LU_NNZ"); - settings_.log.printf("Symbolic nonzeros in factor : %.2e\n", static_cast(lu_nz) / 2.0); + settings_->log.printf("Symbolic nonzeros in factor : %.2e\n", static_cast(lu_nz) / 2.0); // TODO: Is there any way to get nonzeros in the factors? // TODO: Is there any way to get flops for the factorization? RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); handle_ptr_->get_stream().synchronize(); + symbolic_done_ = true; + numeric_factor_valid_ = false; return 0; } i_t factorize(device_csr_matrix_t& Arow) override { raft::common::nvtx::range fun_scope("Factorize: cuDSS"); + if (!symbolic_done_ || !A_created) { + settings_->log.printf( + "Error: cuDSS factorize(device_csr) called before analyze (symbolic_done=%d A_created=%d)\n", + static_cast(symbolic_done_), + static_cast(A_created)); + return -1; + } + // #define PRINT_MATRIX_NORM #ifdef PRINT_MATRIX_NORM cudaStreamSynchronize(stream); csr_matrix_t Arow_host = Arow.to_host(Arow.row_start.stream()); csc_matrix_t A_col(Arow_host.m, Arow_host.n, 1); Arow_host.to_compressed_col(A_col); - settings_.log.printf( + settings_->log.printf( "before factorize || A to factor|| = %.16e hash: %zu\n", A_col.norm1(), A_col.hash()); cudaStreamSynchronize(stream); #endif @@ -545,7 +561,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { auto d_nnz = Arow.row_start.element(Arow.m, Arow.row_start.stream()); if (nnz != d_nnz) { - settings_.log.printf("Error: nnz %d != A_in.col_start[A_in.n] %d\n", nnz, d_nnz); + settings_->log.printf("Error: nnz %d != A_in.col_start[A_in.n] %d\n", nnz, d_nnz); return -1; } @@ -555,11 +571,11 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { f_t start_numeric = tic(); status = cudssExecute( handle, CUDSS_PHASE_FACTORIZATION, solverConfig, solverData, A, cudss_x, cudss_b); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf( + settings_->log.printf( "FAILED: CUDSS call ended unsuccessfully with status = %d, details: cuDSSExecute for " "factorization\n", status); @@ -571,7 +587,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { #endif f_t numeric_time = toc(start_numeric); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } @@ -585,16 +601,16 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { handle_ptr_->get_stream().synchronize(); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); if (info != 0) { - settings_.log.printf("Factorization failed info %d\n", info); + settings_->log.printf("Factorization failed info %d\n", info); return -1; } if (first_factor) { - settings_.log.debug("Factorization time : %.2fs\n", numeric_time); + settings_->log.debug("Factorization time : %.2fs\n", numeric_time); first_factor = false; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf("cuDSS Factorization failed\n"); + settings_->log.printf("cuDSS Factorization failed\n"); return -1; } return 0; @@ -608,15 +624,15 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { FILE* fid = fopen("A.mtx", "w"); A_in.write_matrix_market(fid); fclose(fid); - settings_.log.printf("Wrote A.mtx\n"); + settings_->log.printf("Wrote A.mtx\n"); #endif A_in.to_compressed_row(Arow); #ifdef CHECK_MATRIX - settings_.log.printf("Checking matrices\n"); + settings_->log.printf("Checking matrices\n"); A_in.check_matrix(); Arow.check_matrix(); - settings_.log.printf("Finished checking matrices\n"); + settings_->log.printf("Finished checking matrices\n"); #endif if (A_in.n != n) { printf("Analyze input does not match size %d != %d\n", A_in.n, n); @@ -690,7 +706,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { A_created = true; // Perform symbolic analysis - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } f_t start_analysis = tic(); @@ -700,7 +716,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { "cudssExecute for reordering"); f_t reorder_time = toc(start_analysis); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } @@ -714,8 +730,8 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { f_t symbolic_time = toc(start_symbolic); f_t analysis_time = toc(start_analysis); - settings_.log.printf("Symbolic factorization time : %.2fs\n", symbolic_time); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + settings_->log.printf("Symbolic factorization time : %.2fs\n", symbolic_time); + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); handle_ptr_->get_stream().synchronize(); return CONCURRENT_HALT_RETURN; @@ -726,7 +742,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { cudssDataGet(handle, solverData, CUDSS_DATA_LU_NNZ, &lu_nz, sizeof(int64_t), &size_written), status, "cudssDataGet for LU_NNZ"); - settings_.log.printf("Symbolic nonzeros in factor : %.2e\n", static_cast(lu_nz) / 2.0); + settings_->log.printf("Symbolic nonzeros in factor : %.2e\n", static_cast(lu_nz) / 2.0); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); handle_ptr_->get_stream().synchronize(); // TODO: Is there any way to get nonzeros in the factors? @@ -739,10 +755,10 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { csr_matrix_t Arow(A_in.n, A_in.m, A_in.col_start[A_in.n]); A_in.to_compressed_row(Arow); - if (A_in.n != n) { settings_.log.printf("Error A in n %d != size %d\n", A_in.n, n); } + if (A_in.n != n) { settings_->log.printf("Error A in n %d != size %d\n", A_in.n, n); } if (nnz != A_in.col_start[A_in.n]) { - settings_.log.printf( + settings_->log.printf( "Error: nnz %d != A_in.col_start[A_in.n] %d\n", nnz, A_in.col_start[A_in.n]); return -1; } @@ -766,7 +782,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { "cudssExecute for factorization"); f_t numeric_time = toc(start_numeric); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } @@ -779,16 +795,16 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); handle_ptr_->get_stream().synchronize(); if (info != 0) { - settings_.log.printf("Factorization failed info %d\n", info); + settings_->log.printf("Factorization failed info %d\n", info); return -1; } if (first_factor) { - settings_.log.debug("Factorization time : %.2fs\n", numeric_time); + settings_->log.debug("Factorization time : %.2fs\n", numeric_time); first_factor = false; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf("cuDSS Factorization failed\n"); + settings_->log.printf("cuDSS Factorization failed\n"); return -1; } return 0; @@ -817,11 +833,11 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { { handle_ptr_->get_stream().synchronize(); if (static_cast(b.size()) != n) { - settings_.log.printf("Error: b.size() %d != n %d\n", b.size(), n); + settings_->log.printf("Error: b.size() %d != n %d\n", b.size(), n); return -1; } if (static_cast(x.size()) != n) { - settings_.log.printf("Error: x.size() %d != n %d\n", x.size(), n); + settings_->log.printf("Error: x.size() %d != n %d\n", x.size(), n); return -1; } @@ -831,11 +847,11 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { cudssMatrixSetValues(cudss_x, x.data()), status, "cudssMatrixSetValues for x"); status = cudssExecute(handle, CUDSS_PHASE_SOLVE, solverConfig, solverData, A, cudss_x, cudss_b); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { + if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { return CONCURRENT_HALT_RETURN; } if (status != CUDSS_STATUS_SUCCESS) { - settings_.log.printf( + settings_->log.printf( "FAILED: CUDSS call ended unsuccessfully with status = %d, details: cuDSSExecute for " "solve\n", status); @@ -851,7 +867,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { raft::copy(b_host.data(), b.data(), n, stream); raft::copy(x_host.data(), x.data(), n, stream); cudaStreamSynchronize(stream); - settings_.log.printf("RHS norm %.16e, hash: %zu, Solution norm %.16e, hash: %zu\n", + settings_->log.printf("RHS norm %.16e, hash: %zu, Solution norm %.16e, hash: %zu\n", vector_norm2(b_host), compute_hash(b_host), vector_norm2(x_host), @@ -866,6 +882,63 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { this->positive_definite = positive_definite; } + void rebind_settings(const simplex::simplex_solver_settings_t& settings) override + { + settings_ = &settings; + } + + void invalidate_numeric_factor() override { numeric_factor_valid_ = false; } + + /// Re-point cuDSS CSR wrapper at current device buffers after in-place value refresh. + void rebind_csr_matrix(device_csr_matrix_t& Arow) override + { + if (!symbolic_done_ || !A_created) { return; } + auto d_nnz = Arow.row_start.element(Arow.m, Arow.row_start.stream()); + if (d_nnz != nnz) { return; } + status = cudssMatrixDestroy(A); + if (status != CUDSS_STATUS_SUCCESS) { + settings_->log.printf("cudssMatrixDestroy for A rebind failed: %d\n", status); + return; + } +#if CUDSS_VERSION_MAJOR > 0 || (CUDSS_VERSION_MAJOR == 0 && CUDSS_VERSION_MINOR >= 8) + status = cudssMatrixCreateCsr(&A, + n, + n, + nnz, + Arow.row_start.data(), + nullptr, + Arow.j.data(), + Arow.x.data(), + CUDSS_R_32I, + CUDSS_R_32I, + CUDSS_R_64F, + positive_definite ? CUDSS_MTYPE_SPD : CUDSS_MTYPE_SYMMETRIC, + CUDSS_MVIEW_FULL, + CUDSS_BASE_ZERO); +#else + status = cudssMatrixCreateCsr(&A, + n, + n, + nnz, + Arow.row_start.data(), + nullptr, + Arow.j.data(), + Arow.x.data(), + CUDA_R_32I, + CUDA_R_64F, + positive_definite ? CUDSS_MTYPE_SPD : CUDSS_MTYPE_SYMMETRIC, + CUDSS_MVIEW_FULL, + CUDSS_BASE_ZERO); +#endif + if (status != CUDSS_STATUS_SUCCESS) { + settings_->log.printf("cudssMatrixCreateCsr rebind failed: %d\n", status); + A_created = false; + return; + } + A_created = true; + numeric_factor_valid_ = false; + } + private: raft::handle_t const* handle_ptr_; i_t n; @@ -889,7 +962,10 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { f_t* x_values_d; f_t* b_values_d; - const simplex::simplex_solver_settings_t& settings_; + bool symbolic_done_; + bool numeric_factor_valid_; + const simplex::simplex_solver_settings_t* settings_; + CUgreenCtx barrier_green_ctx; CUstream stream; void* cuGetErrorString_func; diff --git a/cpp/src/dual_simplex/solve.cpp b/cpp/src/dual_simplex/solve.cpp index 7907abd3b9..79a5f0cebd 100644 --- a/cpp/src/dual_simplex/solve.cpp +++ b/cpp/src/dual_simplex/solve.cpp @@ -26,17 +26,90 @@ #include #include +#include +#include + #include #include #include +#include #include +#include #include namespace cuopt::mathematical_optimization::simplex { namespace { +template +bool can_continue_barrier_c_only(const user_problem_t& user_problem, + const simplex_solver_settings_t& settings, + cuopt::cython::barrier_cache_t* session) +{ + if (session == nullptr || !session->c_dirty()) { return false; } + if (session->iteration_data() == nullptr) { return false; } + auto const* front_end = session->front_end_cache(); + if (front_end == nullptr || front_end->barrier_lp == nullptr) { return false; } + if (user_problem.Q_values.empty()) { return false; } + if (settings.barrier_presolve_bound_free_variables != 0) { return false; } + return user_problem.num_cols == front_end->user_num_cols && + user_problem.num_rows == front_end->user_num_rows; +} + +template +void unscale_uncrush_barrier_to_user(const user_problem_t& user_problem, + const raft::handle_t* handle_ptr, + i_t original_num_rows, + i_t original_num_cols, + const lp_problem_t& barrier_lp, + const presolve_info_t& presolve_info, + const std::vector& column_scales, + const std::vector& row_scales, + const simplex_solver_settings_t& barrier_settings, + const lp_solution_t& barrier_solution, + lp_solution_t& solution) +{ + std::vector unscaled_x(barrier_lp.num_cols); + std::vector unscaled_y(barrier_lp.num_rows); + std::vector unscaled_z(barrier_lp.num_cols); + unscale_solution(column_scales, + row_scales, + barrier_solution.x, + barrier_solution.y, + barrier_solution.z, + unscaled_x, + unscaled_y, + unscaled_z); + + // Dummy converted LP: sizes only. Bound-free=0 so uncrush_solution never reads A. + lp_problem_t converted(handle_ptr, original_num_rows, original_num_cols, 0); + lp_solution_t lp_solution(original_num_rows, original_num_cols); + uncrush_solution(presolve_info, + barrier_settings, + converted, + unscaled_x, + unscaled_y, + unscaled_z, + lp_solution.x, + lp_solution.y, + lp_solution.z); + + uncrush_primal_solution(user_problem, converted, lp_solution.x, solution.x); + uncrush_dual_solution( + user_problem, converted, lp_solution.y, lp_solution.z, solution.y, solution.z); + solution.objective = + barrier_solution.user_objective / user_problem.obj_scale - user_problem.obj_constant; + solution.user_objective = barrier_solution.user_objective; + solution.l2_primal_residual = barrier_solution.l2_primal_residual; + solution.l2_dual_residual = barrier_solution.l2_dual_residual; + solution.iterations = barrier_solution.iterations; +} + +} // namespace + +namespace { + template void write_matlab(const std::string& filename, const simplex::lp_problem_t& lp) { @@ -352,14 +425,45 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session, const raft::handle_t* handle_ptr) { lp_status_t status = lp_status_t::UNSET; + simplex_solver_settings_t barrier_settings = settings; + + if (can_continue_barrier_c_only(user_problem, barrier_settings, session)) { + settings.log.printf( + "Barrier: continue from session (skip convert/presolve/scaling)\n"); + auto* front_end = session->front_end_cache(); + lp_solution_t barrier_solution(front_end->barrier_lp->num_rows, + front_end->barrier_lp->num_cols); + barrier::barrier_solver_t barrier_solver( + *front_end->barrier_lp, front_end->presolve_info, barrier_settings); + lp_status_t barrier_status = + barrier_solver.barrier_solve_advanced(start_time, barrier_solution, session); + if (barrier_status == lp_status_t::OPTIMAL) { + unscale_uncrush_barrier_to_user(user_problem, + session->handle_ptr(), + front_end->original_num_rows, + front_end->original_num_cols, + *front_end->barrier_lp, + front_end->presolve_info, + front_end->column_scales, + front_end->row_scales, + barrier_settings, + barrier_solution, + solution); + session->set_c_dirty(false); + } else { + session->clear_front_end_cache(); + } + return barrier_status; + } + lp_problem_t original_lp(handle_ptr, 1, 1, 1); // Convert the user problem to a linear program with only equality constraints std::vector new_slacks; - simplex_solver_settings_t barrier_settings = settings; dualize_info_t dualize_info; convert_user_problem(user_problem, barrier_settings, original_lp, new_slacks, dualize_info); if (!barrier::validate_barrier_cone_layout(original_lp, barrier_settings)) { @@ -389,7 +493,46 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us lp_solution_t barrier_solution(barrier_lp.num_rows, barrier_lp.num_cols); barrier::barrier_solver_t barrier_solver(barrier_lp, presolve_info, barrier_settings); - lp_status_t barrier_status = barrier_solver.solve(start_time, barrier_solution); + lp_status_t barrier_status = barrier_solver.solve(start_time, barrier_solution, session); + + if (session != nullptr) { + if (barrier_status == lp_status_t::OPTIMAL) { + auto front_end = std::make_unique(); + front_end->c_dirty = false; + front_end->user_num_cols = user_problem.num_cols; + front_end->user_num_rows = user_problem.num_rows; + front_end->original_num_cols = original_lp.num_cols; + front_end->original_num_rows = original_lp.num_rows; + front_end->barrier_num_cols = barrier_lp.num_cols; + front_end->barrier_num_rows = barrier_lp.num_rows; + front_end->obj_scale = user_problem.obj_scale; + front_end->obj_constant = user_problem.obj_constant; + front_end->presolve_info = presolve_info; + front_end->column_scales = column_scales; + front_end->row_scales = row_scales; + front_end->barrier_lp = std::make_unique>(barrier_lp); + { + try { + auto crushed = cuopt::cython::crush_user_linear_objective( + *front_end, user_problem.objective.data(), user_problem.num_cols); + front_end->linear_obj_shift.resize(static_cast(barrier_lp.num_cols), 0.0); + if (static_cast(crushed.size()) == barrier_lp.num_cols) { + for (int j = 0; j < barrier_lp.num_cols; ++j) { + front_end->linear_obj_shift[static_cast(j)] = + barrier_lp.objective[static_cast(j)] - + crushed[static_cast(j)]; + } + } + } catch (std::exception const&) { + front_end->linear_obj_shift.assign(static_cast(barrier_lp.num_cols), 0.0); + } + } + session->store_front_end_cache(std::move(front_end)); + } else { + session->clear_front_end_cache(); + } + } + if (barrier_status == lp_status_t::OPTIMAL) { #ifdef COMPUTE_SCALED_RESIDUALS std::vector scaled_residual = barrier_lp.rhs; @@ -682,20 +825,23 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us template lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, const simplex_solver_settings_t& settings, - f_t start_time, - lp_solution_t& solution) + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session) { + f_t start_time = tic(); return solve_linear_program_with_barrier( - user_problem, settings, start_time, solution, user_problem.handle_ptr); + user_problem, settings, start_time, solution, session, user_problem.handle_ptr); } template lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, const simplex_solver_settings_t& settings, - lp_solution_t& solution) + f_t start_time, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session) { - f_t start_time = tic(); - return solve_linear_program_with_barrier(user_problem, settings, start_time, solution); + return solve_linear_program_with_barrier( + user_problem, settings, start_time, solution, session, user_problem.handle_ptr); } template @@ -837,19 +983,22 @@ template lp_status_t solve_linear_program_with_advanced_basis( template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, - lp_solution_t& solution); + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session); template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, double start_time, - lp_solution_t& solution); + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session); template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, double start_time, lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session, const raft::handle_t* handle_ptr); template lp_status_t solve_linear_program(const user_problem_t& user_problem, diff --git a/cpp/src/dual_simplex/solve.hpp b/cpp/src/dual_simplex/solve.hpp index 90c2dbd690..e2cb71745a 100644 --- a/cpp/src/dual_simplex/solve.hpp +++ b/cpp/src/dual_simplex/solve.hpp @@ -17,6 +17,10 @@ namespace cuopt { struct work_limit_context_t; } +namespace cuopt::cython { +class barrier_cache_t; +} // namespace cuopt::cython + namespace cuopt::mathematical_optimization::simplex { template @@ -88,21 +92,26 @@ lp_status_t solve_linear_program_with_advanced_basis( work_limit_context_t* work_unit_context = nullptr); template -lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, - const simplex_solver_settings_t& settings, - lp_solution_t& solution); +lp_status_t solve_linear_program_with_barrier( + const user_problem_t& user_problem, + const simplex_solver_settings_t& settings, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session = nullptr); template -lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, - const simplex_solver_settings_t& settings, - f_t start_time, - lp_solution_t& solution); +lp_status_t solve_linear_program_with_barrier( + const user_problem_t& user_problem, + const simplex_solver_settings_t& settings, + f_t start_time, + lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session = nullptr); template lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, + cuopt::cython::barrier_cache_t* session, const raft::handle_t* handle_ptr); template diff --git a/cpp/src/pdlp/CMakeLists.txt b/cpp/src/pdlp/CMakeLists.txt index 2f90f94872..9673d283dc 100644 --- a/cpp/src/pdlp/CMakeLists.txt +++ b/cpp/src/pdlp/CMakeLists.txt @@ -39,6 +39,7 @@ set(LP_CORE_FILES # C and Python adapter files set(LP_ADAPTER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/utilities/cython_solve.cu + ${CMAKE_CURRENT_SOURCE_DIR}/utilities/barrier_cache.cu ${CMAKE_CURRENT_SOURCE_DIR}/cuopt_c.cpp ) diff --git a/cpp/src/pdlp/solve.cu b/cpp/src/pdlp/solve.cu index 80b3da2c18..e4c2e3e04c 100644 --- a/cpp/src/pdlp/solve.cu +++ b/cpp/src/pdlp/solve.cu @@ -35,6 +35,7 @@ #include #include #include +#include #include #include @@ -62,6 +63,7 @@ #include #include +#include #include #include #include @@ -72,6 +74,52 @@ namespace cuopt::mathematical_optimization { +namespace { + +template +uint64_t fnv1a64_mix(uint64_t hash, uint64_t value) +{ + constexpr uint64_t kFnvPrime = 1099511628211ULL; + constexpr uint64_t kFnvOffset = 14695981039346656037ULL; + if (hash == 0) { hash = kFnvOffset; } + for (int shift = 0; shift < 64; shift += 8) { + hash ^= (value >> shift) & 0xFFULL; + hash *= kFnvPrime; + } + return hash; +} + +template +uint64_t compute_problem_fingerprint(const optimization_problem_t& op) +{ + uint64_t hash = fnv1a64_mix(0, static_cast(op.get_n_variables())); + hash = fnv1a64_mix(hash, static_cast(op.get_n_constraints())); + hash = fnv1a64_mix(hash, static_cast(op.get_nnz())); + + const auto offsets = op.get_constraint_matrix_offsets_host(); + for (i_t off : offsets) { + hash = fnv1a64_mix(hash, static_cast(off)); + } + const auto indices = op.get_constraint_matrix_indices_host(); + for (i_t idx : indices) { + hash = fnv1a64_mix(hash, static_cast(idx)); + } + + if (op.has_quadratic_objective()) { + const auto q_offsets = op.get_quadratic_objective_offsets(); + for (i_t off : q_offsets) { + hash = fnv1a64_mix(hash, static_cast(off)); + } + const auto q_indices = op.get_quadratic_objective_indices(); + for (i_t idx : q_indices) { + hash = fnv1a64_mix(hash, static_cast(idx)); + } + } + return hash; +} + +} // namespace + template extern rmm::device_uvector gpu_cast(const rmm::device_uvector& src, rmm::cuda_stream_view stream); @@ -492,7 +540,8 @@ std::tuple, simplex::lp_status_t, f_t, f_t, f_t const simplex::user_problem_t& user_problem, pdlp_solver_settings_t const& settings, const timer_t& timer, - const raft::handle_t* handle_ptr) + const raft::handle_t* handle_ptr, + cuopt::cython::barrier_cache_t* session = nullptr) { f_t norm_user_objective = vector_norm2(user_problem.objective); f_t norm_rhs = vector_norm2(user_problem.rhs); @@ -529,7 +578,7 @@ std::tuple, simplex::lp_status_t, f_t, f_t, f_t simplex::lp_solution_t solution(user_problem.num_rows, user_problem.num_cols); auto status = simplex::solve_linear_program_with_barrier( - user_problem, barrier_settings, timer.get_tic_start(), solution, handle_ptr); + user_problem, barrier_settings, timer.get_tic_start(), solution, session, handle_ptr); if (status == simplex::lp_status_t::OPTIMAL) { barrier::project_barrier_solution_to_model_variables(user_problem, solution); @@ -553,12 +602,14 @@ template optimization_problem_solution_t run_barrier( mip::problem_t& problem, pdlp_solver_settings_t const& settings, - const timer_t& timer) + const timer_t& timer, + cuopt::cython::barrier_cache_t* session = nullptr) { // Convert data structures to dual simplex format and back simplex::user_problem_t dual_simplex_problem = cuopt_problem_to_user_problem(problem.handle_ptr, problem, false); - auto sol_dual_simplex = run_barrier(dual_simplex_problem, settings, timer, problem.handle_ptr); + auto sol_dual_simplex = + run_barrier(dual_simplex_problem, settings, timer, problem.handle_ptr, session); return convert_dual_simplex_sol(problem, std::get<0>(sol_dual_simplex), std::get<1>(sol_dual_simplex), @@ -1806,7 +1857,7 @@ optimization_problem_solution_t solve_lp_with_method( if (settings.method == method_t::DualSimplex) { return run_dual_simplex(problem, settings, timer); } else if (settings.method == method_t::Barrier) { - return run_barrier(problem, settings, timer); + return run_barrier(problem, settings, timer, settings.barrier_cache); } else if (settings.method == method_t::Concurrent) { return run_concurrent(problem, settings, timer, is_batch_mode); } else { @@ -1834,7 +1885,10 @@ optimization_problem_solution_t solve_qcqp( print_version_info(); // Init libraries before to not include it in solve time - init_handler(op_problem.get_handle_ptr()); + { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C02); + init_handler(op_problem.get_handle_ptr()); + } auto qcqp_timer = cuopt::timer_t(settings.time_limit); @@ -1866,11 +1920,18 @@ optimization_problem_solution_t solve_qcqp( CUOPT_LOG_INFO("Writing user problem to file: %s", settings.user_problem_file.c_str()); op_problem.write_to_mps(settings.user_problem_file); } + { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C03); + [[maybe_unused]] const uint64_t fingerprint = compute_problem_fingerprint(op_problem); + } // Convert data structures to dual simplex format and back simplex::user_problem_t dual_simplex_problem = cuopt_optimization_problem_to_user_problem(op_problem.get_handle_ptr(), op_problem); - auto sol_dual_simplex = - run_barrier(dual_simplex_problem, settings, qcqp_timer, op_problem.get_handle_ptr()); + auto sol_dual_simplex = run_barrier(dual_simplex_problem, + settings, + qcqp_timer, + op_problem.get_handle_ptr(), + settings.barrier_cache); auto solution = convert_dual_simplex_sol(op_problem, std::get<0>(sol_dual_simplex), std::get<1>(sol_dual_simplex), @@ -1992,7 +2053,10 @@ optimization_problem_solution_t solve_lp( // Init libraries before to not include it in solve time // This needs to be called before pdlp is initialized - init_handler(op_problem.get_handle_ptr()); + { + CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C02); + init_handler(op_problem.get_handle_ptr()); + } raft::common::nvtx::range fun_scope("Running solver"); diff --git a/cpp/src/pdlp/utilities/barrier_cache.cu b/cpp/src/pdlp/utilities/barrier_cache.cu new file mode 100644 index 0000000000..748e0663f3 --- /dev/null +++ b/cpp/src/pdlp/utilities/barrier_cache.cu @@ -0,0 +1,174 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace cuopt::cython { + +using barrier_iteration_data_t = + mathematical_optimization::barrier::iteration_data_t; +using barrier_iteration_data_ptr = + std::unique_ptr; + +struct barrier_cache_t::impl { + impl(std::unique_ptr stream_in, std::unique_ptr handle_in) + : stream(std::move(stream_in)), + handle(std::move(handle_in)), + iteration_data(nullptr, &mathematical_optimization::barrier::destroy_iteration_data) + { + } + + std::unique_ptr stream; + std::unique_ptr handle; + std::optional> + symbolic_cache; + barrier_iteration_data_ptr iteration_data; + std::unique_ptr front_end; +}; + +barrier_cache_t::barrier_cache_t(std::unique_ptr stream, + std::unique_ptr handle) + : impl_(std::make_unique(std::move(stream), std::move(handle))) +{ +} + +barrier_cache_t::~barrier_cache_t() = default; + +barrier_cache_t::barrier_cache_t(barrier_cache_t&&) noexcept = default; +barrier_cache_t& barrier_cache_t::operator=(barrier_cache_t&&) noexcept = default; + +std::unique_ptr barrier_cache_t::create(unsigned stream_flags) +{ + auto stream = std::make_unique(static_cast(stream_flags)); + auto handle = std::make_unique(*stream); + return std::unique_ptr( + new barrier_cache_t(std::move(stream), std::move(handle))); +} + +raft::handle_t* barrier_cache_t::handle_ptr() +{ + return impl_->handle.get(); +} + +raft::handle_t const* barrier_cache_t::handle_ptr() const +{ + return impl_->handle.get(); +} + +rmm::cuda_stream_view barrier_cache_t::stream_view() const +{ + return impl_->stream->view(); +} + +mathematical_optimization::barrier::barrier_symbolic_cache_t* +barrier_cache_t::symbolic_cache_for_reuse(raft::handle_t const* handle) +{ + if (handle == nullptr || !impl_->symbolic_cache.has_value() || !impl_->symbolic_cache->valid || + impl_->symbolic_cache->handle_ptr != handle) { + return nullptr; + } + return &(*impl_->symbolic_cache); +} + +void barrier_cache_t::clear_symbolic_cache() +{ + impl_->symbolic_cache.reset(); + clear_iteration_data(); + clear_front_end_cache(); +} + +void barrier_cache_t::store_symbolic_cache( + mathematical_optimization::barrier::iteration_data_t& data) +{ + if (!impl_->symbolic_cache.has_value()) { + impl_->symbolic_cache.emplace(impl_->handle->get_stream()); + } + mathematical_optimization::barrier::barrier_store_symbolic_cache_from_iteration_data( + data, *impl_->symbolic_cache); +} + +void barrier_cache_t::store_iteration_data(barrier_iteration_data_t* data) +{ + impl_->iteration_data.reset(data); +} + +barrier_iteration_data_t* barrier_cache_t::release_iteration_data() +{ + return impl_->iteration_data.release(); +} + +barrier_iteration_data_t* barrier_cache_t::iteration_data() +{ + return impl_->iteration_data.get(); +} + +void barrier_cache_t::clear_iteration_data() { impl_->iteration_data.reset(); } + +void barrier_cache_t::store_front_end_cache(std::unique_ptr cache) +{ + impl_->front_end = std::move(cache); +} + +barrier_front_end_cache_t* barrier_cache_t::front_end_cache() { return impl_->front_end.get(); } + +barrier_front_end_cache_t const* barrier_cache_t::front_end_cache() const +{ + return impl_->front_end.get(); +} + +void barrier_cache_t::clear_front_end_cache() { impl_->front_end.reset(); } + +void barrier_cache_t::set_c_dirty(bool dirty) +{ + if (impl_->front_end) { impl_->front_end->c_dirty = dirty; } +} + +bool barrier_cache_t::c_dirty() const +{ + return impl_->front_end != nullptr && impl_->front_end->c_dirty; +} + +bool barrier_cache_t::has_front_end_cache() const { return impl_->front_end != nullptr; } + +void barrier_cache_t::update_linear_objective(double const* c, int n) +{ + cuopt_expects(impl_->front_end != nullptr, + error_type_t::ValidationError, + "update_q: no front-end cache; Solve with sequence_solve first."); + cuopt_expects(impl_->iteration_data.get() != nullptr, + error_type_t::ValidationError, + "update_q: no cached iteration_data; Solve a QP to Optimal first."); + std::vector crushed; + try { + crushed = crush_user_linear_objective(*impl_->front_end, c, n); + } catch (std::invalid_argument const& e) { + cuopt_expects(false, error_type_t::ValidationError, "%s", e.what()); + } + if (impl_->front_end->linear_obj_shift.size() == crushed.size()) { + for (std::size_t j = 0; j < crushed.size(); ++j) { + crushed[j] += impl_->front_end->linear_obj_shift[j]; + } + } + try { + mathematical_optimization::barrier::apply_barrier_linear_objective( + *impl_->iteration_data, crushed.data(), static_cast(crushed.size())); + } catch (std::invalid_argument const& e) { + cuopt_expects(false, error_type_t::ValidationError, "%s", e.what()); + } + impl_->front_end->c_dirty = true; +} + +} // namespace cuopt::cython diff --git a/cpp/src/pdlp/utilities/barrier_front_end_cache.hpp b/cpp/src/pdlp/utilities/barrier_front_end_cache.hpp new file mode 100644 index 0000000000..d405ef33b9 --- /dev/null +++ b/cpp/src/pdlp/utilities/barrier_front_end_cache.hpp @@ -0,0 +1,98 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +#include +#include +#include + +namespace cuopt::cython { + +/** + * Convert / presolve / scaling state retained on barrier_cache_t after Optimal. + * Enough to crush a new user-space linear objective into barrier space (C) and to + * uncrush a solution (D) without rerunning those algorithms. + */ +struct barrier_front_end_cache_t { + bool c_dirty{false}; + + int user_num_cols{0}; + int user_num_rows{0}; + int original_num_cols{0}; + int original_num_rows{0}; + int barrier_num_cols{0}; + int barrier_num_rows{0}; + double obj_scale{1.0}; + double obj_constant{0.0}; + + cuopt::mathematical_optimization::simplex::presolve_info_t presolve_info; + std::vector column_scales; + std::vector row_scales; + // Barrier linear objective minus crush(user c) from the first solve (Q·ℓ shift, etc.). + std::vector linear_obj_shift; + std::unique_ptr> barrier_lp; +}; + +inline std::vector crush_user_linear_objective(barrier_front_end_cache_t const& fe, + double const* c, + int n) +{ + if (c == nullptr || n != fe.user_num_cols) { + throw std::invalid_argument( + "update_q: linear objective length must match the cached user column count."); + } + if (fe.original_num_cols < fe.user_num_cols) { + throw std::invalid_argument("update_q: cached original column count is smaller than user n."); + } + + std::vector orig(static_cast(fe.original_num_cols), 0.0); + for (int j = 0; j < n; ++j) { + orig[static_cast(j)] = c[j]; + } + for (int j : fe.presolve_info.negated_variables) { + orig[static_cast(j)] *= -1.0; + } + + std::vector presolved; + if (!fe.presolve_info.remaining_variables.empty()) { + presolved.resize(fe.presolve_info.remaining_variables.size()); + for (std::size_t k = 0; k < fe.presolve_info.remaining_variables.size(); ++k) { + presolved[k] = orig[static_cast(fe.presolve_info.remaining_variables[k])]; + } + } else { + presolved = std::move(orig); + } + + auto const& pairs = fe.presolve_info.free_variable_pairs; + if (!pairs.empty()) { + if (pairs.size() % 2 != 0) { + throw std::invalid_argument("update_q: free_variable_pairs size is not even."); + } + std::size_t extra = pairs.size() / 2; + presolved.resize(presolved.size() + extra); + for (std::size_t k = 0; k < extra; ++k) { + int u = pairs[2 * k]; + int v = pairs[2 * k + 1]; + presolved[static_cast(v)] = -presolved[static_cast(u)]; + } + } + + if (static_cast(presolved.size()) != fe.barrier_num_cols || + fe.column_scales.size() != presolved.size()) { + throw std::invalid_argument( + "update_q: crushed objective size does not match barrier columns / column_scales."); + } + for (std::size_t j = 0; j < presolved.size(); ++j) { + presolved[j] /= fe.column_scales[j]; + } + return presolved; +} + +} // namespace cuopt::cython diff --git a/cpp/src/pdlp/utilities/cython_solve.cu b/cpp/src/pdlp/utilities/cython_solve.cu index ed77c4f722..fb45bcd732 100644 --- a/cpp/src/pdlp/utilities/cython_solve.cu +++ b/cpp/src/pdlp/utilities/cython_solve.cu @@ -6,6 +6,7 @@ /* clang-format on */ #include + #include #include #include @@ -18,6 +19,9 @@ #include #include #include +#include +#include + #include #include #include @@ -30,11 +34,26 @@ #include #include +#include + #include namespace cuopt { namespace cython { +namespace { + +bool uses_barrier_session_path( + cuopt::mathematical_optimization::solver_settings_t& solver_settings, + cuopt::mathematical_optimization::io::data_model_view_t const& data_model) +{ + if (data_model.has_quadratic_objective() || data_model.has_quadratic_constraints()) { return true; } + return solver_settings.get_pdlp_settings().method == + cuopt::mathematical_optimization::method_t::Barrier; +} + +} // namespace + /** * @brief Wrapper for linear_programming to expose the API to cython * @@ -96,24 +115,68 @@ std::unique_ptr call_solve( cuopt::mathematical_optimization::io::data_model_view_t* data_model, cuopt::mathematical_optimization::solver_settings_t* solver_settings, unsigned int flags, - bool is_batch_mode) + bool is_batch_mode, + barrier_cache_t* session_in) { raft::common::nvtx::range fun_scope("Call Solve"); + namespace cache_profile = cuopt::linear_programming::cache_profile; + if (cache_profile::enabled()) { cache_profile::reset(); } + + cuopt_expects(data_model != nullptr, + error_type_t::ValidationError, + "call_solve: data_model is null."); + cuopt_expects(solver_settings != nullptr, + error_type_t::ValidationError, + "call_solve: solver_settings is null."); + // Determine memory backend based on execution mode auto memory_backend = cuopt::mathematical_optimization::get_memory_backend_type(); solver_ret_t response; + auto& pdlp_settings = solver_settings->get_pdlp_settings(); + const bool sequence_solve = pdlp_settings.sequence_solve; + const bool barrier_path = uses_barrier_session_path(*solver_settings, *data_model); + const bool want_session = (session_in != nullptr || sequence_solve) && barrier_path && + memory_backend == cuopt::mathematical_optimization::memory_backend_t::GPU && + !is_batch_mode; + + std::unique_ptr owned_session; + barrier_cache_t* active_session = session_in; + pdlp_settings.barrier_cache = nullptr; + + rmm::cuda_stream ephemeral_stream(static_cast(flags)); + raft::handle_t ephemeral_handle(ephemeral_stream); + raft::handle_t* solve_handle = &ephemeral_handle; + // Create problem instance and CUDA resources based on memory backend if (memory_backend == cuopt::mathematical_optimization::memory_backend_t::GPU) { - // GPU memory backend: Create CUDA resources and GPU problem - rmm::cuda_stream stream(static_cast(flags)); - const raft::handle_t handle_{stream}; + if (want_session) { + if (active_session == nullptr) { + const auto handle_start = std::chrono::steady_clock::now(); + owned_session = barrier_cache_t::create(flags); + active_session = owned_session.get(); + if (cache_profile::enabled()) { + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - handle_start).count(); + cache_profile::add(cache_profile::cache_id::C01, elapsed); + } + } + solve_handle = active_session->handle_ptr(); + pdlp_settings.barrier_cache = active_session; + } else { + const auto handle_start = std::chrono::steady_clock::now(); + if (cache_profile::enabled()) { + const double elapsed = + std::chrono::duration(std::chrono::steady_clock::now() - handle_start).count(); + cache_profile::add(cache_profile::cache_id::C01, elapsed); + } + } - auto problem = cuopt::mathematical_optimization::optimization_problem_t(&handle_); + auto problem = cuopt::mathematical_optimization::optimization_problem_t(solve_handle); cuopt::mathematical_optimization::populate_from_data_model_view( - &problem, data_model, solver_settings, &handle_); + &problem, data_model, solver_settings, solve_handle); // Call appropriate solve function and convert to ret struct if (problem.get_problem_category() == mathematical_optimization::problem_category_t::LP) { @@ -142,6 +205,8 @@ std::unique_ptr call_solve( gpu_sols.last_restart_duality_gap_primal_solution_->set_stream(rmm::cuda_stream_per_thread); gpu_sols.last_restart_duality_gap_dual_solution_->set_stream(rmm::cuda_stream_per_thread); + if (owned_session) { response.lp_ret.barrier_cache = std::move(owned_session); } + } else { // MIP solve auto mip_solution_ptr = @@ -200,6 +265,10 @@ std::unique_ptr call_solve( } } + if (cache_profile::enabled()) { cache_profile::log_summary(); } + + pdlp_settings.barrier_cache = nullptr; + return std::make_unique(std::move(response)); } @@ -288,7 +357,7 @@ std::pair>, double> call_batch_solve( #pragma omp parallel for num_threads(max_thread) for (std::size_t i = 0; i < size; ++i) - list[i] = call_solve(data_models[i], solver_settings, cudaStreamNonBlocking, is_batch_mode); + list[i] = call_solve(data_models[i], solver_settings, cudaStreamNonBlocking, is_batch_mode, nullptr); auto end = std::chrono::high_resolution_clock::now(); auto duration = std::chrono::duration_cast(end - start_solver); diff --git a/python/cuopt/cuopt/linear_programming/data_model/data_model.py b/python/cuopt/cuopt/linear_programming/data_model/data_model.py index 7bcdfaea9b..fe9b1d42f4 100644 --- a/python/cuopt/cuopt/linear_programming/data_model/data_model.py +++ b/python/cuopt/cuopt/linear_programming/data_model/data_model.py @@ -228,6 +228,23 @@ def set_objective_coefficients(self, c): """ super().set_objective_coefficients(c) + @catch_cuopt_exception + def update_q(self, c): + """ + Update the linear objective coefficients (c) for a session re-solve. + + Writes user-space ``c`` onto this DataModel. If a Barrier session is + present, also maps ``c`` into the cached barrier workspace and marks + it dirty (quadratic ``Q``, ``A``, and bounds must stay unchanged). + + Parameters + ---------- + c : array-like of float64 + Linear objective coefficients, length equal to the number of + variables on the first session solve. + """ + super().update_q(c) + @catch_cuopt_exception def set_objective_scaling_factor(self, objective_scaling_factor): """ diff --git a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pxd b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pxd index 6c401b59f5..aae627a927 100644 --- a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pxd +++ b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pxd @@ -12,6 +12,7 @@ from libcpp.memory cimport unique_ptr cdef class DataModel: cdef unique_ptr[data_model_view_t[int, double]] c_data_model_view + cdef object barrier_cache_capsule cdef void _set_cpp_quadratic_constraints( self, data_model_view_t[int, double]* c_data_model_view diff --git a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx index ec8bdf3730..2c063282e4 100644 --- a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx +++ b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx @@ -21,6 +21,14 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector +cdef extern from "Python.h": + bint PyCapsule_IsValid(object cap, const char* name) + void* PyCapsule_GetPointer(object cap, const char* name) + +cdef extern from "cuopt/mathematical_optimization/utilities/barrier_cache.hpp" namespace "cuopt::cython": # noqa + cdef cppclass barrier_cache_t: + void update_linear_objective(const double* c, int n) except + + def type_cast(np_obj, np_type, name): if not isinstance(np_obj, np.ndarray): @@ -41,6 +49,7 @@ cdef class DataModel: def __init__(self): self.c_data_model_view.reset(new data_model_view_t[int, double]()) + self.barrier_cache_capsule = None self.maximize = False self.A_values = np.array([]) @@ -68,6 +77,14 @@ cdef class DataModel: self.row_names = np.array([]) self.quadratic_constraints = [] + def has_barrier_cache(self): + """Return whether this data model owns a reusable solver session.""" + return self.barrier_cache_capsule is not None + + def clear_barrier_cache(self): + """Release this data model's reusable solver session and GPU cache.""" + self.barrier_cache_capsule = None + def clear_quadratic_constraints(self): self.quadratic_constraints = [] @@ -158,6 +175,34 @@ cdef class DataModel: def set_objective_coefficients(self, c): self.c = type_cast(c, np.float64, "c") + def update_q(self, c): + """Update linear objective coefficients (user-space ``c``). + + Always writes the DataModel objective. If this model owns a solver + session from a prior Barrier solve, also crushes ``c`` into the cached + ``iteration_data_t`` and sets ``c_dirty`` so a later continue path can + skip convert/presolve. Session crush runs first so a length error + leaves the DataModel coefficients unchanged. + """ + cdef barrier_cache_t* session + cdef double[::1] c_view + new_c = type_cast(c, np.float64, "c") + if self.barrier_cache_capsule is not None: + if not PyCapsule_IsValid( + self.barrier_cache_capsule, b"cuopt.barrier_cache" + ): + raise ValueError("Invalid barrier cache stored on DataModel.") + session = PyCapsule_GetPointer( + self.barrier_cache_capsule, + b"cuopt.barrier_cache", + ) + c_view = np.ascontiguousarray(new_c, dtype=np.float64) + if c_view.shape[0] == 0: + session.update_linear_objective(NULL, 0) + else: + session.update_linear_objective(&c_view[0], c_view.shape[0]) + self.c = new_c + def set_objective_scaling_factor(self, objective_scaling_factor): self.objective_scaling_factor = objective_scaling_factor diff --git a/python/cuopt/cuopt/linear_programming/solver/solver.pxd b/python/cuopt/cuopt/linear_programming/solver/solver.pxd index 04b75ce4e5..09efdc2ae5 100644 --- a/python/cuopt/cuopt/linear_programming/solver/solver.pxd +++ b/python/cuopt/cuopt/linear_programming/solver/solver.pxd @@ -93,6 +93,10 @@ cdef extern from "cuopt/mathematical_optimization/utilities/cython_types.hpp" na vector[double] last_restart_duality_gap_primal_solution_ vector[double] last_restart_duality_gap_dual_solution_ +cdef extern from "cuopt/mathematical_optimization/utilities/barrier_cache.hpp" namespace "cuopt::cython": # noqa + cdef cppclass barrier_cache_t: + pass + cdef extern from "cuopt/mathematical_optimization/utilities/cython_solve.hpp" namespace "cuopt::cython": # noqa # Unified LP solution struct — solutions_ variant accessed via helpers cdef cppclass linear_programming_ret_t: @@ -117,6 +121,7 @@ cdef extern from "cuopt/mathematical_optimization/utilities/cython_solve.hpp" na int nb_iterations_ double solve_time_ method_t solved_by_ + unique_ptr[barrier_cache_t] barrier_cache bool is_gpu() # Unified MIP solution struct — solution_ variant accessed via helpers @@ -144,6 +149,9 @@ cdef extern from "cuopt/mathematical_optimization/utilities/cython_solve.hpp" na cdef unique_ptr[solver_ret_t] call_solve( data_model_view_t[int, double]* data_model, solver_settings_t[int, double]* solver_settings, + unsigned int flags, + bool is_batch_mode, + barrier_cache_t* session_in, ) except + nogil cdef pair[vector[unique_ptr[solver_ret_t]], double] call_batch_solve( # noqa diff --git a/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx b/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx index 1bbb101af9..ca33987108 100644 --- a/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx +++ b/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx @@ -15,7 +15,7 @@ from dateutil.relativedelta import relativedelta from cuopt.utilities import type_cast -from libc.stdint cimport uintptr_t +from libc.stdint cimport uintptr_t, uint32_t from libc.stdlib cimport free, malloc from libc.string cimport memcpy, strcpy, strlen from libcpp cimport bool @@ -25,6 +25,13 @@ from libcpp.string cimport string from libcpp.utility cimport move from libcpp.vector cimport vector +from cpython.pycapsule cimport ( + PyCapsule_Destructor, + PyCapsule_GetPointer, + PyCapsule_IsValid, + PyCapsule_New, +) + from rmm.pylibrmm.device_buffer cimport DeviceBuffer from cuopt.linear_programming.data_model.data_model cimport data_model_view_t @@ -40,6 +47,7 @@ from cuopt.linear_programming.solver.solver cimport ( linear_programming_ret_t, lp_cpu_solutions_t, lp_gpu_solutions_t, + barrier_cache_t, mip_ret_t, mip_termination_status_t, pdlp_solver_mode_t, @@ -76,6 +84,25 @@ cdef extern from "cuopt/mathematical_optimization/utilities/internals.hpp" names cdef cppclass base_solution_callback_t +cdef extern from *: + """ + #include + + static void cuopt_barrier_cache_capsule_dtor(PyObject *cap) noexcept + { + void *p = PyCapsule_GetPointer(cap, "cuopt.barrier_cache"); + if (p != nullptr) { + delete reinterpret_cast(p); + } + } + """ + void cuopt_barrier_cache_capsule_dtor(object cap) noexcept + + +cdef extern from "driver_types.h": + cdef uint32_t cudaStreamNonBlocking + + class MILPTerminationStatus(IntEnum): NoTermination = mip_termination_status_t.NoTermination Optimal = mip_termination_status_t.Optimal @@ -453,6 +480,19 @@ def prepare_solver_settings(SolverSettings settings, data_model=None, mip=False) def Solve(py_data_model_obj, SolverSettings settings, mip=False): cdef DataModel data_model_obj = py_data_model_obj + cdef barrier_cache_t* session_in = NULL + cdef solver_ret_t* sol_ret + + if settings.sequence_solve and data_model_obj.barrier_cache_capsule is not None: + if not PyCapsule_IsValid( + data_model_obj.barrier_cache_capsule, + b"cuopt.barrier_cache", + ): + raise ValueError("Invalid barrier cache stored on DataModel.") + session_in = PyCapsule_GetPointer( + data_model_obj.barrier_cache_capsule, + b"cuopt.barrier_cache", + ) data_model_obj.variable_types = type_cast( data_model_obj.variable_types, "S1", "variable_types" @@ -468,7 +508,22 @@ def Solve(py_data_model_obj, SolverSettings settings, mip=False): sol_ret_ptr = move(call_solve( data_model_obj.c_data_model_view.get(), settings.c_solver_settings.get(), + cudaStreamNonBlocking, + False, + session_in, )) + + sol_ret = sol_ret_ptr.get() + if ( + sol_ret.problem_type == ProblemCategory.LP + and sol_ret.lp_ret.barrier_cache.get() != NULL + ): + data_model_obj.barrier_cache_capsule = PyCapsule_New( + sol_ret.lp_ret.barrier_cache.release(), + b"cuopt.barrier_cache", + cuopt_barrier_cache_capsule_dtor, + ) + return create_solution(move(sol_ret_ptr), data_model_obj) diff --git a/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pxd b/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pxd index 03958d2286..02d1ec241f 100644 --- a/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pxd +++ b/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pxd @@ -30,6 +30,14 @@ cdef extern from "cuopt/mathematical_optimization/pdlp/solver_settings.hpp" name Barrier "cuopt::mathematical_optimization::method_t::Barrier" # noqa Unset "cuopt::mathematical_optimization::method_t::Unset" # noqa + cdef cppclass pdlp_solver_settings_t[i_t, f_t]: + bool sequence_solve + barrier_cache_t* barrier_cache + +cdef extern from "cuopt/mathematical_optimization/utilities/barrier_cache.hpp" namespace "cuopt::cython": # noqa + cdef cppclass barrier_cache_t: + pass + cdef extern from "cuopt/mathematical_optimization/solver_settings.hpp" namespace "cuopt::mathematical_optimization": # noqa cdef cppclass solver_settings_t[i_t, f_t]: @@ -90,9 +98,12 @@ cdef extern from "cuopt/mathematical_optimization/solver_settings.hpp" namespace void load_parameters_from_file(const string& path) except + + pdlp_solver_settings_t[i_t, f_t]& get_pdlp_settings() + cdef class SolverSettings: cdef unique_ptr[solver_settings_t[int, double]] c_solver_settings cdef public dict settings_dict cdef public object pdlp_warm_start_data cdef public list mip_callbacks + cdef public bint sequence_solve diff --git a/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pyx b/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pyx index a5dcc78d18..17c3046105 100644 --- a/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pyx +++ b/python/cuopt/cuopt/linear_programming/solver_settings/solver_settings.pyx @@ -117,6 +117,7 @@ cdef class SolverSettings: self.settings_dict = {} self.pdlp_warm_start_data = None self.mip_callbacks = [] + self.sequence_solve = False def to_base_type(self, value): """Convert a string to a base type. @@ -459,6 +460,16 @@ cdef class SolverSettings: warm_start_data.iterations_since_last_restart # noqa ) + c_solver_settings.get_pdlp_settings().sequence_solve = self.sequence_solve + + def set_sequence_solve(self, enabled): + """Enable barrier cache reuse across a sequence of solves (same sparsity).""" + self.sequence_solve = True if enabled else False + + def get_sequence_solve(self): + """Return whether sequence-solve barrier cache reuse is enabled.""" + return self.sequence_solve + def dump_parameters_to_file(self, path, hyperparameters_only=True): """Apply ``settings_dict`` / warm start to C++, then dump parameters to *path*. From 2c312587b7396d6aa2d0482af7a8eaea14bfefa6 Mon Sep 17 00:00:00 2001 From: Ishika Roy Date: Fri, 28 Aug 2026 03:58:40 +0000 Subject: [PATCH 2/4] cleanup --- .../utilities/barrier_cache.hpp | 44 +- .../utilities/cython_solve.hpp | 2 +- .../utilities/cython_types.hpp | 2 +- .../utilities/solver_cache_profiler.hpp | 152 -- cpp/src/barrier/CMakeLists.txt | 1 - cpp/src/barrier/barrier.cu | 1558 +++++++---------- cpp/src/barrier/barrier.hpp | 22 +- .../barrier_factorization_sparsity_hash.cu | 25 - .../barrier_factorization_sparsity_hash.hpp | 126 -- cpp/src/barrier/barrier_symbolic_cache.hpp | 86 - cpp/src/barrier/sparse_cholesky.cuh | 21 +- cpp/src/dual_simplex/solve.cpp | 136 +- cpp/src/dual_simplex/solve.hpp | 6 +- cpp/src/linear_algebra/vector_math.cuh | 66 + cpp/src/pdlp/solve.cu | 106 +- cpp/src/pdlp/utilities/barrier_cache.cu | 78 +- ...nt_end_cache.hpp => barrier_transform.hpp} | 53 +- cpp/src/pdlp/utilities/cython_solve.cu | 58 +- .../data_model/data_model.py | 7 +- .../data_model/data_model_wrapper.pyx | 18 +- .../linear_programming/solver/solver.pxd | 2 +- .../solver/solver_wrapper.pyx | 6 +- 22 files changed, 919 insertions(+), 1656 deletions(-) delete mode 100644 cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp delete mode 100644 cpp/src/barrier/barrier_factorization_sparsity_hash.cu delete mode 100644 cpp/src/barrier/barrier_factorization_sparsity_hash.hpp delete mode 100644 cpp/src/barrier/barrier_symbolic_cache.hpp rename cpp/src/pdlp/utilities/{barrier_front_end_cache.hpp => barrier_transform.hpp} (59%) diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp index d548aa3dbf..4b3643f80d 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp +++ b/cpp/include/cuopt/mathematical_optimization/utilities/barrier_cache.hpp @@ -17,12 +17,6 @@ namespace cuopt::mathematical_optimization::barrier { template class iteration_data_t; -template -struct barrier_symbolic_cache_t; - -template -void barrier_store_symbolic_cache_from_iteration_data(iteration_data_t& data, - barrier_symbolic_cache_t& cache); void destroy_iteration_data(iteration_data_t* data); @@ -34,15 +28,14 @@ void apply_barrier_linear_objective(iteration_data_t& data, namespace cuopt { namespace CUOPT_EXPORT cython { -struct barrier_front_end_cache_t; +struct barrier_transform_t; /** - * @brief Lean GPU solve session: owns RAFT handle + stream, optional barrier symbolic cache, - * and optional barrier iteration_data_t (GPU IPM workspace) after an Optimal solve. + * @brief GPU solve cache owned by DataModel when sequence_solve is on. * - * Created on first solve when sequence_solve; reused on subsequent solves with the same capsule. - * Per-solve convert/presolve/scaling remain stack-local until the continue path (D); A keeps - * iteration_data_t, B keeps front-end maps + c_dirty. + * After an Optimal full solve, holds iteration_data_t and the user↔barrier transform. + * update_q crushes the new linear objective and sets c_dirty so the next Solve + * reuses that workspace (skip convert/presolve/scaling). */ class barrier_cache_t { public: @@ -56,16 +49,8 @@ class barrier_cache_t { [[nodiscard]] raft::handle_t const* handle_ptr() const; [[nodiscard]] rmm::cuda_stream_view stream_view() const; - /** - * @brief Returns cached symbolic state when valid and @p handle matches the stored handle. - */ - [[nodiscard]] mathematical_optimization::barrier::barrier_symbolic_cache_t* - symbolic_cache_for_reuse(raft::handle_t const* handle); - - void clear_symbolic_cache(); - - void store_symbolic_cache( - mathematical_optimization::barrier::iteration_data_t& data); + /** Drop cached iteration workspace and transform (handle/stream stay). */ + void clear(); /** * @brief Take ownership of barrier iteration workspace. @p data may be null (clears). @@ -78,22 +63,15 @@ class barrier_cache_t { */ mathematical_optimization::barrier::iteration_data_t* release_iteration_data(); - [[nodiscard]] mathematical_optimization::barrier::iteration_data_t* - iteration_data(); - - void clear_iteration_data(); - - void store_front_end_cache(std::unique_ptr cache); - [[nodiscard]] barrier_front_end_cache_t* front_end_cache(); - [[nodiscard]] barrier_front_end_cache_t const* front_end_cache() const; - void clear_front_end_cache(); + void store_transform(std::unique_ptr transform); + [[nodiscard]] barrier_transform_t* transform(); + [[nodiscard]] barrier_transform_t const* transform() const; void set_c_dirty(bool dirty); [[nodiscard]] bool c_dirty() const; - [[nodiscard]] bool has_front_end_cache() const; /** * Crush user-space linear objective into cached iteration_data_t.c / d_c_ and set c_dirty. - * Requires a stored front-end cache and iteration_data from an Optimal solve. + * Requires a stored transform and iteration_data from an Optimal solve. */ void update_linear_objective(double const* c, int n); diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp index fb6a3a21d9..ec7736b5a5 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp +++ b/cpp/include/cuopt/mathematical_optimization/utilities/cython_solve.hpp @@ -56,7 +56,7 @@ std::unique_ptr call_solve( mathematical_optimization::solver_settings_t*, unsigned int flags = cudaStreamNonBlocking, bool is_batch_mode = false, - barrier_cache_t* session_in = nullptr); + barrier_cache_t* cache_in = nullptr); std::pair>, double> solve_batch_remote( std::vector*>, diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp index fd92f93964..f047ca9637 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp +++ b/cpp/include/cuopt/mathematical_optimization/utilities/cython_types.hpp @@ -87,7 +87,7 @@ struct linear_programming_ret_t { double solve_time_{}; mathematical_optimization::method_t solved_by_{}; - /** GPU barrier session (stream + handle + symbolic cache); moved to Python capsule when set. */ + /** GPU barrier cache (stream + handle + iteration workspace); moved to Python capsule when set. */ std::unique_ptr barrier_cache; bool is_gpu() const { return std::holds_alternative(solutions_); } diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp b/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp deleted file mode 100644 index 0b72d45666..0000000000 --- a/cpp/include/cuopt/mathematical_optimization/utilities/solver_cache_profiler.hpp +++ /dev/null @@ -1,152 +0,0 @@ -/* clang-format off */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -/* clang-format on */ - -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace cuopt::linear_programming::cache_profile { - -enum class cache_id : int { - C01 = 0, // raft::handle_t + stream - C02, // cuBLAS / cuSparse warmup (init_handler) - C03, // problem fingerprint (structural hash) - C04, // augmented vs ADAT choice - C05, // ADAT / augmented sparsity pattern - C06, // cuDSS handle + config - C07, // cuDSS symbolic factorization - C08, // dense-column / SOC layout metadata - C09, // device buffer allocation (iteration_data setup) - COUNT -}; - -inline constexpr int num_cache_ids = static_cast(cache_id::COUNT); - -inline const char* cache_id_label(cache_id id) -{ - switch (id) { - case cache_id::C01: return "C01 raft handle+stream"; - case cache_id::C02: return "C02 cuBLAS/cuSparse warmup"; - case cache_id::C03: return "C03 problem fingerprint"; - case cache_id::C04: return "C04 augmented vs ADAT choice"; - case cache_id::C05: return "C05 KKT sparsity pattern"; - case cache_id::C06: return "C06 cuDSS handle+config"; - case cache_id::C07: return "C07 cuDSS symbolic factorization"; - case cache_id::C08: return "C08 dense-column/SOC layout"; - case cache_id::C09: return "C09 device buffer allocation"; - default: return "C?? unknown"; - } -} - -class profiler_t { - public: - static profiler_t& instance() - { - static profiler_t prof; - return prof; - } - - bool enabled() const { return enabled_; } - - void reset() - { - times_.fill(0.0); - } - - void add(cache_id id, double seconds) - { - if (!enabled_) { return; } - times_[static_cast(id)] += seconds; - } - - double get(cache_id id) const { return times_[static_cast(id)]; } - - double total_measured() const - { - double sum = 0.0; - for (double t : times_) { - sum += t; - } - return sum; - } - - void log_summary() const - { - if (!enabled_) { return; } - auto emit = [](const char* fmt, ...) { - va_list args; - va_start(args, fmt); - char buf[512]; - vsnprintf(buf, sizeof(buf), fmt, args); - va_end(args); - CUOPT_LOG_INFO("%s", buf); - fprintf(stderr, "%s\n", buf); - }; - emit("=== Solver cache profile (ms) ==="); - for (int i = 0; i < num_cache_ids; ++i) { - const double ms = times_[i] * 1000.0; - emit("Cache profile: %s %.3f", cache_id_label(static_cast(i)), ms); - } - emit("Cache profile: TOTAL measured %.3f", total_measured() * 1000.0); - emit("=== End solver cache profile ==="); - } - - private: - profiler_t() - { - const char* env = std::getenv("CUOPT_CACHE_PROFILE"); - enabled_ = env != nullptr && env[0] != '\0' && std::strcmp(env, "0") != 0; - } - - bool enabled_{false}; - std::array times_{}; -}; - -inline bool enabled() { return profiler_t::instance().enabled(); } - -inline void reset() { profiler_t::instance().reset(); } - -inline void add(cache_id id, double seconds) { profiler_t::instance().add(id, seconds); } - -inline void log_summary() { profiler_t::instance().log_summary(); } - -class scoped_timer_t { - public: - explicit scoped_timer_t(cache_id id) : id_(id), start_(clock_::now()), active_(enabled()) {} - - ~scoped_timer_t() - { - if (!active_) { return; } - const double elapsed = - std::chrono::duration(clock_::now() - start_).count(); - add(id_, elapsed); - } - - private: - using clock_ = std::chrono::steady_clock; - cache_id id_; - clock_::time_point start_; - bool active_; -}; - -} // namespace cuopt::linear_programming::cache_profile - -#define CUOPT_CACHE_PROFILE_SCOPE(id) \ - ::cuopt::linear_programming::cache_profile::scoped_timer_t CUOPT_CACHE_PROFILE_CONCAT( \ - _cuopt_cache_scope_, __LINE__)(id) - -#define CUOPT_CACHE_PROFILE_CONCAT(a, b) CUOPT_CACHE_PROFILE_CONCAT_IMPL(a, b) -#define CUOPT_CACHE_PROFILE_CONCAT_IMPL(a, b) a##b diff --git a/cpp/src/barrier/CMakeLists.txt b/cpp/src/barrier/CMakeLists.txt index 2d5fb27dc7..650bc733e9 100644 --- a/cpp/src/barrier/CMakeLists.txt +++ b/cpp/src/barrier/CMakeLists.txt @@ -6,7 +6,6 @@ set(BARRIER_SRC_FILES ${CMAKE_CURRENT_SOURCE_DIR}/cusparse_view.cu ${CMAKE_CURRENT_SOURCE_DIR}/barrier.cu - ${CMAKE_CURRENT_SOURCE_DIR}/barrier_factorization_sparsity_hash.cu ${CMAKE_CURRENT_SOURCE_DIR}/device_sparse_matrix.cu ${CMAKE_CURRENT_SOURCE_DIR}/pinned_host_allocator.cu ) diff --git a/cpp/src/barrier/barrier.cu b/cpp/src/barrier/barrier.cu index f824ff8b88..0c6114e266 100644 --- a/cpp/src/barrier/barrier.cu +++ b/cpp/src/barrier/barrier.cu @@ -7,8 +7,6 @@ #include -#include -#include #include #include #include @@ -31,13 +29,12 @@ #include -#include - #include #include #include +#include #include #include @@ -46,8 +43,8 @@ #include #include -#include #include +#include #include #include @@ -238,8 +235,7 @@ class iteration_data_t { i_t num_upper_bounds, const std::vector& direct_free_variables, const csc_matrix_t& Qin, - const simplex_solver_settings_t& settings, - barrier_symbolic_cache_t* adopt_symbolic = nullptr) + const simplex_solver_settings_t& settings) : upper_bounds(num_upper_bounds), c(lp.objective), b(lp.rhs), @@ -358,17 +354,18 @@ class iteration_data_t { transform_reduce_helper_(lp.handle_ptr->get_stream()), transform_reduce_pair_helper_(lp.handle_ptr->get_stream()), sum_reduce_helper_(lp.handle_ptr->get_stream()), + d_scalar_batch_(kNumScalarBatchSlots, lp.handle_ptr->get_stream()), + h_scalar_batch_(kNumScalarBatchSlots), + d_reduce_tmp_(0, lp.handle_ptr->get_stream()), indefinite_Q(false), Q_diagonal(false), symbolic_status(0), - adopted_symbolic_(false), cone_combined_step_(false), cone_sigma_mu_(f_t(0)) { raft::common::nvtx::range fun_scope("Barrier: LP Data Creation"); { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: direct free linear"); // Setup tracking of direct free variables (linear columns only j < cone_start) n_direct_free_linear = direct_free_variables.size(); @@ -391,7 +388,6 @@ class iteration_data_t { bool has_Q = Q.x.size() > 0; indefinite_Q = false; { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: Q setup"); if (has_Q) { Qdiag.resize(lp.num_cols, 0.0); @@ -433,7 +429,6 @@ class iteration_data_t { } if (!lp.second_order_cone_dims.empty()) { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C08); raft::common::nvtx::range scope("Barrier: LP Data: SOC setup"); cone_var_start_ = lp.cone_var_start; i_t total_cone_dim = @@ -454,7 +449,6 @@ class iteration_data_t { } { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: complementarity buffers"); const i_t linear_xz_rhs_size = linear_xz_size(lp.num_cols); d_complementarity_xz_rhs_.resize(linear_xz_rhs_size, stream_view_); @@ -474,7 +468,6 @@ class iteration_data_t { } { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: upper bounds"); // Create the upper bounds vector n_upper_bounds = 0; @@ -488,7 +481,6 @@ class iteration_data_t { std::vector dense_columns_unordered; { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C04); raft::common::nvtx::range scope("Barrier: LP Data: dense columns and augmented"); // Decide if we are going to use the augmented system or not n_dense_columns = 0; @@ -533,7 +525,7 @@ class iteration_data_t { if (n_dense_rows > 0) { settings.log.printf("Dense rows : %d\n", n_dense_rows); } - settings.log.printf("Density estimator time : %.2fs\n", column_density_time); + settings.log.printf("Density estimator time : %.3fs\n", column_density_time); if ((settings.augmented != 0) && (n_dense_columns > 50 || n_dense_rows > 10 || lp.A.m == 0 /* handle case with no constraints */ || @@ -566,7 +558,6 @@ class iteration_data_t { } { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: diag and inv_diag"); // D = I + EET diag.set_scalar(1.0); @@ -598,7 +589,6 @@ class iteration_data_t { if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: AD matrix setup"); // Copy A into AD AD = lp.A; @@ -646,7 +636,6 @@ class iteration_data_t { // device_AD / device_A / ADAT path is only used when forming ADAT (!use_augmented). if (!use_augmented) { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C09); raft::common::nvtx::range scope("Barrier: LP Data: device AD path"); device_AD.copy(AD, handle_ptr->get_stream()); d_original_A_values.resize(device_AD.x.size(), handle_ptr->get_stream()); @@ -665,478 +654,97 @@ class iteration_data_t { if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C06); raft::common::nvtx::range scope("Barrier: LP Data: Cholesky init"); i_t factorization_size = use_augmented ? augmented_system_size(lp.num_cols, lp.num_rows) : lp.num_rows; - auto adopt_augmented_symbolic = [&]() -> bool { - if (has_cones() || !use_augmented || adopt_symbolic == nullptr) { return false; } - if (adopt_symbolic->device_augmented.x.size() == 0) { return false; } - - const barrier_sparsity_hash_t host_hash = hash_augmented_kkt_sparsity(A, AT, Q); - const bool matched = - adopt_symbolic->matches_reuse(host_hash, true, handle_ptr); - if (!matched) { return false; } - - chol = adopt_symbolic->chol; - static_cast*>(chol.get())->rebind_settings(settings); - pinned_device_augmented_ = &adopt_symbolic->device_augmented; - d_augmented_diagonal_indices_.resize(adopt_symbolic->d_augmented_diagonal_indices_.size(), - stream_view_); - raft::copy(d_augmented_diagonal_indices_.data(), - adopt_symbolic->d_augmented_diagonal_indices_.data(), - adopt_symbolic->d_augmented_diagonal_indices_.size(), - stream_view_); - handle_ptr->sync_stream(); - static_cast*>(chol.get())->rebind_csr_matrix(aug_mat()); - adopted_symbolic_ = true; - symbolic_status = 0; - return true; - }; - - auto unpin_adat_workspace = [&]() { - pinned_device_ADAT_ = nullptr; - pinned_device_A_ = nullptr; - pinned_device_AD_ = nullptr; - pinned_d_original_A_values_ = nullptr; - pinned_device_A_x_values_ = nullptr; - pinned_cusparse_info_ = nullptr; - }; - - auto pin_adat_from_cache = [&](barrier_symbolic_cache_t& cache) { - pinned_device_ADAT_ = &cache.device_ADAT; - pinned_device_A_ = &cache.device_A; - pinned_device_AD_ = &cache.device_AD; - pinned_d_original_A_values_ = &cache.d_original_A_values; - pinned_device_A_x_values_ = &cache.device_A_x_values; - pinned_cusparse_info_ = cache.cusparse_info.get(); - }; - - auto adopt_adat_symbolic = [&]() -> bool { - if (has_cones() || use_augmented || adopt_symbolic == nullptr || n_dense_columns > 0) { - return false; - } - if (!adopt_symbolic->valid || adopt_symbolic->use_augmented) { return false; } - - // Gate on the *incoming* A sparsity before pinning SpGEMM workspace. - // Hashing ADAT after pin/form used the cached A and could false-match when - // only the new problem's pattern changed (same idea as augmented host gate). - // device_A already holds the current problem's CSR (uploaded above). - const barrier_sparsity_hash_t a_hash = - hash_device_csr_sparsity_pattern(device_A, stream_view_); - if (!adopt_symbolic->matches_reuse(a_hash, false, handle_ptr)) { - settings_.log.printf( - "Barrier: ADAT A-sparsity hash mismatch; rebuilding symbolic analysis\n"); - adopt_symbolic->clear(); - return false; - } - if (adopt_symbolic->device_A.x.size() == 0) { return false; } - - pin_adat_from_cache(*adopt_symbolic); + if (use_augmented) { + raft::common::nvtx::range form_scope("Barrier: LP Data: form augmented"); + form_augmented(true); + } else { + raft::common::nvtx::range form_scope("Barrier: LP Data: form ADAT"); form_adat(true); - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - unpin_adat_workspace(); - return false; - } - - chol = adopt_symbolic->chol; - static_cast*>(chol.get())->rebind_settings(settings); - handle_ptr->sync_stream(); - static_cast*>(chol.get())->rebind_csr_matrix(adat_mat()); - adopted_symbolic_ = true; - symbolic_status = 0; - return true; - }; + } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - if (!adopt_augmented_symbolic() && !adopt_adat_symbolic()) { + chol = std::make_shared>( + handle_ptr, settings, factorization_size); + chol->set_positive_definite(false); + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } + symbolic_status = 0; + { + raft::common::nvtx::range analyze_scope("Barrier: LP Data: symbolic analysis"); if (use_augmented) { - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C05); - raft::common::nvtx::range form_scope("Barrier: LP Data: form augmented"); - form_augmented(true); - } + symbolic_status = chol->analyze(aug_mat()); } else { - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C05); - raft::common::nvtx::range form_scope("Barrier: LP Data: form ADAT"); - form_adat(true); - } - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - - chol = std::make_shared>( - handle_ptr, settings, factorization_size); - chol->set_positive_definite(false); - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; } - symbolic_status = 0; - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C07); - raft::common::nvtx::range analyze_scope("Barrier: LP Data: symbolic analysis"); - if (use_augmented) { - symbolic_status = chol->analyze(aug_mat()); - } else { - symbolic_status = chol->analyze(adat_mat()); - } + symbolic_status = chol->analyze(adat_mat()); } } } } - [[nodiscard]] bool adopted_symbolic() const { return adopted_symbolic_; } - device_csr_matrix_t& augmented_system() { return aug_mat(); } const device_csr_matrix_t& augmented_system() const { return aug_mat(); } - void store_symbolic_cache(barrier_symbolic_cache_t& cache) + // Attach this solve's settings and rewind iterate-dependent state so IPM can + // Mehrotra-start with the new c. A and Q are unchanged; the previous solve + // left D and the KKT values at its last iterate. Reuse is QP-only (no cones), + // so form_*(false) updates values in the existing CSR; no symbolic rebuild. + bool prepare_for_reuse(const simplex_solver_settings_t& settings) { - if (symbolic_status != 0 || has_cones()) { return; } - auto* cudss_chol = dynamic_cast*>(chol.get()); - if (cudss_chol == nullptr) { return; } - - cache.chol = std::static_pointer_cast>(chol); - cache.handle_ptr = handle_ptr; - - if (use_augmented) { - cache.cusparse_info.reset(); - cache.use_augmented = true; - - if (pinned_device_augmented_ != nullptr) { - // Warm reuse: sparsity_hash unchanged since adopt (values-only refresh); unpin only. - pinned_device_augmented_ = nullptr; - } else { - cache.sparsity_hash = - hash_device_csr_sparsity_pattern(device_augmented, handle_ptr->get_stream()); - // Keep KKT buffers on iteration_data_t so the session can retain the workspace. - } - -#ifndef NDEBUG - const barrier_sparsity_hash_t host_hash = hash_augmented_kkt_sparsity(A, AT, Q); - cuopt_assert(cache.sparsity_hash == host_hash, - "store_symbolic_cache: device/host augmented sparsity hash mismatch"); -#endif - - cache.valid = true; - settings_.log.printf( - "Barrier: stored augmented symbolic cache hash=0x%016llx\n", - static_cast(cache.sparsity_hash)); - return; - } - - if (n_dense_columns > 0) { return; } - - cache.use_augmented = false; - if (pinned_device_ADAT_ != nullptr) { - // Warm reuse: sparsity_hash is the A-pattern hash from adopt; unpin only. - pinned_device_ADAT_ = nullptr; - pinned_device_A_ = nullptr; - pinned_device_AD_ = nullptr; - pinned_d_original_A_values_ = nullptr; - pinned_device_A_x_values_ = nullptr; - pinned_cusparse_info_ = nullptr; - } else { - // Store A sparsity (not ADAT): adopt compares the incoming A CSR before pin. - cache.sparsity_hash = - hash_device_csr_sparsity_pattern(device_A, handle_ptr->get_stream()); - // Keep ADAT/A buffers on iteration_data_t so the session can retain the workspace. + if (chol == nullptr || symbolic_status != 0) { return false; } + settings_ = settings; + if (chol != nullptr) { + static_cast*>(chol.get())->rebind_settings(settings_); } - cache.valid = true; - settings_.log.printf( - "Barrier: stored ADAT symbolic cache hash=0x%016llx\n", - static_cast(cache.sparsity_hash)); - } - - bool refresh_augmented_values() - { - i_t n = A.n; - i_t m = A.m; - i_t nnzA = A.col_start[n]; - i_t nnzQ = Q.n > 0 ? Q.col_start[n] : 0; - - i_t new_nnz = 2 * nnzA + n + m + nnzQ; - csr_matrix_t augmented_CSR(n + m, n + m, new_nnz); - i_t q = 0; - i_t off_diag_Qnz = 0; + { + raft::common::nvtx::range fun_scope("Barrier: reset diagonal scaling"); + const bool has_Q = Q.n > 0; - for (i_t i = 0; i < n; i++) { - augmented_CSR.row_start[i] = q; - if (nnzQ == 0) { - augmented_CSR.j[q] = i; - augmented_CSR.x[q++] = -diag[i] - dual_perturb; + if (has_cones()) { + primal_perturb = 1e-8; + dual_perturb = 1e-8; } else { - const i_t q_col_beg = Q.col_start[i]; - const i_t q_col_end = Q.col_start[i + 1]; - bool has_diagonal = false; - for (i_t p = q_col_beg; p < q_col_end; ++p) { - augmented_CSR.j[q] = Q.i[p]; - if (Q.i[p] == i) { - has_diagonal = true; - augmented_CSR.x[q++] = -Q.x[p] - diag[i] - dual_perturb; - } else { - off_diag_Qnz++; - augmented_CSR.x[q++] = -Q.x[p]; - } - } - if (!has_diagonal) { - augmented_CSR.j[q] = i; - augmented_CSR.x[q++] = -diag[i] - dual_perturb; - } - } - const i_t col_beg = A.col_start[i]; - const i_t col_end = A.col_start[i + 1]; - for (i_t p = col_beg; p < col_end; ++p) { - augmented_CSR.j[q] = A.i[p] + n; - augmented_CSR.x[q++] = A.x[p]; - } - } - - for (i_t k = n; k < n + m; ++k) { - augmented_CSR.row_start[k] = q; - const i_t l = k - n; - const i_t col_beg = AT.col_start[l]; - const i_t col_end = AT.col_start[l + 1]; - for (i_t p = col_beg; p < col_end; ++p) { - augmented_CSR.j[q] = AT.i[p]; - augmented_CSR.x[q++] = AT.x[p]; + primal_perturb = 1e-6; + dual_perturb = 0; } - augmented_CSR.j[q] = k; - augmented_CSR.x[q++] = primal_perturb; - } - augmented_CSR.row_start[n + m] = q; - if (q != static_cast(aug_mat().x.size()) || q != 2 * nnzA + n + m + off_diag_Qnz) { - return false; - } - - augmented_CSR.j.resize(q); - augmented_CSR.x.resize(q); - raft::copy(aug_mat().x.data(), augmented_CSR.x.data(), q, handle_ptr->get_stream()); - RAFT_CHECK_CUDA(handle_ptr->get_stream()); - return true; - } - - bool rebuild_augmented_symbolic() - { - if (!use_augmented) { return false; } - - settings_.log.printf( - "Barrier: augmented nnz mismatch on cached symbolic; rebuilding symbolic analysis\n"); - - adopted_symbolic_ = false; - pinned_device_augmented_ = nullptr; - - const i_t factorization_size = A.n + A.m; - chol = std::make_shared>( - handle_ptr, settings_, factorization_size); - chol->set_positive_definite(false); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } - - form_augmented(true); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } - - symbolic_status = chol->analyze(aug_mat()); - if (symbolic_status != 0) { return false; } - - reset_for_new_solve(); - return true; - } - - bool refresh_adat_values() - { - if (use_augmented || n_dense_columns > 0) { return false; } - - const i_t expected_nnz = static_cast(adat_mat().x.size()); - form_adat(false); - handle_ptr->sync_stream(); - return static_cast(adat_mat().x.size()) == expected_nnz; - } - - bool rebuild_adat_symbolic() - { - if (use_augmented || n_dense_columns > 0) { return false; } - - settings_.log.printf( - "Barrier: ADAT nnz mismatch on cached symbolic; rebuilding symbolic analysis\n"); - - adopted_symbolic_ = false; - pinned_device_ADAT_ = nullptr; - pinned_device_A_ = nullptr; - pinned_device_AD_ = nullptr; - pinned_d_original_A_values_ = nullptr; - pinned_device_A_x_values_ = nullptr; - pinned_cusparse_info_ = nullptr; - cusparse_info_.reset(); - - const i_t factorization_size = A.m; - chol = std::make_shared>( - handle_ptr, settings_, factorization_size); - chol->set_positive_definite(false); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } - form_adat(true); - if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } - - symbolic_status = chol->analyze(adat_mat()); - if (symbolic_status != 0) { return false; } - - reset_for_new_solve(); - return true; - } - - // Restore D = I + EE^T (+ Q_diag on the ADAT path) and the regularization used - // for the first factorization. The IPM loop overwrites both with the values of - // its last iterate, so a solve that reuses this workspace must reset them - // before an initial point is computed. - void reset_diagonal_scaling() - { - raft::common::nvtx::range fun_scope("Barrier: reset diagonal scaling"); - const bool has_Q = Q.n > 0; - - if (has_cones()) { - primal_perturb = 1e-8; - dual_perturb = 1e-8; - } else { - primal_perturb = 1e-6; - dual_perturb = 0; - } - - diag.set_scalar(1.0); - for (i_t k = 0; k < n_upper_bounds; k++) { - diag[upper_bounds[k]] = 2.0; - } - if (has_Q && !use_augmented) { - for (i_t j = 0; j < Q.n; j++) { - diag[j] += Qdiag[j]; + diag.set_scalar(1.0); + for (i_t k = 0; k < n_upper_bounds; k++) { + diag[upper_bounds[k]] = 2.0; } - } - - inv_diag.set_scalar(1.0); - if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { diag.inverse(inv_diag); } - raft::copy(d_diag_.data(), diag.data(), diag.size(), stream_view_); - raft::copy(d_inv_diag.data(), inv_diag.data(), inv_diag.size(), stream_view_); - inv_sqrt_diag.set_scalar(1.0); - if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { inv_diag.sqrt(inv_sqrt_diag); } - } - - // Re-form the ADAT or augmented values from the current diag, keeping the - // cached symbolic factorization whenever the sparsity pattern is unchanged. - bool refresh_linear_system_values() - { - if (use_augmented) { - if (!refresh_augmented_values()) { return rebuild_augmented_symbolic(); } - return true; - } - if (!adopted_symbolic_) { return true; } - if (!refresh_adat_values()) { return rebuild_adat_symbolic(); } - handle_ptr->sync_stream(); - if (chol != nullptr) { chol->rebind_csr_matrix(adat_mat()); } - return true; - } - - bool refresh_lp_numerics(const lp_problem_t& lp) - { - raft::common::nvtx::range fun_scope("Barrier: refresh LP numerics"); - - c = lp.objective; - b = lp.rhs; - raft::copy(d_c_.data(), c.data(), c.size(), stream_view_); - raft::copy(d_b_.data(), b.data(), b.size(), stream_view_); - - AD = A; - if (!use_augmented && n_dense_columns > 0) { AD.remove_columns(cols_to_remove); } - AT.transpose(AD); - - const bool has_Q = Q.n > 0; - if (has_Q) { - for (i_t j = 0; j < Q.n; j++) { - Qdiag[j] = 0.0; - const i_t col_start = Q.col_start[j]; - const i_t col_end = Q.col_start[j + 1]; - for (i_t p = col_start; p < col_end; p++) { - const i_t row = Q.i[p]; - if (j == row) { - Qdiag[j] = Q.x[p]; - break; - } + if (has_Q && !use_augmented) { + for (i_t j = 0; j < Q.n; j++) { + diag[j] += Qdiag[j]; } } - if (d_Q_diag_.size() > 0) { - raft::copy(d_Q_diag_.data(), Qdiag.data(), Qdiag.size(), stream_view_); - } - } - reset_diagonal_scaling(); - - if (!use_augmented) { - ad_mat().copy(AD, handle_ptr->get_stream()); - raft::copy(original_a_values().data(), - ad_mat().x.data(), - ad_mat().x.size(), - handle_ptr->get_stream()); - raft::copy(a_x_values().data(), ad_mat().x.data(), ad_mat().x.size(), handle_ptr->get_stream()); - ad_mat().to_compressed_row(a_mat(), handle_ptr->get_stream()); - RAFT_CHECK_CUDA(handle_ptr->get_stream()); - } - - if (!refresh_linear_system_values()) { return false; } - - cusparse_view_.update_matrix_values(A); - if (Q.n > 0) { cusparse_Q_view_.update_matrix_values(Q); } - - reset_for_new_solve(); - return true; - } - - bool rebind_from_lp(const lp_problem_t& lp, - const std::vector& direct_free_variables, - const csc_matrix_t& Qin, - const simplex_solver_settings_t& settings) - { - raft::common::nvtx::range fun_scope("Barrier: rebind iteration data"); - if (handle_ptr != lp.handle_ptr) { return false; } - if (chol == nullptr || symbolic_status != 0) { return false; } - if (A.m != lp.A.m || A.n != lp.A.n || A.nnz() != lp.A.nnz()) { return false; } - if (A.col_start != lp.A.col_start || A.i != lp.A.i) { return false; } - if (Q.n != Qin.n || Q.m != Qin.m || Q.nnz() != Qin.nnz()) { return false; } - if (Q.col_start != Qin.col_start || Q.i != Qin.i) { return false; } - if (static_cast(direct_free_variables.size()) != n_direct_free_linear) { return false; } - i_t new_n_upper = 0; - for (i_t j = 0; j < lp.num_cols; j++) { - if (lp.upper[j] < inf) { new_n_upper++; } + inv_diag.set_scalar(1.0); + if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { diag.inverse(inv_diag); } + raft::copy(d_diag_.data(), diag.data(), diag.size(), stream_view_); + raft::copy(d_inv_diag.data(), inv_diag.data(), inv_diag.size(), stream_view_); + inv_sqrt_diag.set_scalar(1.0); + if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { inv_diag.sqrt(inv_sqrt_diag); } } - if (new_n_upper != n_upper_bounds) { return false; } - A = lp.A; - Q = Qin; - settings_ = settings; - if (chol != nullptr) { - static_cast*>(chol.get())->rebind_settings(settings_); + if (use_augmented) { + form_augmented(false); + } else { + form_adat(false); + handle_ptr->sync_stream(); + if (chol != nullptr) { chol->rebind_csr_matrix(adat_mat()); } } - adopted_symbolic_ = true; - return refresh_lp_numerics(lp); - } + if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return false; } - bool rebind_settings_for_continue(const simplex_solver_settings_t& settings) - { - if (chol == nullptr || symbolic_status != 0) { return false; } - settings_ = settings; - adopted_symbolic_ = true; - if (chol != nullptr) { - static_cast*>(chol.get())->rebind_settings(settings_); - } - // A and Q are unchanged, so only the iterate-dependent state has to be - // rewound: the initial point is computed from D and the linear system built - // from it, both of which the previous solve left at its final iterate. - reset_diagonal_scaling(); - if (!refresh_linear_system_values()) { return false; } reset_for_new_solve(); return true; } void reset_for_new_solve() { - has_factorization = false; - has_solve_info = false; + has_factorization = false; + has_solve_info = false; relative_primal_residual_save = inf; relative_dual_residual_save = inf; relative_complementarity_residual_save = inf; @@ -1391,7 +999,7 @@ class iteration_data_t { RAFT_CHECK_CUDA(stream_view_); } if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return; } - if (first_call && pinned_cusparse_info_ == nullptr) { + if (first_call) { raft::common::nvtx::range scope("Barrier: Form ADAT: cusparse init"); try { if (!cusparse_info_) { @@ -1416,7 +1024,7 @@ class iteration_data_t { float64_t adat_time = toc(start_form_adat); if (num_factorizations == 0) { - settings_.log.printf("ADAT time : %.2fs\n", adat_time); + settings_.log.printf("ADAT time : %.3fs\n", adat_time); settings_.log.printf("ADAT nonzeros : %.2e\n", static_cast(adat_nnz)); settings_.log.printf( @@ -2438,52 +2046,19 @@ class iteration_data_t { std::vector Qdiag; bool Q_diagonal; rmm::device_uvector d_augmented_diagonal_indices_; - device_csr_matrix_t* pinned_device_augmented_{nullptr}; - device_csr_matrix_t* pinned_device_ADAT_{nullptr}; - device_csr_matrix_t* pinned_device_A_{nullptr}; - device_csc_matrix_t* pinned_device_AD_{nullptr}; - rmm::device_uvector* pinned_d_original_A_values_{nullptr}; - rmm::device_uvector* pinned_device_A_x_values_{nullptr}; - cusparse_info_t* pinned_cusparse_info_{nullptr}; - - device_csr_matrix_t& aug_mat() - { - return pinned_device_augmented_ != nullptr ? *pinned_device_augmented_ : device_augmented; - } - const device_csr_matrix_t& aug_mat() const - { - return pinned_device_augmented_ != nullptr ? *pinned_device_augmented_ : device_augmented; - } - device_csr_matrix_t& adat_mat() - { - return pinned_device_ADAT_ != nullptr ? *pinned_device_ADAT_ : device_ADAT; - } - const device_csr_matrix_t& adat_mat() const - { - return pinned_device_ADAT_ != nullptr ? *pinned_device_ADAT_ : device_ADAT; - } - device_csr_matrix_t& a_mat() - { - return pinned_device_A_ != nullptr ? *pinned_device_A_ : device_A; - } - device_csc_matrix_t& ad_mat() - { - return pinned_device_AD_ != nullptr ? *pinned_device_AD_ : device_AD; - } - rmm::device_uvector& original_a_values() - { - return pinned_d_original_A_values_ != nullptr ? *pinned_d_original_A_values_ - : d_original_A_values; - } - rmm::device_uvector& a_x_values() - { - return pinned_device_A_x_values_ != nullptr ? *pinned_device_A_x_values_ : device_A_x_values; - } + + device_csr_matrix_t& aug_mat() { return device_augmented; } + const device_csr_matrix_t& aug_mat() const { return device_augmented; } + device_csr_matrix_t& adat_mat() { return device_ADAT; } + const device_csr_matrix_t& adat_mat() const { return device_ADAT; } + device_csr_matrix_t& a_mat() { return device_A; } + device_csc_matrix_t& ad_mat() { return device_AD; } + rmm::device_uvector& original_a_values() { return d_original_A_values; } + rmm::device_uvector& a_x_values() { return device_A_x_values; } cusparse_info_t& spgemm_info() { - cuopt_assert(pinned_cusparse_info_ != nullptr || cusparse_info_ != nullptr, - "spgemm_info: cusparse workspace unset"); - return pinned_cusparse_info_ != nullptr ? *pinned_cusparse_info_ : *cusparse_info_; + cuopt_assert(cusparse_info_ != nullptr, "spgemm_info: cusparse workspace unset"); + return *cusparse_info_; } cone_kkt_data_t cone_kkt_data_; @@ -2495,7 +2070,6 @@ class iteration_data_t { bool use_augmented; i_t symbolic_status; - bool adopted_symbolic_; i_t n_direct_free_linear{0}; rmm::device_uvector d_is_direct_free_linear_; // 1 if variable is free in the linear block, else 0 @@ -2597,6 +2171,14 @@ class iteration_data_t { transform_reduce_pair_helper_t transform_reduce_pair_helper_; sum_reduce_helper_t sum_reduce_helper_; + // Staging area for compute_residual_norms_mu_and_objective: several independent GPU + // reductions/dot-products write into slots of d_scalar_batch_, then a single copy into + // h_scalar_batch_ + one stream sync reads them all back at once instead of one sync each. + static constexpr i_t kNumScalarBatchSlots = 12; + rmm::device_uvector d_scalar_batch_; + pinned_dense_vector_t h_scalar_batch_; + rmm::device_buffer d_reduce_tmp_; + bool cone_combined_step_; f_t cone_sigma_mu_; @@ -4524,6 +4106,155 @@ void barrier_solver_t::compute_primal_dual_objective(iteration_data_t< #endif } +// Hot-loop fusion of compute_residual_norms + compute_mu + compute_primal_dual_objective: the +// three functions above each read every reduction/dot-product result back individually via a +// blocking rmm::device_scalar::value(stream) (cudaMemcpyAsync + full stream synchronize), even +// though none of these values are used until well after all of them have been computed. This +// version issues every reduction/dot-product kernel into a slot of data.d_scalar_batch_ and +// defers the host readback to a single copy + a single sync at the end. +template +void barrier_solver_t::compute_residual_norms_mu_and_objective( + iteration_data_t& data, + f_t& primal_residual_norm, + f_t& dual_residual_norm, + f_t& complementarity_residual_norm, + f_t& mu, + f_t& primal_objective, + f_t& dual_objective) +{ + raft::common::nvtx::range fun_scope("Barrier: compute_residual_norms_mu_and_objective"); + + gpu_compute_residuals(data.d_w_, data.d_x_, data.d_y_, data.d_v_, data.d_z_, data); + + constexpr i_t kSlotPrimalResidual = 0; + constexpr i_t kSlotBoundResidual = 1; + constexpr i_t kSlotDualResidual = 2; + constexpr i_t kSlotComplXzLinear = 3; + constexpr i_t kSlotComplWv = 4; + constexpr i_t kSlotComplCone = 5; + constexpr i_t kSlotMuXzSum = 6; + constexpr i_t kSlotMuWvSum = 7; + constexpr i_t kSlotCx = 8; + constexpr i_t kSlotBy = 9; + constexpr i_t kSlotUv = 10; + constexpr i_t kSlotXQx = 11; + + f_t* d_batch = data.d_scalar_batch_.data(); + + const bool has_soc = data.has_cones(); + const i_t linear_xz_size = data.linear_xz_size(data.d_complementarity_xz_residual_.size()); + auto linear_xz_span = + raft::device_span(data.d_complementarity_xz_residual_.data(), linear_xz_size); + + // All enqueue calls below must stay on stream_view_: correctness relies on strict + // single-stream FIFO ordering, so that the single sync at the bottom is enough for every + // result to be ready on the host. + enqueue_norm_inf_into(data.d_primal_residual_.data(), + data.d_primal_residual_.size(), + d_batch + kSlotPrimalResidual, + data.d_reduce_tmp_, + stream_view_); + enqueue_norm_inf_into(data.d_bound_residual_.data(), + data.d_bound_residual_.size(), + d_batch + kSlotBoundResidual, + data.d_reduce_tmp_, + stream_view_); + enqueue_norm_inf_into(data.d_dual_residual_.data(), + data.d_dual_residual_.size(), + d_batch + kSlotDualResidual, + data.d_reduce_tmp_, + stream_view_); + enqueue_norm_inf_into(linear_xz_span.data(), + linear_xz_span.size(), + d_batch + kSlotComplXzLinear, + data.d_reduce_tmp_, + stream_view_); + enqueue_norm_inf_into(data.d_complementarity_wv_residual_.data(), + data.d_complementarity_wv_residual_.size(), + d_batch + kSlotComplWv, + data.d_reduce_tmp_, + stream_view_); + + if (has_soc) { + raft::device_span cone_dot = data.cones().scratch.template get_slot<0>(); + data.cones().segmented_sum( + data.d_complementarity_xz_residual_.data() + data.cone_start(), cone_dot, stream_view_); + enqueue_max_into( + cone_dot.data(), cone_dot.size(), d_batch + kSlotComplCone, data.d_reduce_tmp_, stream_view_); + } + + enqueue_sum_into(data.d_complementarity_xz_residual_.data(), + data.d_complementarity_xz_residual_.size(), + d_batch + kSlotMuXzSum, + data.d_reduce_tmp_, + stream_view_); + enqueue_sum_into(data.d_complementarity_wv_residual_.data(), + data.d_complementarity_wv_residual_.size(), + d_batch + kSlotMuWvSum, + data.d_reduce_tmp_, + stream_view_); + + RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(lp.handle_ptr->get_cublas_handle(), + data.d_c_.size(), + data.d_c_.data(), + 1, + data.d_x_.data(), + 1, + d_batch + kSlotCx, + stream_view_)); + RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(lp.handle_ptr->get_cublas_handle(), + data.d_b_.size(), + data.d_b_.data(), + 1, + data.d_y_.data(), + 1, + d_batch + kSlotBy, + stream_view_)); + RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(lp.handle_ptr->get_cublas_handle(), + data.d_restrict_u_.size(), + data.d_restrict_u_.data(), + 1, + data.d_v_.data(), + 1, + d_batch + kSlotUv, + stream_view_)); + if (data.Q.n > 0) { + auto cusparse_d_x = data.cusparse_view_.create_vector(data.d_x_); + auto cusparse_Qx = data.cusparse_view_.create_vector(data.d_Qx_); + data.cusparse_Q_view_.spmv(1.0, cusparse_d_x, 0.0, cusparse_Qx); + RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(lp.handle_ptr->get_cublas_handle(), + data.d_Qx_.size(), + data.d_Qx_.data(), + 1, + data.d_x_.data(), + 1, + d_batch + kSlotXQx, + stream_view_)); + } + + raft::copy(data.h_scalar_batch_.data(), + data.d_scalar_batch_.data(), + data.kNumScalarBatchSlots, + stream_view_); + stream_view_.synchronize(); + + const f_t* h = data.h_scalar_batch_.data(); + + primal_residual_norm = std::max(h[kSlotPrimalResidual], h[kSlotBoundResidual]); + dual_residual_norm = h[kSlotDualResidual]; + complementarity_residual_norm = std::max(h[kSlotComplXzLinear], h[kSlotComplWv]); + if (has_soc) { + complementarity_residual_norm = std::max(complementarity_residual_norm, h[kSlotComplCone]); + } + + const f_t mu_denom = data.complementarity_degree(data.x.size(), data.n_upper_bounds); + mu = (h[kSlotMuXzSum] + h[kSlotMuWvSum]) / mu_denom; + + const f_t quad_objective = (data.Q.n > 0) ? 0.5 * h[kSlotXQx] : f_t(0); + primal_objective = h[kSlotCx] + quad_objective; + dual_objective = h[kSlotBy] - h[kSlotUv] - quad_objective; +} + template lp_status_t barrier_solver_t::check_for_suboptimal_solution( iteration_data_t& data, @@ -4557,7 +4288,7 @@ lp_status_t barrier_solver_t::check_for_suboptimal_solution( solution); settings.log.printf("\n"); settings.log.printf( - "Suboptimal solution found in %d iterations and %.2f seconds\n", iter, toc(start_time)); + "Suboptimal solution found in %d iterations and %.3f seconds\n", iter, toc(start_time)); settings.log.printf("Objective %+.8e\n", compute_user_objective(lp, primal_objective)); settings.log.printf("Primal infeasibility (abs/rel): %8.2e/%8.2e\n", primal_residual_norm, @@ -4596,7 +4327,7 @@ lp_status_t barrier_solver_t::check_for_suboptimal_solution( solution); settings.log.printf("\n"); settings.log.printf( - "Suboptimal solution found in %d iterations and %.2f seconds\n", iter, toc(start_time)); + "Suboptimal solution found in %d iterations and %.3f seconds\n", iter, toc(start_time)); settings.log.printf("Objective %+.8e\n", compute_user_objective(lp, primal_objective_save)); settings.log.printf("Primal infeasibility (abs/rel): %8.2e/%8.2e\n", data.primal_residual_norm_save, @@ -4620,33 +4351,13 @@ lp_status_t barrier_solver_t::check_for_suboptimal_solution( return lp_status_t::NUMERICAL_ISSUES; } - -template -lp_status_t finish_barrier_session(cuopt::cython::barrier_cache_t* session, - std::unique_ptr>& owned_data, - lp_status_t status) -{ - if (session == nullptr) { return status; } - if (owned_data) { - if (status == lp_status_t::OPTIMAL) { - session->store_symbolic_cache(*owned_data); - session->store_iteration_data(owned_data.release()); - } else { - session->clear_symbolic_cache(); - } - } - return status; -} - template -lp_status_t barrier_solver_t::barrier_solve_advanced( - f_t start_time, - lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session) +lp_status_t barrier_solver_t::barrier_advanced_solve( + f_t start_time, lp_solution_t& solution, cuopt::cython::barrier_cache_t* cache) { - settings.log.printf("Barrier solver started at %.2f seconds\n", toc(start_time)); + settings.log.printf("Barrier solver started at %.3f seconds\n", toc(start_time)); try { - raft::common::nvtx::range fun_scope("Barrier: barrier_solve_advanced"); + raft::common::nvtx::range fun_scope("Barrier: barrier_advanced_solve"); i_t n = lp.num_cols; i_t m = lp.num_rows; @@ -4658,40 +4369,47 @@ lp_status_t barrier_solver_t::barrier_solve_advanced( } std::unique_ptr> owned_data; - if (session != nullptr) { - if (auto* cached = session->release_iteration_data()) { owned_data.reset(cached); } + if (cache != nullptr) { + if (auto* cached = cache->release_iteration_data()) { owned_data.reset(cached); } } if (!owned_data) { - settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + if (cache != nullptr) { cache->clear(); } + settings.log.printf( + "Barrier: cache reuse failed; cached iteration_data is missing or invalid\n"); return lp_status_t::NUMERICAL_ISSUES; } try { - if (!owned_data->rebind_settings_for_continue(settings)) { + if (!owned_data->prepare_for_reuse(settings)) { owned_data.reset(); - if (session != nullptr) { session->clear_symbolic_cache(); } - settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + if (cache != nullptr) { cache->clear(); } + settings.log.printf( + "Barrier: cache reuse failed; cached iteration_data is missing or invalid\n"); return lp_status_t::NUMERICAL_ISSUES; } } catch (const raft::cuda_error&) { owned_data.reset(); - if (session != nullptr) { session->clear_symbolic_cache(); } - settings.log.printf("Barrier: continue failed; cached iteration_data is missing or invalid\n"); + if (cache != nullptr) { cache->clear(); } + settings.log.printf( + "Barrier: cache reuse failed; cached iteration_data is missing or invalid\n"); return lp_status_t::NUMERICAL_ISSUES; } iteration_data_t& data = *owned_data; + auto fail_reuse = [&](lp_status_t status) { + if (cache != nullptr) { cache->clear(); } + return status; + }; if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return finish_barrier_session(session, owned_data, lp_status_t::CONCURRENT_LIMIT); - } - if (data.indefinite_Q) { - return finish_barrier_session(session, owned_data, lp_status_t::NUMERICAL_ISSUES); + return fail_reuse(lp_status_t::CONCURRENT_LIMIT); } + if (data.indefinite_Q) { return fail_reuse(lp_status_t::NUMERICAL_ISSUES); } if (data.symbolic_status != 0) { settings.log.printf("Error in symbolic analysis\n"); - return finish_barrier_session(session, owned_data, lp_status_t::NUMERICAL_ISSUES); + return fail_reuse(lp_status_t::NUMERICAL_ISSUES); } - return run_ipm(start_time, solution, session, owned_data); + settings.log.printf("Barrier setup complete at %.3f seconds\n", toc(start_time)); + return run_ipm(start_time, solution, cache, owned_data); } catch (const raft::cuda_error& e) { settings.log.printf("Error in barrier_solver_t: %s\n", e.what()); return lp_status_t::NUMERICAL_ISSUES; @@ -4705,13 +4423,21 @@ template lp_status_t barrier_solver_t::run_ipm( f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session, + cuopt::cython::barrier_cache_t* cache, std::unique_ptr>& owned_data) { - auto finish_session = [&](lp_status_t status) -> lp_status_t { - return finish_barrier_session(session, owned_data, status); + auto finish_cache = [&](lp_status_t status) -> lp_status_t { + if (cache != nullptr && owned_data) { + if (status == lp_status_t::OPTIMAL) { + cache->store_iteration_data(owned_data.release()); + } else { + cache->clear(); + } + } + return status; }; iteration_data_t& data = *owned_data; + { data.cusparse_dual_residual_ = data.cusparse_view_.create_vector(data.d_dual_residual_); data.cusparse_r1_ = data.cusparse_view_.create_vector(data.d_r1_); data.cusparse_tmp4_ = data.cusparse_view_.create_vector(data.d_tmp4_); @@ -4720,107 +4446,296 @@ lp_status_t barrier_solver_t::run_ipm( data.cusparse_u_ = data.cusparse_view_.create_vector(data.d_u_); data.cusparse_y_residual_ = data.cusparse_view_.create_vector(data.d_y_residual_); data.restrict_u_.resize(data.n_upper_bounds); + } + + settings.log.printf("Elapsed time : %.3fs\n", toc(start_time)); + + if (toc(start_time) > settings.time_limit) { + settings.log.printf("Barrier time limit exceeded\n"); + return finish_cache(lp_status_t::TIME_LIMIT); + } - settings.log.printf("Elapsed time : %.2fs\n", toc(start_time)); + i_t initial_status = initial_point(data); + if (toc(start_time) > settings.time_limit) { + settings.log.printf("Barrier time limit exceeded\n"); + return finish_cache(lp_status_t::TIME_LIMIT); + } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); + } + if (initial_status != 0) { + settings.log.printf("Unable to compute initial point\n"); + return finish_cache(lp_status_t::NUMERICAL_ISSUES); + } + // Upload initial point to device and compute initial residuals/norms on GPU + data.d_complementarity_wv_residual_.resize(data.n_upper_bounds, stream_view_); + data.d_complementarity_wv_rhs_.resize(data.n_upper_bounds, stream_view_); + data.d_x_.resize(data.x.size(), stream_view_); + raft::copy(data.d_x_.data(), data.x.data(), data.x.size(), stream_view_); + data.d_y_.resize(data.y.size(), stream_view_); + raft::copy(data.d_y_.data(), data.y.data(), data.y.size(), stream_view_); + data.d_z_.resize(data.z.size(), stream_view_); + raft::copy(data.d_z_.data(), data.z.data(), data.z.size(), stream_view_); + data.d_w_.resize(data.w.size(), stream_view_); + raft::copy(data.d_w_.data(), data.w.data(), data.w.size(), stream_view_); + data.d_v_.resize(data.v.size(), stream_view_); + raft::copy(data.d_v_.data(), data.v.data(), data.v.size(), stream_view_); + data.d_upper_bounds_.resize(data.upper_bounds.size(), stream_view_); + raft::copy( + data.d_upper_bounds_.data(), data.upper_bounds.data(), data.upper_bounds.size(), stream_view_); + data.d_upper_.resize(lp.upper.size(), stream_view_); + raft::copy(data.d_upper_.data(), lp.upper.data(), lp.upper.size(), stream_view_); + data.d_bound_residual_.resize(data.n_upper_bounds, stream_view_); + + f_t primal_residual_norm, dual_residual_norm, complementarity_residual_norm; + gpu_compute_residual_norms(data.d_w_, + data.d_x_, + data.d_y_, + data.d_v_, + data.d_z_, + data, + primal_residual_norm, + dual_residual_norm, + complementarity_residual_norm); + f_t mu; + compute_mu(data, mu); + + f_t norm_b = vector_norm_inf(data.b, stream_view_); + f_t norm_c = vector_norm_inf(data.c, stream_view_); + + f_t quad_objective = 0.0; + if (data.Q.n > 0) { + dense_vector_t Qx(data.Q.n); + matrix_vector_multiply(data.Q, 1.0, data.x, 0.0, Qx); + quad_objective = 0.5 * data.x.inner_product(Qx); + } + f_t primal_objective = data.c.inner_product(data.x) + quad_objective; + + f_t relative_primal_residual = primal_residual_norm / (1.0 + norm_b); + f_t relative_dual_residual = dual_residual_norm / (1.0 + norm_c); + f_t relative_complementarity_residual = + complementarity_residual_norm / + (1.0 + + std::min(std::abs(compute_user_objective(lp, primal_objective)), std::abs(primal_objective))); + + dense_vector_t upper(lp.upper); + data.gather_upper_bounds(upper, data.restrict_u_); + data.d_restrict_u_.resize(data.restrict_u_.size(), stream_view_); + raft::copy( + data.d_restrict_u_.data(), data.restrict_u_.data(), data.restrict_u_.size(), stream_view_); + f_t dual_objective = + data.b.inner_product(data.y) - data.restrict_u_.inner_product(data.v) - quad_objective; + + f_t objective_gap_abs = std::abs(primal_objective - dual_objective); + f_t objective_gap_rel = + objective_gap_abs / + std::max(f_t(1), std::min(std::abs(primal_objective), std::abs(dual_objective))); + + data.w_save = data.w; + data.x_save = data.x; + data.y_save = data.y; + data.v_save = data.v; + data.z_save = data.z; + + i_t iter = 0; + settings.log.printf("\n"); + settings.log.printf( + " Objective Infeasibility Time\n"); + settings.log.printf( + "Iter Primal Dual Primal Dual Compl. Elapsed\n"); + float64_t elapsed_time = toc(start_time); + settings.log.printf("%3d %+.12e %+.12e %.2e %.2e %.2e %.3f\n", + iter, + compute_user_objective(lp, primal_objective), + compute_user_objective(lp, dual_objective), + relative_primal_residual, + relative_dual_residual, + relative_complementarity_residual, + elapsed_time); + + bool converged = primal_residual_norm < settings.barrier_relative_feasibility_tol && + dual_residual_norm < settings.barrier_relative_optimality_tol && + complementarity_residual_norm < settings.barrier_relative_complementarity_tol; + + const i_t iteration_limit = settings.iteration_limit; + + // Adaptive regularization for the augmented system. + f_t dual_perturb = data.has_cones() ? 1e-8 : 0; + f_t primal_perturb = data.has_cones() ? 1e-8 : 1e-6; + + while (iter < iteration_limit) { + raft::common::nvtx::range fun_scope("Barrier: iteration"); if (toc(start_time) > settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return finish_session(lp_status_t::TIME_LIMIT); + return finish_cache(lp_status_t::TIME_LIMIT); } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); + } + + // Compute the affine step. This is the call that (re)factorizes the + // augmented system, so the IR residual here drives the adaptation of + // dual_perturb / primal_perturb for the next iteration's matrix. + compute_affine_rhs(data); + f_t max_affine_residual = 0.0; - i_t initial_status = initial_point(data); + i_t status; + { + raft::common::nvtx::range fun_scope("Barrier: search_direction (affine)"); + status = + gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_affine_residual); + } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); + } + + if (status < 0) { + return finish_cache(check_for_suboptimal_solution(data, + start_time, + iter, + primal_objective, + primal_residual_norm, + dual_residual_norm, + complementarity_residual_norm, + relative_primal_residual, + relative_dual_residual, + relative_complementarity_residual, + solution)); + } if (toc(start_time) > settings.time_limit) { settings.log.printf("Barrier time limit exceeded\n"); - return finish_session(lp_status_t::TIME_LIMIT); + return finish_cache(lp_status_t::TIME_LIMIT); } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - if (initial_status != 0) { - settings.log.printf("Unable to compute initial point\n"); - return finish_session(lp_status_t::NUMERICAL_ISSUES); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); } - // Upload initial point to device and compute initial residuals/norms on GPU - data.d_complementarity_wv_residual_.resize(data.n_upper_bounds, stream_view_); - data.d_complementarity_wv_rhs_.resize(data.n_upper_bounds, stream_view_); - data.d_x_.resize(data.x.size(), stream_view_); - raft::copy(data.d_x_.data(), data.x.data(), data.x.size(), stream_view_); - data.d_y_.resize(data.y.size(), stream_view_); - raft::copy(data.d_y_.data(), data.y.data(), data.y.size(), stream_view_); - data.d_z_.resize(data.z.size(), stream_view_); - raft::copy(data.d_z_.data(), data.z.data(), data.z.size(), stream_view_); - data.d_w_.resize(data.w.size(), stream_view_); - raft::copy(data.d_w_.data(), data.w.data(), data.w.size(), stream_view_); - data.d_v_.resize(data.v.size(), stream_view_); - raft::copy(data.d_v_.data(), data.v.data(), data.v.size(), stream_view_); - data.d_upper_bounds_.resize(data.upper_bounds.size(), stream_view_); - raft::copy(data.d_upper_bounds_.data(), - data.upper_bounds.data(), - data.upper_bounds.size(), - stream_view_); - data.d_upper_.resize(lp.upper.size(), stream_view_); - raft::copy(data.d_upper_.data(), lp.upper.data(), lp.upper.size(), stream_view_); - data.d_bound_residual_.resize(data.n_upper_bounds, stream_view_); - f_t primal_residual_norm, dual_residual_norm, complementarity_residual_norm; - gpu_compute_residual_norms(data.d_w_, - data.d_x_, - data.d_y_, - data.d_v_, - data.d_z_, - data, - primal_residual_norm, - dual_residual_norm, - complementarity_residual_norm); - f_t mu; - compute_mu(data, mu); - - f_t norm_b = vector_norm_inf(data.b, stream_view_); - f_t norm_c = vector_norm_inf(data.c, stream_view_); - - f_t quad_objective = 0.0; - if (data.Q.n > 0) { - dense_vector_t Qx(data.Q.n); - matrix_vector_multiply(data.Q, 1.0, data.x, 0.0, Qx); - quad_objective = 0.5 * data.x.inner_product(Qx); + f_t mu_aff, sigma, new_mu; + compute_target_mu(data, mu, mu_aff, sigma, new_mu); + + compute_cc_rhs(data, new_mu); + + // Corrector / centering step: reuses the factorization built by the + // affine call above, so the perturbation is fixed for this solve + f_t max_corrector_residual = 0.0; + + { + raft::common::nvtx::range fun_scope("Barrier: search_direction (corrector)"); + status = + gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_corrector_residual); } - f_t primal_objective = data.c.inner_product(data.x) + quad_objective; + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); + } + if (status < 0) { + return finish_cache(check_for_suboptimal_solution(data, + start_time, + iter, + primal_objective, + primal_residual_norm, + dual_residual_norm, + complementarity_residual_norm, + relative_primal_residual, + relative_dual_residual, + relative_complementarity_residual, + solution)); + } + data.has_factorization = false; + data.has_solve_info = false; + if (toc(start_time) > settings.time_limit) { + settings.log.printf("Barrier time limit exceeded\n"); + return finish_cache(lp_status_t::TIME_LIMIT); + } + if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { + settings.log.printf("Barrier solver halted\n"); + return finish_cache(lp_status_t::CONCURRENT_LIMIT); + } + + compute_final_direction(data); + f_t step_primal, step_dual; + compute_primal_dual_step_length(data, settings.barrier_step_scale, step_primal, step_dual); - f_t relative_primal_residual = primal_residual_norm / (1.0 + norm_b); - f_t relative_dual_residual = dual_residual_norm / (1.0 + norm_c); - f_t relative_complementarity_residual = + compute_next_iterate(data, settings.barrier_step_scale, step_primal, step_dual); + + compute_residual_norms_mu_and_objective(data, + primal_residual_norm, + dual_residual_norm, + complementarity_residual_norm, + mu, + primal_objective, + dual_objective); + + relative_primal_residual = primal_residual_norm / (1.0 + norm_b); + relative_dual_residual = dual_residual_norm / (1.0 + norm_c); + relative_complementarity_residual = complementarity_residual_norm / (1.0 + std::min(std::abs(compute_user_objective(lp, primal_objective)), std::abs(primal_objective))); - dense_vector_t upper(lp.upper); - data.gather_upper_bounds(upper, data.restrict_u_); - data.d_restrict_u_.resize(data.restrict_u_.size(), stream_view_); - raft::copy( - data.d_restrict_u_.data(), data.restrict_u_.data(), data.restrict_u_.size(), stream_view_); - f_t dual_objective = - data.b.inner_product(data.y) - data.restrict_u_.inner_product(data.v) - quad_objective; - - f_t objective_gap_abs = std::abs(primal_objective - dual_objective); - f_t objective_gap_rel = + objective_gap_abs = std::abs(primal_objective - dual_objective); + objective_gap_rel = objective_gap_abs / std::max(f_t(1), std::min(std::abs(primal_objective), std::abs(dual_objective))); - data.w_save = data.w; - data.x_save = data.x; - data.y_save = data.y; - data.v_save = data.v; - data.z_save = data.z; + if (relative_primal_residual < settings.barrier_relaxed_feasibility_tol && + relative_dual_residual < settings.barrier_relaxed_optimality_tol && + relative_complementarity_residual < settings.barrier_relaxed_complementarity_tol) { + if (relative_primal_residual < data.relative_primal_residual_save && + relative_dual_residual < data.relative_dual_residual_save && + relative_complementarity_residual < data.relative_complementarity_residual_save && + primal_objective == primal_objective && dual_objective == dual_objective) { + settings.log.debug( + "Saving solution at iter %d: feasibility %.2e, optimality %.2e, complementarity " + "%.2e\n", + iter, + relative_primal_residual, + relative_dual_residual, + relative_complementarity_residual); + raft::copy(data.w.data(), data.d_w_.data(), data.d_w_.size(), stream_view_); + raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); + raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); + raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); + raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + data.w_save = data.w; + data.x_save = data.x; + data.y_save = data.y; + data.v_save = data.v; + data.z_save = data.z; + data.relative_primal_residual_save = relative_primal_residual; + data.relative_dual_residual_save = relative_dual_residual; + data.relative_complementarity_residual_save = relative_complementarity_residual; + data.primal_residual_norm_save = primal_residual_norm; + data.dual_residual_norm_save = dual_residual_norm; + data.complementarity_residual_norm_save = complementarity_residual_norm; + } + } + + iter++; + elapsed_time = toc(start_time); + + if (primal_objective != primal_objective || dual_objective != dual_objective) { + settings.log.printf("Numerical error in objective\n"); + return finish_cache(check_for_suboptimal_solution(data, + start_time, + iter, + primal_objective, + primal_residual_norm, + dual_residual_norm, + complementarity_residual_norm, + relative_primal_residual, + relative_dual_residual, + relative_complementarity_residual, + solution)); + } - i_t iter = 0; - settings.log.printf("\n"); - settings.log.printf( - " Objective Infeasibility Time\n"); - settings.log.printf( - "Iter Primal Dual Primal Dual Compl. Elapsed\n"); - float64_t elapsed_time = toc(start_time); - settings.log.printf("%3d %+.12e %+.12e %.2e %.2e %.2e %.1f\n", + settings.log.printf("%3d %+.12e %+.12e %.2e %.2e %.2e %.3f\n", iter, compute_user_objective(lp, primal_objective), compute_user_objective(lp, dual_objective), @@ -4829,174 +4744,57 @@ lp_status_t barrier_solver_t::run_ipm( relative_complementarity_residual, elapsed_time); - bool converged = primal_residual_norm < settings.barrier_relative_feasibility_tol && - dual_residual_norm < settings.barrier_relative_optimality_tol && - complementarity_residual_norm < settings.barrier_relative_complementarity_tol; - - const i_t iteration_limit = settings.iteration_limit; + bool primal_feasible = relative_primal_residual < settings.barrier_relative_feasibility_tol; + bool dual_feasible = relative_dual_residual < settings.barrier_relative_optimality_tol; + bool small_gap = + relative_complementarity_residual < settings.barrier_relative_complementarity_tol; + bool small_objective_gap = + !data.has_cones() || objective_gap_rel < settings.barrier_relaxed_complementarity_tol; - // Adaptive regularization for the augmented system. - f_t dual_perturb = data.has_cones() ? 1e-8 : 0; - f_t primal_perturb = data.has_cones() ? 1e-8 : 1e-6; + converged = primal_feasible && dual_feasible && small_gap && small_objective_gap; - while (iter < iteration_limit) { - raft::common::nvtx::range fun_scope("Barrier: iteration"); - - if (toc(start_time) > settings.time_limit) { - settings.log.printf("Barrier time limit exceeded\n"); - return finish_session(lp_status_t::TIME_LIMIT); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - - // Compute the affine step. This is the call that (re)factorizes the - // augmented system, so the IR residual here drives the adaptation of - // dual_perturb / primal_perturb for the next iteration's matrix. - compute_affine_rhs(data); - f_t max_affine_residual = 0.0; - - i_t status; - { - raft::common::nvtx::range fun_scope("Barrier: search_direction (affine)"); - status = - gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_affine_residual); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - - if (status < 0) { - return finish_session(check_for_suboptimal_solution(data, - start_time, - iter, - primal_objective, - primal_residual_norm, - dual_residual_norm, - complementarity_residual_norm, - relative_primal_residual, - relative_dual_residual, - relative_complementarity_residual, - solution)); - } - if (toc(start_time) > settings.time_limit) { - settings.log.printf("Barrier time limit exceeded\n"); - return finish_session(lp_status_t::TIME_LIMIT); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - - f_t mu_aff, sigma, new_mu; - compute_target_mu(data, mu, mu_aff, sigma, new_mu); - - compute_cc_rhs(data, new_mu); - - // Corrector / centering step: reuses the factorization built by the - // affine call above, so the perturbation is fixed for this solve - f_t max_corrector_residual = 0.0; - - { - raft::common::nvtx::range fun_scope("Barrier: search_direction (corrector)"); - status = - gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_corrector_residual); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - if (status < 0) { - return finish_session(check_for_suboptimal_solution(data, - start_time, - iter, - primal_objective, - primal_residual_norm, - dual_residual_norm, - complementarity_residual_norm, - relative_primal_residual, - relative_dual_residual, - relative_complementarity_residual, - solution)); - } - data.has_factorization = false; - data.has_solve_info = false; - if (toc(start_time) > settings.time_limit) { - settings.log.printf("Barrier time limit exceeded\n"); - return finish_session(lp_status_t::TIME_LIMIT); - } - if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { - settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); - } - - compute_final_direction(data); - f_t step_primal, step_dual; - compute_primal_dual_step_length(data, settings.barrier_step_scale, step_primal, step_dual); - - compute_next_iterate(data, settings.barrier_step_scale, step_primal, step_dual); - - compute_residual_norms( - data, primal_residual_norm, dual_residual_norm, complementarity_residual_norm); - - compute_mu(data, mu); - - compute_primal_dual_objective(data, primal_objective, dual_objective); - - relative_primal_residual = primal_residual_norm / (1.0 + norm_b); - relative_dual_residual = dual_residual_norm / (1.0 + norm_c); - relative_complementarity_residual = - complementarity_residual_norm / - (1.0 + std::min(std::abs(compute_user_objective(lp, primal_objective)), - std::abs(primal_objective))); - - objective_gap_abs = std::abs(primal_objective - dual_objective); - objective_gap_rel = - objective_gap_abs / - std::max(f_t(1), std::min(std::abs(primal_objective), std::abs(dual_objective))); - - if (relative_primal_residual < settings.barrier_relaxed_feasibility_tol && - relative_dual_residual < settings.barrier_relaxed_optimality_tol && - relative_complementarity_residual < settings.barrier_relaxed_complementarity_tol) { - if (relative_primal_residual < data.relative_primal_residual_save && - relative_dual_residual < data.relative_dual_residual_save && - relative_complementarity_residual < data.relative_complementarity_residual_save && - primal_objective == primal_objective && dual_objective == dual_objective) { - settings.log.debug( - "Saving solution at iter %d: feasibility %.2e, optimality %.2e, complementarity " - "%.2e\n", - iter, - relative_primal_residual, - relative_dual_residual, - relative_complementarity_residual); - raft::copy(data.w.data(), data.d_w_.data(), data.d_w_.size(), stream_view_); - raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); - raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); - raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); - raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); - data.w_save = data.w; - data.x_save = data.x; - data.y_save = data.y; - data.v_save = data.v; - data.z_save = data.z; - data.relative_primal_residual_save = relative_primal_residual; - data.relative_dual_residual_save = relative_dual_residual; - data.relative_complementarity_residual_save = relative_complementarity_residual; - data.primal_residual_norm_save = primal_residual_norm; - data.dual_residual_norm_save = dual_residual_norm; - data.complementarity_residual_norm_save = complementarity_residual_norm; - } - } - - iter++; - elapsed_time = toc(start_time); + if (converged) { + settings.log.printf("\n"); + settings.log.printf( + "Optimal solution found in %d iterations and %.3fs\n", iter, toc(start_time)); + settings.log.printf("Objective %+.8e\n", compute_user_objective(lp, primal_objective)); + settings.log.printf("Primal infeasibility (abs/rel): %8.2e/%8.2e\n", + primal_residual_norm, + relative_primal_residual); + settings.log.printf("Dual infeasibility (abs/rel): %8.2e/%8.2e\n", + dual_residual_norm, + relative_dual_residual); + settings.log.printf("Complementarity gap (abs/rel): %8.2e/%8.2e\n", + complementarity_residual_norm, + relative_complementarity_residual); + settings.log.printf("\n"); + raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); + raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); + raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); + raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + data.to_solution(lp, + iter, + primal_objective, + compute_user_objective(lp, primal_objective), + primal_residual_norm, + data.cusparse_view_, + solution); + return finish_cache(lp_status_t::OPTIMAL); + } - if (primal_objective != primal_objective || dual_objective != dual_objective) { - settings.log.printf("Numerical error in objective\n"); - return finish_session(check_for_suboptimal_solution(data, + // Check if the solution is getting worse + if (data.Q.n > 0 && + ((!primal_feasible && + relative_primal_residual > 100 * data.relative_primal_residual_save) || + (!dual_feasible && relative_dual_residual > 100 * data.relative_dual_residual_save) || + (!small_gap && relative_complementarity_residual > + 10000 * data.relative_complementarity_residual_save))) { + if (data.relative_primal_residual_save < settings.barrier_relaxed_feasibility_tol && + data.relative_dual_residual_save < settings.barrier_relaxed_optimality_tol && + data.relative_complementarity_residual_save < + settings.barrier_relaxed_complementarity_tol) { + return finish_cache(check_for_suboptimal_solution(data, start_time, iter, primal_objective, @@ -5008,101 +4806,29 @@ lp_status_t barrier_solver_t::run_ipm( relative_complementarity_residual, solution)); } - - settings.log.printf("%3d %+.12e %+.12e %.2e %.2e %.2e %.1f\n", - iter, - compute_user_objective(lp, primal_objective), - compute_user_objective(lp, dual_objective), - relative_primal_residual, - relative_dual_residual, - relative_complementarity_residual, - elapsed_time); - - bool primal_feasible = relative_primal_residual < settings.barrier_relative_feasibility_tol; - bool dual_feasible = relative_dual_residual < settings.barrier_relative_optimality_tol; - bool small_gap = - relative_complementarity_residual < settings.barrier_relative_complementarity_tol; - bool small_objective_gap = - !data.has_cones() || objective_gap_rel < settings.barrier_relaxed_complementarity_tol; - - converged = primal_feasible && dual_feasible && small_gap && small_objective_gap; - - if (converged) { - settings.log.printf("\n"); - settings.log.printf( - "Optimal solution found in %d iterations and %.3fs\n", iter, toc(start_time)); - settings.log.printf("Objective %+.8e\n", compute_user_objective(lp, primal_objective)); - settings.log.printf("Primal infeasibility (abs/rel): %8.2e/%8.2e\n", - primal_residual_norm, - relative_primal_residual); - settings.log.printf("Dual infeasibility (abs/rel): %8.2e/%8.2e\n", - dual_residual_norm, - relative_dual_residual); - settings.log.printf("Complementarity gap (abs/rel): %8.2e/%8.2e\n", - complementarity_residual_norm, - relative_complementarity_residual); - settings.log.printf("\n"); - raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); - raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); - raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); - raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); - data.to_solution(lp, - iter, - primal_objective, - compute_user_objective(lp, primal_objective), - primal_residual_norm, - data.cusparse_view_, - solution); - return finish_session(lp_status_t::OPTIMAL); - } - - // Check if the solution is getting worse - if (data.Q.n > 0 && - ((!primal_feasible && - relative_primal_residual > 100 * data.relative_primal_residual_save) || - (!dual_feasible && relative_dual_residual > 100 * data.relative_dual_residual_save) || - (!small_gap && relative_complementarity_residual > - 10000 * data.relative_complementarity_residual_save))) { - if (data.relative_primal_residual_save < settings.barrier_relaxed_feasibility_tol && - data.relative_dual_residual_save < settings.barrier_relaxed_optimality_tol && - data.relative_complementarity_residual_save < - settings.barrier_relaxed_complementarity_tol) { - return finish_session(check_for_suboptimal_solution(data, - start_time, - iter, - primal_objective, - primal_residual_norm, - dual_residual_norm, - complementarity_residual_norm, - relative_primal_residual, - relative_dual_residual, - relative_complementarity_residual, - solution)); - } - } } - raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); - raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); - raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); - raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); - data.to_solution(lp, - iter, - primal_objective, - compute_user_objective(lp, primal_objective), - primal_residual_norm, - data.cusparse_view_, - solution); - return finish_session(lp_status_t::ITERATION_LIMIT); + } + raft::copy(data.x.data(), data.d_x_.data(), data.d_x_.size(), stream_view_); + raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); + raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); + raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + data.to_solution(lp, + iter, + primal_objective, + compute_user_objective(lp, primal_objective), + primal_residual_norm, + data.cusparse_view_, + solution); + return finish_cache(lp_status_t::ITERATION_LIMIT); } template lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session) + cuopt::cython::barrier_cache_t* cache) { - settings.log.printf("Barrier solver started at %.2f seconds\n", toc(start_time)); + settings.log.printf("Barrier solver started at %.3f seconds\n", toc(start_time)); try { raft::common::nvtx::range fun_scope("Barrier: solve"); @@ -5136,73 +4862,29 @@ lp_status_t barrier_solver_t::solve(f_t start_time, csc_matrix_t Q(lp.num_cols, 0, 0); std::unique_ptr> owned_data; - auto finish_session = [&](lp_status_t status) -> lp_status_t { - return finish_barrier_session(session, owned_data, status); - }; - + if (cache != nullptr) { cache->store_iteration_data(nullptr); } if (lp.Q.n > 0) { create_Q(lp, Q); } - bool reused_iteration_data = false; - if (session != nullptr) { - if (auto* cached = session->release_iteration_data()) { owned_data.reset(cached); } - if (owned_data) { - try { - bool rebound = owned_data->rebind_from_lp( - lp, presolve_info.direct_free_variables, Q, settings); - if (rebound) { - reused_iteration_data = true; - } else { - owned_data.reset(); - session->clear_symbolic_cache(); - } - } catch (const raft::cuda_error&) { - owned_data.reset(); - session->clear_symbolic_cache(); - } - } - } - if (!owned_data) { - barrier_symbolic_cache_t* adopt_cache = nullptr; - if (session != nullptr) { adopt_cache = session->symbolic_cache_for_reuse(lp.handle_ptr); } - owned_data = std::make_unique>( - lp, num_upper_bounds, presolve_info.direct_free_variables, Q, settings, adopt_cache); - } + owned_data = std::make_unique>( + lp, num_upper_bounds, presolve_info.direct_free_variables, Q, settings); iteration_data_t& data = *owned_data; - if (reused_iteration_data) { - settings.log.printf("Barrier: reusing cuDSS symbolic analysis (sparsity hash match)\n"); - } else if (data.adopted_symbolic()) { - try { - if (!data.refresh_lp_numerics(lp)) { - settings.log.printf( - "Barrier: hash match but numeric refresh and symbolic rebuild failed\n"); - if (session != nullptr) { session->clear_symbolic_cache(); } - return finish_session(lp_status_t::NUMERICAL_ISSUES); - } - if (data.adopted_symbolic()) { - settings.log.printf("Barrier: reusing cuDSS symbolic analysis (sparsity hash match)\n"); - } else { - settings.log.printf( - "Barrier: rebuilt cuDSS symbolic analysis (%s nnz mismatch)\n", - data.use_augmented ? "augmented" : "adat"); - } - } catch (const raft::cuda_error&) { - settings.log.printf( - "Barrier: hash match but numeric refresh failed (CUDA); clearing symbolic cache\n"); - if (session != nullptr) { session->clear_symbolic_cache(); } - return finish_session(lp_status_t::NUMERICAL_ISSUES); - } - } if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { settings.log.printf("Barrier solver halted\n"); - return finish_session(lp_status_t::CONCURRENT_LIMIT); + if (cache != nullptr) { cache->clear(); } + return lp_status_t::CONCURRENT_LIMIT; + } + if (data.indefinite_Q) { + if (cache != nullptr) { cache->clear(); } + return lp_status_t::NUMERICAL_ISSUES; } - if (data.indefinite_Q) { return finish_session(lp_status_t::NUMERICAL_ISSUES); } if (data.symbolic_status != 0) { settings.log.printf("Error in symbolic analysis\n"); - return finish_session(lp_status_t::NUMERICAL_ISSUES); + if (cache != nullptr) { cache->clear(); } + return lp_status_t::NUMERICAL_ISSUES; } - return run_ipm(start_time, solution, session, owned_data); + settings.log.printf("Barrier setup complete at %.3f seconds\n", toc(start_time)); + return run_ipm(start_time, solution, cache, owned_data); } catch (const raft::cuda_error& e) { settings.log.printf("Error in barrier_solver_t: %s\n", e.what()); return lp_status_t::NUMERICAL_ISSUES; @@ -5212,14 +4894,6 @@ lp_status_t barrier_solver_t::solve(f_t start_time, } } - -template -void barrier_store_symbolic_cache_from_iteration_data(iteration_data_t& data, - barrier_symbolic_cache_t& cache) -{ - data.store_symbolic_cache(cache); -} - #ifdef DUAL_SIMPLEX_INSTANTIATE_DOUBLE template bool validate_barrier_cone_layout( const lp_problem_t& problem, const simplex_solver_settings_t& settings); @@ -5227,9 +4901,6 @@ template class barrier_solver_t; template class sparse_cholesky_base_t; template class sparse_cholesky_cudss_t; template class iteration_data_t; - -template void barrier_store_symbolic_cache_from_iteration_data( - iteration_data_t& data, barrier_symbolic_cache_t& cache); #endif void destroy_iteration_data(iteration_data_t* data) { delete data; } @@ -5244,7 +4915,8 @@ void apply_barrier_linear_objective(iteration_data_t& data, "update_q: barrier linear objective size does not match cached iteration_data_t."); } std::copy(barrier_c, barrier_c + n, data.c.data()); - raft::copy(data.d_c_.data(), data.c.data(), static_cast(n), data.handle_ptr->get_stream()); + raft::copy( + data.d_c_.data(), data.c.data(), static_cast(n), data.handle_ptr->get_stream()); } } // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier.hpp b/cpp/src/barrier/barrier.hpp index 1bf31cb191..93cc2a197d 100644 --- a/cpp/src/barrier/barrier.hpp +++ b/cpp/src/barrier/barrier.hpp @@ -43,17 +43,18 @@ class barrier_solver_t { const simplex::simplex_solver_settings_t& settings); simplex::lp_status_t solve(f_t start_time, simplex::lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session = nullptr); - // Continue path: cached iteration_data_t already has the updated linear objective. - // Rebind settings, compute a new initial point, run IPM. Same status/solution contract as solve(). - simplex::lp_status_t barrier_solve_advanced(f_t start_time, - simplex::lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session); + cuopt::cython::barrier_cache_t* cache = nullptr); + // Cache reuse: cached iteration_data_t already has the updated linear objective. + // Prepare the workspace, compute a new initial point, run IPM. Same status/solution contract as + // solve(). + simplex::lp_status_t barrier_advanced_solve(f_t start_time, + simplex::lp_solution_t& solution, + cuopt::cython::barrier_cache_t* cache); private: simplex::lp_status_t run_ipm(f_t start_time, simplex::lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session, + cuopt::cython::barrier_cache_t* cache, std::unique_ptr>& owned_data); void my_pop_range(bool debug) const; void create_Q(const simplex::lp_problem_t& lp, csc_matrix_t& Q); @@ -81,6 +82,13 @@ class barrier_solver_t { void compute_primal_dual_objective(iteration_data_t& data, f_t& primal_objective, f_t& dual_objective); + void compute_residual_norms_mu_and_objective(iteration_data_t& data, + f_t& primal_residual_norm, + f_t& dual_residual_norm, + f_t& complementarity_residual_norm, + f_t& mu, + f_t& primal_objective, + f_t& dual_objective); // To be able to directly pass lambdas to transform functions public: diff --git a/cpp/src/barrier/barrier_factorization_sparsity_hash.cu b/cpp/src/barrier/barrier_factorization_sparsity_hash.cu deleted file mode 100644 index 7ecd48bd03..0000000000 --- a/cpp/src/barrier/barrier_factorization_sparsity_hash.cu +++ /dev/null @@ -1,25 +0,0 @@ -/* clang-format off */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -/* clang-format on */ - -#include - -namespace cuopt::mathematical_optimization::barrier { - -template -barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( - device_csr_matrix_t& mat, rmm::cuda_stream_view stream) -{ - const csr_matrix_t host = mat.to_host(stream); - return hash_host_csr_sparsity_pattern(host.m, host.row_start, host.j); -} - -template barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( - device_csr_matrix_t&, rmm::cuda_stream_view); -template barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( - device_csr_matrix_t&, rmm::cuda_stream_view); - -} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp b/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp deleted file mode 100644 index 799b267ce1..0000000000 --- a/cpp/src/barrier/barrier_factorization_sparsity_hash.hpp +++ /dev/null @@ -1,126 +0,0 @@ -/* clang-format off */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -/* clang-format on */ - -#pragma once - -#include - -#include - -#include -#include - -namespace cuopt::mathematical_optimization::barrier { - -using barrier_sparsity_hash_t = std::uint64_t; - -/// FNV-1a style mix for incremental hashing. -inline barrier_sparsity_hash_t barrier_hash_combine(barrier_sparsity_hash_t h, std::uint64_t value) -{ - constexpr barrier_sparsity_hash_t kPrime = 1099511628211ULL; - h ^= value; - h *= kPrime; - return h; -} - -inline barrier_sparsity_hash_t barrier_hash_u64(std::uint64_t value) -{ - return barrier_hash_combine(1469598103934665603ULL, value); -} - -/** - * @brief Hash CSR sparsity (row_start + col indices); numeric values are ignored. - */ -template -barrier_sparsity_hash_t hash_host_csr_sparsity_pattern(i_t num_rows, - const std::vector& row_start, - const std::vector& col_indices) -{ - barrier_sparsity_hash_t h = barrier_hash_u64(static_cast(num_rows)); - h = barrier_hash_combine(h, static_cast(col_indices.size())); - for (i_t k = 0; k <= num_rows; ++k) { - h = barrier_hash_combine(h, static_cast(row_start[static_cast(k)])); - } - for (i_t col : col_indices) { - h = barrier_hash_combine(h, static_cast(col)); - } - return h; -} - -/** - * @brief Hash the sparsity pattern of the augmented KKT matrix passed to cuDSS (host CSR). - * - * Must match the index layout produced by iteration_data_t::form_augmented(true). - */ -template -barrier_sparsity_hash_t hash_augmented_kkt_sparsity(const csc_matrix_t& A, - const csc_matrix_t& AT, - const csc_matrix_t& Q) -{ - const i_t n = A.n; - const i_t m = A.m; - const i_t size = n + m; - - std::vector row_start(static_cast(size + 1), 0); - std::vector col_indices; - col_indices.reserve(static_cast(2) * static_cast(A.col_start[n]) + - static_cast(n + m) + - (Q.n > 0 ? static_cast(Q.col_start[n]) : 0)); - - i_t q = 0; - for (i_t i = 0; i < n; ++i) { - row_start[static_cast(i)] = q; - if (Q.n == 0) { - col_indices.push_back(i); - ++q; - } else { - const i_t q_col_beg = Q.col_start[i]; - const i_t q_col_end = Q.col_start[i + 1]; - bool has_diagonal = false; - for (i_t p = q_col_beg; p < q_col_end; ++p) { - col_indices.push_back(Q.i[p]); - ++q; - if (Q.i[p] == i) { has_diagonal = true; } - } - if (!has_diagonal) { - col_indices.push_back(i); - ++q; - } - } - const i_t col_beg = A.col_start[i]; - const i_t col_end = A.col_start[i + 1]; - for (i_t p = col_beg; p < col_end; ++p) { - col_indices.push_back(A.i[p] + n); - ++q; - } - } - - for (i_t k = n; k < n + m; ++k) { - row_start[static_cast(k)] = q; - const i_t l = k - n; - const i_t col_beg = AT.col_start[l]; - const i_t col_end = AT.col_start[l + 1]; - for (i_t p = col_beg; p < col_end; ++p) { - col_indices.push_back(AT.i[p]); - ++q; - } - col_indices.push_back(k); - ++q; - } - row_start[static_cast(size)] = q; - - return hash_host_csr_sparsity_pattern(size, row_start, col_indices); -} - -/** - * @brief Hash CSR sparsity from a device matrix (copies row/col indices to host). - */ -template -barrier_sparsity_hash_t hash_device_csr_sparsity_pattern( - device_csr_matrix_t& mat, rmm::cuda_stream_view stream); - -} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/barrier_symbolic_cache.hpp b/cpp/src/barrier/barrier_symbolic_cache.hpp deleted file mode 100644 index 63826a2530..0000000000 --- a/cpp/src/barrier/barrier_symbolic_cache.hpp +++ /dev/null @@ -1,86 +0,0 @@ -/* clang-format off */ -/* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -/* clang-format on */ - -#pragma once - -#include -#include -#include - -#include - -#include -#include - -namespace cuopt::mathematical_optimization::barrier { - -using barrier_sparsity_hash_t = std::uint64_t; - -template -class sparse_cholesky_cudss_t; - -/** - * @brief Cached cuDSS symbolic state and GPU buffers for hash-gated barrier reuse. - * - * Holds reordering + symbolic factorization in @p chol, a sparsity hash used to gate reuse, - * and path-specific GPU workspace (augmented KKT or ADAT + cuSPARSE). - * - * Hash meaning: augmented store uses device KKT CSR (adopt uses matching host synthetic); - * ADAT store/adopt use the constraint-matrix @c device_A CSR pattern (not ADAT), so adopt can - * reject before pinning SpGEMM workspace. - */ -template -struct barrier_symbolic_cache_t { - std::shared_ptr> chol; - barrier_sparsity_hash_t sparsity_hash{0}; - raft::handle_t const* handle_ptr{nullptr}; - bool use_augmented{false}; - bool valid{false}; - - // --- Augmented KKT (use_augmented == true) --- - device_csr_matrix_t device_augmented; - rmm::device_uvector d_augmented_diagonal_indices_; - - // --- ADAT (use_augmented == false) --- - device_csr_matrix_t device_ADAT; - device_csc_matrix_t device_AD; - device_csr_matrix_t device_A; - rmm::device_uvector d_original_A_values; - rmm::device_uvector device_A_x_values; - std::unique_ptr> cusparse_info; - - explicit barrier_symbolic_cache_t(rmm::cuda_stream_view stream) - : device_augmented(stream), - d_augmented_diagonal_indices_(0, stream), - device_ADAT(stream), - device_AD(stream), - device_A(stream), - d_original_A_values(0, stream), - device_A_x_values(0, stream) - { - } - - void clear() - { - chol.reset(); - sparsity_hash = 0; - handle_ptr = nullptr; - use_augmented = false; - valid = false; - cusparse_info.reset(); - } - - [[nodiscard]] bool matches_reuse(barrier_sparsity_hash_t hash, - bool augmented, - raft::handle_t const* handle) const - { - return valid && handle != nullptr && handle_ptr == handle && use_augmented == augmented && - sparsity_hash == hash; - } -}; - -} // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index f0a5b07f1b..22b96c4fa0 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -499,7 +499,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { return -1; } f_t reordering_time = toc(start_symbolic); - settings_->log.printf("Reordering time : %.2fs\n", reordering_time); + settings_->log.printf("Reordering time : %.3fs\n", reordering_time); start_symbolic_factor = tic(); status = cudssExecute( @@ -517,7 +517,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { } RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); f_t symbolic_factorization_time = toc(start_symbolic_factor); - settings_->log.printf("Symbolic factorization time : %.2fs\n", symbolic_factorization_time); + settings_->log.printf("Symbolic factorization time : %.3fs\n", symbolic_factorization_time); int64_t lu_nz = 0; size_t size_written = 0; CUDSS_CALL_AND_CHECK( @@ -540,7 +540,8 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { if (!symbolic_done_ || !A_created) { settings_->log.printf( - "Error: cuDSS factorize(device_csr) called before analyze (symbolic_done=%d A_created=%d)\n", + "Error: cuDSS factorize(device_csr) called before analyze (symbolic_done=%d " + "A_created=%d)\n", static_cast(symbolic_done_), static_cast(A_created)); return -1; @@ -606,7 +607,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { } if (first_factor) { - settings_->log.debug("Factorization time : %.2fs\n", numeric_time); + settings_->log.debug("Factorization time : %.3fs\n", numeric_time); first_factor = false; } if (status != CUDSS_STATUS_SUCCESS) { @@ -730,7 +731,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { f_t symbolic_time = toc(start_symbolic); f_t analysis_time = toc(start_analysis); - settings_->log.printf("Symbolic factorization time : %.2fs\n", symbolic_time); + settings_->log.printf("Symbolic factorization time : %.3fs\n", symbolic_time); if (settings_->concurrent_halt != nullptr && *settings_->concurrent_halt == 1) { RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); handle_ptr_->get_stream().synchronize(); @@ -800,7 +801,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { } if (first_factor) { - settings_->log.debug("Factorization time : %.2fs\n", numeric_time); + settings_->log.debug("Factorization time : %.3fs\n", numeric_time); first_factor = false; } if (status != CUDSS_STATUS_SUCCESS) { @@ -868,10 +869,10 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { raft::copy(x_host.data(), x.data(), n, stream); cudaStreamSynchronize(stream); settings_->log.printf("RHS norm %.16e, hash: %zu, Solution norm %.16e, hash: %zu\n", - vector_norm2(b_host), - compute_hash(b_host), - vector_norm2(x_host), - compute_hash(x_host)); + vector_norm2(b_host), + compute_hash(b_host), + vector_norm2(x_host), + compute_hash(x_host)); #endif return 0; diff --git a/cpp/src/dual_simplex/solve.cpp b/cpp/src/dual_simplex/solve.cpp index 79a5f0cebd..929b082456 100644 --- a/cpp/src/dual_simplex/solve.cpp +++ b/cpp/src/dual_simplex/solve.cpp @@ -27,7 +27,7 @@ #include #include -#include +#include #include @@ -42,21 +42,6 @@ namespace cuopt::mathematical_optimization::simplex { namespace { -template -bool can_continue_barrier_c_only(const user_problem_t& user_problem, - const simplex_solver_settings_t& settings, - cuopt::cython::barrier_cache_t* session) -{ - if (session == nullptr || !session->c_dirty()) { return false; } - if (session->iteration_data() == nullptr) { return false; } - auto const* front_end = session->front_end_cache(); - if (front_end == nullptr || front_end->barrier_lp == nullptr) { return false; } - if (user_problem.Q_values.empty()) { return false; } - if (settings.barrier_presolve_bound_free_variables != 0) { return false; } - return user_problem.num_cols == front_end->user_num_cols && - user_problem.num_rows == front_end->user_num_rows; -} - template void unscale_uncrush_barrier_to_user(const user_problem_t& user_problem, const raft::handle_t* handle_ptr, @@ -425,37 +410,45 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session, + cuopt::cython::barrier_cache_t* cache, const raft::handle_t* handle_ptr) { - lp_status_t status = lp_status_t::UNSET; + lp_status_t status = lp_status_t::UNSET; simplex_solver_settings_t barrier_settings = settings; - if (can_continue_barrier_c_only(user_problem, barrier_settings, session)) { - settings.log.printf( - "Barrier: continue from session (skip convert/presolve/scaling)\n"); - auto* front_end = session->front_end_cache(); - lp_solution_t barrier_solution(front_end->barrier_lp->num_rows, - front_end->barrier_lp->num_cols); + auto const* xf = + (cache != nullptr && cache->c_dirty()) ? cache->transform() : nullptr; + const bool reuse_c_only = + xf != nullptr && xf->barrier_lp != nullptr && !user_problem.Q_values.empty() && + user_problem.second_order_cone_dims.empty() && xf->second_order_cone_dims.empty() && + xf->barrier_lp->second_order_cone_dims.empty() && + settings.barrier_presolve_bound_free_variables == 0 && + user_problem.num_cols == xf->user_num_cols && + user_problem.num_rows == xf->user_num_rows; + + if (reuse_c_only) { + settings.log.printf("Barrier: reusing cache (skip convert/presolve/scaling)\n"); + lp_solution_t barrier_solution(xf->barrier_lp->num_rows, + xf->barrier_lp->num_cols); barrier::barrier_solver_t barrier_solver( - *front_end->barrier_lp, front_end->presolve_info, barrier_settings); + *xf->barrier_lp, xf->presolve_info, barrier_settings); lp_status_t barrier_status = - barrier_solver.barrier_solve_advanced(start_time, barrier_solution, session); + barrier_solver.barrier_advanced_solve(start_time, barrier_solution, cache); if (barrier_status == lp_status_t::OPTIMAL) { unscale_uncrush_barrier_to_user(user_problem, - session->handle_ptr(), - front_end->original_num_rows, - front_end->original_num_cols, - *front_end->barrier_lp, - front_end->presolve_info, - front_end->column_scales, - front_end->row_scales, + cache->handle_ptr(), + xf->original_num_rows, + xf->original_num_cols, + *xf->barrier_lp, + xf->presolve_info, + xf->column_scales, + xf->row_scales, barrier_settings, barrier_solution, solution); - session->set_c_dirty(false); + cache->set_c_dirty(false); } else { - session->clear_front_end_cache(); + cache->clear(); } return barrier_status; } @@ -465,7 +458,9 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us // Convert the user problem to a linear program with only equality constraints std::vector new_slacks; dualize_info_t dualize_info; - convert_user_problem(user_problem, barrier_settings, original_lp, new_slacks, dualize_info); + { + convert_user_problem(user_problem, barrier_settings, original_lp, new_slacks, dualize_info); + } if (!barrier::validate_barrier_cone_layout(original_lp, barrier_settings)) { return lp_status_t::NUMERICAL_ISSUES; } @@ -475,7 +470,10 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us // Presolve the linear program presolve_info_t presolve_info; lp_problem_t presolved_lp(handle_ptr, 1, 1, 1); - const i_t ok = presolve(original_lp, barrier_settings, presolved_lp, presolve_info); + i_t ok; + { + ok = presolve(original_lp, barrier_settings, presolved_lp, presolve_info); + } if (ok == CONCURRENT_HALT_RETURN) { return lp_status_t::CONCURRENT_LIMIT; } if (ok == TIME_LIMIT_RETURN) { return lp_status_t::TIME_LIMIT; } if (ok == -1) { return lp_status_t::INFEASIBLE; } @@ -487,49 +485,53 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us presolved_lp.A.col_start[presolved_lp.num_cols]); std::vector column_scales; std::vector row_scales; - scaling(presolved_lp, barrier_settings, barrier_lp, column_scales, row_scales); + { + scaling(presolved_lp, barrier_settings, barrier_lp, column_scales, row_scales); + } // Solve using barrier lp_solution_t barrier_solution(barrier_lp.num_rows, barrier_lp.num_cols); barrier::barrier_solver_t barrier_solver(barrier_lp, presolve_info, barrier_settings); - lp_status_t barrier_status = barrier_solver.solve(start_time, barrier_solution, session); + lp_status_t barrier_status = barrier_solver.solve(start_time, barrier_solution, cache); - if (session != nullptr) { + if (cache != nullptr) { if (barrier_status == lp_status_t::OPTIMAL) { - auto front_end = std::make_unique(); - front_end->c_dirty = false; - front_end->user_num_cols = user_problem.num_cols; - front_end->user_num_rows = user_problem.num_rows; - front_end->original_num_cols = original_lp.num_cols; - front_end->original_num_rows = original_lp.num_rows; - front_end->barrier_num_cols = barrier_lp.num_cols; - front_end->barrier_num_rows = barrier_lp.num_rows; - front_end->obj_scale = user_problem.obj_scale; - front_end->obj_constant = user_problem.obj_constant; - front_end->presolve_info = presolve_info; - front_end->column_scales = column_scales; - front_end->row_scales = row_scales; - front_end->barrier_lp = std::make_unique>(barrier_lp); + auto xf = std::make_unique(); + xf->user_num_cols = user_problem.num_cols; + xf->user_num_rows = user_problem.num_rows; + xf->original_num_cols = original_lp.num_cols; + xf->original_num_rows = original_lp.num_rows; + xf->obj_scale = user_problem.obj_scale; + xf->obj_constant = user_problem.obj_constant; + xf->row_sense = user_problem.row_sense; + xf->cone_var_start = user_problem.cone_var_start; + xf->second_order_cone_dims = user_problem.second_order_cone_dims; + xf->expanded_original_num_cols = user_problem.original_num_cols; + xf->original_col_to_expanded_col = user_problem.original_col_to_expanded_col; + xf->presolve_info = presolve_info; + xf->column_scales = column_scales; + xf->row_scales = row_scales; + xf->barrier_lp = std::make_unique>(barrier_lp); { try { auto crushed = cuopt::cython::crush_user_linear_objective( - *front_end, user_problem.objective.data(), user_problem.num_cols); - front_end->linear_obj_shift.resize(static_cast(barrier_lp.num_cols), 0.0); + *xf, user_problem.objective.data(), user_problem.num_cols); + xf->linear_obj_shift.resize(static_cast(barrier_lp.num_cols), 0.0); if (static_cast(crushed.size()) == barrier_lp.num_cols) { for (int j = 0; j < barrier_lp.num_cols; ++j) { - front_end->linear_obj_shift[static_cast(j)] = + xf->linear_obj_shift[static_cast(j)] = barrier_lp.objective[static_cast(j)] - crushed[static_cast(j)]; } } } catch (std::exception const&) { - front_end->linear_obj_shift.assign(static_cast(barrier_lp.num_cols), 0.0); + xf->linear_obj_shift.assign(static_cast(barrier_lp.num_cols), 0.0); } } - session->store_front_end_cache(std::move(front_end)); + cache->store_transform(std::move(xf)); } else { - session->clear_front_end_cache(); + cache->clear(); } } @@ -826,11 +828,11 @@ template lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, const simplex_solver_settings_t& settings, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session) + cuopt::cython::barrier_cache_t* cache) { f_t start_time = tic(); return solve_linear_program_with_barrier( - user_problem, settings, start_time, solution, session, user_problem.handle_ptr); + user_problem, settings, start_time, solution, cache, user_problem.handle_ptr); } template @@ -838,10 +840,10 @@ lp_status_t solve_linear_program_with_barrier(const user_problem_t& us const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session) + cuopt::cython::barrier_cache_t* cache) { return solve_linear_program_with_barrier( - user_problem, settings, start_time, solution, session, user_problem.handle_ptr); + user_problem, settings, start_time, solution, cache, user_problem.handle_ptr); } template @@ -984,21 +986,21 @@ template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session); + cuopt::cython::barrier_cache_t* cache); template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, double start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session); + cuopt::cython::barrier_cache_t* cache); template lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, double start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session, + cuopt::cython::barrier_cache_t* cache, const raft::handle_t* handle_ptr); template lp_status_t solve_linear_program(const user_problem_t& user_problem, diff --git a/cpp/src/dual_simplex/solve.hpp b/cpp/src/dual_simplex/solve.hpp index e2cb71745a..6dc9d436d9 100644 --- a/cpp/src/dual_simplex/solve.hpp +++ b/cpp/src/dual_simplex/solve.hpp @@ -96,7 +96,7 @@ lp_status_t solve_linear_program_with_barrier( const user_problem_t& user_problem, const simplex_solver_settings_t& settings, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session = nullptr); + cuopt::cython::barrier_cache_t* cache = nullptr); template lp_status_t solve_linear_program_with_barrier( @@ -104,14 +104,14 @@ lp_status_t solve_linear_program_with_barrier( const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session = nullptr); + cuopt::cython::barrier_cache_t* cache = nullptr); template lp_status_t solve_linear_program_with_barrier(const user_problem_t& user_problem, const simplex_solver_settings_t& settings, f_t start_time, lp_solution_t& solution, - cuopt::cython::barrier_cache_t* session, + cuopt::cython::barrier_cache_t* cache, const raft::handle_t* handle_ptr); template diff --git a/cpp/src/linear_algebra/vector_math.cuh b/cpp/src/linear_algebra/vector_math.cuh index ac9d24001b..fe3e11bd99 100644 --- a/cpp/src/linear_algebra/vector_math.cuh +++ b/cpp/src/linear_algebra/vector_math.cuh @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -68,6 +69,71 @@ f_t device_custom_vector_norm_inf(InputIteratorT in, i_t size, rmm::cuda_stream_ return d_out.value(stream_view); } +// Same reduction as device_custom_vector_norm_inf, but writes into a caller-supplied device +// pointer (and reuses a caller-supplied temp-storage buffer) instead of allocating a private +// rmm::device_scalar and blocking on .value(). Lets callers batch several reductions and defer +// the host readback to a single copy + sync. +template +void enqueue_norm_inf_into( + InputIteratorT in, i_t size, f_t* out, rmm::device_buffer& tmp, rmm::cuda_stream_view stream_view) +{ + if (size == 0) { + RAFT_CUDA_TRY(cudaMemsetAsync(out, 0, sizeof(f_t), stream_view.value())); + return; + } + size_t temp_storage_bytes = 0; + f_t init = 0; + auto custom_op = norm_inf_max{}; + cub::DeviceReduce::Reduce( + nullptr, temp_storage_bytes, in, out, size, custom_op, init, stream_view); + + tmp.resize(temp_storage_bytes, stream_view); + + cub::DeviceReduce::Reduce( + tmp.data(), temp_storage_bytes, in, out, size, custom_op, init, stream_view); +} + +// Sum reduction into a caller-supplied device pointer/temp-storage buffer, deferring the host +// readback (see enqueue_norm_inf_into). +template +void enqueue_sum_into( + InputIteratorT in, i_t size, f_t* out, rmm::device_buffer& tmp, rmm::cuda_stream_view stream_view) +{ + if (size == 0) { + RAFT_CUDA_TRY(cudaMemsetAsync(out, 0, sizeof(f_t), stream_view.value())); + return; + } + size_t temp_storage_bytes = 0; + cub::DeviceReduce::Sum(nullptr, temp_storage_bytes, in, out, size, stream_view); + + tmp.resize(temp_storage_bytes, stream_view); + + cub::DeviceReduce::Sum(tmp.data(), temp_storage_bytes, in, out, size, stream_view); +} + +// Max reduction (with a floor of 0, matching this codebase's existing +// thrust::reduce(..., f_t(0), thrust::maximum()) usage) into a caller-supplied device +// pointer/temp-storage buffer, deferring the host readback (see enqueue_norm_inf_into). +template +void enqueue_max_into( + InputIteratorT in, i_t size, f_t* out, rmm::device_buffer& tmp, rmm::cuda_stream_view stream_view) +{ + if (size == 0) { + RAFT_CUDA_TRY(cudaMemsetAsync(out, 0, sizeof(f_t), stream_view.value())); + return; + } + size_t temp_storage_bytes = 0; + f_t init = 0; + auto custom_op = thrust::maximum{}; + cub::DeviceReduce::Reduce( + nullptr, temp_storage_bytes, in, out, size, custom_op, init, stream_view); + + tmp.resize(temp_storage_bytes, stream_view); + + cub::DeviceReduce::Reduce( + tmp.data(), temp_storage_bytes, in, out, size, custom_op, init, stream_view); +} + template f_t device_vector_norm_inf(const rmm::device_uvector& in, rmm::cuda_stream_view stream_view) { diff --git a/cpp/src/pdlp/solve.cu b/cpp/src/pdlp/solve.cu index e4c2e3e04c..5fa0fbedbf 100644 --- a/cpp/src/pdlp/solve.cu +++ b/cpp/src/pdlp/solve.cu @@ -35,7 +35,8 @@ #include #include #include -#include +#include +#include #include #include @@ -77,45 +78,26 @@ namespace cuopt::mathematical_optimization { namespace { template -uint64_t fnv1a64_mix(uint64_t hash, uint64_t value) +simplex::user_problem_t user_problem_from_transform( + raft::handle_t const* handle_ptr, + optimization_problem_t& model, + cuopt::cython::barrier_transform_t const& xf) { - constexpr uint64_t kFnvPrime = 1099511628211ULL; - constexpr uint64_t kFnvOffset = 14695981039346656037ULL; - if (hash == 0) { hash = kFnvOffset; } - for (int shift = 0; shift < 64; shift += 8) { - hash ^= (value >> shift) & 0xFFULL; - hash *= kFnvPrime; - } - return hash; -} - -template -uint64_t compute_problem_fingerprint(const optimization_problem_t& op) -{ - uint64_t hash = fnv1a64_mix(0, static_cast(op.get_n_variables())); - hash = fnv1a64_mix(hash, static_cast(op.get_n_constraints())); - hash = fnv1a64_mix(hash, static_cast(op.get_nnz())); - - const auto offsets = op.get_constraint_matrix_offsets_host(); - for (i_t off : offsets) { - hash = fnv1a64_mix(hash, static_cast(off)); - } - const auto indices = op.get_constraint_matrix_indices_host(); - for (i_t idx : indices) { - hash = fnv1a64_mix(hash, static_cast(idx)); - } - - if (op.has_quadratic_objective()) { - const auto q_offsets = op.get_quadratic_objective_offsets(); - for (i_t off : q_offsets) { - hash = fnv1a64_mix(hash, static_cast(off)); - } - const auto q_indices = op.get_quadratic_objective_indices(); - for (i_t idx : q_indices) { - hash = fnv1a64_mix(hash, static_cast(idx)); - } - } - return hash; + simplex::user_problem_t user_problem(handle_ptr); + user_problem.num_rows = xf.user_num_rows; + user_problem.num_cols = xf.user_num_cols; + user_problem.objective = model.get_objective_coefficients_host(); + user_problem.row_sense = xf.row_sense; + user_problem.rhs.assign(static_cast(xf.user_num_rows), f_t(0)); + user_problem.obj_scale = static_cast(xf.obj_scale); + user_problem.obj_constant = static_cast(xf.obj_constant); + // Nonempty Q so the cache-reuse path accepts this as a QP (it rejects empty Q). + user_problem.Q_values.assign(1, f_t(1)); + user_problem.cone_var_start = xf.cone_var_start; + user_problem.second_order_cone_dims = xf.second_order_cone_dims; + user_problem.original_num_cols = xf.expanded_original_num_cols; + user_problem.original_col_to_expanded_col = xf.original_col_to_expanded_col; + return user_problem; } } // namespace @@ -541,7 +523,7 @@ std::tuple, simplex::lp_status_t, f_t, f_t, f_t pdlp_solver_settings_t const& settings, const timer_t& timer, const raft::handle_t* handle_ptr, - cuopt::cython::barrier_cache_t* session = nullptr) + cuopt::cython::barrier_cache_t* cache = nullptr) { f_t norm_user_objective = vector_norm2(user_problem.objective); f_t norm_rhs = vector_norm2(user_problem.rhs); @@ -578,7 +560,7 @@ std::tuple, simplex::lp_status_t, f_t, f_t, f_t simplex::lp_solution_t solution(user_problem.num_rows, user_problem.num_cols); auto status = simplex::solve_linear_program_with_barrier( - user_problem, barrier_settings, timer.get_tic_start(), solution, session, handle_ptr); + user_problem, barrier_settings, timer.get_tic_start(), solution, cache, handle_ptr); if (status == simplex::lp_status_t::OPTIMAL) { barrier::project_barrier_solution_to_model_variables(user_problem, solution); @@ -603,13 +585,13 @@ optimization_problem_solution_t run_barrier( mip::problem_t& problem, pdlp_solver_settings_t const& settings, const timer_t& timer, - cuopt::cython::barrier_cache_t* session = nullptr) + cuopt::cython::barrier_cache_t* cache = nullptr) { // Convert data structures to dual simplex format and back simplex::user_problem_t dual_simplex_problem = cuopt_problem_to_user_problem(problem.handle_ptr, problem, false); auto sol_dual_simplex = - run_barrier(dual_simplex_problem, settings, timer, problem.handle_ptr, session); + run_barrier(dual_simplex_problem, settings, timer, problem.handle_ptr, cache); return convert_dual_simplex_sol(problem, std::get<0>(sol_dual_simplex), std::get<1>(sol_dual_simplex), @@ -1885,14 +1867,22 @@ optimization_problem_solution_t solve_qcqp( print_version_info(); // Init libraries before to not include it in solve time - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C02); - init_handler(op_problem.get_handle_ptr()); - } + init_handler(op_problem.get_handle_ptr()); auto qcqp_timer = cuopt::timer_t(settings.time_limit); - if (problem_checking) { + auto* cache = settings.barrier_cache; + auto const* xf = (cache != nullptr && cache->c_dirty()) ? cache->transform() : nullptr; + const bool reuse_from_cache = + settings.user_problem_file.empty() && xf != nullptr && xf->barrier_lp != nullptr && + settings.barrier_presolve_bound_free_variables == 0 && + op_problem.has_quadratic_objective() && !op_problem.has_quadratic_constraints() && + xf->second_order_cone_dims.empty() && + static_cast(xf->row_sense.size()) == xf->user_num_rows && + op_problem.get_n_variables() == xf->user_num_cols && + op_problem.get_n_constraints() == xf->user_num_rows; + + if (problem_checking && !reuse_from_cache) { problem_checking_t::check_problem_representation(op_problem); if (problem_checking_t::has_crossing_bounds(op_problem)) { return optimization_problem_solution_t( @@ -1920,19 +1910,20 @@ optimization_problem_solution_t solve_qcqp( CUOPT_LOG_INFO("Writing user problem to file: %s", settings.user_problem_file.c_str()); op_problem.write_to_mps(settings.user_problem_file); } - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C03); - [[maybe_unused]] const uint64_t fingerprint = compute_problem_fingerprint(op_problem); + simplex::user_problem_t dual_simplex_problem(op_problem.get_handle_ptr()); + if (reuse_from_cache) { + dual_simplex_problem = user_problem_from_transform( + op_problem.get_handle_ptr(), op_problem, *settings.barrier_cache->transform()); + } else { + dual_simplex_problem = cuopt_optimization_problem_to_user_problem( + op_problem.get_handle_ptr(), op_problem); } - // Convert data structures to dual simplex format and back - simplex::user_problem_t dual_simplex_problem = - cuopt_optimization_problem_to_user_problem(op_problem.get_handle_ptr(), op_problem); auto sol_dual_simplex = run_barrier(dual_simplex_problem, settings, qcqp_timer, op_problem.get_handle_ptr(), settings.barrier_cache); - auto solution = convert_dual_simplex_sol(op_problem, + auto solution = convert_dual_simplex_sol(op_problem, std::get<0>(sol_dual_simplex), std::get<1>(sol_dual_simplex), std::get<2>(sol_dual_simplex), @@ -2053,10 +2044,7 @@ optimization_problem_solution_t solve_lp( // Init libraries before to not include it in solve time // This needs to be called before pdlp is initialized - { - CUOPT_CACHE_PROFILE_SCOPE(cuopt::linear_programming::cache_profile::cache_id::C02); - init_handler(op_problem.get_handle_ptr()); - } + init_handler(op_problem.get_handle_ptr()); raft::common::nvtx::range fun_scope("Running solver"); diff --git a/cpp/src/pdlp/utilities/barrier_cache.cu b/cpp/src/pdlp/utilities/barrier_cache.cu index 748e0663f3..9690d8a59e 100644 --- a/cpp/src/pdlp/utilities/barrier_cache.cu +++ b/cpp/src/pdlp/utilities/barrier_cache.cu @@ -8,10 +8,8 @@ #include #include -#include -#include +#include -#include #include #include #include @@ -33,10 +31,9 @@ struct barrier_cache_t::impl { std::unique_ptr stream; std::unique_ptr handle; - std::optional> - symbolic_cache; barrier_iteration_data_ptr iteration_data; - std::unique_ptr front_end; + std::unique_ptr transform; + bool c_dirty{false}; }; barrier_cache_t::barrier_cache_t(std::unique_ptr stream, @@ -73,31 +70,11 @@ rmm::cuda_stream_view barrier_cache_t::stream_view() const return impl_->stream->view(); } -mathematical_optimization::barrier::barrier_symbolic_cache_t* -barrier_cache_t::symbolic_cache_for_reuse(raft::handle_t const* handle) +void barrier_cache_t::clear() { - if (handle == nullptr || !impl_->symbolic_cache.has_value() || !impl_->symbolic_cache->valid || - impl_->symbolic_cache->handle_ptr != handle) { - return nullptr; - } - return &(*impl_->symbolic_cache); -} - -void barrier_cache_t::clear_symbolic_cache() -{ - impl_->symbolic_cache.reset(); - clear_iteration_data(); - clear_front_end_cache(); -} - -void barrier_cache_t::store_symbolic_cache( - mathematical_optimization::barrier::iteration_data_t& data) -{ - if (!impl_->symbolic_cache.has_value()) { - impl_->symbolic_cache.emplace(impl_->handle->get_stream()); - } - mathematical_optimization::barrier::barrier_store_symbolic_cache_from_iteration_data( - data, *impl_->symbolic_cache); + impl_->iteration_data.reset(); + impl_->transform.reset(); + impl_->c_dirty = false; } void barrier_cache_t::store_iteration_data(barrier_iteration_data_t* data) @@ -110,56 +87,39 @@ barrier_iteration_data_t* barrier_cache_t::release_iteration_data() return impl_->iteration_data.release(); } -barrier_iteration_data_t* barrier_cache_t::iteration_data() -{ - return impl_->iteration_data.get(); -} - -void barrier_cache_t::clear_iteration_data() { impl_->iteration_data.reset(); } - -void barrier_cache_t::store_front_end_cache(std::unique_ptr cache) +void barrier_cache_t::store_transform(std::unique_ptr transform) { - impl_->front_end = std::move(cache); + impl_->transform = std::move(transform); } -barrier_front_end_cache_t* barrier_cache_t::front_end_cache() { return impl_->front_end.get(); } +barrier_transform_t* barrier_cache_t::transform() { return impl_->transform.get(); } -barrier_front_end_cache_t const* barrier_cache_t::front_end_cache() const -{ - return impl_->front_end.get(); -} +barrier_transform_t const* barrier_cache_t::transform() const { return impl_->transform.get(); } -void barrier_cache_t::clear_front_end_cache() { impl_->front_end.reset(); } - -void barrier_cache_t::set_c_dirty(bool dirty) -{ - if (impl_->front_end) { impl_->front_end->c_dirty = dirty; } -} +void barrier_cache_t::set_c_dirty(bool dirty) { impl_->c_dirty = dirty; } bool barrier_cache_t::c_dirty() const { - return impl_->front_end != nullptr && impl_->front_end->c_dirty; + return impl_->c_dirty && impl_->transform != nullptr && impl_->iteration_data.get() != nullptr; } -bool barrier_cache_t::has_front_end_cache() const { return impl_->front_end != nullptr; } - void barrier_cache_t::update_linear_objective(double const* c, int n) { - cuopt_expects(impl_->front_end != nullptr, + cuopt_expects(impl_->transform != nullptr, error_type_t::ValidationError, - "update_q: no front-end cache; Solve with sequence_solve first."); + "update_q: no barrier transform; Solve with sequence_solve first."); cuopt_expects(impl_->iteration_data.get() != nullptr, error_type_t::ValidationError, "update_q: no cached iteration_data; Solve a QP to Optimal first."); std::vector crushed; try { - crushed = crush_user_linear_objective(*impl_->front_end, c, n); + crushed = crush_user_linear_objective(*impl_->transform, c, n); } catch (std::invalid_argument const& e) { cuopt_expects(false, error_type_t::ValidationError, "%s", e.what()); } - if (impl_->front_end->linear_obj_shift.size() == crushed.size()) { + if (impl_->transform->linear_obj_shift.size() == crushed.size()) { for (std::size_t j = 0; j < crushed.size(); ++j) { - crushed[j] += impl_->front_end->linear_obj_shift[j]; + crushed[j] += impl_->transform->linear_obj_shift[j]; } } try { @@ -168,7 +128,7 @@ void barrier_cache_t::update_linear_objective(double const* c, int n) } catch (std::invalid_argument const& e) { cuopt_expects(false, error_type_t::ValidationError, "%s", e.what()); } - impl_->front_end->c_dirty = true; + impl_->c_dirty = true; } } // namespace cuopt::cython diff --git a/cpp/src/pdlp/utilities/barrier_front_end_cache.hpp b/cpp/src/pdlp/utilities/barrier_transform.hpp similarity index 59% rename from cpp/src/pdlp/utilities/barrier_front_end_cache.hpp rename to cpp/src/pdlp/utilities/barrier_transform.hpp index d405ef33b9..f4c6185498 100644 --- a/cpp/src/pdlp/utilities/barrier_front_end_cache.hpp +++ b/cpp/src/pdlp/utilities/barrier_transform.hpp @@ -16,22 +16,26 @@ namespace cuopt::cython { /** - * Convert / presolve / scaling state retained on barrier_cache_t after Optimal. - * Enough to crush a new user-space linear objective into barrier space (C) and to - * uncrush a solution (D) without rerunning those algorithms. + * User ↔ barrier transform retained on barrier_cache_t after Optimal: + * convert / presolve / scaling, plus the scaled LP. + * Enough to crush a new user-space linear objective into barrier space and to + * uncrush a solution without rerunning those algorithms. */ -struct barrier_front_end_cache_t { - bool c_dirty{false}; - +struct barrier_transform_t { int user_num_cols{0}; int user_num_rows{0}; int original_num_cols{0}; int original_num_rows{0}; - int barrier_num_cols{0}; - int barrier_num_rows{0}; double obj_scale{1.0}; double obj_constant{0.0}; + // Enough of the user problem for reuse uncrush without rebuilding A. + std::vector row_sense; + int cone_var_start{0}; + std::vector second_order_cone_dims; + int expanded_original_num_cols{0}; + std::vector original_col_to_expanded_col; + cuopt::mathematical_optimization::simplex::presolve_info_t presolve_info; std::vector column_scales; std::vector row_scales; @@ -40,37 +44,40 @@ struct barrier_front_end_cache_t { std::unique_ptr> barrier_lp; }; -inline std::vector crush_user_linear_objective(barrier_front_end_cache_t const& fe, +inline std::vector crush_user_linear_objective(barrier_transform_t const& xf, double const* c, int n) { - if (c == nullptr || n != fe.user_num_cols) { + if (c == nullptr || n != xf.user_num_cols) { throw std::invalid_argument( "update_q: linear objective length must match the cached user column count."); } - if (fe.original_num_cols < fe.user_num_cols) { + if (xf.original_num_cols < xf.user_num_cols) { throw std::invalid_argument("update_q: cached original column count is smaller than user n."); } + if (xf.barrier_lp == nullptr) { + throw std::invalid_argument("update_q: cached barrier LP is missing."); + } - std::vector orig(static_cast(fe.original_num_cols), 0.0); + std::vector orig(static_cast(xf.original_num_cols), 0.0); for (int j = 0; j < n; ++j) { orig[static_cast(j)] = c[j]; } - for (int j : fe.presolve_info.negated_variables) { + for (int j : xf.presolve_info.negated_variables) { orig[static_cast(j)] *= -1.0; } std::vector presolved; - if (!fe.presolve_info.remaining_variables.empty()) { - presolved.resize(fe.presolve_info.remaining_variables.size()); - for (std::size_t k = 0; k < fe.presolve_info.remaining_variables.size(); ++k) { - presolved[k] = orig[static_cast(fe.presolve_info.remaining_variables[k])]; + if (!xf.presolve_info.remaining_variables.empty()) { + presolved.resize(xf.presolve_info.remaining_variables.size()); + for (std::size_t k = 0; k < xf.presolve_info.remaining_variables.size(); ++k) { + presolved[k] = orig[static_cast(xf.presolve_info.remaining_variables[k])]; } } else { presolved = std::move(orig); } - auto const& pairs = fe.presolve_info.free_variable_pairs; + auto const& pairs = xf.presolve_info.free_variable_pairs; if (!pairs.empty()) { if (pairs.size() % 2 != 0) { throw std::invalid_argument("update_q: free_variable_pairs size is not even."); @@ -78,19 +85,19 @@ inline std::vector crush_user_linear_objective(barrier_front_end_cache_t std::size_t extra = pairs.size() / 2; presolved.resize(presolved.size() + extra); for (std::size_t k = 0; k < extra; ++k) { - int u = pairs[2 * k]; - int v = pairs[2 * k + 1]; + int u = pairs[2 * k]; + int v = pairs[2 * k + 1]; presolved[static_cast(v)] = -presolved[static_cast(u)]; } } - if (static_cast(presolved.size()) != fe.barrier_num_cols || - fe.column_scales.size() != presolved.size()) { + if (static_cast(presolved.size()) != xf.barrier_lp->num_cols || + xf.column_scales.size() != presolved.size()) { throw std::invalid_argument( "update_q: crushed objective size does not match barrier columns / column_scales."); } for (std::size_t j = 0; j < presolved.size(); ++j) { - presolved[j] /= fe.column_scales[j]; + presolved[j] /= xf.column_scales[j]; } return presolved; } diff --git a/cpp/src/pdlp/utilities/cython_solve.cu b/cpp/src/pdlp/utilities/cython_solve.cu index fb45bcd732..78c2c5c192 100644 --- a/cpp/src/pdlp/utilities/cython_solve.cu +++ b/cpp/src/pdlp/utilities/cython_solve.cu @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -41,19 +40,6 @@ namespace cuopt { namespace cython { -namespace { - -bool uses_barrier_session_path( - cuopt::mathematical_optimization::solver_settings_t& solver_settings, - cuopt::mathematical_optimization::io::data_model_view_t const& data_model) -{ - if (data_model.has_quadratic_objective() || data_model.has_quadratic_constraints()) { return true; } - return solver_settings.get_pdlp_settings().method == - cuopt::mathematical_optimization::method_t::Barrier; -} - -} // namespace - /** * @brief Wrapper for linear_programming to expose the API to cython * @@ -116,13 +102,10 @@ std::unique_ptr call_solve( cuopt::mathematical_optimization::solver_settings_t* solver_settings, unsigned int flags, bool is_batch_mode, - barrier_cache_t* session_in) + barrier_cache_t* cache_in) { raft::common::nvtx::range fun_scope("Call Solve"); - namespace cache_profile = cuopt::linear_programming::cache_profile; - if (cache_profile::enabled()) { cache_profile::reset(); } - cuopt_expects(data_model != nullptr, error_type_t::ValidationError, "call_solve: data_model is null."); @@ -137,13 +120,15 @@ std::unique_ptr call_solve( auto& pdlp_settings = solver_settings->get_pdlp_settings(); const bool sequence_solve = pdlp_settings.sequence_solve; - const bool barrier_path = uses_barrier_session_path(*solver_settings, *data_model); - const bool want_session = (session_in != nullptr || sequence_solve) && barrier_path && + const bool barrier_path = + data_model->has_quadratic_objective() || data_model->has_quadratic_constraints() || + pdlp_settings.method == cuopt::mathematical_optimization::method_t::Barrier; + const bool want_cache = (cache_in != nullptr || sequence_solve) && barrier_path && memory_backend == cuopt::mathematical_optimization::memory_backend_t::GPU && !is_batch_mode; - std::unique_ptr owned_session; - barrier_cache_t* active_session = session_in; + std::unique_ptr owned_cache; + barrier_cache_t* active_cache = cache_in; pdlp_settings.barrier_cache = nullptr; rmm::cuda_stream ephemeral_stream(static_cast(flags)); @@ -152,26 +137,13 @@ std::unique_ptr call_solve( // Create problem instance and CUDA resources based on memory backend if (memory_backend == cuopt::mathematical_optimization::memory_backend_t::GPU) { - if (want_session) { - if (active_session == nullptr) { - const auto handle_start = std::chrono::steady_clock::now(); - owned_session = barrier_cache_t::create(flags); - active_session = owned_session.get(); - if (cache_profile::enabled()) { - const double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - handle_start).count(); - cache_profile::add(cache_profile::cache_id::C01, elapsed); - } - } - solve_handle = active_session->handle_ptr(); - pdlp_settings.barrier_cache = active_session; - } else { - const auto handle_start = std::chrono::steady_clock::now(); - if (cache_profile::enabled()) { - const double elapsed = - std::chrono::duration(std::chrono::steady_clock::now() - handle_start).count(); - cache_profile::add(cache_profile::cache_id::C01, elapsed); + if (want_cache) { + if (active_cache == nullptr) { + owned_cache = barrier_cache_t::create(flags); + active_cache = owned_cache.get(); } + solve_handle = active_cache->handle_ptr(); + pdlp_settings.barrier_cache = active_cache; } auto problem = cuopt::mathematical_optimization::optimization_problem_t(solve_handle); @@ -205,7 +177,7 @@ std::unique_ptr call_solve( gpu_sols.last_restart_duality_gap_primal_solution_->set_stream(rmm::cuda_stream_per_thread); gpu_sols.last_restart_duality_gap_dual_solution_->set_stream(rmm::cuda_stream_per_thread); - if (owned_session) { response.lp_ret.barrier_cache = std::move(owned_session); } + if (owned_cache) { response.lp_ret.barrier_cache = std::move(owned_cache); } } else { // MIP solve @@ -265,8 +237,6 @@ std::unique_ptr call_solve( } } - if (cache_profile::enabled()) { cache_profile::log_summary(); } - pdlp_settings.barrier_cache = nullptr; return std::make_unique(std::move(response)); diff --git a/python/cuopt/cuopt/linear_programming/data_model/data_model.py b/python/cuopt/cuopt/linear_programming/data_model/data_model.py index fe9b1d42f4..b995815108 100644 --- a/python/cuopt/cuopt/linear_programming/data_model/data_model.py +++ b/python/cuopt/cuopt/linear_programming/data_model/data_model.py @@ -231,17 +231,18 @@ def set_objective_coefficients(self, c): @catch_cuopt_exception def update_q(self, c): """ - Update the linear objective coefficients (c) for a session re-solve. + Update the linear objective coefficients (c) for a sequence re-solve. - Writes user-space ``c`` onto this DataModel. If a Barrier session is + Writes user-space ``c`` onto this DataModel. If a barrier cache is present, also maps ``c`` into the cached barrier workspace and marks it dirty (quadratic ``Q``, ``A``, and bounds must stay unchanged). + Cache reuse is QP-only: quadratic constraints take a full solve. Parameters ---------- c : array-like of float64 Linear objective coefficients, length equal to the number of - variables on the first session solve. + variables on the first ``sequence_solve``. """ super().update_q(c) diff --git a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx index 2c063282e4..3484232d9d 100644 --- a/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx +++ b/python/cuopt/cuopt/linear_programming/data_model/data_model_wrapper.pyx @@ -78,11 +78,11 @@ cdef class DataModel: self.quadratic_constraints = [] def has_barrier_cache(self): - """Return whether this data model owns a reusable solver session.""" + """Return whether this data model owns a reusable solver cache.""" return self.barrier_cache_capsule is not None def clear_barrier_cache(self): - """Release this data model's reusable solver session and GPU cache.""" + """Release this data model's reusable barrier cache.""" self.barrier_cache_capsule = None def clear_quadratic_constraints(self): @@ -179,12 +179,12 @@ cdef class DataModel: """Update linear objective coefficients (user-space ``c``). Always writes the DataModel objective. If this model owns a solver - session from a prior Barrier solve, also crushes ``c`` into the cached - ``iteration_data_t`` and sets ``c_dirty`` so a later continue path can - skip convert/presolve. Session crush runs first so a length error + cache from a prior Barrier solve, also crushes ``c`` into the cached + ``iteration_data_t`` and sets ``c_dirty`` so a later reuse can + skip convert/presolve. Crush runs first so a length error leaves the DataModel coefficients unchanged. """ - cdef barrier_cache_t* session + cdef barrier_cache_t* cache cdef double[::1] c_view new_c = type_cast(c, np.float64, "c") if self.barrier_cache_capsule is not None: @@ -192,15 +192,15 @@ cdef class DataModel: self.barrier_cache_capsule, b"cuopt.barrier_cache" ): raise ValueError("Invalid barrier cache stored on DataModel.") - session = PyCapsule_GetPointer( + cache = PyCapsule_GetPointer( self.barrier_cache_capsule, b"cuopt.barrier_cache", ) c_view = np.ascontiguousarray(new_c, dtype=np.float64) if c_view.shape[0] == 0: - session.update_linear_objective(NULL, 0) + cache.update_linear_objective(NULL, 0) else: - session.update_linear_objective(&c_view[0], c_view.shape[0]) + cache.update_linear_objective(&c_view[0], c_view.shape[0]) self.c = new_c def set_objective_scaling_factor(self, objective_scaling_factor): diff --git a/python/cuopt/cuopt/linear_programming/solver/solver.pxd b/python/cuopt/cuopt/linear_programming/solver/solver.pxd index 09efdc2ae5..6fa11d874c 100644 --- a/python/cuopt/cuopt/linear_programming/solver/solver.pxd +++ b/python/cuopt/cuopt/linear_programming/solver/solver.pxd @@ -151,7 +151,7 @@ cdef extern from "cuopt/mathematical_optimization/utilities/cython_solve.hpp" na solver_settings_t[int, double]* solver_settings, unsigned int flags, bool is_batch_mode, - barrier_cache_t* session_in, + barrier_cache_t* cache_in, ) except + nogil cdef pair[vector[unique_ptr[solver_ret_t]], double] call_batch_solve( # noqa diff --git a/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx b/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx index ca33987108..af0702e302 100644 --- a/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx +++ b/python/cuopt/cuopt/linear_programming/solver/solver_wrapper.pyx @@ -480,7 +480,7 @@ def prepare_solver_settings(SolverSettings settings, data_model=None, mip=False) def Solve(py_data_model_obj, SolverSettings settings, mip=False): cdef DataModel data_model_obj = py_data_model_obj - cdef barrier_cache_t* session_in = NULL + cdef barrier_cache_t* cache_in = NULL cdef solver_ret_t* sol_ret if settings.sequence_solve and data_model_obj.barrier_cache_capsule is not None: @@ -489,7 +489,7 @@ def Solve(py_data_model_obj, SolverSettings settings, mip=False): b"cuopt.barrier_cache", ): raise ValueError("Invalid barrier cache stored on DataModel.") - session_in = PyCapsule_GetPointer( + cache_in = PyCapsule_GetPointer( data_model_obj.barrier_cache_capsule, b"cuopt.barrier_cache", ) @@ -510,7 +510,7 @@ def Solve(py_data_model_obj, SolverSettings settings, mip=False): settings.c_solver_settings.get(), cudaStreamNonBlocking, False, - session_in, + cache_in, )) sol_ret = sol_ret_ptr.get() From fa633cd7fa2a165ed860fadef973bf6f1c316cf1 Mon Sep 17 00:00:00 2001 From: Alice Boucher <160623740+aliceb-nv@users.noreply.github.com> Date: Tue, 25 Aug 2026 22:23:23 +0200 Subject: [PATCH 3/4] Fix device_scalar constructors after RMM change (#1795) This PR fixes build after the latest RMM merge broke our pipeline (https://github.com/rapidsai/rmm/commit/6646d15835adba33d70eba714ca4939bb8c8e872). device_scalar no longer accepts a r-value constructor. Replaced with common constants as inline constexpr that are passed instead of r-value constants. - [ ] I am familiar with the [Contributing Guidelines](https://github.com/NVIDIA/cuopt/blob/HEAD/CONTRIBUTING.md). - Testing - [ ] New or existing tests cover these changes - [ ] Added tests - [ ] Created an issue to follow-up - [ ] NA - Documentation - [ ] The documentation is up to date with these changes - [ ] Added new documentation - [ ] NA --- cpp/src/barrier/cusparse_view.cu | 7 +-- .../feasibility_jump/feasibility_jump.cu | 5 +- .../feasibility_jump/feasibility_jump.cuh | 50 +++++++++---------- .../mip_heuristics/feasibility_jump/utils.cuh | 5 +- .../local_search/rounding/simple_rounding.cu | 7 +-- .../problem/load_balanced_problem.cu | 5 +- cpp/src/mip_heuristics/problem/problem.cu | 3 +- .../problem/problem_helpers.cuh | 3 +- cpp/src/pdlp/cusparse_view.cu | 17 ++++--- .../distributed_algorithms.cu | 6 ++- .../optimal_batch_size_handler.cu | 5 +- cpp/src/pdlp/pdhg.cu | 10 ++-- cpp/src/pdlp/pdlp.cu | 15 +++--- .../restart_strategy/pdlp_restart_strategy.cu | 14 +++--- .../weighted_average_solution.cu | 5 +- .../adaptive_step_size_strategy.cu | 5 +- .../convergence_information.cu | 13 ++--- .../infeasibility_information.cu | 11 ++-- cpp/src/routing/ges/squeeze.cu | 9 ++-- .../local_search/cycle_finder/cycle.hpp | 5 +- .../cycle_finder/cycle_finder.hpp | 7 +-- cpp/src/utilities/device_scalar_init.hpp | 37 ++++++++++++++ 22 files changed, 152 insertions(+), 92 deletions(-) create mode 100644 cpp/src/utilities/device_scalar_init.hpp diff --git a/cpp/src/barrier/cusparse_view.cu b/cpp/src/barrier/cusparse_view.cu index 5e5a54139e..e67eb7bfe0 100644 --- a/cpp/src/barrier/cusparse_view.cu +++ b/cpp/src/barrier/cusparse_view.cu @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -171,9 +172,9 @@ cusparse_view_t::cusparse_view_t(raft::handle_t const* handle_ptr, A_T_data_(0, handle_ptr->get_stream()), spmv_buffer_(0, handle_ptr->get_stream()), spmv_buffer_transpose_(0, handle_ptr->get_stream()), - d_one_(f_t(1), handle_ptr->get_stream()), - d_minus_one_(f_t(-1), handle_ptr->get_stream()), - d_zero_(f_t(0), handle_ptr->get_stream()) + d_one_(one_v, handle_ptr->get_stream()), + d_minus_one_(neg_one_v, handle_ptr->get_stream()), + d_zero_(zero_v, handle_ptr->get_stream()) { RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu index 1853ecfcbd..4efd73e454 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -52,8 +53,8 @@ fj_t::fj_t(mip_solver_context_t& context_, fj_settings_t in_ cstr_right_weights(pb_ptr->n_constraints, pb_ptr->handle_ptr->get_stream()), cstr_left_weights(pb_ptr->n_constraints, pb_ptr->handle_ptr->get_stream()), weight_update_increment(1.0), - objective_weight(0.0, pb_ptr->handle_ptr->get_stream()), - max_cstr_weight(0, pb_ptr->handle_ptr->get_stream()), + objective_weight(zero_v, pb_ptr->handle_ptr->get_stream()), + max_cstr_weight(zero_v, pb_ptr->handle_ptr->get_stream()), climber_views(0, pb_ptr->handle_ptr->get_stream()), objective_vars(0, pb_ptr->handle_ptr->get_stream()), constraint_lower_bounds_csr(pb_ptr->coefficients.size(), pb_ptr->handle_ptr->get_stream()), diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh index 8d1f39ce22..507998536a 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -367,21 +368,20 @@ class fj_t { climber_data_t(fj_t& in_fj) : fj(in_fj), - selected_var(std::numeric_limits::max(), fj.handle_ptr->get_stream()), - violation_score(0, fj.handle_ptr->get_stream()), - weighted_violation_score(0, fj.handle_ptr->get_stream()), - constraints_changed_count(0, fj.handle_ptr->get_stream()), - local_minimums_reached(0, fj.handle_ptr->get_stream()), - iterations(0, fj.handle_ptr->get_stream()), - best_excess(-std::numeric_limits::infinity(), fj.handle_ptr->get_stream()), - best_objective(+std::numeric_limits::infinity(), fj.handle_ptr->get_stream()), - saved_solution_objective(+std::numeric_limits::infinity(), - fj.handle_ptr->get_stream()), - incumbent_quality(+std::numeric_limits::infinity(), fj.handle_ptr->get_stream()), - incumbent_objective(0.0, fj.handle_ptr->get_stream()), - iterations_until_feasible_counter(0, fj.handle_ptr->get_stream()), - full_refresh_iteration(0, fj.handle_ptr->get_stream()), - best_jump_idx(cub::KeyValuePair{}, fj.handle_ptr->get_stream()), + selected_var(max_v, fj.handle_ptr->get_stream()), + violation_score(zero_v, fj.handle_ptr->get_stream()), + weighted_violation_score(zero_v, fj.handle_ptr->get_stream()), + constraints_changed_count(zero_v, fj.handle_ptr->get_stream()), + local_minimums_reached(zero_v, fj.handle_ptr->get_stream()), + iterations(zero_v, fj.handle_ptr->get_stream()), + best_excess(neg_inf_v, fj.handle_ptr->get_stream()), + best_objective(inf_v, fj.handle_ptr->get_stream()), + saved_solution_objective(inf_v, fj.handle_ptr->get_stream()), + incumbent_quality(inf_v, fj.handle_ptr->get_stream()), + incumbent_objective(zero_v, fj.handle_ptr->get_stream()), + iterations_until_feasible_counter(zero_v, fj.handle_ptr->get_stream()), + full_refresh_iteration(zero_v, fj.handle_ptr->get_stream()), + best_jump_idx(zero_v>, fj.handle_ptr->get_stream()), violated_constraints(fj.pb_ptr->n_constraints, fj.handle_ptr->get_stream()), candidate_variables(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()), iteration_related_variables(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()), @@ -406,20 +406,20 @@ class fj_t { jump_candidate_count(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()), jump_locks(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()), fractional_variables(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()), - small_move_tabu(0, fj.handle_ptr->get_stream()), - handle_fractionals_only(false, fj.handle_ptr->get_stream()), - saved_best_fractional_count(0, fj.handle_ptr->get_stream()), + small_move_tabu(zero_v, fj.handle_ptr->get_stream()), + handle_fractionals_only(false_v, fj.handle_ptr->get_stream()), + saved_best_fractional_count(zero_v, fj.handle_ptr->get_stream()), candidate_arrived_workids(fj.pb_ptr->coefficients.size(), fj.handle_ptr->get_stream()), grid_score_buf(0, fj.handle_ptr->get_stream()), grid_var_buf(0, fj.handle_ptr->get_stream()), grid_delta_buf(0, fj.handle_ptr->get_stream()), - last_minimum_iteration(0, fj.handle_ptr->get_stream()), - last_improving_minimum(0, fj.handle_ptr->get_stream()), - last_iter_candidates(0, fj.handle_ptr->get_stream()), - relvar_count_last_update(0, fj.handle_ptr->get_stream()), - load_balancing_skip(0, fj.handle_ptr->get_stream()), - break_condition(0, fj.handle_ptr->get_stream()), - temp_break_condition(0, fj.handle_ptr->get_stream()), + last_minimum_iteration(zero_v, fj.handle_ptr->get_stream()), + last_improving_minimum(zero_v, fj.handle_ptr->get_stream()), + last_iter_candidates(zero_v, fj.handle_ptr->get_stream()), + relvar_count_last_update(zero_v, fj.handle_ptr->get_stream()), + load_balancing_skip(zero_v, fj.handle_ptr->get_stream()), + break_condition(zero_v, fj.handle_ptr->get_stream()), + temp_break_condition(zero_v, fj.handle_ptr->get_stream()), cub_storage_bytes(0, fj.handle_ptr->get_stream()), dot_product_buffer(fj.pb_ptr->n_variables, fj.handle_ptr->get_stream()) { diff --git a/cpp/src/mip_heuristics/feasibility_jump/utils.cuh b/cpp/src/mip_heuristics/feasibility_jump/utils.cuh index a24e4b6a7f..1b2862d558 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/utils.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/utils.cuh @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -100,8 +101,8 @@ struct bitmap_t { template struct contiguous_set_t { contiguous_set_t(i_t max_size, const rmm::cuda_stream_view& stream) - : set_size(0, stream), - lock(0, stream), + : set_size(zero_v, stream), + lock(zero_v, stream), contents(max_size, stream), index_map(max_size, stream), validity_bitmap(max_size, stream) diff --git a/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu b/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu index 404185fe26..2d5aae0b0d 100644 --- a/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu +++ b/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -42,7 +43,7 @@ bool check_brute_force_rounding(solution_t& solution) rmm::device_uvector var_map(n_integers_to_round, solution.handle_ptr->get_stream()); rmm::device_uvector constraint_buf(n_configs * solution.problem_ptr->n_constraints, solution.handle_ptr->get_stream()); - rmm::device_scalar best_config(-1, solution.handle_ptr->get_stream()); + rmm::device_scalar best_config(neg_one_v, solution.handle_ptr->get_stream()); thrust::copy_if( solution.handle_ptr->get_thrust_policy(), solution.problem_ptr->integer_indices.begin(), @@ -80,7 +81,7 @@ bool invoke_simple_rounding(solution_t& solution) solution_t sol_copy(*solution.problem_ptr); sol_copy.copy_from(solution); - rmm::device_scalar successful(true, solution.handle_ptr->get_stream()); + rmm::device_scalar successful(true_v, solution.handle_ptr->get_stream()); i_t TPB = 128; simple_rounding_kernel <<<2048, TPB, 0, solution.handle_ptr->get_stream()>>>(solution.view(), successful.data()); @@ -125,7 +126,7 @@ void invoke_random_round_nearest(solution_t& solution, i_t n_target_ra CUOPT_LOG_TRACE("before random roundin n_integers %d total n_integers %d", n_integers, solution.problem_ptr->n_integer_vars); - rmm::device_scalar n_randomly_rounded(0, solution.handle_ptr->get_stream()); + rmm::device_scalar n_randomly_rounded(zero_v, solution.handle_ptr->get_stream()); random_nearest_rounding_kernel<<get_stream()>>>( solution.view(), cuopt::seed_generator::get_seed(), n_randomly_rounded.data()); i_t h_n_random_rounds = n_randomly_rounded.value(solution.handle_ptr->get_stream()); diff --git a/cpp/src/mip_heuristics/problem/load_balanced_problem.cu b/cpp/src/mip_heuristics/problem/load_balanced_problem.cu index 3199750679..4911a15de8 100644 --- a/cpp/src/mip_heuristics/problem/load_balanced_problem.cu +++ b/cpp/src/mip_heuristics/problem/load_balanced_problem.cu @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -206,7 +207,7 @@ void create_constraint_graph(const raft::handle_t* handle_ptr, make_span(reorg_ids), make_span(offsets), make_span(coeff), make_span(edge), bounds, pb.view()); if (debug) { - rmm::device_scalar errors(0, handle_ptr->get_stream()); + rmm::device_scalar errors(zero_v, handle_ptr->get_stream()); check_constraint_data <<get_stream()>>>(make_span(reorg_ids), make_span(offsets), @@ -253,7 +254,7 @@ void create_variable_graph(const raft::handle_t* handle_ptr, pb.view()); if (debug) { - rmm::device_scalar errors(0, handle_ptr->get_stream()); + rmm::device_scalar errors(zero_v, handle_ptr->get_stream()); check_variable_data <<get_stream()>>>(make_span(reorg_ids), make_span(offsets), diff --git a/cpp/src/mip_heuristics/problem/problem.cu b/cpp/src/mip_heuristics/problem/problem.cu index ccba2d5f2b..746d47f4a6 100644 --- a/cpp/src/mip_heuristics/problem/problem.cu +++ b/cpp/src/mip_heuristics/problem/problem.cu @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -1541,7 +1542,7 @@ void problem_t::substitute_variables(const std::vector& var_indic fixing_helpers.variable_fix_mask.end(), -1); - rmm::device_scalar objective_offset(0., handle_ptr->get_stream()); + rmm::device_scalar objective_offset(zero_v, handle_ptr->get_stream()); constexpr f_t zero_value = f_t(0.); rmm::device_uvector objective_offset_delta_per_variable(d_var_indices.size(), handle_ptr->get_stream()); diff --git a/cpp/src/mip_heuristics/problem/problem_helpers.cuh b/cpp/src/mip_heuristics/problem/problem_helpers.cuh index c5a60b1bf7..388fae4ecd 100644 --- a/cpp/src/mip_heuristics/problem/problem_helpers.cuh +++ b/cpp/src/mip_heuristics/problem/problem_helpers.cuh @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -216,7 +217,7 @@ static bool check_transpose_validity(const rmm::device_uvector& coefficient { if (offsets.size() <= 1) { return true; } - rmm::device_scalar failed(false, handle_ptr->get_stream()); + rmm::device_scalar failed(false_v, handle_ptr->get_stream()); kernel_check_transpose_validity <<get_stream()>>>( raft::device_span(coefficients.data(), coefficients.size()), diff --git a/cpp/src/pdlp/cusparse_view.cu b/cpp/src/pdlp/cusparse_view.cu index 130ac8d532..9d3a0cc67c 100644 --- a/cpp/src/pdlp/cusparse_view.cu +++ b/cpp/src/pdlp/cusparse_view.cu @@ -6,6 +6,7 @@ /* clang-format on */ #include +#include #include #include @@ -608,8 +609,8 @@ cusparse_view_t::cusparse_view_t( _reflected_primal_solution.data()); } - const rmm::device_scalar alpha{1, handle_ptr->get_stream()}; - const rmm::device_scalar beta{0, handle_ptr->get_stream()}; + const rmm::device_scalar alpha{one_v, handle_ptr->get_stream()}; + const rmm::device_scalar beta{zero_v, handle_ptr->get_stream()}; size_t buffer_size_non_transpose = 0; RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmv_buffersize(handle_ptr_->get_cusparse_handle(), @@ -810,8 +811,8 @@ cusparse_view_t::cusparse_view_t( const_cast(A_T_indices_.data()), A_T_float_.data()); - const rmm::device_scalar alpha_d{1.0, handle_ptr->get_stream()}; - const rmm::device_scalar beta_d{0.0, handle_ptr->get_stream()}; + const rmm::device_scalar alpha_d{one_v, handle_ptr->get_stream()}; + const rmm::device_scalar beta_d{zero_v, handle_ptr->get_stream()}; size_t buffer_size_non_transpose_mixed = mixed_precision_spmv_buffersize(handle_ptr_->get_cusparse_handle(), @@ -974,8 +975,8 @@ cusparse_view_t::cusparse_view_t( CUSPARSE_ORDER_COL); } - const rmm::device_scalar alpha{1, handle_ptr->get_stream()}; - const rmm::device_scalar beta{1, handle_ptr->get_stream()}; + const rmm::device_scalar alpha{one_v, handle_ptr->get_stream()}; + const rmm::device_scalar beta{one_v, handle_ptr->get_stream()}; size_t buffer_size_non_transpose = 0; RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmv_buffersize(handle_ptr_->get_cusparse_handle(), @@ -1161,8 +1162,8 @@ cusparse_view_t::cusparse_view_t( primal_gradient.create(op_problem.n_variables, _primal_gradient); dual_gradient.create(op_problem.n_constraints, _dual_gradient); - const rmm::device_scalar alpha{1, handle_ptr->get_stream()}; - const rmm::device_scalar beta{1, handle_ptr->get_stream()}; + const rmm::device_scalar alpha{one_v, handle_ptr->get_stream()}; + const rmm::device_scalar beta{one_v, handle_ptr->get_stream()}; size_t buffer_size_non_transpose = 0; RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmv_buffersize(handle_ptr_->get_cusparse_handle(), diff --git a/cpp/src/pdlp/distributed_pdlp/distributed_algorithms.cu b/cpp/src/pdlp/distributed_pdlp/distributed_algorithms.cu index 5e5c23a272..09cd3cb836 100644 --- a/cpp/src/pdlp/distributed_pdlp/distributed_algorithms.cu +++ b/cpp/src/pdlp/distributed_pdlp/distributed_algorithms.cu @@ -12,6 +12,8 @@ #include +#include + #include #include @@ -62,8 +64,8 @@ void multi_gpu_engine_t::distributed_bound_objective_rescaling(f_t c_s for_each_shard([&](auto& s) { const auto& scaled = s.sub_pdlp->get_initial_scaling_strategy().get_scaled_op_problem(); const auto stream = s.stream.view(); - rmm::device_scalar d_bound_sq(f_t(0), stream); - rmm::device_scalar d_obj_sq(f_t(0), stream); + rmm::device_scalar d_bound_sq(zero_v, stream); + rmm::device_scalar d_obj_sq(zero_v, stream); compute_sum_bounds_squared(scaled.constraint_lower_bounds, scaled.constraint_upper_bounds, diff --git a/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu b/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu index 04991c652c..8c8cdce0b4 100644 --- a/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu +++ b/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -29,8 +30,8 @@ struct SpMM_benchmarks_context_t { y(static_cast(dual_size) * current_batch_size, handle_ptr->get_stream()), buffer_non_transpose_batch(0, handle_ptr->get_stream()), buffer_transpose_batch(0, handle_ptr->get_stream()), - alpha(1, handle_ptr->get_stream()), - beta(0, handle_ptr->get_stream()), + alpha(one_v, handle_ptr->get_stream()), + beta(zero_v, handle_ptr->get_stream()), A(A), A_T(A_T), handle_ptr(handle_ptr) diff --git a/cpp/src/pdlp/pdhg.cu b/cpp/src/pdlp/pdhg.cu index 69ac810455..81a5a2f0e5 100644 --- a/cpp/src/pdlp/pdhg.cu +++ b/cpp/src/pdlp/pdhg.cu @@ -19,6 +19,8 @@ #include +#include + #ifdef CUPDLP_DEBUG_MODE #include #endif @@ -91,16 +93,16 @@ pdhg_solver_t::pdhg_solver_t( climber_strategies, hyper_params, enable_mixed_precision_spmv}, - reusable_device_scalar_value_1_{1.0, stream_view_}, - reusable_device_scalar_value_0_{0.0, stream_view_}, - reusable_device_scalar_value_neg_1_{f_t(-1.0), stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, + reusable_device_scalar_value_neg_1_{neg_one_v, stream_view_}, reusable_device_scalar_1_{stream_view_}, // In both multi stream and SpMM PDLP CUDA Graphs are causing issue // Currently graph capture is not supported for cuSparse SpMM // TODO enable once cuSparse SpMM supports graph capture graph_all{stream_view_, is_legacy_batch_mode || batch_mode_}, graph_prim_proj_gradient_dual{stream_view_, is_legacy_batch_mode}, - d_total_pdhg_iterations_{0, stream_view_}, + d_total_pdhg_iterations_{zero_v, stream_view_}, climber_strategies_(climber_strategies), hyper_params_(hyper_params), new_bounds_climber_id_{new_bounds.size(), stream_view_}, diff --git a/cpp/src/pdlp/pdlp.cu b/cpp/src/pdlp/pdlp.cu index 62b50825e7..217ea4260a 100644 --- a/cpp/src/pdlp/pdlp.cu +++ b/cpp/src/pdlp/pdlp.cu @@ -23,6 +23,7 @@ #include "distributed_pdlp/multi_gpu_engine.hpp" #include +#include #include #include @@ -265,8 +266,8 @@ pdlp_solver_t::pdlp_solver_t(mip::problem_t& op_problem, climber_strategies_}, initial_primal_{0, stream_view_}, initial_dual_{0, stream_view_}, - reusable_device_scalar_value_1_{f_t(1.0), stream_view_}, - reusable_device_scalar_value_0_{f_t(0.0), stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, batch_solution_to_return_{pdlp_termination_status_t::TimeLimit, stream_view_}, best_primal_solution_so_far{pdlp_termination_status_t::TimeLimit, stream_view_}, inside_mip_{false} @@ -3311,7 +3312,7 @@ void pdlp_solver_t::compute_initial_step_size() if (!settings_.hyper_params.initial_step_size_max_singular_value) { // set stepsize relative to maximum absolute value of A - rmm::device_scalar abs_max_element{0.0, stream_view_}; + rmm::device_scalar abs_max_element{zero_v, stream_view_}; void* d_temp_storage = NULL; size_t temp_storage_bytes = 0; @@ -3354,8 +3355,8 @@ void pdlp_solver_t::compute_initial_step_size() rmm::device_scalar norm_q(stream_view_); rmm::device_scalar sigma_max_sq(stream_view_); rmm::device_scalar residual_norm(stream_view_); - rmm::device_scalar reusable_device_scalar_value_1_(1, stream_view_); - rmm::device_scalar reusable_device_scalar_value_0_(0, stream_view_); + rmm::device_scalar reusable_device_scalar_value_1_(one_v, stream_view_); + rmm::device_scalar reusable_device_scalar_value_0_(zero_v, stream_view_); cusparseDnVecDescr_t vecZ, vecQ, vecATQ; RAFT_CUSPARSE_TRY( @@ -3496,13 +3497,13 @@ void pdlp_solver_t::compute_initial_primal_weight() // Here we use the combined bounds of the op_problem_scaled which may or may not be scaled yet // based on pdlp config pdlp::combine_constraint_bounds(op_problem_scaled_, op_problem_scaled_.combined_bounds); - rmm::device_scalar c_vec_norm{0.0, stream_view_}; + rmm::device_scalar c_vec_norm{zero_v, stream_view_}; pdlp::my_l2_weighted_norm(op_problem_scaled_.objective_coefficients, settings_.hyper_params.initial_primal_weight_c_scaling, c_vec_norm, stream_view_); - rmm::device_scalar b_vec_norm{0.0, stream_view_}; + rmm::device_scalar b_vec_norm{zero_v, stream_view_}; if (settings_.hyper_params.initial_primal_weight_combined_bounds) { // => same as sqrt(dot(b,b)) pdlp::my_l2_weighted_norm(op_problem_scaled_.combined_bounds, diff --git a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu index 5d3258502a..954935b06e 100644 --- a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu +++ b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu @@ -17,6 +17,8 @@ #include +#include + #ifdef CUPDLP_DEBUG_MODE #include #endif @@ -88,8 +90,8 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( : static_cast(primal_size_h_ + dual_size_h_), stream_view_}, dual_norm_weight_{stream_view_}, - restart_triggered_{0, stream_view_}, - candidate_is_avg_{0, stream_view_}, + restart_triggered_{zero_v, stream_view_}, + candidate_is_avg_{zero_v, stream_view_}, avg_duality_gap_{handle_ptr_, hyper_params.never_restart_to_average ? 0 : primal_size, hyper_params.never_restart_to_average ? 0 : dual_size, @@ -189,10 +191,10 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( test_radius_squared_{stream_view_}, testing_range_low_{stream_view_}, testing_range_high_{stream_view_}, - reusable_device_scalar_value_1_{f_t(1.0), stream_view_}, - reusable_device_scalar_value_0_{f_t(0.0), stream_view_}, - reusable_device_scalar_value_0_i_t_{i_t(0), stream_view_}, - reusable_device_scalar_value_neg_1_{f_t(-1.0), stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, + reusable_device_scalar_value_0_i_t_{zero_v, stream_view_}, + reusable_device_scalar_value_neg_1_{neg_one_v, stream_view_}, dot_product_storage(0, stream_view_), dot_product_bytes{0}, tmp_kkt_score_{stream_view_}, diff --git a/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu b/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu index 9dc75e0620..50ad27334b 100644 --- a/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu +++ b/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -27,8 +28,8 @@ weighted_average_solution_t::weighted_average_solution_t(raft::handle_ dual_size_h_(dual_size), sum_primal_solutions_{static_cast(primal_size_h_), stream_view_}, sum_dual_solutions_{static_cast(dual_size_h_), stream_view_}, - sum_primal_solution_weights_{0.0, stream_view_}, - sum_dual_solution_weights_{0.0, stream_view_}, + sum_primal_solution_weights_{zero_v, stream_view_}, + sum_dual_solution_weights_{zero_v, stream_view_}, iterations_since_last_restart_{0}, graph(stream_view_, is_batch_mode) { diff --git a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu index a97ed13449..e52c329166 100644 --- a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu +++ b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu @@ -15,6 +15,7 @@ #include +#include #include #include @@ -56,8 +57,8 @@ adaptive_step_size_strategy_t::adaptive_step_size_strategy_t( interaction_{climber_strategies.size(), stream_view_}, norm_squared_delta_primal_{climber_strategies.size(), stream_view_}, norm_squared_delta_dual_{climber_strategies.size(), stream_view_}, - reusable_device_scalar_value_1_{f_t(1.0), stream_view_}, - reusable_device_scalar_value_0_{f_t(0.0), stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, dot_product_storage(0, stream_view_), graph(stream_view_, is_legacy_batch_mode), climber_strategies_(climber_strategies), diff --git a/cpp/src/pdlp/termination_strategy/convergence_information.cu b/cpp/src/pdlp/termination_strategy/convergence_information.cu index f72524d076..6dbe20ecc6 100644 --- a/cpp/src/pdlp/termination_strategy/convergence_information.cu +++ b/cpp/src/pdlp/termination_strategy/convergence_information.cu @@ -12,6 +12,7 @@ #include #include #include +#include #include @@ -62,12 +63,12 @@ convergence_information_t::convergence_information_t( objective_offsets_{climber_strategies.size(), stream_view_}, primal_objective_{climber_strategies.size(), stream_view_}, dual_objective_{climber_strategies.size(), stream_view_}, - reduced_cost_dual_objective_{f_t(0.0), stream_view_}, + reduced_cost_dual_objective_{zero_v, stream_view_}, l2_primal_residual_{climber_strategies.size(), stream_view_}, l2_dual_residual_{climber_strategies.size(), stream_view_}, linf_primal_residual_{climber_strategies.size(), stream_view_}, linf_dual_residual_{climber_strategies.size(), stream_view_}, - nb_violated_constraints_{0, stream_view_}, + nb_violated_constraints_{zero_v, stream_view_}, gap_{climber_strategies.size(), stream_view_}, abs_objective_{climber_strategies.size(), stream_view_}, primal_residual_{climber_strategies.size() * dual_size_h_, stream_view_}, @@ -78,9 +79,9 @@ convergence_information_t::convergence_information_t( ? static_cast(dual_size_h_ * climber_strategies.size()) : 0, stream_view_}, - reusable_device_scalar_value_1_{1.0, stream_view_}, - reusable_device_scalar_value_0_{0.0, stream_view_}, - reusable_device_scalar_value_neg_1_{-1.0, stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, + reusable_device_scalar_value_neg_1_{neg_one_v, stream_view_}, segmented_sum_handler_{stream_view_}, dual_dot_{climber_strategies.size(), stream_view_}, sum_primal_slack_{climber_strategies.size(), stream_view_}, @@ -236,7 +237,7 @@ void convergence_information_t::distributed_init_l2_norms( const auto& problem = *s.sub_pdlp->get_current_termination_strategy().get_convergence_information().problem_ptr; const auto stream = s.stream.view(); - rmm::device_scalar d_rhs_sq(f_t(0), stream); + rmm::device_scalar d_rhs_sq(zero_v, stream); compute_sum_bounds_squared(problem.constraint_lower_bounds, problem.constraint_upper_bounds, diff --git a/cpp/src/pdlp/termination_strategy/infeasibility_information.cu b/cpp/src/pdlp/termination_strategy/infeasibility_information.cu index 47c2148693..7e38ffa845 100644 --- a/cpp/src/pdlp/termination_strategy/infeasibility_information.cu +++ b/cpp/src/pdlp/termination_strategy/infeasibility_information.cu @@ -10,6 +10,7 @@ #include #include #include +#include #include @@ -65,8 +66,8 @@ infeasibility_information_t::infeasibility_information_t( dual_ray_inf_norm_(climber_strategies.size(), stream_view_), max_dual_ray_infeasibility_{climber_strategies.size(), stream_view_}, dual_ray_linear_objective_{climber_strategies.size(), stream_view_}, - reduced_cost_dual_objective_{0.0, stream_view_}, - reduced_cost_inf_norm_{0.0, stream_view_}, + reduced_cost_dual_objective_{zero_v, stream_view_}, + reduced_cost_inf_norm_{zero_v, stream_view_}, // If infeasibility_detection is off, no need to allocate all those homogenous_primal_residual_{(!infeasibility_detection) ? 0 : static_cast(dual_size_h_), stream_view_}, @@ -91,9 +92,9 @@ infeasibility_information_t::infeasibility_information_t( stream_view_}, sum_primal_slack_{climber_strategies.size(), stream_view_}, sum_dual_slack_{climber_strategies.size(), stream_view_}, - reusable_device_scalar_value_1_{1.0, stream_view_}, - reusable_device_scalar_value_0_{0.0, stream_view_}, - reusable_device_scalar_value_neg_1_{-1.0, stream_view_}, + reusable_device_scalar_value_1_{one_v, stream_view_}, + reusable_device_scalar_value_0_{zero_v, stream_view_}, + reusable_device_scalar_value_neg_1_{neg_one_v, stream_view_}, scaling_strategy_(scaling_strategy), segmented_sum_handler_(stream_view_), climber_strategies_(climber_strategies), diff --git a/cpp/src/routing/ges/squeeze.cu b/cpp/src/routing/ges/squeeze.cu index c81fa60199..5de35d153a 100644 --- a/cpp/src/routing/ges/squeeze.cu +++ b/cpp/src/routing/ges/squeeze.cu @@ -1,6 +1,6 @@ /* clang-format off */ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ /* clang-format on */ @@ -11,13 +11,14 @@ namespace cuopt { namespace routing { namespace detail { +static const cand_t empty_move{0, 0, std::numeric_limits::max()}; + template bool guided_ejection_search_t::repair_empty_routes() { int counter = 0; auto min_vehicles = solution_ptr->problem_ptr->data_view_ptr->get_min_vehicles(); - rmm::device_scalar best_move({0, 0, std::numeric_limits::max()}, - solution_ptr->sol_handle->get_stream()); + rmm::device_scalar best_move(empty_move, solution_ptr->sol_handle->get_stream()); // Try every request to non empty route auto const n_blocks = solution_ptr->get_num_requests() * solution_ptr->get_n_routes(); auto const n_empty_routes = solution_ptr->get_num_empty_vehicles(); @@ -288,7 +289,7 @@ void guided_ejection_search_t::squeeze( size_t sh_size = solution_ptr->check_routes_can_insert_and_get_sh_size() + sizeof(cand_t); const i_t TPB = std::min( 128, raft::alignTo(solution_ptr->get_max_active_nodes_for_all_routes(), raft::WarpSize)); - rmm::device_scalar best_move({0, 0, std::numeric_limits::max()}, stream); + rmm::device_scalar best_move(empty_move, stream); solution_ptr->d_lock.set_value_to_zero_async(stream); bool is_set = set_shmem_of_kernel(find_best_squeeze_pos, sh_size); diff --git a/cpp/src/routing/local_search/cycle_finder/cycle.hpp b/cpp/src/routing/local_search/cycle_finder/cycle.hpp index 7e3e275e13..e6a8aec57b 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle.hpp +++ b/cpp/src/routing/local_search/cycle_finder/cycle.hpp @@ -8,6 +8,7 @@ #pragma once #include +#include #include #include "../../solution/solution_handle.cuh" @@ -25,8 +26,8 @@ struct ret_cycles_t { ret_cycles_t(size_t max_size, rmm::cuda_stream_view stream_view) : paths(max_size, stream_view), offsets(max_size, stream_view), - n_cycles_(0, stream_view), - curr_iter_n_starts(0, stream_view) + n_cycles_(zero_v, stream_view), + curr_iter_n_starts(zero_v, stream_view) { } diff --git a/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp b/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp index fc9f7f1c1f..803f6a2393 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp +++ b/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp @@ -52,10 +52,11 @@ struct path_t { : key_ptr(max_routes, handle_ptr_->get_stream()), cost_ptr(max_routes, handle_ptr_->get_stream()), level_ptr(max_routes, handle_ptr_->get_stream()), - n_cycles(0, handle_ptr_->get_stream()), - all_mask(device_bitset_t{}, handle_ptr_->get_stream()), - all_found(false, handle_ptr_->get_stream()) + n_cycles(handle_ptr_->get_stream()), + all_mask(handle_ptr_->get_stream()), + all_found(handle_ptr_->get_stream()) { + reset(handle_ptr_->get_stream()); } void reset(rmm::cuda_stream_view stream) diff --git a/cpp/src/utilities/device_scalar_init.hpp b/cpp/src/utilities/device_scalar_init.hpp new file mode 100644 index 0000000000..2e269b2af0 --- /dev/null +++ b/cpp/src/utilities/device_scalar_init.hpp @@ -0,0 +1,37 @@ +/* clang-format off */ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +/* clang-format on */ + +#pragma once + +#include + +namespace cuopt { + +// inline constants to use as constructor arguments for rmm::device_scalar +// since the rvalue constructor is deleted + +template +inline constexpr T zero_v{}; +template +inline constexpr T one_v = T(1); +template +inline constexpr T neg_one_v = T(-1); +template +inline constexpr T inf_v = std::numeric_limits::infinity(); +template +inline constexpr T neg_inf_v = -std::numeric_limits::infinity(); +template +inline constexpr T max_v = std::numeric_limits::max(); +template +inline constexpr T min_v = std::numeric_limits::min(); +template +inline constexpr T lowest_v = std::numeric_limits::lowest(); + +inline constexpr bool true_v = true; +inline constexpr bool false_v = false; + +} // namespace cuopt From 62d595b2eb475bbb72b1667efe79741fa9ab272b Mon Sep 17 00:00:00 2001 From: root Date: Mon, 31 Aug 2026 17:45:02 +0000 Subject: [PATCH 4/4] Fix CUDA 13 settings_ pointer access in sparse_cholesky. The cache-reuse rebind left one destructor check as settings_. instead of settings_->, which only compiles on CU13 wheels. Signed-off-by: root --- cpp/src/barrier/sparse_cholesky.cuh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 22b96c4fa0..216e8a7ebe 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -379,7 +379,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { CUDA_CALL_AND_CHECK_EXIT(cudaStreamSynchronize(stream), "cudaStreamSynchronize"); #if CUDART_VERSION >= 13000 - if (settings_.concurrent_halt != nullptr && settings_->num_gpus == 1) { + if (settings_->concurrent_halt != nullptr && settings_->num_gpus == 1) { auto cuStreamDestroy_func = cuopt::get_driver_entry_point("cuStreamDestroy"); CU_CHECK(reinterpret_cast(cuStreamDestroy_func)(stream), reinterpret_cast(cuGetErrorString_func));