From adb4f6565ff190687656980637fb05bbba29df0e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 28 Aug 2026 13:08:49 -0500 Subject: [PATCH 1/2] Use cuda::stream_ref stream synchronization --- cpp/src/routing/utilities/cython.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/routing/utilities/cython.cu b/cpp/src/routing/utilities/cython.cu index 5a0b9bf6b2..8563c18671 100644 --- a/cpp/src/routing/utilities/cython.cu +++ b/cpp/src/routing/utilities/cython.cu @@ -125,7 +125,7 @@ std::vector> call_batch_solve( auto routing_solution = cuopt::routing::solve(*data_models[i], *settings); // Make sure current solve is finished - stream_pool.get_stream(i).synchronize(); + stream_pool.get_stream(i).sync(); // Create buffers and reassociate them with the original stream so they // outlive the local stream which will be destroyed at end of loop iteration From 4e37290599644d6776d21ada4a8cdb1023b49c6f Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 1 Sep 2026 05:54:50 -0500 Subject: [PATCH 2/2] Migrate stream view APIs to cuda::stream_ref --- .../cpu_pdlp_warm_start_data.hpp | 4 +- .../mip/solver_settings.hpp | 2 +- .../mip/solver_solution.hpp | 6 +- .../optimization_problem.hpp | 4 +- .../optimization_problem_solution.hpp | 26 +-- .../pdlp/pdlp_warm_start_data.hpp | 2 +- .../pdlp/solver_settings.hpp | 4 +- .../pdlp/solver_solution.hpp | 9 +- .../solver_settings.hpp | 6 +- .../utilities/segmented_sum_handler.cuh | 14 +- cpp/include/cuopt/routing/assignment.hpp | 6 +- .../distance_engine/waypoint_matrix.hpp | 2 +- cpp/src/barrier/barrier.cu | 141 ++++++------ cpp/src/barrier/barrier.hpp | 2 +- cpp/src/barrier/csr_kkt_build.cuh | 29 ++- cpp/src/barrier/cusparse_view.cu | 15 +- cpp/src/barrier/device_sparse_matrix.cuh | 106 +++++---- cpp/src/barrier/iterative_refinement.hpp | 2 +- cpp/src/barrier/second_order_cone_kernels.cuh | 216 +++++++++--------- .../barrier/second_order_cone_reduction.cuh | 20 +- cpp/src/barrier/sparse_cholesky.cuh | 30 +-- cpp/src/linear_algebra/sort_csr.cuh | 6 +- cpp/src/linear_algebra/vector_math.cuh | 14 +- cpp/src/math_optimization/solver_settings.cu | 6 +- .../diversity/assignment_hash_map.cu | 12 +- .../diversity/recombiners/recombiner.cuh | 10 +- .../feasibility_jump/feasibility_jump.cu | 31 ++- .../feasibility_jump/feasibility_jump.cuh | 18 +- .../feasibility_jump_kernels.cu | 134 ++++++----- .../feasibility_jump_kernels.cuh | 32 +-- .../mip_heuristics/feasibility_jump/utils.cuh | 16 +- .../local_search/rounding/bounds_repair.cu | 2 +- .../local_search/rounding/constraint_prop.cu | 6 +- .../local_search/rounding/simple_rounding.cu | 35 +-- .../mip_heuristics/mip_scaling_strategy.cu | 30 +-- .../mip_heuristics/mip_scaling_strategy.cuh | 2 +- cpp/src/mip_heuristics/presolve/block_bve.cu | 24 +- .../presolve/bounds_presolve.cu | 4 +- .../conditional_bound_strengthening.cu | 6 +- .../presolve/load_balanced_bounds_presolve.cu | 2 +- .../load_balanced_bounds_presolve.cuh | 4 +- .../load_balanced_bounds_presolve_helpers.cuh | 2 +- .../mip_heuristics/presolve/multi_probe.cu | 10 +- .../mip_heuristics/presolve/probing_cache.cu | 8 +- .../presolve/semi_continuous.cu | 6 +- .../presolve/semi_continuous.cuh | 2 +- .../presolve/third_party_presolve.cpp | 6 +- .../presolve/third_party_presolve.hpp | 2 +- .../mip_heuristics/problem/presolve_data.cu | 2 +- .../mip_heuristics/problem/presolve_data.cuh | 6 +- cpp/src/mip_heuristics/problem/problem.cu | 31 +-- cpp/src/mip_heuristics/problem/problem.cuh | 2 +- .../problem/problem_helpers.cuh | 8 +- .../mip_heuristics/relaxed_lp/lp_state.cuh | 4 +- .../solution/feasibility_test.cuh | 4 +- cpp/src/mip_heuristics/solution/solution.cu | 11 +- cpp/src/mip_heuristics/solve.cu | 7 +- cpp/src/mip_heuristics/solver.cu | 7 +- cpp/src/mip_heuristics/solver_settings.cu | 2 +- cpp/src/mip_heuristics/solver_solution.cu | 10 +- cpp/src/mip_heuristics/utils.cuh | 10 +- cpp/src/pdlp/cpu_pdlp_warm_start_data.cu | 20 +- cpp/src/pdlp/cuopt_c_internal.hpp | 8 +- cpp/src/pdlp/cusparse_view.cu | 79 ++++--- .../pdlp/distributed_pdlp/multi_gpu_engine.cu | 9 +- .../distributed_pdlp/multi_gpu_engine.hpp | 8 +- .../initial_scaling.cu | 65 +++--- .../initial_scaling.cuh | 2 +- .../optimal_batch_size_handler.cu | 18 +- cpp/src/pdlp/optimization_problem.cu | 31 ++- cpp/src/pdlp/pdhg.cu | 60 ++--- cpp/src/pdlp/pdhg.hpp | 2 +- cpp/src/pdlp/pdlp.cu | 140 ++++++------ cpp/src/pdlp/pdlp.cuh | 2 +- cpp/src/pdlp/pdlp_warm_start_data.cu | 2 +- .../localized_duality_gap_container.cu | 6 +- .../restart_strategy/pdlp_restart_strategy.cu | 123 +++++----- .../pdlp_restart_strategy.cuh | 2 +- .../weighted_average_solution.cu | 40 ++-- .../weighted_average_solution.hpp | 2 +- cpp/src/pdlp/saddle_point.cu | 14 +- cpp/src/pdlp/saddle_point.hpp | 2 +- cpp/src/pdlp/solve.cu | 25 +- cpp/src/pdlp/solver_settings.cu | 4 +- cpp/src/pdlp/solver_solution.cu | 22 +- .../adaptive_step_size_strategy.cu | 67 +++--- .../adaptive_step_size_strategy.hpp | 2 +- cpp/src/pdlp/swap_and_resize_helper.cuh | 2 +- .../convergence_information.cu | 112 ++++----- .../convergence_information.hpp | 2 +- .../infeasibility_information.cu | 58 ++--- .../infeasibility_information.hpp | 2 +- .../termination_strategy.cu | 22 +- .../termination_strategy.hpp | 2 +- cpp/src/pdlp/translate.hpp | 8 +- cpp/src/pdlp/utilities/ping_pong_graph.cu | 3 +- cpp/src/pdlp/utilities/ping_pong_graph.cuh | 4 +- cpp/src/pdlp/utils.cuh | 40 ++-- .../routing/adapters/assignment_adapter.cuh | 6 +- cpp/src/routing/assignment.cu | 13 +- cpp/src/routing/cpu_routing_problem.cu | 8 +- .../routing/crossovers/optimal_eax_cycles.cu | 24 +- cpp/src/routing/crossovers/ox_graph.hpp | 6 +- cpp/src/routing/crossovers/ox_recombiner.cuh | 18 +- cpp/src/routing/cuda_graph.cuh | 10 +- .../distance_engine/waypoint_matrix.cpp | 8 +- cpp/src/routing/fleet_info.cu | 8 +- cpp/src/routing/fleet_info.hpp | 9 +- cpp/src/routing/fleet_order_constraints.hpp | 2 +- cpp/src/routing/generator/generator.cu | 34 +-- .../routing/ges/compute_fragment_ejections.cu | 6 +- .../ges/compute_fragment_ejections.cuh | 2 +- cpp/src/routing/ges/eject_until_feasible.cu | 10 +- cpp/src/routing/ges/ejection_pool.cuh | 6 +- cpp/src/routing/ges/execute_insertion.cu | 16 +- cpp/src/routing/ges/guided_ejection_search.cu | 10 +- .../brute_force_lexico.cu | 20 +- .../lexicographic_search.cu | 27 +-- cpp/src/routing/ges/squeeze.cu | 83 +++---- .../routing/local_search/breaks_insertion.cu | 8 +- .../local_search/compute_compatible.cu | 60 ++--- .../local_search/compute_insertions.cu | 8 +- .../local_search/cycle_finder/cycle.hpp | 4 +- .../local_search/cycle_finder/cycle_finder.cu | 58 ++--- .../cycle_finder/cycle_finder.hpp | 7 +- .../local_search/cycle_finder/cycle_graph.hpp | 4 +- .../local_search/cycle_finder/device_map.cuh | 4 +- .../routing/local_search/fill_gpu_graph.cu | 6 +- .../local_search/hvrp/vehicle_assignment.cu | 19 +- .../local_search/hvrp/vehicle_assignment.cuh | 2 +- cpp/src/routing/local_search/perform_moves.cu | 15 +- .../routing/local_search/prize_collection.cu | 9 +- cpp/src/routing/local_search/random_cross.cu | 16 +- cpp/src/routing/local_search/sliding_tsp.cu | 16 +- .../routing/local_search/sliding_window.cu | 22 +- cpp/src/routing/local_search/two_opt.cu | 6 +- .../local_search/vrp/nodes_to_search.cu | 2 +- .../routing/local_search/vrp/vrp_execute.cu | 10 +- .../routing/local_search/vrp/vrp_search.cu | 4 +- cpp/src/routing/order_info.cu | 8 +- cpp/src/routing/order_info.hpp | 4 +- cpp/src/routing/route/break_route.cuh | 2 +- cpp/src/routing/route/capacity_route.cuh | 4 +- cpp/src/routing/route/distance_route.cuh | 2 +- cpp/src/routing/route/mismatch_route.cuh | 2 +- cpp/src/routing/route/pdp_route.cuh | 4 +- cpp/src/routing/route/prize_route.cuh | 2 +- cpp/src/routing/route/service_time_route.cuh | 2 +- cpp/src/routing/route/tasks_route.cuh | 2 +- cpp/src/routing/route/time_route.cuh | 2 +- cpp/src/routing/route/tsp_route.cuh | 2 +- .../route/vehicle_fixed_cost_route.cuh | 5 +- cpp/src/routing/solution/pool_allocator.cuh | 6 +- cpp/src/routing/solution/route_node_map.cuh | 6 +- cpp/src/routing/solution/solution.cu | 34 +-- cpp/src/routing/solution/solution_handle.cuh | 8 +- .../util_kernels/compute_backward_forward.cu | 4 +- .../routing/util_kernels/runtime_checks.cu | 8 +- .../routing/util_kernels/set_initial_nodes.cu | 12 +- cpp/src/routing/utilities/check_input.cu | 93 ++++---- cpp/src/routing/utilities/check_input.hpp | 22 +- cpp/src/routing/utilities/cython.cu | 2 +- cpp/src/routing/utilities/md_utils.hpp | 10 +- cpp/src/utilities/copy_helpers.hpp | 29 ++- cpp/src/utilities/event_handler.cuh | 12 +- cpp/src/utilities/manual_cuda_graph.cuh | 14 +- cpp/src/utilities/vector_helpers.cuh | 18 +- .../distance_engine/waypoint_matrix_test.cpp | 8 +- .../dual_simplex/unit_tests/solve_barrier.cu | 7 +- cpp/tests/linear_programming/pdlp_test.cu | 24 +- .../utilities/pdlp_test_utilities.cuh | 2 +- cpp/tests/mip/bounds_standardization_test.cu | 7 +- cpp/tests/mip/elim_var_remap_test.cu | 7 +- cpp/tests/mip/multi_probe_test.cu | 7 +- cpp/tests/routing/level0/l0_routing_test.cu | 14 +- .../routing/level0/l0_vehicle_order_match.cu | 2 +- cpp/tests/routing/routing_test.cuh | 2 +- cpp/tests/routing/unit_tests/top_k.cu | 27 +-- cpp/tests/socp/general_quadratic_test.cu | 7 +- cpp/tests/socp/solve_barrier_socp.cu | 7 +- cpp/tests/socp/sparse_augmented_kkt_test.cu | 3 +- 181 files changed, 1661 insertions(+), 1578 deletions(-) diff --git a/cpp/include/cuopt/mathematical_optimization/cpu_pdlp_warm_start_data.hpp b/cpp/include/cuopt/mathematical_optimization/cpu_pdlp_warm_start_data.hpp index 1a76da0fa3..117235b8eb 100644 --- a/cpp/include/cuopt/mathematical_optimization/cpu_pdlp_warm_start_data.hpp +++ b/cpp/include/cuopt/mathematical_optimization/cpu_pdlp_warm_start_data.hpp @@ -114,12 +114,12 @@ struct pdlp_warm_start_data_t; // Convert GPU → CPU warmstart (D2H copy) template cpu_pdlp_warm_start_data_t convert_to_cpu_warmstart( - const pdlp_warm_start_data_t& gpu_data, rmm::cuda_stream_view stream); + const pdlp_warm_start_data_t& gpu_data, cuda::stream_ref stream); // Convert CPU → GPU warmstart (H2D copy) template pdlp_warm_start_data_t convert_to_gpu_warmstart( - const cpu_pdlp_warm_start_data_t& cpu_data, rmm::cuda_stream_view stream); + const cpu_pdlp_warm_start_data_t& cpu_data, cuda::stream_ref stream); } // namespace CUOPT_EXPORT mathematical_optimization } // namespace cuopt diff --git a/cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp b/cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp index ed1080d9a3..a282559c90 100644 --- a/cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp +++ b/cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp @@ -90,7 +90,7 @@ class mip_solver_settings_t { */ void add_initial_solution(const f_t* initial_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); /** * @brief Get the callback for the user solution diff --git a/cpp/include/cuopt/mathematical_optimization/mip/solver_solution.hpp b/cpp/include/cuopt/mathematical_optimization/mip/solver_solution.hpp index 1ad58b9e10..3746fac530 100644 --- a/cpp/include/cuopt/mathematical_optimization/mip/solver_solution.hpp +++ b/cpp/include/cuopt/mathematical_optimization/mip/solver_solution.hpp @@ -52,8 +52,8 @@ class mip_solution_t : public base_solution_t { mip_solution_t(mip_termination_status_t termination_status, solver_stats_t stats, - rmm::cuda_stream_view stream_view); - mip_solution_t(const cuopt::logic_error& error_status, rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); + mip_solution_t(const cuopt::logic_error& error_status, cuda::stream_ref stream_view); bool is_mip() const override { return true; } const rmm::device_uvector& get_solution() const; @@ -76,7 +76,7 @@ class mip_solution_t : public base_solution_t { i_t get_num_simplex_iterations() const; const std::vector& get_variable_names() const; const std::vector>& get_solution_pool() const; - void write_to_sol_file(std::string_view filename, rmm::cuda_stream_view stream_view) const; + void write_to_sol_file(std::string_view filename, cuda::stream_ref stream_view) const; void log_detailed_summary() const; void log_summary() const; diff --git a/cpp/include/cuopt/mathematical_optimization/optimization_problem.hpp b/cpp/include/cuopt/mathematical_optimization/optimization_problem.hpp index bdfc2ffbd4..743c705746 100644 --- a/cpp/include/cuopt/mathematical_optimization/optimization_problem.hpp +++ b/cpp/include/cuopt/mathematical_optimization/optimization_problem.hpp @@ -350,7 +350,7 @@ class optimization_problem_t : public optimization_problem_interface_t * @tparam other_f_t Target floating-point type (e.g. float when this is double) */ template - optimization_problem_t convert_to_other_prec(rmm::cuda_stream_view stream) const; + optimization_problem_t convert_to_other_prec(cuda::stream_ref stream) const; /** * @brief Returns nullptr since this is already a GPU problem. @@ -388,7 +388,7 @@ class optimization_problem_t : public optimization_problem_interface_t private: raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; problem_category_t problem_category_ = problem_category_t::LP; bool maximize_{false}; diff --git a/cpp/include/cuopt/mathematical_optimization/optimization_problem_solution.hpp b/cpp/include/cuopt/mathematical_optimization/optimization_problem_solution.hpp index 577d5727ec..b3706473b3 100644 --- a/cpp/include/cuopt/mathematical_optimization/optimization_problem_solution.hpp +++ b/cpp/include/cuopt/mathematical_optimization/optimization_problem_solution.hpp @@ -65,7 +65,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { solution_.get_primal_solution().data(), solution_.get_primal_solution().size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -77,7 +77,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { solution_.get_dual_solution().data(), solution_.get_dual_solution().size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -88,7 +88,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { auto stream = reduced_cost.stream(); std::vector result(reduced_cost.size()); raft::copy(result.data(), reduced_cost.data(), reduced_cost.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -154,7 +154,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { ws.current_primal_solution_.data(), ws.current_primal_solution_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -167,7 +167,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { std::vector result(ws.current_dual_solution_.size()); raft::copy( result.data(), ws.current_dual_solution_.data(), ws.current_dual_solution_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -180,7 +180,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { std::vector result(ws.initial_primal_average_.size()); raft::copy( result.data(), ws.initial_primal_average_.data(), ws.initial_primal_average_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -193,7 +193,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { std::vector result(ws.initial_dual_average_.size()); raft::copy( result.data(), ws.initial_dual_average_.data(), ws.initial_dual_average_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -205,7 +205,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { auto stream = ws.current_ATY_.stream(); std::vector result(ws.current_ATY_.size()); raft::copy(result.data(), ws.current_ATY_.data(), ws.current_ATY_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -218,7 +218,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { std::vector result(ws.sum_primal_solutions_.size()); raft::copy( result.data(), ws.sum_primal_solutions_.data(), ws.sum_primal_solutions_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -230,7 +230,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { auto stream = ws.sum_dual_solutions_.stream(); std::vector result(ws.sum_dual_solutions_.size()); raft::copy(result.data(), ws.sum_dual_solutions_.data(), ws.sum_dual_solutions_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -245,7 +245,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { ws.last_restart_duality_gap_primal_solution_.data(), ws.last_restart_duality_gap_primal_solution_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -260,7 +260,7 @@ class gpu_lp_solution_t : public lp_solution_interface_t { ws.last_restart_duality_gap_dual_solution_.data(), ws.last_restart_duality_gap_dual_solution_.size(), stream); - stream.synchronize(); + stream.sync(); return result; } @@ -406,7 +406,7 @@ class gpu_mip_solution_t : public mip_solution_interface_t { std::vector result(solution_.get_solution().size()); raft::copy( result.data(), solution_.get_solution().data(), solution_.get_solution().size(), stream); - stream.synchronize(); + stream.sync(); return result; } diff --git a/cpp/include/cuopt/mathematical_optimization/pdlp/pdlp_warm_start_data.hpp b/cpp/include/cuopt/mathematical_optimization/pdlp/pdlp_warm_start_data.hpp index 52a800c3c2..548b2886a6 100644 --- a/cpp/include/cuopt/mathematical_optimization/pdlp/pdlp_warm_start_data.hpp +++ b/cpp/include/cuopt/mathematical_optimization/pdlp/pdlp_warm_start_data.hpp @@ -67,7 +67,7 @@ struct pdlp_warm_start_data_t { // Copy constructor using the view version for the cython_solver pdlp_warm_start_data_t(const pdlp_warm_start_data_view_t& other, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); // Copy constructor for when copying the solver_settings object in the PDLP object pdlp_warm_start_data_t(const pdlp_warm_start_data_t& other); diff --git a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp index bd07aaac32..fc7391be77 100644 --- a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp +++ b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp @@ -151,7 +151,7 @@ class pdlp_solver_settings_t { */ void set_initial_primal_solution(const f_t* initial_primal_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); /** * @brief Set an initial dual solution. @@ -165,7 +165,7 @@ class pdlp_solver_settings_t { */ void set_initial_dual_solution(const f_t* initial_dual_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); /** TODO batch mode: tmp * @brief Set an initial step size. diff --git a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_solution.hpp b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_solution.hpp index be48ea4baf..7020795f5d 100644 --- a/cpp/include/cuopt/mathematical_optimization/pdlp/solver_solution.hpp +++ b/cpp/include/cuopt/mathematical_optimization/pdlp/solver_solution.hpp @@ -105,7 +105,7 @@ class optimization_problem_solution_t : public base_solution_t { * @param[in] stream_view An rmm view to a stream. All computations will go through this stream */ optimization_problem_solution_t(pdlp_termination_status_t termination_status_, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); /** * @brief Construct an optimization problem solution that serves as PDLP solver output @@ -115,8 +115,7 @@ class optimization_problem_solution_t : public base_solution_t { * 'Optimal', 'PrimalInfeasible', 'DualInfeasible', 'TimeLimit' * @param[in] stream_view An rmm view to a stream. All computations will go through this stream */ - optimization_problem_solution_t(cuopt::logic_error error_status_, - rmm::cuda_stream_view stream_view); + optimization_problem_solution_t(cuopt::logic_error error_status_, cuda::stream_ref stream_view); /** * @brief Construct an optimization problem solution that serves as PDLP solver output * @@ -271,7 +270,7 @@ class optimization_problem_solution_t : public base_solution_t { * @param stream_view Non-owning stream view object */ void write_to_file(std::string_view filename, - rmm::cuda_stream_view stream_view, + cuda::stream_ref stream_view, bool generate_variable_values = true); /** @@ -281,7 +280,7 @@ class optimization_problem_solution_t : public base_solution_t { * @param filename Name of the output file * @param stream_view Non-owning stream view object */ - void write_to_sol_file(std::string_view filename, rmm::cuda_stream_view stream_view) const; + void write_to_sol_file(std::string_view filename, cuda::stream_ref stream_view) const; /** * @brief Copy solution from another solution object diff --git a/cpp/include/cuopt/mathematical_optimization/solver_settings.hpp b/cpp/include/cuopt/mathematical_optimization/solver_settings.hpp index 6b47805702..6cc910dfc6 100644 --- a/cpp/include/cuopt/mathematical_optimization/solver_settings.hpp +++ b/cpp/include/cuopt/mathematical_optimization/solver_settings.hpp @@ -52,10 +52,10 @@ class solver_settings_t { void set_initial_pdlp_primal_solution(const f_t* initial_primal_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); void set_initial_pdlp_dual_solution(const f_t* initial_dual_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); void set_pdlp_warm_start_data(const f_t* current_primal_solution, const f_t* current_dual_solution, const f_t* initial_primal_average, @@ -82,7 +82,7 @@ class solver_settings_t { // MIP Settings void add_initial_mip_solution(const f_t* initial_solution, i_t size, - rmm::cuda_stream_view stream = rmm::cuda_stream_default); + cuda::stream_ref stream = rmm::cuda_stream_default); void set_mip_callback(internals::base_solution_callback_t* callback = nullptr, void* user_data = nullptr); diff --git a/cpp/include/cuopt/mathematical_optimization/utilities/segmented_sum_handler.cuh b/cpp/include/cuopt/mathematical_optimization/utilities/segmented_sum_handler.cuh index dd4284dab3..4a54288245 100644 --- a/cpp/include/cuopt/mathematical_optimization/utilities/segmented_sum_handler.cuh +++ b/cpp/include/cuopt/mathematical_optimization/utilities/segmented_sum_handler.cuh @@ -13,7 +13,7 @@ namespace cuopt { template struct segmented_sum_handler_t { - segmented_sum_handler_t(rmm::cuda_stream_view stream_view) : stream_view_(stream_view) {} + segmented_sum_handler_t(cuda::stream_ref stream_view) : stream_view_(stream_view) {} template void segmented_sum_helper(InputIteratorT input, @@ -22,7 +22,7 @@ struct segmented_sum_handler_t { i_t problem_size) { cub::DeviceSegmentedReduce::Sum( - nullptr, byte_needed_, input, output, batch_size, problem_size, stream_view_); + nullptr, byte_needed_, input, output, batch_size, problem_size, stream_view_.get()); segmented_sum_storage_.resize(byte_needed_, stream_view_); @@ -32,7 +32,7 @@ struct segmented_sum_handler_t { output, batch_size, problem_size, - stream_view_); + stream_view_.get()); } template @@ -51,9 +51,9 @@ struct segmented_sum_handler_t { problem_size, reduction_op, initial_value, - stream_view_.value()); + stream_view_.get()); - segmented_sum_storage_.resize(byte_needed_, stream_view_.value()); + segmented_sum_storage_.resize(byte_needed_, stream_view_.get()); cub::DeviceSegmentedReduce::Reduce(segmented_sum_storage_.data(), byte_needed_, @@ -63,12 +63,12 @@ struct segmented_sum_handler_t { problem_size, reduction_op, initial_value, - stream_view_.value()); + stream_view_.get()); } size_t byte_needed_; rmm::device_buffer segmented_sum_storage_; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; }; } // namespace cuopt diff --git a/cpp/include/cuopt/routing/assignment.hpp b/cpp/include/cuopt/routing/assignment.hpp index b138382d6d..5ad334f36f 100644 --- a/cpp/include/cuopt/routing/assignment.hpp +++ b/cpp/include/cuopt/routing/assignment.hpp @@ -54,14 +54,14 @@ class assignment_t { * @param status Solution status. * @param stream_view Non-owning stream_view object. */ - assignment_t(solution_status_t status, rmm::cuda_stream_view stream_view); + assignment_t(solution_status_t status, cuda::stream_ref stream_view); /** * @brief Constructor. * * @param error_status Error status. * @param stream_view Non-owning stream_view object. */ - assignment_t(cuopt::logic_error error_status, rmm::cuda_stream_view stream_view); + assignment_t(cuopt::logic_error error_status, cuda::stream_ref stream_view); /** * @brief Constructor. * @@ -200,7 +200,7 @@ class assignment_t { * @param filename Name of the output file * @param stream_view Non-owning stream view object */ - void to_csv(std::string_view filename, rmm::cuda_stream_view stream_view); + void to_csv(std::string_view filename, cuda::stream_ref stream_view); /** * @brief Returns the final status as a human readable string diff --git a/cpp/include/cuopt/routing/distance_engine/waypoint_matrix.hpp b/cpp/include/cuopt/routing/distance_engine/waypoint_matrix.hpp index e8c04c941d..3d656ca52d 100644 --- a/cpp/include/cuopt/routing/distance_engine/waypoint_matrix.hpp +++ b/cpp/include/cuopt/routing/distance_engine/waypoint_matrix.hpp @@ -168,7 +168,7 @@ class waypoint_matrix_t { f_t const* weights, f_t& out_cost); raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_{}; + cuda::stream_ref stream_view_{}; i_t const* offsets_; i_t n_vertices_; i_t const* indices_; diff --git a/cpp/src/barrier/barrier.cu b/cpp/src/barrier/barrier.cu index dca523669c..b6ca6985db 100644 --- a/cpp/src/barrier/barrier.cu +++ b/cpp/src/barrier/barrier.cu @@ -136,35 +136,35 @@ bool should_use_adaptive_regularization(const simplex_solver_settings_t [[maybe_unused]] static void pairwise_multiply( - f_t* a, f_t* b, f_t* out, int size, rmm::cuda_stream_view stream) + f_t* a, f_t* b, f_t* out, int size, cuda::stream_ref stream) { cub::DeviceTransform::Transform( - cuda::std::make_tuple(a, b), out, size, cuda::std::multiplies<>{}, stream.value()); + cuda::std::make_tuple(a, b), out, size, cuda::std::multiplies<>{}, stream.get()); } // out[i] = is_direct_free_linear[i] ? 0 : a[i] * b[i] template [[maybe_unused]] static void pairwise_multiply_skip_direct_free_linear( - f_t* a, f_t* b, int* is_direct_free_linear, f_t* out, int size, rmm::cuda_stream_view stream) + f_t* a, f_t* b, int* is_direct_free_linear, f_t* out, int size, cuda::stream_ref stream) { cub::DeviceTransform::Transform( cuda::std::make_tuple(a, b, is_direct_free_linear), out, size, [] __host__ __device__(f_t x_j, f_t d_j, int free_j) { return free_j ? f_t{0} : x_j * d_j; }, - stream.value()); + stream.get()); } template [[maybe_unused]] static void axpy( - f_t alpha, f_t* x, f_t beta, f_t* y, f_t* out, int size, rmm::cuda_stream_view stream) + f_t alpha, f_t* x, f_t beta, f_t* y, f_t* out, int size, cuda::stream_ref stream) { cub::DeviceTransform::Transform( cuda::std::make_tuple(x, y), out, size, [alpha, beta] __host__ __device__(f_t a, f_t b) { return alpha * a + beta * b; }, - stream.value()); + stream.get()); } // Step size computation for nonnegative and free variables. Fuses two independent @@ -180,7 +180,7 @@ static f2_t max_nonnegative_step_length_pair_in_range( i_t len, const rmm::device_uvector& is_direct_free_linear, bool apply_direct_free_mask, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (len <= 0) { return f2_t{f_t(1), f_t(1)}; } @@ -211,7 +211,7 @@ static void recover_linear_orthant_dz(raft::device_span target, raft::device_span x, raft::device_span dz, raft::device_span is_direct_free_linear, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (dz.empty()) return; @@ -224,18 +224,18 @@ static void recover_linear_orthant_dz(raft::device_span target, if (is_direct_free) return f_t(0); return target_val - (z_val * dx_val) / x_val; }, - stream.value()); + stream.get()); RAFT_CHECK_CUDA(stream); } template static void negate_complementarity_rhs(raft::device_span out, raft::device_span residual, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (out.empty()) return; cub::DeviceTransform::Transform( - residual.data(), out.data(), out.size(), [] HD(f_t rhs) { return -rhs; }, stream.value()); + residual.data(), out.data(), out.size(), [] HD(f_t rhs) { return -rhs; }, stream.get()); } template @@ -244,7 +244,7 @@ static void fill_linear_cc_rhs(raft::device_span out, raft::device_span dz_aff, f_t new_mu, raft::device_span is_direct_free_linear, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (out.empty()) return; cub::DeviceTransform::Transform( @@ -254,7 +254,7 @@ static void fill_linear_cc_rhs(raft::device_span out, [new_mu] HD(f_t dx_aff_val, f_t dz_aff_val, i_t is_direct_free_linear) { return is_direct_free_linear ? f_t(0) : (-(dx_aff_val * dz_aff_val) + new_mu); }, - stream.value()); + stream.get()); RAFT_CHECK_CUDA(stream); } @@ -659,7 +659,7 @@ class iteration_data_t { d_inv_diag_prime.data(), d_num_flag.data(), inv_diag.size(), - stream_view_)); + stream_view_.get())); d_flag_buffer.resize(flag_buffer_size, stream_view_); } @@ -1103,7 +1103,7 @@ class iteration_data_t { d_inv_diag_prime.data(), d_num_flag.data(), d_inv_diag.size(), - stream_view_); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); } else { d_inv_diag_prime.resize(inv_diag.size(), stream_view_); @@ -1539,7 +1539,7 @@ class iteration_data_t { return chol->solve(d_b, d_x); } else { raft::copy(inv_diag.data(), d_inv_diag.data(), d_inv_diag.size(), stream_view_); - stream_view_.synchronize(); + stream_view_.sync(); dense_vector_t b = host_copy(d_b, stream_view_); dense_vector_t x = host_copy(d_x, stream_view_); @@ -1549,7 +1549,7 @@ class iteration_data_t { raft::copy(d_b.data(), b.data(), b.size(), stream_view_); d_x.resize(x.size(), stream_view_); raft::copy(d_x.data(), x.data(), x.size(), stream_view_); - stream_view_.synchronize(); // host x can go out of scope before copy finishes + stream_view_.sync(); // host x can go out of scope before copy finishes return out; } @@ -1917,7 +1917,7 @@ class iteration_data_t { u.data(), u.size(), cuda::std::multiplies<>{}, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); // y = alpha * A * w + beta * v = alpha * A * Dinv * A^T * y + beta * v @@ -1939,7 +1939,7 @@ class iteration_data_t { u.data(), u.size(), cuda::std::multiplies<>{}, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); cusparse_view_.spmv(alpha, u, beta, v); } @@ -2286,7 +2286,7 @@ class iteration_data_t { bool cone_combined_step_; f_t cone_sigma_mu_; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; const simplex_solver_settings_t& settings_; }; @@ -2542,7 +2542,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) // x = Dinv*(F*u - A'*q) // Fu <- -1.0 * A' * q + 1.0 * Fu data.cusparse_view_.transpose_spmv(-1.0, q, 1.0, Fu); - data.handle_ptr->get_stream().synchronize(); + data.handle_ptr->get_stream().sync(); // x <- Dinv * (F*u - A'*q) data.inv_diag.pairwise_product(Fu, data.x); @@ -2560,7 +2560,7 @@ int barrier_solver_t::initial_point(iteration_data_t& data) dense_vector_t init_primal_residual(lp.num_rows); init_primal_residual = lp.rhs; data.cusparse_view_.spmv(1.0, data.x, -1.0, init_primal_residual); - data.handle_ptr->get_stream().synchronize(); + data.handle_ptr->get_stream().sync(); #ifdef PRINT_INFO settings.log.printf("||b - A * x||: %.16e\n", vector_norm2(init_primal_residual)); #endif @@ -2748,7 +2748,7 @@ void barrier_solver_t::gpu_compute_residuals(const rmm::device_uvector data.d_bound_residual_.data(), data.d_upper_bounds_.size(), [] HD(f_t upper_j, f_t w_k, f_t x_j) { return upper_j - w_k - x_j; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); } @@ -2757,7 +2757,7 @@ void barrier_solver_t::gpu_compute_residuals(const rmm::device_uvector data.d_dual_residual_.data(), data.d_dual_residual_.size(), cuda::std::minus<>{}, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); auto descr_dual_residual = data.cusparse_view_.create_vector(data.d_dual_residual_); if (data.Q.n > 0) { data.cusparse_Q_view_.spmv(1.0, cusparse_d_x, 1.0, descr_dual_residual); } @@ -2773,7 +2773,7 @@ void barrier_solver_t::gpu_compute_residuals(const rmm::device_uvector thrust::make_permutation_iterator(data.d_dual_residual_.data(), data.d_upper_bounds_.data()), data.d_upper_bounds_.size(), [] HD(f_t dual_residual_j, f_t v_k) { return dual_residual_j + v_k; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); } @@ -2782,14 +2782,14 @@ void barrier_solver_t::gpu_compute_residuals(const rmm::device_uvector data.d_complementarity_xz_residual_.data(), data.d_complementarity_xz_residual_.size(), cuda::std::multiplies<>{}, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); // Compute complementarity_wv_residual = w.*v cub::DeviceTransform::Transform(cuda::std::make_tuple(d_w.data(), d_v.data()), data.d_complementarity_wv_residual_.data(), data.d_complementarity_wv_residual_.size(), cuda::std::multiplies<>{}, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); } @@ -2891,13 +2891,13 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t{}, - stream_view_.value()); + stream_view_.get()); } RAFT_CHECK_CUDA(stream_view_); @@ -2911,7 +2911,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t f_t(0)) return diag_j; return diag_j + free_var_reg; }, - stream_view_.value()); + stream_view_.get()); } else { cub::DeviceTransform::Transform( cuda::std::make_tuple(data.d_diag_.data(), data.d_is_direct_free_linear_.data()), @@ -2945,7 +2945,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t 0) { cub::DeviceTransform::Transform( cuda::std::make_tuple(data.d_bound_rhs_.data(), @@ -3034,7 +3034,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t(data.d_xz_residual_.data(), linear_size), @@ -3464,7 +3464,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t(d_dv_residual, stream_view_); max_residual = std::max(max_residual, dv_residual_norm); @@ -3524,7 +3524,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t(data.d_dual_residual_, stream_view_); @@ -3545,7 +3545,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t::gpu_compute_search_direction(iteration_data_t(data.d_dw_residual_, stream_view_); @@ -3585,7 +3585,7 @@ i_t barrier_solver_t::gpu_compute_search_direction(iteration_data_t(data.d_wv_residual_, stream_view_); @@ -3603,7 +3603,7 @@ void fill_linear_complementarity_target(iteration_data_t& data, raft::device_span target, raft::device_span xz_rhs, raft::device_span x, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (target.empty()) return; cub::DeviceTransform::Transform( @@ -3614,7 +3614,7 @@ void fill_linear_complementarity_target(iteration_data_t& data, if (is_direct_free_linear) return f_t(0); return complementarity_xz_rhs / x_val; }, - stream.value()); + stream.get()); RAFT_CHECK_CUDA(stream); } @@ -3622,7 +3622,7 @@ template void fill_affine_cone_complementarity_target(iteration_data_t& data, i_t cone_var_start, i_t m_c, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (m_c == 0) return; auto& cones = data.cones(); @@ -3631,7 +3631,7 @@ void fill_affine_cone_complementarity_target(iteration_data_t& data, auto cone_target = raft::device_span(data.d_complementarity_target_.data() + cone_var_start, m_c); cub::DeviceTransform::Transform( - cones.z.data(), cone_target.data(), m_c, [] HD(f_t z_val) { return -z_val; }, stream.value()); + cones.z.data(), cone_target.data(), m_c, [] HD(f_t z_val) { return -z_val; }, stream.get()); RAFT_CUDA_TRY(cudaPeekAtLastError()); RAFT_CHECK_CUDA(stream); } @@ -3641,7 +3641,7 @@ void fill_corrector_cone_complementarity_target(iteration_data_t& data i_t cone_var_start, i_t m_c, f_t sigma_mu, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (m_c == 0) return; auto& cones = data.cones(); @@ -3851,17 +3851,18 @@ void barrier_solver_t::compute_cc_rhs(iteration_data_t& data data.d_complementarity_wv_rhs_.data(), data.d_complementarity_wv_rhs_.size(), [new_mu] HD(f_t dw_aff, f_t dv_aff) { return -(dw_aff * dv_aff) + new_mu; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); // Zero the corrector RHS on device - RAFT_CUDA_TRY(cudaMemsetAsync(data.d_h_.data(), 0, sizeof(f_t) * data.d_h_.size(), stream_view_)); + RAFT_CUDA_TRY( + cudaMemsetAsync(data.d_h_.data(), 0, sizeof(f_t) * data.d_h_.size(), stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - data.d_dual_rhs_.data(), 0, sizeof(f_t) * data.d_dual_rhs_.size(), stream_view_)); + data.d_dual_rhs_.data(), 0, sizeof(f_t) * data.d_dual_rhs_.size(), stream_view_.get())); if (data.n_upper_bounds > 0) { RAFT_CUDA_TRY(cudaMemsetAsync( - data.d_bound_rhs_.data(), 0, sizeof(f_t) * data.d_bound_rhs_.size(), stream_view_)); + data.d_bound_rhs_.data(), 0, sizeof(f_t) * data.d_bound_rhs_.size(), stream_view_.get())); RAFT_CUDA_TRY( - cudaMemsetAsync(data.d_dw_.data(), 0, sizeof(f_t) * data.d_dw_.size(), stream_view_)); + cudaMemsetAsync(data.d_dw_.data(), 0, sizeof(f_t) * data.d_dw_.size(), stream_view_.get())); } data.cone_combined_step_ = has_soc; data.cone_sigma_mu_ = has_soc ? new_mu : f_t(0); @@ -3894,7 +3895,7 @@ void barrier_solver_t::compute_final_direction(iteration_data_t thrust::tuple { return {dw + dw_aff, dv + dv_aff}; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); cub::DeviceTransform::Transform( cuda::std::make_tuple( @@ -3904,14 +3905,14 @@ void barrier_solver_t::compute_final_direction(iteration_data_t thrust::tuple { return {dx + dx_aff, dz + dz_aff}; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); cub::DeviceTransform::Transform( cuda::std::make_tuple(data.d_dy_aff_.data(), data.d_dy_.data()), data.d_dy_.data(), data.d_dy_.size(), [] HD(f_t dy_aff, f_t dy) { return dy + dy_aff; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); } @@ -3968,7 +3969,7 @@ void barrier_solver_t::compute_next_iterate(iteration_data_t [step_primal, step_dual] HD(f_t w, f_t v, f_t dw, f_t dv) -> thrust::tuple { return {w + step_primal * dw, v + step_dual * dv}; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); cub::DeviceTransform::Transform( cuda::std::make_tuple(data.d_x_.data(), data.d_z_.data(), data.d_dx_.data(), data.d_dz_.data()), @@ -3977,7 +3978,7 @@ void barrier_solver_t::compute_next_iterate(iteration_data_t [step_primal, step_dual] HD(f_t x, f_t z, f_t dx, f_t dz) -> thrust::tuple { return {x + step_primal * dx, z + step_dual * dz}; }, - stream_view_.value()); + stream_view_.get()); RAFT_CHECK_CUDA(stream_view_); cub::DeviceTransform::Transform( cuda::std::make_tuple(data.d_y_.data(), data.d_dy_.data()), @@ -4191,7 +4192,7 @@ lp_status_t barrier_solver_t::check_for_suboptimal_solution( raft::copy(data.y.data(), data.d_y_.data(), data.d_y_.size(), stream_view_); raft::copy(data.z.data(), data.d_z_.data(), data.d_z_.size(), stream_view_); raft::copy(data.v.data(), data.d_v_.data(), data.d_v_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); data.to_solution(lp, iter, primal_objective, @@ -4582,7 +4583,7 @@ lp_status_t barrier_solver_t::solve(f_t start_time, lp_solution_t::solve(f_t start_time, lp_solution_t::solve(f_t start_time, lp_solution_t& lp; const simplex::simplex_solver_settings_t& settings; const simplex::presolve_info_t& presolve_info; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; }; } // namespace cuopt::mathematical_optimization::barrier diff --git a/cpp/src/barrier/csr_kkt_build.cuh b/cpp/src/barrier/csr_kkt_build.cuh index 4003c96fc6..8d2db0516b 100644 --- a/cpp/src/barrier/csr_kkt_build.cuh +++ b/cpp/src/barrier/csr_kkt_build.cuh @@ -27,7 +27,7 @@ constexpr int augmented_csr_block_size = 256; // Cone -> augmented-KKT-CSR assembly data. template struct cone_kkt_data_t { - explicit cone_kkt_data_t(rmm::cuda_stream_view stream) + explicit cone_kkt_data_t(cuda::stream_ref stream) : sparse_ids_by_cone(0, stream), dense_ids_by_cone(0, stream), dense_cone_entry_rank(0, stream), @@ -509,7 +509,7 @@ __global__ void fill_augmented_csr_row_kernel(i_t factorization_size, template void build_augmented_csr_metadata(const cone_data_t& cones, cone_kkt_data_t& metadata, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { raft::common::nvtx::range scope("Barrier: augmented: device CSR metadata"); const i_t n_cones = cones.n_cones; @@ -524,7 +524,7 @@ void build_augmented_csr_metadata(const cone_data_t& cones, i_t(-1)); if (n_sparse > 0) { const size_t grid = raft::ceildiv(n_sparse, augmented_csr_block_size); - scatter_sparse_ids_by_cone_kernel<<>>( + scatter_sparse_ids_by_cone_kernel<<>>( cuopt::make_span(metadata.sparse_ids_by_cone), cuopt::make_span(cones.sparse_cone_ids), n_sparse); @@ -548,14 +548,14 @@ void build_augmented_csr_metadata(const cone_data_t& cones, rmm::exec_policy(stream), is_dense_cone.begin(), is_dense_cone.end(), dense_prefix.begin()); const size_t grid = raft::ceildiv(n_cones, augmented_csr_block_size); - build_dense_ids_by_cone_kernel<<>>( + build_dense_ids_by_cone_kernel<<>>( cuopt::make_span(metadata.dense_ids_by_cone), cuopt::make_span(cones.cone_is_sparse), cuopt::make_span(dense_prefix), n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); - compact_dense_cone_ids_kernel<<>>( + compact_dense_cone_ids_kernel<<>>( cuopt::make_span(metadata.dense_cone_ids), cuopt::make_span(dense_prefix), cuopt::make_span(cones.cone_is_sparse), @@ -564,12 +564,11 @@ void build_augmented_csr_metadata(const cone_data_t& cones, rmm::device_uvector dense_block_sizes(n_dense, stream); const size_t dense_grid = raft::ceildiv(n_dense, augmented_csr_block_size); - build_dense_block_sizes_kernel - <<>>( - cuopt::make_span(dense_block_sizes), - cuopt::make_span(metadata.dense_cone_ids), - cuopt::make_span(cones.cone_offsets), - n_dense); + build_dense_block_sizes_kernel<<>>( + cuopt::make_span(dense_block_sizes), + cuopt::make_span(metadata.dense_cone_ids), + cuopt::make_span(cones.cone_offsets), + n_dense); RAFT_CUDA_TRY(cudaPeekAtLastError()); thrust::exclusive_scan(rmm::exec_policy(stream), @@ -612,7 +611,7 @@ void build_augmented_csr_metadata(const cone_data_t& cones, const size_t entry_grid = raft::ceildiv(m_c, augmented_csr_block_size); build_dense_cone_entry_rank_kernel - <<>>( + <<>>( cuopt::make_span(metadata.dense_cone_entry_rank), cuopt::make_span(cones.element_cone_ids), cuopt::make_span(cones.cone_is_sparse), @@ -639,7 +638,7 @@ i_t build_augmented_csr_on_device(i_t n, cone_kkt_data_t& cone_data, rmm::device_uvector& augmented_diagonal_indices, device_csr_matrix_t& device_augmented, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { const i_t factorization_size = n + m + p; const csc_view_t A_view = A.view(); @@ -650,7 +649,7 @@ i_t build_augmented_csr_on_device(i_t n, { raft::common::nvtx::range scope("Barrier: augmented: device CSR count"); const size_t grid = raft::ceildiv(factorization_size, augmented_csr_block_size); - count_augmented_row_nnz_kernel<<>>( + count_augmented_row_nnz_kernel<<>>( factorization_size, n, m, @@ -715,7 +714,7 @@ i_t build_augmented_csr_on_device(i_t n, raft::common::nvtx::range scope("Barrier: augmented: device CSR fill"); auto views = make_cone_kkt_views(cone_data, augmented_diagonal_indices); const size_t grid = raft::ceildiv(factorization_size, augmented_csr_block_size); - fill_augmented_csr_row_kernel<<>>( + fill_augmented_csr_row_kernel<<>>( factorization_size, n, m, diff --git a/cpp/src/barrier/cusparse_view.cu b/cpp/src/barrier/cusparse_view.cu index 477200c5e9..92ca242fe0 100644 --- a/cpp/src/barrier/cusparse_view.cu +++ b/cpp/src/barrier/cusparse_view.cu @@ -145,7 +145,7 @@ void cusparse_view_t::init_spmv_buffer_and_preprocess(cusparseSpMatDes y, spmv_alg, &buffer_size_spmv, - handle_ptr_->get_stream())); + handle_ptr_->get_stream().get())); buffer.resize(buffer_size_spmv, handle_ptr_->get_stream()); my_cusparsespmv_preprocess(handle_ptr_->get_cusparse_handle(), @@ -157,7 +157,7 @@ void cusparse_view_t::init_spmv_buffer_and_preprocess(cusparseSpMatDes y, spmv_alg, buffer.data(), - handle_ptr_->get_stream()); + handle_ptr_->get_stream().get()); } template @@ -177,9 +177,10 @@ cusparse_view_t::cusparse_view_t(raft::handle_t const* handle_ptr, d_zero_(zero_v, handle_ptr->get_stream()) { RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); // TMP matrix data should already be on the GPU constexpr bool debug = false; if (debug) { printf("A hash: %zu\n", A.hash()); } @@ -304,7 +305,7 @@ void cusparse_view_t::spmv(f_t alpha, y, get_spmv_alg(rows_), (f_t*)spmv_buffer_.data(), - handle_ptr_->get_stream()); + handle_ptr_->get_stream().get()); } template @@ -359,7 +360,7 @@ void cusparse_view_t::transpose_spmv(f_t alpha, y, get_spmv_alg(A_T_offsets_.size() - 1), (f_t*)spmv_buffer_transpose_.data(), - handle_ptr_->get_stream()); + handle_ptr_->get_stream().get()); } template class cusparse_view_t; diff --git a/cpp/src/barrier/device_sparse_matrix.cuh b/cpp/src/barrier/device_sparse_matrix.cuh index 974e2b0f4a..1d1c11a8ad 100644 --- a/cpp/src/barrier/device_sparse_matrix.cuh +++ b/cpp/src/barrier/device_sparse_matrix.cuh @@ -34,18 +34,18 @@ struct sum_reduce_helper_t { rmm::device_scalar out; size_t buffer_size; - sum_reduce_helper_t(rmm::cuda_stream_view stream_view) - : buffer_data(0, stream_view), out(stream_view) + sum_reduce_helper_t(cuda::stream_ref stream_view) : buffer_data(0, stream_view), out(stream_view) { } template - f_t sum(InputIteratorT input, i_t size, rmm::cuda_stream_view stream_view) + f_t sum(InputIteratorT input, i_t size, cuda::stream_ref stream_view) { buffer_size = 0; - cub::DeviceReduce::Sum(nullptr, buffer_size, input, out.data(), size, stream_view); + cub::DeviceReduce::Sum(nullptr, buffer_size, input, out.data(), size, stream_view.get()); buffer_data.resize(buffer_size, stream_view); - cub::DeviceReduce::Sum(buffer_data.data(), buffer_size, input, out.data(), size, stream_view); + cub::DeviceReduce::Sum( + buffer_data.data(), buffer_size, input, out.data(), size, stream_view.get()); return out.value(stream_view); } }; @@ -56,7 +56,7 @@ struct transform_reduce_helper_t { rmm::device_scalar out; size_t buffer_size; - transform_reduce_helper_t(rmm::cuda_stream_view stream_view) + transform_reduce_helper_t(cuda::stream_ref stream_view) : buffer_data(0, stream_view), out(stream_view) { } @@ -67,10 +67,17 @@ struct transform_reduce_helper_t { TransformOpT transform_op, f_t init, i_t size, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { - cub::DeviceReduce::TransformReduce( - nullptr, buffer_size, input, out.data(), size, reduce_op, transform_op, init, stream_view); + cub::DeviceReduce::TransformReduce(nullptr, + buffer_size, + input, + out.data(), + size, + reduce_op, + transform_op, + init, + stream_view.get()); buffer_data.resize(buffer_size, stream_view); @@ -82,7 +89,7 @@ struct transform_reduce_helper_t { reduce_op, transform_op, init, - stream_view); + stream_view.get()); return out.value(stream_view); } @@ -108,7 +115,7 @@ struct transform_reduce_pair_helper_t { rmm::device_scalar> out; size_t buffer_size; - transform_reduce_pair_helper_t(rmm::cuda_stream_view stream_view) + transform_reduce_pair_helper_t(cuda::stream_ref stream_view) : buffer_data(0, stream_view), out(stream_view) { } @@ -120,11 +127,18 @@ struct transform_reduce_pair_helper_t { TransformOpT transform_op, f2_t init, i_t size, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { f2_min_t reduce_op{}; - cub::DeviceReduce::TransformReduce( - nullptr, buffer_size, input, out.data(), size, reduce_op, transform_op, init, stream_view); + cub::DeviceReduce::TransformReduce(nullptr, + buffer_size, + input, + out.data(), + size, + reduce_op, + transform_op, + init, + stream_view.get()); buffer_data.resize(buffer_size, stream_view); @@ -136,7 +150,7 @@ struct transform_reduce_pair_helper_t { reduce_op, transform_op, init, - stream_view); + stream_view.get()); return out.value(stream_view); } @@ -152,12 +166,12 @@ struct csc_view_t { template class device_csc_matrix_t { public: - device_csc_matrix_t(rmm::cuda_stream_view stream) + device_csc_matrix_t(cuda::stream_ref stream) : col_start(0, stream), i(0, stream), x(0, stream), col_index(0, stream) { } - device_csc_matrix_t(i_t rows, i_t cols, i_t nz, rmm::cuda_stream_view stream) + device_csc_matrix_t(i_t rows, i_t cols, i_t nz, cuda::stream_ref stream) : m(rows), n(cols), nz_max(nz), @@ -179,7 +193,7 @@ class device_csc_matrix_t { { } - device_csc_matrix_t(const csc_matrix_t& A, rmm::cuda_stream_view stream) + device_csc_matrix_t(const csc_matrix_t& A, cuda::stream_ref stream) : m(A.m), n(A.n), nz_max(A.col_start[A.n]), @@ -193,7 +207,7 @@ class device_csc_matrix_t { x = cuopt::device_copy(A.x, stream); } - void resize_to_nnz(i_t nnz, rmm::cuda_stream_view stream) + void resize_to_nnz(i_t nnz, cuda::stream_ref stream) { col_start.resize(n + 1, stream); i.resize(nnz, stream); @@ -201,7 +215,7 @@ class device_csc_matrix_t { nz_max = nnz; } - csc_matrix_t to_host(rmm::cuda_stream_view stream) + csc_matrix_t to_host(cuda::stream_ref stream) { csc_matrix_t A(m, n, nz_max); A.col_start = cuopt::host_copy(col_start, stream); @@ -210,7 +224,7 @@ class device_csc_matrix_t { return A; } - void copy(const csc_matrix_t& A, rmm::cuda_stream_view stream) + void copy(const csc_matrix_t& A, cuda::stream_ref stream) { m = A.m; n = A.n; @@ -224,7 +238,7 @@ class device_csc_matrix_t { } /** Reset to an empty (all-zero col_start, no nonzeros) matrix of the given shape. */ - void reset_empty(i_t rows, i_t cols, rmm::cuda_stream_view stream) + void reset_empty(i_t rows, i_t cols, cuda::stream_ref stream) { m = rows; n = cols; @@ -235,12 +249,13 @@ class device_csc_matrix_t { /** Same semantics as csc_matrix_t::to_compressed_row, entirely on * device. */ - void to_compressed_row(device_csr_matrix_t& Arow, rmm::cuda_stream_view stream) const; + void to_compressed_row(device_csr_matrix_t& Arow, cuda::stream_ref stream) const; - void form_col_index(rmm::cuda_stream_view stream) + void form_col_index(cuda::stream_ref stream) { col_index.resize(x.size(), stream); - RAFT_CUDA_TRY(cudaMemsetAsync(col_index.data(), 0, sizeof(i_t) * col_index.size(), stream)); + RAFT_CUDA_TRY( + cudaMemsetAsync(col_index.data(), 0, sizeof(i_t) * col_index.size(), stream.get())); // Scatter 1 when there is a col start in col_index if (col_start.size() > 2) { @@ -259,17 +274,21 @@ class device_csc_matrix_t { // Inclusive cumulative sum to have the corresponding column for each entry rmm::device_buffer d_temp_storage; size_t temp_storage_bytes{0}; - cub::DeviceScan::InclusiveSum( - nullptr, temp_storage_bytes, col_index.data(), col_index.data(), col_index.size(), stream); + cub::DeviceScan::InclusiveSum(nullptr, + temp_storage_bytes, + col_index.data(), + col_index.data(), + col_index.size(), + stream.get()); d_temp_storage.resize(temp_storage_bytes, stream); cub::DeviceScan::InclusiveSum(d_temp_storage.data(), temp_storage_bytes, col_index.data(), col_index.data(), col_index.size(), - stream); + stream.get()); // Have to sync since InclusiveSum is being run on local data (d_temp_storage) - stream.synchronize(); + stream.sync(); } csc_view_t view() @@ -293,12 +312,9 @@ class device_csc_matrix_t { template class device_csr_matrix_t { public: - device_csr_matrix_t(rmm::cuda_stream_view stream) - : row_start(0, stream), j(0, stream), x(0, stream) - { - } + device_csr_matrix_t(cuda::stream_ref stream) : row_start(0, stream), j(0, stream), x(0, stream) {} - device_csr_matrix_t(i_t rows, i_t cols, i_t nz, rmm::cuda_stream_view stream) + device_csr_matrix_t(i_t rows, i_t cols, i_t nz, cuda::stream_ref stream) : m(rows), n(cols), nz_max(nz), @@ -318,7 +334,7 @@ class device_csr_matrix_t { { } - device_csr_matrix_t(const csr_matrix_t& A, rmm::cuda_stream_view stream) + device_csr_matrix_t(const csr_matrix_t& A, cuda::stream_ref stream) : m(A.m), n(A.n), nz_max(A.row_start[A.m]), @@ -331,7 +347,7 @@ class device_csr_matrix_t { x = cuopt::device_copy(A.x, stream); } - void resize_to_nnz(i_t nnz, rmm::cuda_stream_view stream) + void resize_to_nnz(i_t nnz, cuda::stream_ref stream) { row_start.resize(m + 1, stream); j.resize(nnz, stream); @@ -339,7 +355,7 @@ class device_csr_matrix_t { nz_max = nnz; } - csr_matrix_t to_host(rmm::cuda_stream_view stream) + csr_matrix_t to_host(cuda::stream_ref stream) { csr_matrix_t A(m, n, nz_max); A.row_start = cuopt::host_copy(row_start, stream); @@ -348,7 +364,7 @@ class device_csr_matrix_t { return A; } - void copy(csr_matrix_t& A, rmm::cuda_stream_view stream) + void copy(csr_matrix_t& A, cuda::stream_ref stream) { m = A.m; n = A.n; @@ -374,7 +390,7 @@ class device_csr_matrix_t { template void device_csc_matrix_t::to_compressed_row(device_csr_matrix_t& Arow, - rmm::cuda_stream_view stream) const + cuda::stream_ref stream) const { static_assert(std::is_signed_v); @@ -394,13 +410,13 @@ void device_csc_matrix_t::to_compressed_row(device_csr_matrix_t row_counts(m, stream); - RAFT_CUDA_TRY(cudaMemsetAsync(row_counts.data(), 0, sizeof(i_t) * m, stream)); + RAFT_CUDA_TRY(cudaMemsetAsync(row_counts.data(), 0, sizeof(i_t) * m, stream.get())); thrust::for_each(exec, thrust::make_counting_iterator(0), @@ -413,13 +429,13 @@ void device_csc_matrix_t::to_compressed_row(device_csr_matrix_tget_stream()); - RAFT_CUDA_TRY(cudaStreamSynchronize(op.data_.handle_ptr->get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(op.data_.handle_ptr->get_stream().get())); return err; } diff --git a/cpp/src/barrier/second_order_cone_kernels.cuh b/cpp/src/barrier/second_order_cone_kernels.cuh index b605b04192..f493ea6801 100644 --- a/cpp/src/barrier/second_order_cone_kernels.cuh +++ b/cpp/src/barrier/second_order_cone_kernels.cuh @@ -96,10 +96,7 @@ struct cone_scratch_t { // TODO: Consider moving this out to the barrier layer when we wire it in rmm::device_uvector temp_cone; // [n_cone_entries] - cone_scratch_t(i_t n_cones_in, - size_t n_cone_entries_in, - size_t n_large, - rmm::cuda_stream_view stream) + cone_scratch_t(i_t n_cones_in, size_t n_cone_entries_in, size_t n_large, cuda::stream_ref stream) : n_cones(n_cones_in), n_cone_entries(n_cone_entries_in), slots(0, stream), @@ -193,7 +190,7 @@ struct cone_data_t { cone_data_t(std::span cone_dimensions_host, raft::device_span x_in, raft::device_span z_in, - rmm::cuda_stream_view stream, + cuda::stream_ref stream, i_t soc_threshold_in = 100) : n_cones(cone_dimensions_host.size()), n_cone_entries( @@ -446,7 +443,7 @@ __global__ void __launch_bounds__(soc_block_size) * 1: ||z_tail||^2 -> z_scale */ template -void launch_nt_scaling(cone_data_t& cones, rmm::cuda_stream_view stream) +void launch_nt_scaling(cone_data_t& cones, cuda::stream_ref stream) { auto x_scale = cones.scratch.template get_slot<0>(); auto z_scale = cones.scratch.template get_slot<1>(); @@ -475,15 +472,14 @@ void launch_nt_scaling(cone_data_t& cones, rmm::cuda_stream_view strea const size_t cone_grid_dim = raft::ceildiv(static_cast(cones.n_cones), soc_block_size); - nt_finalize_scaling_scalars_kernel - <<>>( - cones.x, cones.z, x_scale, z_scale, cuopt::make_span(cones.eta), cone_offsets, cones.n_cones); + nt_finalize_scaling_scalars_kernel<<>>( + cones.x, cones.z, x_scale, z_scale, cuopt::make_span(cones.eta), cone_offsets, cones.n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); const size_t element_grid_dim = raft::ceildiv(cones.n_cone_entries, soc_block_size); auto w = cuopt::make_span(cones.w); - nt_write_w_kernel<<>>( + nt_write_w_kernel<<>>( cones.x, cones.z, x_scale, z_scale, w, cone_offsets, element_cone_ids); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -495,24 +491,24 @@ void launch_nt_scaling(cone_data_t& cones, rmm::cuda_stream_view strea }); cones.segmented_sum(unnormalized_tail_sq_terms, w_scale, stream); - nt_finalize_w_scale_kernel<<>>( + nt_finalize_w_scale_kernel<<>>( w, w_scale, w_scale, cone_offsets, cones.n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); nt_normalize_w_kernel - <<>>(w, w_scale, element_cone_ids); + <<>>(w, w_scale, element_cone_ids); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Persist lambda while w_scale still stores sqrt(det_J(w_tmp)). nt_write_lambda_kernel - <<>>(cones.x, - cones.z, - x_scale, - z_scale, - w_scale, - cuopt::make_span(cones.lambda), - cone_offsets, - element_cone_ids); + <<>>(cones.x, + cones.z, + x_scale, + z_scale, + w_scale, + cuopt::make_span(cones.lambda), + cone_offsets, + element_cone_ids); RAFT_CUDA_TRY(cudaPeekAtLastError()); // w_scale is overwritten from here @@ -524,7 +520,7 @@ void launch_nt_scaling(cone_data_t& cones, rmm::cuda_stream_view strea }); cones.segmented_sum(normalized_tail_terms, w_scale, stream); - nt_finalize_head_kernel<<>>( + nt_finalize_head_kernel<<>>( cuopt::make_span(cones.w), w_scale, cone_offsets, cones.n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -598,22 +594,22 @@ __global__ void update_scaling_sparse_kernel(raft::device_span w, * iteration. Call after `launch_nt_scaling` has updated w and eta. */ template -void launch_update_scaling_sparse(cone_data_t& cones, rmm::cuda_stream_view stream) +void launch_update_scaling_sparse(cone_data_t& cones, cuda::stream_ref stream) { if (!cones.has_sparse_cones()) { return; } const i_t n_sparse = cones.n_sparse_cones; update_scaling_sparse_kernel - <<>>(cuopt::make_span(cones.w), - cuopt::make_span(cones.eta), - cuopt::make_span(cones.d), - cuopt::make_span(cones.sparse_v), - cuopt::make_span(cones.sparse_u), - cuopt::make_span(cones.cone_offsets), - cuopt::make_span(cones.sparse_cone_dims), - cuopt::make_span(cones.sparse_cone_ids), - cuopt::make_span(cones.sparse_entry_offsets), - n_sparse); + <<>>(cuopt::make_span(cones.w), + cuopt::make_span(cones.eta), + cuopt::make_span(cones.d), + cuopt::make_span(cones.sparse_v), + cuopt::make_span(cones.sparse_u), + cuopt::make_span(cones.cone_offsets), + cuopt::make_span(cones.sparse_cone_dims), + cuopt::make_span(cones.sparse_cone_ids), + cuopt::make_span(cones.sparse_entry_offsets), + n_sparse); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -833,7 +829,7 @@ template void apply_w_inv(raft::device_span v, raft::device_span out, cone_data_t& cones, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto w = cuopt::make_span(cones.w); auto eta = cuopt::make_span(cones.eta); @@ -850,7 +846,7 @@ void apply_w_inv(raft::device_span v, cones.segmented_sum(tail_terms, tail_dot, stream); const size_t grid_dim = raft::ceildiv(out.size(), soc_block_size); - apply_w_inv_write_kernel<<>>( + apply_w_inv_write_kernel<<>>( v, out, w, eta, tail_dot, cone_offsets, element_cone_ids); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -869,7 +865,7 @@ template void apply_w(raft::device_span v, raft::device_span out, cone_data_t& cones, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto w = cuopt::make_span(cones.w); auto eta = cuopt::make_span(cones.eta); @@ -886,7 +882,7 @@ void apply_w(raft::device_span v, cones.segmented_sum(tail_terms, tail_dot, stream); const size_t grid_dim = raft::ceildiv(out.size(), soc_block_size); - apply_w_write_kernel<<>>( + apply_w_write_kernel<<>>( v, out, w, eta, tail_dot, cone_offsets, element_cone_ids); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -902,7 +898,7 @@ template void apply_hessian(raft::device_span v, raft::device_span out, cone_data_t& cones, - rmm::cuda_stream_view stream, + cuda::stream_ref stream, f_t output_scale = 1, raft::device_span bias = {}, f_t bias_scale = 0, @@ -921,18 +917,18 @@ void apply_hessian(raft::device_span v, const size_t grid_dim = raft::ceildiv(out.size(), soc_block_size); apply_hessian_kernel - <<>>(v, - out, - w, - eta, - wv_dot, - cone_offsets, - element_cone_ids, - cuopt::make_span(cones.cone_is_sparse), - dense_cones_only, - bias, - output_scale, - bias_scale); + <<>>(v, + out, + w, + eta, + wv_dot, + cone_offsets, + element_cone_ids, + cuopt::make_span(cones.cone_is_sparse), + dense_cones_only, + bias, + output_scale, + bias_scale); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -948,7 +944,7 @@ void recover_cone_dz_from_target(raft::device_span dx, cone_data_t& cones, raft::device_span cone_target, raft::device_span dz, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { apply_hessian(dx, dz, cones, stream, -1, cone_target, 1); } @@ -961,7 +957,7 @@ template void launch_dense_hessian_matvec(raft::device_span x, cone_data_t& cones, raft::device_span out, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto out_input = raft::device_span(out.data(), out.size()); apply_hessian(x, out, cones, stream, 1, out_input, 1, true); @@ -1053,7 +1049,7 @@ void scatter_sparse_hessian_into_augmented(cone_data_t& cones, const rmm::device_uvector& exp_v_row, const rmm::device_uvector& exp_u_row, const rmm::device_uvector& sparse_expansion_D, - rmm::cuda_stream_view stream, + cuda::stream_ref stream, f_t dual_perturb) { if (!cones.has_sparse_cones()) { return; } @@ -1062,24 +1058,23 @@ void scatter_sparse_hessian_into_augmented(cone_data_t& cones, const size_t E = cones.n_sparse_cone_entries; const size_t entry_grid = raft::ceildiv(E, soc_block_size); scatter_sparse_hessian_into_augmented_kernel - <<>>( - cuopt::make_span(augmented_x), - cuopt::make_span(Hs_diag), - cuopt::make_span(cones.eta), - cuopt::make_span(cones.d), - cuopt::make_span(cones.sparse_cone_ids), - cuopt::make_span(cones.sparse_entry_offsets), - n_sparse, - cuopt::make_span(hessian_diag_csr_indices), - cuopt::make_span(q_values), - cuopt::make_span(cones.sparse_v), - cuopt::make_span(cones.sparse_u), - cuopt::make_span(exp_v_col), - cuopt::make_span(exp_u_col), - cuopt::make_span(exp_v_row), - cuopt::make_span(exp_u_row), - cuopt::make_span(sparse_expansion_D), - dual_perturb); + <<>>(cuopt::make_span(augmented_x), + cuopt::make_span(Hs_diag), + cuopt::make_span(cones.eta), + cuopt::make_span(cones.d), + cuopt::make_span(cones.sparse_cone_ids), + cuopt::make_span(cones.sparse_entry_offsets), + n_sparse, + cuopt::make_span(hessian_diag_csr_indices), + cuopt::make_span(q_values), + cuopt::make_span(cones.sparse_v), + cuopt::make_span(cones.sparse_u), + cuopt::make_span(exp_v_col), + cuopt::make_span(exp_u_col), + cuopt::make_span(exp_v_row), + cuopt::make_span(exp_u_row), + cuopt::make_span(sparse_expansion_D), + dual_perturb); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -1171,7 +1166,7 @@ void launch_sparse_augmented_matvec(raft::device_span x, i_t cone_var_start, i_t n_primal, i_t m_constraints, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (!cones.has_sparse_cones()) { return; } @@ -1181,21 +1176,21 @@ void launch_sparse_augmented_matvec(raft::device_span x, "expansion output size mismatch"); sparse_augmented_matvec_kernel - <<>>(x, - r1, - y_exp, - Hs_diag, - cuopt::make_span(cones.sparse_v), - cuopt::make_span(cones.sparse_u), - cuopt::make_span(cones.eta), - cuopt::make_span(cones.sparse_cone_ids), - cuopt::make_span(cones.sparse_cone_dims), - cuopt::make_span(cones.sparse_entry_offsets), - cuopt::make_span(cones.cone_offsets), - cone_var_start, - n_primal, - m_constraints, - n_sparse); + <<>>(x, + r1, + y_exp, + Hs_diag, + cuopt::make_span(cones.sparse_v), + cuopt::make_span(cones.sparse_u), + cuopt::make_span(cones.eta), + cuopt::make_span(cones.sparse_cone_ids), + cuopt::make_span(cones.sparse_cone_dims), + cuopt::make_span(cones.sparse_entry_offsets), + cuopt::make_span(cones.cone_offsets), + cone_var_start, + n_primal, + m_constraints, + n_sparse); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -1251,7 +1246,7 @@ void scatter_dense_hessian_into_augmented(const cone_data_t& cones, const rmm::device_uvector& q_values, const rmm::device_uvector& dense_block_offsets, const rmm::device_uvector& dense_cone_ids, - rmm::cuda_stream_view stream, + cuda::stream_ref stream, f_t dual_perturb_value) { const size_t count = csr_indices.size(); @@ -1261,16 +1256,16 @@ void scatter_dense_hessian_into_augmented(const cone_data_t& cones, const i_t n_dense = cones.n_dense_cones(); const size_t grid = raft::ceildiv(count, soc_block_size); scatter_dense_hessian_into_augmented_kernel - <<>>(cuopt::make_span(augmented_x), - cuopt::make_span(csr_indices), - cuopt::make_span(q_values), - cuopt::make_span(cones.w), - cuopt::make_span(cones.eta), - cuopt::make_span(cones.cone_offsets), - cuopt::make_span(dense_block_offsets), - cuopt::make_span(dense_cone_ids), - n_dense, - dual_perturb_value); + <<>>(cuopt::make_span(augmented_x), + cuopt::make_span(csr_indices), + cuopt::make_span(q_values), + cuopt::make_span(cones.w), + cuopt::make_span(cones.eta), + cuopt::make_span(cones.cone_offsets), + cuopt::make_span(dense_block_offsets), + cuopt::make_span(dense_cone_ids), + n_dense, + dual_perturb_value); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -1460,14 +1455,14 @@ void launch_cone_step_length(segmented_sum_t& partitions, raft::device_span alpha, raft::device_span> large_sums, f_t alpha_max, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { constexpr int warps_per_cta = 8; if (!partitions.small_cone_ids.is_empty()) { const auto n_small = partitions.small_cone_ids.size(); const auto grid = (n_small + warps_per_cta - 1) / warps_per_cta; step_length_small_kernel - <<>>( + <<>>( u, du, alpha, @@ -1480,7 +1475,7 @@ void launch_cone_step_length(segmented_sum_t& partitions, if (!partitions.medium_cone_ids.is_empty()) { constexpr int medium_block_dim = 256; step_length_medium_kernel - <<>>( + <<>>( u, du, alpha, @@ -1512,14 +1507,14 @@ void launch_cone_step_length(segmented_sum_t& partitions, input, large_sums.data() + i, dim, - stream.value())); + stream.get())); } raft::device_span> large_sums_c(large_sums.data(), large_sums.size()); constexpr int large_solve_block_dim = 256; const auto grid = raft::ceildiv(n_large, static_cast(large_solve_block_dim)); - step_length_large_solve_kernel<<>>( + step_length_large_solve_kernel<<>>( u, du, alpha, @@ -1541,7 +1536,7 @@ f_t compute_cone_step_length(cone_data_t& cones, raft::device_span dx, raft::device_span dz, f_t alpha_max, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto alpha_primal = cuopt::make_span(cones.scratch.step_alpha_primal); auto alpha_dual = cuopt::make_span(cones.scratch.step_alpha_dual); @@ -1584,7 +1579,7 @@ void compute_combined_cone_rhs_term(raft::device_span dx_aff, cone_data_t& cones, f_t sigma_mu, raft::device_span out, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto cone_offsets = cuopt::make_span(cones.cone_offsets); auto element_cone_ids = cuopt::make_span(cones.element_cone_ids); @@ -1608,16 +1603,15 @@ void compute_combined_cone_rhs_term(raft::device_span dx_aff, // Stage both head vectors first because every tail entry needs them. const size_t cone_grid_dim = raft::ceildiv(static_cast(cones.n_cones), soc_block_size); - gather_cone_heads_kernel<<>>( + gather_cone_heads_kernel<<>>( scaled_dx, slot_1, cone_offsets, cones.n_cones); - gather_cone_heads_kernel<<>>( + gather_cone_heads_kernel<<>>( scaled_dz, slot_2, cone_offsets, cones.n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); const size_t element_grid_dim = raft::ceildiv(cones.n_cone_entries, soc_block_size); - combined_cone_shift_write_kernel - <<>>( - out, scaled_dx, scaled_dz, slot_0, slot_1, slot_2, cone_offsets, element_cone_ids, sigma_mu); + combined_cone_shift_write_kernel<<>>( + out, scaled_dx, scaled_dz, slot_0, slot_1, slot_2, cone_offsets, element_cone_ids, sigma_mu); RAFT_CUDA_TRY(cudaPeekAtLastError()); auto shift = raft::device_span(out.data(), out.size()); @@ -1641,13 +1635,13 @@ void compute_combined_cone_rhs_term(raft::device_span dx_aff, cones.segmented_sum(lambda_tail_sq_terms, slot_1, stream); jordan_divide_by_lambda_scalar_kernel - <<>>( + <<>>( shift, nt_point, slot_0, slot_1, slot_0, slot_1, cone_offsets, cones.n_cones); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Note that we implicitly multiply by -1 here since we are writing -p. jordan_divide_by_lambda_write_kernel - <<>>( + <<>>( shift, nt_point, slot_0, slot_1, cone_offsets, element_cone_ids, scratch_cone); RAFT_CUDA_TRY(cudaPeekAtLastError()); diff --git a/cpp/src/barrier/second_order_cone_reduction.cuh b/cpp/src/barrier/second_order_cone_reduction.cuh index bed06572a9..b576e4ccf9 100644 --- a/cpp/src/barrier/second_order_cone_reduction.cuh +++ b/cpp/src/barrier/second_order_cone_reduction.cuh @@ -83,7 +83,7 @@ struct segmented_sum_t { private: template - void prepare_workspace_for_type(rmm::cuda_stream_view stream) + void prepare_workspace_for_type(cuda::stream_ref stream) { auto input = thrust::make_constant_iterator(value_t{}); auto output = thrust::make_discard_iterator(); @@ -95,7 +95,7 @@ struct segmented_sum_t { input + large_cone_offsets[i], output + large_cone_ids[i], large_cone_dimensions[i], - stream.value())); + stream.get())); cub_workspace_bytes = std::max(cub_workspace_bytes, temp_storage_bytes); } @@ -106,14 +106,14 @@ struct segmented_sum_t { public: template - void prepare_workspace(rmm::cuda_stream_view stream) + void prepare_workspace(cuda::stream_ref stream) { prepare_workspace_for_type(stream); (prepare_workspace_for_type(stream), ...); } template - void operator()(InputIt input, OutputIt output, value_t init, rmm::cuda_stream_view stream) + void operator()(InputIt input, OutputIt output, value_t init, cuda::stream_ref stream) { if (!small_cone_ids.is_empty()) { // Each warp reduces one small cone. `warps_per_cta` only controls how @@ -122,7 +122,7 @@ struct segmented_sum_t { const auto n_small = small_cone_ids.size(); const auto grid = (n_small + warps_per_cta - 1) / warps_per_cta; warp_per_cone_reduce_kernel - <<>>( + <<>>( input, cuopt::make_span(small_cone_ids), cone_offsets, output, init); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -131,7 +131,7 @@ struct segmented_sum_t { constexpr int medium_block_dim = 256; const auto n_medium = medium_cone_ids.size(); block_per_cone_reduce_kernel - <<>>( + <<>>( input, cuopt::make_span(medium_cone_ids), cone_offsets, output, init); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -147,20 +147,20 @@ struct segmented_sum_t { input + large_cone_offsets[i], output + large_cone_ids[i], large_cone_dimensions[i], - stream.value())); + stream.get())); } } } template - void operator()(InputIt input, raft::device_span output, rmm::cuda_stream_view stream) + void operator()(InputIt input, raft::device_span output, cuda::stream_ref stream) { operator()(input, output.data(), f_t{0}, stream); } segmented_sum_t(std::span cone_dimensions_host, raft::device_span cone_offsets_in, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) : cone_offsets(cone_offsets_in), small_cone_ids(0, stream), medium_cone_ids(0, stream), @@ -199,7 +199,7 @@ struct segmented_sum_t { cuopt::device_copy(large_cone_ids_device, large_cone_ids, stream); need_sync = true; } - if (need_sync) { stream.synchronize(); } + if (need_sync) { stream.sync(); } } }; diff --git a/cpp/src/barrier/sparse_cholesky.cuh b/cpp/src/barrier/sparse_cholesky.cuh index 01045847d1..2cf1ab1dec 100644 --- a/cpp/src/barrier/sparse_cholesky.cuh +++ b/cpp/src/barrier/sparse_cholesky.cuh @@ -144,7 +144,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { positive_definite(true), A_created(false), settings_(settings), - stream(handle_ptr->get_stream()) + stream(handle_ptr->get_stream().get()) { int major, minor, patch; cudssGetProperty(MAJOR_VERSION, &major); @@ -221,7 +221,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { // 4. Create the green context and stream for that green // context CUstream barrier_green_ctx_stream; i_t stream_priority; - cudaStream_t cuda_stream = handle_ptr_->get_stream(); + cudaStream_t cuda_stream = handle_ptr_->get_stream().get(); cudaError_t priority_result = cudaStreamGetPriority(cuda_stream, &stream_priority); RAFT_CUDA_TRY(priority_result); auto cuGreenCtxCreate_func = cuopt::get_driver_entry_point("cuGreenCtxCreate"); @@ -347,7 +347,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { status, "cudssMatrixCreateDn for x"); #endif - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); } ~sparse_cholesky_cudss_t() override @@ -381,7 +381,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { CU_CHECK( reinterpret_cast(cuGreenCtxDestroy_func)(barrier_green_ctx), reinterpret_cast(cuGetErrorString_func)); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); } #endif } @@ -522,7 +522,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { // TODO: Is there any way to get nonzeros in the factors? // TODO: Is there any way to get flops for the factorization? RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); return 0; } @@ -582,7 +582,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { status, "cudssDataGet for info"); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); if (info != 0) { settings_.log.printf("Factorization failed info %d\n", info); @@ -717,7 +717,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { settings_.log.printf("Symbolic factorization time : %.2fs\n", symbolic_time); if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); return CONCURRENT_HALT_RETURN; } int64_t lu_nz = 0; @@ -728,7 +728,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { "cudssDataGet for LU_NNZ"); settings_.log.printf("Symbolic nonzeros in factor : %.2e\n", static_cast(lu_nz) / 2.0); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); // TODO: Is there any way to get nonzeros in the factors? // TODO: Is there any way to get flops for the factorization? @@ -753,7 +753,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { "cudaMemcpy for csr_values"); CUDA_CALL_AND_CHECK(cudaStreamSynchronize(stream), "cudaStreamSynchronize"); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); CUDSS_CALL_AND_CHECK( cudssMatrixSetValues(A, csr_values_d), status, "cudssMatrixSetValues for A"); @@ -777,7 +777,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { status, "cudssDataGet for info"); RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); if (info != 0) { settings_.log.printf("Factorization failed info %d\n", info); return -1; @@ -798,13 +798,13 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { { auto d_b = cuopt::device_copy(b, handle_ptr_->get_stream()); auto d_x = cuopt::device_copy(x, handle_ptr_->get_stream()); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); i_t out = solve(d_b, d_x); raft::copy(x.data(), d_x.data(), d_x.size(), handle_ptr_->get_stream()); // Sync so that data is on the host - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); for (i_t i = 0; i < n; i++) { if (x[i] != x[i]) { return -1; } @@ -815,7 +815,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { i_t solve(rmm::device_uvector& b, rmm::device_uvector& x) override { - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); if (static_cast(b.size()) != n) { settings_.log.printf("Error: b.size() %d != n %d\n", b.size(), n); return -1; @@ -843,7 +843,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { } CUDA_CALL_AND_CHECK(cudaStreamSynchronize(stream), "cudaStreamSynchronize"); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); #ifdef PRINT_RHS_AND_SOLUTION_HASH dense_vector_t b_host(n); @@ -874,7 +874,7 @@ class sparse_cholesky_cudss_t : public sparse_cholesky_base_t { bool positive_definite; cudaError_t cuda_error; cudssStatus_t status; - // rmm::cuda_stream_view stream; + // cuda::stream_ref stream; cudssHandle_t handle; cudssDeviceMemHandler_t mem_handler; cudssConfig_t solverConfig; diff --git a/cpp/src/linear_algebra/sort_csr.cuh b/cpp/src/linear_algebra/sort_csr.cuh index 23b9fd2d57..2cada2e1bd 100644 --- a/cpp/src/linear_algebra/sort_csr.cuh +++ b/cpp/src/linear_algebra/sort_csr.cuh @@ -37,7 +37,7 @@ void sort_csr(optimization_problem_t& op_problem) num_segments, op_problem.get_constraint_matrix_offsets().data(), op_problem.get_constraint_matrix_offsets().data() + 1, - stream_view); + stream_view.get()); d_tmp_storage_bytes.resize(tmp_storage_bytes, stream_view); cub::DeviceSegmentedSort::SortPairs(d_tmp_storage_bytes.data(), tmp_storage_bytes, @@ -49,9 +49,9 @@ void sort_csr(optimization_problem_t& op_problem) num_segments, op_problem.get_constraint_matrix_offsets().data(), op_problem.get_constraint_matrix_offsets().data() + 1, - stream_view); + stream_view.get()); RAFT_CHECK_CUDA(stream_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } } // namespace mathematical_optimization diff --git a/cpp/src/linear_algebra/vector_math.cuh b/cpp/src/linear_algebra/vector_math.cuh index ac9d24001b..57f5509213 100644 --- a/cpp/src/linear_algebra/vector_math.cuh +++ b/cpp/src/linear_algebra/vector_math.cuh @@ -37,7 +37,7 @@ struct norm_inf_max { }; template -f_t device_custom_vector_norm_inf(InputIteratorT in, i_t size, rmm::cuda_stream_view stream_view) +f_t device_custom_vector_norm_inf(InputIteratorT in, i_t size, cuda::stream_ref stream_view) { if (size == 0) { return 0; } // FIXME: Tmp storage stored in vector_math class. @@ -53,7 +53,7 @@ f_t device_custom_vector_norm_inf(InputIteratorT in, i_t size, rmm::cuda_stream_ size, custom_op, init, - stream_view); + stream_view.get()); d_temp_storage.resize(temp_storage_bytes, stream_view); @@ -64,18 +64,18 @@ f_t device_custom_vector_norm_inf(InputIteratorT in, i_t size, rmm::cuda_stream_ size, custom_op, init, - stream_view); + stream_view.get()); return d_out.value(stream_view); } template -f_t device_vector_norm_inf(const rmm::device_uvector& in, rmm::cuda_stream_view stream_view) +f_t device_vector_norm_inf(const rmm::device_uvector& in, cuda::stream_ref stream_view) { return device_custom_vector_norm_inf(in.data(), in.size(), stream_view); } template -f_t device_vector_norm_inf(raft::device_span in, rmm::cuda_stream_view stream_view) +f_t device_vector_norm_inf(raft::device_span in, cuda::stream_ref stream_view) { return device_custom_vector_norm_inf(in.data(), in.size(), stream_view); } @@ -83,14 +83,14 @@ f_t device_vector_norm_inf(raft::device_span in, rmm::cuda_stream_vie // TMP we should just have a CPU and GPU version to do the comparison // Should never have to norm inf a CPU vector if we are using the GPU template -f_t vector_norm_inf(const std::vector& x, rmm::cuda_stream_view stream_view) +f_t vector_norm_inf(const std::vector& x, cuda::stream_ref stream_view) { const auto d_x = device_copy(x, stream_view); return device_vector_norm_inf(d_x, stream_view); } template -f_t vector_norm_inf(raft::host_span x, rmm::cuda_stream_view stream_view) +f_t vector_norm_inf(raft::host_span x, cuda::stream_ref stream_view) { rmm::device_uvector d_x(x.size(), stream_view); raft::copy(d_x.data(), x.data(), x.size(), stream_view); diff --git a/cpp/src/math_optimization/solver_settings.cu b/cpp/src/math_optimization/solver_settings.cu index 67ba811eb4..2e04767f04 100644 --- a/cpp/src/math_optimization/solver_settings.cu +++ b/cpp/src/math_optimization/solver_settings.cu @@ -420,7 +420,7 @@ std::string solver_settings_t::get_parameter_as_string(const std::stri template void solver_settings_t::set_initial_pdlp_primal_solution(const f_t* solution, i_t size, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { pdlp_settings.set_initial_primal_solution(solution, size, stream); } @@ -428,7 +428,7 @@ void solver_settings_t::set_initial_pdlp_primal_solution(const f_t* so template void solver_settings_t::set_initial_pdlp_dual_solution(const f_t* solution, i_t size, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { pdlp_settings.set_initial_dual_solution(solution, size, stream); } @@ -492,7 +492,7 @@ const rmm::device_uvector& solver_settings_t::get_initial_pdlp_du template void solver_settings_t::add_initial_mip_solution(const f_t* solution, i_t size, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { mip_settings.add_initial_solution(solution, size, stream); } diff --git a/cpp/src/mip_heuristics/diversity/assignment_hash_map.cu b/cpp/src/mip_heuristics/diversity/assignment_hash_map.cu index c9d20c97fe..7b53829f66 100644 --- a/cpp/src/mip_heuristics/diversity/assignment_hash_map.cu +++ b/cpp/src/mip_heuristics/diversity/assignment_hash_map.cu @@ -84,9 +84,11 @@ size_t assignment_hash_map_t::hash_solution(solution_t& solu fill_integer_assignment(solution); thrust::fill( solution.handle_ptr->get_thrust_policy(), reduction_buffer.begin(), reduction_buffer.end(), 0); - hash_solution_kernel - <<<(integer_assignment.size() + TPB - 1) / TPB, TPB, 0, solution.handle_ptr->get_stream()>>>( - cuopt::make_span(integer_assignment), cuopt::make_span(reduction_buffer)); + hash_solution_kernel<<<(integer_assignment.size() + TPB - 1) / TPB, + TPB, + 0, + solution.handle_ptr->get_stream().get()>>>( + cuopt::make_span(integer_assignment), cuopt::make_span(reduction_buffer)); RAFT_CHECK_CUDA(solution.handle_ptr->get_stream()); // Get the number of blocks used in the hash_solution_kernel int num_blocks = (integer_assignment.size() + TPB - 1) / TPB; @@ -103,7 +105,7 @@ size_t assignment_hash_map_t::hash_solution(solution_t& solu num_blocks, combine_hash(), 0, - solution.handle_ptr->get_stream()); + solution.handle_ptr->get_stream().get()); // Allocate temporary storage temp_storage.resize(temp_storage_bytes, solution.handle_ptr->get_stream()); @@ -117,7 +119,7 @@ size_t assignment_hash_map_t::hash_solution(solution_t& solu num_blocks, combine_hash(), 0, - solution.handle_ptr->get_stream()); + solution.handle_ptr->get_stream().get()); // Return early since we've already computed the hash sum return hash_sum.value(solution.handle_ptr->get_stream()); diff --git a/cpp/src/mip_heuristics/diversity/recombiners/recombiner.cuh b/cpp/src/mip_heuristics/diversity/recombiners/recombiner.cuh index 0e9c64e796..bc42d5e1df 100644 --- a/cpp/src/mip_heuristics/diversity/recombiners/recombiner.cuh +++ b/cpp/src/mip_heuristics/diversity/recombiners/recombiner.cuh @@ -86,11 +86,11 @@ class recombiner_t { const i_t TPB = 128; i_t n_blocks = (a.problem_ptr->n_integer_vars + TPB - 1) / TPB; assign_same_variables_kernel - <<get_stream()>>>(a.view(), - b.view(), - offspring.view(), - cuopt::make_span(remaining_indices), - n_remaining.data()); + <<get_stream().get()>>>(a.view(), + b.view(), + offspring.view(), + cuopt::make_span(remaining_indices), + n_remaining.data()); i_t remaining_variables = this->n_remaining.value(a.handle_ptr->get_stream()); auto vec_remaining_indices = diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu index 4efd73e454..e73742040c 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cu @@ -119,7 +119,7 @@ fj_t::~fj_t() } template -void fj_t::reset_weights(const rmm::cuda_stream_view& climber_stream, f_t weight) +void fj_t::reset_weights(const cuda::stream_ref& climber_stream, f_t weight) { // unless reset explicitly, the values are kept across runs and across climbers max_cstr_weight.set_value_async(weight, climber_stream); @@ -277,7 +277,7 @@ void fj_t::copy_weights(const weight_t& weights, } template -void fj_t::climber_data_t::clear_sets(const rmm::cuda_stream_view& stream) +void fj_t::climber_data_t::clear_sets(const cuda::stream_ref& stream) { violated_constraints.clear(stream); candidate_variables.clear(stream); @@ -286,7 +286,7 @@ void fj_t::climber_data_t::clear_sets(const rmm::cuda_stream_view& str } template -void fj_t::device_init(const rmm::cuda_stream_view& stream) +void fj_t::device_init(const cuda::stream_ref& stream) { thrust::for_each(rmm::exec_policy(stream), thrust::counting_iterator(0), @@ -314,7 +314,7 @@ void fj_t::climber_init(i_t climber_idx) } template -void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& climber_stream) +void fj_t::climber_init(i_t climber_idx, const cuda::stream_ref& climber_stream) { raft::common::nvtx::range scope("climber_init"); @@ -452,7 +452,7 @@ void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& f_t excess = climber->violation_score.value(climber_stream); climber->best_excess.set_value_async(excess, climber_stream); } - climber_stream.synchronize(); + climber_stream.sync(); climber->break_condition.set_value_to_zero_async(climber_stream); climber->temp_break_condition.set_value_to_zero_async(climber_stream); @@ -468,9 +468,9 @@ void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& climber->iterations_until_feasible_counter.set_value_to_zero_async(climber_stream); climber->small_move_tabu.set_value_to_zero_async(climber_stream); - climber_stream.synchronize(); + climber_stream.sync(); - climber_stream.synchronize(); + climber_stream.sync(); view = climber->view(); @@ -496,7 +496,7 @@ void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& row_size_it_bin, row_size_bin_prefix_sum.data(), pb_ptr->binary_indices.size(), - climber_stream); + climber_stream.get()); if (i == 0 && temp_storage_bytes > climber->cub_storage_bytes.size()) climber->cub_storage_bytes.resize(temp_storage_bytes, climber_stream); } @@ -507,7 +507,7 @@ void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& row_size_it_nonbin, row_size_nonbin_prefix_sum.data(), pb_ptr->nonbinary_indices.size(), - climber_stream); + climber_stream.get()); if (i == 0 && temp_storage_bytes > climber->cub_storage_bytes.size()) climber->cub_storage_bytes.resize(temp_storage_bytes, climber_stream); } @@ -530,7 +530,7 @@ void fj_t::climber_init(i_t climber_idx, const rmm::cuda_stream_view& pb_ptr->n_variables, pb_ptr->related_variables_offsets.begin(), pb_ptr->related_variables_offsets.begin() + 1, - climber_stream); + climber_stream.get()); if (i == 0 && temp_storage_bytes > climber->cub_storage_bytes.size()) climber->cub_storage_bytes.resize(temp_storage_bytes, climber_stream); } @@ -600,8 +600,7 @@ void fj_t::run_step_device(i_t climber_idx, bool use_graph) // TODO: switch to conditional graph nodes once we switch to CTK >= 12.4 template -void fj_t::load_balancing_score_update(const rmm::cuda_stream_view& stream, - i_t climber_idx) +void fj_t::load_balancing_score_update(const cuda::stream_ref& stream, i_t climber_idx) { auto [grid_load_balancing_prepare, blocks_load_balancing_prepare] = load_balancing_prepare_launch_dims; @@ -657,7 +656,7 @@ void fj_t::load_balancing_score_update(const rmm::cuda_stream_view& st } template -void fj_t::run_step_device(const rmm::cuda_stream_view& climber_stream, +void fj_t::run_step_device(const cuda::stream_ref& climber_stream, i_t climber_idx, bool use_graph) { @@ -720,7 +719,7 @@ void fj_t::run_step_device(const rmm::cuda_stream_view& climber_stream data.candidate_variables.contents.data(), data.candidate_variables.set_size.data(), pb_ptr->n_variables, - climber_stream); + climber_stream.get()); if (compaction_temp_storage_bytes > data.cub_storage_bytes.size()) { data.cub_storage_bytes.resize(compaction_temp_storage_bytes, climber_stream); } @@ -768,7 +767,7 @@ void fj_t::run_step_device(const rmm::cuda_stream_view& climber_stream data.candidate_variables.contents.data(), data.candidate_variables.set_size.data(), pb_ptr->n_variables, - climber_stream); + climber_stream.get()); launch_select_variable_kernel(dim3(1), dim3(256), kernel_args, climber_stream); @@ -808,7 +807,7 @@ void fj_t::round_remaining_fractionals(solution_t& solution, } template -void fj_t::refresh_lhs_and_violation(const rmm::cuda_stream_view& stream, i_t climber_idx) +void fj_t::refresh_lhs_and_violation(const cuda::stream_ref& stream, i_t climber_idx) { auto& data = *climbers[climber_idx]; auto v = data.view(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh index 07c3be022f..e5f33b2a7c 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump.cuh @@ -228,23 +228,21 @@ class fj_t { bool randomize_params = false); i_t alloc_max_climbers(i_t desired_climbers); void resize_vectors(const raft::handle_t* handle_ptr); - void device_init(const rmm::cuda_stream_view& stream); + void device_init(const cuda::stream_ref& stream); void climber_init(i_t climber_idx); - void climber_init(i_t climber_idx, const rmm::cuda_stream_view& stream); + void climber_init(i_t climber_idx, const cuda::stream_ref& stream); void set_fj_settings(fj_settings_t settings_); - void reset_weights(const rmm::cuda_stream_view& stream, f_t weight = 10.); + void reset_weights(const cuda::stream_ref& stream, f_t weight = 10.); void randomize_weights(const raft::handle_t* handle_ptr); void copy_weights(const weight_t& weights, const raft::handle_t* handle_ptr, std::optional new_size = std::nullopt); i_t host_loop(solution_t& solution, i_t climber_idx = 0); void run_step_device(i_t climber_idx = 0, bool use_graph = true); - void run_step_device(const rmm::cuda_stream_view& stream, - i_t climber_idx = 0, - bool use_graph = true); - void refresh_lhs_and_violation(const rmm::cuda_stream_view& stream, i_t climber_idx = 0); + void run_step_device(const cuda::stream_ref& stream, i_t climber_idx = 0, bool use_graph = true); + void refresh_lhs_and_violation(const cuda::stream_ref& stream, i_t climber_idx = 0); // load balancing - void load_balancing_score_update(const rmm::cuda_stream_view& stream, i_t climber_idx = 0); + void load_balancing_score_update(const cuda::stream_ref& stream, i_t climber_idx = 0); // executed after a roudning FJ run if any fractionals remain to eliminate them void round_remaining_fractionals(solution_t& solution, i_t climber_idx = 0); @@ -438,7 +436,7 @@ class fj_t { dot_product_buffer.data(), incumbent_objective.data(), fj.pb_ptr->n_variables, - fj.handle_ptr->get_stream()); + fj.handle_ptr->get_stream().get()); // Allocate temporary storage cub_storage_bytes.resize(temp_storage_bytes, fj.handle_ptr->get_stream()); @@ -633,7 +631,7 @@ class fj_t { }; view_t view(); - void clear_sets(const rmm::cuda_stream_view& stream); + void clear_sets(const cuda::stream_ref& stream); }; void populate_climber_views(); diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cu b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cu index 441cfcc01f..3c69af1387 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cu +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cu @@ -1439,10 +1439,10 @@ template void launch_load_balancing_prepare_iteration(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchCooperativeKernel( - (void*)load_balancing_prepare_iteration, grid, blocks, kernel_args, 0, stream)); + (void*)load_balancing_prepare_iteration, grid, blocks, kernel_args, 0, stream.get())); } template @@ -1457,10 +1457,10 @@ template void launch_update_assignment_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)update_assignment_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)update_assignment_kernel, grid, blocks, kernel_args, 0, stream.get())); } template @@ -1531,7 +1531,7 @@ template void launch_compute_mtm_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY( cudaLaunchCooperativeKernel((void*)compute_mtm_moves_kernel, @@ -1539,27 +1539,27 @@ void launch_compute_mtm_moves_kernel(dim3 grid, blocks, kernel_args, 0, - stream)); + stream.get())); } template void launch_load_balancing_sanity_checks(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchCooperativeKernel( - (void*)load_balancing_sanity_checks, grid, blocks, kernel_args, 0, stream)); + (void*)load_balancing_sanity_checks, grid, blocks, kernel_args, 0, stream.get())); } template void launch_handle_local_minimum_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchCooperativeKernel( - (void*)handle_local_minimum_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)handle_local_minimum_kernel, grid, blocks, kernel_args, 0, stream.get())); } template @@ -1574,110 +1574,134 @@ template void launch_update_changed_constraints_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)update_changed_constraints_kernel, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)update_changed_constraints_kernel, + grid, + blocks, + kernel_args, + 0, + stream.get())); } template void launch_update_lift_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)update_lift_moves_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)update_lift_moves_kernel, grid, blocks, kernel_args, 0, stream.get())); } template void launch_update_breakthrough_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)update_breakthrough_moves_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)update_breakthrough_moves_kernel, grid, blocks, kernel_args, 0, stream.get())); } template void launch_select_variable_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)select_variable_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)select_variable_kernel, grid, blocks, kernel_args, 0, stream.get())); } template void launch_init_lhs_and_violation(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)init_lhs_and_violation, grid, blocks, kernel_args, 0, stream)); + (void*)init_lhs_and_violation, grid, blocks, kernel_args, 0, stream.get())); } template void launch_update_best_solution_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)update_best_solution_kernel, grid, blocks, kernel_args, 0, stream)); + (void*)update_best_solution_kernel, grid, blocks, kernel_args, 0, stream.get())); } template void launch_load_balancing_compute_workid_mappings(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)load_balancing_compute_workid_mappings, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)load_balancing_compute_workid_mappings, + grid, + blocks, + kernel_args, + 0, + stream.get())); } template void launch_load_balancing_init_cstr_bounds_csr(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)load_balancing_init_cstr_bounds_csr, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)load_balancing_init_cstr_bounds_csr, + grid, + blocks, + kernel_args, + 0, + stream.get())); } template void launch_load_balancing_compute_scores_binary(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)load_balancing_compute_scores_binary, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)load_balancing_compute_scores_binary, + grid, + blocks, + kernel_args, + 0, + stream.get())); } template void launch_load_balancing_mtm_compute_candidates(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)load_balancing_mtm_compute_candidates, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)load_balancing_mtm_compute_candidates, + grid, + blocks, + kernel_args, + 0, + stream.get())); } template void launch_load_balancing_mtm_compute_scores(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { - RAFT_CUDA_TRY(cudaLaunchKernel( - (void*)load_balancing_mtm_compute_scores, grid, blocks, kernel_args, 0, stream)); + RAFT_CUDA_TRY(cudaLaunchKernel((void*)load_balancing_mtm_compute_scores, + grid, + blocks, + kernel_args, + 0, + stream.get())); } // to save from compilation time, separate those and instantiate separately rather being part of a @@ -1734,11 +1758,11 @@ void launch_load_balancing_mtm_compute_scores(dim3 grid, template __global__ void select_variable_kernel( \ typename fj_t::climber_data_t::view_t fj); \ template void launch_load_balancing_prepare_iteration( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template std::pair get_launch_dims_update_assignment_kernel( \ int TPB, const raft::handle_t* handle_ptr); \ template void launch_update_assignment_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template std::pair \ get_launch_dims_compute_mtm_moves_kernel( \ int TPB, const raft::handle_t* handle_ptr); \ @@ -1763,37 +1787,37 @@ void launch_load_balancing_mtm_compute_scores(dim3 grid, template std::pair get_launch_dims_load_balancing_prepare_iteration( \ int TPB, const raft::handle_t* handle_ptr); \ template void launch_compute_mtm_moves_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_compute_mtm_moves_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_sanity_checks( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_handle_local_minimum_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template std::pair get_launch_dims_update_changed_constraints_kernel( \ int TPB, const raft::handle_t* handle_ptr); \ template void launch_update_changed_constraints_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_update_lift_moves_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_update_breakthrough_moves_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_select_variable_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_init_lhs_and_violation( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_update_best_solution_kernel( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_compute_workid_mappings( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_init_cstr_bounds_csr( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_compute_scores_binary( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_mtm_compute_candidates( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); \ + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); \ template void launch_load_balancing_mtm_compute_scores( \ - dim3 grid, dim3 blocks, void** kernel_args, rmm::cuda_stream_view stream); + dim3 grid, dim3 blocks, void** kernel_args, cuda::stream_ref stream); #if MIP_INSTANTIATE_FLOAT CUOPT_INSTANTIATE(float) diff --git a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cuh b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cuh index 3bc4b2ebaa..7a2c691da5 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/feasibility_jump_kernels.cuh @@ -102,7 +102,7 @@ template void launch_load_balancing_prepare_iteration(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template std::pair get_launch_dims_update_assignment_kernel(int TPB, @@ -112,7 +112,7 @@ template void launch_update_assignment_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template std::pair get_launch_dims_compute_mtm_moves_kernel(int TPB, @@ -150,19 +150,19 @@ template void launch_compute_mtm_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_sanity_checks(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_handle_local_minimum_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template std::pair get_launch_dims_update_changed_constraints_kernel( @@ -172,66 +172,66 @@ template void launch_update_changed_constraints_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_update_lift_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_update_breakthrough_moves_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_select_variable_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_init_lhs_and_violation(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_update_best_solution_kernel(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_compute_workid_mappings(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_init_cstr_bounds_csr(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_compute_scores_binary(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_mtm_compute_candidates(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void launch_load_balancing_mtm_compute_scores(dim3 grid, dim3 blocks, void** kernel_args, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/feasibility_jump/utils.cuh b/cpp/src/mip_heuristics/feasibility_jump/utils.cuh index 1b2862d558..7e95a8e867 100644 --- a/cpp/src/mip_heuristics/feasibility_jump/utils.cuh +++ b/cpp/src/mip_heuristics/feasibility_jump/utils.cuh @@ -36,23 +36,23 @@ template struct bitmap_t { static constexpr int bits_per_word = sizeof(word_t) * CHAR_BIT; - bitmap_t(size_t size, const rmm::cuda_stream_view& stream) + bitmap_t(size_t size, const cuda::stream_ref& stream) : validity_bitmap(size > 0 ? (size - 1) / bits_per_word + 1 : 0, stream) { clear(stream); } - void clear(const rmm::cuda_stream_view& stream) + void clear(const cuda::stream_ref& stream) { cudaMemsetAsync( - validity_bitmap.data(), 0, sizeof(word_t) * validity_bitmap.size(), stream.value()); + validity_bitmap.data(), 0, sizeof(word_t) * validity_bitmap.size(), stream.get()); } void clear(const raft::handle_t* handle_ptr) { thrust::uninitialized_fill( handle_ptr->get_thrust_policy(), validity_bitmap.begin(), validity_bitmap.end(), 0); } - void resize(size_t size, const rmm::cuda_stream_view& stream) + void resize(size_t size, const cuda::stream_ref& stream) { validity_bitmap.resize(size > 0 ? (size - 1) / bits_per_word + 1 : 0, stream); } @@ -100,7 +100,7 @@ struct bitmap_t { template struct contiguous_set_t { - contiguous_set_t(i_t max_size, const rmm::cuda_stream_view& stream) + contiguous_set_t(i_t max_size, const cuda::stream_ref& stream) : set_size(zero_v, stream), lock(zero_v, stream), contents(max_size, stream), @@ -110,12 +110,12 @@ struct contiguous_set_t { clear(stream); } - void clear(const rmm::cuda_stream_view& stream) + void clear(const cuda::stream_ref& stream) { set_size.set_value_to_zero_async(stream); // can't use thrust::fill, needs a memset node in order to be recorded in CUDA graphs // works bcs (uint8_t)-1 == 0xFF => (repeated 4 times) 0xFFFFFFFF == (uint32_t)-1 - cudaMemsetAsync(index_map.data(), -1, sizeof(i_t) * index_map.size(), stream.value()); + cudaMemsetAsync(index_map.data(), -1, sizeof(i_t) * index_map.size(), stream.get()); validity_bitmap.clear(stream); } @@ -127,7 +127,7 @@ struct contiguous_set_t { set_size.set_value_to_zero_async(handle_ptr->get_stream()); } - void resize(size_t size, const rmm::cuda_stream_view& stream) + void resize(size_t size, const cuda::stream_ref& stream) { contents.resize(size, stream); index_map.resize(size, stream); diff --git a/cpp/src/mip_heuristics/local_search/rounding/bounds_repair.cu b/cpp/src/mip_heuristics/local_search/rounding/bounds_repair.cu index ddc6db68a7..f1b3011514 100644 --- a/cpp/src/mip_heuristics/local_search/rounding/bounds_repair.cu +++ b/cpp/src/mip_heuristics/local_search/rounding/bounds_repair.cu @@ -254,7 +254,7 @@ void bounds_repair_t::compute_damages(problem_t& problem, i_ CUOPT_LOG_TRACE("Bounds repair: Computing damanges!"); // TODO check performance, we can apply load balancing here const i_t TPB = 256; - compute_damages_kernel<<get_stream()>>>( + compute_damages_kernel<<get_stream().get()>>>( problem.view(), candidates.view(), make_span(cstr_violations_up), diff --git a/cpp/src/mip_heuristics/local_search/rounding/constraint_prop.cu b/cpp/src/mip_heuristics/local_search/rounding/constraint_prop.cu index 861432b720..e2e9e51456 100644 --- a/cpp/src/mip_heuristics/local_search/rounding/constraint_prop.cu +++ b/cpp/src/mip_heuristics/local_search/rounding/constraint_prop.cu @@ -88,7 +88,7 @@ void sort_subsections(raft::device_span vars, n_subsections, offsets.data(), offsets.data() + 1, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); // Allocate temporary storage d_temp_storage.resize(temp_storage_bytes, handle_ptr->get_stream()); @@ -104,7 +104,7 @@ void sort_subsections(raft::device_span vars, n_subsections, offsets.data(), offsets.data() + 1, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); handle_ptr->sync_stream(); } @@ -176,7 +176,7 @@ void constraint_prop_t::sort_by_implied_slack_consumption(solution_t - <<get_stream()>>>( + <<get_stream().get()>>>( sol.problem_ptr->view(), vars, min_activity, diff --git a/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu b/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu index 2d5aae0b0d..872bb8f0bf 100644 --- a/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu +++ b/cpp/src/mip_heuristics/local_search/rounding/simple_rounding.cu @@ -54,16 +54,18 @@ bool check_brute_force_rounding(solution_t& solution) // // try all configs in parallel and compute feasibility brute_force_check_kernel - <<get_stream()>>>(solution.view(), - n_integers_to_round, - cuopt::make_span(var_map), - cuopt::make_span(constraint_buf), - best_config.data()); + <<get_stream().get()>>>( + solution.view(), + n_integers_to_round, + cuopt::make_span(var_map), + cuopt::make_span(constraint_buf), + best_config.data()); if (best_config.value(solution.handle_ptr->get_stream()) != -1) { CUOPT_LOG_DEBUG("Feasible found during brute force rounding!"); // apply the feasible rounding - apply_feasible_rounding_kernel<<<1, TPB, 0, solution.handle_ptr->get_stream()>>>( - solution.view(), n_integers_to_round, cuopt::make_span(var_map), best_config.data()); + apply_feasible_rounding_kernel + <<<1, TPB, 0, solution.handle_ptr->get_stream().get()>>>( + solution.view(), n_integers_to_round, cuopt::make_span(var_map), best_config.data()); solution.handle_ptr->sync_stream(); bool feas = solution.compute_feasibility(); cuopt_assert(feas, "Solution must be feasible!"); @@ -84,7 +86,7 @@ bool invoke_simple_rounding(solution_t& solution) rmm::device_scalar successful(true_v, solution.handle_ptr->get_stream()); i_t TPB = 128; simple_rounding_kernel - <<<2048, TPB, 0, solution.handle_ptr->get_stream()>>>(solution.view(), successful.data()); + <<<2048, TPB, 0, solution.handle_ptr->get_stream().get()>>>(solution.view(), successful.data()); if (!successful.value(solution.handle_ptr->get_stream())) { CUOPT_LOG_DEBUG("Simple rounding failed"); solution.copy_from(sol_copy); @@ -112,7 +114,7 @@ void invoke_round_nearest(solution_t& solution) if (simple_round) { return; } i_t n_blocks = (solution.problem_ptr->n_integer_vars + TPB - 1) / TPB; - nearest_rounding_kernel<<get_stream()>>>( + nearest_rounding_kernel<<get_stream().get()>>>( solution.view(), cuopt::seed_generator::get_seed()); RAFT_CHECK_CUDA(solution.handle_ptr->get_stream()); } @@ -127,8 +129,9 @@ void invoke_random_round_nearest(solution_t& solution, i_t n_target_ra n_integers, solution.problem_ptr->n_integer_vars); rmm::device_scalar n_randomly_rounded(zero_v, solution.handle_ptr->get_stream()); - random_nearest_rounding_kernel<<get_stream()>>>( - solution.view(), cuopt::seed_generator::get_seed(), n_randomly_rounded.data()); + random_nearest_rounding_kernel + <<get_stream().get()>>>( + solution.view(), cuopt::seed_generator::get_seed(), n_randomly_rounded.data()); i_t h_n_random_rounds = n_randomly_rounded.value(solution.handle_ptr->get_stream()); CUOPT_LOG_TRACE("Randomly rounded integers %d", h_n_random_rounds); i_t additional_roundings_needed = n_target_random_rounds - h_n_random_rounds; @@ -143,11 +146,11 @@ void invoke_random_round_nearest(solution_t& solution, i_t n_target_ra shuffled_indices.end(), rng); random_rounding_kernel - <<<1, 1, 0, solution.handle_ptr->get_stream()>>>(solution.view(), - cuopt::seed_generator::get_seed(), - shuffled_indices.data(), - n_randomly_rounded.data(), - additional_roundings_needed); + <<<1, 1, 0, solution.handle_ptr->get_stream().get()>>>(solution.view(), + cuopt::seed_generator::get_seed(), + shuffled_indices.data(), + n_randomly_rounded.data(), + additional_roundings_needed); h_n_random_rounds = n_randomly_rounded.value(solution.handle_ptr->get_stream()); CUOPT_LOG_TRACE("Randomly rounded integers, after adding close integers too %d", h_n_random_rounds); diff --git a/cpp/src/mip_heuristics/mip_scaling_strategy.cu b/cpp/src/mip_heuristics/mip_scaling_strategy.cu index 8ff8310f61..c635ac1405 100644 --- a/cpp/src/mip_heuristics/mip_scaling_strategy.cu +++ b/cpp/src/mip_heuristics/mip_scaling_strategy.cu @@ -149,7 +149,7 @@ void compute_row_inf_norm( rmm::device_uvector& temp_storage, size_t temp_storage_bytes, rmm::device_uvector& row_inf_norm, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { const auto& matrix_values = op_problem.get_constraint_matrix_values(); const auto& matrix_offsets = op_problem.get_constraint_matrix_offsets(); @@ -165,7 +165,7 @@ void compute_row_inf_norm( matrix_offsets.data() + 1, max_op_t{}, f_t(0), - stream_view)); + stream_view.get())); } template @@ -174,7 +174,7 @@ void compute_row_integer_gcd( rmm::device_uvector& temp_storage, size_t temp_storage_bytes, rmm::device_uvector& row_integer_gcd, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { const auto& matrix_values = op_problem.get_constraint_matrix_values(); const auto& matrix_indices = op_problem.get_constraint_matrix_indices(); @@ -203,7 +203,7 @@ void compute_row_integer_gcd( matrix_offsets.data() + 1, gcd_op_t{}, std::int64_t{0}, - stream_view)); + stream_view.get())); } template @@ -236,7 +236,7 @@ void compute_big_m_skip_rows( matrix_offsets.data() + 1, max_op_t{}, f_t(0), - stream_view)); + stream_view.get())); size_t min_bytes = temp_storage_bytes; RAFT_CUDA_TRY(cub::DeviceSegmentedReduce::Reduce(temp_storage.data(), min_bytes, @@ -247,7 +247,7 @@ void compute_big_m_skip_rows( matrix_offsets.data() + 1, min_op_t{}, std::numeric_limits::infinity(), - stream_view)); + stream_view.get())); size_t count_bytes = temp_storage_bytes; RAFT_CUDA_TRY(cub::DeviceSegmentedReduce::Reduce(temp_storage.data(), count_bytes, @@ -258,7 +258,7 @@ void compute_big_m_skip_rows( matrix_offsets.data() + 1, thrust::plus{}, i_t(0), - stream_view)); + stream_view.get())); auto row_begin = thrust::make_zip_iterator( thrust::make_tuple(row_inf_norm.begin(), row_min_nonzero.begin(), row_nonzero_count.begin())); @@ -357,7 +357,7 @@ rmm::device_uvector capture_pre_scaling_integer_gcd( const cuopt::mathematical_optimization::optimization_problem_t& op_problem, rmm::device_uvector& temp_storage, size_t temp_storage_bytes, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { const i_t n_rows = op_problem.get_n_constraints(); rmm::device_uvector gcd(static_cast(n_rows), stream_view); @@ -371,7 +371,7 @@ void assert_integer_coefficient_integrality( rmm::device_uvector& temp_storage, size_t temp_storage_bytes, const rmm::device_uvector& pre_scaling_gcd, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { const auto* handle_ptr = op_problem.get_handle_ptr(); const i_t n_rows = op_problem.get_n_constraints(); @@ -415,7 +415,7 @@ size_t dry_run_cub( rmm::device_uvector& row_min_nonzero, rmm::device_uvector& row_nonzero_count, rmm::device_uvector& row_integer_gcd, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { const auto& matrix_values = op_problem.get_constraint_matrix_values(); const auto& matrix_indices = op_problem.get_constraint_matrix_indices(); @@ -435,7 +435,7 @@ size_t dry_run_cub( matrix_offsets.data() + 1, max_op_t{}, f_t(0), - stream_view)); + stream_view.get())); temp_storage_bytes = std::max(temp_storage_bytes, current_required_bytes); auto coeff_nonzero_min_iter = @@ -449,7 +449,7 @@ size_t dry_run_cub( matrix_offsets.data() + 1, min_op_t{}, std::numeric_limits::infinity(), - stream_view)); + stream_view.get())); temp_storage_bytes = std::max(temp_storage_bytes, current_required_bytes); auto coeff_nonzero_count_iter = @@ -463,7 +463,7 @@ size_t dry_run_cub( matrix_offsets.data() + 1, thrust::plus{}, i_t(0), - stream_view)); + stream_view.get())); temp_storage_bytes = std::max(temp_storage_bytes, current_required_bytes); if (variable_types.size() == static_cast(op_problem.get_n_variables())) { @@ -482,7 +482,7 @@ size_t dry_run_cub( matrix_offsets.data() + 1, gcd_op_t{}, std::int64_t{0}, - stream_view)); + stream_view.get())); temp_storage_bytes = std::max(temp_storage_bytes, current_required_bytes); } @@ -678,7 +678,7 @@ void mip_scaling_strategy_t::scale_problem(bool do_objective_scaling) ref_log2_values.data() + median_idx, sizeof(double), cudaMemcpyDeviceToHost, - stream_view_)); + stream_view_.get())); handle_ptr_->sync_stream(); f_t target_norm = static_cast(exp2(h_median_log2)); cuopt_assert(std::isfinite(static_cast(target_norm)), "target_norm must be finite"); diff --git a/cpp/src/mip_heuristics/mip_scaling_strategy.cuh b/cpp/src/mip_heuristics/mip_scaling_strategy.cuh index 3355eee5ca..59ec6d372c 100644 --- a/cpp/src/mip_heuristics/mip_scaling_strategy.cuh +++ b/cpp/src/mip_heuristics/mip_scaling_strategy.cuh @@ -26,7 +26,7 @@ class mip_scaling_strategy_t { private: raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; optimization_problem_type_t& op_problem_scaled_; }; diff --git a/cpp/src/mip_heuristics/presolve/block_bve.cu b/cpp/src/mip_heuristics/presolve/block_bve.cu index 6874833e15..c8d6ea3314 100644 --- a/cpp/src/mip_heuristics/presolve/block_bve.cu +++ b/cpp/src/mip_heuristics/presolve/block_bve.cu @@ -762,22 +762,22 @@ double bve_project_batch_gpu(const raft::handle_t& handle, // sentinel 0xFFFFFFFF (every byte 0xFF) marks a boundary pattern with no feasible interior // yet RAFT_CUDA_TRY( - cudaMemsetAsync(d_witness.data(), 0xFF, d_witness.size() * sizeof(uint32_t), stream)); + cudaMemsetAsync(d_witness.data(), 0xFF, d_witness.size() * sizeof(uint32_t), stream.get())); // one warp per row, one CTA per (block, m, am) assignment, grid-strided const int64_t total = (int64_t)num * (int64_t)patterns * ((int64_t)1 << na); const int grid = std::min(total, int64_t{65535}); - bve_enumerate_kernel<<>>(num, - nb, - na, - nrows, - tol, - d_coeffs.data(), - d_local_var.data(), - d_row_start.data(), - d_lower.data(), - d_upper.data(), - d_witness.data()); + bve_enumerate_kernel<<>>(num, + nb, + na, + nrows, + tol, + d_coeffs.data(), + d_local_var.data(), + d_row_start.data(), + d_lower.data(), + d_upper.data(), + d_witness.data()); RAFT_CUDA_TRY(cudaGetLastError()); // Unscaled op counts: host pack/unpack touches + one coeff read per assignment. diff --git a/cpp/src/mip_heuristics/presolve/bounds_presolve.cu b/cpp/src/mip_heuristics/presolve/bounds_presolve.cu index e5a7f249f1..98b166087d 100644 --- a/cpp/src/mip_heuristics/presolve/bounds_presolve.cu +++ b/cpp/src/mip_heuristics/presolve/bounds_presolve.cu @@ -100,7 +100,7 @@ void bound_presolve_t::calculate_activity(problem_t& pb) constexpr auto n_threads = 256; calc_activity_kernel - <<get_stream()>>>(pb.view(), upd.view()); + <<get_stream().get()>>>(pb.view(), upd.view()); } template @@ -122,7 +122,7 @@ bool bound_presolve_t::calculate_bounds_update(problem_t& pb pb.tolerances.absolute_tolerance / context.settings.semi_continuous_big_m; upd.bounds_changed.set_value_async(zero, pb.handle_ptr->get_stream()); update_bounds_kernel - <<get_stream()>>>(pb.view(), upd.view()); + <<get_stream().get()>>>(pb.view(), upd.view()); RAFT_CHECK_CUDA(pb.handle_ptr->get_stream()); i_t h_bounds_changed = upd.bounds_changed.value(pb.handle_ptr->get_stream()); return h_bounds_changed != zero; diff --git a/cpp/src/mip_heuristics/presolve/conditional_bound_strengthening.cu b/cpp/src/mip_heuristics/presolve/conditional_bound_strengthening.cu index b1f27de8a6..7033ae93a5 100644 --- a/cpp/src/mip_heuristics/presolve/conditional_bound_strengthening.cu +++ b/cpp/src/mip_heuristics/presolve/conditional_bound_strengthening.cu @@ -85,7 +85,7 @@ void spgemm_cusparse([[maybe_unused]] rmm::device_uvector& offsetsA, auto stream = offsetsA.stream(); cusparseHandle_t handle; cusparseCreate(&handle); - cusparseSetStream(handle, stream); + cusparseSetStream(handle, stream.get()); int m = offsetsA.size() - 1; int n = offsetsB.size() - 1; @@ -215,7 +215,7 @@ void spgemm_cusparse([[maybe_unused]] rmm::device_uvector& offsetsA, check_cusparse_status(cusparseSpGEMM_copy( handle, opA, opB, &alpha, matA, matB, &beta, matC, computeType, alg, spgemmDesc)); - stream.synchronize(); + stream.sync(); cusparseSpGEMM_destroyDescr(spgemmDesc); cusparseDestroySpMat(matA); @@ -642,7 +642,7 @@ struct len_from_offset { // Ideally this should be precomputed and stored in the problem, but that also means we need to // update it every time the problem is modified, so we will compute it here for now template -i_t get_max_row_size(rmm::device_uvector& offsets, rmm::cuda_stream_view stream_view) +i_t get_max_row_size(rmm::device_uvector& offsets, cuda::stream_ref stream_view) { auto begin = thrust::make_zip_iterator(thrust::make_tuple(offsets.begin(), offsets.begin() + 1)); auto end = thrust::make_zip_iterator(thrust::make_tuple(offsets.end() - 1, offsets.end())); diff --git a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cu b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cu index 017bf32e91..b8b86f1588 100644 --- a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cu +++ b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cu @@ -90,7 +90,7 @@ load_balanced_bounds_presolve_t::~load_balanced_bounds_presolve_t() } template -std::pair sub_warp_meta(rmm::cuda_stream_view stream, +std::pair sub_warp_meta(cuda::stream_ref stream, rmm::device_uvector& d_warp_offsets, rmm::device_uvector& d_warp_id_offsets, const std::vector& bin_offsets, diff --git a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cuh b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cuh index 341d9cd262..bd7a758b99 100644 --- a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cuh +++ b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve.cuh @@ -54,9 +54,9 @@ class managed_stream_pool { * * This function is thread safe with respect to other calls to the same function. * - * @return rmm::cuda_stream_view + * @return cuda::stream_ref */ - rmm::cuda_stream_view get_stream() const noexcept + cuda::stream_ref get_stream() const noexcept { int stream_id = (next_stream++) % streams_.size(); end_unsycned = std::max(stream_id, end_unsycned); diff --git a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve_helpers.cuh b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve_helpers.cuh index 6f8a811309..24651ed2d1 100644 --- a/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve_helpers.cuh +++ b/cpp/src/mip_heuristics/presolve/load_balanced_bounds_presolve_helpers.cuh @@ -80,7 +80,7 @@ struct heavy_vertex_meta_t { }; template -i_t create_heavy_item_block_segments(rmm::cuda_stream_view stream, +i_t create_heavy_item_block_segments(cuda::stream_ref stream, rmm::device_uvector& vertex_id, rmm::device_uvector& pseudo_block_id, rmm::device_uvector& item_block_segments, diff --git a/cpp/src/mip_heuristics/presolve/multi_probe.cu b/cpp/src/mip_heuristics/presolve/multi_probe.cu index 394d89f580..9f8d5038ab 100644 --- a/cpp/src/mip_heuristics/presolve/multi_probe.cu +++ b/cpp/src/mip_heuristics/presolve/multi_probe.cu @@ -115,11 +115,11 @@ void multi_probe_t::calculate_activity(problem_t& pb, auto& upd = skip_0 ? upd_1 : upd_0; constexpr auto n_threads = 256; calc_activity_kernel - <<get_stream()>>>(pb.view(), upd.view()); + <<get_stream().get()>>>(pb.view(), upd.view()); } else { constexpr auto n_threads = 256; calc_activity_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( pb.view(), upd_0.view(), upd_1.view()); } RAFT_CHECK_CUDA(handle_ptr->get_stream()); @@ -150,7 +150,7 @@ bool multi_probe_t::calculate_bounds_update(problem_t& pb, } else if (skip_0) { upd_1.bounds_changed.set_value_async(zero, handle_ptr->get_stream()); update_bounds_kernel - <<get_stream()>>>(pb.view(), upd_1.view()); + <<get_stream().get()>>>(pb.view(), upd_1.view()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); i_t h_bounds_changed_1 = upd_1.bounds_changed.value(handle_ptr->get_stream()); CUOPT_LOG_TRACE("Bounds changed upd 1 %d", h_bounds_changed_1); @@ -158,7 +158,7 @@ bool multi_probe_t::calculate_bounds_update(problem_t& pb, } else if (skip_1) { upd_0.bounds_changed.set_value_async(zero, handle_ptr->get_stream()); update_bounds_kernel - <<get_stream()>>>(pb.view(), upd_0.view()); + <<get_stream().get()>>>(pb.view(), upd_0.view()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); i_t h_bounds_changed_0 = upd_0.bounds_changed.value(handle_ptr->get_stream()); CUOPT_LOG_TRACE("Bounds changed upd 0 %d", h_bounds_changed_0); @@ -167,7 +167,7 @@ bool multi_probe_t::calculate_bounds_update(problem_t& pb, upd_0.bounds_changed.set_value_async(zero, handle_ptr->get_stream()); upd_1.bounds_changed.set_value_async(zero, handle_ptr->get_stream()); update_bounds_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( pb.view(), upd_0.view(), upd_1.view()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); i_t h_bounds_changed_0 = upd_0.bounds_changed.value(handle_ptr->get_stream()); diff --git a/cpp/src/mip_heuristics/presolve/probing_cache.cu b/cpp/src/mip_heuristics/presolve/probing_cache.cu index d331f27f80..8a25ebdd83 100644 --- a/cpp/src/mip_heuristics/presolve/probing_cache.cu +++ b/cpp/src/mip_heuristics/presolve/probing_cache.cu @@ -804,7 +804,7 @@ std::vector compute_priority_indices_by_implied_integers(problem_t{}, 0, - problem.handle_ptr->get_stream()); + problem.handle_ptr->get_stream().get()); rmm::device_uvector temp_storage(temp_storage_bytes, problem.handle_ptr->get_stream()); @@ -820,7 +820,7 @@ std::vector compute_priority_indices_by_implied_integers(problem_t{}, 0, - problem.handle_ptr->get_stream()); + problem.handle_ptr->get_stream().get()); // keeps the count of number of other integers that this variables shares a constraint with rmm::device_uvector count_per_variable(problem.n_variables, problem.handle_ptr->get_stream()); @@ -842,7 +842,7 @@ std::vector compute_priority_indices_by_implied_integers(problem_t{}, 0, - problem.handle_ptr->get_stream()); + problem.handle_ptr->get_stream().get()); temp_storage.resize(temp_storage_bytes, problem.handle_ptr->get_stream()); d_temp_storage = thrust::raw_pointer_cast(temp_storage.data()); @@ -857,7 +857,7 @@ std::vector compute_priority_indices_by_implied_integers(problem_t{}, 0, - problem.handle_ptr->get_stream()); + problem.handle_ptr->get_stream().get()); thrust::for_each(problem.handle_ptr->get_thrust_policy(), thrust::make_counting_iterator(0), thrust::make_counting_iterator(problem.n_variables), diff --git a/cpp/src/mip_heuristics/presolve/semi_continuous.cu b/cpp/src/mip_heuristics/presolve/semi_continuous.cu index 33b7efff0e..2241a24827 100644 --- a/cpp/src/mip_heuristics/presolve/semi_continuous.cu +++ b/cpp/src/mip_heuristics/presolve/semi_continuous.cu @@ -372,7 +372,7 @@ template void expand_initial_solutions_for_semi_continuous( mip_solver_settings_t& settings, const std::vector& semi_continuous_binary_to_original_indices, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (semi_continuous_binary_to_original_indices.empty()) { return; } @@ -399,7 +399,7 @@ template void append_semi_continuous_auxiliaries_to_assignment( template void strip_semi_continuous_auxiliaries_from_assignment(std::vector&, int); template void expand_initial_solutions_for_semi_continuous(mip_solver_settings_t&, const std::vector&, - rmm::cuda_stream_view); + cuda::stream_ref); #endif #if MIP_INSTANTIATE_DOUBLE @@ -412,7 +412,7 @@ template void append_semi_continuous_auxiliaries_to_assignment( template void strip_semi_continuous_auxiliaries_from_assignment(std::vector&, int); template void expand_initial_solutions_for_semi_continuous(mip_solver_settings_t&, const std::vector&, - rmm::cuda_stream_view); + cuda::stream_ref); #endif } // namespace cuopt::mathematical_optimization::mip diff --git a/cpp/src/mip_heuristics/presolve/semi_continuous.cuh b/cpp/src/mip_heuristics/presolve/semi_continuous.cuh index ce3edc16a5..fe37ec80dc 100644 --- a/cpp/src/mip_heuristics/presolve/semi_continuous.cuh +++ b/cpp/src/mip_heuristics/presolve/semi_continuous.cuh @@ -55,7 +55,7 @@ template void expand_initial_solutions_for_semi_continuous( mip_solver_settings_t& settings, const std::vector& semi_continuous_binary_to_original_indices, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); template void append_semi_continuous_auxiliaries_to_assignment( diff --git a/cpp/src/mip_heuristics/presolve/third_party_presolve.cpp b/cpp/src/mip_heuristics/presolve/third_party_presolve.cpp index b6976e430b..58e4452bd5 100644 --- a/cpp/src/mip_heuristics/presolve/third_party_presolve.cpp +++ b/cpp/src/mip_heuristics/presolve/third_party_presolve.cpp @@ -1205,7 +1205,7 @@ void third_party_presolve_t::undo_from_device(rmm::device_uvector problem_category_t category, bool status_to_skip, bool dual_postsolve, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { std::vector h_primal(primal_solution.size()); std::vector h_dual(dual_solution.size()); @@ -1213,7 +1213,7 @@ void third_party_presolve_t::undo_from_device(rmm::device_uvector raft::copy(h_primal.data(), primal_solution.data(), primal_solution.size(), stream_view); raft::copy(h_dual.data(), dual_solution.data(), dual_solution.size(), stream_view); raft::copy(h_rc.data(), reduced_costs.data(), reduced_costs.size(), stream_view); - stream_view.synchronize(); + stream_view.sync(); undo(h_primal, h_dual, h_rc, category, status_to_skip, dual_postsolve); @@ -1223,7 +1223,7 @@ void third_party_presolve_t::undo_from_device(rmm::device_uvector raft::copy(primal_solution.data(), h_primal.data(), h_primal.size(), stream_view); raft::copy(dual_solution.data(), h_dual.data(), h_dual.size(), stream_view); raft::copy(reduced_costs.data(), h_rc.data(), h_rc.size(), stream_view); - stream_view.synchronize(); + stream_view.sync(); } template diff --git a/cpp/src/mip_heuristics/presolve/third_party_presolve.hpp b/cpp/src/mip_heuristics/presolve/third_party_presolve.hpp index 79eb28d01e..7b25084d07 100644 --- a/cpp/src/mip_heuristics/presolve/third_party_presolve.hpp +++ b/cpp/src/mip_heuristics/presolve/third_party_presolve.hpp @@ -130,7 +130,7 @@ class third_party_presolve_t { problem_category_t category, bool status_to_skip, bool dual_postsolve, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); // Host-only postsolve. Resizes the vectors to original-problem dimensions. void undo(std::vector& primal_solution, diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cu b/cpp/src/mip_heuristics/problem/presolve_data.cu index 3c621bc2cd..0068b8e3dc 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cu +++ b/cpp/src/mip_heuristics/problem/presolve_data.cu @@ -150,7 +150,7 @@ void presolve_data_t::post_process_assignment( problem_t& problem, rmm::device_uvector& current_assignment, bool resize_to_original_problem, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { raft::common::nvtx::range fun_scope("post_process_assignment"); cuopt_assert(current_assignment.size() == variable_mapping.size(), "size mismatch"); diff --git a/cpp/src/mip_heuristics/problem/presolve_data.cuh b/cpp/src/mip_heuristics/problem/presolve_data.cuh index 713bb24c0d..1155307a23 100644 --- a/cpp/src/mip_heuristics/problem/presolve_data.cuh +++ b/cpp/src/mip_heuristics/problem/presolve_data.cuh @@ -53,7 +53,7 @@ struct postsolve_reconstruction_t { template class presolve_data_t { public: - presolve_data_t(const optimization_problem_t& problem, rmm::cuda_stream_view stream) + presolve_data_t(const optimization_problem_t& problem, cuda::stream_ref stream) : variable_offsets(problem.get_n_variables(), 0), additional_var_used(problem.get_n_variables(), false), additional_var_id_per_var(problem.get_n_variables(), -1), @@ -65,7 +65,7 @@ class presolve_data_t { { } - presolve_data_t(const presolve_data_t& other, rmm::cuda_stream_view stream) + presolve_data_t(const presolve_data_t& other, cuda::stream_ref stream) : variable_offsets(other.variable_offsets), additional_var_used(other.additional_var_used), additional_var_id_per_var(other.additional_var_id_per_var), @@ -106,7 +106,7 @@ class presolve_data_t { void post_process_assignment(problem_t& problem, rmm::device_uvector& current_assignment, bool resize_to_original_problem, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); void post_process_assignment(problem_t& problem, rmm::device_uvector& current_assignment, bool resize_to_original_problem = true) diff --git a/cpp/src/mip_heuristics/problem/problem.cu b/cpp/src/mip_heuristics/problem/problem.cu index 0264147781..2eeee225c1 100644 --- a/cpp/src/mip_heuristics/problem/problem.cu +++ b/cpp/src/mip_heuristics/problem/problem.cu @@ -454,7 +454,7 @@ void csr_to_csc_transpose(const i_t* csr_offsets, rmm::device_uvector next_pos(n_cols, stream); raft::copy(next_pos.data(), csc_offsets, n_cols, stream); - csr_to_csc_scatter_kernel<<>>( + csr_to_csc_scatter_kernel<<>>( n_rows, csr_offsets, csr_indices, csr_values, next_pos.data(), csc_indices, csc_values); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -473,7 +473,7 @@ void csr_to_csc_transpose(const i_t* csr_offsets, n_cols, csc_offsets, csc_offsets + 1, - stream); + stream.get()); rmm::device_uvector temp_storage(temp_storage_bytes, stream); cub::DeviceSegmentedSort::SortPairs(temp_storage.data(), @@ -486,12 +486,12 @@ void csr_to_csc_transpose(const i_t* csr_offsets, n_cols, csc_offsets, csc_offsets + 1, - stream); + stream.get()); // Copy sorted results back raft::copy(csc_indices, row_ind_sorted.data(), nnz, stream); raft::copy(csc_values, val_sorted.data(), nnz, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); } template @@ -500,9 +500,10 @@ void problem_t::compute_transpose_of_problem() raft::common::nvtx::range fun_scope("compute_transpose_of_problem"); csrsort_cusparse(coefficients, variables, offsets, n_constraints, n_variables, handle_ptr); RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); // Resize what is needed for LP reverse_offsets.resize(n_variables + 1, handle_ptr->get_stream()); reverse_constraints.resize(nnz, handle_ptr->get_stream()); @@ -1018,7 +1019,7 @@ void problem_t::compute_related_variables(double time_limit) related_variables.size() / (f_t)1e6); thrust::fill(handle_ptr->get_thrust_policy(), varmap.begin(), varmap.end(), 0); - compute_related_vars_unique<<<1024, 128, 0, handle_ptr->get_stream()>>>( + compute_related_vars_unique<<<1024, 128, 0, handle_ptr->get_stream().get()>>>( pb_view, slice_begin, slice_end, make_span(varmap)); // prefix sum to generate offsets @@ -1508,7 +1509,7 @@ void problem_t::substitute_variables(const std::vector& var_indic offsets.data() + 1, cuda::std::plus<>{}, initial_value, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); rmm::device_uvector temp_storage(temp_storage_bytes, handle_ptr->get_stream()); d_temp_storage = thrust::raw_pointer_cast(temp_storage.data()); @@ -1523,7 +1524,7 @@ void problem_t::substitute_variables(const std::vector& var_indic offsets.data() + 1, cuda::std::plus<>{}, initial_value, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); thrust::for_each( handle_ptr->get_thrust_policy(), @@ -1632,7 +1633,7 @@ void problem_t::fix_given_variables(problem_t& original_prob original_problem.offsets.data() + 1, cuda::std::plus<>{}, initial_value, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); rmm::device_uvector temp_storage(temp_storage_bytes, handle_ptr->get_stream()); d_temp_storage = thrust::raw_pointer_cast(temp_storage.data()); @@ -1647,7 +1648,7 @@ void problem_t::fix_given_variables(problem_t& original_prob original_problem.offsets.data() + 1, cuda::std::plus<>{}, initial_value, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); thrust::for_each( handle_ptr->get_thrust_policy(), @@ -1790,7 +1791,7 @@ void problem_t::remove_given_variables(problem_t& original_p presolve_data.var_flags.resize(variable_map.size(), handle_ptr->get_stream()); const i_t TPB = 64; // compute new offsets - compute_new_offsets<<get_stream()>>>( + compute_new_offsets<<get_stream().get()>>>( original_problem.view(), view(), cuopt::make_span(variable_map)); RAFT_CHECK_CUDA(handle_ptr->get_stream()); thrust::exclusive_scan(handle_ptr->get_thrust_policy(), @@ -1800,7 +1801,7 @@ void problem_t::remove_given_variables(problem_t& original_p rmm::device_uvector write_pos(n_constraints, handle_ptr->get_stream()); thrust::fill(handle_ptr->get_thrust_policy(), write_pos.begin(), write_pos.end(), 0); // compute new csr - compute_new_csr<<get_stream()>>>( + compute_new_csr<<get_stream().get()>>>( original_problem.view(), view(), cuopt::make_span(variable_map), cuopt::make_span(write_pos)); RAFT_CHECK_CUDA(handle_ptr->get_stream()); // assign nnz, number of variables etc. @@ -2167,7 +2168,7 @@ bool problem_t::pre_process_assignment(rmm::device_uvector& assig template void problem_t::post_process_assignment(rmm::device_uvector& current_assignment, bool resize_to_original_problem, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { presolve_data.post_process_assignment( *this, current_assignment, resize_to_original_problem, stream); diff --git a/cpp/src/mip_heuristics/problem/problem.cuh b/cpp/src/mip_heuristics/problem/problem.cuh index 3ea3973d1d..4833809f83 100644 --- a/cpp/src/mip_heuristics/problem/problem.cuh +++ b/cpp/src/mip_heuristics/problem/problem.cuh @@ -102,7 +102,7 @@ class problem_t { bool pre_process_assignment(rmm::device_uvector& assignment); void post_process_assignment(rmm::device_uvector& current_assignment, bool resize_to_original_problem, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); void post_process_assignment(rmm::device_uvector& current_assignment, bool resize_to_original_problem = true) { diff --git a/cpp/src/mip_heuristics/problem/problem_helpers.cuh b/cpp/src/mip_heuristics/problem/problem_helpers.cuh index 388fae4ecd..cff6dfb686 100644 --- a/cpp/src/mip_heuristics/problem/problem_helpers.cuh +++ b/cpp/src/mip_heuristics/problem/problem_helpers.cuh @@ -143,7 +143,7 @@ static void convert_to_maximization_problem(mip::problem_t& op_problem op_problem.objective_coefficients.data(), op_problem.objective_coefficients.size(), mip::negate(), - op_problem.handle_ptr->get_stream()); + op_problem.handle_ptr->get_stream().get()); } // Negate objective scaling factor and objective offset so that primal / dual stay same sign after // negating objective coeffs @@ -219,7 +219,7 @@ static bool check_transpose_validity(const rmm::device_uvector& coefficient rmm::device_scalar failed(false_v, handle_ptr->get_stream()); kernel_check_transpose_validity - <<get_stream()>>>( + <<get_stream().get()>>>( raft::device_span(coefficients.data(), coefficients.size()), raft::device_span(offsets.data(), offsets.size()), raft::device_span(variables.data(), variables.size()), @@ -366,7 +366,7 @@ static void csrsort_cusparse(rmm::device_uvector& values, auto stream = offsets.stream(); cusparseHandle_t handle; cusparseCreate(&handle); - cusparseSetStream(handle, stream); + cusparseSetStream(handle, stream.get()); i_t nnz = values.size(); i_t m = rows; @@ -411,7 +411,7 @@ static void convert_greater_to_less(mip::problem_t& problem) constexpr i_t TPB = 256; kernel_convert_greater_to_less - <<get_stream()>>>( + <<get_stream().get()>>>( raft::device_span(problem.coefficients.data(), problem.coefficients.size()), raft::device_span(problem.offsets.data(), problem.offsets.size()), raft::device_span(problem.constraint_lower_bounds.data(), diff --git a/cpp/src/mip_heuristics/relaxed_lp/lp_state.cuh b/cpp/src/mip_heuristics/relaxed_lp/lp_state.cuh index 6c9191ad3b..0463164707 100644 --- a/cpp/src/mip_heuristics/relaxed_lp/lp_state.cuh +++ b/cpp/src/mip_heuristics/relaxed_lp/lp_state.cuh @@ -19,7 +19,7 @@ class problem_t; template class lp_state_t { public: - lp_state_t(problem_t& problem, rmm::cuda_stream_view stream) + lp_state_t(problem_t& problem, cuda::stream_ref stream) : prev_primal(problem.n_variables, stream), prev_dual(problem.n_constraints, stream) { thrust::fill( @@ -47,7 +47,7 @@ class lp_state_t { lp_state_t(lp_state_t&& other) noexcept = default; lp_state_t& operator=(lp_state_t&& other) noexcept = default; - void resize(problem_t& problem, rmm::cuda_stream_view stream) + void resize(problem_t& problem, cuda::stream_ref stream) { prev_primal.resize(problem.n_variables, stream); prev_dual.resize(problem.n_constraints, stream); diff --git a/cpp/src/mip_heuristics/solution/feasibility_test.cuh b/cpp/src/mip_heuristics/solution/feasibility_test.cuh index 140603c763..6df15e1251 100644 --- a/cpp/src/mip_heuristics/solution/feasibility_test.cuh +++ b/cpp/src/mip_heuristics/solution/feasibility_test.cuh @@ -86,7 +86,7 @@ void solution_t::test_absolute_feasibility() { i_t TPB = 64; i_t n_blocks = (problem_ptr->n_constraints + TPB - 1) / TPB; - test_feasibility_kernel<<get_stream()>>>(view()); + test_feasibility_kernel<<get_stream().get()>>>(view()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); } @@ -96,7 +96,7 @@ void solution_t::test_variable_bounds(bool check_integer, i_t* is_feas i_t TPB = 64; i_t n_blocks = (problem_ptr->n_variables + TPB - 1) / TPB; test_variable_bounds_kernel - <<get_stream()>>>(view(), check_integer, is_feasible); + <<get_stream().get()>>>(view(), check_integer, is_feasible); RAFT_CHECK_CUDA(handle_ptr->get_stream()); } diff --git a/cpp/src/mip_heuristics/solution/solution.cu b/cpp/src/mip_heuristics/solution/solution.cu index 3b00fca7a8..9c96d057d5 100644 --- a/cpp/src/mip_heuristics/solution/solution.cu +++ b/cpp/src/mip_heuristics/solution/solution.cu @@ -296,7 +296,7 @@ void solution_t::compute_constraints() i_t TPB = 64; compute_constraint_values - <<n_constraints, TPB, 0, handle_ptr->get_stream()>>>(view()); + <<n_constraints, TPB, 0, handle_ptr->get_stream().get()>>>(view()); RAFT_CHECK_CUDA(handle_ptr->get_stream()); } @@ -313,11 +313,12 @@ f_t solution_t::compute_l2_residual() upper_excess.data(), problem_ptr->n_constraints, [] __device__(f_t lower, f_t upper) -> f_t { return max(abs(lower), abs(upper)); }, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); pdlp::my_l2_norm(combined_excess, l2_residual, handle_ptr); return l2_residual.value(handle_ptr->get_stream()); } diff --git a/cpp/src/mip_heuristics/solve.cu b/cpp/src/mip_heuristics/solve.cu index 162a5ba291..aa83dbf5dd 100644 --- a/cpp/src/mip_heuristics/solve.cu +++ b/cpp/src/mip_heuristics/solve.cu @@ -77,9 +77,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } template diff --git a/cpp/src/mip_heuristics/solver.cu b/cpp/src/mip_heuristics/solver.cu index f8eac0c4d8..7ae7e33677 100644 --- a/cpp/src/mip_heuristics/solver.cu +++ b/cpp/src/mip_heuristics/solver.cu @@ -44,9 +44,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } template diff --git a/cpp/src/mip_heuristics/solver_settings.cu b/cpp/src/mip_heuristics/solver_settings.cu index 8b454c949b..215d8f6916 100644 --- a/cpp/src/mip_heuristics/solver_settings.cu +++ b/cpp/src/mip_heuristics/solver_settings.cu @@ -16,7 +16,7 @@ namespace cuopt::mathematical_optimization { template void mip_solver_settings_t::add_initial_solution(const f_t* initial_solution, i_t size, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { cuopt_expects( initial_solution != nullptr, error_type_t::ValidationError, "initial_solution cannot be null"); diff --git a/cpp/src/mip_heuristics/solver_solution.cu b/cpp/src/mip_heuristics/solver_solution.cu index 1997d684dc..554c767cf8 100644 --- a/cpp/src/mip_heuristics/solver_solution.cu +++ b/cpp/src/mip_heuristics/solver_solution.cu @@ -46,7 +46,7 @@ mip_solution_t::mip_solution_t(rmm::device_uvector solution, template mip_solution_t::mip_solution_t(mip_termination_status_t termination_status, solver_stats_t stats, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) : solution_(0, stream_view), objective_(0), mip_gap_(0), @@ -61,7 +61,7 @@ mip_solution_t::mip_solution_t(mip_termination_status_t termination_st template mip_solution_t::mip_solution_t(const cuopt::logic_error& error_status, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) : solution_(0, stream_view), objective_(0), mip_gap_(0), @@ -202,7 +202,7 @@ const std::vector>& mip_solution_t::get_solut template void mip_solution_t::write_to_sol_file(std::string_view filename, - rmm::cuda_stream_view stream_view) const + cuda::stream_ref stream_view) const { std::string status = get_termination_status_string(); // Override for no termination @@ -215,8 +215,8 @@ void mip_solution_t::write_to_sol_file(std::string_view filename, auto& var_names = get_variable_names(); std::vector solution; solution.resize(solution_.size()); - raft::copy(solution.data(), solution_.data(), solution_.size(), stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + raft::copy(solution.data(), solution_.data(), solution_.size(), stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); solution_writer_t::write_solution_to_sol_file( std::string(filename), status, objective_value, var_names, solution); diff --git a/cpp/src/mip_heuristics/utils.cuh b/cpp/src/mip_heuristics/utils.cuh index faf4718a5e..2401dfba2e 100644 --- a/cpp/src/mip_heuristics/utils.cuh +++ b/cpp/src/mip_heuristics/utils.cuh @@ -31,7 +31,7 @@ constexpr int default_int_lower = std::numeric_limits::min(); constexpr double zero_bound = 0.; template -inline uint32_t compute_hash(raft::device_span values, rmm::cuda_stream_view stream) +inline uint32_t compute_hash(raft::device_span values, cuda::stream_ref stream) { auto h_contents = cuopt::host_copy(values, stream); RAFT_CHECK_CUDA(stream); @@ -39,7 +39,7 @@ inline uint32_t compute_hash(raft::device_span values, rmm::cuda_stream_vie } template -inline uint32_t compute_hash(const rmm::device_uvector& values, rmm::cuda_stream_view stream) +inline uint32_t compute_hash(const rmm::device_uvector& values, cuda::stream_ref stream) { auto h_contents = cuopt::host_copy(values, stream); RAFT_CHECK_CUDA(stream); @@ -261,7 +261,7 @@ f_t compute_objective_from_vec(const rmm::device_uvector& assignment, template f_t compute_objective_from_vec(const rmm::device_uvector& assignment, const rmm::device_uvector& objective_coefficients, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { cuopt_assert(assignment.size() == objective_coefficients.size(), "Size mismatch!"); f_t computed_obj = thrust::inner_product(rmm::exec_policy(stream), @@ -331,9 +331,9 @@ static __global__ void run_lambda_kernel(F f) // run a printf statement from the device side, useful for debugging without having to deal with // explicit memcpys template -static void inline run_device_lambda(const rmm::cuda_stream_view& stream, Func f) +static void inline run_device_lambda(const cuda::stream_ref& stream, Func f) { - run_lambda_kernel<<<1, 1, 0, stream.value()>>>(f); + run_lambda_kernel<<<1, 1, 0, stream.get()>>>(f); } template diff --git a/cpp/src/pdlp/cpu_pdlp_warm_start_data.cu b/cpp/src/pdlp/cpu_pdlp_warm_start_data.cu index f2af28ba51..8a8c0987ed 100644 --- a/cpp/src/pdlp/cpu_pdlp_warm_start_data.cu +++ b/cpp/src/pdlp/cpu_pdlp_warm_start_data.cu @@ -16,33 +16,33 @@ namespace cuopt::mathematical_optimization { // Helper to copy device_uvector to std::vector (D2H) template std::vector device_to_host_vector(const rmm::device_uvector& device_vec, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (device_vec.size() == 0) return std::vector(); std::vector host_vec(device_vec.size()); raft::copy(host_vec.data(), device_vec.data(), device_vec.size(), stream); - stream.synchronize(); + stream.sync(); return host_vec; } // Helper to copy std::vector to device_uvector (H2D) template rmm::device_uvector host_to_device_vector(const std::vector& host_vec, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (host_vec.empty()) return rmm::device_uvector(0, stream); rmm::device_uvector device_vec(host_vec.size(), stream); raft::copy(device_vec.data(), host_vec.data(), host_vec.size(), stream); - stream.synchronize(); + stream.sync(); return device_vec; } // Convert GPU → CPU warmstart (D2H copy) template cpu_pdlp_warm_start_data_t convert_to_cpu_warmstart( - const pdlp_warm_start_data_t& gpu_data, rmm::cuda_stream_view stream) + const pdlp_warm_start_data_t& gpu_data, cuda::stream_ref stream) { cpu_pdlp_warm_start_data_t cpu_data; @@ -77,7 +77,7 @@ cpu_pdlp_warm_start_data_t convert_to_cpu_warmstart( // Convert CPU → GPU warmstart (H2D copy) template pdlp_warm_start_data_t convert_to_gpu_warmstart( - const cpu_pdlp_warm_start_data_t& cpu_data, rmm::cuda_stream_view stream) + const cpu_pdlp_warm_start_data_t& cpu_data, cuda::stream_ref stream) { pdlp_warm_start_data_t gpu_data; @@ -111,17 +111,17 @@ pdlp_warm_start_data_t convert_to_gpu_warmstart( #if MIP_INSTANTIATE_DOUBLE template CUOPT_EXPORT cpu_pdlp_warm_start_data_t convert_to_cpu_warmstart( - const pdlp_warm_start_data_t&, rmm::cuda_stream_view); + const pdlp_warm_start_data_t&, cuda::stream_ref); template CUOPT_EXPORT pdlp_warm_start_data_t convert_to_gpu_warmstart( - const cpu_pdlp_warm_start_data_t&, rmm::cuda_stream_view); + const cpu_pdlp_warm_start_data_t&, cuda::stream_ref); #endif #if MIP_INSTANTIATE_FLOAT || PDLP_INSTANTIATE_FLOAT template CUOPT_EXPORT cpu_pdlp_warm_start_data_t convert_to_cpu_warmstart( - const pdlp_warm_start_data_t&, rmm::cuda_stream_view); + const pdlp_warm_start_data_t&, cuda::stream_ref); template CUOPT_EXPORT pdlp_warm_start_data_t convert_to_gpu_warmstart( - const cpu_pdlp_warm_start_data_t&, rmm::cuda_stream_view); + const cpu_pdlp_warm_start_data_t&, cuda::stream_ref); #endif } // namespace cuopt::mathematical_optimization diff --git a/cpp/src/pdlp/cuopt_c_internal.hpp b/cpp/src/pdlp/cuopt_c_internal.hpp index 338e998df2..85ae11a659 100644 --- a/cpp/src/pdlp/cuopt_c_internal.hpp +++ b/cpp/src/pdlp/cuopt_c_internal.hpp @@ -29,8 +29,7 @@ struct problem_and_stream_view_t { { if (mem_backend == memory_backend_t::GPU) { // Use RAII locals so partial allocations are cleaned up if a later new throws - std::unique_ptr sv( - new rmm::cuda_stream_view(rmm::cuda_stream_per_thread)); + std::unique_ptr sv(new cuda::stream_ref(rmm::cuda_stream_per_thread)); std::unique_ptr h(new raft::handle_t(*sv)); std::unique_ptr> gp( new optimization_problem_t(h.get())); @@ -115,9 +114,8 @@ struct problem_and_stream_view_t { memory_backend_t memory_backend; optimization_problem_t* gpu_problem; cpu_optimization_problem_t* cpu_problem; - rmm::cuda_stream_view* - stream_view_ptr; // nullptr for CPU memory backend to avoid CUDA initialization - raft::handle_t* handle_ptr; // nullptr for CPU memory backend to avoid CUDA initialization + cuda::stream_ref* stream_view_ptr; // nullptr for CPU memory backend to avoid CUDA initialization + raft::handle_t* handle_ptr; // nullptr for CPU memory backend to avoid CUDA initialization }; struct solution_and_stream_view_t { diff --git a/cpp/src/pdlp/cusparse_view.cu b/cpp/src/pdlp/cusparse_view.cu index 9d3a0cc67c..83e7412354 100644 --- a/cpp/src/pdlp/cusparse_view.cu +++ b/cpp/src/pdlp/cusparse_view.cu @@ -622,7 +622,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_non_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose.resize(buffer_size_non_transpose, handle_ptr->get_stream()); size_t buffer_size_transpose = 0; @@ -636,7 +636,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose.resize(buffer_size_transpose, handle_ptr->get_stream()); @@ -653,7 +653,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, &buffer_size_transpose_batch, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose_batch.resize(buffer_size_transpose_batch, handle_ptr->get_stream()); size_t buffer_size_non_transpose_batch = 0; @@ -668,7 +668,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, &buffer_size_non_transpose_batch, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose_batch.resize(buffer_size_non_transpose_batch, handle_ptr->get_stream()); // In row row the buffer size may be different @@ -685,7 +685,7 @@ cusparse_view_t::cusparse_view_t( batch_current_AtYs, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &buffer_size_transpose_batch_row_row, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose_batch_row_row_.resize(buffer_size_transpose_batch_row_row, handle_ptr->get_stream()); size_t buffer_size_non_transpose_batch_row_row = 0; @@ -700,7 +700,7 @@ cusparse_view_t::cusparse_view_t( batch_dual_gradients, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &buffer_size_non_transpose_batch_row_row, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose_batch_row_row_.resize(buffer_size_non_transpose_batch_row_row, handle_ptr->get_stream()); } @@ -715,7 +715,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, buffer_non_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmv_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -726,7 +726,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, buffer_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmm_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -737,7 +737,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, buffer_transpose_batch.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmm_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -749,7 +749,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, buffer_non_transpose_batch.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); if (batch_mode_) { my_cusparsespmm_preprocess( handle_ptr_->get_cusparse_handle(), @@ -762,7 +762,7 @@ cusparse_view_t::cusparse_view_t( batch_current_AtYs, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, buffer_transpose_batch_row_row_.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmm_preprocess( handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -774,7 +774,7 @@ cusparse_view_t::cusparse_view_t( batch_dual_gradients, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, buffer_non_transpose_batch_row_row_.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); } #endif @@ -789,13 +789,13 @@ cusparse_view_t::cusparse_view_t( A_float_.data(), op_problem_scaled.nnz, double_to_float_functor{}, - handle_ptr->get_stream().value())); + handle_ptr->get_stream().get())); RAFT_CUDA_TRY(cub::DeviceTransform::Transform(A_T_.data(), A_T_float_.data(), op_problem_scaled.nnz, double_to_float_functor{}, - handle_ptr->get_stream().value())); + handle_ptr->get_stream().get())); A_mixed_.create(op_problem_scaled.n_constraints, op_problem_scaled.n_variables, @@ -823,7 +823,7 @@ cusparse_view_t::cusparse_view_t( beta_d.data(), dual_solution, CUSPARSE_SPMV_CSR_ALG2, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); buffer_non_transpose_mixed_.resize(buffer_size_non_transpose_mixed, handle_ptr->get_stream()); size_t buffer_size_transpose_mixed = @@ -835,7 +835,7 @@ cusparse_view_t::cusparse_view_t( beta_d.data(), c, CUSPARSE_SPMV_CSR_ALG2, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); buffer_transpose_mixed_.resize(buffer_size_transpose_mixed, handle_ptr->get_stream()); #if CUDA_VER_12_4_UP @@ -848,7 +848,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, buffer_non_transpose_mixed_.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); mixed_precision_spmv_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -859,7 +859,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, buffer_transpose_mixed_.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); #endif } } @@ -919,8 +919,9 @@ cusparse_view_t::cusparse_view_t( std::cout << "PDLP cusparse view init" << std::endl; #endif - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr_->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr_->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); // setup cusparse view A.create(op_problem.n_constraints, @@ -988,7 +989,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_non_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose.resize(buffer_size_non_transpose, handle_ptr->get_stream()); size_t buffer_size_transpose = 0; @@ -1002,7 +1003,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose.resize(buffer_size_transpose, handle_ptr->get_stream()); @@ -1019,7 +1020,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, &buffer_size_transpose_batch, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose_batch.resize(buffer_size_transpose_batch, handle_ptr->get_stream()); size_t buffer_size_non_transpose_batch = 0; RAFT_CUSPARSE_TRY( @@ -1033,7 +1034,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, &buffer_size_non_transpose_batch, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose_batch.resize(buffer_size_non_transpose_batch, handle_ptr->get_stream()); } @@ -1047,7 +1048,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, buffer_non_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmv_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -1058,7 +1059,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, buffer_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); if (batch_mode_) { my_cusparsespmm_preprocess(handle_ptr_->get_cusparse_handle(), @@ -1071,7 +1072,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, buffer_non_transpose_batch.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmm_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -1083,7 +1084,7 @@ cusparse_view_t::cusparse_view_t( batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, buffer_transpose_batch.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); } #endif } @@ -1133,8 +1134,9 @@ cusparse_view_t::cusparse_view_t( std::cout << "Restart Strategy cusparse view init" << std::endl; #endif - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr_->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr_->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); // Need to reinstanciate the cuSparse views // Copying them from the existing cuSparse view is a bad practice and creates segfault post @@ -1175,7 +1177,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_non_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_non_transpose.resize(buffer_size_non_transpose, handle_ptr->get_stream()); size_t buffer_size_transpose = 0; @@ -1189,7 +1191,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, &buffer_size_transpose, - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); buffer_transpose.resize(buffer_size_transpose, handle_ptr->get_stream()); @@ -1203,7 +1205,7 @@ cusparse_view_t::cusparse_view_t( dual_solution, CUSPARSE_SPMV_CSR_ALG2, buffer_non_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); my_cusparsespmv_preprocess(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -1214,7 +1216,7 @@ cusparse_view_t::cusparse_view_t( c, CUSPARSE_SPMV_CSR_ALG2, buffer_transpose.data(), - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); #endif } @@ -1260,15 +1262,15 @@ void cusparse_view_t::update_mixed_precision_matrices() A_float_.data(), A_.size(), double_to_float_functor{}, - handle_ptr_->get_stream().value())); + handle_ptr_->get_stream().get())); RAFT_CUDA_TRY(cub::DeviceTransform::Transform(A_T_.data(), A_T_float_.data(), A_T_.size(), double_to_float_functor{}, - handle_ptr_->get_stream().value())); + handle_ptr_->get_stream().get())); - handle_ptr_->get_stream().synchronize(); + handle_ptr_->get_stream().sync(); } } @@ -1391,7 +1393,8 @@ void cusparse_view_t::create_spmv_op_plans(bool is_reflected) if (!is_cusparse_runtime_spmvop_supported() || !(std::is_same_v)) { return; } static const auto buffer_size = dynamic_load_runtime::function("cusparseSpMVOp_bufferSize"); - CUSPARSE_CHECK(cusparseSetStream(handle_ptr_->get_cusparse_handle(), handle_ptr_->get_stream())); + CUSPARSE_CHECK( + cusparseSetStream(handle_ptr_->get_cusparse_handle(), handle_ptr_->get_stream().get())); // Prepare buffers for At_y SpMVOp size_t buffer_size_transpose = 0; RAFT_CUSPARSE_TRY((*buffer_size)(handle_ptr_->get_cusparse_handle(), diff --git a/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.cu b/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.cu index 5cf5b2af46..811282008f 100644 --- a/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.cu +++ b/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.cu @@ -111,15 +111,14 @@ void multi_gpu_engine_t::synchronize_shards() } template -void multi_gpu_engine_t::graph_capture_fork_to_shards(rmm::cuda_stream_view master_stream) +void multi_gpu_engine_t::graph_capture_fork_to_shards(cuda::stream_ref master_stream) { graph_master_ready_event_->record(master_stream); for_each_shard([&](auto& s) { graph_master_ready_event_->stream_wait(s.stream.view()); }); } template -void multi_gpu_engine_t::graph_capture_join_from_shards( - rmm::cuda_stream_view master_stream) +void multi_gpu_engine_t::graph_capture_join_from_shards(cuda::stream_ref master_stream) { for_each_shard([&](auto& s, int r) { graph_shard_ready_events_[r]->record(s.stream.view()); }); for (auto& e : graph_shard_ready_events_) { @@ -128,14 +127,14 @@ void multi_gpu_engine_t::graph_capture_join_from_shards( } template -void multi_gpu_engine_t::sync_await_master(rmm::cuda_stream_view master_stream) +void multi_gpu_engine_t::sync_await_master(cuda::stream_ref master_stream) { sync_master_ready_event_->record(master_stream); for_each_shard([&](auto& s) { sync_master_ready_event_->stream_wait(s.stream.view()); }); } template -void multi_gpu_engine_t::sync_await_shards(rmm::cuda_stream_view master_stream) +void multi_gpu_engine_t::sync_await_shards(cuda::stream_ref master_stream) { for_each_shard([&](auto& s, int r) { sync_shard_ready_events_[r]->record(s.stream.view()); }); for (auto& e : sync_shard_ready_events_) { diff --git a/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.hpp b/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.hpp index f5bd95b832..1052b79908 100644 --- a/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.hpp +++ b/cpp/src/pdlp/distributed_pdlp/multi_gpu_engine.hpp @@ -536,18 +536,18 @@ struct multi_gpu_engine_t { std::vector> sync_shard_ready_events_; // Forks master stream to shards, so that the captured graph can see the work on the shards - void graph_capture_fork_to_shards(rmm::cuda_stream_view master_stream); + void graph_capture_fork_to_shards(cuda::stream_ref master_stream); // Joins shards back to master stream for correct graph capture - void graph_capture_join_from_shards(rmm::cuda_stream_view master_stream); + void graph_capture_join_from_shards(cuda::stream_ref master_stream); // Functionnaly same as graph_capture_fork_to_shards but on a different event to avoid race // conditions Can be used as a way to sync shards with master stream - void sync_await_master(rmm::cuda_stream_view master_stream); + void sync_await_master(cuda::stream_ref master_stream); // Same as sync_await_master // Can be used as a way to sync master stream with shards - void sync_await_shards(rmm::cuda_stream_view master_stream); + void sync_await_shards(cuda::stream_ref master_stream); }; } // namespace cuopt::mathematical_optimization::pdlp diff --git a/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cu b/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cu index 5925ec9aea..4f7b5ca209 100644 --- a/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cu +++ b/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cu @@ -101,10 +101,12 @@ pdlp_initial_scaling_strategy_t::pdlp_initial_scaling_strategy_t( cuopt_assert(original_batch_size_ > 0, "Original batch size must be positive"); // start with all one for scaling vectors + RAFT_CUDA_TRY(cudaMemsetAsync(iteration_constraint_matrix_scaling_.data(), + 0.0, + sizeof(f_t) * dual_size_h_, + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_constraint_matrix_scaling_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_)); - RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_variable_scaling_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_)); + iteration_variable_scaling_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_.get())); thrust::fill(handle_ptr_->get_thrust_policy(), cummulative_constraint_matrix_scaling_.begin(), cummulative_constraint_matrix_scaling_.end(), @@ -231,10 +233,12 @@ template void pdlp_initial_scaling_strategy_t::ruiz_iter_local() { // Reset the iteration_scaling vectors to all 0 + RAFT_CUDA_TRY(cudaMemsetAsync(iteration_constraint_matrix_scaling_.data(), + 0, + sizeof(f_t) * dual_size_h_, + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_constraint_matrix_scaling_.data(), 0, sizeof(f_t) * dual_size_h_, stream_view_)); - RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_variable_scaling_.data(), 0, sizeof(f_t) * primal_size_h_, stream_view_)); + iteration_variable_scaling_.data(), 0, sizeof(f_t) * primal_size_h_, stream_view_.get())); // Inf-norm over rows and columns. Split into two kernels so the distributed path can // touch only owned entries. @@ -243,15 +247,20 @@ void pdlp_initial_scaling_strategy_t::ruiz_iter_local() i_t number_of_blocks = op_problem_scaled_.n_constraints / block_size; if (op_problem_scaled_.n_constraints % block_size) number_of_blocks++; i_t number_of_threads = std::min(op_problem_scaled_.n_variables, (i_t)block_size); - inf_norm_row_kernel<<>>( + inf_norm_row_kernel<<>>( op_problem_scaled_.view(), this->view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); i_t number_of_blocks_col = op_problem_scaled_.n_variables / block_size; if (op_problem_scaled_.n_variables % block_size) number_of_blocks_col++; i_t number_of_threads_col = std::min(op_problem_scaled_.n_constraints, (i_t)block_size); - inf_norm_col_kernel<<>>( - op_problem_scaled_.view(), this->view(), A_T_.data(), A_T_offsets_.data(), A_T_indices_.data()); + inf_norm_col_kernel + <<>>( + op_problem_scaled_.view(), + this->view(), + A_T_.data(), + A_T_offsets_.data(), + A_T_indices_.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); if (running_mip_) { reset_integer_variables(); } @@ -263,14 +272,14 @@ void pdlp_initial_scaling_strategy_t::ruiz_iter_local() iteration_constraint_matrix_scaling_.data(), dual_size_h_, a_divides_sqrt_b_bounded(), - stream_view_); + stream_view_.get()); raft::linalg::binaryOp(cummulative_variable_scaling_.data(), cummulative_variable_scaling_.data(), iteration_variable_scaling_.data(), primal_size_h_, a_divides_sqrt_b_bounded(), - stream_view_); + stream_view_.get()); } template @@ -378,10 +387,12 @@ template void pdlp_initial_scaling_strategy_t::pock_chambolle_scaling(f_t alpha) { // Reset the iteration_scaling vectors to all 0 + RAFT_CUDA_TRY(cudaMemsetAsync(iteration_constraint_matrix_scaling_.data(), + 0.0, + sizeof(f_t) * dual_size_h_, + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_constraint_matrix_scaling_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_)); - RAFT_CUDA_TRY(cudaMemsetAsync( - iteration_variable_scaling_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_)); + iteration_variable_scaling_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_.get())); EXE_CUOPT_EXPECTS( alpha >= 0.0 && alpha <= 2.0, @@ -396,13 +407,13 @@ void pdlp_initial_scaling_strategy_t::pock_chambolle_scaling(f_t alpha constexpr i_t number_of_threads = 128; pock_chambolle_scaling_kernel_row - <<>>( + <<>>( op_problem_scaled_.view(), alpha, this->view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Use transposed matrix instead to compute column-wise more easily pock_chambolle_scaling_kernel_col - <<>>( + <<>>( op_problem_scaled_.view(), alpha, this->view(), @@ -420,13 +431,13 @@ void pdlp_initial_scaling_strategy_t::pock_chambolle_scaling(f_t alpha iteration_constraint_matrix_scaling_.data(), dual_size_h_, a_divides_sqrt_b_bounded(), - stream_view_); + stream_view_.get()); raft::linalg::binaryOp(cummulative_variable_scaling_.data(), cummulative_variable_scaling_.data(), iteration_variable_scaling_.data(), primal_size_h_, a_divides_sqrt_b_bounded(), - stream_view_); + stream_view_.get()); } template @@ -516,10 +527,10 @@ void pdlp_initial_scaling_strategy_t::swap_context( const auto [grid_size, block_size] = kernel_config_from_batch_size(static_cast(swap_pairs.size())); scaling_swap_rescaling_kernel - <<>>(thrust::raw_pointer_cast(swap_pairs.data()), - static_cast(swap_pairs.size()), - make_span(bound_rescaling_), - make_span(objective_rescaling_)); + <<>>(thrust::raw_pointer_cast(swap_pairs.data()), + static_cast(swap_pairs.size()), + make_span(bound_rescaling_), + make_span(objective_rescaling_)); RAFT_CUDA_TRY(cudaPeekAtLastError()); for (const auto& pair : swap_pairs) { @@ -553,7 +564,7 @@ void pdlp_initial_scaling_strategy_t::apply_cummulative_scaling_to_pro i_t number_of_blocks = op_problem_scaled_.n_constraints / block_size; if (op_problem_scaled_.n_constraints % block_size) number_of_blocks++; i_t number_of_threads = std::min(op_problem_scaled_.n_variables, block_size); - scale_problem_kernel<<>>( + scale_problem_kernel<<>>( this->view(), op_problem_scaled_.view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -563,7 +574,7 @@ void pdlp_initial_scaling_strategy_t::apply_cummulative_scaling_to_pro i_t number_of_threads_transposed = std::min(op_problem_scaled_.n_constraints, block_size); scale_transposed_problem_kernel - <<>>( + <<>>( this->view(), A_T_.data(), A_T_offsets_.data(), A_T_indices_.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -583,7 +594,7 @@ void pdlp_initial_scaling_strategy_t::apply_cummulative_scaling_to_pro op_problem_scaled_.variable_bounds.data(), op_problem_scaled_.variable_bounds.size(), divide_check_zero(), - stream_view_.value()); + stream_view_.get()); if (pdhg_solver_ptr_ && pdhg_solver_ptr_->get_new_bounds_idx().size() != 0) { cub::DeviceTransform::Transform( @@ -662,7 +673,7 @@ void pdlp_initial_scaling_strategy_t::apply_bound_objective_rescaling_ f_t bound_rescaling) -> thrust::tuple { return {constraint_lower_bound * bound_rescaling, constraint_upper_bound * bound_rescaling}; }, - stream_view_.value()); + stream_view_.get()); // In batch mode we don't scale the variable bounds (here) because they are shared across // climbers. While the variable bounds are the same across climbers, there can be different @@ -688,7 +699,7 @@ void pdlp_initial_scaling_strategy_t::apply_bound_objective_rescaling_ op_problem_scaled_.objective_coefficients.data(), op_problem_scaled_.objective_coefficients.size(), cuda::std::multiplies{}, - stream_view_.value()); + stream_view_.get()); } template diff --git a/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cuh b/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cuh index df67024630..5a1e247552 100644 --- a/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cuh +++ b/cpp/src/pdlp/initial_scaling_strategy/initial_scaling.cuh @@ -146,7 +146,7 @@ class pdlp_initial_scaling_strategy_t { void reset_integer_variables(); raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; i_t primal_size_h_; i_t dual_size_h_; diff --git a/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu b/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu index 8c8cdce0b4..ba6eecdd76 100644 --- a/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu +++ b/cpp/src/pdlp/optimal_batch_size_handler/optimal_batch_size_handler.cu @@ -62,7 +62,7 @@ struct SpMM_benchmarks_context_t { y_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &buffer_size_non_transpose_batch, - stream_view)); + stream_view.get())); size_t buffer_size_transpose_batch = 0; RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsespmm_bufferSize( @@ -76,7 +76,7 @@ struct SpMM_benchmarks_context_t { x_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &buffer_size_transpose_batch, - stream_view)); + stream_view.get())); buffer_transpose_batch = rmm::device_buffer(buffer_size_transpose_batch, stream_view); buffer_non_transpose_batch = rmm::device_buffer(buffer_size_non_transpose_batch, stream_view); @@ -94,7 +94,7 @@ struct SpMM_benchmarks_context_t { x_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, buffer_transpose_batch.data(), - stream_view); + stream_view.get()); my_cusparsespmm_preprocess( handle_ptr->get_cusparse_handle(), @@ -107,7 +107,7 @@ struct SpMM_benchmarks_context_t { y_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, buffer_non_transpose_batch.data(), - stream_view); + stream_view.get()); #endif // First empty run for warm up @@ -129,7 +129,7 @@ struct SpMM_benchmarks_context_t { y_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, (f_t*)buffer_non_transpose_batch.data(), - stream_view)); + stream_view.get())); RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsespmm( handle_ptr->get_cusparse_handle(), @@ -142,7 +142,7 @@ struct SpMM_benchmarks_context_t { x_descr, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, (f_t*)buffer_transpose_batch.data(), - stream_view)); + stream_view.get())); } cusparse_dn_mat_descr_wrapper_t x_descr; @@ -169,7 +169,7 @@ static double evaluate_node(cusparse_sp_mat_descr_wrapper_t& A, { cuopt_assert(current_batch_size > 0, "Current batch size must be greater than 0"); - rmm::cuda_stream_view stream_view = handle_ptr->get_stream(); + cuda::stream_ref stream_view = handle_ptr->get_stream(); SpMM_benchmarks_context_t spmm_benchmarks_context( A, A_T, primal_size, dual_size, current_batch_size, handle_ptr); @@ -219,7 +219,7 @@ int optimal_batch_size_handler(const optimization_problem_t& op_proble std::pow(2, std::floor(std::log2(std::min(initial_batch_size, max_batch_size)))); int optimal_batch_size = current_batch_size; double best_ratio; - rmm::cuda_stream_view stream_view = op_problem.get_handle_ptr()->get_stream(); + cuda::stream_ref stream_view = op_problem.get_handle_ptr()->get_stream(); mip::problem_t problem(op_problem); @@ -244,7 +244,7 @@ int optimal_batch_size_handler(const optimization_problem_t& op_proble problem.reverse_coefficients.data()); // Sync before starting anything to make sure everything is done - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); // Evaluate current, left and right nodes to pick a direction diff --git a/cpp/src/pdlp/optimization_problem.cu b/cpp/src/pdlp/optimization_problem.cu index 95457e2556..9a4770d8cb 100644 --- a/cpp/src/pdlp/optimization_problem.cu +++ b/cpp/src/pdlp/optimization_problem.cu @@ -57,7 +57,7 @@ namespace cuopt::mathematical_optimization { template optimization_problem_t::optimization_problem_t(raft::handle_t const* handle_ptr) : handle_ptr_(handle_ptr), - stream_view_(handle_ptr != nullptr ? handle_ptr->get_stream() : rmm::cuda_stream_view{}), + stream_view_(handle_ptr != nullptr ? handle_ptr->get_stream() : cuda::stream_ref{}), A_(0, stream_view_), A_indices_(0, stream_view_), A_offsets_(0, stream_view_), @@ -1023,7 +1023,7 @@ static bool csr_matrices_equivalent_with_permutation(const rmm::device_uvector& d_row_perm_inv, const rmm::device_uvector& d_col_perm_inv, i_t n_cols, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { const i_t nnz = static_cast(this_values.size()); if (nnz != static_cast(other_values.size())) { return false; } @@ -1538,25 +1538,25 @@ struct cast_op { }; template -rmm::device_uvector gpu_cast(const rmm::device_uvector& src, rmm::cuda_stream_view stream) +rmm::device_uvector gpu_cast(const rmm::device_uvector& src, cuda::stream_ref stream) { rmm::device_uvector dst(src.size(), stream); if (src.size() > 0) { RAFT_CUDA_TRY(cub::DeviceTransform::Transform( - src.data(), dst.data(), src.size(), cast_op{}, stream.value())); + src.data(), dst.data(), src.size(), cast_op{}, stream.get())); } return dst; } template rmm::device_uvector gpu_cast(const rmm::device_uvector&, - rmm::cuda_stream_view); + cuda::stream_ref); template rmm::device_uvector gpu_cast(const rmm::device_uvector&, - rmm::cuda_stream_view); + cuda::stream_ref); template template optimization_problem_t optimization_problem_t::convert_to_other_prec( - rmm::cuda_stream_view stream) const + cuda::stream_ref stream) const { optimization_problem_t other(handle_ptr_); @@ -1577,43 +1577,43 @@ optimization_problem_t optimization_problem_t::convert static_cast(A_indices_.size()), A_offsets_.data(), static_cast(A_offsets_.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (c_.size() > 0) { auto other_c = gpu_cast(c_, stream); other.set_objective_coefficients(other_c.data(), static_cast(other_c.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (b_.size() > 0) { auto other_b = gpu_cast(b_, stream); other.set_constraint_bounds(other_b.data(), static_cast(other_b.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (constraint_lower_bounds_.size() > 0) { auto other_clb = gpu_cast(constraint_lower_bounds_, stream); other.set_constraint_lower_bounds(other_clb.data(), static_cast(other_clb.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (constraint_upper_bounds_.size() > 0) { auto other_cub = gpu_cast(constraint_upper_bounds_, stream); other.set_constraint_upper_bounds(other_cub.data(), static_cast(other_cub.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (variable_lower_bounds_.size() > 0) { auto other_vlb = gpu_cast(variable_lower_bounds_, stream); other.set_variable_lower_bounds(other_vlb.data(), static_cast(other_vlb.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (variable_upper_bounds_.size() > 0) { auto other_vub = gpu_cast(variable_upper_bounds_, stream); other.set_variable_upper_bounds(other_vub.data(), static_cast(other_vub.size())); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } if (variable_types_.size() > 0) { @@ -1641,8 +1641,7 @@ template class CUOPT_EXPORT optimization_problem_t; #if PDLP_INSTANTIATE_FLOAT || MIP_INSTANTIATE_FLOAT template CUOPT_EXPORT optimization_problem_t - optimization_problem_t::convert_to_other_prec( - rmm::cuda_stream_view) const; + optimization_problem_t::convert_to_other_prec(cuda::stream_ref) const; #endif } // namespace cuopt::mathematical_optimization diff --git a/cpp/src/pdlp/pdhg.cu b/cpp/src/pdlp/pdhg.cu index 81a5a2f0e5..550f50a42e 100644 --- a/cpp/src/pdlp/pdhg.cu +++ b/cpp/src/pdlp/pdhg.cu @@ -172,7 +172,7 @@ new_bounds_groups_t copy_new_bounds_to_groups( const rmm::device_uvector& new_bounds_lower, const rmm::device_uvector& new_bounds_upper, i_t batch_size, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { cuopt_assert(new_bounds_climber_id.size() == new_bounds_idx.size(), "New bounds climber id and index sizes must match"); @@ -191,7 +191,7 @@ new_bounds_groups_t copy_new_bounds_to_groups( raft::copy(h_idx.data(), new_bounds_idx.data(), n_entries, stream_view); raft::copy(h_lower.data(), new_bounds_lower.data(), n_entries, stream_view); raft::copy(h_upper.data(), new_bounds_upper.data(), n_entries, stream_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } new_bounds_groups_t groups(batch_size); @@ -210,7 +210,7 @@ void copy_groups_to_new_bounds(const new_bounds_groups_t& groups, rmm::device_uvector& new_bounds_idx, rmm::device_uvector& new_bounds_lower, rmm::device_uvector& new_bounds_upper, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { size_t n_entries = 0; for (i_t c = 0; c < group_count; ++c) { @@ -411,7 +411,7 @@ void pdhg_solver_t::compute_next_dual_solution(rmm::device_uvector::compute_next_dual_solution(rmm::device_uvector::compute_next_dual_solution(rmm::device_uvector(dual_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -460,7 +460,7 @@ void pdhg_solver_t::spmvop_At_y() cusparse_view_.dual_solution, cusparse_view_.current_AtY, cusparse_view_.current_AtY, - stream_view_.value()); + stream_view_.get()); return; } #endif @@ -473,7 +473,7 @@ void pdhg_solver_t::spmvop_At_y() cusparse_view_.current_AtY, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); } template @@ -488,7 +488,7 @@ void pdhg_solver_t::spmvop_A_x() cusparse_view_.reflected_primal_solution, cusparse_view_.dual_gradient, cusparse_view_.dual_gradient, - stream_view_.value()); + stream_view_.get()); return; } #endif @@ -502,7 +502,7 @@ void pdhg_solver_t::spmvop_A_x() cusparse_view_.dual_gradient, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); } template @@ -529,7 +529,7 @@ void pdhg_solver_t::compute_At_y() cusparse_view_.current_AtY, CUSPARSE_SPMV_CSR_ALG2, cusparse_view_.buffer_transpose_mixed_.data(), - stream_view_); + stream_view_.get()); } else { spmvop_At_y(); } @@ -544,7 +544,7 @@ void pdhg_solver_t::compute_At_y() cusparse_view_.current_AtY, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); } } else { RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsespmm( @@ -558,7 +558,7 @@ void pdhg_solver_t::compute_At_y() cusparse_view_.batch_current_AtYs, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, (f_t*)cusparse_view_.buffer_transpose_batch_row_row_.data(), - stream_view_)); + stream_view_.get())); } } @@ -587,7 +587,7 @@ void pdhg_solver_t::compute_A_x() cusparse_view_.dual_gradient, CUSPARSE_SPMV_CSR_ALG2, cusparse_view_.buffer_non_transpose_mixed_.data(), - stream_view_); + stream_view_.get()); } else { spmvop_A_x(); } @@ -602,7 +602,7 @@ void pdhg_solver_t::compute_A_x() cusparse_view_.dual_gradient, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); } } else { RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsespmm( @@ -616,7 +616,7 @@ void pdhg_solver_t::compute_A_x() cusparse_view_.batch_dual_gradients, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, (f_t*)cusparse_view_.buffer_non_transpose_batch_row_row_.data(), - stream_view_)); + stream_view_.get())); } } @@ -636,7 +636,7 @@ void pdhg_solver_t::spmv_At_into(cusparseDnVecDescr_t in_desc, out_desc, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); } // out_desc = A @ in_desc, the counterpart of spmv_At_into on this shard's local A. @@ -654,7 +654,7 @@ void pdhg_solver_t::spmv_A_into(cusparseDnVecDescr_t in_desc, out_desc, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); } template @@ -678,7 +678,7 @@ void pdhg_solver_t::compute_primal_projection_with_gradient( tmp_primal_.data()), primal_size_h_, primal_projection(primal_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -764,7 +764,7 @@ void pdhg_solver_t::primal_reflected_major_projection_transform( potential_next_primal_solution_.data(), dual_slack_.data(), reflected_primal_.data()), primal_size_h_, primal_reflected_major_projection(primal_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -807,7 +807,7 @@ void pdhg_solver_t::primal_reflected_projection_transform( reflected_primal_.data(), primal_size_h_, primal_reflected_projection(primal_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -851,7 +851,7 @@ void pdhg_solver_t::dual_reflected_major_projection_transform( thrust::make_zip_iterator(potential_next_dual_solution_.data(), reflected_dual_.data()), dual_size_h_, dual_reflected_major_projection(dual_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -894,7 +894,7 @@ void pdhg_solver_t::dual_reflected_projection_transform( reflected_dual_.data(), dual_size_h_, dual_reflected_projection(dual_step_size.data()), - stream_view_.value()); + stream_view_.get()); } template @@ -1217,7 +1217,7 @@ void pdhg_solver_t::refine_initial_primal_projection( make_span(bound_rescaling), make_span(current_saddle_point_state_.get_primal_solution()), problem_ptr->n_variables}, - stream_view_.value()); + stream_view_.get()); } template @@ -1265,7 +1265,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( reflected_primal_.data(), batch_size_divisor_, problem_ptr->objective_coefficients.size() > static_cast(primal_size_h_)}, - stream_view_.value()); + stream_view_.get()); } if (new_bounds_idx_.size() != 0) { #ifdef CUPDLP_DEBUG_MODE @@ -1297,7 +1297,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( make_span(reflected_primal_), (int)climber_strategies_.size(), problem_ptr->objective_coefficients.size() > static_cast(primal_size_h_)}, - stream_view_.value()); + stream_view_.get()); } #ifdef CUPDLP_DEBUG_MODE print("potential_next_primal_solution_", potential_next_primal_solution_); @@ -1329,7 +1329,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( reflected_dual_.data(), batch_size_divisor_, problem_ptr->constraint_lower_bounds.size() > static_cast(dual_size_h_)}, - stream_view_.value()); + stream_view_.get()); } #ifdef CUPDLP_DEBUG_MODE @@ -1380,7 +1380,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( reflected_primal_.data(), (int)climber_strategies_.size(), problem_ptr->objective_coefficients.size() > static_cast(primal_size_h_)}, - stream_view_.value()); + stream_view_.get()); } if (new_bounds_idx_.size() != 0) { #ifdef CUPDLP_DEBUG_MODE @@ -1410,7 +1410,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( make_span(reflected_primal_), (int)climber_strategies_.size(), problem_ptr->objective_coefficients.size() > static_cast(primal_size_h_)}, - stream_view_.value()); + stream_view_.get()); } #ifdef CUPDLP_DEBUG_MODE print("reflected_primal_", reflected_primal_); @@ -1445,7 +1445,7 @@ void pdhg_solver_t::compute_next_primal_dual_solution_reflected( reflected_dual_.data(), (int)climber_strategies_.size(), problem_ptr->constraint_lower_bounds.size() > static_cast(dual_size_h_)}, - stream_view_.value()); + stream_view_.get()); } #ifdef CUPDLP_DEBUG_MODE print("reflected_dual_", reflected_dual_); diff --git a/cpp/src/pdlp/pdhg.hpp b/cpp/src/pdlp/pdhg.hpp index f6eb9931fa..a116ce7a66 100644 --- a/cpp/src/pdlp/pdhg.hpp +++ b/cpp/src/pdlp/pdhg.hpp @@ -138,7 +138,7 @@ class pdhg_solver_t { bool batch_mode_{false}; raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; mip::problem_t* problem_ptr; diff --git a/cpp/src/pdlp/pdlp.cu b/cpp/src/pdlp/pdlp.cu index 217ea4260a..b7f795c89c 100644 --- a/cpp/src/pdlp/pdlp.cu +++ b/cpp/src/pdlp/pdlp.cu @@ -945,7 +945,7 @@ template optimization_problem_solution_t pdlp_solver_t::finalize_batch_return() { current_termination_strategy_.fill_gpu_terms_stats(total_pdlp_iterations_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); current_termination_strategy_.convert_gpu_terms_stats_to_host( batch_solution_to_return_.get_additional_termination_informations()); return optimization_problem_solution_t{ @@ -1086,7 +1086,7 @@ pdlp_solver_t::check_batch_termination(const timer_t& timer) sb_view_.mark_solved(climber_strategies_[i].original_index); } } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); return current_termination_strategy_.fill_return_problem_solution( internal_solver_iterations_, pdhg_solver_, @@ -1484,11 +1484,11 @@ static void compute_stats(const rmm::device_uvector& vec, n, cuda::minimum<>{}, std::numeric_limits::max(), - stream)); + stream.get())); RAFT_CUDA_TRY(cub::DeviceReduce::Reduce( - d_temp, bytes_2, abs_iter, d_largest.data(), n, cuda::maximum<>{}, f_t(0), stream)); + d_temp, bytes_2, abs_iter, d_largest.data(), n, cuda::maximum<>{}, f_t(0), stream.get())); RAFT_CUDA_TRY(cub::DeviceReduce::Reduce( - d_temp, bytes_3, abs_iter, d_sum.data(), n, cuda::std::plus<>{}, f_t(0), stream)); + d_temp, bytes_3, abs_iter, d_sum.data(), n, cuda::std::plus<>{}, f_t(0), stream.get())); size_t max_bytes = std::max({bytes_1, bytes_2, bytes_3}); rmm::device_buffer temp_buf(max_bytes, stream); @@ -1500,11 +1500,23 @@ static void compute_stats(const rmm::device_uvector& vec, n, cuda::minimum<>{}, std::numeric_limits::max(), - stream)); - RAFT_CUDA_TRY(cub::DeviceReduce::Reduce( - temp_buf.data(), bytes_2, abs_iter, d_largest.data(), n, cuda::maximum<>{}, f_t(0), stream)); - RAFT_CUDA_TRY(cub::DeviceReduce::Reduce( - temp_buf.data(), bytes_3, abs_iter, d_sum.data(), n, cuda::std::plus<>{}, f_t(0), stream)); + stream.get())); + RAFT_CUDA_TRY(cub::DeviceReduce::Reduce(temp_buf.data(), + bytes_2, + abs_iter, + d_largest.data(), + n, + cuda::maximum<>{}, + f_t(0), + stream.get())); + RAFT_CUDA_TRY(cub::DeviceReduce::Reduce(temp_buf.data(), + bytes_3, + abs_iter, + d_sum.data(), + n, + cuda::std::plus<>{}, + f_t(0), + stream.get())); smallest = d_smallest.value(stream); largest = d_largest.value(stream); @@ -1628,7 +1640,7 @@ void pdlp_solver_t::update_primal_dual_solutions( RAFT_CUDA_TRY(cudaMemsetAsync(saddle.get_current_AtY().data(), f_t(0.0), sizeof(f_t) * saddle.get_current_AtY().size(), - stream_view_)); + stream_view_.get())); // Scale if should compute initial step size after scaling if (!settings_.hyper_params.compute_initial_step_size_before_scaling) { @@ -1797,13 +1809,13 @@ void pdlp_solver_t::swap_context( const auto [grid_size, block_size] = kernel_config_from_batch_size(static_cast(swap_pairs.size())); pdlp_swap_device_vectors_kernel - <<>>(thrust::raw_pointer_cast(swap_pairs.data()), - static_cast(swap_pairs.size()), - make_span(primal_weight_), - make_span(best_primal_weight_), - make_span(step_size_), - make_span(primal_step_size_), - make_span(dual_step_size_)); + <<>>(thrust::raw_pointer_cast(swap_pairs.data()), + static_cast(swap_pairs.size()), + make_span(primal_weight_), + make_span(best_primal_weight_), + make_span(step_size_), + make_span(primal_step_size_), + make_span(dual_step_size_)); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Swap unscaled problem's per-climber fields (COL-major blocks) if (problem_ptr->objective_coefficients.size() > static_cast(primal_size_h_)) { @@ -1862,7 +1874,7 @@ void pdlp_solver_t::swap_all_context( host_vector_swap(climber_strategies_, pair.left, pair.right); } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -1881,7 +1893,7 @@ void pdlp_solver_t::resize_all_context(i_t new_size) // Resize PDLP own context resize_context(new_size); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -2025,7 +2037,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_current_AtYs, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &new_buf_size, - stream_view_)); + stream_view_.get())); pdhg_cusparse_view.buffer_transpose_batch_row_row_.resize(new_buf_size, stream_view_); // PDHG row-row: A * batch_reflected_primal_solutions -> batch_dual_gradients @@ -2040,7 +2052,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_dual_gradients, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, &new_buf_size, - stream_view_)); + stream_view_.get())); pdhg_cusparse_view.buffer_non_transpose_batch_row_row_.resize(new_buf_size, stream_view_); // Adaptive step size: A_T * batch_potential_next_dual_solution -> batch_next_AtYs @@ -2055,7 +2067,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_next_AtYs, CUSPARSE_SPMM_CSR_ALG3, &new_buf_size, - stream_view_)); + stream_view_.get())); pdhg_cusparse_view.buffer_transpose_batch.resize(new_buf_size, stream_view_); // Convergence info: A_T * batch_dual_solutions -> batch_tmp_primals @@ -2070,7 +2082,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( current_op_problem_evaluation_cusparse_view_.batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, &new_buf_size, - stream_view_)); + stream_view_.get())); current_op_problem_evaluation_cusparse_view_.buffer_transpose_batch.resize(new_buf_size, stream_view_); @@ -2086,7 +2098,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( current_op_problem_evaluation_cusparse_view_.batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, &new_buf_size, - stream_view_)); + stream_view_.get())); current_op_problem_evaluation_cusparse_view_.buffer_non_transpose_batch.resize(new_buf_size, stream_view_); } @@ -2106,7 +2118,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_current_AtYs, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, pdhg_cusparse_view.buffer_transpose_batch_row_row_.data(), - stream_view_); + stream_view_.get()); my_cusparsespmm_preprocess( handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -2118,7 +2130,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_dual_gradients, (deterministic_batch_pdlp) ? CUSPARSE_SPMM_CSR_ALG3 : CUSPARSE_SPMM_CSR_ALG2, pdhg_cusparse_view.buffer_non_transpose_batch_row_row_.data(), - stream_view_); + stream_view_.get()); // Adaptive step size strategy SpMM preprocess my_cusparsespmm_preprocess(handle_ptr_->get_cusparse_handle(), @@ -2131,7 +2143,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( pdhg_cusparse_view.batch_next_AtYs, CUSPARSE_SPMM_CSR_ALG3, (f_t*)pdhg_cusparse_view.buffer_transpose_batch.data(), - stream_view_); + stream_view_.get()); // Convergence information SpMM preprocess my_cusparsespmm_preprocess( @@ -2145,7 +2157,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( current_op_problem_evaluation_cusparse_view_.batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)current_op_problem_evaluation_cusparse_view_.buffer_transpose_batch.data(), - stream_view_); + stream_view_.get()); my_cusparsespmm_preprocess( handle_ptr_->get_cusparse_handle(), @@ -2158,7 +2170,7 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( current_op_problem_evaluation_cusparse_view_.batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)current_op_problem_evaluation_cusparse_view_.buffer_non_transpose_batch.data(), - stream_view_); + stream_view_.get()); #endif // Set PDHG graphs to uninitialized so that next call can start a new graph. @@ -2168,14 +2180,14 @@ void pdlp_solver_t::resize_and_swap_all_context_loop( // graph_all_non_major (reflected non-major). pdhg_solver_.get_graph_all() = ping_pong_graph_t(stream_view_, true); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } // delta = reflected - current, for both primal and dual, written into the // saddle-point delta buffers. Shared by the single-GPU and per-shard // (distributed) paths so the two only differ by which pdhg/stream they pass. template -static void compute_primal_dual_deltas(pdhg_solver_t& pdhg, rmm::cuda_stream_view stream) +static void compute_primal_dual_deltas(pdhg_solver_t& pdhg, cuda::stream_ref stream) { cub::DeviceTransform::Transform( cuda::std::make_tuple(pdhg.get_reflected_primal().data(), pdhg.get_primal_solution().data()), @@ -2285,7 +2297,7 @@ void pdlp_solver_t::compute_fixed_error(std::vector& has_restarte } else { // Sync to make sure all previous cuSparse operations are finished before setting the // potential_next_dual_solution - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); // Make potential_next_dual_solution point towards reflected dual solution to reuse the code RAFT_CUSPARSE_TRY(cusparseDnVecSetValues(cusparse_view.potential_next_dual_solution, @@ -2301,7 +2313,7 @@ void pdlp_solver_t::compute_fixed_error(std::vector& has_restarte if (batch_mode_) { const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); - kernel_compute_fixed_error<<>>( + kernel_compute_fixed_error<<>>( make_span(step_size_strategy_.get_norm_squared_delta_primal()), make_span(step_size_strategy_.get_norm_squared_delta_dual()), make_span(primal_weight_), @@ -2309,7 +2321,7 @@ void pdlp_solver_t::compute_fixed_error(std::vector& has_restarte make_span(step_size_strategy_.get_interaction()), make_span(restart_strategy_.fixed_point_error_)); RAFT_CUDA_TRY(cudaStreamSynchronize( - stream_view_)); // To make sure all the data is written from device to host + stream_view_.get())); // To make sure all the data is written from device to host RAFT_CUDA_TRY(cudaPeekAtLastError()); #ifdef CUPDLP_DEBUG_MODE @@ -2326,7 +2338,7 @@ void pdlp_solver_t::compute_fixed_error(std::vector& has_restarte // Sync to make sure all previous cuSparse operations are finished before setting the // potential_next_dual_solution - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); // Put back, already done in multi-gpu side if (!is_distributed_master()) { @@ -2387,10 +2399,10 @@ void pdlp_solver_t::transpose_problem_fields(bool to_row) transposed.data(), *output_ld)); raft::copy(field.data(), transposed.data(), field.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); }; - RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_)); + RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_.get())); // We need to swap the scaled version because they can be dynamically resized and swapped. transpose_field(op_problem_scaled_.objective_coefficients, primal_size_h_); transpose_field(op_problem_scaled_.constraint_lower_bounds, dual_size_h_); @@ -2412,7 +2424,7 @@ void pdlp_solver_t::transpose_primal_dual_to_row( rmm::device_uvector dual_slack_transposed( is_dual_slack_empty ? 0 : primal_size_h_ * climber_strategies_.size(), stream_view_); - RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_)); + RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_.get())); CUBLAS_CHECK(cublasGeam(handle_ptr_->get_cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, @@ -2475,7 +2487,7 @@ void pdlp_solver_t::transpose_primal_dual_to_row( dual_size_h_ * climber_strategies_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -2491,7 +2503,7 @@ void pdlp_solver_t::transpose_primal_dual_back_to_col( rmm::device_uvector dual_slack_transposed( is_dual_slack_empty ? 0 : primal_size_h_ * climber_strategies_.size(), stream_view_); - RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_)); + RAFT_CUBLAS_TRY(cublasSetStream(handle_ptr_->get_cublas_handle(), stream_view_.get())); CUBLAS_CHECK(cublasGeam(handle_ptr_->get_cublas_handle(), CUBLAS_OP_T, CUBLAS_OP_N, @@ -2555,7 +2567,7 @@ void pdlp_solver_t::transpose_primal_dual_back_to_col( dual_size_h_ * climber_strategies_.size(), stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -2746,7 +2758,7 @@ optimization_problem_solution_t pdlp_solver_t::run_solver(co pdhg_solver_.get_primal_solution().data(), pdhg_solver_.get_primal_solution().size(), clamp(), - stream_view_.value()); + stream_view_.get()); } else { cub::DeviceTransform::Transform( cuda::std::make_tuple(pdhg_solver_.get_primal_solution().data(), @@ -2754,7 +2766,7 @@ optimization_problem_solution_t pdlp_solver_t::run_solver(co pdhg_solver_.get_primal_solution().data(), pdhg_solver_.get_primal_solution().size(), clamp(), - stream_view_.value()); + stream_view_.get()); } pdhg_solver_.refine_initial_primal_projection( @@ -2770,7 +2782,7 @@ optimization_problem_solution_t pdlp_solver_t::run_solver(co unscaled_primal_avg_solution_.data(), primal_size_h_, clamp(), - stream_view_.value()); + stream_view_.get()); } } @@ -3190,7 +3202,7 @@ void pdlp_solver_t::halpern_update() (f_t(1.0) - reflection_coefficient) * current_primal; return weight * reflected + (f_t(1.0) - weight) * initial_primal; }, - stream_view_.value()); + stream_view_.get()); #ifdef CUPDLP_DEBUG_MODE print("pdhg_solver_.get_reflected_dual()", pdhg_solver_.get_reflected_dual()); @@ -3214,7 +3226,7 @@ void pdlp_solver_t::halpern_update() (f_t(1.0) - reflection_coefficient) * current_dual; return weight * reflected + (f_t(1.0) - weight) * initial_dual; }, - stream_view_.value()); + stream_view_.get()); #ifdef CUPDLP_DEBUG_MODE print("halpen_update current primal", @@ -3324,7 +3336,7 @@ void pdlp_solver_t::compute_initial_step_size() op_problem_scaled_.nnz, red_op, 0.0, - stream_view_); + stream_view_.get()); // Allocate temporary storage rmm::device_buffer cub_tmp{temp_storage_bytes, stream_view_}; // Run max-reduction @@ -3335,12 +3347,12 @@ void pdlp_solver_t::compute_initial_step_size() op_problem_scaled_.nnz, red_op, 0.0, - stream_view_); + stream_view_.get()); raft::linalg::eltwiseDivideCheckZero( - step_size_.data(), step_size_.data(), abs_max_element.data(), 1, stream_view_); + step_size_.data(), step_size_.data(), abs_max_element.data(), 1, stream_view_.get()); // Sync since we are using local variable - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } else { i_t m = op_problem_scaled_.n_constraints; i_t n = op_problem_scaled_.n_variables; @@ -3383,7 +3395,7 @@ void pdlp_solver_t::compute_initial_step_size() d_q.data(), d_q.size(), divide_by_device_scalar_t{norm_q.data()}, - stream_view_.value()); + stream_view_.get()); // A_t_q = A_t @ d_q RAFT_CUSPARSE_TRY( @@ -3396,7 +3408,7 @@ void pdlp_solver_t::compute_initial_step_size() vecATQ, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_transpose.data(), - stream_view_.value())); + stream_view_.get())); // z = A @ A_t_q RAFT_CUSPARSE_TRY( @@ -3409,7 +3421,7 @@ void pdlp_solver_t::compute_initial_step_size() vecZ, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view_.buffer_non_transpose.data(), - stream_view_.value())); + stream_view_.get())); // sigma_max_sq = dot(q, z) RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(handle_ptr_->get_cublas_handle(), m, @@ -3418,14 +3430,14 @@ void pdlp_solver_t::compute_initial_step_size() d_z.data(), primal_stride, sigma_max_sq.data(), - stream_view_.value())); + stream_view_.get())); // d_q := -sigma_max_sq * d_q + d_z cub::DeviceTransform::Transform(cuda::std::make_tuple(d_q.data(), d_z.data()), d_q.data(), d_q.size(), residual_fma_neg_scalar_t{sigma_max_sq.data()}, - stream_view_.value()); + stream_view_.get()); my_l2_norm(d_q, residual_norm, handle_ptr_); @@ -3440,7 +3452,7 @@ void pdlp_solver_t::compute_initial_step_size() handle_ptr_->get_thrust_policy(), step_size_.begin(), step_size_.end(), step_size); // Sync since we are using local variable - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); RAFT_CUSPARSE_TRY(cusparseDestroyDnVec(vecZ)); RAFT_CUSPARSE_TRY(cusparseDestroyDnVec(vecQ)); RAFT_CUSPARSE_TRY(cusparseDestroyDnVec(vecATQ)); @@ -3535,16 +3547,16 @@ void pdlp_solver_t::compute_initial_primal_weight() const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); compute_weights_initial_primal_weight_from_squared_norms - <<>>(b_vec_norm.data(), - c_vec_norm.data(), - make_span(primal_weight_), - make_span(best_primal_weight_), - climber_strategies_.size(), - settings_.hyper_params); + <<>>(b_vec_norm.data(), + c_vec_norm.data(), + make_span(primal_weight_), + make_span(best_primal_weight_), + climber_strategies_.size(), + settings_.hyper_params); RAFT_CUDA_TRY(cudaPeekAtLastError()); // Sync since we are using local variable - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template diff --git a/cpp/src/pdlp/pdlp.cuh b/cpp/src/pdlp/pdlp.cuh index 60b2c1c354..73f68f94cc 100644 --- a/cpp/src/pdlp/pdlp.cuh +++ b/cpp/src/pdlp/pdlp.cuh @@ -176,7 +176,7 @@ class pdlp_solver_t { bool batch_mode_{false}; raft::handle_t const* handle_ptr_; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; // Intentionnaly take a copy to avoid an unintentional modification in the calling context const pdlp_solver_settings_t settings_; mip::shared_strong_branching_context_view_t sb_view_{settings_.shared_sb_solved}; diff --git a/cpp/src/pdlp/pdlp_warm_start_data.cu b/cpp/src/pdlp/pdlp_warm_start_data.cu index a214f1a165..353a35ed89 100644 --- a/cpp/src/pdlp/pdlp_warm_start_data.cu +++ b/cpp/src/pdlp/pdlp_warm_start_data.cu @@ -79,7 +79,7 @@ pdlp_warm_start_data_t::pdlp_warm_start_data_t() template pdlp_warm_start_data_t::pdlp_warm_start_data_t( - const pdlp_warm_start_data_view_t& other, rmm::cuda_stream_view stream_view) + const pdlp_warm_start_data_view_t& other, cuda::stream_ref stream_view) : current_primal_solution_(other.current_primal_solution_.size(), stream_view), current_dual_solution_(other.current_dual_solution_.size(), stream_view), initial_primal_average_(other.initial_primal_average_.size(), stream_view), diff --git a/cpp/src/pdlp/restart_strategy/localized_duality_gap_container.cu b/cpp/src/pdlp/restart_strategy/localized_duality_gap_container.cu index 7610e4f7dc..eb356022eb 100644 --- a/cpp/src/pdlp/restart_strategy/localized_duality_gap_container.cu +++ b/cpp/src/pdlp/restart_strategy/localized_duality_gap_container.cu @@ -52,11 +52,11 @@ localized_duality_gap_container_t::localized_duality_gap_container_t( RAFT_CUDA_TRY(cudaMemsetAsync(primal_solution_.data(), f_t(0.0), sizeof(f_t) * primal_solution_.size(), - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); RAFT_CUDA_TRY(cudaMemsetAsync(dual_solution_.data(), f_t(0.0), sizeof(f_t) * dual_solution_.size(), - handle_ptr->get_stream())); + handle_ptr->get_stream().get())); } template @@ -96,7 +96,7 @@ void localized_duality_gap_container_t::swap_context( const auto [grid_size, block_size] = kernel_config_from_batch_size(static_cast(swap_pairs.size())); localized_duality_gap_swap_device_vectors_kernel - <<>>( + <<>>( thrust::raw_pointer_cast(swap_pairs.data()), static_cast(swap_pairs.size()), make_span(primal_distance_traveled_), diff --git a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu index 954935b06e..18883ee52b 100644 --- a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu +++ b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cu @@ -216,11 +216,11 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( RAFT_CUDA_TRY(cudaMemsetAsync(last_restart_duality_gap_.primal_solution_.data(), 0.0, sizeof(f_t) * last_restart_duality_gap_.primal_solution_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync(last_restart_duality_gap_.dual_solution_.data(), 0.0, sizeof(f_t) * last_restart_duality_gap_.dual_solution_.size(), - stream_view_)); + stream_view_.get())); // Trigger the costly (costly for ms instances) GetDeviceProperty only if need trust region // restart @@ -231,13 +231,13 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( problem_ptr->constraint_upper_bounds.data(), dual_size_h_, transform_constraint_lower_bounds(), - stream_view_); + stream_view_.get()); raft::linalg::binaryOp(transformed_constraint_upper_bounds_.data(), problem_ptr->constraint_lower_bounds.data(), problem_ptr->constraint_upper_bounds.data(), dual_size_h_, transform_constraint_upper_bounds(), - stream_view_); + stream_view_.get()); // Check that device support CooperativeLaunch int dev = 0; @@ -287,7 +287,7 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( reusable_device_scalar_1_.data(), climber_strategies_.size(), primal_size_h_, - stream_view_); + stream_view_.get()); dot_product_bytes = std::max(dot_product_bytes, byte_needed); cub::DeviceSegmentedReduce::Sum( @@ -297,7 +297,7 @@ pdlp_restart_strategy_t::pdlp_restart_strategy_t( reusable_device_scalar_1_.data(), climber_strategies_.size(), dual_size_h_, - stream_view_); + stream_view_.get()); dot_product_bytes = std::max(dot_product_bytes, byte_needed); dot_product_storage.resize(dot_product_bytes, stream_view_); @@ -351,12 +351,12 @@ bool pdlp_restart_strategy_t::run_trust_region_restart( reusable_device_scalar_value_1_.data(), primal_step_size.data(), 1, - stream_view_); + stream_view_.get()); raft::linalg::eltwiseDivideCheckZero(dual_norm_weight_.data(), reusable_device_scalar_value_1_.data(), dual_step_size.data(), 1, - stream_view_); + stream_view_.get()); i_t restart = should_do_artificial_restart(total_number_of_iterations); @@ -447,11 +447,11 @@ f_t pdlp_restart_strategy_t::compute_kkt_score( const rmm::device_uvector& gap, const rmm::device_uvector& primal_weight) { - kernel_compute_kkt_score<<<1, 1, 0, stream_view_>>>(l2_primal_residual.data(), - l2_dual_residual.data(), - gap.data(), - primal_weight.data(), - tmp_kkt_score_.data()); + kernel_compute_kkt_score<<<1, 1, 0, stream_view_.get()>>>(l2_primal_residual.data(), + l2_dual_residual.data(), + gap.data(), + primal_weight.data(), + tmp_kkt_score_.data()); return tmp_kkt_score_.value(stream_view_); } @@ -928,10 +928,10 @@ void pdlp_restart_strategy_t::cupdlpx_restart( if (batch_mode_) { const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); kernel_compute_next_cupdlpx_primal_weight - <<>>(view, climber_strategies_.size()); + <<>>(view, climber_strategies_.size()); RAFT_CUDA_TRY(cudaPeekAtLastError()); RAFT_CUDA_TRY(cudaStreamSynchronize( - stream_view_)); // To make sure all the data is written from device to host + stream_view_.get())); // To make sure all the data is written from device to host #ifdef CUPDLP_DEBUG_MODE RAFT_CUDA_TRY(cudaDeviceSynchronize()); #endif @@ -981,7 +981,7 @@ void pdlp_restart_strategy_t::cupdlpx_restart( // Small copy helper to use in both single-GPU and distributed paths. auto commit_potential_next_as_last_restart = [](pdlp_restart_strategy_t& rest, pdhg_solver_t& solver, - rmm::cuda_stream_view stream) { + cuda::stream_ref stream) { raft::copy(rest.last_restart_duality_gap_.primal_solution_.data(), solver.get_potential_next_primal_solution().data(), rest.last_restart_duality_gap_.primal_solution_.size(), @@ -1232,11 +1232,12 @@ void pdlp_restart_strategy_t::compute_new_primal_weight( cuopt_assert(!batch_mode_, "compute_new_primal_weight not supported in batch mode"); - compute_new_primal_weight_kernel<<<1, 1, 0, stream_view_>>>(duality_gap.view(), - primal_weight.data(), - step_size.data(), - primal_step_size.data(), - dual_step_size.data()); + compute_new_primal_weight_kernel + <<<1, 1, 0, stream_view_.get()>>>(duality_gap.view(), + primal_weight.data(), + step_size.data(), + primal_step_size.data(), + dual_step_size.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -1282,7 +1283,7 @@ void pdlp_restart_strategy_t::distance_squared_moved_from_last_restart new_solution.data(), new_solution.size(), a_sub_scalar_times_b(reusable_device_scalar_value_1_.data()), - stream_view_); + stream_view_.get()); if (!batch_mode_) { RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(handle_ptr_->get_cublas_handle(), @@ -1292,7 +1293,7 @@ void pdlp_restart_strategy_t::distance_squared_moved_from_last_restart tmp.data(), stride, distance_moved.data(), - stream_view_)); + stream_view_.get())); } else { cub::DeviceSegmentedReduce::Sum( dot_product_storage.data(), @@ -1301,7 +1302,7 @@ void pdlp_restart_strategy_t::distance_squared_moved_from_last_restart distance_moved.data(), climber_strategies_.size(), size_of_solutions_h, - stream_view_); + stream_view_.get()); } } @@ -1348,7 +1349,7 @@ void pdlp_restart_strategy_t::update_last_restart_information( { raft::common::nvtx::range fun_scope("update_last_restart_information"); - compute_distance_traveled_last_restart_kernel<<<1, 1, 0, stream_view_>>>( + compute_distance_traveled_last_restart_kernel<<<1, 1, 0, stream_view_.get()>>>( duality_gap.view(), primal_weight.data(), last_restart_duality_gap_.distance_traveled_.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -1383,8 +1384,8 @@ __global__ void pick_restart_candidate_kernel( template i_t pdlp_restart_strategy_t::pick_restart_candidate() { - pick_restart_candidate_kernel - <<<1, 1, 0, stream_view_>>>(avg_duality_gap_.view(), current_duality_gap_.view(), this->view()); + pick_restart_candidate_kernel<<<1, 1, 0, stream_view_.get()>>>( + avg_duality_gap_.view(), current_duality_gap_.view(), this->view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); i_t restart_to_average_h = candidate_is_avg_.value(stream_view_); @@ -1394,7 +1395,7 @@ i_t pdlp_restart_strategy_t::pick_restart_candidate() candidate_duality_gap_ = ¤t_duality_gap_; } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); return restart_to_average_h; } @@ -1447,15 +1448,15 @@ void pdlp_restart_strategy_t::should_do_adaptive_restart_normalized_du // 2 * primal_weight + lri.dual_distance_moved_last_restart_period ^ 2 / primal_weight, compute_distance_traveled_last_restart_kernel - <<<1, 1, 0, stream_view_>>>(candidate_duality_gap.view(), - primal_weight.data(), - last_restart_duality_gap_.distance_traveled_.data()); + <<<1, 1, 0, stream_view_.get()>>>(candidate_duality_gap.view(), + primal_weight.data(), + last_restart_duality_gap_.distance_traveled_.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); bound_optimal_objective( last_restart_duality_gap_cusparse_view_, last_restart_duality_gap_, tmp_primal, tmp_dual); - adaptive_restart_triggered<<<1, 1, 0, stream_view_>>>( + adaptive_restart_triggered<<<1, 1, 0, stream_view_.get()>>>( candidate_duality_gap.view(), last_restart_duality_gap_.view(), this->view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); @@ -1552,7 +1553,7 @@ void pdlp_restart_strategy_t::compute_localized_duality_gaps( current_duality_gap_cusparse_view_, current_duality_gap_, tmp_primal, tmp_dual); compute_normalized_gaps_kernel - <<<1, 1, 0, stream_view_>>>(avg_duality_gap_.view(), current_duality_gap_.view()); + <<<1, 1, 0, stream_view_.get()>>>(avg_duality_gap_.view(), current_duality_gap_.view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -1589,7 +1590,8 @@ void pdlp_restart_strategy_t::compute_bound(const rmm::device_uvector< #ifdef PDLP_DEBUG_MODE std::cout << "Compute bound" << std::endl; #endif - raft::linalg::eltwiseSub(tmp.data(), solution_tr.data(), solution.data(), size, stream_view_); + raft::linalg::eltwiseSub( + tmp.data(), solution_tr.data(), solution.data(), size, stream_view_.get()); RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(handle_ptr_->get_cublas_handle(), size, @@ -1598,9 +1600,9 @@ void pdlp_restart_strategy_t::compute_bound(const rmm::device_uvector< gradient.data(), stride, bound.data(), - stream_view_)); + stream_view_.get())); - raft::linalg::eltwiseAdd(bound.data(), bound.data(), lagrangian.data(), 1, stream_view_); + raft::linalg::eltwiseAdd(bound.data(), bound.data(), lagrangian.data(), 1, stream_view_.get()); } template @@ -1947,7 +1949,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( duality_gap.dual_gradient_.data(), dual_size_h_, negate_t(), - stream_view_); + stream_view_.get()); // Use high_radius_squared_ to store objective_vector l2_norm my_l2_norm(objective_vector_, high_radius_squared_, handle_ptr_); @@ -1974,10 +1976,12 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( const f_t zero_float = f_t(0.0); high_radius_squared_.set_value_async(zero_float, stream_view_); low_radius_squared_.set_value_async(zero_float, stream_view_); + RAFT_CUDA_TRY(cudaMemsetAsync(direction_full_.data(), + 0, + sizeof(f_t) * (primal_size_h_ + dual_size_h_), + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - direction_full_.data(), 0, sizeof(f_t) * (primal_size_h_ + dual_size_h_), stream_view_)); - RAFT_CUDA_TRY(cudaMemsetAsync( - threshold_.data(), 0, sizeof(f_t) * (primal_size_h_ + dual_size_h_), stream_view_)); + threshold_.data(), 0, sizeof(f_t) * (primal_size_h_ + dual_size_h_), stream_view_.get())); /* ----- */ // Determine the direction which each component has moved and the threshold for when the @@ -1990,7 +1994,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( thrust::make_zip_iterator(thrust::make_tuple(lower_bound_.data(), upper_bound_.data())), primal_size_h_, extract_bounds_t(), - stream_view_.value()); + stream_view_.get()); raft::copy(lower_bound_.data() + primal_size_h_, transformed_constraint_lower_bounds_.data(), dual_size_h_, @@ -2131,7 +2135,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( dimBlock, kernel_args, 0, - stream_view_)); + stream_view_.get())); // Find max threshold for the join problem const f_t* max_threshold = @@ -2146,7 +2150,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( // target_threshold which was computed before the loop in the direction_and_threshold_kernel // Otherwise use the test_threshold determined in the loop // { - target_threshold_determination_kernel<<<1, 1, 0, stream_view_>>>( + target_threshold_determination_kernel<<<1, 1, 0, stream_view_.get()>>>( this->view(), duality_gap.distance_traveled_.data(), max_threshold, max_threshold); RAFT_CUDA_TRY(cudaPeekAtLastError()); // } @@ -2160,13 +2164,13 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( unsorted_direction_full_.data(), primal_size_h_, a_add_scalar_times_b(target_threshold_.data()), - stream_view_); + stream_view_.get()); raft::linalg::binaryOp(duality_gap.dual_solution_tr_.data(), duality_gap.dual_solution_.data(), unsorted_direction_full_.data() + primal_size_h_, dual_size_h_, a_add_scalar_times_b(target_threshold_.data()), - stream_view_); + stream_view_.get()); // project by max(min(x[i], upperbound[i]),lowerbound[i]) for primal part using f_t2 = typename type_2::type; cub::DeviceTransform::Transform(cuda::std::make_tuple(duality_gap.primal_solution_tr_.data(), @@ -2174,7 +2178,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( duality_gap.primal_solution_tr_.data(), primal_size_h_, clamp(), - stream_view_.value()); + stream_view_.get()); // project by max(min(y[i], upperbound[i]),lowerbound[i]) raft::linalg::ternaryOp(duality_gap.dual_solution_tr_.data(), @@ -2183,7 +2187,7 @@ void pdlp_restart_strategy_t::solve_bound_constrained_trust_region( transformed_constraint_upper_bounds_.data(), dual_size_h_, constraint_clamp(), - stream_view_); + stream_view_.get()); // } } @@ -2245,7 +2249,7 @@ void pdlp_restart_strategy_t::compute_distance_traveled_from_last_rest // distance_traveled = primal_distance * 0.5 * primal_weight // + dual_distance * 0.5 / primal_weight - compute_distance_traveled_last_restart_kernel<<<1, 1, 0, stream_view_>>>( + compute_distance_traveled_last_restart_kernel<<<1, 1, 0, stream_view_.get()>>>( duality_gap.view(), primal_weight.data(), duality_gap.distance_traveled_.data()); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -2276,7 +2280,7 @@ void pdlp_restart_strategy_t::compute_primal_gradient( cusparse_view.primal_gradient, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); } template @@ -2344,21 +2348,22 @@ void pdlp_restart_strategy_t::compute_dual_gradient( cusparse_view.dual_gradient, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); // tmp_dual will contain the subgradient i_t number_of_blocks = dual_size_h_ / block_size; if (dual_size_h_ % block_size) number_of_blocks++; i_t number_of_threads = std::min(dual_size_h_, block_size); - compute_subgradient_kernel<<>>( - this->view(), problem_ptr->view(), duality_gap.view(), tmp_dual.data()); + compute_subgradient_kernel + <<>>( + this->view(), problem_ptr->view(), duality_gap.view(), tmp_dual.data()); // dual gradient = subgradient - primal_product (tmp_dual-dual_gradient) raft::linalg::eltwiseSub(duality_gap.dual_gradient_.data(), tmp_dual.data(), duality_gap.dual_gradient_.data(), dual_size_h_, - stream_view_); + stream_view_.get()); } template @@ -2389,7 +2394,7 @@ void pdlp_restart_strategy_t::compute_lagrangian_value( problem_ptr->objective_coefficients.data(), primal_stride, reusable_device_scalar_1_.data(), - stream_view_)); + stream_view_.get())); // third term, let beta be 0 to not add what is in tmp_primal, compute it and compute dot RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsespmv(handle_ptr_->get_cusparse_handle(), @@ -2401,7 +2406,7 @@ void pdlp_restart_strategy_t::compute_lagrangian_value( cusparse_view.tmp_primal, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); RAFT_CUBLAS_TRY(raft::linalg::detail::cublasdot(handle_ptr_->get_cublas_handle(), primal_size_h_, @@ -2410,7 +2415,7 @@ void pdlp_restart_strategy_t::compute_lagrangian_value( tmp_primal.data(), primal_stride, reusable_device_scalar_2_.data(), - stream_view_)); + stream_view_.get())); // fourth term //tmp_dual still contains subgradient from the dual_gradient computation reusable_device_scalar_3_.set_value_to_zero_async(stream_view_); @@ -2421,19 +2426,19 @@ void pdlp_restart_strategy_t::compute_lagrangian_value( tmp_dual.data(), dual_stride, reusable_device_scalar_3_.data(), - stream_view_)); + stream_view_.get())); // subtract third term from second up raft::linalg::eltwiseSub(reusable_device_scalar_1_.data(), reusable_device_scalar_1_.data(), reusable_device_scalar_2_.data(), 1, - stream_view_); + stream_view_.get()); raft::linalg::eltwiseAdd(duality_gap.lagrangian_value_.data(), reusable_device_scalar_1_.data(), reusable_device_scalar_3_.data(), 1, - stream_view_); + stream_view_.get()); } template diff --git a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cuh b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cuh index 68ef7503a0..41255c8dd6 100644 --- a/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cuh +++ b/cpp/src/pdlp/restart_strategy/pdlp_restart_strategy.cuh @@ -306,7 +306,7 @@ class pdlp_restart_strategy_t { rmm::device_uvector& dual_step_size); raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; public: const bool batch_mode_{false}; diff --git a/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu b/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu index 50ad27334b..59364143a0 100644 --- a/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu +++ b/cpp/src/pdlp/restart_strategy/weighted_average_solution.cu @@ -33,19 +33,19 @@ weighted_average_solution_t::weighted_average_solution_t(raft::handle_ iterations_since_last_restart_{0}, graph(stream_view_, is_batch_mode) { - RAFT_CUDA_TRY( - cudaMemsetAsync(sum_primal_solutions_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(sum_dual_solutions_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_)); + RAFT_CUDA_TRY(cudaMemsetAsync( + sum_primal_solutions_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync( + sum_dual_solutions_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_.get())); } template void weighted_average_solution_t::reset_weighted_average_solution() { - RAFT_CUDA_TRY( - cudaMemsetAsync(sum_primal_solutions_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(sum_dual_solutions_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_)); + RAFT_CUDA_TRY(cudaMemsetAsync( + sum_primal_solutions_.data(), 0.0, sizeof(f_t) * primal_size_h_, stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync( + sum_dual_solutions_.data(), 0.0, sizeof(f_t) * dual_size_h_, stream_view_.get())); sum_primal_solution_weights_.set_value_to_zero_async(stream_view_); sum_dual_solution_weights_.set_value_to_zero_async(stream_view_); iterations_since_last_restart_ = 0; @@ -78,20 +78,20 @@ void weighted_average_solution_t::add_current_solution_to_weighted_ave sum_primal_solutions_.data(), primal_size_h_, a_add_scalar_times_b(weight.data()), - stream_view_.value()); + stream_view_.get()); cub::DeviceTransform::Transform( cuda::std::make_tuple(sum_dual_solutions_.data(), dual_solution), sum_dual_solutions_.data(), dual_size_h_, a_add_scalar_times_b(weight.data()), - stream_view_.value()); + stream_view_.get()); // update weight sums and count (add weight and +1 respectively) - add_weight_sums<<<1, 1, 0, stream_view_>>>(weight.data(), - weight.data(), - sum_primal_solution_weights_.data(), - sum_dual_solution_weights_.data()); + add_weight_sums<<<1, 1, 0, stream_view_.get()>>>(weight.data(), + weight.data(), + sum_primal_solution_weights_.data(), + sum_dual_solution_weights_.data()); }); iterations_since_last_restart_ += 1; @@ -103,10 +103,10 @@ void weighted_average_solution_t::compute_averages(rmm::device_uvector { // no iterations have added to the sum, so avg is all zero vector if (!iterations_since_last_restart_) { + RAFT_CUDA_TRY(cudaMemsetAsync( + avg_primal.data(), f_t(0.0), sizeof(f_t) * primal_size_h_, stream_view_.get())); RAFT_CUDA_TRY( - cudaMemsetAsync(avg_primal.data(), f_t(0.0), sizeof(f_t) * primal_size_h_, stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(avg_dual.data(), f_t(0.0), sizeof(f_t) * dual_size_h_, stream_view_)); + cudaMemsetAsync(avg_dual.data(), f_t(0.0), sizeof(f_t) * dual_size_h_, stream_view_.get())); return; } @@ -114,19 +114,19 @@ void weighted_average_solution_t::compute_averages(rmm::device_uvector f_t sum_primal_solution_weights_h = sum_primal_solution_weights_.value(stream_view_); f_t sum_dual_solution_weights_h = sum_dual_solution_weights_.value(stream_view_); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); // compute sum_primal_solutions/primal_size raft::linalg::divideScalar(avg_primal.data(), sum_primal_solutions_.data(), sum_primal_solution_weights_h, primal_size_h_, - stream_view_); + stream_view_.get()); raft::linalg::divideScalar(avg_dual.data(), sum_dual_solutions_.data(), sum_dual_solution_weights_h, dual_size_h_, - stream_view_); + stream_view_.get()); } template diff --git a/cpp/src/pdlp/restart_strategy/weighted_average_solution.hpp b/cpp/src/pdlp/restart_strategy/weighted_average_solution.hpp index 777f9ea804..e6fa4eb1cf 100644 --- a/cpp/src/pdlp/restart_strategy/weighted_average_solution.hpp +++ b/cpp/src/pdlp/restart_strategy/weighted_average_solution.hpp @@ -36,7 +36,7 @@ class weighted_average_solution_t { private: raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; i_t primal_size_h_; i_t dual_size_h_; diff --git a/cpp/src/pdlp/saddle_point.cu b/cpp/src/pdlp/saddle_point.cu index b92fdc2fb3..85bdb61412 100644 --- a/cpp/src/pdlp/saddle_point.cu +++ b/cpp/src/pdlp/saddle_point.cu @@ -47,13 +47,15 @@ saddle_point_state_t::saddle_point_state_t(raft::handle_t const* handl handle_ptr->get_thrust_policy(), dual_solution_.data(), dual_solution_.end(), f_t(0)); RAFT_CUDA_TRY(cudaMemsetAsync( - delta_primal_.data(), 0, sizeof(f_t) * delta_primal_.size(), handle_ptr->get_stream())); + delta_primal_.data(), 0, sizeof(f_t) * delta_primal_.size(), handle_ptr->get_stream().get())); RAFT_CUDA_TRY(cudaMemsetAsync( - delta_dual_.data(), 0, sizeof(f_t) * delta_dual_.size(), handle_ptr->get_stream())); + delta_dual_.data(), 0, sizeof(f_t) * delta_dual_.size(), handle_ptr->get_stream().get())); + RAFT_CUDA_TRY(cudaMemsetAsync(primal_gradient_.data(), + 0, + sizeof(f_t) * primal_gradient_.size(), + handle_ptr->get_stream().get())); RAFT_CUDA_TRY(cudaMemsetAsync( - primal_gradient_.data(), 0, sizeof(f_t) * primal_gradient_.size(), handle_ptr->get_stream())); - RAFT_CUDA_TRY(cudaMemsetAsync( - dual_gradient_.data(), 0, sizeof(f_t) * dual_gradient_.size(), handle_ptr->get_stream())); + dual_gradient_.data(), 0, sizeof(f_t) * dual_gradient_.size(), handle_ptr->get_stream().get())); // No need to 0 init current/next AtY, they are directlty written as result of SpMV } @@ -97,7 +99,7 @@ void saddle_point_state_t::resize_context(i_t new_size) template void saddle_point_state_t::copy(saddle_point_state_t& other, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { EXE_CUOPT_EXPECTS(this->primal_size_ == other.get_primal_size(), "Size of primal solution must be the same in order to copy"); diff --git a/cpp/src/pdlp/saddle_point.hpp b/cpp/src/pdlp/saddle_point.hpp index c6fc962016..01bf512c72 100644 --- a/cpp/src/pdlp/saddle_point.hpp +++ b/cpp/src/pdlp/saddle_point.hpp @@ -81,7 +81,7 @@ class saddle_point_state_t { * * @throws cuopt::logic_error if the solutions are not of the same size */ - void copy(saddle_point_state_t& other, rmm::cuda_stream_view stream); + void copy(saddle_point_state_t& other, cuda::stream_ref stream); i_t get_primal_size() const; i_t get_dual_size() const; diff --git a/cpp/src/pdlp/solve.cu b/cpp/src/pdlp/solve.cu index d08a36d178..81b6ee25ad 100644 --- a/cpp/src/pdlp/solve.cu +++ b/cpp/src/pdlp/solve.cu @@ -74,16 +74,17 @@ namespace cuopt::mathematical_optimization { template extern rmm::device_uvector gpu_cast(const rmm::device_uvector& src, - rmm::cuda_stream_view stream); + cuda::stream_ref stream); // This serves as both a warm up but also a mandatory initial call to setup cuSparse and cuBLAS static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } // Corresponds to the first good general settings we found @@ -335,7 +336,7 @@ std::atomic global_concurrent_halt{0}; template void adjust_dual_solution_and_reduced_cost(rmm::device_uvector& dual_solution, rmm::device_uvector& reduced_cost, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { // y <- -y cub::DeviceTransform::Transform( @@ -1280,9 +1281,9 @@ template static optimization_problem_solution_t run_batch_pdlp_splitting( optimization_problem_t& problem, pdlp_solver_settings_t const& settings) { - rmm::cuda_stream_view stream = problem.get_handle_ptr()->get_stream(); - const i_t n_vars = problem.get_n_variables(); - const i_t n_constraints = problem.get_n_constraints(); + cuda::stream_ref stream = problem.get_handle_ptr()->get_stream(); + const i_t n_vars = problem.get_n_variables(); + const i_t n_constraints = problem.get_n_constraints(); // Splitting path only supports un-expanded problems + per-climber variable-bound overrides. cuopt_expects(problem.get_objective_coefficients().size() == static_cast(n_vars), @@ -1607,8 +1608,8 @@ optimization_problem_solution_t run_concurrent( { try { auto call_barrier_thread = [&]() { - rmm::cuda_stream_view barrier_stream = rmm::cuda_stream_per_thread; - barrier_handle_ptr = std::make_unique(barrier_stream); + cuda::stream_ref barrier_stream = rmm::cuda_stream_per_thread; + barrier_handle_ptr = std::make_unique(barrier_stream); run_barrier_thread(dual_simplex_problem, settings_pdlp, sol_barrier_ptr, @@ -1929,7 +1930,7 @@ optimization_problem_solution_t solve_qcqp( template static std::optional> terminal_solution_from_presolve_status(mip::third_party_presolve_status_t status, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { switch (status) { case mip::third_party_presolve_status_t::INFEASIBLE: @@ -2358,7 +2359,7 @@ cuopt::mathematical_optimization::io::mps_data_model_t op_problem_to_m raft::copy(h_constr_lb.data(), d_constr_lb.data(), d_constr_lb.size(), stream); raft::copy(h_constr_ub.data(), d_constr_ub.data(), d_constr_ub.size(), stream); raft::copy(h_var_types_enum.data(), d_var_types.data(), d_var_types.size(), stream); - stream.synchronize(); + stream.sync(); if (!h_offsets.empty()) { mps.set_csr_constraint_matrix( diff --git a/cpp/src/pdlp/solver_settings.cu b/cpp/src/pdlp/solver_settings.cu index 33d8f1a64b..1e2706081d 100644 --- a/cpp/src/pdlp/solver_settings.cu +++ b/cpp/src/pdlp/solver_settings.cu @@ -36,7 +36,7 @@ void pdlp_solver_settings_t::set_optimality_tolerance(f_t eps_optimal) template void pdlp_solver_settings_t::set_initial_primal_solution( - const f_t* initial_primal_solution, i_t size, rmm::cuda_stream_view stream) + const f_t* initial_primal_solution, i_t size, cuda::stream_ref stream) { cuopt_expects(initial_primal_solution != nullptr, error_type_t::ValidationError, @@ -49,7 +49,7 @@ void pdlp_solver_settings_t::set_initial_primal_solution( template void pdlp_solver_settings_t::set_initial_dual_solution(const f_t* initial_dual_solution, i_t size, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { cuopt_expects(initial_dual_solution != nullptr, error_type_t::ValidationError, diff --git a/cpp/src/pdlp/solver_solution.cu b/cpp/src/pdlp/solver_solution.cu index 08e5ee00a8..e02fb2dcfe 100644 --- a/cpp/src/pdlp/solver_solution.cu +++ b/cpp/src/pdlp/solver_solution.cu @@ -25,7 +25,7 @@ namespace cuopt::mathematical_optimization { template optimization_problem_solution_t::optimization_problem_solution_t( - pdlp_termination_status_t termination_status, rmm::cuda_stream_view stream_view) + pdlp_termination_status_t termination_status, cuda::stream_ref stream_view) : primal_solution_{0, stream_view}, dual_solution_{0, stream_view}, reduced_cost_{0, stream_view}, @@ -38,7 +38,7 @@ optimization_problem_solution_t::optimization_problem_solution_t( template optimization_problem_solution_t::optimization_problem_solution_t( - cuopt::logic_error error_status_, rmm::cuda_stream_view stream_view) + cuopt::logic_error error_status_, cuda::stream_ref stream_view) : primal_solution_{0, stream_view}, dual_solution_{0, stream_view}, reduced_cost_{0, stream_view}, @@ -210,7 +210,7 @@ void optimization_problem_solution_t::write_additional_termination_sta template void optimization_problem_solution_t::write_to_file(std::string_view filename, - rmm::cuda_stream_view stream_view, + cuda::stream_ref stream_view, bool generate_variable_values) { raft::common::nvtx::range fun_scope("write final solution to file"); @@ -235,11 +235,10 @@ void optimization_problem_solution_t::write_to_file(std::string_view f dual_solution.resize(dual_solution_.size()); reduced_cost.resize(reduced_cost_.size()); raft::copy( - primal_solution.data(), primal_solution_.data(), primal_solution_.size(), stream_view.value()); - raft::copy( - dual_solution.data(), dual_solution_.data(), dual_solution_.size(), stream_view.value()); - raft::copy(reduced_cost.data(), reduced_cost_.data(), reduced_cost_.size(), stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + primal_solution.data(), primal_solution_.data(), primal_solution_.size(), stream_view.get()); + raft::copy(dual_solution.data(), dual_solution_.data(), dual_solution_.size(), stream_view.get()); + raft::copy(reduced_cost.data(), reduced_cost_.data(), reduced_cost_.size(), stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); myfile << "{ " << std::endl; myfile << "\t\"Termination reason\" : \"" << get_termination_status_string() << "\"," @@ -431,7 +430,7 @@ optimization_problem_solution_t::get_pdlp_warm_start_data() template void optimization_problem_solution_t::write_to_sol_file( - std::string_view filename, rmm::cuda_stream_view stream_view) const + std::string_view filename, cuda::stream_ref stream_view) const { cuopt_expects(termination_stats_.size() == 1, error_type_t::ValidationError, @@ -446,9 +445,8 @@ void optimization_problem_solution_t::write_to_sol_file( auto objective_value = get_objective_value(0); std::vector solution; solution.resize(primal_solution_.size()); - raft::copy( - solution.data(), primal_solution_.data(), primal_solution_.size(), stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + raft::copy(solution.data(), primal_solution_.data(), primal_solution_.size(), stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); solution_writer_t::write_solution_to_sol_file( std::string(filename), status, objective_value, var_names_, solution); } diff --git a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu index e52c329166..1e6f9b938a 100644 --- a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu +++ b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.cu @@ -78,7 +78,7 @@ adaptive_step_size_strategy_t::adaptive_step_size_strategy_t( interaction_.data(), climber_strategies_.size(), primal_size_, - stream_view_.value())); + stream_view_.get())); dot_product_bytes = std::max(dot_product_bytes, byte_needed); RAFT_CUDA_TRY(cub::DeviceSegmentedReduce::Sum( @@ -88,7 +88,7 @@ adaptive_step_size_strategy_t::adaptive_step_size_strategy_t( norm_squared_delta_primal_.data(), climber_strategies_.size(), primal_size_, - stream_view_.value())); + stream_view_.get())); dot_product_bytes = std::max(dot_product_bytes, byte_needed); RAFT_CUDA_TRY(cub::DeviceSegmentedReduce::Sum( @@ -98,10 +98,10 @@ adaptive_step_size_strategy_t::adaptive_step_size_strategy_t( norm_squared_delta_dual_.data(), climber_strategies_.size(), dual_size_, - stream_view_.value())); + stream_view_.get())); dot_product_bytes = std::max(dot_product_bytes, byte_needed); - dot_product_storage.resize(dot_product_bytes, stream_view_.value()); + dot_product_storage.resize(dot_product_bytes, stream_view_.get()); } } @@ -141,12 +141,11 @@ void adaptive_step_size_strategy_t::swap_context( const auto [grid_size, block_size] = kernel_config_from_batch_size(static_cast(swap_pairs.size())); adaptive_step_size_swap_device_vectors_kernel - <<>>( - thrust::raw_pointer_cast(swap_pairs.data()), - static_cast(swap_pairs.size()), - make_span(interaction_), - make_span(norm_squared_delta_primal_), - make_span(norm_squared_delta_dual_)); + <<>>(thrust::raw_pointer_cast(swap_pairs.data()), + static_cast(swap_pairs.size()), + make_span(interaction_), + make_span(norm_squared_delta_primal_), + make_span(norm_squared_delta_dual_)); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -158,9 +157,9 @@ void adaptive_step_size_strategy_t::resize_context(i_t new_size) cuopt_assert(new_size > 0, "New size must be greater than 0"); cuopt_assert(new_size < batch_size, "New size must be less than batch size"); - interaction_.resize(new_size, stream_view_.value()); - norm_squared_delta_primal_.resize(new_size, stream_view_.value()); - norm_squared_delta_dual_.resize(new_size, stream_view_.value()); + interaction_.resize(new_size, stream_view_.get()); + norm_squared_delta_primal_.resize(new_size, stream_view_.get()); + norm_squared_delta_dual_.resize(new_size, stream_view_.get()); } template @@ -275,19 +274,19 @@ i_t adaptive_step_size_strategy_t::get_valid_step_size() const template f_t adaptive_step_size_strategy_t::get_interaction(i_t i) const { - return interaction_.element(i, stream_view_.value()); + return interaction_.element(i, stream_view_.get()); } template f_t adaptive_step_size_strategy_t::get_norm_squared_delta_primal(i_t i) const { - return norm_squared_delta_primal_.element(i, stream_view_.value()); + return norm_squared_delta_primal_.element(i, stream_view_.get()); } template f_t adaptive_step_size_strategy_t::get_norm_squared_delta_dual(i_t i) const { - return norm_squared_delta_dual_.element(i, stream_view_.value()); + return norm_squared_delta_dual_.element(i, stream_view_.get()); } template @@ -352,13 +351,13 @@ void adaptive_step_size_strategy_t::compute_step_sizes( pdhg_solver.get_saddle_point_state()); // Compute n_lim, n_next and decide if step size is valid compute_step_sizes_from_movement_and_interaction - <<<1, 1, 0, stream_view_.value()>>>(this->view(), - primal_step_size.data(), - dual_step_size.data(), - pdhg_solver.get_d_total_pdhg_iterations().data()); + <<<1, 1, 0, stream_view_.get()>>>(this->view(), + primal_step_size.data(), + dual_step_size.data(), + pdhg_solver.get_d_total_pdhg_iterations().data()); }); // Steam sync so that next call can see modification made to host var valid_step_size - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -421,7 +420,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( cusparse_view.next_AtY, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_transpose.data(), - stream_view_.value())); + stream_view_.get())); } else { // TODO later batch mode: handle if not all restart RAFT_CUSPARSE_TRY( @@ -435,7 +434,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( cusparse_view.batch_next_AtYs, CUSPARSE_SPMM_CSR_ALG3, (f_t*)cusparse_view.buffer_transpose_batch.data(), - stream_view_.value())); + stream_view_.get())); } // Compute Ay' - Ay = next_Aty - current_Aty @@ -446,7 +445,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( tmp_primal.data(), tmp_primal.size(), cuda::std::minus<>{}, - stream_view_.value()); + stream_view_.get()); if (!batch_mode_) { // compute interaction (x'-x) . (A(y'-y)) @@ -458,7 +457,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( current_saddle_point_state.get_delta_primal().data(), primal_stride, interaction_.data(), - stream_view_.value())); + stream_view_.get())); // Compute movement // compute euclidean norm squared which is @@ -476,7 +475,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( current_saddle_point_state.get_delta_primal().data(), primal_stride, norm_squared_delta_primal_.data(), - stream_view_.value())); + stream_view_.get())); RAFT_CUBLAS_TRY( raft::linalg::detail::cublasdot(handle_ptr_->get_cublas_handle(), @@ -486,7 +485,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( current_saddle_point_state.get_delta_dual().data(), dual_stride, norm_squared_delta_dual_.data(), - stream_view_.value())); + stream_view_.get())); } else { // TODO later batch mode: remove this once you want to do per climber restart cub::DeviceSegmentedReduce::Sum( @@ -499,7 +498,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( interaction_.data(), climber_strategies_.size(), primal_size_, - stream_view_.value()); + stream_view_.get()); cub::DeviceSegmentedReduce::Sum( dot_product_storage.data(), @@ -509,7 +508,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( norm_squared_delta_primal_.data(), climber_strategies_.size(), primal_size_, - stream_view_.value()); + stream_view_.get()); cub::DeviceSegmentedReduce::Sum( dot_product_storage.data(), @@ -519,7 +518,7 @@ void adaptive_step_size_strategy_t::compute_interaction_and_movement( norm_squared_delta_dual_.data(), climber_strategies_.size(), dual_size_, - stream_view_.value()); + stream_view_.get()); } } @@ -562,10 +561,10 @@ void adaptive_step_size_strategy_t::get_primal_and_dual_stepsizes( cuopt_assert(step_size_->size() == climber_strategies_.size(), "step size must be the same size as the number of climber strategies"); compute_actual_stepsizes - <<>>(this->view(), - make_span(primal_step_size), - make_span(dual_step_size), - climber_strategies_.size()); + <<>>(this->view(), + make_span(primal_step_size), + make_span(dual_step_size), + climber_strategies_.size()); RAFT_CUDA_TRY(cudaPeekAtLastError()); } diff --git a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.hpp b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.hpp index ebe546c2de..8792a08606 100644 --- a/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.hpp +++ b/cpp/src/pdlp/step_size_strategy/adaptive_step_size_strategy.hpp @@ -106,7 +106,7 @@ class adaptive_step_size_strategy_t { const bool batch_mode_; raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; i_t primal_size_; i_t dual_size_; diff --git a/cpp/src/pdlp/swap_and_resize_helper.cuh b/cpp/src/pdlp/swap_and_resize_helper.cuh index e09ba4f9ed..cc73401538 100644 --- a/cpp/src/pdlp/swap_and_resize_helper.cuh +++ b/cpp/src/pdlp/swap_and_resize_helper.cuh @@ -83,7 +83,7 @@ void matrix_swap(rmm::device_uvector& matrix, [] HD(thrust::tuple values) -> thrust::tuple { return thrust::make_tuple(thrust::get<1>(values), thrust::get<0>(values)); }, - matrix.stream().value()); + matrix.stream().get()); } template diff --git a/cpp/src/pdlp/termination_strategy/convergence_information.cu b/cpp/src/pdlp/termination_strategy/convergence_information.cu index 685870b699..8adee0c65e 100644 --- a/cpp/src/pdlp/termination_strategy/convergence_information.cu +++ b/cpp/src/pdlp/termination_strategy/convergence_information.cu @@ -90,20 +90,22 @@ convergence_information_t::convergence_information_t( { // Zero-init per-climber scalars RAFT_CUDA_TRY(cudaMemsetAsync( - primal_objective_.data(), 0, sizeof(f_t) * primal_objective_.size(), stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(dual_objective_.data(), 0, sizeof(f_t) * dual_objective_.size(), stream_view_)); - RAFT_CUDA_TRY(cudaMemsetAsync(gap_.data(), 0, sizeof(f_t) * gap_.size(), stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(abs_objective_.data(), 0, sizeof(f_t) * abs_objective_.size(), stream_view_)); + primal_objective_.data(), 0, sizeof(f_t) * primal_objective_.size(), stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - l2_dual_residual_.data(), 0, sizeof(f_t) * l2_dual_residual_.size(), stream_view_)); + dual_objective_.data(), 0, sizeof(f_t) * dual_objective_.size(), stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync(gap_.data(), 0, sizeof(f_t) * gap_.size(), stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - l2_primal_residual_.data(), 0, sizeof(f_t) * l2_primal_residual_.size(), stream_view_)); + abs_objective_.data(), 0, sizeof(f_t) * abs_objective_.size(), stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - linf_primal_residual_.data(), 0, sizeof(f_t) * linf_primal_residual_.size(), stream_view_)); + l2_dual_residual_.data(), 0, sizeof(f_t) * l2_dual_residual_.size(), stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( - linf_dual_residual_.data(), 0, sizeof(f_t) * linf_dual_residual_.size(), stream_view_)); + l2_primal_residual_.data(), 0, sizeof(f_t) * l2_primal_residual_.size(), stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync(linf_primal_residual_.data(), + 0, + sizeof(f_t) * linf_primal_residual_.size(), + stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync( + linf_dual_residual_.data(), 0, sizeof(f_t) * linf_dual_residual_.size(), stream_view_.get())); init_objective_offsets(); init_reduction_storage(); @@ -111,9 +113,9 @@ convergence_information_t::convergence_information_t( // Zero the residual workspace (reused each iteration by compute_convergence_information). RAFT_CUDA_TRY(cudaMemsetAsync( - primal_residual_.data(), 0.0, sizeof(f_t) * primal_residual_.size(), stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(dual_residual_.data(), 0.0, sizeof(f_t) * dual_residual_.size(), stream_view_)); + primal_residual_.data(), 0.0, sizeof(f_t) * primal_residual_.size(), stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync( + dual_residual_.data(), 0.0, sizeof(f_t) * dual_residual_.size(), stream_view_.get())); } // --------------------------------------------------------------------------- @@ -285,7 +287,7 @@ void convergence_information_t::init_reduction_storage() bound_value_.begin(), dual_objective_.data(), dual_size_h_, - stream_view_); + stream_view_.get()); size_t temp_storage_bytes_2 = 0; cub::DeviceReduce::Sum(d_temp_storage, @@ -293,7 +295,7 @@ void convergence_information_t::init_reduction_storage() bound_value_.begin(), reduced_cost_dual_objective_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); size_of_buffer_ = std::max({temp_storage_bytes_1, temp_storage_bytes_2}); this->rmm_tmp_buffer_ = rmm::device_buffer{size_of_buffer_, stream_view_}; @@ -359,21 +361,21 @@ void convergence_information_t::swap_context( const auto [grid_size, block_size] = kernel_config_from_batch_size(static_cast(swap_pairs.size())); convergence_information_swap_device_vectors_kernel - <<>>(thrust::raw_pointer_cast(swap_pairs.data()), - static_cast(swap_pairs.size()), - make_span(primal_objective_), - make_span(dual_objective_), - make_span(l2_primal_residual_), - make_span(l2_dual_residual_), - make_span(linf_primal_residual_), - make_span(linf_dual_residual_), - make_span(gap_), - make_span(abs_objective_), - make_span(dual_dot_), - make_span(sum_primal_slack_), - make_span(objective_offsets_), - make_span(l2_norm_primal_linear_objective_), - make_span(l2_norm_primal_right_hand_side_)); + <<>>(thrust::raw_pointer_cast(swap_pairs.data()), + static_cast(swap_pairs.size()), + make_span(primal_objective_), + make_span(dual_objective_), + make_span(l2_primal_residual_), + make_span(l2_dual_residual_), + make_span(linf_primal_residual_), + make_span(linf_dual_residual_), + make_span(gap_), + make_span(abs_objective_), + make_span(dual_dot_), + make_span(sum_primal_slack_), + make_span(objective_offsets_), + make_span(l2_norm_primal_linear_objective_), + make_span(l2_norm_primal_right_hand_side_)); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -690,14 +692,14 @@ void convergence_information_t::compute_convergence_information( // behaviour const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); compute_remaining_stats_kernel - <<>>(this->view(), climber_strategies_.size()); + <<>>(this->view(), climber_strategies_.size()); RAFT_CUDA_TRY(cudaPeekAtLastError()); // cleanup for next termination evaluation RAFT_CUDA_TRY(cudaMemsetAsync( - primal_residual_.data(), 0.0, sizeof(f_t) * primal_residual_.size(), stream_view_)); - RAFT_CUDA_TRY( - cudaMemsetAsync(dual_residual_.data(), 0.0, sizeof(f_t) * dual_residual_.size(), stream_view_)); + primal_residual_.data(), 0.0, sizeof(f_t) * primal_residual_.size(), stream_view_.get())); + RAFT_CUDA_TRY(cudaMemsetAsync( + dual_residual_.data(), 0.0, sizeof(f_t) * dual_residual_.size(), stream_view_.get())); } template @@ -726,7 +728,7 @@ void convergence_information_t::compute_primal_residual( cusparse_view.tmp_dual, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); } else { RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmm(handle_ptr_->get_cusparse_handle(), @@ -739,7 +741,7 @@ void convergence_information_t::compute_primal_residual( cusparse_view.batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)cusparse_view.buffer_non_transpose_batch.data(), - stream_view_)); + stream_view_.get())); } if (!hyper_params_.use_reflected_primal_dual) { @@ -754,7 +756,7 @@ void convergence_information_t::compute_primal_residual( problem_ptr->constraint_upper_bounds.data(), dual_size_h_, violation(), - stream_view_); + stream_view_.get()); } else { cuopt_assert(primal_residual_.size() == primal_slack_.size(), "Both vectors should had the same size"); @@ -774,7 +776,7 @@ void convergence_information_t::compute_primal_residual( raft::max(dual, f_t(0.0)) * finite_or_zero(lower) + raft::min(dual, f_t(0.0)) * finite_or_zero(upper)}; }, - stream_view_.value()); + stream_view_.get()); } #ifdef PDLP_DEBUG_MODE @@ -811,7 +813,7 @@ void convergence_information_t::compute_primal_objective_owned_partial problem_ptr->objective_coefficients.data(), primal_stride, primal_objective_.data(), - stream_view_)); + stream_view_.get())); } template @@ -846,7 +848,7 @@ template void convergence_information_t::apply_primal_objective_scaling_and_offset() { const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); - apply_objective_scaling_and_offset<<>>( + apply_objective_scaling_and_offset<<>>( make_span(primal_objective_), problem_ptr->presolve_data.objective_scaling_factor, make_span(objective_offsets_), @@ -888,7 +890,7 @@ void convergence_information_t::compute_dual_residual( cusparse_view.tmp_primal, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); } else { RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmm(handle_ptr_->get_cusparse_handle(), @@ -901,7 +903,7 @@ void convergence_information_t::compute_dual_residual( cusparse_view.batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)cusparse_view.buffer_transpose_batch.data(), - stream_view_)); + stream_view_.get())); } // Substract with the objective vector manually to avoid possible cusparse bug w/ nonzero beta and @@ -922,7 +924,7 @@ void convergence_information_t::compute_dual_residual( dual_residual_.data(), dual_residual_.size(), cuda::std::minus<>{}, - stream_view_.value()); + stream_view_.get()); } else { cuopt_expects(!batch_mode_, error_type_t::ValidationError, @@ -935,7 +937,7 @@ void convergence_information_t::compute_dual_residual( tmp_primal.data(), // primal_gradient reduced_cost_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); } } @@ -963,7 +965,7 @@ void convergence_information_t::compute_dual_objective_owned_partial( primal_solution.data(), primal_stride, dual_dot_.data(), - stream_view_)); + stream_view_.get())); // sum_primal_slack_ = Σ primal_slack_[0:n_owned_cstr] // primal_slack_ is assumed populated for owned cstrs by a prior @@ -973,7 +975,7 @@ void convergence_information_t::compute_dual_objective_owned_partial( primal_slack_.data(), sum_primal_slack_.data(), static_cast(n_owned_cstr), - stream_view_); + stream_view_.get()); // dual_objective_ = dual_dot_ + sum_primal_slack_ (still a partial sum). cub::DeviceTransform::Transform(cuda::std::make_tuple(dual_dot_.data(), sum_primal_slack_.data()), @@ -1008,14 +1010,14 @@ void convergence_information_t::compute_dual_objective( problem_ptr->constraint_upper_bounds.data(), dual_size_h_, constraint_bound_value_reduced_cost_product(), - stream_view_); + stream_view_.get()); cub::DeviceReduce::Sum(rmm_tmp_buffer_.data(), size_of_buffer_, bound_value_.begin(), dual_objective_.data(), dual_size_h_, - stream_view_); + stream_view_.get()); compute_reduced_costs_dual_objective_contribution(); @@ -1023,7 +1025,7 @@ void convergence_information_t::compute_dual_objective( dual_objective_.data(), reduced_cost_dual_objective_.data(), 1, - stream_view_); + stream_view_.get()); } else { // Reflected path. if (!batch_mode_) { @@ -1064,7 +1066,7 @@ template void convergence_information_t::apply_dual_objective_scaling_and_offset() { const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); - apply_objective_scaling_and_offset<<>>( + apply_objective_scaling_and_offset<<>>( make_span(dual_objective_), problem_ptr->presolve_data.objective_scaling_factor, make_span(objective_offsets_), @@ -1084,7 +1086,7 @@ void convergence_information_t::compute_reduced_cost_from_primal_gradi bound_value_.data(), primal_size_h_, bound_value_gradient(), - stream_view_.value()); + stream_view_.get()); if (hyper_params_.handle_some_primal_gradients_on_finite_bounds_as_residuals) { raft::linalg::ternaryOp(reduced_cost_.data(), @@ -1093,14 +1095,14 @@ void convergence_information_t::compute_reduced_cost_from_primal_gradi primal_gradient.data(), primal_size_h_, copy_gradient_if_should_be_reduced_cost(), - stream_view_); + stream_view_.get()); } else { raft::linalg::binaryOp(reduced_cost_.data(), bound_value_.data(), primal_gradient.data(), primal_size_h_, copy_gradient_if_finite_bounds(), - stream_view_); + stream_view_.get()); } } @@ -1117,7 +1119,7 @@ void convergence_information_t::compute_reduced_costs_dual_objective_c bound_value_.data(), primal_size_h_, bound_value_reduced_cost_product(), - stream_view_.value()); + stream_view_.get()); // sum over bound_value*reduced_cost, but should be -inf if any element is -inf cub::DeviceReduce::Sum(rmm_tmp_buffer_.data(), @@ -1125,7 +1127,7 @@ void convergence_information_t::compute_reduced_costs_dual_objective_c bound_value_.begin(), reduced_cost_dual_objective_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); } template diff --git a/cpp/src/pdlp/termination_strategy/convergence_information.hpp b/cpp/src/pdlp/termination_strategy/convergence_information.hpp index 1bcb2fc0ab..a1ed2bb497 100644 --- a/cpp/src/pdlp/termination_strategy/convergence_information.hpp +++ b/cpp/src/pdlp/termination_strategy/convergence_information.hpp @@ -190,7 +190,7 @@ class convergence_information_t { raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; i_t primal_size_h_; i_t dual_size_h_; diff --git a/cpp/src/pdlp/termination_strategy/infeasibility_information.cu b/cpp/src/pdlp/termination_strategy/infeasibility_information.cu index 7e38ffa845..ad14ba68f8 100644 --- a/cpp/src/pdlp/termination_strategy/infeasibility_information.cu +++ b/cpp/src/pdlp/termination_strategy/infeasibility_information.cu @@ -104,11 +104,11 @@ infeasibility_information_t::infeasibility_information_t( RAFT_CUDA_TRY(cudaMemsetAsync(homogenous_primal_residual_.data(), 0.0, sizeof(f_t) * homogenous_primal_residual_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync(homogenous_dual_residual_.data(), 0.0, sizeof(f_t) * homogenous_dual_residual_.size(), - stream_view_)); + stream_view_.get())); // variable bounds in the homogenous primal are 0.0 if the original bound was finite, and // otherwise it is -inf for lower bounds and inf for upper bounds @@ -116,12 +116,12 @@ infeasibility_information_t::infeasibility_information_t( problem_ptr->constraint_lower_bounds.data(), dual_size_h_, zero_if_is_finite(), - stream_view_); + stream_view_.get()); raft::linalg::unaryOp(homogenous_dual_upper_bounds_.data(), problem_ptr->constraint_upper_bounds.data(), dual_size_h_, zero_if_is_finite(), - stream_view_); + stream_view_.get()); void* d_temp_storage = NULL; size_t temp_storage_bytes_1 = 0; @@ -130,7 +130,7 @@ infeasibility_information_t::infeasibility_information_t( bound_value_.begin(), dual_ray_linear_objective_.data(), dual_size_h_, - stream_view_); + stream_view_.get()); size_t temp_storage_bytes_2 = 0; cub::DeviceReduce::Sum(d_temp_storage, @@ -138,7 +138,7 @@ infeasibility_information_t::infeasibility_information_t( bound_value_.begin(), reduced_cost_dual_objective_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); size_of_buffer_ = std::max({temp_storage_bytes_1, temp_storage_bytes_2}); this->rmm_tmp_buffer_ = rmm::device_buffer{size_of_buffer_, stream_view_}; @@ -146,20 +146,20 @@ infeasibility_information_t::infeasibility_information_t( RAFT_CUDA_TRY(cudaMemsetAsync(dual_ray_linear_objective_.data(), 0, sizeof(f_t) * dual_ray_linear_objective_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync(max_dual_ray_infeasibility_.data(), 0, sizeof(f_t) * max_dual_ray_infeasibility_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync(primal_ray_linear_objective_.data(), 0, sizeof(f_t) * primal_ray_linear_objective_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync(max_primal_ray_infeasibility_.data(), 0, sizeof(f_t) * max_primal_ray_infeasibility_.size(), - stream_view_)); + stream_view_.get())); } } @@ -327,7 +327,7 @@ void infeasibility_information_t::compute_infeasibility_information( scaled_cusparse_view_.batch_tmp_duals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)scaled_cusparse_view_.buffer_non_transpose_batch.data(), - stream_view_)); + stream_view_.get())); RAFT_CUSPARSE_TRY( raft::sparse::detail::cusparsespmm(handle_ptr_->get_cusparse_handle(), CUSPARSE_OPERATION_NON_TRANSPOSE, @@ -339,7 +339,7 @@ void infeasibility_information_t::compute_infeasibility_information( scaled_cusparse_view_.batch_tmp_primals, CUSPARSE_SPMM_CSR_ALG3, (f_t*)scaled_cusparse_view_.buffer_transpose_batch.data(), - stream_view_)); + stream_view_.get())); #ifdef CUPDLP_DEBUG_MODE print("primal_product", current_pdhg_solver.get_dual_tmp_resource()); @@ -507,12 +507,12 @@ void infeasibility_information_t::compute_infeasibility_information( reusable_device_scalar_value_1_.data(), primal_ray_inf_norm_.data(), 1, - stream_view_); + stream_view_.get()); raft::linalg::eltwiseMultiply(neg_primal_ray_inf_norm_inverse_.data(), primal_ray_inf_norm_inverse_.data(), reusable_device_scalar_value_neg_1_.data(), 1, - stream_view_); + stream_view_.get()); compute_homogenous_primal_residual(op_problem_cusparse_view_, current_pdhg_solver.get_dual_tmp_resource()); @@ -531,14 +531,14 @@ void infeasibility_information_t::compute_infeasibility_information( my_inf_norm(dual_ray, dual_ray_inf_norm_, handle_ptr_); my_inf_norm(reduced_cost_, reduced_cost_inf_norm_, handle_ptr_); - compute_remaining_stats_kernel<<<1, 1, 0, stream_view_>>>(this->view()); + compute_remaining_stats_kernel<<<1, 1, 0, stream_view_.get()>>>(this->view()); RAFT_CUDA_TRY(cudaPeekAtLastError()); // reset for next round RAFT_CUDA_TRY(cudaMemsetAsync(homogenous_primal_residual_.data(), 0.0, sizeof(f_t) * homogenous_primal_residual_.size(), - stream_view_)); + stream_view_.get())); RAFT_CUDA_TRY(cudaMemsetAsync( homogenous_dual_residual_.data(), 0.0, sizeof(f_t) * homogenous_dual_residual_.size())); } @@ -558,7 +558,7 @@ void infeasibility_information_t::compute_homogenous_primal_residual( cusparse_view.tmp_dual, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_non_transpose.data(), - stream_view_)); + stream_view_.get())); raft::linalg::ternaryOp(homogenous_primal_residual_.data(), tmp_dual.data(), @@ -566,7 +566,7 @@ void infeasibility_information_t::compute_homogenous_primal_residual( homogenous_dual_upper_bounds_.data(), dual_size_h_, violation(), - stream_view_); + stream_view_.get()); } template @@ -599,14 +599,14 @@ void infeasibility_information_t::compute_homogenous_primal_objective( problem_ptr->objective_coefficients.data(), primal_stride, primal_ray_linear_objective_.data(), - stream_view_)); + stream_view_.get())); // just to scale from the primal ray scaling raft::linalg::eltwiseMultiply(primal_ray_linear_objective_.data(), primal_ray_linear_objective_.data(), primal_ray_inf_norm_inverse_.data(), 1, - stream_view_); + stream_view_.get()); } template @@ -628,7 +628,7 @@ void infeasibility_information_t::compute_homogenous_dual_residual( cusparse_view.tmp_primal, CUSPARSE_SPMV_CSR_ALG2, (f_t*)cusparse_view.buffer_transpose.data(), - stream_view_)); + stream_view_.get())); compute_reduced_cost_from_primal_gradient(tmp_primal, primal_ray); // primal gradient is now in temp @@ -637,7 +637,7 @@ void infeasibility_information_t::compute_homogenous_dual_residual( tmp_primal.data(), // primal_gradient reduced_cost_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); } template @@ -650,14 +650,14 @@ void infeasibility_information_t::compute_homogenous_dual_objective( problem_ptr->constraint_upper_bounds.data(), dual_size_h_, constraint_bound_value_reduced_cost_product(), - stream_view_); + stream_view_.get()); cub::DeviceReduce::Sum(rmm_tmp_buffer_.data(), size_of_buffer_, bound_value_.begin(), dual_ray_linear_objective_.data(), dual_size_h_, - stream_view_); + stream_view_.get()); #ifdef PDLP_DEBUG_MODE std::cout << "-compute_homogenous_dual_objective:\n" @@ -671,7 +671,7 @@ void infeasibility_information_t::compute_homogenous_dual_objective( dual_ray_linear_objective_.data(), reduced_cost_dual_objective_.data(), 1, - stream_view_); + stream_view_.get()); #ifdef PDLP_DEBUG_MODE std::cout << " reduced_cost_dual_objective_=" << reduced_cost_dual_objective_.value(stream_view_) << std::endl; @@ -690,7 +690,7 @@ void infeasibility_information_t::compute_reduced_cost_from_primal_gra bound_value_.data(), primal_size_h_, bound_value_gradient(), - stream_view_.value()); + stream_view_.get()); if (hyper_params_.handle_some_primal_gradients_on_finite_bounds_as_residuals) { raft::linalg::ternaryOp(reduced_cost_.data(), @@ -699,14 +699,14 @@ void infeasibility_information_t::compute_reduced_cost_from_primal_gra primal_gradient.data(), primal_size_h_, copy_gradient_if_should_be_reduced_cost(), - stream_view_); + stream_view_.get()); } else { raft::linalg::binaryOp(reduced_cost_.data(), bound_value_.data(), primal_gradient.data(), primal_size_h_, copy_gradient_if_finite_bounds(), - stream_view_); + stream_view_.get()); } } @@ -730,7 +730,7 @@ void infeasibility_information_t::compute_reduced_costs_dual_objective bound_value_.begin(), reduced_cost_dual_objective_.data(), primal_size_h_, - stream_view_); + stream_view_.get()); } template diff --git a/cpp/src/pdlp/termination_strategy/infeasibility_information.hpp b/cpp/src/pdlp/termination_strategy/infeasibility_information.hpp index 2ccae0633f..29df789838 100644 --- a/cpp/src/pdlp/termination_strategy/infeasibility_information.hpp +++ b/cpp/src/pdlp/termination_strategy/infeasibility_information.hpp @@ -85,7 +85,7 @@ class infeasibility_information_t { void compute_reduced_costs_dual_objective_contribution(); raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; i_t primal_size_h_; i_t dual_size_h_; diff --git a/cpp/src/pdlp/termination_strategy/termination_strategy.cu b/cpp/src/pdlp/termination_strategy/termination_strategy.cu index 13acee138c..26d9e883c6 100644 --- a/cpp/src/pdlp/termination_strategy/termination_strategy.cu +++ b/cpp/src/pdlp/termination_strategy/termination_strategy.cu @@ -188,7 +188,7 @@ void pdlp_termination_strategy_t::evaluate_termination_criteria( check_termination_criteria(); // Sync to make sure the termination status is updated - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -420,13 +420,13 @@ void pdlp_termination_strategy_t::check_termination_criteria() #endif const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); check_termination_criteria_kernel - <<>>(convergence_information_.view(), - infeasibility_information_.view(), - make_span(termination_status_), - settings_.tolerances, - settings_.detect_infeasibility, - settings_.per_constraint_residual, - climber_strategies_.size()); + <<>>(convergence_information_.view(), + infeasibility_information_.view(), + make_span(termination_status_), + settings_.tolerances, + settings_.detect_infeasibility, + settings_.per_constraint_residual, + climber_strategies_.size()); RAFT_CUDA_TRY(cudaPeekAtLastError()); } @@ -499,7 +499,7 @@ void pdlp_termination_strategy_t::fill_gpu_terms_stats(i_t number_of_i const bool accept_primal_feasible = settings_.first_primal_feasible || settings_.all_primal_feasible; const auto [grid_size, block_size] = kernel_config_from_batch_size(climber_strategies_.size()); - fill_gpu_terms_stats_kernel<<>>( + fill_gpu_terms_stats_kernel<<>>( make_span(termination_status_), make_span(original_index_), gpu_batch_additional_termination_information_.view(), @@ -509,7 +509,7 @@ void pdlp_termination_strategy_t::fill_gpu_terms_stats(i_t number_of_i settings_.per_constraint_residual, force_all); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); } template @@ -641,7 +641,7 @@ pdlp_termination_strategy_t::fill_return_problem_solution( } } - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view_.get())); if (deep_copy) { cuopt_assert( diff --git a/cpp/src/pdlp/termination_strategy/termination_strategy.hpp b/cpp/src/pdlp/termination_strategy/termination_strategy.hpp index 948f7674d0..fb8ba107c3 100644 --- a/cpp/src/pdlp/termination_strategy/termination_strategy.hpp +++ b/cpp/src/pdlp/termination_strategy/termination_strategy.hpp @@ -214,7 +214,7 @@ class pdlp_termination_strategy_t { void check_termination_criteria(); raft::handle_t const* handle_ptr_{nullptr}; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; mip::problem_t* problem_ptr; diff --git a/cpp/src/pdlp/translate.hpp b/cpp/src/pdlp/translate.hpp index d45d25ecfd..135d3168f6 100644 --- a/cpp/src/pdlp/translate.hpp +++ b/cpp/src/pdlp/translate.hpp @@ -354,14 +354,14 @@ void translate_to_crossover_problem(const mip::problem_t& problem, csr_A.j = std::vector(cuopt::host_copy(problem.variables, stream)); csr_A.row_start = std::vector(cuopt::host_copy(problem.offsets, stream)); - stream.synchronize(); + stream.sync(); CUOPT_LOG_DEBUG("Converting to compressed column"); csr_A.to_compressed_col(lp.A); CUOPT_LOG_DEBUG("Converted to compressed column"); std::vector slack(problem.n_constraints); std::vector tmp_x = cuopt::host_copy(sol.get_primal_solution(), stream); - stream.synchronize(); + stream.sync(); matrix_vector_multiply(lp.A, f_t(1.0), tmp_x, f_t(0.0), slack); CUOPT_LOG_DEBUG("Multiplied A and x"); @@ -400,7 +400,7 @@ void translate_to_crossover_problem(const mip::problem_t& problem, std::copy(lower.begin(), lower.begin() + problem.n_variables, lp.lower.begin()); std::copy(upper.begin(), upper.begin() + problem.n_variables, lp.upper.begin()); - problem.handle_ptr->get_stream().synchronize(); + problem.handle_ptr->get_stream().sync(); for (i_t i = 0; i < m; ++i) { lp.lower[problem.n_variables + i] = constraint_lower[i]; lp.upper[problem.n_variables + i] = constraint_upper[i]; @@ -420,7 +420,7 @@ void translate_to_crossover_problem(const mip::problem_t& problem, initial_solution.y = cuopt::host_copy(sol.get_dual_solution(), stream); std::vector tmp_z = cuopt::host_copy(sol.get_reduced_cost(), stream); - stream.synchronize(); + stream.sync(); std::copy(tmp_z.begin(), tmp_z.begin() + problem.n_variables, initial_solution.z.begin()); for (i_t j = problem.n_variables; j < n; ++j) { initial_solution.z[j] = initial_solution.y[j - problem.n_variables]; diff --git a/cpp/src/pdlp/utilities/ping_pong_graph.cu b/cpp/src/pdlp/utilities/ping_pong_graph.cu index eb1f31116e..00a08f3611 100644 --- a/cpp/src/pdlp/utilities/ping_pong_graph.cu +++ b/cpp/src/pdlp/utilities/ping_pong_graph.cu @@ -10,8 +10,7 @@ namespace cuopt::mathematical_optimization::pdlp { template -ping_pong_graph_t::ping_pong_graph_t(rmm::cuda_stream_view stream_view, - bool is_legacy_batch_mode) +ping_pong_graph_t::ping_pong_graph_t(cuda::stream_ref stream_view, bool is_legacy_batch_mode) : stream_view_(stream_view), is_legacy_batch_mode_(is_legacy_batch_mode) { } diff --git a/cpp/src/pdlp/utilities/ping_pong_graph.cuh b/cpp/src/pdlp/utilities/ping_pong_graph.cuh index 4f895fffd1..913e429500 100644 --- a/cpp/src/pdlp/utilities/ping_pong_graph.cuh +++ b/cpp/src/pdlp/utilities/ping_pong_graph.cuh @@ -26,7 +26,7 @@ namespace cuopt::mathematical_optimization::pdlp { template class ping_pong_graph_t { public: - ping_pong_graph_t(rmm::cuda_stream_view stream_view, bool is_legacy_batch_mode = false); + ping_pong_graph_t(cuda::stream_ref stream_view, bool is_legacy_batch_mode = false); ~ping_pong_graph_t() = default; // Non-copyable because the underlying manual_cuda_graph_t owns a @@ -64,7 +64,7 @@ class ping_pong_graph_t { private: manual_cuda_graph_t even_graph_; manual_cuda_graph_t odd_graph_; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; bool is_legacy_batch_mode_{false}; }; diff --git a/cpp/src/pdlp/utils.cuh b/cpp/src/pdlp/utils.cuh index 25cd790a48..c2ba2bbe79 100644 --- a/cpp/src/pdlp/utils.cuh +++ b/cpp/src/pdlp/utils.cuh @@ -338,7 +338,7 @@ template void inline compute_sum_bounds_squared(const rmm::device_uvector& constraint_lower_bounds, const rmm::device_uvector& constraint_upper_bounds, rmm::device_scalar& out, - rmm::cuda_stream_view stream_view, + cuda::stream_ref stream_view, std::size_t n) { cuopt_assert(constraint_lower_bounds.size() == constraint_upper_bounds.size(), @@ -356,7 +356,7 @@ void inline compute_sum_bounds_squared(const rmm::device_uvector& constrain cuda::std::plus<>{}, rhs_sum_of_squares_t{}, f_t(0), - stream_view); + stream_view.get()); d_temp_storage.resize(bytes, stream_view); @@ -369,8 +369,8 @@ void inline compute_sum_bounds_squared(const rmm::device_uvector& constrain cuda::std::plus<>{}, rhs_sum_of_squares_t{}, f_t(0), - stream_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } // Weighted sum of squares of the first n entries of `values` (no fused sqrt). @@ -379,7 +379,7 @@ template void inline compute_sum_weighted_squares(const rmm::device_uvector& values, f_t weight, rmm::device_scalar& out, - rmm::cuda_stream_view stream_view, + cuda::stream_ref stream_view, std::size_t n) { cuopt_assert(n <= values.size(), "n exceeds values size"); @@ -394,7 +394,7 @@ void inline compute_sum_weighted_squares(const rmm::device_uvector& values, cuda::std::plus<>{}, weighted_square_op{weight}, f_t(0), - stream_view); + stream_view.get()); d_temp_storage.resize(bytes, stream_view); @@ -406,8 +406,8 @@ void inline compute_sum_weighted_squares(const rmm::device_uvector& values, cuda::std::plus<>{}, weighted_square_op{weight}, f_t(0), - stream_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } // Like compute_sum_bounds_squared, but writes sqrt(sum of squares) (the L2 norm). @@ -415,7 +415,7 @@ template void inline compute_sum_bounds(const rmm::device_uvector& constraint_lower_bounds, const rmm::device_uvector& constraint_upper_bounds, f_t* out, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { rmm::device_buffer d_temp_storage; size_t bytes = 0; @@ -428,7 +428,7 @@ void inline compute_sum_bounds(const rmm::device_uvector& constraint_lower_ cuda::std::plus<>{}, rhs_sum_of_squares_t{}, f_t(0), - stream_view); + stream_view.get()); d_temp_storage.resize(bytes, stream_view); @@ -441,15 +441,15 @@ void inline compute_sum_bounds(const rmm::device_uvector& constraint_lower_ cuda::std::plus<>{}, rhs_sum_of_squares_t{}, f_t(0), - stream_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view)); + stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } template void inline compute_sum_bounds(const rmm::device_uvector& constraint_lower_bounds, const rmm::device_uvector& constraint_upper_bounds, rmm::device_scalar& out, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { compute_sum_bounds(constraint_lower_bounds, constraint_upper_bounds, out.data(), stream_view); } @@ -688,7 +688,7 @@ void inline my_l2_norm(const f_t* in, f_t* out, size_t size, raft::handle_t cons { constexpr int stride = 1; RAFT_CUBLAS_TRY(raft::linalg::detail::cublasnrm2( - handle_ptr->get_cublas_handle(), size, in, stride, out, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), size, in, stride, out, handle_ptr->get_stream().get())); } template @@ -712,7 +712,7 @@ void inline my_l2_weighted_norm(const f_t* input_vector, size_t size, f_t weight, rmm::device_scalar& result, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto fin_op = [] __device__(f_t in) { return raft::sqrt(in); }; auto main_op = [weight] __device__(f_t in, i_t _) { return in * in * weight; }; @@ -721,7 +721,7 @@ void inline my_l2_weighted_norm(const f_t* input_vector, (i_t)size, 1, f_t(0.0), - stream, + stream.get(), false, main_op, raft::Sum(), @@ -732,7 +732,7 @@ template void inline my_l2_weighted_norm(rmm::device_uvector& input_vector, f_t weight, rmm::device_scalar& result, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { my_l2_weighted_norm(input_vector.data(), input_vector.size(), weight, result, stream); } @@ -779,10 +779,10 @@ void inline my_inf_norm(const rmm::device_uvector& input_vector, void* d_temp = nullptr; size_t temp_bytes = 0; - cub::DeviceReduce::Max(d_temp, temp_bytes, abs_iter, result, n, stream); + cub::DeviceReduce::Max(d_temp, temp_bytes, abs_iter, result, n, stream.get()); rmm::device_buffer temp_buf(temp_bytes, stream); - cub::DeviceReduce::Max(temp_buf.data(), temp_bytes, abs_iter, result, n, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + cub::DeviceReduce::Max(temp_buf.data(), temp_bytes, abs_iter, result, n, stream.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); } template diff --git a/cpp/src/routing/adapters/assignment_adapter.cuh b/cpp/src/routing/adapters/assignment_adapter.cuh index c41c3e161e..9e823d8464 100644 --- a/cpp/src/routing/adapters/assignment_adapter.cuh +++ b/cpp/src/routing/adapters/assignment_adapter.cuh @@ -27,7 +27,7 @@ assignment_t ges_solver_t::get_ges_assignment( // the stream should be the initial handle stream and not the sol_handle stream as this data will // be exported auto stream = problem.handle_ptr->get_stream(); - stream.synchronize(); + stream.sync(); const auto& problem = *sol.problem_ptr; i_t n_output_nodes = sol.get_n_routes() * 2 + sol.get_num_depot_excluded_orders() + @@ -39,7 +39,7 @@ assignment_t ges_solver_t::get_ges_assignment( rmm::device_uvector route_locations_out(0, stream); rmm::device_uvector node_types_out(0, stream); auto accepted_out = cuopt::device_copy(accepted, stream); - stream.synchronize(); + stream.sync(); std::vector node_types_out_h(n_output_nodes); std::vector route_out_h(n_output_nodes); std::vector truck_id_out_h(n_output_nodes); @@ -150,7 +150,7 @@ assignment_t ges_solver_t::get_ges_assignment( auto unserviced_nodes_h = sol.get_unserviced_nodes(); auto unserviced_nodes = cuopt::device_copy(unserviced_nodes_h, stream); - stream.synchronize(); + stream.sync(); std::map objective_values; for (int i = 0; i < (int)objective_t::SIZE; ++i) { diff --git a/cpp/src/routing/assignment.cu b/cpp/src/routing/assignment.cu index be40bda183..a04838de3c 100644 --- a/cpp/src/routing/assignment.cu +++ b/cpp/src/routing/assignment.cu @@ -21,7 +21,7 @@ const std::string solution_string_t::empty = "cuOpt solver did not run."; const std::string solution_string_t::error = "An error occured while running the cuOpt solver."; template -assignment_t::assignment_t(solution_status_t status, rmm::cuda_stream_view stream_view) +assignment_t::assignment_t(solution_status_t status, cuda::stream_ref stream_view) : status_(status), route_(0, stream_view), arrival_stamp_(0, stream_view), @@ -36,7 +36,7 @@ assignment_t::assignment_t(solution_status_t status, rmm::cuda_stream_view } template -assignment_t::assignment_t(cuopt::logic_error error_status, rmm::cuda_stream_view stream_view) +assignment_t::assignment_t(cuopt::logic_error error_status, cuda::stream_ref stream_view) : status_(solution_status_t::ERROR), route_(0, stream_view), arrival_stamp_(0, stream_view), @@ -188,7 +188,7 @@ const rmm::device_uvector& assignment_t::get_accepted() const noexcept } template -void assignment_t::to_csv(std::string_view filename, rmm::cuda_stream_view stream_view) +void assignment_t::to_csv(std::string_view filename, cuda::stream_ref stream_view) { std::vector route; std::vector arrival_stamp; @@ -196,10 +196,9 @@ void assignment_t::to_csv(std::string_view filename, rmm::cuda_stream_view route.resize(route_.size()); arrival_stamp.resize(arrival_stamp_.size()); truck_id.resize(truck_id_.size()); - raft::copy(route.data(), route_.data(), route_.size(), stream_view.value()); - raft::copy( - arrival_stamp.data(), arrival_stamp_.data(), arrival_stamp_.size(), stream_view.value()); - raft::copy(truck_id.data(), truck_id_.data(), truck_id_.size(), stream_view.value()); + raft::copy(route.data(), route_.data(), route_.size(), stream_view.get()); + raft::copy(arrival_stamp.data(), arrival_stamp_.data(), arrival_stamp_.size(), stream_view.get()); + raft::copy(truck_id.data(), truck_id_.data(), truck_id_.size(), stream_view.get()); std::ofstream myfile(filename.data()); std::cout << "truck_id,\troute,\tarrival_time\n"; for (size_t i = 0; i < route.size(); i++) diff --git a/cpp/src/routing/cpu_routing_problem.cu b/cpp/src/routing/cpu_routing_problem.cu index fb61c7f8cf..35532a6946 100644 --- a/cpp/src/routing/cpu_routing_problem.cu +++ b/cpp/src/routing/cpu_routing_problem.cu @@ -72,14 +72,14 @@ namespace { template std::unique_ptr> copy_vector(std::vector const& host, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (host.empty()) { return nullptr; } return std::make_unique>(cuopt::device_copy(host, stream)); } std::unique_ptr> copy_u8_as_bool(std::vector const& host, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { if (host.empty()) { return nullptr; } std::vector as_bool(host.begin(), host.end()); @@ -87,7 +87,7 @@ std::unique_ptr> copy_u8_as_bool(std::vector // as_bool is a local temporary and the H2D copy above is async; drain the // stream before it goes out of scope so the copy does not read freed host // memory. - stream.synchronize(); + stream.sync(); return d; } @@ -302,7 +302,7 @@ cpu_routing_problem_t::to_device(raft::handle_t* handle) const data->init_types = copy_vector(types, stream); // types is a local temporary feeding an async H2D copy; drain before it // goes out of scope. - stream.synchronize(); + stream.sync(); int32_t n_nodes = static_cast(initial_solutions.routes.size()); int32_t n_sols = static_cast(initial_solutions.sol_offsets.size()); diff --git a/cpp/src/routing/crossovers/optimal_eax_cycles.cu b/cpp/src/routing/crossovers/optimal_eax_cycles.cu index d5547d2c21..9d9e67ffdc 100644 --- a/cpp/src/routing/crossovers/optimal_eax_cycles.cu +++ b/cpp/src/routing/crossovers/optimal_eax_cycles.cu @@ -151,7 +151,7 @@ void optimal_cycles_t::get_min_delta_and_index( eax_cycle_delta.data(), index_delta_pair.data(), num_items, - sol.sol.sol_handle->get_stream()); + sol.sol.sol_handle->get_stream().get()); // Allocate temporary storage if (d_cub_storage_bytes.size() < temp_storage_bytes) { d_cub_storage_bytes.resize(temp_storage_bytes, sol.sol.sol_handle->get_stream()); @@ -162,7 +162,7 @@ void optimal_cycles_t::get_min_delta_and_index( eax_cycle_delta.data(), index_delta_pair.data(), num_items, - sol.sol.sol_handle->get_stream()); + sol.sol.sol_handle->get_stream().get()); } template @@ -179,8 +179,9 @@ bool optimal_cycles_t::insert_cycle_to_found_position( return false; } // prepare the rotations once and copy them to respective device arrays - insert_optimal_rotation_kernel<<<1, TPB, sh_size, solution.sol_handle->get_stream()>>>( - solution.view(), index_delta_pair.data(), eax_fragment.view(), n_rotations); + insert_optimal_rotation_kernel + <<<1, TPB, sh_size, solution.sol_handle->get_stream().get()>>>( + solution.view(), index_delta_pair.data(), eax_fragment.view(), n_rotations); solution.compute_route_id_per_node(); solution.compute_cost(); return true; @@ -216,19 +217,20 @@ bool optimal_cycles_t::add_cycles_request( constexpr i_t TPB = 128; // prepare the rotations once and copy them to respective device arrays - create_rotations_kernel<<<1, TPB, 0, solution.sol_handle->get_stream()>>>( + create_rotations_kernel<<<1, TPB, 0, solution.sol_handle->get_stream().get()>>>( solution.view(), raft::device_span>(d_cycle.data(), d_cycle.size()), eax_fragment.view(), n_rotations); i_t n_blocks = (n_rotations * n_positions + TPB - 1) / TPB; - find_optimal_position_kernel<<get_stream()>>>( - solution.view(), - resource.ls.move_candidates.view(), - eax_fragment.view(), - n_rotations, - raft::device_span(eax_cycle_delta.data(), eax_cycle_delta.size())); + find_optimal_position_kernel + <<get_stream().get()>>>( + solution.view(), + resource.ls.move_candidates.view(), + eax_fragment.view(), + n_rotations, + raft::device_span(eax_cycle_delta.data(), eax_cycle_delta.size())); get_min_delta_and_index(sol, n_rotations * n_positions); bool success = insert_cycle_to_found_position(sol, n_rotations); diff --git a/cpp/src/routing/crossovers/ox_graph.hpp b/cpp/src/routing/crossovers/ox_graph.hpp index 55d2e39a0b..b85801e2d0 100644 --- a/cpp/src/routing/crossovers/ox_graph.hpp +++ b/cpp/src/routing/crossovers/ox_graph.hpp @@ -13,7 +13,7 @@ namespace detail { template struct ox_graph_t { - ox_graph_t(i_t n_buckets_, i_t size, i_t max_nodes_per_row, rmm::cuda_stream_view stream) + ox_graph_t(i_t n_buckets_, i_t size, i_t max_nodes_per_row, cuda::stream_ref stream) : row_sizes(n_buckets_ * size, stream), route_ids(n_buckets_ * size, stream), // allocate with the max size @@ -39,7 +39,7 @@ struct ox_graph_t { std::vector buckets; }; - host_t to_host(rmm::cuda_stream_view stream) + host_t to_host(cuda::stream_ref stream) { host_t h; h.row_sizes = host_copy(row_sizes, stream); @@ -50,7 +50,7 @@ struct ox_graph_t { return h; } - void resize(i_t n_buckets_, i_t size, i_t max_nodes_per_row, rmm::cuda_stream_view stream) + void resize(i_t n_buckets_, i_t size, i_t max_nodes_per_row, cuda::stream_ref stream) { n_buckets = n_buckets_; row_sizes.resize(n_buckets * size, stream); diff --git a/cpp/src/routing/crossovers/ox_recombiner.cuh b/cpp/src/routing/crossovers/ox_recombiner.cuh index cefbd8df15..6902c2ab14 100644 --- a/cpp/src/routing/crossovers/ox_recombiner.cuh +++ b/cpp/src/routing/crossovers/ox_recombiner.cuh @@ -94,7 +94,7 @@ struct OX { ox_graph_t d_graph; ox_graph_t transpose_graph; - explicit OX(size_t nodes_number, const costs& weight, rmm::cuda_stream_view stream_view) + explicit OX(size_t nodes_number, const costs& weight, cuda::stream_ref stream_view) : mt(rd()), problem_size(nodes_number), graph(problem_size), @@ -518,7 +518,7 @@ struct OX { } } - void test_transpose_graph(rmm::cuda_stream_view stream) + void test_transpose_graph(cuda::stream_ref stream) { std::vector>> h_transpose_graph(offspring.size()); for (size_t i = 0; i < h_transpose_graph.size(); ++i) { @@ -592,7 +592,7 @@ struct OX { num_segments, row_offsets.data(), row_offsets.data() + 1, - stream_view); + stream_view.get()); d_tmp_storage_bytes.resize(tmp_storage_bytes, stream_view); cub::DeviceSegmentedSort::SortPairs(d_tmp_storage_bytes.data(), tmp_storage_bytes, @@ -604,7 +604,7 @@ struct OX { num_segments, row_offsets.data(), row_offsets.data() + 1, - stream_view); + stream_view.get()); RAFT_CHECK_CUDA(stream_view); thrust::gather(policy, val_map.begin(), val_map.end(), graph.buckets.data(), gather_int.data()); @@ -622,7 +622,7 @@ struct OX { auto const n_blocks = n_buckets * d_graph.get_num_vertices(); transpose_graph.reset(A.sol.sol_handle); - transpose_graph_kernel<<get_stream()>>>( + transpose_graph_kernel<<get_stream().get()>>>( d_graph.view(), transpose_graph.view(), max_route_len); RAFT_CHECK_CUDA(A.sol.sol_handle->get_stream()); sort_graph_edges(A, transpose_graph); @@ -646,7 +646,7 @@ struct OX { async_fill(d_path_cost, std::numeric_limits::max(), A.sol.sol_handle->get_stream()); async_fill(d_predecessor, -1, A.sol.sol_handle->get_stream()); async_fill(d_predecessor_vehicle, -1, A.sol.sol_handle->get_stream()); - bellman_ford_init<<<1, 1, 0, A.sol.sol_handle->get_stream()>>>( + bellman_ford_init<<<1, 1, 0, A.sol.sol_handle->get_stream().get()>>>( raft::device_span(d_path_cost.data(), d_path_cost.size()), raft::device_span(d_predecessor.data(), d_predecessor.size()), raft::device_span(d_predecessor_vehicle.data(), d_predecessor_vehicle.size())); @@ -665,7 +665,7 @@ struct OX { // routes number exceeds num nodes. Stop the search here if (n_blocks == 0) { break; } bellman_ford_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( A.sol.view(), transpose_graph.view(), raft::device_span(d_path_cost.data(), d_path_cost.size()), @@ -846,7 +846,7 @@ struct OX { } void adj_to_host(std::vector>>& h_graph, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto tmp_graph = d_graph.to_host(stream); for (int veh = 0; veh < n_buckets; ++veh) { @@ -971,7 +971,7 @@ struct OX { return; } calculate_edge_costs_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( A.sol.view(), d_graph.view(), raft::device_span(d_offspring.data(), d_offspring.size()), diff --git a/cpp/src/routing/cuda_graph.cuh b/cpp/src/routing/cuda_graph.cuh index 1fb2425d2c..42838eacb5 100644 --- a/cpp/src/routing/cuda_graph.cuh +++ b/cpp/src/routing/cuda_graph.cuh @@ -18,19 +18,19 @@ namespace detail { // This is not a thread-safe class, be careful on multi-threading struct cuda_graph_t { - void start_capture(rmm::cuda_stream_view stream) + void start_capture(cuda::stream_ref stream) { // Use ThreadLocal mode to allow multi-threaded batch execution // Global mode blocks other streams from performing operations during capture - cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal); + cudaStreamBeginCapture(stream.get(), cudaStreamCaptureModeThreadLocal); capture_started = true; } - void end_capture(rmm::cuda_stream_view stream) + void end_capture(cuda::stream_ref stream) { cuopt_assert(capture_started, "start_capture was not called before end_capture!"); cuopt_expects(capture_started, error_type_t::RuntimeError, "A runtime error occurred!"); - cudaStreamEndCapture(stream, &graph); + cudaStreamEndCapture(stream.get(), &graph); capture_started = false; if (graph_created) { // If the graph fails to update, errorNode will be set to the @@ -52,7 +52,7 @@ struct cuda_graph_t { cudaGraphDestroy(graph); } - void launch_graph(rmm::cuda_stream_view stream) { cudaGraphLaunch(instance, stream); } + void launch_graph(cuda::stream_ref stream) { cudaGraphLaunch(instance, stream.get()); } bool graph_created = false; bool capture_started = false; diff --git a/cpp/src/routing/distance_engine/waypoint_matrix.cpp b/cpp/src/routing/distance_engine/waypoint_matrix.cpp index 030c8790ea..e02d2ce970 100644 --- a/cpp/src/routing/distance_engine/waypoint_matrix.cpp +++ b/cpp/src/routing/distance_engine/waypoint_matrix.cpp @@ -248,7 +248,7 @@ void waypoint_matrix_t::compute_cost_matrix(f_t* d_cost_matrix, std::vector cost_matrix = mpsp(target_locations, n_target_locations); raft::copy(d_cost_matrix, cost_matrix.data(), cost_matrix.size(), stream_view_); - stream_view_.synchronize(); + stream_view_.sync(); } // Location values are greater or equal to n_target_locations @@ -293,7 +293,7 @@ waypoint_matrix_t::compute_waypoint_sequence(i_t const* target_locatio std::vector h_locations(n_locations); raft::copy(h_locations.data(), locations, n_locations, stream_view_); - stream_view_.synchronize(); + stream_view_.sync(); // Locations validity checks check_locations(h_locations.data(), n_locations, n_target_locations); @@ -321,7 +321,7 @@ waypoint_matrix_t::compute_waypoint_sequence(i_t const* target_locatio raft::copy(paths_offsets_out.data(), paths_offsets.data(), paths_offsets.size(), stream_view_); raft::copy(paths_list_out.data(), paths_list.data(), paths_list.size(), stream_view_); - stream_view_.synchronize(); + stream_view_.sync(); return {std::make_unique(paths_offsets_out.release()), std::make_unique(paths_list_out.release())}; @@ -406,7 +406,7 @@ void waypoint_matrix_t::compute_shortest_path_costs(f_t* d_custom_matr raft::copy( d_custom_matrix, shortest_path_matrix.data(), shortest_path_matrix.size(), stream_view_); - stream_view_.synchronize(); + stream_view_.sync(); } template class CUOPT_EXPORT waypoint_matrix_t; diff --git a/cpp/src/routing/fleet_info.cu b/cpp/src/routing/fleet_info.cu index 317191f51f..b1fb91e31c 100644 --- a/cpp/src/routing/fleet_info.cu +++ b/cpp/src/routing/fleet_info.cu @@ -153,9 +153,9 @@ void populate_fleet_info(data_model_view_t const& data_model, if (auto [start_locations, return_locations] = data_model.get_vehicle_locations(); start_locations != nullptr) { raft::copy( - fleet_info_.v_start_locations_.data(), start_locations, fleet_size, stream_view.value()); + fleet_info_.v_start_locations_.data(), start_locations, fleet_size, stream_view.get()); raft::copy( - fleet_info_.v_return_locations_.data(), return_locations, fleet_size, stream_view.value()); + fleet_info_.v_return_locations_.data(), return_locations, fleet_size, stream_view.get()); is_homogenous = is_homogenous && all_entries_are_equal(handle_ptr_, fleet_info_.v_start_locations_.data(), fleet_size); @@ -176,7 +176,7 @@ void populate_fleet_info(data_model_view_t const& data_model, if (auto drop_return_trip = data_model.get_drop_return_trips(); drop_return_trip) { raft::copy( - fleet_info_.v_drop_return_trip_.data(), drop_return_trip, fleet_size, stream_view.value()); + fleet_info_.v_drop_return_trip_.data(), drop_return_trip, fleet_size, stream_view.get()); is_homogenous = is_homogenous && all_entries_are_equal(handle_ptr_, fleet_info_.v_drop_return_trip_.data(), fleet_size); @@ -189,7 +189,7 @@ void populate_fleet_info(data_model_view_t const& data_model, if (auto skip_first_trip = data_model.get_skip_first_trips(); skip_first_trip) { raft::copy( - fleet_info_.v_skip_first_trip_.data(), skip_first_trip, fleet_size, stream_view.value()); + fleet_info_.v_skip_first_trip_.data(), skip_first_trip, fleet_size, stream_view.get()); is_homogenous = is_homogenous && all_entries_are_equal(handle_ptr_, fleet_info_.v_skip_first_trip_.data(), fleet_size); diff --git a/cpp/src/routing/fleet_info.hpp b/cpp/src/routing/fleet_info.hpp index 1a37c66554..f84b0ff252 100644 --- a/cpp/src/routing/fleet_info.hpp +++ b/cpp/src/routing/fleet_info.hpp @@ -55,7 +55,7 @@ class fleet_info_t { constexpr bool is_homogenous() const { return is_homogenous_; } - void resize(i_t size, rmm::cuda_stream_view stream) + void resize(i_t size, cuda::stream_ref stream) { v_earliest_time_.resize(size, stream); v_latest_time_.resize(size, stream); @@ -70,7 +70,7 @@ class fleet_info_t { v_buckets_.resize(size, stream); } - auto to_host(rmm::cuda_stream_view stream) + auto to_host(cuda::stream_ref stream) { host_t h; h.break_offset = host_copy(v_break_offset_, stream); @@ -237,7 +237,7 @@ class fleet_info_t { constexpr raft::device_span get_break_vector(i_t truck_id, const rmm::device_uvector& vec, - rmm::cuda_stream_view stream) const + cuda::stream_ref stream) const { if (!vec.is_empty()) { i_t offset = v_break_offset_.element(truck_id, stream); @@ -248,8 +248,7 @@ class fleet_info_t { } } - constexpr VehicleInfo get_vehicle_info(const i_t vehicle_id, - rmm::cuda_stream_view stream) const + constexpr VehicleInfo get_vehicle_info(const i_t vehicle_id, cuda::stream_ref stream) const { return v_vehicle_infos_.element(vehicle_id, stream); } diff --git a/cpp/src/routing/fleet_order_constraints.hpp b/cpp/src/routing/fleet_order_constraints.hpp index c6be63a87c..2911f5e807 100644 --- a/cpp/src/routing/fleet_order_constraints.hpp +++ b/cpp/src/routing/fleet_order_constraints.hpp @@ -67,7 +67,7 @@ struct fleet_order_constraints_t { i_t n_vehicles; }; - host_t to_host(rmm::cuda_stream_view stream) + host_t to_host(cuda::stream_ref stream) { host_t h; h.order_service_times = host_copy(order_service_times, stream); diff --git a/cpp/src/routing/generator/generator.cu b/cpp/src/routing/generator/generator.cu index 587792ef11..7cc23fe930 100644 --- a/cpp/src/routing/generator/generator.cu +++ b/cpp/src/routing/generator/generator.cu @@ -119,7 +119,7 @@ detail::fleet_order_constraints_t generate_fleet_order_constraints( n_orders - 1, params.min_service_time, params.max_service_time + 1, - handle.get_stream()); + handle.get_stream().get()); } return fleet_order_constraints; } @@ -188,7 +188,7 @@ coordinates_t generate_coordinates(raft::handle_t& handle, params.n_locations, n_cols, n_clusters, - handle.get_stream(), + handle.get_stream().get(), false, (f_t*)nullptr, (f_t*)nullptr, @@ -228,12 +228,12 @@ d_mdarray_t generate_matrices(raft::handle_t& handle, rmm::device_uvector v_rands(params.n_locations * params.n_locations, handle.get_stream()); detail::build_cost_matrix - <<>>(cost_matrix.data(), - std::get<0>(coordinates).data(), - std::get<1>(coordinates).data(), - params.n_locations, - params.asymmetric, - asymmetry_scalar); + <<>>(cost_matrix.data(), + std::get<0>(coordinates).data(), + std::get<1>(coordinates).data(), + params.n_locations, + params.asymmetric, + asymmetry_scalar); RAFT_CHECK_CUDA(handle.get_stream()); auto seed = params.seed; @@ -248,7 +248,7 @@ d_mdarray_t generate_matrices(raft::handle_t& handle, v_rands.size(), static_cast(1.1), static_cast(1.5), - handle.get_stream()); + handle.get_stream().get()); auto matrix_span = matrices.get_cost_matrix(vehicle_type, matrix_type); @@ -309,7 +309,7 @@ rmm::device_uvector generate_vehicle_capacities(raft::handle_t& handle, fleet_size, static_cast(h_min_capacities[i]), static_cast(h_max_capacities[i] + 1), - handle.get_stream()); + handle.get_stream().get()); } return capacities; } @@ -334,7 +334,7 @@ rmm::device_uvector generate_demands(raft::handle_t& handle, params.n_locations - 1, static_cast(h_min_demand[i]), static_cast(h_max_demand[i] + 1), - handle.get_stream()); + handle.get_stream().get()); } return demands; } @@ -467,7 +467,7 @@ rmm ::device_uvector create_service_time(raft::handle_t& handle, v_service_time.size() - 1, params.min_service_time, params.max_service_time + 1, - handle.get_stream()); + handle.get_stream().get()); return v_service_time; } @@ -488,11 +488,11 @@ time_window_t generate_time_windows(raft::handle_t& handle, auto time_matrix = matrices.get_time_matrix(0); auto v_service_time = create_service_time(handle, params); detail::fill_time_windows - <<>>(time_matrix, - v_earliest_time.data(), - v_latest_time.data(), - params.tw_tightness, - params.n_locations); + <<>>(time_matrix, + v_earliest_time.data(), + v_latest_time.data(), + params.tw_tightness, + params.n_locations); handle.sync_stream(); RAFT_CHECK_CUDA(handle.get_stream()); diff --git a/cpp/src/routing/ges/compute_fragment_ejections.cu b/cpp/src/routing/ges/compute_fragment_ejections.cu index de5cd14020..9d1c2c3a78 100644 --- a/cpp/src/routing/ges/compute_fragment_ejections.cu +++ b/cpp/src/routing/ges/compute_fragment_ejections.cu @@ -122,7 +122,7 @@ bool set_shmem_for_kernel_get_best_insertion_ejection_solution(size_t dynamic_sh template void launch_kernel_get_best_insertion_ejection_solution( - dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, rmm::cuda_stream_view stream) + dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, cuda::stream_ref stream) { RAFT_CUDA_TRY(cudaLaunchKernel( (void*)kernel_get_best_insertion_ejection_solution, @@ -130,7 +130,7 @@ void launch_kernel_get_best_insertion_ejection_solution( blocks, kernel_args, shmem_bytes, - stream)); + stream.get())); } #define CUOPT_INSTANTIATE_GET_BEST_INSERTION_EJECTION(BLOCK_SIZE, REQ) \ @@ -141,7 +141,7 @@ void launch_kernel_get_best_insertion_ejection_solution( size_t dynamic_shmem_size); \ template void \ launch_kernel_get_best_insertion_ejection_solution( \ - dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, rmm::cuda_stream_view stream); + dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, cuda::stream_ref stream); CUOPT_INSTANTIATE_GET_BEST_INSERTION_EJECTION(32, PDP) CUOPT_INSTANTIATE_GET_BEST_INSERTION_EJECTION(64, PDP) diff --git a/cpp/src/routing/ges/compute_fragment_ejections.cuh b/cpp/src/routing/ges/compute_fragment_ejections.cuh index 6e560474d1..941585c7a1 100644 --- a/cpp/src/routing/ges/compute_fragment_ejections.cuh +++ b/cpp/src/routing/ges/compute_fragment_ejections.cuh @@ -40,7 +40,7 @@ bool set_shmem_for_kernel_get_best_insertion_ejection_solution(size_t dynamic_sh template void launch_kernel_get_best_insertion_ejection_solution( - dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, rmm::cuda_stream_view stream); + dim3 grid, dim3 blocks, size_t shmem_bytes, void** kernel_args, cuda::stream_ref stream); template ::eject_until_feasible(bool add_slack_to_sol) bool is_set = set_shmem_of_kernel(eject_until_feasible_kernel, sh_size); cuopt_assert(is_set, "Not enough shared memory on device for get_all_feasible_insertion!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); - eject_until_feasible_kernel<<>>( + eject_until_feasible_kernel<<>>( view(), add_slack_to_sol, problem_ptr->seed_gen.get_seed()); compute_cost(); global_runtime_checks(false, true, "eject_until_feasible"); @@ -381,9 +381,9 @@ void solution_t::populate_ep_with_unserved( rmm::device_scalar ep_index_out(EP.index_, stream); const i_t TPB = 256; populate_ep_with_unserved_kernel - <<<1, TPB, 0, stream>>>(view(), EP.view(), ep_index_out.data()); + <<<1, TPB, 0, stream.get()>>>(view(), EP.view(), ep_index_out.data()); EP.index_ = ep_index_out.value(stream); - stream.synchronize(); + stream.sync(); if (EP.size() > 1) { thrust::default_random_engine g(problem_ptr->seed_gen.get_seed()); thrust::shuffle( @@ -404,11 +404,11 @@ void solution_t::populate_ep_with_selected_unserved( auto unserviced_view = raft::device_span(unserviced_device.data(), unserviced_device.size()); - populate_ep_with_selected_unserved_kernel<<<1, TPB, 0, stream>>>( + populate_ep_with_selected_unserved_kernel<<<1, TPB, 0, stream.get()>>>( view(), unserviced_view, EP.view(), ep_index_out.data(), problem_ptr->seed_gen.get_seed()); RAFT_CHECK_CUDA(stream); EP.index_ = ep_index_out.value(stream); - stream.synchronize(); + stream.sync(); } template void solution_t::eject_until_feasible(bool); diff --git a/cpp/src/routing/ges/ejection_pool.cuh b/cpp/src/routing/ges/ejection_pool.cuh index afd566f475..8ae3f94151 100644 --- a/cpp/src/routing/ges/ejection_pool.cuh +++ b/cpp/src/routing/ges/ejection_pool.cuh @@ -41,7 +41,7 @@ __global__ static void device_random_shuffle(elemt_t* data, int size, int64_t se */ template ::max()> struct ejection_pool_t { - ejection_pool_t(int max_ejection_pool_size, rmm::cuda_stream_view stream) + ejection_pool_t(int max_ejection_pool_size, cuda::stream_ref stream) : stack_(max_ejection_pool_size, stream), index_(-1), stream_(stream) { } @@ -63,7 +63,7 @@ struct ejection_pool_t { // replace with thrust shuffle // how to get sol_handle::get_thrust_policy? if (size() > 1) - device_random_shuffle<<<1, 1, 0, stream_>>>(stack_.data(), size(), seed); + device_random_shuffle<<<1, 1, 0, stream_.get()>>>(stack_.data(), size(), seed); } bool empty() const @@ -154,7 +154,7 @@ struct ejection_pool_t { rmm::device_uvector stack_; int index_; - rmm::cuda_stream_view stream_; + cuda::stream_ref stream_; std::uniform_int_distribution dist{0, std::numeric_limits::max()}; std::mt19937 gen{66742}; }; diff --git a/cpp/src/routing/ges/execute_insertion.cu b/cpp/src/routing/ges/execute_insertion.cu index ddec22acee..112dea7e2a 100644 --- a/cpp/src/routing/ges/execute_insertion.cu +++ b/cpp/src/routing/ges/execute_insertion.cu @@ -308,12 +308,12 @@ bool guided_ejection_search_t::execute_best_insertion_ejectio <<<1, 1024, shared_for_delete_array + shared_for_tmp_route, - solution_ptr->sol_handle->get_stream()>>>(solution_ptr->view(), - d_request, - (uint64_t*)feasible_candidates_data_.data(), - EP.view(), - fragment_step, - fragment_size); + solution_ptr->sol_handle->get_stream().get()>>>(solution_ptr->view(), + d_request, + (uint64_t*)feasible_candidates_data_.data(), + EP.view(), + fragment_step, + fragment_size); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); // Update EP index, route_id contains the amount we deleted found_sol_t selected_move = @@ -365,7 +365,7 @@ bool guided_ejection_search_t::perform_insertion( } execute_feasible_insert - <<<1, 1024, shared_for_tmp_route, solution_ptr->sol_handle->get_stream()>>>( + <<<1, 1024, shared_for_tmp_route, solution_ptr->sol_handle->get_stream().get()>>>( solution_ptr->view(), request, selected_candidate); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); return true; @@ -398,7 +398,7 @@ i_t guided_ejection_search_t::find_single_insertion( <<sol_handle->get_stream()>>>( + solution_ptr->sol_handle->get_stream().get()>>>( solution_ptr->view(), request, feasible_move_t(cuopt::make_span(feasible_candidates_data_), diff --git a/cpp/src/routing/ges/guided_ejection_search.cu b/cpp/src/routing/ges/guided_ejection_search.cu index 1e88375a92..18f2dd0170 100644 --- a/cpp/src/routing/ges/guided_ejection_search.cu +++ b/cpp/src/routing/ges/guided_ejection_search.cu @@ -270,7 +270,7 @@ bool guided_ejection_search_t::guided_ejection_search_loop(i_ } // Increase penalty counter for this request - incr_p_scores<<<1, 1, 0, solution_ptr->sol_handle->get_stream()>>>( + incr_p_scores<<<1, 1, 0, solution_ptr->sol_handle->get_stream().get()>>>( request, p_scores_.data(), depot_included); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); @@ -522,9 +522,9 @@ void guided_ejection_search_t::route_minimizer_loop() std::tie(vehicle_id, random_route_id) = next_route_id(); if (random_route_id < 0) { break; } // Save solution state before ges loop in case of route restoration - stream.synchronize(); + stream.sync(); ges_loop_save_state.copy_device_solution(*solution_ptr); - stream.synchronize(); + stream.sync(); solution_ptr->remove_routes(EP, std::vector{random_route_id}); // Routes can be empty when number of vehicles is more than number of requests @@ -532,9 +532,9 @@ void guided_ejection_search_t::route_minimizer_loop() // If ges loop left early, restore state if (!guided_ejection_search_loop(counter, true)) { - stream.synchronize(); + stream.sync(); solution_ptr->copy_device_solution(ges_loop_save_state); - stream.synchronize(); + stream.sync(); } solution_ptr->global_runtime_checks(true, true, "route_minimizer_loop"); } diff --git a/cpp/src/routing/ges/lexicographic_search/brute_force_lexico.cu b/cpp/src/routing/ges/lexicographic_search/brute_force_lexico.cu index 020c5e89ab..2d0241103b 100644 --- a/cpp/src/routing/ges/lexicographic_search/brute_force_lexico.cu +++ b/cpp/src/routing/ges/lexicographic_search/brute_force_lexico.cu @@ -202,15 +202,15 @@ std::vector guided_ejection_search_t::brute_force_lexico size_t shared_size = shared_size_for_route + shared_size_for_intra_indices; i_t n_blocks = combinations.size(); brute_force_lexico_kernel - <<>>(d_combinations.data(), - sol.view(), - route.view(), - n_ejections, - req, - global_min_p.data(), - global_sequence.data(), - EP.view(), - p_scores_.data()); + <<>>(d_combinations.data(), + sol.view(), + route.view(), + n_ejections, + req, + global_min_p.data(), + global_sequence.data(), + EP.view(), + p_scores_.data()); // copy the best result and keep it here sol.sol_handle->sync_stream(); } @@ -219,7 +219,7 @@ std::vector guided_ejection_search_t::brute_force_lexico std::vector sequence(global_sequence.element(0, stream) + 3); // copy including pickup and delivery raft::copy(sequence.data(), global_sequence.data() + 1, sequence.size(), stream); - stream.synchronize(); + stream.sync(); return sequence; } return std::vector{}; diff --git a/cpp/src/routing/ges/lexicographic_search/lexicographic_search.cu b/cpp/src/routing/ges/lexicographic_search/lexicographic_search.cu index 8be74cd348..1548a6744a 100644 --- a/cpp/src/routing/ges/lexicographic_search/lexicographic_search.cu +++ b/cpp/src/routing/ges/lexicographic_search/lexicographic_search.cu @@ -713,13 +713,14 @@ bool guided_ejection_search_t::run_lexicographic_search( solution_ptr->d_lock.set_value_async(zero, stream); global_random_counter_.set_value_async(zero, stream); lexicographic_search - <<>>(solution_ptr->view(), - k_max, - request_id, - p_scores_.data(), - global_min_p_.data(), - global_sequence_.data(), - global_random_counter_.data()); + <<>>( + solution_ptr->view(), + k_max, + request_id, + p_scores_.data(), + global_min_p_.data(), + global_sequence_.data(), + global_random_counter_.data()); solution_ptr->sol_handle->sync_stream(); RAFT_CHECK_CUDA(stream); // If global_min_p_ != max do the move @@ -731,12 +732,12 @@ bool guided_ejection_search_t::run_lexicographic_search( return false; } execute_lexico_move - <<<1, threads_per_block_lexico, shared_for_tmp_route, stream>>>(solution_ptr->view(), - request_id, - global_min_p_.data(), - global_sequence_.data(), - EP.view(), - p_scores_.data()); + <<<1, threads_per_block_lexico, shared_for_tmp_route, stream.get()>>>(solution_ptr->view(), + request_id, + global_min_p_.data(), + global_sequence_.data(), + EP.view(), + p_scores_.data()); RAFT_CHECK_CUDA(stream); i_t removed_size = global_sequence_.element(1, stream); if constexpr (REQUEST == request_t::PDP) { removed_size = (removed_size - 1) / 2; } diff --git a/cpp/src/routing/ges/squeeze.cu b/cpp/src/routing/ges/squeeze.cu index 5de35d153a..41488b46fa 100644 --- a/cpp/src/routing/ges/squeeze.cu +++ b/cpp/src/routing/ges/squeeze.cu @@ -37,7 +37,7 @@ bool guided_ejection_search_t::repair_empty_routes() // reset the best move stored best_move.set_value_async(uninit_cand, solution_ptr->sol_handle->get_stream()); find_best_empty_route_move - <<sol_handle->get_stream()>>>( + <<sol_handle->get_stream().get()>>>( solution_ptr->view(), best_move.data(), include_objective, default_weights, excess_limit); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); @@ -51,8 +51,8 @@ bool guided_ejection_search_t::repair_empty_routes() if (!set_shmem_of_kernel(execute_best_empty_route_move, sh_route)) { break; } execute_best_empty_route_move - <<<1, TPB, sh_route, solution_ptr->sol_handle->get_stream()>>>(solution_ptr->view(), - best_move.data()); + <<<1, TPB, sh_route, solution_ptr->sol_handle->get_stream().get()>>>(solution_ptr->view(), + best_move.data()); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); ++counter; } @@ -95,23 +95,23 @@ i_t guided_ejection_search_t::try_multiple_insert(i_t n_inser cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); // insert the request greedily to a position that will generate the least excess find_all_squeeze_pos - <<>>(solution_ptr->view(), - EP.view(), - cuopt::make_span(best_squeeze_per_cand), - cuopt::make_span(best_squeeze_per_route), - include_objective, - weights, - excess_limit, - n_insertions, - inserted_requests.data()); + <<>>(solution_ptr->view(), + EP.view(), + cuopt::make_span(best_squeeze_per_cand), + cuopt::make_span(best_squeeze_per_route), + include_objective, + weights, + excess_limit, + n_insertions, + inserted_requests.data()); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); if constexpr (squeeze_mode) { size_t move_blocks = solution_ptr->get_num_requests(); extract_best_per_route - <<>>(solution_ptr->view(), - cuopt::make_span(best_squeeze_per_cand), - cuopt::make_span(best_squeeze_per_route)); + <<>>(solution_ptr->view(), + cuopt::make_span(best_squeeze_per_cand), + cuopt::make_span(best_squeeze_per_route)); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); } @@ -121,19 +121,20 @@ i_t guided_ejection_search_t::try_multiple_insert(i_t n_inser cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); // execute squeeze moves execute_all_move - <<>>(solution_ptr->view(), - cuopt::make_span(best_squeeze_per_cand), - cuopt::make_span(best_squeeze_per_route), - inserted_requests.data(), - number_of_inserted.data()); + <<>>( + solution_ptr->view(), + cuopt::make_span(best_squeeze_per_cand), + cuopt::make_span(best_squeeze_per_route), + inserted_requests.data(), + number_of_inserted.data()); RAFT_CHECK_CUDA(stream); auto n_inserted = number_of_inserted.value(stream); if (n_inserted == 0) { // Some of the attempted requests could not be inserted in this call or following ones // after perturbations - increase_multiple_p_scores - <<<1, 64, 0, stream>>>(EP.view(), p_scores_.data(), inserted_requests.data(), n_insertions); + increase_multiple_p_scores<<<1, 64, 0, stream.get()>>>( + EP.view(), p_scores_.data(), inserted_requests.data(), n_insertions); break; } counter += n_inserted; @@ -141,7 +142,7 @@ i_t guided_ejection_search_t::try_multiple_insert(i_t n_inser solution_ptr->compute_cost(); solution_ptr->global_runtime_checks(false, false, "try_multiple_insert_end"); - stream.synchronize(); + stream.sync(); return counter; } @@ -171,8 +172,9 @@ i_t guided_ejection_search_t::try_multiple_feasible_insertion i_t successful_insertions = try_multiple_insert( n_insertions, default_weights, std::numeric_limits::epsilon(), include_objective); - eject_inserted_requests<<<1, 32, 0, solution_ptr->sol_handle->get_stream()>>>( - EP.view(), inserted_requests.data(), n_insertions); + eject_inserted_requests + <<<1, 32, 0, solution_ptr->sol_handle->get_stream().get()>>>( + EP.view(), inserted_requests.data(), n_insertions); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); // Index is not updated in device view @@ -210,7 +212,7 @@ void guided_ejection_search_t::squeeze_all_ep() if (successful_insertions == 0) { run_batches = false; } eject_inserted_requests - <<<1, 32, 0, solution_ptr->sol_handle->get_stream()>>>( + <<<1, 32, 0, solution_ptr->sol_handle->get_stream().get()>>>( EP.view(), inserted_requests.data(), batch_size); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); @@ -299,29 +301,30 @@ void guided_ejection_search_t::squeeze( i_t route_id = dist_candidate(gen_candidate) % solution_ptr->get_n_routes(); // insert the request greedily to a position that will generate the least excess find_best_squeeze_pos - <<<1, TPB, sh_size, stream>>>(solution_ptr->view(), - request, - best_move.data(), - include_objective, - local_search_ptr_->move_candidates.weights, - route_id); + <<<1, TPB, sh_size, stream.get()>>>(solution_ptr->view(), + request, + best_move.data(), + include_objective, + local_search_ptr_->move_candidates.weights, + route_id); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); } else { find_best_squeeze_pos - <<>>(solution_ptr->view(), - request, - best_move.data(), - include_objective, - local_search_ptr_->move_candidates.weights); + <<>>(solution_ptr->view(), + request, + best_move.data(), + include_objective, + local_search_ptr_->move_candidates.weights); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); } cuopt_assert(best_move.value(stream).cost_counter.cost != std::numeric_limits::max(), "At least a move should be found in squeeze"); // execute squeeze - execute_move<<<1, 1, 0, stream>>>(solution_ptr->view(), request, best_move.data()); + execute_move + <<<1, 1, 0, stream.get()>>>(solution_ptr->view(), request, best_move.data()); solution_ptr->compute_cost(); solution_ptr->global_runtime_checks(false, false, "squeeze"); - stream.synchronize(); + stream.sync(); } template @@ -378,7 +381,7 @@ void guided_ejection_search_t::squeeze_breaks() return; } - squeeze_breaks_kernel<<>>( + squeeze_breaks_kernel<<>>( solution_ptr->view(), false, local_search_ptr_->move_candidates.weights); RAFT_CHECK_CUDA(solution_ptr->sol_handle->get_stream()); solution_ptr->compute_cost(); diff --git a/cpp/src/routing/local_search/breaks_insertion.cu b/cpp/src/routing/local_search/breaks_insertion.cu index 8fd06d83f1..3aada78d07 100644 --- a/cpp/src/routing/local_search/breaks_insertion.cu +++ b/cpp/src/routing/local_search/breaks_insertion.cu @@ -167,12 +167,12 @@ void find_break_insertions(solution_t& sol, } find_break_insertions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.include_objective, move_candidates.weights, move_candidates.breaks_move_candidates.view()); - RAFT_CUDA_TRY(cudaStreamSynchronize(sol.sol_handle->get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(sol.sol_handle->get_stream().get())); } } @@ -254,8 +254,8 @@ bool local_search_t::perform_break_moves(solution_t, shared_size)) { return false; } execute_break_moves - <<get_stream()>>>(sol.view(), - move_candidates.view()); + <<get_stream().get()>>>(sol.view(), + move_candidates.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); sol.compute_cost(); diff --git a/cpp/src/routing/local_search/compute_compatible.cu b/cpp/src/routing/local_search/compute_compatible.cu index 457e970632..02e1c965c8 100644 --- a/cpp/src/routing/local_search/compute_compatible.cu +++ b/cpp/src/routing/local_search/compute_compatible.cu @@ -448,7 +448,7 @@ void local_search_t::calculate_route_compatibility( i_t TPB = 128; i_t n_blocks = sol.n_routes * sol.get_num_requests(); calculate_route_compatibility_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.route_compatibility.data(), move_candidates.viables.compatibility_matrix.data()); @@ -635,42 +635,44 @@ void initialize_incompatible(problem_t& problem, solution_t - <<get_stream()>>>( + <<get_stream().get()>>>( problem.view(), viables.compatibility_matrix.data(), sol_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); n_blocks = (problem.get_num_orders() * problem.get_num_orders() - 1 + TPB) / TPB; initialize_viable_kernel - <<get_stream()>>>(problem.view(), - viables.viable_to_pickups.data(), - viables.viable_from_pickups.data(), - viables.n_viable_to_pickups.data(), - viables.n_viable_from_pickups.data(), - viables.viable_to_deliveries.data(), - viables.viable_from_deliveries.data(), - viables.n_viable_to_deliveries.data(), - viables.n_viable_from_deliveries.data(), - sol_view, - is_problem_run); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + <<get_stream().get()>>>( + problem.view(), + viables.viable_to_pickups.data(), + viables.viable_from_pickups.data(), + viables.n_viable_to_pickups.data(), + viables.n_viable_from_pickups.data(), + viables.viable_to_deliveries.data(), + viables.viable_from_deliveries.data(), + viables.n_viable_to_deliveries.data(), + viables.n_viable_from_deliveries.data(), + sol_view, + is_problem_run); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); } else { initialize_incompatible_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( problem.view(), viables.compatibility_matrix.data(), sol_view); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); n_blocks = (problem.get_num_orders() * problem.get_num_orders() - 1 + TPB) / TPB; initialize_viable_kernel - <<get_stream()>>>(problem.view(), - viables.viable_to_pickups.data(), - viables.viable_from_pickups.data(), - viables.n_viable_to_pickups.data(), - viables.n_viable_from_pickups.data(), - viables.viable_to_deliveries.data(), - viables.viable_from_deliveries.data(), - viables.n_viable_to_deliveries.data(), - viables.n_viable_from_deliveries.data(), - sol_view, - is_problem_run); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + <<get_stream().get()>>>( + problem.view(), + viables.viable_to_pickups.data(), + viables.viable_from_pickups.data(), + viables.n_viable_to_pickups.data(), + viables.n_viable_from_pickups.data(), + viables.viable_to_deliveries.data(), + viables.viable_from_deliveries.data(), + viables.n_viable_to_deliveries.data(), + viables.n_viable_from_deliveries.data(), + sol_view, + is_problem_run); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); } problem.sort_viable_matrix(viables.viable_to_pickups, viables.viable_from_pickups); problem.sort_viable_matrix(viables.viable_to_deliveries, viables.viable_from_deliveries); diff --git a/cpp/src/routing/local_search/compute_insertions.cu b/cpp/src/routing/local_search/compute_insertions.cu index 1f69065446..0ba278649a 100644 --- a/cpp/src/routing/local_search/compute_insertions.cu +++ b/cpp/src/routing/local_search/compute_insertions.cu @@ -830,7 +830,7 @@ void find_insertions(solution_t& sol, "Not enough shared memory on device for computing local search insertions!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); find_insertions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); } else { // for cross the load-balance factor is always 4 @@ -846,7 +846,7 @@ void find_insertions(solution_t& sol, "Not enough shared memory on device for computing local search insertions!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); find_insertions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); } else if (search_type == search_type_t::RANDOM) { // we don't search for relocates in random. @@ -858,7 +858,7 @@ void find_insertions(solution_t& sol, "Not enough shared memory on device for computing local search insertions!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); find_insertions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); } } @@ -891,7 +891,7 @@ void find_unserviced_insertions(solution_t& sol, cuopt_assert(is_set, "Not enough shared memory on device for computing local search insertions!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); find_insertions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); sol.sol_handle->sync_stream(); diff --git a/cpp/src/routing/local_search/cycle_finder/cycle.hpp b/cpp/src/routing/local_search/cycle_finder/cycle.hpp index e6a8aec57b..b363eec7fe 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle.hpp +++ b/cpp/src/routing/local_search/cycle_finder/cycle.hpp @@ -23,7 +23,7 @@ namespace detail { template struct ret_cycles_t { - ret_cycles_t(size_t max_size, rmm::cuda_stream_view stream_view) + ret_cycles_t(size_t max_size, cuda::stream_ref stream_view) : paths(max_size, stream_view), offsets(max_size, stream_view), n_cycles_(zero_v, stream_view), @@ -46,7 +46,7 @@ struct ret_cycles_t { i_t n_cycles; }; - host_t to_host(rmm::cuda_stream_view stream) + host_t to_host(cuda::stream_ref stream) { host_t h; h.paths = host_copy(paths, stream); diff --git a/cpp/src/routing/local_search/cycle_finder/cycle_finder.cu b/cpp/src/routing/local_search/cycle_finder/cycle_finder.cu index 65d654b06b..1c15777ca9 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle_finder.cu +++ b/cpp/src/routing/local_search/cycle_finder/cycle_finder.cu @@ -33,12 +33,14 @@ bool ExactCycleFinder::call_init(graph_t& graph) bool is_set = set_shmem_of_kernel(init_kernel, sh_size); if (!is_set) { return false; } - init_kernel<<get_stream()>>>( - graph.view(), d_valid_paths.subspan(level)); + init_kernel + <<get_stream().get()>>>( + graph.view(), d_valid_paths.subspan(level)); RAFT_CHECK_CUDA(handle_ptr->get_stream()); // we have a safe-guard in the kernel for the global array stores // do the safe guard here for the occupied size - clamp_occupied<<<1, 1, 0, handle_ptr->get_stream()>>>(d_valid_paths.subspan(level)); + clamp_occupied + <<<1, 1, 0, handle_ptr->get_stream().get()>>>(d_valid_paths.subspan(level)); return true; } @@ -79,7 +81,7 @@ void ExactCycleFinder::sort_cycle_costs_by_key(int n_items n_items, begin_bit, end_bit, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); // Allocate temporary storage if (d_cub_storage_bytes.size() < temp_storage_bytes) { @@ -95,7 +97,7 @@ void ExactCycleFinder::sort_cycle_costs_by_key(int n_items n_items, begin_bit, end_bit, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); } template @@ -112,7 +114,7 @@ bool ExactCycleFinder::call_find(graph_t& graph, if (last_level) { if (!set_shmem_of_kernel(find_kernel, sh_size)) { return false; } find_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( level, graph.view(), d_valid_paths.subspan(level - 1), @@ -122,7 +124,7 @@ bool ExactCycleFinder::call_find(graph_t& graph, } else { if (!set_shmem_of_kernel(find_kernel, sh_size)) { return false; } find_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( level, graph.view(), d_valid_paths.subspan(level - 1), @@ -136,23 +138,23 @@ bool ExactCycleFinder::call_find(graph_t& graph, } template -void detail::device_map_t::clear(rmm::cuda_stream_view stream) +void detail::device_map_t::clear(cuda::stream_ref stream) { auto max_vals = max_level * max_available; auto n_threads = 256; auto n_blocks = std::min((max_vals + n_threads - 1) / n_threads, max_blocks); - clear_map<<>>(this->view()); + clear_map<<>>(this->view()); RAFT_CHECK_CUDA(stream); } template bool test_empty(typename detail::device_map_t, double>::view_t const map_view, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto max_vals = map_view.max_available; auto n_threads = 256; auto n_blocks = (max_vals + n_threads - 1) / n_threads; - test_empty, double><<>>(map_view); + test_empty, double><<>>(map_view); RAFT_CHECK_CUDA(stream); return true; } @@ -187,21 +189,21 @@ void ExactCycleFinder::get_cycle(graph_t& graph, cuopt_func_call(d_ret.total_cycle_cost = 0.); for (i_t cycle_id = 0; cycle_id < n_cycles; ++cycle_id) { init_cycle - <<<1, 1, 0, handle_ptr->get_stream()>>>(d_ret.view(), best_cycles.subspan(cycle_id)); + <<<1, 1, 0, handle_ptr->get_stream().get()>>>(d_ret.view(), best_cycles.subspan(cycle_id)); RAFT_CHECK_CUDA(handle_ptr->get_stream()); i_t level = level_vec[cycle_id]; for (int i = level; i > 0; --i) { extend_cycle - <<get_stream()>>>(graph.view(), - d_valid_paths.subspan(i), - best_cycles.subspan(cycle_id), - d_ret.view(), - i, - (level + 1) - i); + <<get_stream().get()>>>(graph.view(), + d_valid_paths.subspan(i), + best_cycles.subspan(cycle_id), + d_ret.view(), + i, + (level + 1) - i); RAFT_CHECK_CUDA(handle_ptr->get_stream()); } - close_cycle<<<1, 1, 0, handle_ptr->get_stream()>>>( + close_cycle<<<1, 1, 0, handle_ptr->get_stream().get()>>>( d_ret.view(), best_cycles.subspan(cycle_id), level + 1); cuopt_func_call(d_ret.total_cycle_cost += best_cycles.cost_ptr.element(cycle_id, handle_ptr->get_stream())); @@ -300,7 +302,7 @@ void ExactCycleFinder::sort_occupied(int level, curr_map.occupied_indices.data(), curr_level_occupied, [] __device__(int2 a, int2 b) -> bool { return a.y < b.y; }, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); // Allocate temporary storage if (d_cub_storage_bytes.size() < temp_storage_bytes) { d_cub_storage_bytes.resize(temp_storage_bytes, handle_ptr->get_stream()); @@ -312,7 +314,7 @@ void ExactCycleFinder::sort_occupied(int level, curr_map.occupied_indices.data(), curr_level_occupied, [] __device__(int2 a, int2 b) -> bool { return a.y < b.y; }, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); // do an exclusive scan for the offsets of heads, this will be used in kernels temp_storage_bytes = 0; @@ -321,7 +323,7 @@ void ExactCycleFinder::sort_occupied(int level, curr_map.size_per_head.data(), curr_map.size_per_head.data(), graph.get_num_vertices() + 1, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); // Allocate temporary storage if (d_cub_storage_bytes.size() < temp_storage_bytes) { d_cub_storage_bytes.resize(temp_storage_bytes, handle_ptr->get_stream()); @@ -332,7 +334,7 @@ void ExactCycleFinder::sort_occupied(int level, curr_map.size_per_head.data(), curr_map.size_per_head.data(), graph.get_num_vertices() + 1, - handle_ptr->get_stream()); + handle_ptr->get_stream().get()); } template @@ -349,11 +351,11 @@ void ExactCycleFinder::find_best_cycles( sort_cycle_costs_by_key(cycle_candidates.size * cycle_candidates.n_paths); // record best cycles record_best_cycles - <<<1, 1, 0, handle_ptr->get_stream()>>>(cycle_candidates.size * cycle_candidates.n_paths, - graph.view(), - cycle_candidates.view(), - best_cycles.view(), - sorted_key_indices.data()); + <<<1, 1, 0, handle_ptr->get_stream().get()>>>(cycle_candidates.size * cycle_candidates.n_paths, + graph.view(), + cycle_candidates.view(), + best_cycles.view(), + sorted_key_indices.data()); get_cycle(graph, ret); cuopt_assert(check_cycle(graph, ret), "Recomputed cost mismatch"); } diff --git a/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp b/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp index 73a334ffd6..cd0e36cd09 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp +++ b/cpp/src/routing/local_search/cycle_finder/cycle_finder.hpp @@ -60,13 +60,14 @@ struct path_t { reset(handle_ptr_->get_stream()); } - void reset(rmm::cuda_stream_view stream) + void reset(cuda::stream_ref stream) { n_cycles.set_value_to_zero_async(stream); all_found.set_value_to_zero_async(stream); // device_bitset_t is all zeros when cleared; memset avoids a host-source copy, which // is not capturable into a CUDA graph on CUDA 13. - RAFT_CUDA_TRY(cudaMemsetAsync(all_mask.data(), 0, sizeof(device_bitset_t), stream)); + RAFT_CUDA_TRY( + cudaMemsetAsync(all_mask.data(), 0, sizeof(device_bitset_t), stream.get())); } struct view_t { @@ -111,7 +112,7 @@ struct path_t { template struct cycle_candidates_t { - cycle_candidates_t(size_t size_, int n_paths_, rmm::cuda_stream_view stream) + cycle_candidates_t(size_t size_, int n_paths_, cuda::stream_ref stream) : keys(size_ * n_paths_, stream), costs(size_ * n_paths_, stream), level_vec(size_ * n_paths_, stream), diff --git a/cpp/src/routing/local_search/cycle_finder/cycle_graph.hpp b/cpp/src/routing/local_search/cycle_finder/cycle_graph.hpp index 3c28f78bc4..548e811c0f 100644 --- a/cpp/src/routing/local_search/cycle_finder/cycle_graph.hpp +++ b/cpp/src/routing/local_search/cycle_finder/cycle_graph.hpp @@ -29,7 +29,7 @@ constexpr int max_graph_nodes_per_row = 1024; template struct graph_t { - graph_t(i_t size, rmm::cuda_stream_view stream) + graph_t(i_t size, cuda::stream_ref stream) : row_sizes(size, stream), route_ids(size, stream), // allocate with the max size @@ -45,7 +45,7 @@ struct graph_t { std::vector weights; }; - host_t to_host(rmm::cuda_stream_view stream) + host_t to_host(cuda::stream_ref stream) { host_t h; h.row_sizes = host_copy(row_sizes, stream); diff --git a/cpp/src/routing/local_search/cycle_finder/device_map.cuh b/cpp/src/routing/local_search/cycle_finder/device_map.cuh index 3a2b7a31f4..ebe70dee64 100644 --- a/cpp/src/routing/local_search/cycle_finder/device_map.cuh +++ b/cpp/src/routing/local_search/cycle_finder/device_map.cuh @@ -58,7 +58,7 @@ struct device_map_t { { } - void clear(rmm::cuda_stream_view stream); + void clear(cuda::stream_ref stream); uint32_t get_max_size() const { @@ -66,7 +66,7 @@ struct device_map_t { return adj_max_size; } - size_t get_size(int level, rmm::cuda_stream_view stream) const + size_t get_size(int level, cuda::stream_ref stream) const { return std::min(get_max_size(), occupied.element(level, stream)); } diff --git a/cpp/src/routing/local_search/fill_gpu_graph.cu b/cpp/src/routing/local_search/fill_gpu_graph.cu index 5cb0e6c81e..036b84fcfa 100644 --- a/cpp/src/routing/local_search/fill_gpu_graph.cu +++ b/cpp/src/routing/local_search/fill_gpu_graph.cu @@ -158,13 +158,13 @@ void local_search_t::fill_gpu_graph(solution_tsync_stream(); const auto stream = solution.sol_handle->get_stream(); move_candidates.graph.special_index = solution.get_num_orders() + solution.n_routes; - fill_intra_candidates<<>>( + fill_intra_candidates<<>>( solution.view(), move_candidates.view(), solution.problem_ptr->seed_gen.get_seed()); // +1 for special node i_t n_blocks = solution.get_num_requests() + 1; fill_graph_kernel - <<>>(solution.view(), move_candidates.view()); - stream.synchronize(); + <<>>(solution.view(), move_candidates.view()); + stream.sync(); } template void local_search_t::fill_gpu_graph( solution_t&); diff --git a/cpp/src/routing/local_search/hvrp/vehicle_assignment.cu b/cpp/src/routing/local_search/hvrp/vehicle_assignment.cu index 7767ec9cdd..5884c50533 100644 --- a/cpp/src/routing/local_search/hvrp/vehicle_assignment.cu +++ b/cpp/src/routing/local_search/hvrp/vehicle_assignment.cu @@ -24,7 +24,7 @@ auto compute_route_costs(solution_t& sol, if (!is_set) { return false; } compute_route_costs_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); return true; @@ -44,7 +44,8 @@ auto compute_route_cost_differences(solution_t& sol, if (!is_set) { return false; } compute_route_cost_differences_kernel - <<get_stream()>>>(sol.view(), vehicle_assignment.view()); + <<get_stream().get()>>>(sol.view(), + vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); return true; } @@ -61,7 +62,8 @@ auto compute_route_vehicle_assignments(solution_t& sol, if (!is_set) { return false; } compute_route_vehicle_assignments_kernel - <<get_stream()>>>(sol.view(), vehicle_assignment.view()); + <<get_stream().get()>>>(sol.view(), + vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); return true; } @@ -77,8 +79,9 @@ auto update_assignment(solution_t& sol, if (!is_set) { return false; } auto k_iter = vehicle_assignment.get_k_regrets() - 1; - update_assignment_kernel<<get_stream()>>>( - sol.view(), move_candidates.view(), vehicle_assignment.view()); + update_assignment_kernel + <<get_stream().get()>>>( + sol.view(), move_candidates.view(), vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); return true; } @@ -91,7 +94,7 @@ void reset_vehicle_availability(solution_t& sol, async_fill(vehicle_assignment.vehicle_availability, -1, sol.sol_handle->get_stream()); auto k_iter = vehicle_assignment.get_k_regrets() - 1; reset_vehicle_availability_kernel - <<get_stream()>>>(sol.view(), vehicle_assignment.view()); + <<get_stream().get()>>>(sol.view(), vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } @@ -142,7 +145,7 @@ auto find_best_assignment(solution_t& sol, bool is_set = set_shmem_of_kernel(find_best_assignment_kernel, shmem); if (!is_set) { return false; } find_best_assignment_kernel - <<<1, TPB, shmem, sol.sol_handle->get_stream()>>>(sol.view(), vehicle_assignment.view()); + <<<1, TPB, shmem, sol.sol_handle->get_stream().get()>>>(sol.view(), vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); return true; } @@ -159,7 +162,7 @@ auto update_solution(solution_t& sol, bool is_set = set_shmem_of_kernel(update_solution_kernel, shmem); if (!is_set) { return false; } update_solution_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), vehicle_assignment.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); diff --git a/cpp/src/routing/local_search/hvrp/vehicle_assignment.cuh b/cpp/src/routing/local_search/hvrp/vehicle_assignment.cuh index da8fef1258..27ccd29213 100644 --- a/cpp/src/routing/local_search/hvrp/vehicle_assignment.cuh +++ b/cpp/src/routing/local_search/hvrp/vehicle_assignment.cuh @@ -59,7 +59,7 @@ struct vehicle_assignment_t { gl_lock.set_value_to_zero_async(sol_handle_->get_stream()); } - void resize(i_t n_routes, i_t n_buckets, rmm::cuda_stream_view stream_view) + void resize(i_t n_routes, i_t n_buckets, cuda::stream_ref stream_view) { k_regrets = std::min(n_buckets, k_max_regrets); auto k_iter = k_regrets - 1; diff --git a/cpp/src/routing/local_search/perform_moves.cu b/cpp/src/routing/local_search/perform_moves.cu index d4c1144256..4563cb811a 100644 --- a/cpp/src/routing/local_search/perform_moves.cu +++ b/cpp/src/routing/local_search/perform_moves.cu @@ -419,7 +419,7 @@ bool local_search_t::populate_cross_moves( return false; } populate_cross_list_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( solution.view(), move_candidates.view()); sh_size = sizeof(i_t) * (solution.n_routes + 1) * solution.n_routes; @@ -428,8 +428,8 @@ bool local_search_t::populate_cross_moves( return false; } populate_cross_moves_kernel - <<<1, TPB, sh_size, solution.sol_handle->get_stream()>>>(solution.view(), - move_candidates.view()); + <<<1, TPB, sh_size, solution.sol_handle->get_stream().get()>>>(solution.view(), + move_candidates.view()); solution.sol_handle->sync_stream(); return true; } @@ -442,11 +442,12 @@ void local_search_t::populate_move_path( auto n_cycles = move_candidates.cycles.n_cycles_.value(solution.sol_handle->get_stream()); if (n_cycles) { populate_move_path_kernel - <<get_stream()>>>(solution.view(), - move_candidates.view()); + <<get_stream().get()>>>(solution.view(), + move_candidates.view()); } populate_intra_candidates - <<<1, 128, 0, solution.sol_handle->get_stream()>>>(solution.view(), move_candidates.view()); + <<<1, 128, 0, solution.sol_handle->get_stream().get()>>>(solution.view(), + move_candidates.view()); } template @@ -464,7 +465,7 @@ void local_search_t::perform_moves(solution_t - <<>>(solution.view(), move_candidates.view()); + <<>>(solution.view(), move_candidates.view()); solution.compute_route_id_per_node(); solution.compute_cost(); solution.global_runtime_checks(false, false, "perform_moves_end"); diff --git a/cpp/src/routing/local_search/prize_collection.cu b/cpp/src/routing/local_search/prize_collection.cu index 6d10d310c2..a5967c806c 100644 --- a/cpp/src/routing/local_search/prize_collection.cu +++ b/cpp/src/routing/local_search/prize_collection.cu @@ -228,8 +228,8 @@ bool local_search_t::perform_prize_collection(solution_t - <<get_stream()>>>(sol.view(), - move_candidates.view()); + <<get_stream().get()>>>(sol.view(), + move_candidates.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); if (!move_candidates.prize_move_candidates.has_improving_routes(sol.sol_handle)) { return false; } @@ -237,8 +237,9 @@ bool local_search_t::perform_prize_collection(solution_t::size()); if (!set_shmem_of_kernel(execute_moves, shared_size)) { return false; } - execute_moves<<get_stream()>>>( - sol.view(), move_candidates.view()); + execute_moves + <<get_stream().get()>>>(sol.view(), + move_candidates.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); sol.compute_cost(); diff --git a/cpp/src/routing/local_search/random_cross.cu b/cpp/src/routing/local_search/random_cross.cu index 7d90c96eb6..246a9ee11a 100644 --- a/cpp/src/routing/local_search/random_cross.cu +++ b/cpp/src/routing/local_search/random_cross.cu @@ -203,7 +203,7 @@ void select_random_route_pairs(solution_t& sol, return; } select_random_route_pairs_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } @@ -216,7 +216,7 @@ void pick_random_move_per_route_pair(solution_t& sol, i_t n_route_pair = sol.n_routes * sol.n_routes; auto nblocks = (n_route_pair + nthreads - 1) / nthreads; pick_random_move_per_route_pair_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } @@ -228,8 +228,9 @@ void get_offsets_of_route_pairs(solution_t& sol, { constexpr i_t nthreads = 256; auto nblocks = ((n_random_moves + 1) + nthreads - 1) / nthreads; - extract_offsets_kernel<<get_stream()>>>( - sol.view(), move_candidates.view(), n_random_moves); + extract_offsets_kernel + <<get_stream().get()>>>( + sol.view(), move_candidates.view(), n_random_moves); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } @@ -248,7 +249,7 @@ i_t sort_random_moves_by_route_pair_idx(solution_t& sol, random_candidates.moves_per_route_pair.data(), n_random_moves, [] __device__(int2 a, int2 b) -> bool { return a.y < b.y; }, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); // Allocate temporary storage if (random_candidates.d_cub_storage_bytes.size() < temp_storage_bytes) { random_candidates.d_cub_storage_bytes.resize(temp_storage_bytes, sol.sol_handle->get_stream()); @@ -260,7 +261,7 @@ i_t sort_random_moves_by_route_pair_idx(solution_t& sol, random_candidates.moves_per_route_pair.data(), n_random_moves, [] __device__(int2 a, int2 b) -> bool { return a.y < b.y; }, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); return n_random_moves; } @@ -272,7 +273,8 @@ void local_search_t::populate_random_moves(solution_t - <<get_stream()>>>(sol.view(), move_candidates.view()); + <<get_stream().get()>>>(sol.view(), + move_candidates.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); // sort valid moves by route pair index i_t n_random_moves = sort_random_moves_by_route_pair_idx(sol, move_candidates); diff --git a/cpp/src/routing/local_search/sliding_tsp.cu b/cpp/src/routing/local_search/sliding_tsp.cu index bf206018b5..eda5cb6887 100644 --- a/cpp/src/routing/local_search/sliding_tsp.cu +++ b/cpp/src/routing/local_search/sliding_tsp.cu @@ -427,7 +427,7 @@ void resize_temp_storage(solution_t& sol, distances_ptr, distances_ptr, n_nodes + 1, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); if (temp_storage_bytes > 0) { move_candidates.temp_storage.resize(temp_storage_bytes, sol.sol_handle->get_stream()); @@ -446,11 +446,11 @@ void compute_cumulative_distances(solution_t& sol, auto n_fill_blocks = (sol.get_num_orders() + n_threads - 1) / n_threads; if (reverse) { fill_reverse_distances_kernel - <<get_stream()>>>(sol.view()); + <<get_stream().get()>>>(sol.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } else { fill_forward_distances_kernel - <<get_stream()>>>(sol.view()); + <<get_stream().get()>>>(sol.view()); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); } @@ -460,7 +460,7 @@ void compute_cumulative_distances(solution_t& sol, distances_ptr, distances_ptr, n_nodes + 2, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); if (n_temp_storage_bytes > 0) { cuopt_expects(n_temp_storage_bytes == temp_storage_bytes, @@ -473,7 +473,7 @@ void compute_cumulative_distances(solution_t& sol, distances_ptr, distances_ptr, n_nodes + 2, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); } template @@ -510,7 +510,7 @@ bool local_search_t::perform_sliding_tsp( if (!set_shmem_of_kernel(find_sliding_moves_tsp, sh_size)) { return false; } find_sliding_moves_tsp - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), cuopt::make_span(sampled_tsp_data_), @@ -526,7 +526,7 @@ bool local_search_t::perform_sliding_tsp( async_fill(moved_region_node_infos_, NodeInfo{}, sol.sol_handle->get_stream()); set_moved_regions_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), cuopt::make_span(moved_region_node_infos_)); RAFT_CHECK_CUDA(sol.sol_handle->get_stream()); @@ -548,7 +548,7 @@ bool local_search_t::perform_sliding_tsp( }); execute_sliding_moves_tsp - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), cuopt::make_span(sampled_tsp_data_), diff --git a/cpp/src/routing/local_search/sliding_window.cu b/cpp/src/routing/local_search/sliding_window.cu index 2d676d9b38..831e223593 100644 --- a/cpp/src/routing/local_search/sliding_window.cu +++ b/cpp/src/routing/local_search/sliding_window.cu @@ -1065,21 +1065,21 @@ bool local_search_t::perform_sliding_window( <<get_stream()>>>(solution.view(), - found_sliding_solution_data_.data(), - move_candidates.view(), - locks_.data(), - blocks_per_node); + solution.sol_handle->get_stream().get()>>>(solution.view(), + found_sliding_solution_data_.data(), + move_candidates.view(), + locks_.data(), + blocks_per_node); } else { kernel_perform_sliding_window <<get_stream()>>>(solution.view(), - found_sliding_solution_data_.data(), - move_candidates.view(), - locks_.data(), - blocks_per_node); + solution.sol_handle->get_stream().get()>>>(solution.view(), + found_sliding_solution_data_.data(), + move_candidates.view(), + locks_.data(), + blocks_per_node); } sliding_cuda_graph.end_capture(solution.sol_handle->get_stream()); sliding_cuda_graph.launch_graph(solution.sol_handle->get_stream()); @@ -1104,7 +1104,7 @@ bool local_search_t::perform_sliding_window( // One block for each found route execute_sliding_move - <<get_stream()>>>( + <<get_stream().get()>>>( solution.view(), found_sliding_solution_data_.data(), move_candidates.view(), diff --git a/cpp/src/routing/local_search/two_opt.cu b/cpp/src/routing/local_search/two_opt.cu index abe6e8a928..5973aabfce 100644 --- a/cpp/src/routing/local_search/two_opt.cu +++ b/cpp/src/routing/local_search/two_opt.cu @@ -393,7 +393,7 @@ bool local_search_t::perform_two_opt( if (!set_shmem_of_kernel(find_two_opt_moves, sh_size)) { return false; } find_two_opt_moves - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), cuopt::make_span(two_opt_cand_data_), @@ -434,7 +434,7 @@ bool local_search_t::perform_two_opt( sol.sol_handle->get_stream()); async_fill(moved_regions_, 0, sol.sol_handle->get_stream()); execute_recycle - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), cuopt::make_span(sampled_nodes_data_), @@ -442,7 +442,7 @@ bool local_search_t::perform_two_opt( } else { if (!set_shmem_of_kernel(execute_two_opt_moves, sh_size)) { return false; } execute_two_opt_moves - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), cuopt::make_span(two_opt_cand_data_), diff --git a/cpp/src/routing/local_search/vrp/nodes_to_search.cu b/cpp/src/routing/local_search/vrp/nodes_to_search.cu index f1e8b708d7..5f15cf71cd 100644 --- a/cpp/src/routing/local_search/vrp/nodes_to_search.cu +++ b/cpp/src/routing/local_search/vrp/nodes_to_search.cu @@ -56,7 +56,7 @@ void run_extract_kernel(solution_t& sol, i_t TPB = 256; i_t n_blocks = sol.get_n_routes(); extract_nodes_to_search_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), nodes_to_search.view(), restore_phase); } diff --git a/cpp/src/routing/local_search/vrp/vrp_execute.cu b/cpp/src/routing/local_search/vrp/vrp_execute.cu index d65ec4fb36..2b6c14f5fe 100644 --- a/cpp/src/routing/local_search/vrp/vrp_execute.cu +++ b/cpp/src/routing/local_search/vrp/vrp_execute.cu @@ -380,8 +380,8 @@ i_t extract_non_overlapping_moves(solution_t& sol, i_t TPB = 128; i_t n_blocks_for_compact = (sol.n_routes * sol.n_routes + TPB - 1) / TPB; compact_best_route_pair_moves - <<get_stream()>>>(sol.view(), - move_candidates.view()); + <<get_stream().get()>>>(sol.view(), + move_candidates.view()); i_t n_best_route_pair_moves = move_candidates.vrp_move_candidates.n_best_route_pair_moves.value(sol.sol_handle->get_stream()); n_best_route_pair_moves = std::min(n_best_route_pair_moves, max_n_best_route_pair_moves); @@ -393,7 +393,7 @@ i_t extract_non_overlapping_moves(solution_t& sol, "Not enough shared memory on device for extract_non_overlapping_moves_kernel!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); extract_non_overlapping_moves_kernel - <<<1, TPB, sh_size, sol.sol_handle->get_stream()>>>( + <<<1, TPB, sh_size, sol.sol_handle->get_stream().get()>>>( sol.view(), move_candidates.view(), sol.problem_ptr->seed_gen.get_seed()); return move_candidates.vrp_move_candidates.n_of_selected_moves.value( sol.sol_handle->get_stream()); @@ -407,7 +407,7 @@ void find_max_added_size(solution_t& sol, i_t TPB = 32; i_t n_blocks = n_moves_found; find_max_added_size_kernel - <<get_stream()>>>(sol.view(), move_candidates.view()); + <<get_stream().get()>>>(sol.view(), move_candidates.view()); } template @@ -454,7 +454,7 @@ bool execute_vrp_moves(solution_t& sol, dimBlock, kernelArgs, sh_size, - sol.sol_handle->get_stream()); + sol.sol_handle->get_stream().get()); sol.compute_route_id_per_node(); sol.compute_cost(); // move_candidates.vrp_execute_graph.end_capture(sol.sol_handle->get_stream()); diff --git a/cpp/src/routing/local_search/vrp/vrp_search.cu b/cpp/src/routing/local_search/vrp/vrp_search.cu index 1f71458856..11804f5fc2 100644 --- a/cpp/src/routing/local_search/vrp/vrp_search.cu +++ b/cpp/src/routing/local_search/vrp/vrp_search.cu @@ -652,7 +652,7 @@ bool find_vrp_moves(solution_t& sol, if (sol.problem_ptr->is_cvrp()) { compute_reverse_distances - <<get_stream()>>>(sol.view()); + <<get_stream().get()>>>(sol.view()); } i_t TPB = std::min(max_n_neighbors, sol.problem_ptr->get_num_orders()); size_t size_of_frag = dimensions_route_t::get_shared_size( @@ -672,7 +672,7 @@ bool find_vrp_moves(solution_t& sol, move_candidates.vrp_move_candidates.find_kernel_graph.start_capture(sol.sol_handle->get_stream()); move_candidates.vrp_move_candidates.reset(sol.sol_handle); find_vrp_moves_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( sol.view(), move_candidates.view(), recycle); move_candidates.vrp_move_candidates.find_kernel_graph.end_capture(sol.sol_handle->get_stream()); move_candidates.vrp_move_candidates.find_kernel_graph.launch_graph(sol.sol_handle->get_stream()); diff --git a/cpp/src/routing/order_info.cu b/cpp/src/routing/order_info.cu index 1d7e4de236..f9d52fb35e 100644 --- a/cpp/src/routing/order_info.cu +++ b/cpp/src/routing/order_info.cu @@ -36,9 +36,9 @@ void populate_time_windows(data_model_view_t const& data_model, raft::copy(order_info_.v_earliest_time_.data(), earliest, order_info_.get_num_orders(), - stream_view.value()); + stream_view.get()); raft::copy( - order_info_.v_latest_time_.data(), latest, order_info_.get_num_orders(), stream_view.value()); + order_info_.v_latest_time_.data(), latest, order_info_.get_num_orders(), stream_view.get()); } else { // subtract -1 to ensure that we can set max values for service times // in vehicle order match @@ -113,7 +113,7 @@ void check_depot_times(data_model_view_t const& data_model) i_t depot_earliest, depot_latest; raft::copy(&depot_earliest, earliest, 1, handle_ptr->get_stream()); raft::copy(&depot_latest, latest, 1, handle_ptr->get_stream()); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); rmm::device_uvector v_latest_time(n_orders, handle_ptr->get_stream()); rmm::device_uvector v_earliest_time(n_orders, handle_ptr->get_stream()); @@ -195,7 +195,7 @@ void populate_order_info(data_model_view_t const& data_model, thrust::max_element(handle_ptr_->get_thrust_policy(), temp_abs.begin(), temp_abs.end()); i_t h_max_element; raft::copy(&h_max_element, max_element_ptr, 1, stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); cuopt_expects(norders - 1 == h_max_element, error_type_t::ValidationError, "Index given is too big or an index in the delivery pickup pairs is missing!"); diff --git a/cpp/src/routing/order_info.hpp b/cpp/src/routing/order_info.hpp index d20c46a8ed..3b4d96ff77 100644 --- a/cpp/src/routing/order_info.hpp +++ b/cpp/src/routing/order_info.hpp @@ -43,7 +43,7 @@ class order_info_t { return is_pdp() ? get_num_depot_excluded_orders() / 2 : get_num_depot_excluded_orders(); } - void resize(i_t size, bool is_pickup, rmm::cuda_stream_view stream) + void resize(i_t size, bool is_pickup, cuda::stream_ref stream) { v_demand_.resize(size, stream); v_earliest_time_.resize(size, stream); @@ -57,7 +57,7 @@ class order_info_t { bool is_pdp() const { return !v_pair_indices_.is_empty(); } - auto to_host(rmm::cuda_stream_view stream) + auto to_host(cuda::stream_ref stream) { host_t h; h.earliest_time = cuopt::host_copy(v_earliest_time_, stream); diff --git a/cpp/src/routing/route/break_route.cuh b/cpp/src/routing/route/break_route.cuh index 1d5b3472f9..2fcbe4d7df 100644 --- a/cpp/src/routing/route/break_route.cuh +++ b/cpp/src/routing/route/break_route.cuh @@ -44,7 +44,7 @@ class break_route_t { break_route_t& operator=(break_route_t&& break_route) = default; - void resize(i_t max_nodes_per_route, rmm::cuda_stream_view stream) + void resize(i_t max_nodes_per_route, cuda::stream_ref stream) { breaks_forward.resize(max_nodes_per_route, stream); breaks_backward.resize(max_nodes_per_route, stream); diff --git a/cpp/src/routing/route/capacity_route.cuh b/cpp/src/routing/route/capacity_route.cuh index 3ee61c2c85..b480b2f5e3 100644 --- a/cpp/src/routing/route/capacity_route.cuh +++ b/cpp/src/routing/route/capacity_route.cuh @@ -53,7 +53,7 @@ class capacity_route_t { capacity_route_t& operator=(capacity_route_t&& capacity_route) = default; - void resize(i_t max_nodes_per_route, rmm::cuda_stream_view stream) + void resize(i_t max_nodes_per_route, cuda::stream_ref stream) { i_t n_dims = dim_info.n_capacity_dimensions; if (n_dims == 0) { return; } @@ -72,7 +72,7 @@ class capacity_route_t { std::min(old_stride, new_stride) * sizeof(i_t), n_dims, cudaMemcpyDeviceToDevice, - stream.value())); + stream.get())); } vec = std::move(new_vec); }; diff --git a/cpp/src/routing/route/distance_route.cuh b/cpp/src/routing/route/distance_route.cuh index a5f98c13ce..d5c63fd1f8 100644 --- a/cpp/src/routing/route/distance_route.cuh +++ b/cpp/src/routing/route/distance_route.cuh @@ -47,7 +47,7 @@ class distance_route_t { distance_route_t& operator=(distance_route_t&& distance_route) = default; - void resize(i_t max_nodes_per_route, rmm::cuda_stream_view stream) + void resize(i_t max_nodes_per_route, cuda::stream_ref stream) { distance_forward.resize(max_nodes_per_route, stream); distance_backward.resize(max_nodes_per_route, stream); diff --git a/cpp/src/routing/route/mismatch_route.cuh b/cpp/src/routing/route/mismatch_route.cuh index 78975750e0..99e652eb22 100644 --- a/cpp/src/routing/route/mismatch_route.cuh +++ b/cpp/src/routing/route/mismatch_route.cuh @@ -42,7 +42,7 @@ class mismatch_route_t { mismatch_route_t& operator=(mismatch_route_t&& mismatch_route) = default; - void resize(i_t max_nodes_per_route, rmm::cuda_stream_view stream) + void resize(i_t max_nodes_per_route, cuda::stream_ref stream) { mismatch_forward.resize(max_nodes_per_route, stream); mismatch_backward.resize(max_nodes_per_route, stream); diff --git a/cpp/src/routing/route/pdp_route.cuh b/cpp/src/routing/route/pdp_route.cuh index dd20e2fec3..a901abf624 100644 --- a/cpp/src/routing/route/pdp_route.cuh +++ b/cpp/src/routing/route/pdp_route.cuh @@ -60,7 +60,7 @@ class request_route_t get_node(i_t idx) const diff --git a/cpp/src/routing/solution/pool_allocator.cuh b/cpp/src/routing/solution/pool_allocator.cuh index d78df69517..4220988301 100644 --- a/cpp/src/routing/solution/pool_allocator.cuh +++ b/cpp/src/routing/solution/pool_allocator.cuh @@ -44,7 +44,7 @@ class pool_allocator_t { public: pool_allocator_t(const Problem& problem_, i_t n_solutions_, - rmm::cuda_stream_view stream_, + cuda::stream_ref stream_, i_t desired_n_routes = -1) : problem(problem_), stream(stream_) { @@ -70,10 +70,10 @@ class pool_allocator_t { } } - void sync_all_streams() const { stream.synchronize(); } + void sync_all_streams() const { stream.sync(); } // problem description - rmm::cuda_stream_view stream; + cuda::stream_ref stream; const Problem& problem; std::vector>> sol_handles; // keep a thread safe pool of local search and ges objects that can be reused diff --git a/cpp/src/routing/solution/route_node_map.cuh b/cpp/src/routing/solution/route_node_map.cuh index a4a1b171aa..c94b580841 100644 --- a/cpp/src/routing/solution/route_node_map.cuh +++ b/cpp/src/routing/solution/route_node_map.cuh @@ -20,7 +20,7 @@ namespace detail { template class route_node_map_t { public: - route_node_map_t(const int num_orders, rmm::cuda_stream_view stream) + route_node_map_t(const int num_orders, cuda::stream_ref stream) : route_id_per_node(num_orders, stream), intra_route_idx_per_node(num_orders, stream) { thrust::fill(rmm::exec_policy(stream), route_id_per_node.begin(), route_id_per_node.end(), -1); @@ -30,13 +30,13 @@ class route_node_map_t { -1); } - route_node_map_t(const route_node_map_t& other, rmm::cuda_stream_view stream) + route_node_map_t(const route_node_map_t& other, cuda::stream_ref stream) : route_id_per_node(other.route_id_per_node, stream), intra_route_idx_per_node(other.intra_route_idx_per_node, stream) { } - void copy_from(const route_node_map_t& other, rmm::cuda_stream_view stream) + void copy_from(const route_node_map_t& other, cuda::stream_ref stream) { raft::copy(intra_route_idx_per_node.data(), other.intra_route_idx_per_node.data(), diff --git a/cpp/src/routing/solution/solution.cu b/cpp/src/routing/solution/solution.cu index cbf7ed9384..cbdf1c2bf3 100644 --- a/cpp/src/routing/solution/solution.cu +++ b/cpp/src/routing/solution/solution.cu @@ -171,8 +171,9 @@ void solution_t::add_nodes_to_route( bool is_set = set_shmem_of_kernel(insert_nodes_to_route_kernel, sh_size); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); i_t TPB = 256; - insert_nodes_to_route_kernel<<<1, TPB, sh_size, sol_handle->get_stream()>>>( - view(), route_id, intra_idx, n_nodes_to_insert, temp_nodes.data()); + insert_nodes_to_route_kernel + <<<1, TPB, sh_size, sol_handle->get_stream().get()>>>( + view(), route_id, intra_idx, n_nodes_to_insert, temp_nodes.data()); thrust::fill(sol_handle->get_thrust_policy(), routes_to_search.data() + route_id, routes_to_search.data() + route_id + 1, @@ -193,7 +194,8 @@ void solution_t::add_nodes_to_best( bool is_set = set_shmem_of_kernel(insert_node_to_best_kernel, sh_size); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); insert_node_to_best_kernel - <<<1, TPB, sh_size, sol_handle->get_stream()>>>(view(), node, include_objective, weights); + <<<1, TPB, sh_size, sol_handle->get_stream().get()>>>( + view(), node, include_objective, weights); sol_handle->sync_stream(); } this->global_runtime_checks(false, false, "add_nodes_to_best"); @@ -214,7 +216,7 @@ bool solution_t::remove_nodes(const std::vector>& cuopt_assert(is_set, "Not enough shared memory on device for remove_nodes!"); cuopt_expects(is_set, error_type_t::OutOfMemoryError, "Not enough shared memory on device"); i_t TPB = 256; - remove_nodes_kernel<<<1, TPB, sh_size, sol_handle->get_stream()>>>( + remove_nodes_kernel<<<1, TPB, sh_size, sol_handle->get_stream().get()>>>( view(), n_nodes_to_eject, temp_nodes.data(), empty_route_produced.data()); sol_handle->sync_stream(); return !empty_route_produced.value(sol_handle->get_stream()); @@ -323,7 +325,7 @@ void solution_t::random_init_routes() { raft::common::nvtx::range fun_scope("random_init_routes"); auto stream = sol_handle->get_stream(); - stream.synchronize(); + stream.sync(); const i_t one = 1; d_sol_found.set_value_async(one, stream); std::vector indices(get_num_requests()); @@ -343,7 +345,7 @@ void solution_t::random_init_routes() } } set_initial_nodes(d_indices, n_routes); - stream.synchronize(); + stream.sync(); } template @@ -542,7 +544,7 @@ void solution_t::copy_device_solution(solution_t - <<get_stream()>>>(view(), src_sol.view()); + <<get_stream().get()>>>(view(), src_sol.view()); RAFT_CHECK_CUDA(sol_handle->get_stream()); cuopt_assert(route_node_map.intra_route_idx_per_node.size() == (size_t)get_num_orders(), @@ -585,7 +587,8 @@ void solution_t::compute_cost() objective_cost.set_value_async(zero_obj, sol_handle->get_stream()); n_infeasible_routes.set_value_to_zero_async(sol_handle->get_stream()); if (get_n_routes() < 1) return; - compute_cost_kernel<<get_stream()>>>(view()); + compute_cost_kernel + <<get_stream().get()>>>(view()); } template @@ -627,10 +630,10 @@ void solution_t::shift_move_routes( if (n_blocks > 0) { // Decrement route_id_per_node for this route remap_route_nodes - <<get_stream()>>>( + <<get_stream().get()>>>( routes_view.data(), route_node_map.view(), route_ids_device_copy.data(), route_ids.size()); RAFT_CHECK_CUDA(sol_handle->get_stream()); - shift_routes_kernel<<<1, 1, 0, sol_handle->get_stream()>>>( + shift_routes_kernel<<<1, 1, 0, sol_handle->get_stream().get()>>>( view(), route_ids_device_copy.data(), route_ids.size()); RAFT_CHECK_CUDA(sol_handle->get_stream()); } @@ -679,7 +682,7 @@ void solution_t::remove_routes( cuopt_assert(ejection_pool.index_ >= 0, "Index should be at least 0"); set_deleted_routes_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( view(), cuopt::make_span(routes_view), cuopt::make_span(temp_int_vector), @@ -706,7 +709,7 @@ void solution_t::remove_routes(const std::vector& routes "route to remove should be in range"); } set_deleted_routes_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( view(), cuopt::make_span(routes_view), cuopt::make_span(temp_int_vector)); shift_move_routes(routes_to_remove, temp_int_vector); } @@ -732,7 +735,8 @@ i_t solution_t::compute_max_active() { raft::common::nvtx::range fun_scope("compute_max_active"); i_t TPB = 1024; - compute_max_active_kernel<<<1, TPB, 0, sol_handle->get_stream()>>>(view()); + compute_max_active_kernel + <<<1, TPB, 0, sol_handle->get_stream().get()>>>(view()); max_active_nodes = max_active_nodes_for_all_routes.value(sol_handle->get_stream()); return max_active_nodes; } @@ -742,8 +746,8 @@ void solution_t::compute_route_id_per_node() { raft::common::nvtx::range fun_scope("compute_route_id_per_node"); i_t TPB = 256; - compute_route_id_kernel - <<get_stream()>>>(routes_view.data(), route_node_map.view()); + compute_route_id_kernel<<get_stream().get()>>>( + routes_view.data(), route_node_map.view()); global_runtime_checks(false, false, "compute_route_id_per_node"); } diff --git a/cpp/src/routing/solution/solution_handle.cuh b/cpp/src/routing/solution/solution_handle.cuh index 2a74ac7341..80c78b043f 100644 --- a/cpp/src/routing/solution/solution_handle.cuh +++ b/cpp/src/routing/solution/solution_handle.cuh @@ -30,7 +30,7 @@ class solution_handle_t { solution_handle_t(solution_handle_t&&) = delete; solution_handle_t& operator=(solution_handle_t&&) = delete; - solution_handle_t(rmm::cuda_stream_view stream) + solution_handle_t(cuda::stream_ref stream) : dev_id_([]() -> i_t { i_t cur_dev = -1; RAFT_CUDA_TRY(cudaGetDevice(&cur_dev)); @@ -42,9 +42,9 @@ class solution_handle_t { } rmm::exec_policy& get_thrust_policy() const noexcept { return *thrust_policy_; } - rmm::cuda_stream_view get_stream() const noexcept { return stream_view_; } + cuda::stream_ref get_stream() const noexcept { return stream_view_; } i_t get_device() const { return dev_id_; } - void sync_stream() const { stream_view_.synchronize(); }; + void sync_stream() const { stream_view_.sync(); }; const cudaDeviceProp& get_device_properties() const { @@ -72,7 +72,7 @@ class solution_handle_t { mutable bool device_prop_initialized_{false}; mutable bool shared_attr_initialized_{false}; - rmm::cuda_stream_view stream_view_{}; + cuda::stream_ref stream_view_{}; // this is a shared pointer to be able to copy construct and keep a copy of a solution std::shared_ptr thrust_policy_{nullptr}; }; diff --git a/cpp/src/routing/util_kernels/compute_backward_forward.cu b/cpp/src/routing/util_kernels/compute_backward_forward.cu index bdde6336f4..c94dc3f0fa 100644 --- a/cpp/src/routing/util_kernels/compute_backward_forward.cu +++ b/cpp/src/routing/util_kernels/compute_backward_forward.cu @@ -46,7 +46,7 @@ void solution_t::compute_backward_forward() constexpr i_t TPB = 32; if (n_routes) { compute_backward_forward_kernel - <<get_stream()>>>(view().routes); + <<get_stream().get()>>>(view().routes); sol_handle->sync_stream(); } } @@ -58,7 +58,7 @@ void solution_t::compute_actual_arrival_times() constexpr i_t TPB = 32; if (n_routes && problem_ptr->dimensions_info.has_dimension(dim_t::TIME)) compute_actual_arrival_kernel - <<get_stream()>>>(view().routes); + <<get_stream().get()>>>(view().routes); } template void solution_t::compute_backward_forward(); diff --git a/cpp/src/routing/util_kernels/runtime_checks.cu b/cpp/src/routing/util_kernels/runtime_checks.cu index b1142f2e04..2ca4f08cea 100644 --- a/cpp/src/routing/util_kernels/runtime_checks.cu +++ b/cpp/src/routing/util_kernels/runtime_checks.cu @@ -246,7 +246,7 @@ __global__ void check_breaks(typename solution_t::view_t solu template bool global_runtime_checks_(solution_t& solution, - rmm::cuda_stream_view stream, + cuda::stream_ref stream, bool all_nodes_should_be_served, bool check_feasible) { @@ -255,12 +255,12 @@ bool global_runtime_checks_(solution_t& solution, solution.run_coherence_check(); async_fill(solution.runtime_check_histo, 0, solution.sol_handle->get_stream()); - fill_histo<<>>( + fill_histo<<>>( solution.view(), solution.runtime_check_histo.data()); const bool depot_included = solution.problem_ptr->order_info.depot_included_; check_histogram - <<<(solution.get_num_depot_excluded_orders() + 32 - 1) / 32, 32, 0, stream>>>( + <<<(solution.get_num_depot_excluded_orders() + 32 - 1) / 32, 32, 0, stream.get()>>>( solution.runtime_check_histo.data(), solution.get_num_orders(), all_nodes_should_be_served, @@ -268,7 +268,7 @@ bool global_runtime_checks_(solution_t& solution, if (solution.problem_ptr->get_max_break_dimensions() > 0) { auto sh_size = solution.problem_ptr->get_max_break_dimensions() * sizeof(i_t); - check_breaks<<>>( + check_breaks<<>>( solution.view(), all_nodes_should_be_served); } diff --git a/cpp/src/routing/util_kernels/set_initial_nodes.cu b/cpp/src/routing/util_kernels/set_initial_nodes.cu index 675357d0a3..5e4a91ad93 100644 --- a/cpp/src/routing/util_kernels/set_initial_nodes.cu +++ b/cpp/src/routing/util_kernels/set_initial_nodes.cu @@ -227,10 +227,10 @@ void solution_t::set_initial_nodes(const rmm::device_uvector< -1); constexpr i_t TPB = 32; i_t n_blocks = (desired_n_routes + TPB - 1) / TPB; - set_initial_nodes_kernel - <<get_stream()>>>(view(), problem_ptr->view(), d_indices.data()); + set_initial_nodes_kernel<<get_stream().get()>>>( + view(), problem_ptr->view(), d_indices.data()); - sol_handle->get_stream().synchronize(); + sol_handle->get_stream().sync(); } template @@ -239,7 +239,7 @@ void solution_t::set_nodes_data_of_solution() constexpr i_t TPB = 32; i_t n_blocks = n_routes; set_nodes_data_of_solution_kernel - <<get_stream()>>>(view(), problem_ptr->view()); + <<get_stream().get()>>>(view(), problem_ptr->view()); } template @@ -247,7 +247,7 @@ void solution_t::set_nodes_data_of_route(i_t route_id) { constexpr i_t TPB = 32; set_nodes_data_of_route_kernel - <<<1, TPB, 0, sol_handle->get_stream()>>>(view(), problem_ptr->view(), route_id); + <<<1, TPB, 0, sol_handle->get_stream().get()>>>(view(), problem_ptr->view(), route_id); } template @@ -257,7 +257,7 @@ void solution_t::set_nodes_data_of_new_routes(i_t added_route constexpr i_t TPB = 32; i_t starting_route_id = prev_route_size; set_nodes_data_of_new_routes_kernel - <<get_stream()>>>( + <<get_stream().get()>>>( view(), problem_ptr->view(), starting_route_id); } diff --git a/cpp/src/routing/utilities/check_input.cu b/cpp/src/routing/utilities/check_input.cu index eccc3179bb..8d0370e60d 100644 --- a/cpp/src/routing/utilities/check_input.cu +++ b/cpp/src/routing/utilities/check_input.cu @@ -33,13 +33,13 @@ namespace detail { * @param stream_view Stream view */ template -void transform_absolute(rmm::device_uvector& v, rmm::cuda_stream_view stream_view) +void transform_absolute(rmm::device_uvector& v, cuda::stream_ref stream_view) { thrust::transform( rmm::exec_policy(stream_view), v.begin(), v.end(), v.begin(), [] __device__(T x) -> T { return x < 0 ? -x : x; }); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); } /** @@ -57,7 +57,7 @@ bool check_pickup_tw(const i_t* pickup_indices, const i_t* earliest_time, const i_t* latest_time, size_t n_requests, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { typedef typename rmm::device_uvector::iterator IterConstInt; thrust::permutation_iterator pickup_iter(earliest_time, @@ -70,7 +70,7 @@ bool check_pickup_tw(const i_t* pickup_indices, zip_iterator, zip_iterator + n_requests, [] __device__(const auto& x) -> bool { return thrust::get<0>(x) > thrust::get<1>(x); }); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return !violates_sanity; } @@ -87,7 +87,7 @@ bool check_pickup_demands(const i_t* pickup_indices, const i_t* delivery_indices, const i_t* demands, size_t n_requests, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { typedef typename rmm::device_uvector::iterator IterConstInt; thrust::permutation_iterator pickup_iter(demands, pickup_indices); @@ -98,7 +98,7 @@ bool check_pickup_demands(const i_t* pickup_indices, zip_iterator, zip_iterator + n_requests, [] __device__(const auto& x) -> bool { return thrust::get<0>(x) != -thrust::get<1>(x); }); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return !violates_sanity; } @@ -107,7 +107,7 @@ bool check_pdp_values(const i_t* pickup_indices, const i_t* delivery_indices, const v_t* values, size_t n_requests, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { auto pickup_iter = thrust::make_permutation_iterator(values, pickup_indices); auto delivery_iter = thrust::make_permutation_iterator(values, delivery_indices); @@ -117,7 +117,7 @@ bool check_pdp_values(const i_t* pickup_indices, zip_iterator, zip_iterator + n_requests, [] __device__(const auto& x) -> bool { return thrust::get<0>(x) != thrust::get<1>(x); }); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return !violates_sanity; } @@ -138,8 +138,8 @@ bool is_symmetric_matrix(f_t const* matrix, i_t width, raft::handle_t const* han transposed_matrix.data_handle(), width, width, - handle_ptr->get_stream()); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream())); + handle_ptr->get_stream().get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_ptr->get_stream().get())); return thrust::equal(handle_ptr->get_thrust_policy(), matrix, @@ -158,13 +158,13 @@ template bool is_symmetric_matrix(float const*, int, raft::handle_t template bool check_min_latest_with_depot(rmm::device_uvector& v_latest_time, i_t depot_earliest, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { i_t min_latest; i_t* min_latest_ptr = thrust::min_element( rmm::exec_policy(stream_view), v_latest_time.begin() + 1, v_latest_time.end()); - raft::copy(&min_latest, min_latest_ptr, 1, stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + raft::copy(&min_latest, min_latest_ptr, 1, stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return min_latest >= depot_earliest; } @@ -178,13 +178,13 @@ bool check_min_latest_with_depot(rmm::device_uvector& v_latest_time, template bool check_max_earliest_with_depot(rmm::device_uvector& v_earliest_time, i_t depot_latest, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { i_t max_earliest; i_t* max_earliest_ptr = thrust::max_element( rmm::exec_policy(stream_view), v_earliest_time.begin() + 1, v_earliest_time.end()); - raft::copy(&max_earliest, max_earliest_ptr, 1, stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + raft::copy(&max_earliest, max_earliest_ptr, 1, stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return max_earliest <= depot_latest; } @@ -198,7 +198,7 @@ bool check_max_earliest_with_depot(rmm::device_uvector& v_earliest_time, template bool check_earliest_with_latest(rmm::device_uvector& v_earliest_time, rmm::device_uvector& v_latest_time, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { return thrust::equal(rmm::exec_policy(stream_view), v_earliest_time.begin(), @@ -220,14 +220,14 @@ bool check_min_max_values(const T* ptr, size_t size, const RefType min_value, const RefType max_value, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { T min, max; thrust::pair pair = thrust::minmax_element(rmm::exec_policy(stream_view), ptr, ptr + size); - raft::copy(&min, pair.first, 1, stream_view.value()); - raft::copy(&max, pair.second, 1, stream_view.value()); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.value())); + raft::copy(&min, pair.first, 1, stream_view.get()); + raft::copy(&max, pair.second, 1, stream_view.get()); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream_view.get())); return (min >= static_cast(min_value)) && (max <= static_cast(max_value)); } @@ -241,7 +241,7 @@ void check_guess(i_t const* guess_id, i_t fleet_size, bool const* drop_return_trip, bool const* skip_first_trip, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { cuopt_expects(check_min_max_values(truck_id, size, 0, fleet_size - 1, stream_view), error_type_t::ValidationError, @@ -262,7 +262,7 @@ void check_guess(i_t const* guess_id, d_int_drop_return_trip.data(), id); raft::update_host( - h_drop_return_trip.data(), d_int_drop_return_trip.data(), fleet_size, stream_view.value()); + h_drop_return_trip.data(), d_int_drop_return_trip.data(), fleet_size, stream_view.get()); thrust::transform(rmm::exec_policy(stream_view), skip_first_trip, @@ -270,7 +270,7 @@ void check_guess(i_t const* guess_id, d_int_skip_first_trip.data(), id); raft::update_host( - h_skip_first_trip.data(), d_int_skip_first_trip.data(), fleet_size, stream_view.value()); + h_skip_first_trip.data(), d_int_skip_first_trip.data(), fleet_size, stream_view.get()); raft::update_host(h_guess_id.data(), guess_id, size, stream_view); raft::update_host(h_truck_id.data(), truck_id, size, stream_view); @@ -341,7 +341,7 @@ bool check_no_circular_precedence(i_t node_id, i_t const* preceding_nodes, i_t n_preceding_nodes, std::unordered_map> precedence, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { for (const auto& pair : precedence) { auto other_node = pair.first; @@ -372,7 +372,7 @@ bool check_no_circular_precedence(i_t node_id, * @return bool Whether the item exists */ template -bool check_exists(T item_id, T const* device_ptr, T n_items, rmm::cuda_stream_view stream_view) +bool check_exists(T item_id, T const* device_ptr, T n_items, cuda::stream_ref stream_view) { auto end_ptr = device_ptr + n_items; auto iter_end = thrust::find(rmm::exec_policy(stream_view), device_ptr, end_ptr, item_id); @@ -383,74 +383,69 @@ template bool check_min_max_values(const uint8_t* ptr, size_t size, const int min_value, const int max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const int* ptr, size_t size, const int min_value, const int max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const int* ptr, size_t size, const int16_t min_value, const int16_t max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const int* ptr, size_t size, const uint16_t min_value, const uint16_t max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const float* ptr, size_t size, const float min_value, const float max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const double* ptr, size_t size, const double min_value, const double max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); -template void transform_absolute(rmm::device_uvector& v, - rmm::cuda_stream_view stream_view); +template void transform_absolute(rmm::device_uvector& v, cuda::stream_ref stream_view); template bool check_no_circular_precedence( int node_id, int const* preceding_nodes, int n_preceding_nodes, std::unordered_map> precedence, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_exists(int item_id, int const* device_ptr, int n_items, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_earliest_with_latest(rmm::device_uvector&, rmm::device_uvector&, - rmm::cuda_stream_view); -template bool check_max_earliest_with_depot(rmm::device_uvector&, - int, - rmm::cuda_stream_view); + cuda::stream_ref); +template bool check_max_earliest_with_depot(rmm::device_uvector&, int, cuda::stream_ref); template bool check_pickup_tw( - int const*, int const*, int const*, int const*, unsigned long, rmm::cuda_stream_view); + int const*, int const*, int const*, int const*, unsigned long, cuda::stream_ref); template bool check_pickup_demands( - int const*, int const*, int const*, unsigned long, rmm::cuda_stream_view); + int const*, int const*, int const*, unsigned long, cuda::stream_ref); template bool check_pdp_values( - int const*, int const*, uint8_t const*, unsigned long, rmm::cuda_stream_view); + int const*, int const*, uint8_t const*, unsigned long, cuda::stream_ref); template bool check_pdp_values( - int const*, int const*, int const*, unsigned long, rmm::cuda_stream_view); + int const*, int const*, int const*, unsigned long, cuda::stream_ref); template bool check_pdp_values( - int const*, int const*, float const*, unsigned long, rmm::cuda_stream_view); + int const*, int const*, float const*, unsigned long, cuda::stream_ref); -template bool check_min_latest_with_depot(rmm::device_uvector&, - int, - rmm::cuda_stream_view); +template bool check_min_latest_with_depot(rmm::device_uvector&, int, cuda::stream_ref); template void check_guess(int const*, int const*, int const*, @@ -460,7 +455,7 @@ template void check_guess(int const*, int, bool const*, bool const*, - rmm::cuda_stream_view); + cuda::stream_ref); } // namespace detail } // namespace routing diff --git a/cpp/src/routing/utilities/check_input.hpp b/cpp/src/routing/utilities/check_input.hpp index 07ce697c1a..8575beba07 100644 --- a/cpp/src/routing/utilities/check_input.hpp +++ b/cpp/src/routing/utilities/check_input.hpp @@ -23,21 +23,21 @@ bool is_symmetric_matrix(f_t const* matrix, i_t width, raft::handle_t const* han template bool check_min_latest_with_depot(rmm::device_uvector& v_latest_time, i_t depot_earliest, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_max_earliest_with_depot(rmm::device_uvector& v_earliest_time, i_t depot_latest, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_earliest_with_latest(rmm::device_uvector& v_earliest_time, rmm::device_uvector& v_latest_time, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_min_max_values(const T* ptr, size_t size, const RefType min_value, const RefType max_value, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template void check_guess(i_t const* guess_id, @@ -49,7 +49,7 @@ void check_guess(i_t const* guess_id, i_t fleet_size, bool const* drop_return_trip, bool const* skip_first_trip, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_pickup_tw(const i_t* pickup_indices, @@ -57,34 +57,34 @@ bool check_pickup_tw(const i_t* pickup_indices, const i_t* earliest_time, const i_t* latest_time, size_t n_requests, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_pickup_demands(const i_t* pickup_indices, const i_t* delivery_indices, const i_t* demands, size_t n_requests, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_pdp_values(const i_t* pickup_indices, const i_t* delivery_indices, const v_t* values, size_t n_requests, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template bool check_no_circular_precedence(i_t node_id, i_t const* preceding_nodes, i_t n_preceding_nodes, std::unordered_map> precedence, - rmm::cuda_stream_view stream_view); + cuda::stream_ref stream_view); template -bool check_exists(T item_id, T const* device_ptr, T n_items, rmm::cuda_stream_view stream_view); +bool check_exists(T item_id, T const* device_ptr, T n_items, cuda::stream_ref stream_view); template -void transform_absolute(rmm::device_uvector& v, rmm::cuda_stream_view stream_view); +void transform_absolute(rmm::device_uvector& v, cuda::stream_ref stream_view); } // namespace detail } // namespace routing diff --git a/cpp/src/routing/utilities/cython.cu b/cpp/src/routing/utilities/cython.cu index 8563c18671..7c1e0170e3 100644 --- a/cpp/src/routing/utilities/cython.cu +++ b/cpp/src/routing/utilities/cython.cu @@ -152,7 +152,7 @@ std::vector> call_batch_solve( // Restore the old stream raft::resource::set_cuda_stream(*(data_models[i]->get_handle_ptr()), old_stream); - old_stream.synchronize(); + old_stream.sync(); } return list; diff --git a/cpp/src/routing/utilities/md_utils.hpp b/cpp/src/routing/utilities/md_utils.hpp index 7de1be3f0a..9adbba2204 100644 --- a/cpp/src/routing/utilities/md_utils.hpp +++ b/cpp/src/routing/utilities/md_utils.hpp @@ -95,8 +95,8 @@ struct h_mdarray_t { template struct d_mdarray_t { - d_mdarray_t(rmm::cuda_stream_view stream_) : buffer(0, stream_), stream(stream_) {} - d_mdarray_t(std::vector const& extent_, rmm::cuda_stream_view stream_) + d_mdarray_t(cuda::stream_ref stream_) : buffer(0, stream_), stream(stream_) {} + d_mdarray_t(std::vector const& extent_, cuda::stream_ref stream_) : buffer(0, stream_), stream(stream_) { cuopt_assert(extent_.size() == NCON_DIMS, "Wrong dimensions"); @@ -138,7 +138,7 @@ struct d_mdarray_t { size_t extent[NCON_DIMS]; rmm::device_uvector buffer; - rmm::cuda_stream_view stream; + cuda::stream_ref stream; }; namespace detail { @@ -196,7 +196,7 @@ template auto create_device_mdarray(size_t nlocations, uint8_t n_vehicle_types, uint8_t n_matrix_types, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { std::vector full_matrix_extent{n_vehicle_types, n_matrix_types, nlocations, nlocations}; d_mdarray_t matrices{full_matrix_extent, stream}; @@ -204,7 +204,7 @@ auto create_device_mdarray(size_t nlocations, } inline auto get_unique_vehicle_types(const raft::device_span& vehicle_types, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { auto h_vehicle_types = cuopt::host_copy(vehicle_types, stream); diff --git a/cpp/src/utilities/copy_helpers.hpp b/cpp/src/utilities/copy_helpers.hpp index 6aa9efbab8..255770edc4 100644 --- a/cpp/src/utilities/copy_helpers.hpp +++ b/cpp/src/utilities/copy_helpers.hpp @@ -119,12 +119,12 @@ __host__ __device__ inline typename scalar_type::type& get_upper(f_t2& val * @return auto */ template -auto host_copy(T const* device_ptr, size_t size, rmm::cuda_stream_view stream_view) +auto host_copy(T const* device_ptr, size_t size, cuda::stream_ref stream_view) { if (!device_ptr) return std::vector{}; std::vector host_vec(size); raft::copy(host_vec.data(), device_ptr, size, stream_view); - stream_view.synchronize(); + stream_view.sync(); return host_vec; } @@ -137,7 +137,7 @@ auto host_copy(T const* device_ptr, size_t size, rmm::cuda_stream_view stream_vi * @param[in] stream_view * @return auto */ -inline auto host_copy(bool const* device_ptr, size_t size, rmm::cuda_stream_view stream_view) +inline auto host_copy(bool const* device_ptr, size_t size, cuda::stream_ref stream_view) { if (!device_ptr) { return std::vector(0); } rmm::device_uvector d_int_vec(size, stream_view); @@ -150,7 +150,7 @@ inline auto host_copy(bool const* device_ptr, size_t size, rmm::cuda_stream_view for (size_t i = 0; i < h_int_vec.size(); ++i) { h_bool_vec[i] = static_cast(h_int_vec[i]); } - stream_view.synchronize(); + stream_view.sync(); return h_bool_vec; } @@ -163,11 +163,11 @@ inline auto host_copy(bool const* device_ptr, size_t size, rmm::cuda_stream_view * @return auto */ template -auto host_copy(rmm::device_uvector const& device_vec, rmm::cuda_stream_view stream_view) +auto host_copy(rmm::device_uvector const& device_vec, cuda::stream_ref stream_view) { std::vector host_vec(device_vec.size()); raft::copy(host_vec.data(), device_vec.data(), device_vec.size(), stream_view); - stream_view.synchronize(); + stream_view.sync(); return host_vec; } @@ -180,7 +180,7 @@ auto host_copy(rmm::device_uvector const& device_vec, rmm::cuda_stream_view s * @return auto */ template -auto host_copy(raft::device_span const& device_vec, rmm::cuda_stream_view stream_view) +auto host_copy(raft::device_span const& device_vec, cuda::stream_ref stream_view) { return host_copy(device_vec.data(), device_vec.size(), stream_view); } @@ -194,7 +194,7 @@ auto host_copy(raft::device_span const& device_vec, rmm::cuda_stream_view str * @return auto */ template -auto host_copy(rmm::device_uvector const& device_vec, rmm::cuda_stream_view stream_view) +auto host_copy(rmm::device_uvector const& device_vec, cuda::stream_ref stream_view) { return host_copy(device_vec.data(), device_vec.size(), stream_view); } @@ -209,7 +209,7 @@ auto host_copy(rmm::device_uvector const& device_vec, rmm::cuda_stream_view s */ template inline rmm::device_uvector device_copy(rmm::device_uvector const& device_vec, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { rmm::device_uvector device_vec_copy(device_vec.size(), stream_view); raft::copy(device_vec_copy.data(), device_vec.data(), device_vec.size(), stream_view); @@ -227,7 +227,7 @@ inline rmm::device_uvector device_copy(rmm::device_uvector const& device_v template inline void device_copy(rmm::device_uvector& device_vec, std::vector const& host_vec, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { device_vec.resize(host_vec.size(), stream_view); raft::copy(device_vec.data(), host_vec.data(), host_vec.size(), stream_view); @@ -242,8 +242,7 @@ inline void device_copy(rmm::device_uvector& device_vec, * @return device_vec */ template -inline auto device_copy(std::vector const& host_vec, - rmm::cuda_stream_view stream_view) +inline auto device_copy(std::vector const& host_vec, cuda::stream_ref stream_view) { rmm::device_uvector device_vec(host_vec.size(), stream_view); raft::copy(device_vec.data(), host_vec.data(), host_vec.size(), stream_view); @@ -257,7 +256,7 @@ inline auto device_copy(std::vector const& host_vec, * @param[in] stream_view * @return device_vec */ -inline auto device_copy(std::vector const& host_vec, rmm::cuda_stream_view stream_view) +inline auto device_copy(std::vector const& host_vec, cuda::stream_ref stream_view) { std::vector host_vec_int(host_vec.size()); for (size_t i = 0; i < host_vec.size(); ++i) { @@ -340,7 +339,7 @@ raft::device_span make_span(rmm::device_uvector const& container) template inline void expand_device_copy(rmm::device_uvector& device_vec, std::vector const& host_vec, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { if (host_vec.size() > device_vec.size()) { device_vec.resize(host_vec.size(), stream_view); } raft::copy(device_vec.data(), host_vec.data(), host_vec.size(), stream_view); @@ -349,7 +348,7 @@ inline void expand_device_copy(rmm::device_uvector& device_vec, template inline void expand_device_copy(rmm::device_uvector& dst_vec, rmm::device_uvector const& src_vec, - rmm::cuda_stream_view stream_view) + cuda::stream_ref stream_view) { if (src_vec.size() > dst_vec.size()) { dst_vec.resize(src_vec.size(), stream_view); } raft::copy(dst_vec.data(), src_vec.data(), src_vec.size(), stream_view); diff --git a/cpp/src/utilities/event_handler.cuh b/cpp/src/utilities/event_handler.cuh index 452fe37804..5dacb6ff06 100644 --- a/cpp/src/utilities/event_handler.cuh +++ b/cpp/src/utilities/event_handler.cuh @@ -21,19 +21,19 @@ class event_handler_t { event_handler_t(const event_handler_t&) = delete; event_handler_t& operator=(const event_handler_t&) = delete; - void record(rmm::cuda_stream_view stream_view) + void record(cuda::stream_ref stream_view) { - RAFT_CUDA_TRY(cudaEventRecord(event_, stream_view)); + RAFT_CUDA_TRY(cudaEventRecord(event_, stream_view.get())); } - void record_with_flags(rmm::cuda_stream_view stream_view, int flags) + void record_with_flags(cuda::stream_ref stream_view, int flags) { - RAFT_CUDA_TRY(cudaEventRecordWithFlags(event_, stream_view, flags)); + RAFT_CUDA_TRY(cudaEventRecordWithFlags(event_, stream_view.get(), flags)); } - void stream_wait(rmm::cuda_stream_view stream_view) + void stream_wait(cuda::stream_ref stream_view) { - RAFT_CUDA_TRY(cudaStreamWaitEvent(stream_view, event_)); + RAFT_CUDA_TRY(cudaStreamWaitEvent(stream_view.get(), event_)); } float elapsed_time_since_ms(const event_handler_t& start) diff --git a/cpp/src/utilities/manual_cuda_graph.cuh b/cpp/src/utilities/manual_cuda_graph.cuh index d61cf04af8..bc517b3bec 100644 --- a/cpp/src/utilities/manual_cuda_graph.cuh +++ b/cpp/src/utilities/manual_cuda_graph.cuh @@ -68,18 +68,18 @@ class manual_cuda_graph_t { ~manual_cuda_graph_t() { destroy(); } template - void run(rmm::cuda_stream_view stream, F&& work) + void run(cuda::stream_ref stream, F&& work) { if (instance_ != nullptr) { - RAFT_CUDA_TRY(cudaGraphLaunch(instance_, stream.value())); + RAFT_CUDA_TRY(cudaGraphLaunch(instance_, stream.get())); return; } // RAII: if user code throws mid-capture, end capture so the stream isn't // left in capture state. Errors are swallowed -- we're already unwinding. - capture_guard_t guard{stream.value()}; + capture_guard_t guard{stream.get()}; - RAFT_CUDA_TRY(cudaStreamBeginCapture(stream.value(), cudaStreamCaptureModeThreadLocal)); + RAFT_CUDA_TRY(cudaStreamBeginCapture(stream.get(), cudaStreamCaptureModeThreadLocal)); guard.capture_active = true; cudaGraph_t captured = nullptr; @@ -92,7 +92,7 @@ class manual_cuda_graph_t { // call). End the capture and let its status disambiguate: if the capture was // invalidated the recorded work was never issued, so recover by re-running // `work` eagerly; otherwise the error is genuine and is rethrown. - cudaError_t catch_end_err = cudaStreamEndCapture(stream.value(), &captured); + cudaError_t catch_end_err = cudaStreamEndCapture(stream.get(), &captured); guard.capture_active = false; if (catch_end_err == cudaErrorStreamCaptureInvalidated) { cudaGetLastError(); @@ -103,7 +103,7 @@ class manual_cuda_graph_t { throw; } - cudaError_t end_err = cudaStreamEndCapture(stream.value(), &captured); + cudaError_t end_err = cudaStreamEndCapture(stream.get(), &captured); guard.capture_active = false; if (end_err == cudaErrorStreamCaptureInvalidated) { @@ -124,7 +124,7 @@ class manual_cuda_graph_t { RAFT_CUDA_TRY_NO_THROW(cudaGraphDestroy(captured)); RAFT_CUDA_TRY(inst_err); - RAFT_CUDA_TRY(cudaGraphLaunch(instance_, stream.value())); + RAFT_CUDA_TRY(cudaGraphLaunch(instance_, stream.get())); } bool is_initialized() const noexcept { return instance_ != nullptr; } diff --git a/cpp/src/utilities/vector_helpers.cuh b/cpp/src/utilities/vector_helpers.cuh index b2c6cabbac..2821f76f20 100644 --- a/cpp/src/utilities/vector_helpers.cuh +++ b/cpp/src/utilities/vector_helpers.cuh @@ -39,37 +39,35 @@ __global__ void sequence_with_multiplier_kernel(T* data_ptr, int mult, size_t si } template -void async_fill(rmm::device_uvector& vec, T item, rmm::cuda_stream_view stream) +void async_fill(rmm::device_uvector& vec, T item, cuda::stream_ref stream) { constexpr size_t TPB = 256; size_t n_blocks = (vec.size() + TPB - 1) / TPB; - fill_kernel<<>>(vec.data(), item, vec.size()); + fill_kernel<<>>(vec.data(), item, vec.size()); } template -void async_fill(T* vec, T item, size_t size, rmm::cuda_stream_view stream) +void async_fill(T* vec, T item, size_t size, cuda::stream_ref stream) { constexpr size_t TPB = 256; size_t n_blocks = (size + TPB - 1) / TPB; - fill_kernel<<>>(vec, item, size); + fill_kernel<<>>(vec, item, size); } template -void async_sequence(rmm::device_uvector& vec, rmm::cuda_stream_view stream) +void async_sequence(rmm::device_uvector& vec, cuda::stream_ref stream) { constexpr size_t TPB = 256; size_t n_blocks = (vec.size() + TPB - 1) / TPB; - sequence_kernel<<>>(vec.data(), vec.size()); + sequence_kernel<<>>(vec.data(), vec.size()); } template -void async_sequence_with_multiplier(rmm::device_uvector& vec, - int mult, - rmm::cuda_stream_view stream) +void async_sequence_with_multiplier(rmm::device_uvector& vec, int mult, cuda::stream_ref stream) { constexpr size_t TPB = 256; size_t n_blocks = (vec.size() + TPB - 1) / TPB; - sequence_with_multiplier_kernel<<>>(vec.data(), mult, vec.size()); + sequence_with_multiplier_kernel<<>>(vec.data(), mult, vec.size()); } template diff --git a/cpp/tests/distance_engine/waypoint_matrix_test.cpp b/cpp/tests/distance_engine/waypoint_matrix_test.cpp index 88d4c53229..25091589fb 100644 --- a/cpp/tests/distance_engine/waypoint_matrix_test.cpp +++ b/cpp/tests/distance_engine/waypoint_matrix_test.cpp @@ -59,7 +59,7 @@ class waypoint_matrix_waypoints_sequence_test_t std::vector h_cost_matrix(this->target_locations.size() * this->target_locations.size()); raft::copy(h_cost_matrix.data(), d_cost_matrix.data(), h_cost_matrix.size(), stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); for (size_t i = 0; i != h_cost_matrix.size(); ++i) EXPECT_EQ(h_cost_matrix[i], expected_cost_matrix[i]); @@ -78,7 +78,7 @@ class waypoint_matrix_waypoints_sequence_test_t h_sequence_offsets.size(), stream); raft::copy(h_full_path.data(), (i_t*)d_full_path.get()->data(), h_full_path.size(), stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); for (size_t i = 0; i != h_sequence_offsets.size(); ++i) EXPECT_EQ(h_sequence_offsets[i], expected_sequence_offsets[i]); @@ -154,7 +154,7 @@ class waypoint_matrix_shortest_path_cost_t std::vector h_custom_matrix(this->target_locations.size() * this->target_locations.size()); raft::copy(h_custom_matrix.data(), d_custom_matrix.data(), h_custom_matrix.size(), stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); for (size_t i = 0; i != h_custom_matrix.size(); ++i) EXPECT_EQ(h_custom_matrix[i], ref_custom_matrix[i]); @@ -207,7 +207,7 @@ class waypoint_matrix_cost_matrix_test_t std::vector h_cost_matrix(this->target_locations.size() * this->target_locations.size()); raft::copy(h_cost_matrix.data(), d_cost_matrix.data(), h_cost_matrix.size(), stream); - RAFT_CUDA_TRY(cudaStreamSynchronize(stream)); + RAFT_CUDA_TRY(cudaStreamSynchronize(stream.get())); for (size_t i = 0; i != h_cost_matrix.size(); ++i) EXPECT_NEAR(h_cost_matrix[i], this->ref_cost_matrix[i], 0.001f); diff --git a/cpp/tests/dual_simplex/unit_tests/solve_barrier.cu b/cpp/tests/dual_simplex/unit_tests/solve_barrier.cu index 16640c6c60..de2f65bbfb 100644 --- a/cpp/tests/dual_simplex/unit_tests/solve_barrier.cu +++ b/cpp/tests/dual_simplex/unit_tests/solve_barrier.cu @@ -34,9 +34,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } TEST(barrier, chess_set) diff --git a/cpp/tests/linear_programming/pdlp_test.cu b/cpp/tests/linear_programming/pdlp_test.cu index d707082168..f144ad2606 100644 --- a/cpp/tests/linear_programming/pdlp_test.cu +++ b/cpp/tests/linear_programming/pdlp_test.cu @@ -503,7 +503,7 @@ TEST(pdlp_class, initial_solution_test) solver_settings); auto pdlp_timer = timer_t(solver_settings.time_limit); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); } @@ -518,7 +518,7 @@ TEST(pdlp_class, initial_solution_test) auto d_initial_primal = device_copy(initial_primal, handle_.get_stream()); solver.set_initial_primal_solution(d_initial_primal); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); } @@ -530,7 +530,7 @@ TEST(pdlp_class, initial_solution_test) auto d_initial_dual = device_copy(initial_dual, handle_.get_stream()); solver.set_initial_dual_solution(d_initial_dual); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); } @@ -545,7 +545,7 @@ TEST(pdlp_class, initial_solution_test) auto d_initial_dual = device_copy(initial_dual, handle_.get_stream()); solver.set_initial_dual_solution(d_initial_dual); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); } @@ -557,7 +557,7 @@ TEST(pdlp_class, initial_solution_test) auto pdlp_timer = timer_t(solver_settings.time_limit); solver_settings.hyper_params.update_step_size_on_initial_solution = true; solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); solver_settings.hyper_params.update_step_size_on_initial_solution = false; @@ -568,7 +568,7 @@ TEST(pdlp_class, initial_solution_test) auto pdlp_timer = timer_t(solver_settings.time_limit); solver_settings.hyper_params.update_primal_weight_on_initial_solution = true; solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); solver_settings.hyper_params.update_primal_weight_on_initial_solution = false; @@ -580,7 +580,7 @@ TEST(pdlp_class, initial_solution_test) solver_settings.hyper_params.update_primal_weight_on_initial_solution = true; solver_settings.hyper_params.update_step_size_on_initial_solution = true; solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); solver_settings.hyper_params.update_primal_weight_on_initial_solution = false; @@ -598,7 +598,7 @@ TEST(pdlp_class, initial_solution_test) auto d_initial_primal = device_copy(initial_primal, handle_.get_stream()); solver.set_initial_primal_solution(d_initial_primal); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); solver_settings.hyper_params.update_step_size_on_initial_solution = false; @@ -612,7 +612,7 @@ TEST(pdlp_class, initial_solution_test) auto d_initial_dual = device_copy(initial_dual, handle_.get_stream()); solver.set_initial_dual_solution(d_initial_dual); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NEAR(initial_step_size_afiro, solver.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(initial_primal_weight_afiro, solver.get_primal_weight_h(0), factor_tolerance); solver_settings.hyper_params.update_step_size_on_initial_solution = false; @@ -799,7 +799,7 @@ TEST(pdlp_class, initial_primal_weight_step_size_test) solver.set_initial_primal_weight(test_initial_primal_weight); solver.set_initial_step_size(test_initial_step_size); solver.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_EQ(test_initial_step_size, solver.get_step_size_h(0)); EXPECT_EQ(test_initial_primal_weight, solver.get_primal_weight_h(0)); } @@ -834,7 +834,7 @@ TEST(pdlp_class, initial_primal_weight_step_size_test) solver2.set_initial_primal_solution(d_initial_primal); solver2.set_initial_dual_solution(d_initial_dual); solver2.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); const double sovler2_step_size = solver2.get_step_size_h(0); const double sovler2_primal_weight = solver2.get_primal_weight_h(0); EXPECT_NOT_NEAR(previous_step_size, sovler2_step_size, factor_tolerance); @@ -851,7 +851,7 @@ TEST(pdlp_class, initial_primal_weight_step_size_test) solver3.set_initial_dual_solution(d_initial_dual); solver3.set_initial_dual_solution(d_initial_dual); solver3.run_solver(pdlp_timer); - RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream())); + RAFT_CUDA_TRY(cudaStreamSynchronize(handle_.get_stream().get())); EXPECT_NOT_NEAR(sovler2_step_size, solver3.get_step_size_h(0), factor_tolerance); EXPECT_NEAR(sovler2_primal_weight, solver3.get_primal_weight_h(0), factor_tolerance); } diff --git a/cpp/tests/linear_programming/utilities/pdlp_test_utilities.cuh b/cpp/tests/linear_programming/utilities/pdlp_test_utilities.cuh index 7e3f83dae9..83f322189b 100644 --- a/cpp/tests/linear_programming/utilities/pdlp_test_utilities.cuh +++ b/cpp/tests/linear_programming/utilities/pdlp_test_utilities.cuh @@ -53,7 +53,7 @@ static cuopt::mathematical_optimization::optimization_problem_solution_t static void assign_device_uvector_from_host(rmm::device_uvector& target, const std::vector& src, - rmm::cuda_stream_view stream) + cuda::stream_ref stream) { target.resize(src.size(), stream); raft::copy(target.data(), src.data(), src.size(), stream); diff --git a/cpp/tests/mip/bounds_standardization_test.cu b/cpp/tests/mip/bounds_standardization_test.cu index fffaec4989..aa920f5c0d 100644 --- a/cpp/tests/mip/bounds_standardization_test.cu +++ b/cpp/tests/mip/bounds_standardization_test.cu @@ -35,9 +35,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } void test_bounds_standardization_test(std::string test_instance) diff --git a/cpp/tests/mip/elim_var_remap_test.cu b/cpp/tests/mip/elim_var_remap_test.cu index 1cbc1cc60f..56521d614a 100644 --- a/cpp/tests/mip/elim_var_remap_test.cu +++ b/cpp/tests/mip/elim_var_remap_test.cu @@ -38,9 +38,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } std::vector select_k_random(int population_size, int sample_size) diff --git a/cpp/tests/mip/multi_probe_test.cu b/cpp/tests/mip/multi_probe_test.cu index 9438bf6183..61be30e15c 100644 --- a/cpp/tests/mip/multi_probe_test.cu +++ b/cpp/tests/mip/multi_probe_test.cu @@ -37,9 +37,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } std::tuple, std::vector, std::vector> select_k_random( diff --git a/cpp/tests/routing/level0/l0_routing_test.cu b/cpp/tests/routing/level0/l0_routing_test.cu index 28bd8db9c7..2b13a09ecc 100644 --- a/cpp/tests/routing/level0/l0_routing_test.cu +++ b/cpp/tests/routing/level0/l0_routing_test.cu @@ -372,11 +372,11 @@ class routing_retail_test_t : public base_test_t, raft::copy(this->vehicle_earliest_d.data(), this->vehicle_earliest_h.data(), input_.n_vehicles, - this->stream_view_.value()); + this->stream_view_.get()); raft::copy(this->vehicle_latest_d.data(), this->vehicle_latest_h.data(), input_.n_vehicles, - this->stream_view_.value()); + this->stream_view_.get()); data_model.set_vehicle_time_windows(this->vehicle_earliest_d.data(), this->vehicle_latest_d.data()); } @@ -392,7 +392,7 @@ class routing_retail_test_t : public base_test_t, raft::copy(d_int_drop_return_trip.data(), this->drop_return_trips_h.data(), input_.n_vehicles, - this->stream_view_.value()); + this->stream_view_.get()); thrust::transform(this->handle_.get_thrust_policy(), d_int_drop_return_trip.begin(), d_int_drop_return_trip.end(), @@ -402,13 +402,13 @@ class routing_retail_test_t : public base_test_t, raft::copy(d_int_skip_first_trip.data(), this->skip_first_trips_h.data(), input_.n_vehicles, - this->stream_view_.value()); + this->stream_view_.get()); thrust::transform(this->handle_.get_thrust_policy(), d_int_skip_first_trip.begin(), d_int_skip_first_trip.end(), d_skip_first_trip.begin(), id); - RAFT_CUDA_TRY(cudaStreamSynchronize(this->stream_view_.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(this->stream_view_.get())); data_model.set_drop_return_trips(d_drop_return_trip.data()); data_model.set_skip_first_trips(d_skip_first_trip.data()); } @@ -423,11 +423,11 @@ class routing_retail_test_t : public base_test_t, raft::copy(this->random_demand_d.data(), shuffled_vec.data(), this->n_orders, - this->stream_view_.value()); + this->stream_view_.get()); raft::copy(this->mixed_capacity_d.data(), input_.mixed_capacity_h.data(), this->n_vehicles, - this->stream_view_.value()); + this->stream_view_.get()); data_model.add_capacity_dimension( "random", this->random_demand_d.data(), this->mixed_capacity_d.data()); } diff --git a/cpp/tests/routing/level0/l0_vehicle_order_match.cu b/cpp/tests/routing/level0/l0_vehicle_order_match.cu index f99d1a33df..bed479c565 100644 --- a/cpp/tests/routing/level0/l0_vehicle_order_match.cu +++ b/cpp/tests/routing/level0/l0_vehicle_order_match.cu @@ -58,7 +58,7 @@ class vehicle_order_test_t : public base_test_t, public ::testing::Tes d_int_vec.end(), d_drop_return_trip.begin(), cuda::std::identity{}); - RAFT_CUDA_TRY(cudaStreamSynchronize(this->stream_view_.value())); + RAFT_CUDA_TRY(cudaStreamSynchronize(this->stream_view_.get())); } data_model.set_drop_return_trips(d_drop_return_trip.data()); diff --git a/cpp/tests/routing/routing_test.cuh b/cpp/tests/routing/routing_test.cuh index cdafbbf1f7..dd942f3235 100644 --- a/cpp/tests/routing/routing_test.cuh +++ b/cpp/tests/routing/routing_test.cuh @@ -689,7 +689,7 @@ class base_test_t { } raft::handle_t handle_; - rmm::cuda_stream_view stream_view_; + cuda::stream_ref stream_view_; HighResTimer hr_timer_; bool multi_capacity_{false}; diff --git a/cpp/tests/routing/unit_tests/top_k.cu b/cpp/tests/routing/unit_tests/top_k.cu index c6d377a63f..6c6bdaab68 100644 --- a/cpp/tests/routing/unit_tests/top_k.cu +++ b/cpp/tests/routing/unit_tests/top_k.cu @@ -98,7 +98,7 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW raft::copy(d_input_cost.data(), h_input_cost.data(), h_input_cost.size(), this->stream_view_); - this->stream_view_.synchronize(); + this->stream_view_.sync(); call_top_k(d_input_cost, d_output_cost, d_out_index); verify_top_k(h_input_cost, d_output_cost, d_out_index); @@ -159,11 +159,12 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW rmm::device_uvector& out_index) { constexpr int TPB = 128; - top_k_indices<<stream_view_>>>(width, - cuopt::make_span(input_cost), - cuopt::make_span(output_cost), - cuopt::make_span(out_index)); - this->stream_view_.synchronize(); + top_k_indices + <<stream_view_.get()>>>(width, + cuopt::make_span(input_cost), + cuopt::make_span(output_cost), + cuopt::make_span(out_index)); + this->stream_view_.sync(); RAFT_CUDA_TRY(cudaGetLastError()); } @@ -171,15 +172,15 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW rmm::device_uvector& d_output_cost, rmm::device_uvector& d_out_index) { - this->stream_view_.synchronize(); + this->stream_view_.sync(); std::vector h_output_cost(d_output_cost.size()); raft::copy( h_output_cost.data(), d_output_cost.data(), d_output_cost.size(), this->stream_view_); - this->stream_view_.synchronize(); + this->stream_view_.sync(); std::vector h_sorted_index(d_out_index.size()); raft::copy(h_sorted_index.data(), d_out_index.data(), d_out_index.size(), this->stream_view_); - this->stream_view_.synchronize(); + this->stream_view_.sync(); std::vector sorted_data(width); for (int i = 0; i < width; ++i) { // copy row i @@ -238,7 +239,7 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW rmm::device_uvector d_cub_storage_bytes(0, this->stream_view_); d_cub_storage_bytes.resize(tmp_storage_bytes, this->stream_view_); double elapsed_ms; - this->stream_view_.synchronize(); + this->stream_view_.sync(); { time_it t(&elapsed_ms); for (int i = 0; i < iter; ++i) { @@ -254,7 +255,7 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW segment_marker.data() + 1, this->stream_view_); } - this->stream_view_.synchronize(); + this->stream_view_.sync(); } return elapsed_ms; } @@ -268,13 +269,13 @@ class top_cand_test_t : public routing_test_t, public ::testing::TestW raft::copy(d_input_cost.data(), input_cost.data(), input_cost.size(), this->stream_view_); double elapsed_ms; - this->stream_view_.synchronize(); + this->stream_view_.sync(); { time_it t(&elapsed_ms); for (int i = 0; i < iter; ++i) { call_top_k(d_input_cost, d_output_cost, d_out_index); } - this->stream_view_.synchronize(); + this->stream_view_.sync(); } return elapsed_ms; } diff --git a/cpp/tests/socp/general_quadratic_test.cu b/cpp/tests/socp/general_quadratic_test.cu index b2a5afeafb..182c978eef 100644 --- a/cpp/tests/socp/general_quadratic_test.cu +++ b/cpp/tests/socp/general_quadratic_test.cu @@ -40,9 +40,10 @@ using qc_t = optimization_problem_interface_t::quadratic_constraint_t; static void init_handler(const raft::handle_t* handle_ptr) { RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } // Test: general convex quadratic constraint with dense PD Q matrix. diff --git a/cpp/tests/socp/solve_barrier_socp.cu b/cpp/tests/socp/solve_barrier_socp.cu index 68e2cb2d31..59fa339904 100644 --- a/cpp/tests/socp/solve_barrier_socp.cu +++ b/cpp/tests/socp/solve_barrier_socp.cu @@ -27,9 +27,10 @@ static void init_handler(const raft::handle_t* handle_ptr) { // Init cuBlas / cuSparse context here to avoid having it during solving time RAFT_CUBLAS_TRY(raft::linalg::detail::cublassetpointermode( - handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream())); - RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode( - handle_ptr->get_cusparse_handle(), CUSPARSE_POINTER_MODE_DEVICE, handle_ptr->get_stream())); + handle_ptr->get_cublas_handle(), CUBLAS_POINTER_MODE_DEVICE, handle_ptr->get_stream().get())); + RAFT_CUSPARSE_TRY(raft::sparse::detail::cusparsesetpointermode(handle_ptr->get_cusparse_handle(), + CUSPARSE_POINTER_MODE_DEVICE, + handle_ptr->get_stream().get())); } TEST(barrier, cone_metadata_reindexed_when_slack_is_inserted_before_cones) diff --git a/cpp/tests/socp/sparse_augmented_kkt_test.cu b/cpp/tests/socp/sparse_augmented_kkt_test.cu index b4dae0591b..1c8cd8ccce 100644 --- a/cpp/tests/socp/sparse_augmented_kkt_test.cu +++ b/cpp/tests/socp/sparse_augmented_kkt_test.cu @@ -23,8 +23,7 @@ namespace cuopt::mathematical_optimization::barrier::test { namespace { // Packed Hs_diag reference: eta^2 on every entry, head scaled by rank-2 corner d. -std::vector expected_Hs_diag(const cone_data_t& cones, - rmm::cuda_stream_view stream) +std::vector expected_Hs_diag(const cone_data_t& cones, cuda::stream_ref stream) { const int E = static_cast(cones.n_sparse_cone_entries); auto d_host = cuopt::host_copy(cones.d, stream);