From c5c868617f5eaf76f6f3b5317139674f15b4fb86 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 09:25:49 -0700 Subject: [PATCH 01/14] initial implementation --- .../linear_algebra/CPreconditioner.hpp | 5 +- Common/include/linear_algebra/CSysMatrix.hpp | 90 +++- Common/src/linear_algebra/CSysMatrix.cpp | 115 ++++- .../linear_algebra/CSysPreconditionerGPU.cu | 480 ++++++++++++++++++ 4 files changed, 648 insertions(+), 42 deletions(-) diff --git a/Common/include/linear_algebra/CPreconditioner.hpp b/Common/include/linear_algebra/CPreconditioner.hpp index d28c729c4f0..e23f5de381f 100644 --- a/Common/include/linear_algebra/CPreconditioner.hpp +++ b/Common/include/linear_algebra/CPreconditioner.hpp @@ -40,7 +40,7 @@ /*! * \brief Applies a preconditioner that only has a host implementation to vectors that live * on the device: bring the input down, apply, put the result back. - * \note This is what keeps ILU, LU-SGS, Linelet and PaStiX usable on the GPU path. The + * \note This is what keeps LU-SGS, Linelet and PaStiX usable on the GPU path. The * transfers are issued by one thread with the team synchronized around them, the apply * itself is the normal OpenMP parallel host code. */ @@ -199,7 +199,8 @@ class CILUPreconditioner final : public CPreconditioner { * \param[out] v - CSysVector that is the result of the preconditioning. */ inline void operator()(const CSysVector& u, CSysVector& v) const override { - ApplyPreconditionerOnHost(u, v, [&] { sparse_matrix.ComputeILUPreconditioner(u, v, geometry, config); }); + /*--- No host bracket, ILU has a device implementation and ComputeILUPreconditioner dispatches to it. ---*/ + sparse_matrix.ComputeILUPreconditioner(u, v, geometry, config); } /*! diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index 3eddf892a7b..7afdfe8d3e2 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -250,6 +250,7 @@ class CSysMatrix { LDU mat; /*!< \brief Host matrix (values owned via aligned_alloc; pattern from geometry). */ LDU gpu; /*!< \brief Device matrix (all pointers to GPU memory). */ LDU ilu; /*!< \brief ILU factorization, host (values owned; pattern from geometry). */ + LDU gpu_ilu; /*!< \brief ILU factorization, device (values and pattern in GPU memory). */ ScalarType* d_invM = nullptr; /*!< \brief Device inverse diagonal blocks for the Jacobi preconditioner. */ /*--- Quantized off-diagonal storage (used when quantized_mode == true). ---*/ @@ -271,7 +272,12 @@ class CSysMatrix { * Populated by QuantizeDiagonalBlocks(). */ QuantType* q_blocks_d; /*!< \brief Same as q_blocks_l for the diagonal entries. */ - bool useCuda = false; /*!< \brief Whether CUDA is enabled. */ + bool useCuda = false; /*!< \brief Whether CUDA is enabled. */ + + /*!< \brief Whether the inverse diagonal blocks are only needed on the device. False for the + * Linelet preconditioner, which builds the Jacobi one but reads invM on the host. */ + bool jacobi_on_device = false; + const su2uint* l_to_u_transp; /*!< \brief L-entry index -> U-entry index of its transpose. */ const su2uint* u_to_l_transp; /*!< \brief U-entry index -> L-entry index of its transpose. */ @@ -284,9 +290,39 @@ class CSysMatrix { unsigned short ilu_fill_in; /*!< \brief Fill level for the ILU preconditioner. */ - /*!< \brief Level structure for alternative shared memory parallelization of ILU. */ + /*!< \brief Level structure for alternative shared memory parallelization of ILU. + * Rows within a level are independent, rows in level k only depend on rows in levels < k. + * The same table drives the forward (increasing level) and backward (decreasing level) + * sweeps, because the U pattern is the transpose of the L pattern. */ CCompressedSparsePatternUL levels_ilu; + /*--- Device copy of levels_ilu. The rows of a level are not contiguous in the matrix, so + * the kernels have to go through this table to find the rows they work on. The offsets stay + * on the host because they size the grid of the per-level kernel launches. ---*/ + vector ilu_level_ptr; /*!< \brief Start of each level in d_ilu_level_idx, size nLevels+1. */ + su2uint* d_ilu_level_idx = nullptr; /*!< \brief Row indices, grouped by level. */ + + /*--- The per-level kernel launch sequence (init + one kernel per level for the factorization, + * one per level for each of the forward/backward sweeps) is identical on every call: same + * grid/block sizes, same device pointers (all fixed members, allocated once). It is captured + * once into a CUDA graph and replayed, which removes host-side launch overhead without + * changing the parallelization (unlike a persistent cooperative-groups kernel, this does not + * cap per-level parallelism to the occupancy-resident block count). ---*/ + /*--- Types are forward-declared as opaque structs (matching the real cudaGraphExec_t / + * cudaStream_t typedefs) so this header does not need to include the CUDA runtime. ---*/ + mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; + mutable struct CUgraphExec_st* ilu_apply_graph_exec = nullptr; + mutable const ScalarType* ilu_apply_graph_vec = nullptr; /*!< \brief Pointers the apply graph + * was captured with, to detect when + * it must be recaptured. */ + mutable ScalarType* ilu_apply_graph_prod = nullptr; + /*--- The legacy default stream cannot be captured into a graph, so the ILU graphs are + * captured and replayed on this dedicated stream instead; every launch on it is followed by + * a sync back to the host before control returns to the rest of the (single-stream) solver, + * so this does not change execution order relative to everything else, which stays on the + * default stream. ---*/ + mutable struct CUstream_st* ilu_stream = nullptr; + ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ /*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ @@ -506,6 +542,31 @@ class CSysMatrix { * ScalarType buffer and delegates to the scalar GaussElimination overload. */ inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; + /*--- Hooks for GPU versions of the preconditioners (implemented is in CSysMatrixGPU.cu). ---*/ + + /*! + * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildJacobiPreconditionerGPU(); + + /*! + * \brief Apply the Jacobi preconditioner on the GPU/device side. + */ + void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const; + + /*! + * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildILUPreconditionerGPU(); + + /*! + * \brief Apply the ILU preconditioner on the device. + */ + void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; + public: /*! * \brief Constructor of the class. @@ -1053,23 +1114,6 @@ class CSysMatrix { void GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Performs first step of the LU_SGS Preconditioner building - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUFirstSymmetricIteration(ScalarType& vec, ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - - /*! - * \brief Performs second step of the LU_SGS Preconditioner building - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUSecondSymmetricIteration(ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - /*! * \brief Performs Gaussian Elimination between diagional blocks of the matrix and the prod vector * \param[in] geometry - Geometrical definition of the problem. @@ -1101,14 +1145,6 @@ class CSysMatrix { void ComputeJacobiPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Apply the Jacobi preconditioner on the GPU/device side. - * \note This helper is intended as the implementation hook for GPU-resident Krylov solvers. - * The actual implementation belongs in CSysMatrixGPU.cu. - */ - void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, - CGeometry* geometry, const CConfig* config) const; - /*! * \brief Build the ILU preconditioner. */ diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index a0013e24300..b931ccb6d55 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -131,6 +131,17 @@ CSysMatrix::~CSysMatrix() { GPUMemoryAllocation::gpu_free(gpu.row_ptr_u); GPUMemoryAllocation::gpu_free(gpu.col_ind_u); GPUMemoryAllocation::gpu_free(d_invM); + GPUMemoryAllocation::gpu_free(gpu_ilu.d); + GPUMemoryAllocation::gpu_free(gpu_ilu.l); + GPUMemoryAllocation::gpu_free(gpu_ilu.u); + GPUMemoryAllocation::gpu_free(gpu_ilu.row_ptr_l); + GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_l); + GPUMemoryAllocation::gpu_free(gpu_ilu.row_ptr_u); + GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_u); + GPUMemoryAllocation::gpu_free(d_ilu_level_idx); + if (ilu_build_graph_exec != nullptr) cudaGraphExecDestroy(ilu_build_graph_exec); + if (ilu_apply_graph_exec != nullptr) cudaGraphExecDestroy(ilu_apply_graph_exec); + if (ilu_stream != nullptr) cudaStreamDestroy(ilu_stream); } #ifdef USE_MKL @@ -182,6 +193,10 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi const bool ilu_needed = (prec == ILU); const bool diag_needed = (prec == JACOBI) || (prec == LINELET); + + /*--- Linelet also builds the Jacobi preconditioner but reads the inverse diagonal blocks on + * the host, so only plain Jacobi can keep them exclusively on the device. ---*/ + jacobi_on_device = useCuda && (prec == JACOBI); #ifndef CODI_REVERSE_TYPE const bool q_lus_needed = allow_quant && !useCuda && (prec == Q_LU_SGS); #else @@ -269,8 +284,22 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi ilu.col_ind_u = pat_ilu.u.innerIdx(); ilu.nnz_u = pat_ilu.u.getNumNonZeros(); - if (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels()) { - levels_ilu = computeLevels(pat_ilu.l); + /*--- The GPU implementation is only level-scheduled, so the levels are not optional there. ---*/ + if (useCuda || (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels())) { + /*--- The pattern spans all points but only the domain rows are factorized, so drop the + * halo rows. This cannot change the levels of the domain rows: a row can only depend on + * rows with a lower index, and every halo row has a higher index than every domain row. ---*/ + const auto all_levels = computeLevels(pat_ilu.l); + std::vector> levels; + for (auto level = 0ul; level < all_levels.getOuterSize(); ++level) { + std::vector rows; + for (auto k = 0ul; k < all_levels.getNumNonZeros(level); ++k) { + const auto iPoint = all_levels.getInnerIdx(level, k); + if (iPoint < nPointDomain) rows.push_back(iPoint); + } + if (!rows.empty()) levels.push_back(std::move(rows)); + } + levels_ilu = CCompressedSparsePatternUL(levels); } } @@ -284,10 +313,37 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi if (diag_needed) allocAndInit(invM, nPointDomain * nVar * nEqn); - if (useCuda && diag_needed) { + if (jacobi_on_device) { d_invM = GPUMemoryAllocation::gpu_alloc(nPointDomain * nVar * nEqn * sizeof(ScalarType)); } + if (useCuda && ilu_needed) { + /*--- The factors are built and used on the device, only the pattern and the level table + * are uploaded (once, here) because they do not change. ---*/ + gpu_ilu.nnz_l = ilu.nnz_l; + gpu_ilu.nnz_u = ilu.nnz_u; + gpu_ilu.d = GPUMemoryAllocation::gpu_alloc(nPointDomain * nVar * nEqn * sizeof(ScalarType)); + gpu_ilu.l = GPUMemoryAllocation::gpu_alloc(ilu.nnz_l * nVar * nEqn * sizeof(ScalarType)); + gpu_ilu.u = GPUMemoryAllocation::gpu_alloc(ilu.nnz_u * nVar * nEqn * sizeof(ScalarType)); + gpu_ilu.row_ptr_l = GPUMemoryAllocation::gpu_alloc_cpy(ilu.row_ptr_l, (nPointDomain + 1) * sizeof(su2uint)); + gpu_ilu.col_ind_l = GPUMemoryAllocation::gpu_alloc_cpy(ilu.col_ind_l, ilu.nnz_l * sizeof(su2uint)); + gpu_ilu.row_ptr_u = GPUMemoryAllocation::gpu_alloc_cpy(ilu.row_ptr_u, (nPointDomain + 1) * sizeof(su2uint)); + gpu_ilu.col_ind_u = GPUMemoryAllocation::gpu_alloc_cpy(ilu.col_ind_u, ilu.nnz_u * sizeof(su2uint)); + + /*--- Flatten the level structure, the index type differs from the one of the pattern. ---*/ + std::vector level_idx; + level_idx.reserve(nPointDomain); + ilu_level_ptr.clear(); + ilu_level_ptr.push_back(0); + for (auto level = 0ul; level < levels_ilu.getOuterSize(); ++level) { + for (auto k = 0ul; k < levels_ilu.getNumNonZeros(level); ++k) { + level_idx.push_back(static_cast(levels_ilu.getInnerIdx(level, k))); + } + ilu_level_ptr.push_back(static_cast(level_idx.size())); + } + d_ilu_level_idx = GPUMemoryAllocation::gpu_alloc_cpy(level_idx.data(), level_idx.size() * sizeof(su2uint)); + } + /*--- Thread parallel initialization. ---*/ int num_threads = omp_get_max_threads(); @@ -811,18 +867,11 @@ template void CSysMatrix::BuildJacobiPreconditioner() { SU2_ZONE_SCOPED - /*--- Build Jacobi preconditioner (M = D), compute and store the inverses of the diagonal blocks. ---*/ - SU2_OMP_FOR_DYN(omp_heavy_size) - for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) - InverseDiagonalBlock(iPoint, &(invM[iPoint * nVar * nVar])); - END_SU2_OMP_FOR - - if (useCuda) { + if (jacobi_on_device) { #ifdef SU2_ENABLE_CUDA_KERNELS if constexpr (su2_gpu_capable_v) { - BEGIN_SU2_DEVICE_REGION - gpuErrChk(cudaMemcpy(d_invM, invM, nPointDomain * nVar * nVar * sizeof(ScalarType), cudaMemcpyHostToDevice)); - END_SU2_DEVICE_REGION + SU2_DEVICE_REGION(BuildJacobiPreconditionerGPU();) + return; } else { SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); } @@ -833,6 +882,12 @@ void CSysMatrix::BuildJacobiPreconditioner() { CURRENT_FUNCTION); #endif } + + /*--- Build Jacobi preconditioner (M = D), compute and store the inverses of the diagonal blocks. ---*/ + SU2_OMP_FOR_DYN(omp_heavy_size) + for (unsigned long iPoint = 0; iPoint < nPointDomain; iPoint++) + InverseDiagonalBlock(iPoint, &(invM[iPoint * nVar * nVar])); + END_SU2_OMP_FOR } template @@ -872,6 +927,23 @@ void CSysMatrix::ComputeJacobiPreconditioner(const CSysVector void CSysMatrix::BuildILUPreconditioner() { SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(BuildILUPreconditionerGPU();) + return; + } else { + SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); + } +#else + SU2_MPI::Error( + "\nError in building ILU preconditioner\nENABLE_CUDA is set to YES\nPlease compile with CUDA options " + "enabled in Meson to access GPU Functions", + CURRENT_FUNCTION); +#endif + } + const auto blockSize = nVar * nVar; ScalarType Lij[MAXNVAR * MAXNVAR], Lij_Ujk[MAXNVAR * MAXNVAR]; @@ -1002,6 +1074,23 @@ template void CSysMatrix::ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + SU2_DEVICE_REGION(ComputeILUPreconditionerGPU(vec, prod);) + return; + } else { + SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); + } +#else + SU2_MPI::Error( + "\nError in applying ILU preconditioner\nENABLE_CUDA is set to YES\nPlease compile with CUDA options " + "enabled in Meson to access GPU Functions", + CURRENT_FUNCTION); +#endif + } + /*--- Coherent view of vectors. ---*/ SU2_OMP_BARRIER diff --git a/Common/src/linear_algebra/CSysPreconditionerGPU.cu b/Common/src/linear_algebra/CSysPreconditionerGPU.cu index 794726fbce3..9d24a4cfa26 100644 --- a/Common/src/linear_algebra/CSysPreconditionerGPU.cu +++ b/Common/src/linear_algebra/CSysPreconditionerGPU.cu @@ -25,6 +25,8 @@ * License along with SU2. If not, see . */ +#include + #include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/linear_algebra/GPUComms.cuh" @@ -49,6 +51,316 @@ __global__ void ApplyJacobiPreconditionerKernel(const ScalarType* invM, const Sc } } +/*--- ILU. The factorization and both triangular solves are level scheduled: the rows of a + * level are independent of each other and only depend on rows of previous levels, so each + * level is one kernel launch and the launch boundaries provide the synchronization. The rows + * of a level are scattered through the matrix, hence the indirection through the level table. + * Throughout, one CUDA block works on one row. ---*/ + +/*! + * \brief The pointers of an LDU-partitioned matrix, all in device memory. This mirrors the + * private CSysMatrix::LDU, which the kernels cannot name. + */ +template +struct DeviceLDU { + ScalarType* d; + ScalarType* l; + ScalarType* u; + const su2uint* row_ptr_l; + const su2uint* col_ind_l; + const su2uint* row_ptr_u; + const su2uint* col_ind_u; +}; + +/*! + * \brief Start of block (i,j), or nullptr if it is not a nonzero of the pattern. + */ +template +__device__ FORCEINLINE ScalarType* GetBlockILU(const DeviceLDU& M, unsigned long nVar, + unsigned long block_i, unsigned long block_j) { + const auto blockSize = nVar * nVar; + if (block_i == block_j) return M.d + block_i * blockSize; + + const bool lower = block_j < block_i; + const auto* row_ptr = lower ? M.row_ptr_l : M.row_ptr_u; + const auto* col_ind = lower ? M.col_ind_l : M.col_ind_u; + auto* vals = lower ? M.l : M.u; + + for (auto k = row_ptr[block_i]; k < row_ptr[block_i + 1]; ++k) { + if (col_ind[k] == block_j) return vals + k * blockSize; + } + return nullptr; +} + +/*! + * \brief Device version of the pivot regularization used by the host factorization, + * it has to match to give the same factors. + */ +template +__device__ FORCEINLINE void RegularizePivotDevice(ScalarType& pivot) { + const float eps = 1e-12; + if (fabs(pivot) < eps) pivot = copysign(ScalarType(eps), pivot); +} + +/*! + * \brief Invert a small dense matrix, \p A is the (destroyed) input and \p M the inverse. + * \note Serial port of CSysMatrix::MatrixInverse, run by one thread of the block. + */ +template +__device__ void MatrixInverseDevice(unsigned long nVar, ScalarType* A, ScalarType* M) { + for (auto iVar = 0ul; iVar < nVar; ++iVar) + for (auto jVar = 0ul; jVar < nVar; ++jVar) M[iVar * nVar + jVar] = ScalarType(iVar == jVar); + + /*--- Transform system in Upper Matrix. ---*/ + for (auto iVar = 1ul; iVar < nVar; ++iVar) { + for (auto jVar = 0ul; jVar < iVar; ++jVar) { + RegularizePivotDevice(A[jVar * nVar + jVar]); + + const ScalarType weight = A[iVar * nVar + jVar] / A[jVar * nVar + jVar]; + + for (auto kVar = jVar; kVar < nVar; ++kVar) A[iVar * nVar + kVar] -= weight * A[jVar * nVar + kVar]; + + /*--- At this stage M is lower triangular so not all cols need updating. ---*/ + for (auto kVar = 0ul; kVar <= jVar; ++kVar) M[iVar * nVar + kVar] -= weight * M[jVar * nVar + kVar]; + } + } + + /*--- Backwards substitution. ---*/ + for (auto iVar = nVar; iVar > 0ul;) { + --iVar; // unsigned type + for (auto jVar = iVar + 1; jVar < nVar; ++jVar) + for (auto kVar = 0ul; kVar < nVar; ++kVar) M[iVar * nVar + kVar] -= A[iVar * nVar + jVar] * M[jVar * nVar + kVar]; + + RegularizePivotDevice(A[iVar * nVar + iVar]); + + for (auto kVar = 0ul; kVar < nVar; ++kVar) M[iVar * nVar + kVar] /= A[iVar * nVar + iVar]; + } +} + +/*! + * \brief Invert the diagonal blocks of the matrix, device version of InverseDiagonalBlock. + * \note Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry, they + * stage the input in shared memory). Dynamic shared memory: nVar*nVar scalars. + */ +template +__global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nVar, + const ScalarType* __restrict__ mat_d, ScalarType* __restrict__ invM) { + const unsigned long iRow = blockIdx.x; + if (iRow >= nRows) return; + + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + + /*--- The inversion destroys its input, so it cannot work on the matrix itself. ---*/ + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* work = reinterpret_cast(smem); + + work[tid] = mat_d[iRow * blockSize + tid]; + __syncthreads(); + + if (tid == 0) MatrixInverseDevice(nVar, work, invM + iRow * blockSize); +} + +/*! + * \brief Copy the matrix into the storage of the factorization, whose pattern may be larger + * (fill-in), entries that the matrix does not have are set to zero. + * \note Device version of the InitIluRow helper of BuildILUPreconditioner. Rows are + * independent, so this is done for the entire matrix before the factorization starts. + * Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry). + */ +template +__global__ void IluInitKernel(unsigned long nRows, unsigned long nVar, DeviceLDU A, + DeviceLDU M) { + const unsigned long iRow = blockIdx.x; + if (iRow >= nRows) return; + + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + + M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; + + /*--- Merge-scan the row of the matrix onto the row of the factorization, both are sorted + * by column index. Every thread walks the scan for its own entry of the blocks. ---*/ + auto scatter = [&](const su2uint* a_row_ptr, const su2uint* a_col_ind, const ScalarType* a_vals, + const su2uint* m_row_ptr, const su2uint* m_col_ind, ScalarType* m_vals) { + auto ka = a_row_ptr[iRow]; + const auto ka_end = a_row_ptr[iRow + 1]; + + for (auto k = m_row_ptr[iRow]; k < m_row_ptr[iRow + 1]; ++k) { + const auto jPoint = m_col_ind[k]; + while (ka < ka_end && a_col_ind[ka] < jPoint) ++ka; + + if (ka < ka_end && a_col_ind[ka] == jPoint) { + m_vals[k * blockSize + tid] = a_vals[ka * blockSize + tid]; + } else { + m_vals[k * blockSize + tid] = ScalarType(0); + } + } + }; + scatter(A.row_ptr_l, A.col_ind_l, A.l, M.row_ptr_l, M.col_ind_l, M.l); + scatter(A.row_ptr_u, A.col_ind_u, A.u, M.row_ptr_u, M.col_ind_u, M.u); +} + +/*! + * \brief Factorize the rows of one level, device version of the BuildIluRow helper. + * \note Grid: one block per row of the level, blockDim.x == nVar*nVar (one thread per block + * entry, so that the small matrix products are one dot product per thread). + * Dynamic shared memory: 2*nVar*nVar scalars. + */ +template +__global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* Lij = reinterpret_cast(smem); + auto* work = Lij + blockSize; + + /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ + for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { + /*--- All threads must be done with the previous entry: Lij is about to be overwritten, + * and the blocks of this row updated below are read here across threads. ---*/ + __syncthreads(); + + /*--- jPoint is the column index (jPoint < iRow). ---*/ + const unsigned long jPoint = M.col_ind_l[kl]; + + /*--- Multiply the block by the inverse of the corresponding diagonal block. ---*/ + auto* Block_ij = M.l + kl * blockSize; + const auto* invUjj = M.d + jPoint * blockSize; + + ScalarType sum = 0; + for (auto k = 0ul; k < nVar; ++k) sum += Block_ij[iVar * nVar + k] * invUjj[k * nVar + jVar]; + Lij[tid] = sum; + __syncthreads(); + + /*--- Lij holds Aij*inv(Ujj). Jump to the upper part of the jPoint row. ---*/ + for (auto ku = M.row_ptr_u[jPoint]; ku < M.row_ptr_u[jPoint + 1]; ++ku) { + /*--- Get the column index (kPoint > jPoint), halo columns are not factorized. ---*/ + const unsigned long kPoint = M.col_ind_u[ku]; + if (kPoint >= nRows) break; + + /*--- If Aik exists, update it: Aik -= Lij * Ujk ---*/ + auto* Block_ik = GetBlockILU(M, nVar, iRow, kPoint); + if (Block_ik == nullptr) continue; + + /*--- Block_ik cannot alias Block_ij because kPoint > jPoint. ---*/ + const auto* Ujk = M.u + ku * blockSize; + ScalarType prod = 0; + for (auto k = 0ul; k < nVar; ++k) prod += Lij[iVar * nVar + k] * Ujk[k * nVar + jVar]; + Block_ik[tid] -= prod; + } + + /*--- Store Lij in the lower triangular part, each thread only writes its own entry. ---*/ + Block_ij[tid] = Lij[tid]; + } + + /*--- Invert the diagonal entry, Uii, for the next levels. The loop above may have updated + * it (when kPoint == iRow), so the whole block has to be done first. ---*/ + __syncthreads(); + work[tid] = M.d[iRow * blockSize + tid]; + __syncthreads(); + if (tid == 0) MatrixInverseDevice(nVar, work, M.d + iRow * blockSize); +} + +/*! + * \brief Forward substitution for the rows of one level, (L+I).prod = vec. + * \note One thread per block entry (like the factorization kernel), so the inner dot product + * over a neighbor block is spread across nVar threads instead of done serially by one. + * Unlike the factorization, forward/backward only ever *read* already-finalized prod + * values (no row-to-row write-through during the loop), so each thread can accumulate + * its own (iVar,jVar) partial product across every neighbor with no synchronization at + * all, and only the final nVar-way reduction (summing over jVar for each iVar) needs one + * __syncthreads() — not one per neighbor. Grid: one block per row of the level, + * blockDim.x == nVar*nVar. Dynamic shared memory: nVar*nVar scalars. + */ +template +__global__ void IluForwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nVar, DeviceLDU M, + const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + + ScalarType acc = 0; + /*--- The columns of L are rows of previous levels, so prod is final for all of them. ---*/ + for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { + const unsigned long jPoint = M.col_ind_l[kl]; + const auto* blk = M.l + kl * nVar * nVar; + acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; + } + partial[tid] = acc; + __syncthreads(); + + if (jVar == 0) { + ScalarType sum = vec[iRow * nVar + iVar]; + for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; + prod[iRow * nVar + iVar] = sum; + } +} + +/*! + * \brief Backward substitution for the rows of one level, U.prod = prod. + * \note Same idea as IluForwardLevelKernel: each thread accumulates its own (iVar,jVar) + * partial product across every neighbor with no synchronization, then one + * __syncthreads() to reduce over jVar and get the elimination result per iVar, then a + * second __syncthreads() before the diagonal multiply (which needs every iVar's result). + * Grid: one block per row of the level, blockDim.x == nVar*nVar. Dynamic shared memory: + * nVar*nVar + nVar scalars. + */ +template +__global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + auto* aux = partial + blockSize; + + ScalarType acc = 0; + /*--- The columns of U are rows of later levels, already updated by this sweep. ---*/ + for (auto ku = M.row_ptr_u[iRow]; ku < M.row_ptr_u[iRow + 1]; ++ku) { + const unsigned long jPoint = M.col_ind_u[ku]; + if (jPoint >= nRows) break; + const auto* blk = M.u + ku * blockSize; + acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; + } + partial[tid] = acc; + __syncthreads(); + + if (jVar == 0) { + ScalarType sum = prod[iRow * nVar + iVar]; + for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; + /*--- The diagonal blocks are stored inverted by the factorization. ---*/ + aux[iVar] = sum; + } + __syncthreads(); + + if (jVar == 0) { + const auto* invUii = M.d + iRow * blockSize; + ScalarType out = 0; + for (auto k = 0ul; k < nVar; ++k) out += invUii[iVar * nVar + k] * aux[k]; + prod[iRow * nVar + iVar] = out; + } +} + } // namespace template @@ -71,12 +383,180 @@ void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector +void CSysMatrix::BuildJacobiPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA Jacobi preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nVar != nEqn) { + SU2_MPI::Error("CUDA Jacobi preconditioner requires square blocks.", CURRENT_FUNCTION); + } + if (nVar * nVar > 1024) { + SU2_MPI::Error("CUDA Jacobi preconditioner uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const auto blockSize = static_cast(nVar * nVar); + InvertDiagonalBlocksKernel + <<(nPointDomain), blockSize, blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, + d_invM); + gpuErrChk(cudaPeekAtLastError()); +} + +template +void CSysMatrix::BuildILUPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nVar != nEqn) { + SU2_MPI::Error("CUDA ILU factorization requires square blocks.", CURRENT_FUNCTION); + } + if (nVar * nVar > 1024) { + SU2_MPI::Error("CUDA ILU factorization uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const DeviceLDU A{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, + gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; + + const auto blockSize = static_cast(nVar * nVar); + const auto shared = 2 * blockSize * sizeof(ScalarType); + + /*--- The legacy default stream cannot be captured, so the graph lives on its own stream, + * created once. Every launch below is followed by a sync back to the host, so this does not + * change execution order relative to the rest of the (single-stream) solver. ---*/ + if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); + + /*--- The launch sequence (init + one kernel per level) is identical on every call: the grid + * and block sizes only depend on the (fixed) sparsity pattern and the device pointers are + * fixed members, allocated once. Capture it into a CUDA graph the first time and replay that + * from then on, which removes the per-level host-side launch overhead without touching the + * parallelization of any individual kernel (unlike a persistent cooperative-groups kernel, + * this does not cap per-level parallelism to an occupancy-resident block count). ---*/ + if (ilu_build_graph_exec == nullptr) { + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); + + IluInitKernel + <<(nPointDomain), blockSize, 0, ilu_stream>>>(nPointDomain, nVar, A, M); + + for (auto level = 0ul; level + 1 < ilu_level_ptr.size(); ++level) { + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluFactorLevelKernel + <<>>(d_ilu_level_idx, begin, size, nPointDomain, nVar, M); + } + + gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); + gpuErrChk(cudaGraphInstantiate(&ilu_build_graph_exec, graph, nullptr, nullptr, 0)); + gpuErrChk(cudaGraphDestroy(graph)); + } + + gpuErrChk(cudaGraphLaunch(ilu_build_graph_exec, ilu_stream)); + gpuErrChk(cudaStreamSynchronize(ilu_stream)); + gpuErrChk(cudaPeekAtLastError()); +} + +template +void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, + CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used before BuildILUPreconditionerGPU.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; + + auto* d_vec = vec.GetDevicePointer(); + auto* d_prod = prod.GetDevicePointer(); + + const auto nLevels = ilu_level_ptr.size() - 1; + + /*--- One thread per block entry, like the factorization kernel: spreads each row's neighbor + * dot products over nVar*nVar threads instead of doing them serially in nVar threads, without + * changing the number of blocks (still one per row), so this does not trade away SM coverage + * the way batching several rows into a block did. ---*/ + const auto threads = static_cast(nVar * nVar); + const auto sharedForward = threads * sizeof(ScalarType); + const auto sharedBackward = (threads + nVar) * sizeof(ScalarType); + + if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); + + /*--- Same idea as BuildILUPreconditionerGPU: the launch sequence only depends on the (fixed) + * level structure, plus the vec/prod device pointers. Those normally are the same temporary + * buffers on every call (owned by CSysSolve / CSysVector, allocated once), so the graph is + * captured once and replayed; if the pointers ever do change the graph is recaptured, which + * is no worse than the un-graphed loop, just not free. ---*/ + if (ilu_apply_graph_exec == nullptr || ilu_apply_graph_vec != d_vec || ilu_apply_graph_prod != d_prod) { + if (ilu_apply_graph_exec != nullptr) { + gpuErrChk(cudaGraphExecDestroy(ilu_apply_graph_exec)); + ilu_apply_graph_exec = nullptr; + } + + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); + + /*--- Forward substitution, levels in increasing order. ---*/ + for (auto level = 0ul; level < nLevels; ++level) { + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluForwardLevelKernel + <<>>(d_ilu_level_idx, begin, size, nVar, M, d_vec, d_prod); + } + + /*--- Backward substitution, levels in decreasing order. ---*/ + for (auto level = nLevels; level > 0;) { + --level; // unsigned type + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluBackwardLevelKernel<<>>( + d_ilu_level_idx, begin, size, nPointDomain, nVar, M, d_prod); + } + + gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); + gpuErrChk(cudaGraphInstantiate(&ilu_apply_graph_exec, graph, nullptr, nullptr, 0)); + gpuErrChk(cudaGraphDestroy(graph)); + ilu_apply_graph_vec = d_vec; + ilu_apply_graph_prod = d_prod; + } + + gpuErrChk(cudaGraphLaunch(ilu_apply_graph_exec, ilu_stream)); + gpuErrChk(cudaStreamSynchronize(ilu_stream)); + gpuErrChk(cudaPeekAtLastError()); +} + +template void CSysMatrix::BuildJacobiPreconditionerGPU(); +template void CSysMatrix::BuildILUPreconditionerGPU(); +template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, + CSysVector& prod) const; + template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; #if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) +template void CSysMatrix::BuildJacobiPreconditionerGPU(); +template void CSysMatrix::BuildILUPreconditionerGPU(); +template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, + CSysVector& prod) const; + template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, From 5ae0e4249d8954bb280760ca765c42946cf958ec Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 09:26:34 -0700 Subject: [PATCH 02/14] dead code --- Common/include/linear_algebra/CSysMatrix.hpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index 7afdfe8d3e2..9b26ff5cab3 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -1114,22 +1114,6 @@ class CSysMatrix { void GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Performs Gaussian Elimination between diagional blocks of the matrix and the prod vector - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUGaussElimination(ScalarType& prod, CGeometry* geometry, const CConfig* config) const; - - /*! - * \brief Multiply CSysVector by the preconditioner all of which are stored on the device - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ - void GPUComputeLU_SGSPreconditioner(ScalarType& vec, ScalarType& prod, CGeometry* geometry, - const CConfig* config) const; - /*! * \brief Build the Jacobi preconditioner. */ From cd5e25f126f36ebc6993ff99951b5a8a64ba5475 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 09:40:04 -0700 Subject: [PATCH 03/14] cleanup --- .../include/linear_algebra/CMatrixInverse.hpp | 99 +++++++++++++++++++ .../linear_algebra/CMatrixVectorProduct.hpp | 24 +---- Common/include/linear_algebra/CSysMatrix.hpp | 18 ++-- Common/src/linear_algebra/CSysMatrix.cpp | 62 +++++------- Common/src/linear_algebra/CSysMatrixGPU.cu | 6 +- .../linear_algebra/CSysPreconditionerGPU.cu | 50 +--------- 6 files changed, 140 insertions(+), 119 deletions(-) create mode 100644 Common/include/linear_algebra/CMatrixInverse.hpp diff --git a/Common/include/linear_algebra/CMatrixInverse.hpp b/Common/include/linear_algebra/CMatrixInverse.hpp new file mode 100644 index 00000000000..f3d64630349 --- /dev/null +++ b/Common/include/linear_algebra/CMatrixInverse.hpp @@ -0,0 +1,99 @@ +/*! + * \file CMatrixInverse.hpp + * \brief Dense small-matrix inversion via Gauss-Jordan elimination, shared between the host + * (CSysMatrix::MatrixInverse) and device (CSysPreconditionerGPU.cu) implementations. + * \author F. Palacios, A. Bueno, T. Economon, P. Gomes + * \version 8.5.0 "Harrier" + * + * SU2 Project Website: https://su2code.github.io + * + * The SU2 Project is maintained by the SU2 Foundation + * (http://su2foundation.org) + * + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) + * + * SU2 is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * SU2 is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with SU2. If not, see . + */ + +#pragma once + +#include + +#ifdef __CUDACC__ +#define SU2_CUDA_HOST_DEVICE __host__ __device__ +#else +#define SU2_CUDA_HOST_DEVICE +#endif + +namespace SU2_LinAlg { + +/*! + * \brief Regularize a pivot that is too small to prevent divide-by-zero, on host and device + * this needs to clamp to the same value so that the two produce the same factors. + */ +template +SU2_CUDA_HOST_DEVICE inline void RegularizePivot(ScalarType& pivot) { + const float eps = 1e-12; +#ifdef __CUDA_ARCH__ + if (fabs(pivot) < eps) pivot = copysign(ScalarType(eps), pivot); +#else + if (std::abs(pivot) < eps) pivot = std::copysign(ScalarType(eps), pivot); +#endif +} + +/*! + * \brief Invert the \p nVar by \p nVar dense matrix \p matrix into \p inverse via Gauss-Jordan + * elimination with partial pivoting on the diagonal. + * \note \p matrix is used as scratch space and destroyed, \p inverse must not alias it. + */ +template +SU2_CUDA_HOST_DEVICE inline void MatrixInverse(unsigned long nVar, ScalarType* matrix, ScalarType* inverse) { +#define A(I, J) matrix[(I)*nVar + (J)] +#define M(I, J) inverse[(I)*nVar + (J)] + + /*--- Initialize the inverse with the identity. ---*/ + for (auto iVar = 0ul; iVar < nVar; iVar++) + for (auto jVar = 0ul; jVar < nVar; jVar++) M(iVar, jVar) = ScalarType(iVar == jVar); + + /*--- Transform system in Upper Matrix. ---*/ + for (auto iVar = 1ul; iVar < nVar; iVar++) { + for (auto jVar = 0ul; jVar < iVar; jVar++) { + RegularizePivot(A(jVar, jVar)); + + const ScalarType weight = A(iVar, jVar) / A(jVar, jVar); + for (auto kVar = jVar; kVar < nVar; kVar++) A(iVar, kVar) -= weight * A(jVar, kVar); + + /*--- At this stage M is lower triangular so not all cols need updating. ---*/ + for (auto kVar = 0ul; kVar <= jVar; kVar++) M(iVar, kVar) -= weight * M(jVar, kVar); + } + } + + /*--- Backwards substitution. ---*/ + for (auto iVar = nVar; iVar > 0ul;) { + iVar--; // unsigned type + for (auto jVar = iVar + 1; jVar < nVar; jVar++) + for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) -= A(iVar, jVar) * M(jVar, kVar); + + RegularizePivot(A(iVar, iVar)); + + for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) /= A(iVar, iVar); + } + +#undef A +#undef M +} + +} // namespace SU2_LinAlg + +#undef SU2_CUDA_HOST_DEVICE diff --git a/Common/include/linear_algebra/CMatrixVectorProduct.hpp b/Common/include/linear_algebra/CMatrixVectorProduct.hpp index 4069ff2fd00..52614a45770 100644 --- a/Common/include/linear_algebra/CMatrixVectorProduct.hpp +++ b/Common/include/linear_algebra/CMatrixVectorProduct.hpp @@ -105,28 +105,6 @@ class CSysMatrixVectorProduct final : public CMatrixVectorProduct { * \param[out] v - CSysVector that is the result of the product */ inline void operator()(const CSysVector& u, CSysVector& v) const override { - if (config->GetCUDA()) { -#ifdef SU2_ENABLE_CUDA_KERNELS - if constexpr (su2_gpu_capable_v) { - BEGIN_SU2_DEVICE_REGION - matrix.GPUMatrixVectorProduct(u, v, geometry, config); - END_SU2_DEVICE_REGION - } else { - SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); - } -#elif defined(HAVE_CUDA) - SU2_MPI::Error( - "\nError in launching Matrix-Vector Product Function\nENABLE_CUDA is set to YES\nThe GPU kernels are not " - "part of the AD libraries, use the primal build for GPU acceleration", - CURRENT_FUNCTION); -#else - SU2_MPI::Error( - "\nError in launching Matrix-Vector Product Function\nENABLE_CUDA is set to YES\nPlease compile with CUDA " - "options enabled in Meson to access GPU Functions", - CURRENT_FUNCTION); -#endif - } else { - matrix.MatrixVectorProduct(u, v, geometry, config); - } + matrix.MatrixVectorProduct(u, v, geometry, config); } }; diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index 9b26ff5cab3..cfef21fb75a 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -542,7 +542,13 @@ class CSysMatrix { * ScalarType buffer and delegates to the scalar GaussElimination overload. */ inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; - /*--- Hooks for GPU versions of the preconditioners (implemented is in CSysMatrixGPU.cu). ---*/ + /*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ + + /*! + * \brief Performs the product of a sparse matrix by a CSysVector on the device. + */ + void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; /*! * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. @@ -1104,16 +1110,6 @@ class CSysMatrix { void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Performs the product of a sparse matrix by a CSysVector. - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; - /*! * \brief Build the Jacobi preconditioner. */ diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index b931ccb6d55..3ec66f031b6 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -28,6 +28,7 @@ #include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/geometry/CGeometry.hpp" +#include "../../include/linear_algebra/CMatrixInverse.hpp" #include "../../include/toolboxes/allocation_toolbox.hpp" #include @@ -744,15 +745,16 @@ void CSysMatrix::MatrixInverse(ScalarType* matrix, ScalarType* inver assert((matrix != inverse) && "Output cannot be the same as the input."); + /*--- Inversion ---*/ +#ifdef USE_MKL_LAPACK + // With MKL_DIRECT_CALL enabled, this is significantly faster than native code on Intel Architectures. #define M(I, J) inverse[(I)*nVar + (J)] - /*--- Initialize the inverse with the identity. ---*/ + /*--- Initialize the inverse with the identity, LAPACKE_?getrs solves for it as the rhs. ---*/ for (auto iVar = 0ul; iVar < nVar; iVar++) for (auto jVar = 0ul; jVar < nVar; jVar++) M(iVar, jVar) = ScalarType(iVar == jVar); +#undef M - /*--- Inversion ---*/ -#ifdef USE_MKL_LAPACK - // With MKL_DIRECT_CALL enabled, this is significantly faster than native code on Intel Architectures. lapack_int ipiv[MAXNVAR]; if constexpr (std::is_same_v) { LAPACKE_dgetrf(LAPACK_ROW_MAJOR, nVar, nVar, matrix, nVar, ipiv); @@ -763,38 +765,9 @@ void CSysMatrix::MatrixInverse(ScalarType* matrix, ScalarType* inver LAPACKE_sgetrs(LAPACK_ROW_MAJOR, 'N', nVar, nVar, matrix, nVar, ipiv, inverse, nVar); } #else -#define A(I, J) matrix[(I)*nVar + (J)] - - /*--- Transform system in Upper Matrix ---*/ - for (auto iVar = 1ul; iVar < nVar; iVar++) { - for (auto jVar = 0ul; jVar < iVar; jVar++) { - /*--- Regularize pivot if too small to prevent divide-by-zero ---*/ - RegularizePivot(A(jVar, jVar), jVar, jVar, "MatrixInverse"); - - ScalarType weight = A(iVar, jVar) / A(jVar, jVar); - for (auto kVar = jVar; kVar < nVar; kVar++) A(iVar, kVar) -= weight * A(jVar, kVar); - - /*--- at this stage M is lower triangular so not all cols need updating ---*/ - for (auto kVar = 0ul; kVar <= jVar; kVar++) M(iVar, kVar) -= weight * M(jVar, kVar); - } - } - - /*--- Backwards substitution ---*/ - for (auto iVar = nVar; iVar > 0ul;) { - iVar--; // unsigned type - for (auto jVar = iVar + 1; jVar < nVar; jVar++) - for (auto kVar = 0ul; kVar < nVar; kVar++) M(iVar, kVar) -= A(iVar, jVar) * M(jVar, kVar); - - /*--- Regularize diagonal if too small ---*/ - RegularizePivot(A(iVar, iVar), iVar, iVar, "DEBUG MatrixInverse backsubst"); - - for (auto kVar = 0ul; kVar < nVar; kVar++) { - M(iVar, kVar) /= A(iVar, iVar); - } - } -#undef A + /*--- Shared with the device implementation, see CMatrixInverse.hpp. ---*/ + SU2_LinAlg::MatrixInverse(nVar, matrix, inverse); #endif -#undef M } template @@ -827,6 +800,25 @@ template void CSysMatrix::MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { SU2_ZONE_SCOPED + + if (useCuda) { +#ifdef SU2_ENABLE_CUDA_KERNELS + if constexpr (su2_gpu_capable_v) { + BEGIN_SU2_DEVICE_REGION + MatrixVectorProductGPU(vec, prod, geometry, config); + END_SU2_DEVICE_REGION + return; + } else { + SU2_MPI::Error("GPU acceleration is not supported for AD scalar types.", CURRENT_FUNCTION); + } +#else + SU2_MPI::Error( + "\nError in launching Matrix-Vector Product Function\nENABLE_CUDA is set to YES\nPlease compile with CUDA " + "options enabled in Meson to access GPU Functions", + CURRENT_FUNCTION); +#endif + } + /*--- Some checks for consistency between CSysMatrix and the CSysVectors ---*/ #ifndef NDEBUG if ((nEqn != vec.GetNVar()) || (nVar != prod.GetNVar())) { diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index 381379a0d81..03f1b98272e 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -76,7 +76,7 @@ void CSysMatrix::HtDTransfer(bool trigger) const { } template -void CSysMatrix::GPUMatrixVectorProduct(const CSysVector& vec, CSysVector& prod, +void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { if (nVar != nEqn) { SU2_MPI::Error("CUDA CSysMatrix block-LDU SpMV requires square blocks.", CURRENT_FUNCTION); @@ -93,13 +93,13 @@ void CSysMatrix::GPUMatrixVectorProduct(const CSysVector gpuErrChk(cudaGetLastError()); } template void CSysMatrix::HtDTransfer(bool trigger) const; -template void CSysMatrix::GPUMatrixVectorProduct(const CSysVector& vec, +template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; #if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) template void CSysMatrix::HtDTransfer(bool trigger) const; -template void CSysMatrix::GPUMatrixVectorProduct(const CSysVector& vec, +template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const; #endif diff --git a/Common/src/linear_algebra/CSysPreconditionerGPU.cu b/Common/src/linear_algebra/CSysPreconditionerGPU.cu index 9d24a4cfa26..61fa7c2f341 100644 --- a/Common/src/linear_algebra/CSysPreconditionerGPU.cu +++ b/Common/src/linear_algebra/CSysPreconditionerGPU.cu @@ -27,6 +27,7 @@ #include +#include "../../include/linear_algebra/CMatrixInverse.hpp" #include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/linear_algebra/GPUComms.cuh" @@ -92,51 +93,6 @@ __device__ FORCEINLINE ScalarType* GetBlockILU(const DeviceLDU& M, u return nullptr; } -/*! - * \brief Device version of the pivot regularization used by the host factorization, - * it has to match to give the same factors. - */ -template -__device__ FORCEINLINE void RegularizePivotDevice(ScalarType& pivot) { - const float eps = 1e-12; - if (fabs(pivot) < eps) pivot = copysign(ScalarType(eps), pivot); -} - -/*! - * \brief Invert a small dense matrix, \p A is the (destroyed) input and \p M the inverse. - * \note Serial port of CSysMatrix::MatrixInverse, run by one thread of the block. - */ -template -__device__ void MatrixInverseDevice(unsigned long nVar, ScalarType* A, ScalarType* M) { - for (auto iVar = 0ul; iVar < nVar; ++iVar) - for (auto jVar = 0ul; jVar < nVar; ++jVar) M[iVar * nVar + jVar] = ScalarType(iVar == jVar); - - /*--- Transform system in Upper Matrix. ---*/ - for (auto iVar = 1ul; iVar < nVar; ++iVar) { - for (auto jVar = 0ul; jVar < iVar; ++jVar) { - RegularizePivotDevice(A[jVar * nVar + jVar]); - - const ScalarType weight = A[iVar * nVar + jVar] / A[jVar * nVar + jVar]; - - for (auto kVar = jVar; kVar < nVar; ++kVar) A[iVar * nVar + kVar] -= weight * A[jVar * nVar + kVar]; - - /*--- At this stage M is lower triangular so not all cols need updating. ---*/ - for (auto kVar = 0ul; kVar <= jVar; ++kVar) M[iVar * nVar + kVar] -= weight * M[jVar * nVar + kVar]; - } - } - - /*--- Backwards substitution. ---*/ - for (auto iVar = nVar; iVar > 0ul;) { - --iVar; // unsigned type - for (auto jVar = iVar + 1; jVar < nVar; ++jVar) - for (auto kVar = 0ul; kVar < nVar; ++kVar) M[iVar * nVar + kVar] -= A[iVar * nVar + jVar] * M[jVar * nVar + kVar]; - - RegularizePivotDevice(A[iVar * nVar + iVar]); - - for (auto kVar = 0ul; kVar < nVar; ++kVar) M[iVar * nVar + kVar] /= A[iVar * nVar + iVar]; - } -} - /*! * \brief Invert the diagonal blocks of the matrix, device version of InverseDiagonalBlock. * \note Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry, they @@ -158,7 +114,7 @@ __global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nV work[tid] = mat_d[iRow * blockSize + tid]; __syncthreads(); - if (tid == 0) MatrixInverseDevice(nVar, work, invM + iRow * blockSize); + if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, invM + iRow * blockSize); } /*! @@ -266,7 +222,7 @@ __global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsi __syncthreads(); work[tid] = M.d[iRow * blockSize + tid]; __syncthreads(); - if (tid == 0) MatrixInverseDevice(nVar, work, M.d + iRow * blockSize); + if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, M.d + iRow * blockSize); } /*! From 90c77a59d2332d63af8e2fa0071aaa2c28f813d1 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 10:34:12 -0700 Subject: [PATCH 04/14] cleanup --- .../linear_algebra/CSysPreconditionerGPU.cu | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/Common/src/linear_algebra/CSysPreconditionerGPU.cu b/Common/src/linear_algebra/CSysPreconditionerGPU.cu index 61fa7c2f341..0bef6a2df26 100644 --- a/Common/src/linear_algebra/CSysPreconditionerGPU.cu +++ b/Common/src/linear_algebra/CSysPreconditionerGPU.cu @@ -497,24 +497,17 @@ void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector::BuildJacobiPreconditionerGPU(); -template void CSysMatrix::BuildILUPreconditionerGPU(); -template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, - CSysVector& prod) const; - -template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, - CSysVector& prod, - CGeometry* geometry, - const CConfig* config) const; +#define INSTANTIATE_MATRIX(TYPE) \ +template void CSysMatrix::BuildJacobiPreconditionerGPU(); \ +template void CSysMatrix::BuildILUPreconditionerGPU(); \ +template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod) const; \ +template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod, \ + CGeometry* geometry, \ + const CConfig* config) const; +INSTANTIATE_MATRIX(su2mixedfloat) #if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) -template void CSysMatrix::BuildJacobiPreconditionerGPU(); -template void CSysMatrix::BuildILUPreconditionerGPU(); -template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, - CSysVector& prod) const; - -template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, - CSysVector& prod, - CGeometry* geometry, - const CConfig* config) const; +INSTANTIATE_MATRIX(passivedouble) #endif From 11ad091da971a30d5376c971513d57c7b5886813 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 10:44:19 -0700 Subject: [PATCH 05/14] cleanup --- Common/src/linear_algebra/CSysMatrixGPU.cu | 498 ++++++++++++++++- .../linear_algebra/CSysPreconditionerGPU.cu | 513 ------------------ Common/src/linear_algebra/meson.build | 2 +- 3 files changed, 489 insertions(+), 524 deletions(-) delete mode 100644 Common/src/linear_algebra/CSysPreconditionerGPU.cu diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index 03f1b98272e..f7bc55b780c 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -1,7 +1,7 @@ /*! * \file CSysMatrixGPU.cu * \brief Implementations of Kernels and Functions for Matrix Operations on the GPU - * \author A. Raj + * \author A. Raj, Jesse Li, P. Gomes * \version 8.5.0 "Harrier" * * SU2 Project Website: https://su2code.github.io @@ -25,9 +25,298 @@ * License along with SU2. If not, see . */ -#include "../../include/linear_algebra/CSysMatrix.hpp" +#include + +#include "../../include/linear_algebra/CMatrixInverse.hpp" +#include "../../include/linear_algebra/CSysMatrix.inl" #include "../../include/linear_algebra/GPUComms.cuh" +namespace { + +template +__global__ void ApplyJacobiPreconditionerKernel(const ScalarType* invM, const ScalarType* vec, ScalarType* prod, + unsigned long nPointDomain, unsigned long nVar) { + const auto iPoint = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; + if (iPoint >= nPointDomain) return; + + const auto block = &invM[iPoint * nVar * nVar]; + const auto rhs = &vec[iPoint * nVar]; + auto out = &prod[iPoint * nVar]; + + for (auto iVar = 0ul; iVar < nVar; ++iVar) { + auto sum = ScalarType(0); + for (auto jVar = 0ul; jVar < nVar; ++jVar) { + sum += block[iVar * nVar + jVar] * rhs[jVar]; + } + out[iVar] = sum; + } +} + +/*--- ILU. The factorization and both triangular solves are level scheduled: the rows of a + * level are independent of each other and only depend on rows of previous levels, so each + * level is one kernel launch and the launch boundaries provide the synchronization. The rows + * of a level are scattered through the matrix, hence the indirection through the level table. + * Throughout, one CUDA block works on one row. ---*/ + +/*! + * \brief The pointers of an LDU-partitioned matrix, all in device memory. This mirrors the + * private CSysMatrix::LDU, which the kernels cannot name. + */ +template +struct DeviceLDU { + ScalarType* d; + ScalarType* l; + ScalarType* u; + const su2uint* row_ptr_l; + const su2uint* col_ind_l; + const su2uint* row_ptr_u; + const su2uint* col_ind_u; +}; + +/*! + * \brief Start of block (i,j), or nullptr if it is not a nonzero of the pattern. + */ +template +__device__ FORCEINLINE ScalarType* GetBlockILU(const DeviceLDU& M, unsigned long nVar, + unsigned long block_i, unsigned long block_j) { + const auto blockSize = nVar * nVar; + if (block_i == block_j) return M.d + block_i * blockSize; + + const bool lower = block_j < block_i; + const auto* row_ptr = lower ? M.row_ptr_l : M.row_ptr_u; + const auto* col_ind = lower ? M.col_ind_l : M.col_ind_u; + auto* vals = lower ? M.l : M.u; + + for (auto k = row_ptr[block_i]; k < row_ptr[block_i + 1]; ++k) { + if (col_ind[k] == block_j) return vals + k * blockSize; + } + return nullptr; +} + +/*! + * \brief Invert the diagonal blocks of the matrix, device version of InverseDiagonalBlock. + * \note Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry, they + * stage the input in shared memory). Dynamic shared memory: nVar*nVar scalars. + */ +template +__global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nVar, + const ScalarType* __restrict__ mat_d, ScalarType* __restrict__ invM) { + const unsigned long iRow = blockIdx.x; + if (iRow >= nRows) return; + + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + + /*--- The inversion destroys its input, so it cannot work on the matrix itself. ---*/ + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* work = reinterpret_cast(smem); + + work[tid] = mat_d[iRow * blockSize + tid]; + __syncthreads(); + + if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, invM + iRow * blockSize); +} + +/*! + * \brief Copy the matrix into the storage of the factorization, whose pattern may be larger + * (fill-in), entries that the matrix does not have are set to zero. + * \note Device version of the InitIluRow helper of BuildILUPreconditioner. Rows are + * independent, so this is done for the entire matrix before the factorization starts. + * Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry). + */ +template +__global__ void IluInitKernel(unsigned long nRows, unsigned long nVar, DeviceLDU A, + DeviceLDU M) { + const unsigned long iRow = blockIdx.x; + if (iRow >= nRows) return; + + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + + M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; + + /*--- Merge-scan the row of the matrix onto the row of the factorization, both are sorted + * by column index. Every thread walks the scan for its own entry of the blocks. ---*/ + auto scatter = [&](const su2uint* a_row_ptr, const su2uint* a_col_ind, const ScalarType* a_vals, + const su2uint* m_row_ptr, const su2uint* m_col_ind, ScalarType* m_vals) { + auto ka = a_row_ptr[iRow]; + const auto ka_end = a_row_ptr[iRow + 1]; + + for (auto k = m_row_ptr[iRow]; k < m_row_ptr[iRow + 1]; ++k) { + const auto jPoint = m_col_ind[k]; + while (ka < ka_end && a_col_ind[ka] < jPoint) ++ka; + + if (ka < ka_end && a_col_ind[ka] == jPoint) { + m_vals[k * blockSize + tid] = a_vals[ka * blockSize + tid]; + } else { + m_vals[k * blockSize + tid] = ScalarType(0); + } + } + }; + scatter(A.row_ptr_l, A.col_ind_l, A.l, M.row_ptr_l, M.col_ind_l, M.l); + scatter(A.row_ptr_u, A.col_ind_u, A.u, M.row_ptr_u, M.col_ind_u, M.u); +} + +/*! + * \brief Factorize the rows of one level, device version of the BuildIluRow helper. + * \note Grid: one block per row of the level, blockDim.x == nVar*nVar (one thread per block + * entry, so that the small matrix products are one dot product per thread). + * Dynamic shared memory: 2*nVar*nVar scalars. + */ +template +__global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* Lij = reinterpret_cast(smem); + auto* work = Lij + blockSize; + + /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ + for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { + /*--- All threads must be done with the previous entry: Lij is about to be overwritten, + * and the blocks of this row updated below are read here across threads. ---*/ + __syncthreads(); + + /*--- jPoint is the column index (jPoint < iRow). ---*/ + const unsigned long jPoint = M.col_ind_l[kl]; + + /*--- Multiply the block by the inverse of the corresponding diagonal block. ---*/ + auto* Block_ij = M.l + kl * blockSize; + const auto* invUjj = M.d + jPoint * blockSize; + + ScalarType sum = 0; + for (auto k = 0ul; k < nVar; ++k) sum += Block_ij[iVar * nVar + k] * invUjj[k * nVar + jVar]; + Lij[tid] = sum; + __syncthreads(); + + /*--- Lij holds Aij*inv(Ujj). Jump to the upper part of the jPoint row. ---*/ + for (auto ku = M.row_ptr_u[jPoint]; ku < M.row_ptr_u[jPoint + 1]; ++ku) { + /*--- Get the column index (kPoint > jPoint), halo columns are not factorized. ---*/ + const unsigned long kPoint = M.col_ind_u[ku]; + if (kPoint >= nRows) break; + + /*--- If Aik exists, update it: Aik -= Lij * Ujk ---*/ + auto* Block_ik = GetBlockILU(M, nVar, iRow, kPoint); + if (Block_ik == nullptr) continue; + + /*--- Block_ik cannot alias Block_ij because kPoint > jPoint. ---*/ + const auto* Ujk = M.u + ku * blockSize; + ScalarType prod = 0; + for (auto k = 0ul; k < nVar; ++k) prod += Lij[iVar * nVar + k] * Ujk[k * nVar + jVar]; + Block_ik[tid] -= prod; + } + + /*--- Store Lij in the lower triangular part, each thread only writes its own entry. ---*/ + Block_ij[tid] = Lij[tid]; + } + + /*--- Invert the diagonal entry, Uii, for the next levels. The loop above may have updated + * it (when kPoint == iRow), so the whole block has to be done first. ---*/ + __syncthreads(); + work[tid] = M.d[iRow * blockSize + tid]; + __syncthreads(); + if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, M.d + iRow * blockSize); +} + +/*! + * \brief Forward substitution for the rows of one level, (L+I).prod = vec. + * \note One thread per block entry (like the factorization kernel), so the inner dot product + * over a neighbor block is spread across nVar threads instead of done serially by one. + * Unlike the factorization, forward/backward only ever *read* already-finalized prod + * values (no row-to-row write-through during the loop), so each thread can accumulate + * its own (iVar,jVar) partial product across every neighbor with no synchronization at + * all, and only the final nVar-way reduction (summing over jVar for each iVar) needs one + * __syncthreads() — not one per neighbor. Grid: one block per row of the level, + * blockDim.x == nVar*nVar. Dynamic shared memory: nVar*nVar scalars. + */ +template +__global__ void IluForwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nVar, DeviceLDU M, + const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + + ScalarType acc = 0; + /*--- The columns of L are rows of previous levels, so prod is final for all of them. ---*/ + for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { + const unsigned long jPoint = M.col_ind_l[kl]; + const auto* blk = M.l + kl * nVar * nVar; + acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; + } + partial[tid] = acc; + __syncthreads(); + + if (jVar == 0) { + ScalarType sum = vec[iRow * nVar + iVar]; + for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; + prod[iRow * nVar + iVar] = sum; + } +} + +/*! + * \brief Backward substitution for the rows of one level, U.prod = prod. + * \note Same idea as IluForwardLevelKernel: each thread accumulates its own (iVar,jVar) + * partial product across every neighbor with no synchronization, then one + * __syncthreads() to reduce over jVar and get the elimination result per iVar, then a + * second __syncthreads() before the diagonal multiply (which needs every iVar's result). + * Grid: one block per row of the level, blockDim.x == nVar*nVar. Dynamic shared memory: + * nVar*nVar + nVar scalars. + */ +template +__global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, + unsigned long level_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M, ScalarType* __restrict__ prod) { + if (blockIdx.x >= level_size) return; + + const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const auto blockSize = nVar * nVar; + const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; + + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* partial = reinterpret_cast(smem); + auto* aux = partial + blockSize; + + ScalarType acc = 0; + /*--- The columns of U are rows of later levels, already updated by this sweep. ---*/ + for (auto ku = M.row_ptr_u[iRow]; ku < M.row_ptr_u[iRow + 1]; ++ku) { + const unsigned long jPoint = M.col_ind_u[ku]; + if (jPoint >= nRows) break; + const auto* blk = M.u + ku * blockSize; + acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; + } + partial[tid] = acc; + __syncthreads(); + + if (jVar == 0) { + ScalarType sum = prod[iRow * nVar + iVar]; + for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; + /*--- The diagonal blocks are stored inverted by the factorization. ---*/ + aux[iVar] = sum; + } + __syncthreads(); + + if (jVar == 0) { + const auto* invUii = M.d + iRow * blockSize; + ScalarType out = 0; + for (auto k = 0ul; k < nVar; ++k) out += invUii[iVar * nVar + k] * aux[k]; + prod[iRow * nVar + iVar] = out; + } +} + /*! * \brief Block-LDU SpMV kernel: y[iRow] = (L + D + U) * x per block-row. * One CUDA block per block-row; threadIdx.x indexes output variable (0..nVar-1). @@ -67,6 +356,186 @@ __global__ void BlockLDU_SpMV_kernel(unsigned long nRows, unsigned long nVar, y[iRow * nVar + iVar] = sum; } +} // namespace + +template +void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, + CSysVector& prod, CGeometry* geometry, + const CConfig* config) const { + (void)geometry; + (void)config; + + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA Jacobi preconditioner used before BuildJacobiPreconditionerGPU.", CURRENT_FUNCTION); + } + + constexpr unsigned threadsPerBlock = 128; + const auto blocks = static_cast((nPointDomain + threadsPerBlock - 1) / threadsPerBlock); + ApplyJacobiPreconditionerKernel<<>>(d_invM, vec.GetDevicePointer(), prod.GetDevicePointer(), + nPointDomain, nVar); + gpuErrChk(cudaPeekAtLastError()); +} + +template +void CSysMatrix::BuildJacobiPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (d_invM == nullptr) { + SU2_MPI::Error("CUDA Jacobi preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nVar != nEqn) { + SU2_MPI::Error("CUDA Jacobi preconditioner requires square blocks.", CURRENT_FUNCTION); + } + if (nVar * nVar > 1024) { + SU2_MPI::Error("CUDA Jacobi preconditioner uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const auto blockSize = static_cast(nVar * nVar); + InvertDiagonalBlocksKernel + <<(nPointDomain), blockSize, blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, + d_invM); + gpuErrChk(cudaPeekAtLastError()); +} + +template +void CSysMatrix::BuildILUPreconditionerGPU() { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used without device storage.", CURRENT_FUNCTION); + } + if (nVar != nEqn) { + SU2_MPI::Error("CUDA ILU factorization requires square blocks.", CURRENT_FUNCTION); + } + if (nVar * nVar > 1024) { + SU2_MPI::Error("CUDA ILU factorization uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + /*--- The matrix is expected to be on the device already, it is uploaded once per solve by + * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ + const DeviceLDU A{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, + gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; + + const auto blockSize = static_cast(nVar * nVar); + const auto shared = 2 * blockSize * sizeof(ScalarType); + + /*--- The legacy default stream cannot be captured, so the graph lives on its own stream, + * created once. Every launch below is followed by a sync back to the host, so this does not + * change execution order relative to the rest of the (single-stream) solver. ---*/ + if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); + + /*--- The launch sequence (init + one kernel per level) is identical on every call: the grid + * and block sizes only depend on the (fixed) sparsity pattern and the device pointers are + * fixed members, allocated once. Capture it into a CUDA graph the first time and replay that + * from then on, which removes the per-level host-side launch overhead without touching the + * parallelization of any individual kernel (unlike a persistent cooperative-groups kernel, + * this does not cap per-level parallelism to an occupancy-resident block count). ---*/ + if (ilu_build_graph_exec == nullptr) { + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); + + IluInitKernel + <<(nPointDomain), blockSize, 0, ilu_stream>>>(nPointDomain, nVar, A, M); + + for (auto level = 0ul; level + 1 < ilu_level_ptr.size(); ++level) { + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluFactorLevelKernel + <<>>(d_ilu_level_idx, begin, size, nPointDomain, nVar, M); + } + + gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); + gpuErrChk(cudaGraphInstantiate(&ilu_build_graph_exec, graph, nullptr, nullptr, 0)); + gpuErrChk(cudaGraphDestroy(graph)); + } + + gpuErrChk(cudaGraphLaunch(ilu_build_graph_exec, ilu_stream)); + gpuErrChk(cudaStreamSynchronize(ilu_stream)); + gpuErrChk(cudaPeekAtLastError()); +} + +template +void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, + CSysVector& prod) const { + SU2_ZONE_SCOPED + + if (gpu_ilu.d == nullptr) { + SU2_MPI::Error("CUDA ILU preconditioner used before BuildILUPreconditionerGPU.", CURRENT_FUNCTION); + } + if (nPointDomain == 0) return; + + const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, + gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; + + auto* d_vec = vec.GetDevicePointer(); + auto* d_prod = prod.GetDevicePointer(); + + const auto nLevels = ilu_level_ptr.size() - 1; + + /*--- One thread per block entry, like the factorization kernel: spreads each row's neighbor + * dot products over nVar*nVar threads instead of doing them serially in nVar threads, without + * changing the number of blocks (still one per row), so this does not trade away SM coverage + * the way batching several rows into a block did. ---*/ + const auto threads = static_cast(nVar * nVar); + const auto sharedForward = threads * sizeof(ScalarType); + const auto sharedBackward = (threads + nVar) * sizeof(ScalarType); + + if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); + + /*--- Same idea as BuildILUPreconditionerGPU: the launch sequence only depends on the (fixed) + * level structure, plus the vec/prod device pointers. Those normally are the same temporary + * buffers on every call (owned by CSysSolve / CSysVector, allocated once), so the graph is + * captured once and replayed; if the pointers ever do change the graph is recaptured, which + * is no worse than the un-graphed loop, just not free. ---*/ + if (ilu_apply_graph_exec == nullptr || ilu_apply_graph_vec != d_vec || ilu_apply_graph_prod != d_prod) { + if (ilu_apply_graph_exec != nullptr) { + gpuErrChk(cudaGraphExecDestroy(ilu_apply_graph_exec)); + ilu_apply_graph_exec = nullptr; + } + + cudaGraph_t graph; + gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); + + /*--- Forward substitution, levels in increasing order. ---*/ + for (auto level = 0ul; level < nLevels; ++level) { + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluForwardLevelKernel + <<>>(d_ilu_level_idx, begin, size, nVar, M, d_vec, d_prod); + } + + /*--- Backward substitution, levels in decreasing order. ---*/ + for (auto level = nLevels; level > 0;) { + --level; // unsigned type + const auto begin = ilu_level_ptr[level]; + const auto size = ilu_level_ptr[level + 1] - begin; + if (size == 0) continue; + IluBackwardLevelKernel<<>>( + d_ilu_level_idx, begin, size, nPointDomain, nVar, M, d_prod); + } + + gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); + gpuErrChk(cudaGraphInstantiate(&ilu_apply_graph_exec, graph, nullptr, nullptr, 0)); + gpuErrChk(cudaGraphDestroy(graph)); + ilu_apply_graph_vec = d_vec; + ilu_apply_graph_prod = d_prod; + } + + gpuErrChk(cudaGraphLaunch(ilu_apply_graph_exec, ilu_stream)); + gpuErrChk(cudaStreamSynchronize(ilu_stream)); + gpuErrChk(cudaPeekAtLastError()); +} + template void CSysMatrix::HtDTransfer(bool trigger) const { if (!trigger) return; @@ -92,14 +561,23 @@ void CSysMatrix::MatrixVectorProductGPU(const CSysVector gpu.row_ptr_u, gpu.col_ind_u, gpu.u, d_vec, d_prod); gpuErrChk(cudaGetLastError()); } -template void CSysMatrix::HtDTransfer(bool trigger) const; -template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, - CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + +#define INSTANTIATE_MATRIX(TYPE) \ +template void CSysMatrix::HtDTransfer(bool trigger) const; \ +template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, \ + CSysVector& prod, \ + CGeometry* geometry, \ + const CConfig* config) const; \ +template void CSysMatrix::BuildJacobiPreconditionerGPU(); \ +template void CSysMatrix::BuildILUPreconditionerGPU(); \ +template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod) const; \ +template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, \ + CSysVector& prod, \ + CGeometry* geometry, \ + const CConfig* config) const; +INSTANTIATE_MATRIX(su2mixedfloat) #if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) -template void CSysMatrix::HtDTransfer(bool trigger) const; -template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, - CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +INSTANTIATE_MATRIX(passivedouble) #endif diff --git a/Common/src/linear_algebra/CSysPreconditionerGPU.cu b/Common/src/linear_algebra/CSysPreconditionerGPU.cu deleted file mode 100644 index 0bef6a2df26..00000000000 --- a/Common/src/linear_algebra/CSysPreconditionerGPU.cu +++ /dev/null @@ -1,513 +0,0 @@ -/*! - * \file CSysPreconditionerGPU.cu - * \brief CUDA/GPU skeleton implementations for matrix-based preconditioners. - * \author Jesse Li - * \version 8.5.0 "Harrier" - * - * SU2 Project Website: https://su2code.github.io - * - * The SU2 Project is maintained by the SU2 Foundation - * (http://su2foundation.org) - * - * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) - * - * SU2 is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * SU2 is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with SU2. If not, see . - */ - -#include - -#include "../../include/linear_algebra/CMatrixInverse.hpp" -#include "../../include/linear_algebra/CSysMatrix.inl" -#include "../../include/linear_algebra/GPUComms.cuh" - -namespace { - -template -__global__ void ApplyJacobiPreconditionerKernel(const ScalarType* invM, const ScalarType* vec, ScalarType* prod, - unsigned long nPointDomain, unsigned long nVar) { - const auto iPoint = static_cast(blockIdx.x) * blockDim.x + threadIdx.x; - if (iPoint >= nPointDomain) return; - - const auto block = &invM[iPoint * nVar * nVar]; - const auto rhs = &vec[iPoint * nVar]; - auto out = &prod[iPoint * nVar]; - - for (auto iVar = 0ul; iVar < nVar; ++iVar) { - auto sum = ScalarType(0); - for (auto jVar = 0ul; jVar < nVar; ++jVar) { - sum += block[iVar * nVar + jVar] * rhs[jVar]; - } - out[iVar] = sum; - } -} - -/*--- ILU. The factorization and both triangular solves are level scheduled: the rows of a - * level are independent of each other and only depend on rows of previous levels, so each - * level is one kernel launch and the launch boundaries provide the synchronization. The rows - * of a level are scattered through the matrix, hence the indirection through the level table. - * Throughout, one CUDA block works on one row. ---*/ - -/*! - * \brief The pointers of an LDU-partitioned matrix, all in device memory. This mirrors the - * private CSysMatrix::LDU, which the kernels cannot name. - */ -template -struct DeviceLDU { - ScalarType* d; - ScalarType* l; - ScalarType* u; - const su2uint* row_ptr_l; - const su2uint* col_ind_l; - const su2uint* row_ptr_u; - const su2uint* col_ind_u; -}; - -/*! - * \brief Start of block (i,j), or nullptr if it is not a nonzero of the pattern. - */ -template -__device__ FORCEINLINE ScalarType* GetBlockILU(const DeviceLDU& M, unsigned long nVar, - unsigned long block_i, unsigned long block_j) { - const auto blockSize = nVar * nVar; - if (block_i == block_j) return M.d + block_i * blockSize; - - const bool lower = block_j < block_i; - const auto* row_ptr = lower ? M.row_ptr_l : M.row_ptr_u; - const auto* col_ind = lower ? M.col_ind_l : M.col_ind_u; - auto* vals = lower ? M.l : M.u; - - for (auto k = row_ptr[block_i]; k < row_ptr[block_i + 1]; ++k) { - if (col_ind[k] == block_j) return vals + k * blockSize; - } - return nullptr; -} - -/*! - * \brief Invert the diagonal blocks of the matrix, device version of InverseDiagonalBlock. - * \note Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry, they - * stage the input in shared memory). Dynamic shared memory: nVar*nVar scalars. - */ -template -__global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nVar, - const ScalarType* __restrict__ mat_d, ScalarType* __restrict__ invM) { - const unsigned long iRow = blockIdx.x; - if (iRow >= nRows) return; - - const auto blockSize = nVar * nVar; - const unsigned long tid = threadIdx.x; - - /*--- The inversion destroys its input, so it cannot work on the matrix itself. ---*/ - extern __shared__ __align__(sizeof(double)) char smem[]; - auto* work = reinterpret_cast(smem); - - work[tid] = mat_d[iRow * blockSize + tid]; - __syncthreads(); - - if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, invM + iRow * blockSize); -} - -/*! - * \brief Copy the matrix into the storage of the factorization, whose pattern may be larger - * (fill-in), entries that the matrix does not have are set to zero. - * \note Device version of the InitIluRow helper of BuildILUPreconditioner. Rows are - * independent, so this is done for the entire matrix before the factorization starts. - * Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry). - */ -template -__global__ void IluInitKernel(unsigned long nRows, unsigned long nVar, DeviceLDU A, - DeviceLDU M) { - const unsigned long iRow = blockIdx.x; - if (iRow >= nRows) return; - - const auto blockSize = nVar * nVar; - const unsigned long tid = threadIdx.x; - - M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; - - /*--- Merge-scan the row of the matrix onto the row of the factorization, both are sorted - * by column index. Every thread walks the scan for its own entry of the blocks. ---*/ - auto scatter = [&](const su2uint* a_row_ptr, const su2uint* a_col_ind, const ScalarType* a_vals, - const su2uint* m_row_ptr, const su2uint* m_col_ind, ScalarType* m_vals) { - auto ka = a_row_ptr[iRow]; - const auto ka_end = a_row_ptr[iRow + 1]; - - for (auto k = m_row_ptr[iRow]; k < m_row_ptr[iRow + 1]; ++k) { - const auto jPoint = m_col_ind[k]; - while (ka < ka_end && a_col_ind[ka] < jPoint) ++ka; - - if (ka < ka_end && a_col_ind[ka] == jPoint) { - m_vals[k * blockSize + tid] = a_vals[ka * blockSize + tid]; - } else { - m_vals[k * blockSize + tid] = ScalarType(0); - } - } - }; - scatter(A.row_ptr_l, A.col_ind_l, A.l, M.row_ptr_l, M.col_ind_l, M.l); - scatter(A.row_ptr_u, A.col_ind_u, A.u, M.row_ptr_u, M.col_ind_u, M.u); -} - -/*! - * \brief Factorize the rows of one level, device version of the BuildIluRow helper. - * \note Grid: one block per row of the level, blockDim.x == nVar*nVar (one thread per block - * entry, so that the small matrix products are one dot product per thread). - * Dynamic shared memory: 2*nVar*nVar scalars. - */ -template -__global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nRows, unsigned long nVar, - DeviceLDU M) { - if (blockIdx.x >= level_size) return; - - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; - const auto blockSize = nVar * nVar; - const unsigned long tid = threadIdx.x; - const auto iVar = tid / nVar, jVar = tid % nVar; - - extern __shared__ __align__(sizeof(double)) char smem[]; - auto* Lij = reinterpret_cast(smem); - auto* work = Lij + blockSize; - - /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ - for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { - /*--- All threads must be done with the previous entry: Lij is about to be overwritten, - * and the blocks of this row updated below are read here across threads. ---*/ - __syncthreads(); - - /*--- jPoint is the column index (jPoint < iRow). ---*/ - const unsigned long jPoint = M.col_ind_l[kl]; - - /*--- Multiply the block by the inverse of the corresponding diagonal block. ---*/ - auto* Block_ij = M.l + kl * blockSize; - const auto* invUjj = M.d + jPoint * blockSize; - - ScalarType sum = 0; - for (auto k = 0ul; k < nVar; ++k) sum += Block_ij[iVar * nVar + k] * invUjj[k * nVar + jVar]; - Lij[tid] = sum; - __syncthreads(); - - /*--- Lij holds Aij*inv(Ujj). Jump to the upper part of the jPoint row. ---*/ - for (auto ku = M.row_ptr_u[jPoint]; ku < M.row_ptr_u[jPoint + 1]; ++ku) { - /*--- Get the column index (kPoint > jPoint), halo columns are not factorized. ---*/ - const unsigned long kPoint = M.col_ind_u[ku]; - if (kPoint >= nRows) break; - - /*--- If Aik exists, update it: Aik -= Lij * Ujk ---*/ - auto* Block_ik = GetBlockILU(M, nVar, iRow, kPoint); - if (Block_ik == nullptr) continue; - - /*--- Block_ik cannot alias Block_ij because kPoint > jPoint. ---*/ - const auto* Ujk = M.u + ku * blockSize; - ScalarType prod = 0; - for (auto k = 0ul; k < nVar; ++k) prod += Lij[iVar * nVar + k] * Ujk[k * nVar + jVar]; - Block_ik[tid] -= prod; - } - - /*--- Store Lij in the lower triangular part, each thread only writes its own entry. ---*/ - Block_ij[tid] = Lij[tid]; - } - - /*--- Invert the diagonal entry, Uii, for the next levels. The loop above may have updated - * it (when kPoint == iRow), so the whole block has to be done first. ---*/ - __syncthreads(); - work[tid] = M.d[iRow * blockSize + tid]; - __syncthreads(); - if (tid == 0) SU2_LinAlg::MatrixInverse(nVar, work, M.d + iRow * blockSize); -} - -/*! - * \brief Forward substitution for the rows of one level, (L+I).prod = vec. - * \note One thread per block entry (like the factorization kernel), so the inner dot product - * over a neighbor block is spread across nVar threads instead of done serially by one. - * Unlike the factorization, forward/backward only ever *read* already-finalized prod - * values (no row-to-row write-through during the loop), so each thread can accumulate - * its own (iVar,jVar) partial product across every neighbor with no synchronization at - * all, and only the final nVar-way reduction (summing over jVar for each iVar) needs one - * __syncthreads() — not one per neighbor. Grid: one block per row of the level, - * blockDim.x == nVar*nVar. Dynamic shared memory: nVar*nVar scalars. - */ -template -__global__ void IluForwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nVar, DeviceLDU M, - const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod) { - if (blockIdx.x >= level_size) return; - - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; - const unsigned long tid = threadIdx.x; - const auto iVar = tid / nVar, jVar = tid % nVar; - - extern __shared__ __align__(sizeof(double)) char smem[]; - auto* partial = reinterpret_cast(smem); - - ScalarType acc = 0; - /*--- The columns of L are rows of previous levels, so prod is final for all of them. ---*/ - for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { - const unsigned long jPoint = M.col_ind_l[kl]; - const auto* blk = M.l + kl * nVar * nVar; - acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; - } - partial[tid] = acc; - __syncthreads(); - - if (jVar == 0) { - ScalarType sum = vec[iRow * nVar + iVar]; - for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; - prod[iRow * nVar + iVar] = sum; - } -} - -/*! - * \brief Backward substitution for the rows of one level, U.prod = prod. - * \note Same idea as IluForwardLevelKernel: each thread accumulates its own (iVar,jVar) - * partial product across every neighbor with no synchronization, then one - * __syncthreads() to reduce over jVar and get the elimination result per iVar, then a - * second __syncthreads() before the diagonal multiply (which needs every iVar's result). - * Grid: one block per row of the level, blockDim.x == nVar*nVar. Dynamic shared memory: - * nVar*nVar + nVar scalars. - */ -template -__global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nRows, unsigned long nVar, - DeviceLDU M, ScalarType* __restrict__ prod) { - if (blockIdx.x >= level_size) return; - - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; - const auto blockSize = nVar * nVar; - const unsigned long tid = threadIdx.x; - const auto iVar = tid / nVar, jVar = tid % nVar; - - extern __shared__ __align__(sizeof(double)) char smem[]; - auto* partial = reinterpret_cast(smem); - auto* aux = partial + blockSize; - - ScalarType acc = 0; - /*--- The columns of U are rows of later levels, already updated by this sweep. ---*/ - for (auto ku = M.row_ptr_u[iRow]; ku < M.row_ptr_u[iRow + 1]; ++ku) { - const unsigned long jPoint = M.col_ind_u[ku]; - if (jPoint >= nRows) break; - const auto* blk = M.u + ku * blockSize; - acc += blk[iVar * nVar + jVar] * prod[jPoint * nVar + jVar]; - } - partial[tid] = acc; - __syncthreads(); - - if (jVar == 0) { - ScalarType sum = prod[iRow * nVar + iVar]; - for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; - /*--- The diagonal blocks are stored inverted by the factorization. ---*/ - aux[iVar] = sum; - } - __syncthreads(); - - if (jVar == 0) { - const auto* invUii = M.d + iRow * blockSize; - ScalarType out = 0; - for (auto k = 0ul; k < nVar; ++k) out += invUii[iVar * nVar + k] * aux[k]; - prod[iRow * nVar + iVar] = out; - } -} - -} // namespace - -template -void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, - CSysVector& prod, CGeometry* geometry, - const CConfig* config) const { - (void)geometry; - (void)config; - - SU2_ZONE_SCOPED - - if (d_invM == nullptr) { - SU2_MPI::Error("CUDA Jacobi preconditioner used before BuildJacobiPreconditionerGPU.", CURRENT_FUNCTION); - } - - constexpr unsigned threadsPerBlock = 128; - const auto blocks = static_cast((nPointDomain + threadsPerBlock - 1) / threadsPerBlock); - ApplyJacobiPreconditionerKernel<<>>(d_invM, vec.GetDevicePointer(), prod.GetDevicePointer(), - nPointDomain, nVar); - gpuErrChk(cudaPeekAtLastError()); -} - -template -void CSysMatrix::BuildJacobiPreconditionerGPU() { - SU2_ZONE_SCOPED - - if (d_invM == nullptr) { - SU2_MPI::Error("CUDA Jacobi preconditioner used without device storage.", CURRENT_FUNCTION); - } - if (nVar != nEqn) { - SU2_MPI::Error("CUDA Jacobi preconditioner requires square blocks.", CURRENT_FUNCTION); - } - if (nVar * nVar > 1024) { - SU2_MPI::Error("CUDA Jacobi preconditioner uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); - } - if (nPointDomain == 0) return; - - /*--- The matrix is expected to be on the device already, it is uploaded once per solve by - * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ - const auto blockSize = static_cast(nVar * nVar); - InvertDiagonalBlocksKernel - <<(nPointDomain), blockSize, blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, - d_invM); - gpuErrChk(cudaPeekAtLastError()); -} - -template -void CSysMatrix::BuildILUPreconditionerGPU() { - SU2_ZONE_SCOPED - - if (gpu_ilu.d == nullptr) { - SU2_MPI::Error("CUDA ILU preconditioner used without device storage.", CURRENT_FUNCTION); - } - if (nVar != nEqn) { - SU2_MPI::Error("CUDA ILU factorization requires square blocks.", CURRENT_FUNCTION); - } - if (nVar * nVar > 1024) { - SU2_MPI::Error("CUDA ILU factorization uses one thread per block entry, nVar is too large.", CURRENT_FUNCTION); - } - if (nPointDomain == 0) return; - - /*--- The matrix is expected to be on the device already, it is uploaded once per solve by - * CSysMatrixVectorProduct, which is created before the preconditioner is built. ---*/ - const DeviceLDU A{gpu.d, gpu.l, gpu.u, gpu.row_ptr_l, - gpu.col_ind_l, gpu.row_ptr_u, gpu.col_ind_u}; - const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, - gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; - - const auto blockSize = static_cast(nVar * nVar); - const auto shared = 2 * blockSize * sizeof(ScalarType); - - /*--- The legacy default stream cannot be captured, so the graph lives on its own stream, - * created once. Every launch below is followed by a sync back to the host, so this does not - * change execution order relative to the rest of the (single-stream) solver. ---*/ - if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); - - /*--- The launch sequence (init + one kernel per level) is identical on every call: the grid - * and block sizes only depend on the (fixed) sparsity pattern and the device pointers are - * fixed members, allocated once. Capture it into a CUDA graph the first time and replay that - * from then on, which removes the per-level host-side launch overhead without touching the - * parallelization of any individual kernel (unlike a persistent cooperative-groups kernel, - * this does not cap per-level parallelism to an occupancy-resident block count). ---*/ - if (ilu_build_graph_exec == nullptr) { - cudaGraph_t graph; - gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); - - IluInitKernel - <<(nPointDomain), blockSize, 0, ilu_stream>>>(nPointDomain, nVar, A, M); - - for (auto level = 0ul; level + 1 < ilu_level_ptr.size(); ++level) { - const auto begin = ilu_level_ptr[level]; - const auto size = ilu_level_ptr[level + 1] - begin; - if (size == 0) continue; - IluFactorLevelKernel - <<>>(d_ilu_level_idx, begin, size, nPointDomain, nVar, M); - } - - gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); - gpuErrChk(cudaGraphInstantiate(&ilu_build_graph_exec, graph, nullptr, nullptr, 0)); - gpuErrChk(cudaGraphDestroy(graph)); - } - - gpuErrChk(cudaGraphLaunch(ilu_build_graph_exec, ilu_stream)); - gpuErrChk(cudaStreamSynchronize(ilu_stream)); - gpuErrChk(cudaPeekAtLastError()); -} - -template -void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, - CSysVector& prod) const { - SU2_ZONE_SCOPED - - if (gpu_ilu.d == nullptr) { - SU2_MPI::Error("CUDA ILU preconditioner used before BuildILUPreconditionerGPU.", CURRENT_FUNCTION); - } - if (nPointDomain == 0) return; - - const DeviceLDU M{gpu_ilu.d, gpu_ilu.l, gpu_ilu.u, gpu_ilu.row_ptr_l, - gpu_ilu.col_ind_l, gpu_ilu.row_ptr_u, gpu_ilu.col_ind_u}; - - auto* d_vec = vec.GetDevicePointer(); - auto* d_prod = prod.GetDevicePointer(); - - const auto nLevels = ilu_level_ptr.size() - 1; - - /*--- One thread per block entry, like the factorization kernel: spreads each row's neighbor - * dot products over nVar*nVar threads instead of doing them serially in nVar threads, without - * changing the number of blocks (still one per row), so this does not trade away SM coverage - * the way batching several rows into a block did. ---*/ - const auto threads = static_cast(nVar * nVar); - const auto sharedForward = threads * sizeof(ScalarType); - const auto sharedBackward = (threads + nVar) * sizeof(ScalarType); - - if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); - - /*--- Same idea as BuildILUPreconditionerGPU: the launch sequence only depends on the (fixed) - * level structure, plus the vec/prod device pointers. Those normally are the same temporary - * buffers on every call (owned by CSysSolve / CSysVector, allocated once), so the graph is - * captured once and replayed; if the pointers ever do change the graph is recaptured, which - * is no worse than the un-graphed loop, just not free. ---*/ - if (ilu_apply_graph_exec == nullptr || ilu_apply_graph_vec != d_vec || ilu_apply_graph_prod != d_prod) { - if (ilu_apply_graph_exec != nullptr) { - gpuErrChk(cudaGraphExecDestroy(ilu_apply_graph_exec)); - ilu_apply_graph_exec = nullptr; - } - - cudaGraph_t graph; - gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); - - /*--- Forward substitution, levels in increasing order. ---*/ - for (auto level = 0ul; level < nLevels; ++level) { - const auto begin = ilu_level_ptr[level]; - const auto size = ilu_level_ptr[level + 1] - begin; - if (size == 0) continue; - IluForwardLevelKernel - <<>>(d_ilu_level_idx, begin, size, nVar, M, d_vec, d_prod); - } - - /*--- Backward substitution, levels in decreasing order. ---*/ - for (auto level = nLevels; level > 0;) { - --level; // unsigned type - const auto begin = ilu_level_ptr[level]; - const auto size = ilu_level_ptr[level + 1] - begin; - if (size == 0) continue; - IluBackwardLevelKernel<<>>( - d_ilu_level_idx, begin, size, nPointDomain, nVar, M, d_prod); - } - - gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); - gpuErrChk(cudaGraphInstantiate(&ilu_apply_graph_exec, graph, nullptr, nullptr, 0)); - gpuErrChk(cudaGraphDestroy(graph)); - ilu_apply_graph_vec = d_vec; - ilu_apply_graph_prod = d_prod; - } - - gpuErrChk(cudaGraphLaunch(ilu_apply_graph_exec, ilu_stream)); - gpuErrChk(cudaStreamSynchronize(ilu_stream)); - gpuErrChk(cudaPeekAtLastError()); -} - -#define INSTANTIATE_MATRIX(TYPE) \ -template void CSysMatrix::BuildJacobiPreconditionerGPU(); \ -template void CSysMatrix::BuildILUPreconditionerGPU(); \ -template void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector& vec, \ - CSysVector& prod) const; \ -template void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector& vec, \ - CSysVector& prod, \ - CGeometry* geometry, \ - const CConfig* config) const; -INSTANTIATE_MATRIX(su2mixedfloat) - -#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION) -INSTANTIATE_MATRIX(passivedouble) -#endif diff --git a/Common/src/linear_algebra/meson.build b/Common/src/linear_algebra/meson.build index 48ef65cb8db..3b84b2373a5 100644 --- a/Common/src/linear_algebra/meson.build +++ b/Common/src/linear_algebra/meson.build @@ -8,5 +8,5 @@ common_src += files(['CSysSolve_b.cpp', if get_option('enable-cuda') # Kept apart from common_src: these are compiled without the CoDiPack defines and so # must only go into the primal library, see common_cuda_src in Common/src/meson.build. - common_cuda_src += files(['CSysMatrixGPU.cu', 'CSysVectorGPU.cu', 'CSysPreconditionerGPU.cu']) + common_cuda_src += files(['CSysMatrixGPU.cu', 'CSysVectorGPU.cu']) endif From 15fa91c3c1b10c6631fdbe6b7a42afccca5cd08f Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 13:36:27 -0700 Subject: [PATCH 06/14] iterative ILU --- .../meshreader/CSU2BinaryMeshReaderBase.hpp | 2 +- .../meshreader/CSU2BinaryMeshReaderFEM.hpp | 2 +- .../meshreader/CSU2BinaryMeshReaderFVM.hpp | 2 +- .../meshreader/CSU2MeshReaderBase.hpp | 2 +- .../CVolumetricMovementFactory.hpp | 2 +- Common/include/linear_algebra/CSysMatrix.hpp | 19 ++++ Common/include/linear_algebra/GPUComms.cuh | 2 +- Common/include/toolboxes/SwapBytes.hpp | 2 +- Common/include/toolboxes/random_toolbox.hpp | 2 +- .../meshreader/CSU2BinaryMeshReaderBase.cpp | 2 +- .../meshreader/CSU2BinaryMeshReaderFEM.cpp | 2 +- .../meshreader/CSU2BinaryMeshReaderFVM.cpp | 2 +- .../meshreader/CSU2MeshReaderBase.cpp | 2 +- Common/src/linear_algebra/CSysMatrix.cpp | 35 ++++++ Common/src/linear_algebra/CSysMatrixGPU.cu | 101 ++++++++---------- Common/src/linear_algebra/CSysVectorGPU.cu | 2 +- Common/src/toolboxes/SwapBytes.cpp | 2 +- TestCases/euler/oneram6/inv_ONERAM6.cfg | 15 +-- .../lam_buoyancy_cavity.cfg | 2 +- .../py_wrapper/custom_source_buoyancy/run.py | 4 +- .../py_wrapper/turbulent_premixed_psi/run.py | 6 +- 21 files changed, 129 insertions(+), 81 deletions(-) diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp index f4b71d94e64..ca66e6aa80c 100644 --- a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderBase.hpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp index bec54f75fcb..d4112d09e58 100644 --- a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFEM.hpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp index 03ba796289b..894bd41486d 100644 --- a/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp +++ b/Common/include/geometry/meshreader/CSU2BinaryMeshReaderFVM.hpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp b/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp index 9e5217f5c2b..0f80b73b23d 100644 --- a/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp +++ b/Common/include/geometry/meshreader/CSU2MeshReaderBase.hpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/grid_movement/CVolumetricMovementFactory.hpp b/Common/include/grid_movement/CVolumetricMovementFactory.hpp index 702decb25e9..970d780676f 100644 --- a/Common/include/grid_movement/CVolumetricMovementFactory.hpp +++ b/Common/include/grid_movement/CVolumetricMovementFactory.hpp @@ -8,7 +8,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index cfef21fb75a..2e356c114ae 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -296,12 +296,31 @@ class CSysMatrix { * sweeps, because the U pattern is the transpose of the L pattern. */ CCompressedSparsePatternUL levels_ilu; + /*!< \brief Coloring of the (domain-only) ILU dependency graph, used only by the GPU iterative + * factorization (see ilu_color_ptr / d_ilu_color_idx below); the host/OMP path is unaffected + * and keeps using levels_ilu exactly as before. */ + CCompressedSparsePatternUL color_ilu; + /*--- Device copy of levels_ilu. The rows of a level are not contiguous in the matrix, so * the kernels have to go through this table to find the rows they work on. The offsets stay * on the host because they size the grid of the per-level kernel launches. ---*/ vector ilu_level_ptr; /*!< \brief Start of each level in d_ilu_level_idx, size nLevels+1. */ su2uint* d_ilu_level_idx = nullptr; /*!< \brief Row indices, grouped by level. */ + /*!< \brief Number of colored Gauss-Seidel sweeps used to build the ILU factorization on the + * device, see IluFactorColorKernel. Fixed (not adaptive) so the result is reproducible. */ + static constexpr int ILU_GPU_COLOR_SWEEPS = 3; + + /*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent + * set (no dependency between same-colored rows in either direction), so far fewer, wider + * colors are needed than levels, but a color launch is only exact as one step of an iterative + * refinement (see BuildILUPreconditionerGPU), not a single pass — this does not change the + * elimination order/pattern, so it converges to the exact same factorization levels_ilu does, + * just reached by iterating instead of substituting. Device copy mirrors ilu_level_ptr / + * d_ilu_level_idx. ---*/ + vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ + su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ + /*--- The per-level kernel launch sequence (init + one kernel per level for the factorization, * one per level for each of the forward/backward sweeps) is identical on every call: same * grid/block sizes, same device pointers (all fixed members, allocated once). It is captured diff --git a/Common/include/linear_algebra/GPUComms.cuh b/Common/include/linear_algebra/GPUComms.cuh index eb727477f21..39268c10e5a 100644 --- a/Common/include/linear_algebra/GPUComms.cuh +++ b/Common/include/linear_algebra/GPUComms.cuh @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * -* Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) +* Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/toolboxes/SwapBytes.hpp b/Common/include/toolboxes/SwapBytes.hpp index 033dc49dd8d..c09a91e8b12 100644 --- a/Common/include/toolboxes/SwapBytes.hpp +++ b/Common/include/toolboxes/SwapBytes.hpp @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/include/toolboxes/random_toolbox.hpp b/Common/include/toolboxes/random_toolbox.hpp index 6ad052d9e52..f2ad39a7aa2 100644 --- a/Common/include/toolboxes/random_toolbox.hpp +++ b/Common/include/toolboxes/random_toolbox.hpp @@ -8,7 +8,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp index ce7d03cb3e3..9e0e9c02a65 100644 --- a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderBase.cpp @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp index e3d4ceed2e6..cbbc4826e9d 100644 --- a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFEM.cpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp index 96905c0512a..4c235416ebd 100644 --- a/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp +++ b/Common/src/geometry/meshreader/CSU2BinaryMeshReaderFVM.cpp @@ -10,7 +10,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp b/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp index ef919ef724b..752d9cbdfd0 100644 --- a/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp +++ b/Common/src/geometry/meshreader/CSU2MeshReaderBase.cpp @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index 3ec66f031b6..af440451e7f 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -140,6 +140,7 @@ CSysMatrix::~CSysMatrix() { GPUMemoryAllocation::gpu_free(gpu_ilu.row_ptr_u); GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_u); GPUMemoryAllocation::gpu_free(d_ilu_level_idx); + GPUMemoryAllocation::gpu_free(d_ilu_color_idx); if (ilu_build_graph_exec != nullptr) cudaGraphExecDestroy(ilu_build_graph_exec); if (ilu_apply_graph_exec != nullptr) cudaGraphExecDestroy(ilu_apply_graph_exec); if (ilu_stream != nullptr) cudaStreamDestroy(ilu_stream); @@ -302,6 +303,27 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi } levels_ilu = CCompressedSparsePatternUL(levels); } + + /*--- Coloring for the GPU iterative factorization, see IluFactorColorKernel. Colors are + * true independent sets of the (domain-only, symmetric) dependency graph, computed the same + * way SU2 already colors edges/elements for OMP loops, just applied to the ILU pattern + * instead. This does not change the elimination order/pattern (nothing here affects L/U + * membership), only how the build is scheduled on the device. ---*/ + if (useCuda) { + std::vector adjPtr(nPointDomain + 1, 0); + std::vector adjIdx; + adjIdx.reserve(ilu.nnz_l + ilu.nnz_u); + for (auto i = 0ul; i < nPointDomain; ++i) { + adjPtr[i] = static_cast(adjIdx.size()); + for (auto k = ilu.row_ptr_l[i]; k < ilu.row_ptr_l[i + 1]; ++k) adjIdx.push_back(ilu.col_ind_l[k]); + for (auto k = ilu.row_ptr_u[i]; k < ilu.row_ptr_u[i + 1]; ++k) { + const auto j = ilu.col_ind_u[k]; + if (j < nPointDomain) adjIdx.push_back(static_cast(j)); + } + } + adjPtr[nPointDomain] = static_cast(adjIdx.size()); + color_ilu = colorSparsePattern(CCompressedSparsePatternUL(adjPtr, adjIdx), 1, true, false); + } } /*--- Preconditioners. ---*/ @@ -343,6 +365,19 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi ilu_level_ptr.push_back(static_cast(level_idx.size())); } d_ilu_level_idx = GPUMemoryAllocation::gpu_alloc_cpy(level_idx.data(), level_idx.size() * sizeof(su2uint)); + + /*--- Flatten the coloring the same way. ---*/ + std::vector color_idx; + color_idx.reserve(nPointDomain); + ilu_color_ptr.clear(); + ilu_color_ptr.push_back(0); + for (auto color = 0ul; color < color_ilu.getOuterSize(); ++color) { + for (auto k = 0ul; k < color_ilu.getNumNonZeros(color); ++k) { + color_idx.push_back(static_cast(color_ilu.getInnerIdx(color, k))); + } + ilu_color_ptr.push_back(static_cast(color_idx.size())); + } + d_ilu_color_idx = GPUMemoryAllocation::gpu_alloc_cpy(color_idx.data(), color_idx.size() * sizeof(su2uint)); } /*--- Thread parallel initialization. ---*/ diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index f7bc55b780c..502479907ff 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -118,34 +118,47 @@ __global__ void InvertDiagonalBlocksKernel(unsigned long nRows, unsigned long nV } /*! - * \brief Copy the matrix into the storage of the factorization, whose pattern may be larger - * (fill-in), entries that the matrix does not have are set to zero. - * \note Device version of the InitIluRow helper of BuildILUPreconditioner. Rows are - * independent, so this is done for the entire matrix before the factorization starts. - * Grid: one block per row, blockDim.x == nVar*nVar (one thread per block entry). + * \brief Factorize the rows of one color, one sweep of an iterative (colored Gauss-Seidel) + * ILU factorization: same order/pattern as the exact level-scheduled algorithm (this + * does not change L/U membership, so it converges to the exact same fixed point), but + * colors are true independent sets (zero dependency between same-colored rows in either + * direction), so a color can be processed with zero races in far fewer, wider launches + * than the number of levels — at the cost of needing several sweeps (repeated passes + * over all colors) instead of one exact pass, because for a fixed order the level count + * is already the minimum number of race-free single-pass groups (Mirsky's theorem). + * \note Every visit of a row (there is one per sweep) resets it from the original matrix first + * (folding in the device version of the InitIluRow helper of BuildILUPreconditioner), + * because the elimination below is a re-evaluation of the row's defining equation using + * the current (possibly stale) values of other rows, not an incremental accumulation. + * Grid: one block per row of the color, blockDim.x == nVar*nVar (one thread per block + * entry, so that the small matrix products are one dot product per thread). Dynamic + * shared memory: 2*nVar*nVar scalars. */ template -__global__ void IluInitKernel(unsigned long nRows, unsigned long nVar, DeviceLDU A, - DeviceLDU M) { - const unsigned long iRow = blockIdx.x; - if (iRow >= nRows) return; +__global__ void IluFactorColorKernel(const su2uint* __restrict__ color_idx, unsigned long color_begin, + unsigned long color_size, unsigned long nRows, unsigned long nVar, + DeviceLDU A, DeviceLDU M) { + if (blockIdx.x >= color_size) return; + const unsigned long iRow = color_idx[color_begin + blockIdx.x]; const auto blockSize = nVar * nVar; const unsigned long tid = threadIdx.x; + const auto iVar = tid / nVar, jVar = tid % nVar; - M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; + extern __shared__ __align__(sizeof(double)) char smem[]; + auto* Lij = reinterpret_cast(smem); + auto* work = Lij + blockSize; - /*--- Merge-scan the row of the matrix onto the row of the factorization, both are sorted - * by column index. Every thread walks the scan for its own entry of the blocks. ---*/ + /*--- Reset this row to the raw matrix entries (device version of InitIluRow, but for one + * row instead of the whole matrix, since here it runs once per row per sweep). ---*/ + M.d[iRow * blockSize + tid] = A.d[iRow * blockSize + tid]; auto scatter = [&](const su2uint* a_row_ptr, const su2uint* a_col_ind, const ScalarType* a_vals, const su2uint* m_row_ptr, const su2uint* m_col_ind, ScalarType* m_vals) { auto ka = a_row_ptr[iRow]; const auto ka_end = a_row_ptr[iRow + 1]; - for (auto k = m_row_ptr[iRow]; k < m_row_ptr[iRow + 1]; ++k) { const auto jPoint = m_col_ind[k]; while (ka < ka_end && a_col_ind[ka] < jPoint) ++ka; - if (ka < ka_end && a_col_ind[ka] == jPoint) { m_vals[k * blockSize + tid] = a_vals[ka * blockSize + tid]; } else { @@ -155,28 +168,7 @@ __global__ void IluInitKernel(unsigned long nRows, unsigned long nVar, DeviceLDU }; scatter(A.row_ptr_l, A.col_ind_l, A.l, M.row_ptr_l, M.col_ind_l, M.l); scatter(A.row_ptr_u, A.col_ind_u, A.u, M.row_ptr_u, M.col_ind_u, M.u); -} - -/*! - * \brief Factorize the rows of one level, device version of the BuildIluRow helper. - * \note Grid: one block per row of the level, blockDim.x == nVar*nVar (one thread per block - * entry, so that the small matrix products are one dot product per thread). - * Dynamic shared memory: 2*nVar*nVar scalars. - */ -template -__global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nRows, unsigned long nVar, - DeviceLDU M) { - if (blockIdx.x >= level_size) return; - - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; - const auto blockSize = nVar * nVar; - const unsigned long tid = threadIdx.x; - const auto iVar = tid / nVar, jVar = tid % nVar; - - extern __shared__ __align__(sizeof(double)) char smem[]; - auto* Lij = reinterpret_cast(smem); - auto* work = Lij + blockSize; + __syncthreads(); /*--- For this row (unknown), loop over its lower diagonal entries. ---*/ for (auto kl = M.row_ptr_l[iRow]; kl < M.row_ptr_l[iRow + 1]; ++kl) { @@ -217,8 +209,8 @@ __global__ void IluFactorLevelKernel(const su2uint* __restrict__ level_idx, unsi Block_ij[tid] = Lij[tid]; } - /*--- Invert the diagonal entry, Uii, for the next levels. The loop above may have updated - * it (when kPoint == iRow), so the whole block has to be done first. ---*/ + /*--- Invert the diagonal entry, Uii, for the rows that depend on it. The loop above may have + * updated it (when kPoint == iRow), so the whole block has to be done first. ---*/ __syncthreads(); work[tid] = M.d[iRow * blockSize + tid]; __syncthreads(); @@ -432,25 +424,26 @@ void CSysMatrix::BuildILUPreconditionerGPU() { * change execution order relative to the rest of the (single-stream) solver. ---*/ if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); - /*--- The launch sequence (init + one kernel per level) is identical on every call: the grid - * and block sizes only depend on the (fixed) sparsity pattern and the device pointers are - * fixed members, allocated once. Capture it into a CUDA graph the first time and replay that - * from then on, which removes the per-level host-side launch overhead without touching the - * parallelization of any individual kernel (unlike a persistent cooperative-groups kernel, - * this does not cap per-level parallelism to an occupancy-resident block count). ---*/ + /*--- The launch sequence (ILU_GPU_COLOR_SWEEPS passes over all colors) is identical on every + * call: the grid and block sizes only depend on the (fixed) sparsity pattern/coloring and the + * device pointers are fixed members, allocated once. Capture it into a CUDA graph the first + * time and replay that from then on, which removes the per-launch host-side overhead without + * touching the parallelization of any individual kernel (unlike a persistent cooperative- + * groups kernel, this does not cap per-color parallelism to an occupancy-resident block + * count). See IluFactorColorKernel for why several sweeps over the (far fewer, wider) colors + * are needed in place of one exact pass over the (many, narrow) levels. ---*/ if (ilu_build_graph_exec == nullptr) { cudaGraph_t graph; gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); - IluInitKernel - <<(nPointDomain), blockSize, 0, ilu_stream>>>(nPointDomain, nVar, A, M); - - for (auto level = 0ul; level + 1 < ilu_level_ptr.size(); ++level) { - const auto begin = ilu_level_ptr[level]; - const auto size = ilu_level_ptr[level + 1] - begin; - if (size == 0) continue; - IluFactorLevelKernel - <<>>(d_ilu_level_idx, begin, size, nPointDomain, nVar, M); + for (int sweep = 0; sweep < ILU_GPU_COLOR_SWEEPS; ++sweep) { + for (auto color = 0ul; color + 1 < ilu_color_ptr.size(); ++color) { + const auto begin = ilu_color_ptr[color]; + const auto size = ilu_color_ptr[color + 1] - begin; + if (size == 0) continue; + IluFactorColorKernel + <<>>(d_ilu_color_idx, begin, size, nPointDomain, nVar, A, M); + } } gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); diff --git a/Common/src/linear_algebra/CSysVectorGPU.cu b/Common/src/linear_algebra/CSysVectorGPU.cu index 2be1215a7bd..9472911cf87 100644 --- a/Common/src/linear_algebra/CSysVectorGPU.cu +++ b/Common/src/linear_algebra/CSysVectorGPU.cu @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/Common/src/toolboxes/SwapBytes.cpp b/Common/src/toolboxes/SwapBytes.cpp index 6f4d4350457..d60567e09ed 100644 --- a/Common/src/toolboxes/SwapBytes.cpp +++ b/Common/src/toolboxes/SwapBytes.cpp @@ -9,7 +9,7 @@ * The SU2 Project is maintained by the SU2 Foundation * (http://su2foundation.org) * - * Copyright 2012-2025, SU2 Contributors (cf. AUTHORS.md) + * Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) * * SU2 is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public diff --git a/TestCases/euler/oneram6/inv_ONERAM6.cfg b/TestCases/euler/oneram6/inv_ONERAM6.cfg index 6517b957871..a0351dbfbb5 100644 --- a/TestCases/euler/oneram6/inv_ONERAM6.cfg +++ b/TestCases/euler/oneram6/inv_ONERAM6.cfg @@ -53,11 +53,12 @@ CFL_NUMBER= 5.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) -ITER= 99999 +ITER= 50 LINEAR_SOLVER= FGMRES -LINEAR_SOLVER_PREC= LU_SGS -LINEAR_SOLVER_ERROR= 1E-6 -LINEAR_SOLVER_ITER= 2 +LINEAR_SOLVER_PREC= ILU +LINEAR_SOLVER_ERROR= 1E-3 +LINEAR_SOLVER_ITER= 10 +ENABLE_CUDA= YES % ----------------------- SLOPE LIMITER DEFINITION ----------------------------% % @@ -68,7 +69,7 @@ SENS_REMOVE_SHARP= YES % -------------------------- MULTIGRID PARAMETERS -----------------------------% % -MGLEVEL= 3 +MGLEVEL= 0 MGCYCLE= W_CYCLE MG_PRE_SMOOTH= ( 1, 2, 3, 3 ) MG_POST_SMOOTH= ( 0, 0, 0, 0 ) @@ -91,7 +92,7 @@ TIME_DISCRE_ADJFLOW= EULER_IMPLICIT % --------------------------- CONVERGENCE PARAMETERS --------------------------& % -CONV_RESIDUAL_MINVAL= -12 +CONV_RESIDUAL_MINVAL= -14 CONV_STARTITER= 25 CONV_CAUCHY_ELEMS= 100 CONV_CAUCHY_EPS= 1E-10 @@ -113,7 +114,7 @@ GRAD_OBJFUNC_FILENAME= of_grad SURFACE_FILENAME= surface_flow SURFACE_ADJ_FILENAME= surface_adjoint OUTPUT_WRT_FREQ= 100 -SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG) +SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG, LINSOL) OUTPUT_FILES= (RESTART_ASCII, CGNS, SURFACE_CGNS) % ----------------------- DESIGN VARIABLE PARAMETERS --------------------------% diff --git a/TestCases/py_wrapper/custom_source_buoyancy/lam_buoyancy_cavity.cfg b/TestCases/py_wrapper/custom_source_buoyancy/lam_buoyancy_cavity.cfg index 6329eaf739c..6f33ec59f60 100644 --- a/TestCases/py_wrapper/custom_source_buoyancy/lam_buoyancy_cavity.cfg +++ b/TestCases/py_wrapper/custom_source_buoyancy/lam_buoyancy_cavity.cfg @@ -4,7 +4,7 @@ % Case description: Buoyancy-driven flow inside a cavity % % Author: Thomas D. Economon % % Date: 2018.06.10 % -% File Version 8.1.0 "Harrier" % +% File Version 8.5.0 "Harrier" % % % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/TestCases/py_wrapper/custom_source_buoyancy/run.py b/TestCases/py_wrapper/custom_source_buoyancy/run.py index f07dfdc3570..190c2db4913 100644 --- a/TestCases/py_wrapper/custom_source_buoyancy/run.py +++ b/TestCases/py_wrapper/custom_source_buoyancy/run.py @@ -2,14 +2,14 @@ ## \file run.py # \brief Buoyancy force using user defines source term -# \version 8.1.0 "Harrier" +# \version 8.5.0 "Harrier" # # SU2 Project Website: https://su2code.github.io # # The SU2 Project is maintained by the SU2 Foundation # (http://su2foundation.org) # -# Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) +# Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) # # SU2 is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/TestCases/py_wrapper/turbulent_premixed_psi/run.py b/TestCases/py_wrapper/turbulent_premixed_psi/run.py index 8a157ef0ae9..27021528fdf 100644 --- a/TestCases/py_wrapper/turbulent_premixed_psi/run.py +++ b/TestCases/py_wrapper/turbulent_premixed_psi/run.py @@ -3,14 +3,14 @@ ## \file run.py # \brief turbulent premixed dump combustor simulation (PSI flame) # phi=0.5, methane-air, U=40 m/s -# \version 8.1.0 "Harrier" +# \version 8.5.0 "Harrier" # # SU2 Project Website: https://su2code.github.io # # The SU2 Project is maintained by the SU2 Foundation # (http://su2foundation.org) # -# Copyright 2012-2024, SU2 Contributors (cf. AUTHORS.md) +# Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md) # # SU2 is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -95,7 +95,7 @@ def update_temperature(SU2Driver, iPoint): iFLOWSOLVER = SU2Driver.GetSolverIndices()['INC.FLOW'] iENTH = 3 - #h = + #h = SU2Driver.Solution(iFLOWSOLVER).Set(iPoint,iENTH, cp_u*(T-Tref)) From ba06805d63a532003e284bda3bcea21c40995973 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 18:10:05 -0700 Subject: [PATCH 07/14] config knob --- Common/include/CConfig.hpp | 6 ++++++ Common/include/linear_algebra/CSysMatrix.hpp | 5 +++-- Common/src/CConfig.cpp | 7 +++++++ Common/src/linear_algebra/CSysMatrix.cpp | 1 + Common/src/linear_algebra/CSysMatrixGPU.cu | 11 +++++++---- config_template.cfg | 7 +++++++ 6 files changed, 31 insertions(+), 6 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 1b2a45dc3ff..19d27bd233e 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -646,6 +646,7 @@ class CConfig { unsigned long Linear_Solver_Prec_Threads; /*!< \brief Number of threads per rank for ILU and LU_SGS preconditioners. */ unsigned short Linear_Solver_ILU_n; /*!< \brief ILU fill=in level. */ bool Linear_Solver_ILU_levels; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ + unsigned short Linear_Solver_ILU_GPU_Sweeps; /*!< \brief Colored Gauss-Seidel sweeps used to build the ILU on the GPU. */ su2double SemiSpan; /*!< \brief Wing Semi span. */ su2double MSW_Alpha; /*!< \brief Coefficient for blending states in the MSW scheme. */ su2double Roe_Kappa; /*!< \brief Relaxation of the Roe scheme. */ @@ -4400,6 +4401,11 @@ class CConfig { */ bool GetLinear_Solver_ILU_levels(void) const { return Linear_Solver_ILU_levels; } + /*! + * \brief Get the number of colored Gauss-Seidel sweeps used to build the ILU factorization on the GPU. + */ + unsigned short GetLinear_Solver_ILU_GPU_Sweeps(void) const { return Linear_Solver_ILU_GPU_Sweeps; } + /*! * \brief Get restart frequency of the linear solver for the implicit formulation. * \return Restart frequency of the linear solver for the implicit formulation. diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index 2e356c114ae..8363ab80e0d 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -308,8 +308,9 @@ class CSysMatrix { su2uint* d_ilu_level_idx = nullptr; /*!< \brief Row indices, grouped by level. */ /*!< \brief Number of colored Gauss-Seidel sweeps used to build the ILU factorization on the - * device, see IluFactorColorKernel. Fixed (not adaptive) so the result is reproducible. */ - static constexpr int ILU_GPU_COLOR_SWEEPS = 3; + * device, see IluFactorColorKernel. Fixed per solve (not adaptive) so the result is + * reproducible; set from config in Initialize(). */ + unsigned short ilu_gpu_color_sweeps = 3; /*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent * set (no dependency between same-colored rows in either direction), so far fewer, wider diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index a9ca92ab145..f5c35917197 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1972,6 +1972,8 @@ void CConfig::SetConfig_Options() { addUnsignedShortOption("LINEAR_SOLVER_ILU_FILL_IN", Linear_Solver_ILU_n, 0); /* DESCRIPTION: Use level scheduling for OMP parallelization of the ILU preconditioner */ addBoolOption("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING", Linear_Solver_ILU_levels, false); + /* DESCRIPTION: Colored Gauss-Seidel sweeps used to build the ILU preconditioner on the GPU */ + addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_SWEEPS", Linear_Solver_ILU_GPU_Sweeps, 1); /* DESCRIPTION: Maximum number of iterations of the linear solver for the implicit formulation */ addUnsignedLongOption("LINEAR_SOLVER_RESTART_FREQUENCY", Linear_Solver_Restart_Frequency, 10); /* DESCRIPTION: Number of vectors used for deflated restarts */ @@ -4139,6 +4141,11 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i } } + if (Linear_Solver_ILU_GPU_Sweeps == 0) { + SU2_MPI::Error("LINEAR_SOLVER_ILU_GPU_SWEEPS must be at least 1, 0 sweeps never factorizes the matrix.", + CURRENT_FUNCTION); + } + Radiation = (Kind_Radiation != RADIATION_MODEL::NONE); /*--- Check for unsupported features. ---*/ diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index af440451e7f..07fb087ac87 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -277,6 +277,7 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi if (ilu_needed) { ilu_fill_in = config->GetLinear_Solver_ILU_n(); + ilu_gpu_color_sweeps = config->GetLinear_Solver_ILU_GPU_Sweeps(); const auto& pat_ilu = geometry->GetSparsePattern(type, ilu_fill_in); ilu.row_ptr_l = pat_ilu.l.outerPtr(); diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index 502479907ff..fd5c43744c1 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -424,19 +424,22 @@ void CSysMatrix::BuildILUPreconditionerGPU() { * change execution order relative to the rest of the (single-stream) solver. ---*/ if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); - /*--- The launch sequence (ILU_GPU_COLOR_SWEEPS passes over all colors) is identical on every + /*--- The launch sequence (ilu_gpu_color_sweeps passes over all colors) is identical on every * call: the grid and block sizes only depend on the (fixed) sparsity pattern/coloring and the * device pointers are fixed members, allocated once. Capture it into a CUDA graph the first * time and replay that from then on, which removes the per-launch host-side overhead without * touching the parallelization of any individual kernel (unlike a persistent cooperative- * groups kernel, this does not cap per-color parallelism to an occupancy-resident block - * count). See IluFactorColorKernel for why several sweeps over the (far fewer, wider) colors - * are needed in place of one exact pass over the (many, narrow) levels. ---*/ + * count). See IluFactorColorKernel for why several sweeps over the colors are needed. + * Note that factors are not reset between calls to BuildILUPreconditionerGPU, so with + * LINEAR_SOLVER_ILU_GPU_SWEEPS set low (even 1), each call refines the previous one's + * result rather than reconverging from scratch, relying on the matrix changing little + * between outer/pseudo-time iterations. ---*/ if (ilu_build_graph_exec == nullptr) { cudaGraph_t graph; gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); - for (int sweep = 0; sweep < ILU_GPU_COLOR_SWEEPS; ++sweep) { + for (unsigned short sweep = 0; sweep < ilu_gpu_color_sweeps; ++sweep) { for (auto color = 0ul; color + 1 < ilu_color_ptr.size(); ++color) { const auto begin = ilu_color_ptr[color]; const auto size = ilu_color_ptr[color + 1] - begin; diff --git a/config_template.cfg b/config_template.cfg index bc6be98103a..ab95850e5bf 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1661,6 +1661,13 @@ DISCADJ_LIN_PREC= ILU % Linear solver ILU preconditioner fill-in level (0 by default) LINEAR_SOLVER_ILU_FILL_IN= 0 % +% Colored Gauss-Seidel sweeps used to build the ILU preconditioner on the GPU (1 by default). +% The factorization is not reset between calls, so with the matrix changing little between +% outer/pseudo-time iterations, each call refines the previous one's result instead of +% reconverging from scratch. For cases that do few outer iterations (e.g. elasticity problems) +% it may useful to increase this number to 3-5. +LINEAR_SOLVER_ILU_GPU_SWEEPS= 1 +% % Minimum error of the linear solver for implicit formulations LINEAR_SOLVER_ERROR= 1E-6 % From 1c8f3bc3ff0b87c0811f54b56892040f417cfb73 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 18:46:03 -0700 Subject: [PATCH 08/14] syncs --- Common/src/linear_algebra/CSysMatrixGPU.cu | 16 ++++++++++++---- Common/src/linear_algebra/CSysVectorGPU.cu | 4 ++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index fd5c43744c1..0b6509b03d7 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -367,7 +367,9 @@ void CSysMatrix::ComputeJacobiPreconditionerGPU(const CSysVector((nPointDomain + threadsPerBlock - 1) / threadsPerBlock); ApplyJacobiPreconditionerKernel<<>>(d_invM, vec.GetDevicePointer(), prod.GetDevicePointer(), nPointDomain, nVar); - gpuErrChk(cudaPeekAtLastError()); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); } template @@ -391,7 +393,9 @@ void CSysMatrix::BuildJacobiPreconditionerGPU() { InvertDiagonalBlocksKernel <<(nPointDomain), blockSize, blockSize * sizeof(ScalarType)>>>(nPointDomain, nVar, gpu.d, d_invM); - gpuErrChk(cudaPeekAtLastError()); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); + gpuErrChk(cudaGetLastError()); } template @@ -456,7 +460,7 @@ void CSysMatrix::BuildILUPreconditionerGPU() { gpuErrChk(cudaGraphLaunch(ilu_build_graph_exec, ilu_stream)); gpuErrChk(cudaStreamSynchronize(ilu_stream)); - gpuErrChk(cudaPeekAtLastError()); + gpuErrChk(cudaGetLastError()); } template @@ -529,11 +533,12 @@ void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector void CSysMatrix::HtDTransfer(bool trigger) const { + SU2_ZONE_SCOPED if (!trigger) return; gpuErrChk(cudaMemcpy(gpu.d, mat.d, sizeof(ScalarType) * nPoint * nVar * nEqn, cudaMemcpyHostToDevice)); gpuErrChk(cudaMemcpy(gpu.l, mat.l, sizeof(ScalarType) * mat.nnz_l * nVar * nEqn, cudaMemcpyHostToDevice)); @@ -543,6 +548,7 @@ void CSysMatrix::HtDTransfer(bool trigger) const { template void CSysMatrix::MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, const CConfig* config) const { + SU2_ZONE_SCOPED if (nVar != nEqn) { SU2_MPI::Error("CUDA CSysMatrix block-LDU SpMV requires square blocks.", CURRENT_FUNCTION); } @@ -555,6 +561,8 @@ void CSysMatrix::MatrixVectorProductGPU(const CSysVector BlockLDU_SpMV_kernel<<>>( nPointDomain, nVar, gpu.row_ptr_l, gpu.col_ind_l, gpu.l, gpu.d, gpu.row_ptr_u, gpu.col_ind_u, gpu.u, d_vec, d_prod); + /*--- Sync so the zone above actually times the kernel, not just the (async) launch call. ---*/ + gpuErrChk(cudaStreamSynchronize(nullptr)); gpuErrChk(cudaGetLastError()); } diff --git a/Common/src/linear_algebra/CSysVectorGPU.cu b/Common/src/linear_algebra/CSysVectorGPU.cu index 9472911cf87..01b146baf76 100644 --- a/Common/src/linear_algebra/CSysVectorGPU.cu +++ b/Common/src/linear_algebra/CSysVectorGPU.cu @@ -66,18 +66,21 @@ void SetUseDeviceExpressions(bool use) { use_device_expressions = use; } template void CSysVector::HtDTransfer(bool trigger) const { + SU2_ZONE_SCOPED if (trigger) gpuErrChk(cudaMemcpy((void*)(d_vec_val), (void*)&vec_val[0], (sizeof(ScalarType) * nElm), cudaMemcpyHostToDevice)); } template void CSysVector::DtHTransfer(bool trigger) const { + SU2_ZONE_SCOPED if (trigger) gpuErrChk(cudaMemcpy((void*)(&vec_val[0]), (void*)d_vec_val, (sizeof(ScalarType) * nElm), cudaMemcpyDeviceToHost)); } template ScalarType CSysVector::GPUDot(const CSysVector& other) const { + SU2_ZONE_SCOPED /*--- Both operands are already on the device, the caller owns the transfers. This * reduces over MPI, so it must be called by a single thread (see SU2_DEVICE_REGION). ---*/ cublasHandle_t handle = GetBlasHandle(); @@ -110,6 +113,7 @@ ScalarType CSysVector::GPUDot(const CSysVector& other) const { template ScalarType CSysVector::GPUNorm() const { + SU2_ZONE_SCOPED return sqrt(GPUDot(*this)); } From ced05dc27c56532fccb7e00ebbc6b2b435f4539c Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 20:32:37 -0700 Subject: [PATCH 09/14] iterative lower solve --- Common/include/CConfig.hpp | 6 +++++ Common/include/linear_algebra/CSysMatrix.hpp | 9 +++++++ Common/src/CConfig.cpp | 8 ++++++ Common/src/linear_algebra/CSysMatrix.cpp | 1 + Common/src/linear_algebra/CSysMatrixGPU.cu | 28 +++++++++++++++----- 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index 19d27bd233e..fa015865033 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -647,6 +647,7 @@ class CConfig { unsigned short Linear_Solver_ILU_n; /*!< \brief ILU fill=in level. */ bool Linear_Solver_ILU_levels; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ unsigned short Linear_Solver_ILU_GPU_Sweeps; /*!< \brief Colored Gauss-Seidel sweeps used to build the ILU on the GPU. */ + unsigned short Linear_Solver_ILU_GPU_Fwd_Sweeps; /*!< \brief Colored Jacobi sweeps for the GPU ILU forward triangular solve. */ su2double SemiSpan; /*!< \brief Wing Semi span. */ su2double MSW_Alpha; /*!< \brief Coefficient for blending states in the MSW scheme. */ su2double Roe_Kappa; /*!< \brief Relaxation of the Roe scheme. */ @@ -4406,6 +4407,11 @@ class CConfig { */ unsigned short GetLinear_Solver_ILU_GPU_Sweeps(void) const { return Linear_Solver_ILU_GPU_Sweeps; } + /*! + * \brief Get the number of colored Jacobi sweeps used for the GPU ILU forward triangular solve. + */ + unsigned short GetLinear_Solver_ILU_GPU_Fwd_Sweeps(void) const { return Linear_Solver_ILU_GPU_Fwd_Sweeps; } + /*! * \brief Get restart frequency of the linear solver for the implicit formulation. * \return Restart frequency of the linear solver for the implicit formulation. diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index 8363ab80e0d..aa6a6026200 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -312,6 +312,15 @@ class CSysMatrix { * reproducible; set from config in Initialize(). */ unsigned short ilu_gpu_color_sweeps = 3; + /*!< \brief Number of colored Jacobi sweeps used for the forward triangular solve (only) when + * applying the ILU preconditioner on the device, see ComputeILUPreconditionerGPU. The backward + * solve stays exact/level-scheduled: a host experiment showed the colored-iterative backward + * solve diverges for this class of matrix (unlike forward, which converges cleanly), so only + * forward is colored. Unlike ilu_gpu_color_sweeps, this cannot rely on cross-call warm + * starting: the apply runs once per Krylov iteration with a new RHS each time, so every sweep + * is paid in full on every call. Set from config in Initialize(). */ + unsigned short ilu_gpu_fwd_sweeps = 4; + /*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent * set (no dependency between same-colored rows in either direction), so far fewer, wider * colors are needed than levels, but a color launch is only exact as one step of an iterative diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index f5c35917197..e1bb41bc235 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1974,6 +1974,8 @@ void CConfig::SetConfig_Options() { addBoolOption("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING", Linear_Solver_ILU_levels, false); /* DESCRIPTION: Colored Gauss-Seidel sweeps used to build the ILU preconditioner on the GPU */ addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_SWEEPS", Linear_Solver_ILU_GPU_Sweeps, 1); + /* DESCRIPTION: Colored Jacobi sweeps used for the GPU ILU forward triangular solve */ + addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_FWD_SWEEPS", Linear_Solver_ILU_GPU_Fwd_Sweeps, 3); /* DESCRIPTION: Maximum number of iterations of the linear solver for the implicit formulation */ addUnsignedLongOption("LINEAR_SOLVER_RESTART_FREQUENCY", Linear_Solver_Restart_Frequency, 10); /* DESCRIPTION: Number of vectors used for deflated restarts */ @@ -4146,6 +4148,12 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i CURRENT_FUNCTION); } + if (Linear_Solver_ILU_GPU_Fwd_Sweeps == 0) { + SU2_MPI::Error( + "LINEAR_SOLVER_ILU_GPU_FWD_SWEEPS must be at least 1, 0 sweeps never solves the forward triangular system.", + CURRENT_FUNCTION); + } + Radiation = (Kind_Radiation != RADIATION_MODEL::NONE); /*--- Check for unsupported features. ---*/ diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index 07fb087ac87..e58ae302281 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -278,6 +278,7 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi if (ilu_needed) { ilu_fill_in = config->GetLinear_Solver_ILU_n(); ilu_gpu_color_sweeps = config->GetLinear_Solver_ILU_GPU_Sweeps(); + ilu_gpu_fwd_sweeps = config->GetLinear_Solver_ILU_GPU_Fwd_Sweeps(); const auto& pat_ilu = geometry->GetSparsePattern(type, ilu_fill_in); ilu.row_ptr_l = pat_ilu.l.outerPtr(); diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index 0b6509b03d7..53774aa6e4a 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -505,13 +505,27 @@ void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector - <<>>(d_ilu_level_idx, begin, size, nVar, M, d_vec, d_prod); + /*--- Forward substitution: colored-iterative, not level-scheduled. IluForwardLevelKernel + * only ever reads already-finalized prod values within a single exact level-scheduled pass; + * across colors that guarantee is gone (a color is wider/less ordered than a level), so an + * L-neighbor in a not-yet-processed color this sweep is read at whatever value it currently + * holds. Repeated full passes over all colors still converge to the same forward-solve + * result (validated on the host: same fixed point, ~4.7x error reduction per sweep, colors + * are true independent sets so this is race-free). Zero prod first: on the very first call + * this buffer may hold unrelated leftover data, and reading it would otherwise make the + * first sweep ill-defined. Unlike the level-scheduled solve this replaces, the sweep count is + * paid on every apply call, not amortized — see ilu_gpu_fwd_sweeps. + * The backward solve stays exact/level-scheduled below: a host experiment showed the + * colored-iterative version diverges to NaN for this class of matrix, so it is not used. ---*/ + gpuErrChk(cudaMemsetAsync(d_prod, 0, nPointDomain * nVar * sizeof(ScalarType), ilu_stream)); + for (unsigned short sweep = 0; sweep < ilu_gpu_fwd_sweeps; ++sweep) { + for (auto color = 0ul; color < ilu_color_ptr.size() - 1; ++color) { + const auto begin = ilu_color_ptr[color]; + const auto size = ilu_color_ptr[color + 1] - begin; + if (size == 0) continue; + IluForwardLevelKernel + <<>>(d_ilu_color_idx, begin, size, nVar, M, d_vec, d_prod); + } } /*--- Backward substitution, levels in decreasing order. ---*/ From 79dd8da21a73dc17bceb526c460b59e164007947 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 22:15:47 -0700 Subject: [PATCH 10/14] cleanup --- Common/include/CConfig.hpp | 15 +- Common/include/linear_algebra/CSysMatrix.hpp | 1582 +++++++++--------- Common/src/CConfig.cpp | 23 +- Common/src/linear_algebra/CSysMatrix.cpp | 23 +- Common/src/linear_algebra/CSysMatrixGPU.cu | 141 +- TestCases/euler/oneram6/inv_ONERAM6.cfg | 1 + config_template.cfg | 14 +- 7 files changed, 899 insertions(+), 900 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index fa015865033..fc0dac4edb2 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -646,8 +646,9 @@ class CConfig { unsigned long Linear_Solver_Prec_Threads; /*!< \brief Number of threads per rank for ILU and LU_SGS preconditioners. */ unsigned short Linear_Solver_ILU_n; /*!< \brief ILU fill=in level. */ bool Linear_Solver_ILU_levels; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ - unsigned short Linear_Solver_ILU_GPU_Sweeps; /*!< \brief Colored Gauss-Seidel sweeps used to build the ILU on the GPU. */ - unsigned short Linear_Solver_ILU_GPU_Fwd_Sweeps; /*!< \brief Colored Jacobi sweeps for the GPU ILU forward triangular solve. */ + /*!< \brief Colored-iterative sweep counts for the GPU ILU preconditioner: [0] builds the + * factorization (Gauss-Seidel), [1]/[2] apply it (Jacobi, forward/backward triangular solve). */ + array Linear_Solver_ILU_GPU_Sweeps{(1, 2, 2}}; su2double SemiSpan; /*!< \brief Wing Semi span. */ su2double MSW_Alpha; /*!< \brief Coefficient for blending states in the MSW scheme. */ su2double Roe_Kappa; /*!< \brief Relaxation of the Roe scheme. */ @@ -4403,14 +4404,10 @@ class CConfig { bool GetLinear_Solver_ILU_levels(void) const { return Linear_Solver_ILU_levels; } /*! - * \brief Get the number of colored Gauss-Seidel sweeps used to build the ILU factorization on the GPU. + * \brief Get the [build, forward, backward] colored-iterative sweep counts for the GPU ILU + * preconditioner, see Linear_Solver_ILU_GPU_Sweeps. */ - unsigned short GetLinear_Solver_ILU_GPU_Sweeps(void) const { return Linear_Solver_ILU_GPU_Sweeps; } - - /*! - * \brief Get the number of colored Jacobi sweeps used for the GPU ILU forward triangular solve. - */ - unsigned short GetLinear_Solver_ILU_GPU_Fwd_Sweeps(void) const { return Linear_Solver_ILU_GPU_Fwd_Sweeps; } + array GetLinear_Solver_ILU_GPU_Sweeps(void) const { return Linear_Solver_ILU_GPU_Sweeps; } /*! * \brief Get restart frequency of the linear solver for the implicit formulation. diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index aa6a6026200..a306e2ac5dd 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -297,395 +297,389 @@ class CSysMatrix { CCompressedSparsePatternUL levels_ilu; /*!< \brief Coloring of the (domain-only) ILU dependency graph, used only by the GPU iterative - * factorization (see ilu_color_ptr / d_ilu_color_idx below); the host/OMP path is unaffected - * and keeps using levels_ilu exactly as before. */ + * factorization and triangular solves (see ilu_color_ptr / d_ilu_color_idx below); the + * host/OMP path is unaffected and keeps using levels_ilu exactly as before. */ CCompressedSparsePatternUL color_ilu; - /*--- Device copy of levels_ilu. The rows of a level are not contiguous in the matrix, so - * the kernels have to go through this table to find the rows they work on. The offsets stay - * on the host because they size the grid of the per-level kernel launches. ---*/ - vector ilu_level_ptr; /*!< \brief Start of each level in d_ilu_level_idx, size nLevels+1. */ - su2uint* d_ilu_level_idx = nullptr; /*!< \brief Row indices, grouped by level. */ - - /*!< \brief Number of colored Gauss-Seidel sweeps used to build the ILU factorization on the - * device, see IluFactorColorKernel. Fixed per solve (not adaptive) so the result is + /*!< \brief Number of colored Gauss-Seidel sweeps used to build and apply the ILU factorization + * on the device, see IluFactorColorKernel. Fixed per solve (not adaptive) so the result is * reproducible; set from config in Initialize(). */ - unsigned short ilu_gpu_color_sweeps = 3; - - /*!< \brief Number of colored Jacobi sweeps used for the forward triangular solve (only) when - * applying the ILU preconditioner on the device, see ComputeILUPreconditionerGPU. The backward - * solve stays exact/level-scheduled: a host experiment showed the colored-iterative backward - * solve diverges for this class of matrix (unlike forward, which converges cleanly), so only - * forward is colored. Unlike ilu_gpu_color_sweeps, this cannot rely on cross-call warm - * starting: the apply runs once per Krylov iteration with a new RHS each time, so every sweep - * is paid in full on every call. Set from config in Initialize(). */ - unsigned short ilu_gpu_fwd_sweeps = 4; - - /*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent - * set (no dependency between same-colored rows in either direction), so far fewer, wider - * colors are needed than levels, but a color launch is only exact as one step of an iterative - * refinement (see BuildILUPreconditionerGPU), not a single pass — this does not change the - * elimination order/pattern, so it converges to the exact same factorization levels_ilu does, - * just reached by iterating instead of substituting. Device copy mirrors ilu_level_ptr / - * d_ilu_level_idx. ---*/ - vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ - su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ - - /*--- The per-level kernel launch sequence (init + one kernel per level for the factorization, - * one per level for each of the forward/backward sweeps) is identical on every call: same - * grid/block sizes, same device pointers (all fixed members, allocated once). It is captured - * once into a CUDA graph and replayed, which removes host-side launch overhead without - * changing the parallelization (unlike a persistent cooperative-groups kernel, this does not - * cap per-level parallelism to the occupancy-resident block count). ---*/ - /*--- Types are forward-declared as opaque structs (matching the real cudaGraphExec_t / - * cudaStream_t typedefs) so this header does not need to include the CUDA runtime. ---*/ - mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; - mutable struct CUgraphExec_st* ilu_apply_graph_exec = nullptr; - mutable const ScalarType* ilu_apply_graph_vec = nullptr; /*!< \brief Pointers the apply graph - * was captured with, to detect when - * it must be recaptured. */ - mutable ScalarType* ilu_apply_graph_prod = nullptr; - /*--- The legacy default stream cannot be captured into a graph, so the ILU graphs are - * captured and replayed on this dedicated stream instead; every launch on it is followed by - * a sync back to the host before control returns to the rest of the (single-stream) solver, - * so this does not change execution order relative to everything else, which stays on the - * default stream. ---*/ - mutable struct CUstream_st* ilu_stream = nullptr; - - ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ - - /*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ - mutable vector> - LineletUpper; /*!< \brief Pointers to the upper blocks of the tri-diag system (working memory). */ - mutable vector> - LineletInvDiag; /*!< \brief Inverse of the diagonal blocks of the tri-diag system (working memory). */ - mutable vector> - LineletVector; /*!< \brief Solution and RHS of the tri-diag system (working memory). */ + array ilu_gpu_sweeps {(1, 2, 2 + } +}; + +/*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent + * set (no dependency between same-colored rows in either direction), so far fewer, wider + * colors are needed than levels, but a color launch is only exact as one step of an iterative + * refinement (see BuildILUPreconditionerGPU and ComputeILUPreconditionerGPU), not a single + * pass — this does not change the elimination order/pattern, so the factorization and both + * triangular solves converge to the exact same result levels_ilu would give, just reached by + * iterating instead of substituting. ---*/ +vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ +su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ + +/*!< \brief Fixed right-hand side of the backward triangular solve (the forward-solve result), + * kept separate from the evolving prod buffer. A color visits every row once per sweep, so + * after the first sweep prod[iRow] holds a solution *estimate*, not the right-hand side + * anymore; reading the right-hand side back out of prod past the first sweep would silently + * solve the wrong equation. Size nPointDomain*nVar, allocated once in Initialize(). */ +ScalarType* d_ilu_backward_rhs = nullptr; + +/*--- The per-color kernel launch sequence (ilu_gpu_sweeps[0] passes for the factorization, + * ilu_gpu_sweeps[1] / ilu_gpu_sweeps[2] passes for the forward/backward triangular solves) + * is identical on every call: same grid/block sizes, same device pointers (all fixed members, + * allocated once). It is captured once into a CUDA graph and replayed, which removes + * host-side launch overhead without changing the parallelization (unlike a persistent + * cooperative-groups kernel, this does not cap per-color parallelism to the occupancy-resident + * block count). ---*/ +/*--- Types are forward-declared as opaque structs (matching the real cudaGraphExec_t / + * cudaStream_t typedefs) so this header does not need to include the CUDA runtime. ---*/ +mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; +mutable struct CUgraphExec_st* ilu_apply_graph_exec = nullptr; +mutable const ScalarType* ilu_apply_graph_vec = nullptr; /*!< \brief Pointers the apply graph + * was captured with, to detect when + * it must be recaptured. */ +mutable ScalarType* ilu_apply_graph_prod = nullptr; +/*--- The legacy default stream cannot be captured into a graph, so the ILU graphs are + * captured and replayed on this dedicated stream instead; every launch on it is followed by + * a sync back to the host before control returns to the rest of the (single-stream) solver, + * so this does not change execution order relative to everything else, which stays on the + * default stream. ---*/ +mutable struct CUstream_st* ilu_stream = nullptr; + +ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ + +/*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ +mutable vector> + LineletUpper; /*!< \brief Pointers to the upper blocks of the tri-diag system (working memory). */ +mutable vector> + LineletInvDiag; /*!< \brief Inverse of the diagonal blocks of the tri-diag system (working memory). */ +mutable vector> + LineletVector; /*!< \brief Solution and RHS of the tri-diag system (working memory). */ #ifdef USE_MKL - using gemm_t = typename mkl_jit_wrapper::gemm_t; - void* MatrixMatrixProductJitter; /*!< \brief Jitter handle for MKL JIT based GEMM. */ - gemm_t MatrixMatrixProductKernel; /*!< \brief MKL JIT based GEMM kernel. */ - void* MatrixVectorProductJitterBetaZero; /*!< \brief Jitter handle for MKL JIT based GEMV. */ - gemm_t MatrixVectorProductKernelBetaZero; /*!< \brief MKL JIT based GEMV kernel. */ - void* MatrixVectorProductJitterBetaOne; /*!< \brief Jitter handle for MKL JIT based GEMV with BETA=1.0. */ - gemm_t MatrixVectorProductKernelBetaOne; /*!< \brief MKL JIT based GEMV kernel with BETA=1.0. */ - void* MatrixVectorProductJitterAlphaMinusOne; /*!< \brief Jitter handle for MKL JIT based GEMV with ALPHA=-1.0 and - BETA=1.0. */ - gemm_t MatrixVectorProductKernelAlphaMinusOne; /*!< \brief MKL JIT based GEMV kernel with ALPHA=-1.0 and BETA=1.0. */ +using gemm_t = typename mkl_jit_wrapper::gemm_t; +void* MatrixMatrixProductJitter; /*!< \brief Jitter handle for MKL JIT based GEMM. */ +gemm_t MatrixMatrixProductKernel; /*!< \brief MKL JIT based GEMM kernel. */ +void* MatrixVectorProductJitterBetaZero; /*!< \brief Jitter handle for MKL JIT based GEMV. */ +gemm_t MatrixVectorProductKernelBetaZero; /*!< \brief MKL JIT based GEMV kernel. */ +void* MatrixVectorProductJitterBetaOne; /*!< \brief Jitter handle for MKL JIT based GEMV with BETA=1.0. */ +gemm_t MatrixVectorProductKernelBetaOne; /*!< \brief MKL JIT based GEMV kernel with BETA=1.0. */ +void* MatrixVectorProductJitterAlphaMinusOne; /*!< \brief Jitter handle for MKL JIT based GEMV with ALPHA=-1.0 and + BETA=1.0. */ +gemm_t MatrixVectorProductKernelAlphaMinusOne; /*!< \brief MKL JIT based GEMV kernel with ALPHA=-1.0 and BETA=1.0. */ #endif #ifdef HAVE_PASTIX - mutable CPastixWrapper pastix_wrapper; +mutable CPastixWrapper pastix_wrapper; #endif - /*! - * \brief Handle type conversion for when we Set, Add, etc. blocks, preserving derivative information (if supported by - * types). - */ - template ::value> = 0> - FORCEINLINE static DstType ActiveAssign(const SrcType& val) { - return SU2_TYPE::GetValue(val); - } +/*! + * \brief Handle type conversion for when we Set, Add, etc. blocks, preserving derivative information (if supported by + * types). + */ +template ::value> = 0> +FORCEINLINE static DstType ActiveAssign(const SrcType& val) { + return SU2_TYPE::GetValue(val); +} - template ::value> = 0> - FORCEINLINE static DstType ActiveAssign(const SrcType& val) { - return val; - } +template ::value> = 0> +FORCEINLINE static DstType ActiveAssign(const SrcType& val) { + return val; +} - /*! - * \brief Handle type conversion for when we Set, Add, etc. blocks, discarding derivative information. - */ - template - FORCEINLINE static ScalarType PassiveAssign(const SrcType& val) { - return SU2_TYPE::GetValue(val); - } +/*! + * \brief Handle type conversion for when we Set, Add, etc. blocks, discarding derivative information. + */ +template +FORCEINLINE static ScalarType PassiveAssign(const SrcType& val) { + return SU2_TYPE::GetValue(val); +} - /*! - * \brief Calculates the matrix-vector product: product = matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[out] product - */ - void MatrixVectorProduct(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; +/*! + * \brief Calculates the matrix-vector product: product = matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[out] product + */ +void MatrixVectorProduct(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; - /*! - * \brief Calculates the matrix-vector product: product += matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[in,out] product - */ - void MatrixVectorProductAdd(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; +/*! + * \brief Calculates the matrix-vector product: product += matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[in,out] product + */ +void MatrixVectorProductAdd(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; - /*! - * \brief Calculates the matrix-vector product: product -= matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[in,out] product - */ - void MatrixVectorProductSub(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; +/*! + * \brief Calculates the matrix-vector product: product -= matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[in,out] product + */ +void MatrixVectorProductSub(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; - /*! - * \brief Calculates the matrix-matrix product - */ - void MatrixMatrixProduct(const ScalarType* matrix_a, const ScalarType* matrix_b, ScalarType* product) const; +/*! + * \brief Calculates the matrix-matrix product + */ +void MatrixMatrixProduct(const ScalarType* matrix_a, const ScalarType* matrix_b, ScalarType* product) const; - /*! - * \brief Subtract b from a and store the result in c. - */ - FORCEINLINE void VectorSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { - for (unsigned long iVar = 0; iVar < nVar; iVar++) c[iVar] = a[iVar] - b[iVar]; - } +/*! + * \brief Subtract b from a and store the result in c. + */ +FORCEINLINE void VectorSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { + for (unsigned long iVar = 0; iVar < nVar; iVar++) c[iVar] = a[iVar] - b[iVar]; +} - /*! - * \brief Subtract b from a and store the result in c. - */ - FORCEINLINE void MatrixSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { - SU2_OMP_SIMD - for (unsigned long iVar = 0; iVar < nVar * nEqn; iVar++) c[iVar] = a[iVar] - b[iVar]; - } +/*! + * \brief Subtract b from a and store the result in c. + */ +FORCEINLINE void MatrixSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { + SU2_OMP_SIMD + for (unsigned long iVar = 0; iVar < nVar * nEqn; iVar++) c[iVar] = a[iVar] - b[iVar]; +} - /*! - * \brief Copy matrix src into dst, transpose if required. - */ - FORCEINLINE void MatrixCopy(const ScalarType* src, ScalarType* dst) const { - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) dst[iVar] = src[iVar]; - } +/*! + * \brief Copy matrix src into dst, transpose if required. + */ +FORCEINLINE void MatrixCopy(const ScalarType* src, ScalarType* dst) const { + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) dst[iVar] = src[iVar]; +} - /*! - * \brief Zero a matrix. - */ - FORCEINLINE void ZeroMatrix(ScalarType* mat) const { - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) mat[iVar] = 0; - } +/*! + * \brief Zero a matrix. + */ +FORCEINLINE void ZeroMatrix(ScalarType* mat) const { + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) mat[iVar] = 0; +} - /*! - * \brief Solve a small (nVar x nVar) linear system using Gaussian elimination. - * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. - * \param[in,out] vec - On entry the rhs, on exit the solution. - */ - void GaussElimination(ScalarType* matrix, ScalarType* vec) const; +/*! + * \brief Solve a small (nVar x nVar) linear system using Gaussian elimination. + * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. + * \param[in,out] vec - On entry the rhs, on exit the solution. + */ +void GaussElimination(ScalarType* matrix, ScalarType* vec) const; - /*! - * \brief Invert a small dense matrix. - * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. - * \param[out] inverse - the matrix inverse. - */ - void MatrixInverse(ScalarType* matrix, ScalarType* inverse) const; +/*! + * \brief Invert a small dense matrix. + * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. + * \param[out] inverse - the matrix inverse. + */ +void MatrixInverse(ScalarType* matrix, ScalarType* inverse) const; - /*! - * \brief Performs the Gauss Elimination algorithm to solve the linear subsystem of the (i,i) subblock and rhs. - * \param[in] block_i - Index of the (i,i) diagonal block. - * \param[in] rhs - Right-hand-side of the linear system. - * \return Solution of the linear system (overwritten on rhs). - */ - inline void GaussElimination(unsigned long block_i, ScalarType* rhs) const; +/*! + * \brief Performs the Gauss Elimination algorithm to solve the linear subsystem of the (i,i) subblock and rhs. + * \param[in] block_i - Index of the (i,i) diagonal block. + * \param[in] rhs - Right-hand-side of the linear system. + * \return Solution of the linear system (overwritten on rhs). + */ +inline void GaussElimination(unsigned long block_i, ScalarType* rhs) const; - /*! - * \brief Inverse diagonal block. - * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. - * \param[out] invBlock - Inverse block. - */ - inline void InverseDiagonalBlock(unsigned long block_i, ScalarType* invBlock) const; +/*! + * \brief Inverse diagonal block. + * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. + * \param[out] invBlock - Inverse block. + */ +inline void InverseDiagonalBlock(unsigned long block_i, ScalarType* invBlock) const; - /*! - * \brief Invert diagonal block (Uii) of the ILU matrix in place. - * \param[in] block_i - Index of the block to invert. - * \return Inverted block. - */ - inline const ScalarType* InvertDiagonalBlockILUMatrix(unsigned long block_i); +/*! + * \brief Invert diagonal block (Uii) of the ILU matrix in place. + * \param[in] block_i - Index of the block to invert. + * \return Inverted block. + */ +inline const ScalarType* InvertDiagonalBlockILUMatrix(unsigned long block_i); - /*! - * \brief Returns the start of the ILU block or nullptr if (i,j) is not a nonzero. - * \param[in] block_i/j - Indexes of the block in the matrix-by-blocks structure. - */ - inline ScalarType* GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j); +/*! + * \brief Returns the start of the ILU block or nullptr if (i,j) is not a nonzero. + * \param[in] block_i/j - Indexes of the block in the matrix-by-blocks structure. + */ +inline ScalarType* GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j); - /*! - * \brief Performs the product of i-th row of the upper part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the upper part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \param[in] col_ub - Exclusive upper bound for column indices considered in multiplication. - * \param[out] prod - Result of the product U(A)*vec. - */ - inline void UpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, - ScalarType* prod) const; +/*! + * \brief Performs the product of i-th row of the upper part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the upper part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \param[in] col_ub - Exclusive upper bound for column indices considered in multiplication. + * \param[out] prod - Result of the product U(A)*vec. + */ +inline void UpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, + ScalarType* prod) const; - /*! - * \brief Performs the product of i-th row of the lower part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the lower part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \param[in] col_lb - Inclusive lower bound for column indices considered in multiplication. - * \param[out] prod - Result of the product L(A)*vec. - */ - inline void LowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, - ScalarType* prod) const; +/*! + * \brief Performs the product of i-th row of the lower part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the lower part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \param[in] col_lb - Inclusive lower bound for column indices considered in multiplication. + * \param[out] prod - Result of the product L(A)*vec. + */ +inline void LowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, + ScalarType* prod) const; - /*! - * \brief Performs the product of i-th row of the diagonal part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the diagonal part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \return prod Result of the product D(A)*vec (stored at *prod_row_vector). - */ - inline void DiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; +/*! + * \brief Performs the product of i-th row of the diagonal part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the diagonal part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \return prod Result of the product D(A)*vec (stored at *prod_row_vector). + */ +inline void DiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; - /*! - * \brief Performs the product of i-th row of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the row of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \return Result of the product (stored at *prod_row_vector). - */ - void RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; +/*! + * \brief Performs the product of i-th row of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the row of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \return Result of the product (stored at *prod_row_vector). + */ +void RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; - /*! - * \brief Computes product += A_k * vec using the quantized representation of block k. - * \note Only valid after QuantizeDiagonalBlocks() has been called. - * \param[in] k - Block index in the CSR flat storage. - * \param[in] vec - Input vector (nEqn entries). - * \param[in,out] prod - Accumulation output (nVar entries). - */ - inline void QuantizedMatVecAdd(const QuantType* qs, const QuantType* qv, const ScalarType* vec, - ScalarType* prod) const; +/*! + * \brief Computes product += A_k * vec using the quantized representation of block k. + * \note Only valid after QuantizeDiagonalBlocks() has been called. + * \param[in] k - Block index in the CSR flat storage. + * \param[in] vec - Input vector (nEqn entries). + * \param[in,out] prod - Accumulation output (nVar entries). + */ +inline void QuantizedMatVecAdd(const QuantType* qs, const QuantType* qv, const ScalarType* vec, ScalarType* prod) const; - /*! \brief Quantize one nVar×nVar block (row-major) into the int8 scale+value arrays. - * Called on the hot assembly path (SetBlocks/UpdateBlocks in Q_LU_SGS mode). */ - void QuantizeBlock(const ScalarType* blk, QuantType* qs, QuantType* qv) const; +/*! \brief Quantize one nVar×nVar block (row-major) into the int8 scale+value arrays. + * Called on the hot assembly path (SetBlocks/UpdateBlocks in Q_LU_SGS mode). */ +void QuantizeBlock(const ScalarType* blk, QuantType* qs, QuantType* qv) const; - /*! \brief Full-row product using quantized L/D/U (Q_LU_SGS SpMV path). */ - inline void QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; +/*! \brief Full-row product using quantized L/D/U (Q_LU_SGS SpMV path). */ +inline void QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; - /*! \brief Upper-triangle product using quantized U (Q_LU_SGS backward sweep). */ - inline void QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, - ScalarType* prod) const; +/*! \brief Upper-triangle product using quantized U (Q_LU_SGS backward sweep). */ +inline void QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, + ScalarType* prod) const; - /*! \brief Lower-triangle product using quantized L (Q_LU_SGS forward sweep). */ - inline void QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, - ScalarType* prod) const; +/*! \brief Lower-triangle product using quantized L (Q_LU_SGS forward sweep). */ +inline void QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, + ScalarType* prod) const; - /*! \brief Diagonal product using quantized D (Q_LU_SGS backward sweep). */ - inline void QuantizedDiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; +/*! \brief Diagonal product using quantized D (Q_LU_SGS backward sweep). */ +inline void QuantizedDiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; - /*! \brief Gauss elimination on the quantized diagonal block: decodes q_blocks_d into a local - * ScalarType buffer and delegates to the scalar GaussElimination overload. */ - inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; +/*! \brief Gauss elimination on the quantized diagonal block: decodes q_blocks_d into a local + * ScalarType buffer and delegates to the scalar GaussElimination overload. */ +inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; - /*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ +/*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ - /*! - * \brief Performs the product of a sparse matrix by a CSysVector on the device. - */ - void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +/*! + * \brief Performs the product of a sparse matrix by a CSysVector on the device. + */ +void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. - * \note Requires the device matrix to be up to date, see HtDTransfer. - */ - void BuildJacobiPreconditionerGPU(); +/*! + * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ +void BuildJacobiPreconditionerGPU(); - /*! - * \brief Apply the Jacobi preconditioner on the GPU/device side. - */ - void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, - CGeometry* geometry, const CConfig* config) const; +/*! + * \brief Apply the Jacobi preconditioner on the GPU/device side. + */ +void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const; - /*! - * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. - * \note Requires the device matrix to be up to date, see HtDTransfer. - */ - void BuildILUPreconditionerGPU(); +/*! + * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ +void BuildILUPreconditionerGPU(); - /*! - * \brief Apply the ILU preconditioner on the device. - */ - void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; +/*! + * \brief Apply the ILU preconditioner on the device. + */ +void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; - public: - /*! - * \brief Constructor of the class. - */ - CSysMatrix(); +public: +/*! + * \brief Constructor of the class. + */ +CSysMatrix(); - /*! - * \brief Destructor of the class. - */ - ~CSysMatrix(); +/*! + * \brief Destructor of the class. + */ +~CSysMatrix(); - /*! - * \brief Initializes the sparse matrix. - * \note The preconditioners require nVar == nEqn (square blocks). - * \param[in] npoint - Number of points including halos. - * \param[in] npointdomain - Number of points excluding halos. - * \param[in] nvar - Number of variables (and rows of the blocks). - * \param[in] neqn - Number of equations (and columns of the blocks). - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[in] needTranspPtr - If the L/U transpose maps should be built, used for "SetDiagonalAsColumnSum". - * \param[in] grad_mode - Gradient smoothing mode, only used to detect the right preconditioner type. - * \param[in] allow_quant - Quantization is only possible with solvers that "set and forget" the off-diagonal - * blocks of the matrix. Solvers that perform multiple updates would lose too much information, so - * that pattern is not supported with quantization (the code will hit null pointers). It is up to - * the solver to declare whether it will "set and forget". - */ - void Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, - bool EdgeConnect, CGeometry* geometry, const CConfig* config, bool needTranspPtr = false, - bool grad_mode = false, bool allow_quant = false); +/*! + * \brief Initializes the sparse matrix. + * \note The preconditioners require nVar == nEqn (square blocks). + * \param[in] npoint - Number of points including halos. + * \param[in] npointdomain - Number of points excluding halos. + * \param[in] nvar - Number of variables (and rows of the blocks). + * \param[in] neqn - Number of equations (and columns of the blocks). + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[in] needTranspPtr - If the L/U transpose maps should be built, used for "SetDiagonalAsColumnSum". + * \param[in] grad_mode - Gradient smoothing mode, only used to detect the right preconditioner type. + * \param[in] allow_quant - Quantization is only possible with solvers that "set and forget" the off-diagonal + * blocks of the matrix. Solvers that perform multiple updates would lose too much information, so + * that pattern is not supported with quantization (the code will hit null pointers). It is up to + * the solver to declare whether it will "set and forget". + */ +void Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, + bool EdgeConnect, CGeometry* geometry, const CConfig* config, bool needTranspPtr = false, + bool grad_mode = false, bool allow_quant = false); - /*! - * \brief Compresses off-diagonal blocks into quantized form for use with USE_QUANTIZATION. - */ - void QuantizeDiagonalBlocks(); +/*! + * \brief Compresses off-diagonal blocks into quantized form for use with USE_QUANTIZATION. + */ +void QuantizeDiagonalBlocks(); - /*! - * \brief Sets to zero all the entries of the sparse matrix. - */ - void SetValZero(); +/*! + * \brief Sets to zero all the entries of the sparse matrix. + */ +void SetValZero(); - /*! - * \brief Sets to zero all the block diagonal entries of the sparse matrix. - */ - void SetValDiagonalZero(); +/*! + * \brief Sets to zero all the block diagonal entries of the sparse matrix. + */ +void SetValDiagonalZero(); - /*! - * \brief Performs the memory copy from host to device. - * \param[in] trigger - boolean value that decides whether to conduct the transfer or not. True by default. - */ - void HtDTransfer(bool trigger = true) const; +/*! + * \brief Performs the memory copy from host to device. + * \param[in] trigger - boolean value that decides whether to conduct the transfer or not. True by default. + */ +void HtDTransfer(bool trigger = true) const; - /*! - * \brief Get a pointer to the start of block "ij" - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \return Pointer to location in memory where the block starts. - */ - FORCEINLINE const ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) const { - if (block_i == block_j) return &mat.d[block_i * nVar * nEqn]; - if (block_j < block_i) { - for (auto index = mat.row_ptr_l[block_i]; index < mat.row_ptr_l[block_i + 1]; ++index) - if (mat.col_ind_l[index] == block_j) return &mat.l[index * nVar * nEqn]; - return nullptr; - } - for (auto index = mat.row_ptr_u[block_i]; index < mat.row_ptr_u[block_i + 1]; ++index) - if (mat.col_ind_u[index] == block_j) return &mat.u[index * nVar * nEqn]; +/*! + * \brief Get a pointer to the start of block "ij" + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \return Pointer to location in memory where the block starts. + */ +FORCEINLINE const ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) const { + if (block_i == block_j) return &mat.d[block_i * nVar * nEqn]; + if (block_j < block_i) { + for (auto index = mat.row_ptr_l[block_i]; index < mat.row_ptr_l[block_i + 1]; ++index) + if (mat.col_ind_l[index] == block_j) return &mat.l[index * nVar * nEqn]; return nullptr; } + for (auto index = mat.row_ptr_u[block_i]; index < mat.row_ptr_u[block_i + 1]; ++index) + if (mat.col_ind_u[index] == block_j) return &mat.u[index * nVar * nEqn]; + return nullptr; +} - /*! - * \brief Get a pointer to the start of block "ij", non-const version. - */ - FORCEINLINE ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) { - const CSysMatrix& const_this = *this; - return const_cast(const_this.GetBlock(block_i, block_j)); - } +/*! + * \brief Get a pointer to the start of block "ij", non-const version. + */ +FORCEINLINE ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) { + const CSysMatrix& const_this = *this; + return const_cast(const_this.GetBlock(block_i, block_j)); +} - /*! - * \brief Read-only view of block (block_i, block_j). In Q_LU_SGS mode values are decoded - * on access inside CBlockView::operator()(i,j); no temporary copy is made. - * \return A CBlockView that evaluates to false if the block is absent. - */ - FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) const { +/*! + * \brief Read-only view of block (block_i, block_j). In Q_LU_SGS mode values are decoded + * on access inside CBlockView::operator()(i,j); no temporary copy is made. + * \return A CBlockView that evaluates to false if the block is absent. + */ +FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) const { #define GET_BLOCK_VIEW_IMPL \ if (!quantized_mode || block_i == block_j) { \ return {GetBlock(block_i, block_j), nullptr, nullptr, nVar}; \ @@ -698,524 +692,524 @@ class CSysMatrix { if (mat.col_ind_u[k] == block_j) return {nullptr, &q_scale_u[k * nVar], &q_blocks_u[k * nVar * nVar], nVar}; \ } \ return {} - GET_BLOCK_VIEW_IMPL; - } + GET_BLOCK_VIEW_IMPL; +} - /*! - * \overload Non const version of GetBlockView. - */ - FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) { - GET_BLOCK_VIEW_IMPL; +/*! + * \overload Non const version of GetBlockView. + */ +FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) { + GET_BLOCK_VIEW_IMPL; #undef GET_BLOCK_VIEW_IMPL - } - - /*! - * \brief Set the value of a scaled block in the sparse matrix. - * \note This is an templated overload for C2Dcontainer specialization su2matrix. - * It assumes that MatrixType supports a member type Scalar and access operator(i, j). - * If the template param Overwrite is false we add to the block (bij += alpha*b). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. - */ - template - inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block(i, j)); }); - } - - /*! - * \overload val_block is a pointer instead of a matrix type. - */ - template ::value> = 0> - inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i * nEqn + j]); }); - } - - /*! - * \overload val_block is a double pointer instead of matrix type. - */ - template - inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i][j]); }); - } +} - /*! - * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. - */ - template - inline void AddBlock(unsigned long block_i, unsigned long block_j, const T& val_block, OtherType alpha = 1.0) { - SetBlock(block_i, block_j, val_block, alpha); - } +/*! + * \brief Set the value of a scaled block in the sparse matrix. + * \note This is an templated overload for C2Dcontainer specialization su2matrix. + * It assumes that MatrixType supports a member type Scalar and access operator(i, j). + * If the template param Overwrite is false we add to the block (bij += alpha*b). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to set to A(i, j). + * \param[in] alpha - Scale factor. + */ +template +inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block(i, j)); }); +} - /*! - * \brief Subtracts the specified block to the sparse matrix (see AddBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to subtract to A(i, j). - */ - template - inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const T& val_block) { - AddBlock(block_i, block_j, val_block, -1); - } +/*! + * \overload val_block is a pointer instead of a matrix type. + */ +template ::value> = 0> +inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i * nEqn + j]); }); +} - /*! - * \brief Returns the 4 blocks ii, ij, ji, jj used by "UpdateBlocks". - * \note This method assumes an FVM-type sparse pattern. - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] iPoint - Row to which we add the blocks. - * \param[in] jPoint - Row from which we subtract the blocks. - * \param[out] bii, bij, bji, bjj - Blocks of the matrix. - */ - inline void GetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, ScalarType*& bii, - ScalarType*& bij, ScalarType*& bji, ScalarType*& bjj) { - const auto blkSz = nVar * nEqn; - bii = &mat.d[iPoint * blkSz]; - bjj = &mat.d[jPoint * blkSz]; - bij = &mat.u[iEdge * blkSz]; - bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; - } +/*! + * \overload val_block is a double pointer instead of matrix type. + */ +template +inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i][j]); }); +} - /*! - * \brief Update 4 blocks ii, ij, ji, jj (add to i* sub from j*). - * \note This method assumes an FVM-type sparse pattern. - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] iPoint - Row to which we add the blocks. - * \param[in] jPoint - Row from which we subtract the blocks. - * \param[in] block_i - Adds to ii, subs from ji. - * \param[in] block_j - Adds to ij, subs from jj. - * \param[in] scale - Scale blocks during update (axpy type op). - */ - template - inline void UpdateBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, - const MatrixType& block_j, OtherType scale = 1) { - const auto blkSz = nVar * nEqn; - auto* bii = &mat.d[iPoint * blkSz]; - auto* bjj = &mat.d[jPoint * blkSz]; +/*! + * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to set to A(i, j). + * \param[in] alpha - Scale factor. + */ +template +inline void AddBlock(unsigned long block_i, unsigned long block_j, const T& val_block, OtherType alpha = 1.0) { + SetBlock(block_i, block_j, val_block, alpha); +} - unsigned long iVar, jVar, offset = 0; +/*! + * \brief Subtracts the specified block to the sparse matrix (see AddBlock). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to subtract to A(i, j). + */ +template +inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const T& val_block) { + AddBlock(block_i, block_j, val_block, -1); +} - if (quantized_mode) { - assert(OverwriteOffDiag); - /*--- Diagonal: full-precision accumulation. Off-diagonal: quantize on the fly. ---*/ - ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; - for (iVar = 0; iVar < nVar; iVar++) - for (jVar = 0; jVar < nEqn; jVar++, ++offset) { - bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); - bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); - bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); - } - QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); - const auto k_l = edge_ptr_l[iEdge]; - QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - return; - } +/*! + * \brief Returns the 4 blocks ii, ij, ji, jj used by "UpdateBlocks". + * \note This method assumes an FVM-type sparse pattern. + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] iPoint - Row to which we add the blocks. + * \param[in] jPoint - Row from which we subtract the blocks. + * \param[out] bii, bij, bji, bjj - Blocks of the matrix. + */ +inline void GetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, ScalarType*& bii, + ScalarType*& bij, ScalarType*& bji, ScalarType*& bjj) { + const auto blkSz = nVar * nEqn; + bii = &mat.d[iPoint * blkSz]; + bjj = &mat.d[jPoint * blkSz]; + bij = &mat.u[iEdge * blkSz]; + bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; +} - auto* bij = &mat.u[iEdge * blkSz]; - auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; - for (iVar = 0; iVar < nVar; iVar++) { - for (jVar = 0; jVar < nEqn; jVar++) { +/*! + * \brief Update 4 blocks ii, ij, ji, jj (add to i* sub from j*). + * \note This method assumes an FVM-type sparse pattern. + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] iPoint - Row to which we add the blocks. + * \param[in] jPoint - Row from which we subtract the blocks. + * \param[in] block_i - Adds to ii, subs from ji. + * \param[in] block_j - Adds to ij, subs from jj. + * \param[in] scale - Scale blocks during update (axpy type op). + */ +template +inline void UpdateBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, + const MatrixType& block_j, OtherType scale = 1) { + const auto blkSz = nVar * nEqn; + auto* bii = &mat.d[iPoint * blkSz]; + auto* bjj = &mat.d[jPoint * blkSz]; + + unsigned long iVar, jVar, offset = 0; + + if (quantized_mode) { + assert(OverwriteOffDiag); + /*--- Diagonal: full-precision accumulation. Off-diagonal: quantize on the fly. ---*/ + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); - if constexpr (OverwriteOffDiag) { - bij[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); - } else { - bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); - } - ++offset; + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); } - } + QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + return; } - /*! - * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of UpdateBlocks. - */ - template - inline void UpdateBlocksSub(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, - const MatrixType& block_i, const MatrixType& block_j) { - UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); - } - - /*! - * \brief SIMD version, does the update for multiple edges and points. - * \note Nothing is updated if the mask is 0. - */ - template - FORCEINLINE void SetBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, - const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { - static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); - static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); - constexpr size_t blkSz = MatTypeSIMD::StaticSize; - assert(blkSz == nVar * nEqn); - - /*--- "Transpose" the blocks, scale, and possibly convert types, - * giving the compiler the chance to vectorize all of these. ---*/ - ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; - - for (size_t i = 0; i < blkSz; ++i) { - SU2_OMP_SIMD_IF_NOT_AD - for (size_t k = 0; k < N; ++k) { - blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); - blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); + auto* bij = &mat.u[iEdge * blkSz]; + auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { + bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); + bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); + if constexpr (OverwriteOffDiag) { + bij[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } else { + bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); } + ++offset; } + } +} - /*--- Update one by one skipping if mask is 0. ---*/ - for (size_t k = 0; k < N; ++k) { - if (mask[k] == 0) continue; - - auto bii = &mat.d[iPoint[k] * blkSz]; - auto bjj = &mat.d[jPoint[k] * blkSz]; +/*! + * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of UpdateBlocks. + */ +template +inline void UpdateBlocksSub(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, + const MatrixType& block_j) { + UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); +} - if (quantized_mode) { - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bii[i] -= blk_i[k][i]; - bjj[i] -= blk_j[k][i]; - } - QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); - const auto k_l = edge_ptr_l[iEdge[k]]; - QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - } else { - auto bij = &mat.u[iEdge[k] * blkSz]; - auto bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bii[i] -= blk_i[k][i]; - bjj[i] -= blk_j[k][i]; - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; - } - } +/*! + * \brief SIMD version, does the update for multiple edges and points. + * \note Nothing is updated if the mask is 0. + */ +template +FORCEINLINE void SetBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, + const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { + static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); + static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); + constexpr size_t blkSz = MatTypeSIMD::StaticSize; + assert(blkSz == nVar * nEqn); + + /*--- "Transpose" the blocks, scale, and possibly convert types, + * giving the compiler the chance to vectorize all of these. ---*/ + ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; + + for (size_t i = 0; i < blkSz; ++i) { + SU2_OMP_SIMD_IF_NOT_AD + for (size_t k = 0; k < N; ++k) { + blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); + blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); } } - /*! - * \brief Sets 2 blocks ij and ji (add to i* sub from j*) associated with - * one edge of an FVM-type sparse pattern. - * \note The parameter Overwrite allows completely writing over the - * current values held by the matrix (true), or updating them (false). - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] block_i - Subs from ji. - * \param[in] block_j - Adds to ij. - * \param[in] scale - Scale blocks during update (axpy type op). - */ - template - inline void SetBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, - OtherType scale = 1) { - const auto blkSz = nVar * nEqn; - unsigned long iVar, jVar, offset = 0; + /*--- Update one by one skipping if mask is 0. ---*/ + for (size_t k = 0; k < N; ++k) { + if (mask[k] == 0) continue; - if (quantized_mode) { - assert(Overwrite); - ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; - for (iVar = 0; iVar < nVar; iVar++) - for (jVar = 0; jVar < nEqn; jVar++, ++offset) { - bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); - } - QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); - const auto k_l = edge_ptr_l[iEdge]; - QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - return; - } + auto bii = &mat.d[iPoint[k] * blkSz]; + auto bjj = &mat.d[jPoint[k] * blkSz]; - ScalarType* bij = &mat.u[iEdge * blkSz]; - ScalarType* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; - for (iVar = 0; iVar < nVar; iVar++) { - for (jVar = 0; jVar < nEqn; jVar++) { - bij[offset] = (Overwrite ? ScalarType(0) : bij[offset]) + PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] = (Overwrite ? ScalarType(0) : bji[offset]) - PassiveAssign(block_i[iVar][jVar] * scale); - ++offset; + if (quantized_mode) { + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + } + QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); + const auto k_l = edge_ptr_l[iEdge[k]]; + QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + } else { + auto bij = &mat.u[iEdge[k] * blkSz]; + auto bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; } } } +} - /*! - * \brief Short-hand for the "additive overwrite" version of SetBlocks. - */ - template - inline void UpdateBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, - OtherType scale = 1) { - SetBlocks(iEdge, block_i, block_j, scale); +/*! + * \brief Sets 2 blocks ij and ji (add to i* sub from j*) associated with + * one edge of an FVM-type sparse pattern. + * \note The parameter Overwrite allows completely writing over the + * current values held by the matrix (true), or updating them (false). + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] block_i - Subs from ji. + * \param[in] block_j - Adds to ij. + * \param[in] scale - Scale blocks during update (axpy type op). + */ +template +inline void SetBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, OtherType scale = 1) { + const auto blkSz = nVar * nEqn; + unsigned long iVar, jVar, offset = 0; + + if (quantized_mode) { + assert(Overwrite); + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } + QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + return; } - /*! - * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of SetBlocks. - */ - template - inline void UpdateBlocksSub(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j) { - SetBlocks(iEdge, block_i, block_j, -1); + ScalarType* bij = &mat.u[iEdge * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { + bij[offset] = (Overwrite ? ScalarType(0) : bij[offset]) + PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] = (Overwrite ? ScalarType(0) : bji[offset]) - PassiveAssign(block_i[iVar][jVar] * scale); + ++offset; + } } +} - /*! - * \brief SIMD version, does the update for multiple edges. - * \note Nothing is updated if the mask is 0. - */ - template - FORCEINLINE void SetBlocks(simd::Array iEdge, const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, - simd::Array mask = 1) { - static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); - static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); - constexpr size_t blkSz = MatTypeSIMD::StaticSize; - assert(blkSz == nVar * nEqn); - - /*--- "Transpose" the blocks, scale, and possibly convert types, - * giving the compiler the chance to vectorize all of these. ---*/ - ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; - - for (size_t i = 0; i < blkSz; ++i) { - SU2_OMP_SIMD_IF_NOT_AD - for (size_t k = 0; k < N; ++k) { - blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); - blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); - } - } +/*! + * \brief Short-hand for the "additive overwrite" version of SetBlocks. + */ +template +inline void UpdateBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, + OtherType scale = 1) { + SetBlocks(iEdge, block_i, block_j, scale); +} - /*--- Update one by one skipping if mask is 0. ---*/ - for (size_t k = 0; k < N; ++k) { - if (mask[k] == 0) continue; +/*! + * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of SetBlocks. + */ +template +inline void UpdateBlocksSub(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j) { + SetBlocks(iEdge, block_i, block_j, -1); +} - if (quantized_mode) { - QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); - const auto k_l = edge_ptr_l[iEdge[k]]; - QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - } else { - ScalarType* bij = &mat.u[iEdge[k] * blkSz]; - ScalarType* bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; - } - } +/*! + * \brief SIMD version, does the update for multiple edges. + * \note Nothing is updated if the mask is 0. + */ +template +FORCEINLINE void SetBlocks(simd::Array iEdge, const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, + simd::Array mask = 1) { + static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); + static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); + constexpr size_t blkSz = MatTypeSIMD::StaticSize; + assert(blkSz == nVar * nEqn); + + /*--- "Transpose" the blocks, scale, and possibly convert types, + * giving the compiler the chance to vectorize all of these. ---*/ + ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; + + for (size_t i = 0; i < blkSz; ++i) { + SU2_OMP_SIMD_IF_NOT_AD + for (size_t k = 0; k < N; ++k) { + blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); + blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); } } - /*! - * \brief Sets the specified block to the (i, i) subblock of the sparse matrix. - * Scales the input block by factor alpha. If the Overwrite parameter is - * false we update instead (bii += alpha*b). - * \param[in] block_i - Diagonal index. - * \param[in] val_block - Block to add to the diagonal of the matrix. - * \param[in] alpha - Scale factor. - */ - template - inline void SetBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { - auto mat_ii = &mat.d[block_i * nVar * nEqn]; - - for (auto iVar = 0ul; iVar < nVar; iVar++) - for (auto jVar = 0ul; jVar < nEqn; jVar++) { - *mat_ii = (Overwrite ? ScalarType(0) : *mat_ii) + PassiveAssign(alpha * val_block[iVar][jVar]); - ++mat_ii; + /*--- Update one by one skipping if mask is 0. ---*/ + for (size_t k = 0; k < N; ++k) { + if (mask[k] == 0) continue; + + if (quantized_mode) { + QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); + const auto k_l = edge_ptr_l[iEdge[k]]; + QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + } else { + ScalarType* bij = &mat.u[iEdge[k] * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; } + } } +} - /*! - * \brief Non overwrite version of SetBlock2Diag, also with scaling. - */ - template - inline void AddBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { - SetBlock2Diag(block_i, val_block, alpha); - } +/*! + * \brief Sets the specified block to the (i, i) subblock of the sparse matrix. + * Scales the input block by factor alpha. If the Overwrite parameter is + * false we update instead (bii += alpha*b). + * \param[in] block_i - Diagonal index. + * \param[in] val_block - Block to add to the diagonal of the matrix. + * \param[in] alpha - Scale factor. + */ +template +inline void SetBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { + auto mat_ii = &mat.d[block_i * nVar * nEqn]; + + for (auto iVar = 0ul; iVar < nVar; iVar++) + for (auto jVar = 0ul; jVar < nEqn; jVar++) { + *mat_ii = (Overwrite ? ScalarType(0) : *mat_ii) + PassiveAssign(alpha * val_block[iVar][jVar]); + ++mat_ii; + } +} - /*! - * \brief Short-hand to AddBlock2Diag with alpha = -1, i.e. subtracts from the current diagonal. - */ - template - inline void SubtractBlock2Diag(unsigned long block_i, const OtherType& val_block) { - AddBlock2Diag(block_i, val_block, -1.0); - } +/*! + * \brief Non overwrite version of SetBlock2Diag, also with scaling. + */ +template +inline void AddBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { + SetBlock2Diag(block_i, val_block, alpha); +} - /*! - * \brief Adds the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). - */ - template - inline void AddVal2Diag(unsigned long block_i, OtherType val_matrix) { - auto d = &mat.d[block_i * nVar * nVar]; - for (auto iVar = 0ul; iVar < nVar; iVar++) d[iVar * (nVar + 1)] += PassiveAssign(val_matrix); - } +/*! + * \brief Short-hand to AddBlock2Diag with alpha = -1, i.e. subtracts from the current diagonal. + */ +template +inline void SubtractBlock2Diag(unsigned long block_i, const OtherType& val_block) { + AddBlock2Diag(block_i, val_block, -1.0); +} - /*! - * \brief Adds the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] iVar - Variable index. - * \param[in] val - Value to add to the diagonal elements of A(i, i). - */ - template - inline void AddVal2Diag(unsigned long block_i, unsigned long iVar, OtherType val) { - mat.d[block_i * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); - } +/*! + * \brief Adds the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). + */ +template +inline void AddVal2Diag(unsigned long block_i, OtherType val_matrix) { + auto d = &mat.d[block_i * nVar * nVar]; + for (auto iVar = 0ul; iVar < nVar; iVar++) d[iVar * (nVar + 1)] += PassiveAssign(val_matrix); +} - /*! - * \brief Sets the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). - */ - template - inline void SetVal2Diag(unsigned long block_i, OtherType val_matrix) { - /*--- Clear entire block before setting its diagonal. ---*/ - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nVar; iVar++) mat.d[block_i * nVar * nVar + iVar] = 0.0; +/*! + * \brief Adds the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] iVar - Variable index. + * \param[in] val - Value to add to the diagonal elements of A(i, i). + */ +template +inline void AddVal2Diag(unsigned long block_i, unsigned long iVar, OtherType val) { + mat.d[block_i * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); +} - AddVal2Diag(block_i, val_matrix); - } +/*! + * \brief Sets the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). + */ +template +inline void SetVal2Diag(unsigned long block_i, OtherType val_matrix) { + /*--- Clear entire block before setting its diagonal. ---*/ + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nVar; iVar++) mat.d[block_i * nVar * nVar + iVar] = 0.0; - /*! - * \brief Deletes the values of a row of the sparse matrix. - * \param[in] block_i - Index of the block. - * \param[in] row - Row within the block. - */ - void DeleteValsRowi(unsigned long block_i, unsigned long row); + AddVal2Diag(block_i, val_matrix); +} - /*! - * \brief Modifies this matrix (A) and a rhs vector (b) such that (A^-1 * b)_i = x_i. - * \param[in] node_i - Index of the node for which to enforce the solution of all DOF's. - * \param[in] x_i - Values to enforce (nVar sized). - * \param[in,out] b - The rhs vector (b := b - A_{*,i} * x_i; b_i = x_i). - */ - template - void EnforceSolutionAtNode(unsigned long node_i, const OtherType* x_i, CSysVector& b); +/*! + * \brief Deletes the values of a row of the sparse matrix. + * \param[in] block_i - Index of the block. + * \param[in] row - Row within the block. + */ +void DeleteValsRowi(unsigned long block_i, unsigned long row); - /*! - * \brief Similar to EnforceSolutionAtNode, but for 0 projection in a given direction. - */ - template - void EnforceZeroProjection(unsigned long node_i, const OtherType* n, CSysVector& b); +/*! + * \brief Modifies this matrix (A) and a rhs vector (b) such that (A^-1 * b)_i = x_i. + * \param[in] node_i - Index of the node for which to enforce the solution of all DOF's. + * \param[in] x_i - Values to enforce (nVar sized). + * \param[in,out] b - The rhs vector (b := b - A_{*,i} * x_i; b_i = x_i). + */ +template +void EnforceSolutionAtNode(unsigned long node_i, const OtherType* x_i, CSysVector& b); - /*! - * \brief Sets the diagonal entries of the matrix as the sum of the blocks in the corresponding column. - */ - void SetDiagonalAsColumnSum(); +/*! + * \brief Similar to EnforceSolutionAtNode, but for 0 projection in a given direction. + */ +template +void EnforceZeroProjection(unsigned long node_i, const OtherType* n, CSysVector& b); - /*! - * \brief Transposes the matrix, any preconditioner that was computed may be invalid. - */ - void TransposeInPlace(); +/*! + * \brief Sets the diagonal entries of the matrix as the sum of the blocks in the corresponding column. + */ +void SetDiagonalAsColumnSum(); - /*! - * \brief Add a scaled sparse matrix to "this" (axpy-type operation, A = A+alpha*B). - * \note Matrices must have the same sparse pattern. - * \param[in] alpha - The scaling constant. - * \param[in] B - Matrix being. - */ - void MatrixMatrixAddition(ScalarType alpha, const CSysMatrix& B); +/*! + * \brief Transposes the matrix, any preconditioner that was computed may be invalid. + */ +void TransposeInPlace(); - /*! - * \brief Performs the product of a sparse matrix by a CSysVector. - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ - void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +/*! + * \brief Add a scaled sparse matrix to "this" (axpy-type operation, A = A+alpha*B). + * \note Matrices must have the same sparse pattern. + * \param[in] alpha - The scaling constant. + * \param[in] B - Matrix being. + */ +void MatrixMatrixAddition(ScalarType alpha, const CSysMatrix& B); - /*! - * \brief Build the Jacobi preconditioner. - */ - void BuildJacobiPreconditioner(); +/*! + * \brief Performs the product of a sparse matrix by a CSysVector. + * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[out] prod - Result of the product. + */ +void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ - void ComputeJacobiPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +/*! + * \brief Build the Jacobi preconditioner. + */ +void BuildJacobiPreconditioner(); - /*! - * \brief Build the ILU preconditioner. - */ - void BuildILUPreconditioner(); +/*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ +void ComputeJacobiPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ - void ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +/*! + * \brief Build the ILU preconditioner. + */ +void BuildILUPreconditioner(); - /*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ - void ComputeLU_SGSPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; +/*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ +void ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Build the Linelet preconditioner. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ - void BuildLineletPreconditioner(const CGeometry* geometry, const CConfig* config); +/*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + */ +void ComputeLU_SGSPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ - void ComputeLineletPreconditioner(const CSysVector& vec, CSysVector& prod, - CGeometry* geometry, const CConfig* config) const; +/*! + * \brief Build the Linelet preconditioner. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ +void BuildLineletPreconditioner(const CGeometry* geometry, const CConfig* config); - /*! - * \brief Compute the linear residual. - * \param[in] sol - Solution (x). - * \param[in] f - Right hand side (b). - * \param[out] res - Residual (Ax-b). - */ - void ComputeResidual(const CSysVector& sol, const CSysVector& f, - CSysVector& res) const; +/*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + */ +void ComputeLineletPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; - /*! - * \brief Factorize matrix using PaStiX. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[in] kind_fact - Type of factorization. - */ - void BuildPastixPreconditioner(CGeometry* geometry, const CConfig* config, unsigned short kind_fact); +/*! + * \brief Compute the linear residual. + * \param[in] sol - Solution (x). + * \param[in] f - Right hand side (b). + * \param[out] res - Residual (Ax-b). + */ +void ComputeResidual(const CSysVector& sol, const CSysVector& f, + CSysVector& res) const; - /*! - * \brief Apply the PaStiX factorization to CSysVec. - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product M*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ - void ComputePastixPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; -}; +/*! + * \brief Factorize matrix using PaStiX. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[in] kind_fact - Type of factorization. + */ +void BuildPastixPreconditioner(CGeometry* geometry, const CConfig* config, unsigned short kind_fact); + +/*! + * \brief Apply the PaStiX factorization to CSysVec. + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product M*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ +void ComputePastixPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; +} +; diff --git a/Common/src/CConfig.cpp b/Common/src/CConfig.cpp index e1bb41bc235..67850acae3d 100644 --- a/Common/src/CConfig.cpp +++ b/Common/src/CConfig.cpp @@ -1972,10 +1972,9 @@ void CConfig::SetConfig_Options() { addUnsignedShortOption("LINEAR_SOLVER_ILU_FILL_IN", Linear_Solver_ILU_n, 0); /* DESCRIPTION: Use level scheduling for OMP parallelization of the ILU preconditioner */ addBoolOption("LINEAR_SOLVER_ILU_LEVEL_SCHEDULING", Linear_Solver_ILU_levels, false); - /* DESCRIPTION: Colored Gauss-Seidel sweeps used to build the ILU preconditioner on the GPU */ - addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_SWEEPS", Linear_Solver_ILU_GPU_Sweeps, 1); - /* DESCRIPTION: Colored Jacobi sweeps used for the GPU ILU forward triangular solve */ - addUnsignedShortOption("LINEAR_SOLVER_ILU_GPU_FWD_SWEEPS", Linear_Solver_ILU_GPU_Fwd_Sweeps, 3); + /* DESCRIPTION: Colored-iterative sweep counts for the GPU ILU preconditioner: (build, forward, backward) */ + addUShortArrayOption("LINEAR_SOLVER_ILU_GPU_SWEEPS", Linear_Solver_ILU_GPU_Sweeps.size(), false, + Linear_Solver_ILU_GPU_Sweeps.data()); /* DESCRIPTION: Maximum number of iterations of the linear solver for the implicit formulation */ addUnsignedLongOption("LINEAR_SOLVER_RESTART_FREQUENCY", Linear_Solver_Restart_Frequency, 10); /* DESCRIPTION: Number of vectors used for deflated restarts */ @@ -4143,15 +4142,13 @@ void CConfig::SetPostprocessing(SU2_COMPONENT val_software, unsigned short val_i } } - if (Linear_Solver_ILU_GPU_Sweeps == 0) { - SU2_MPI::Error("LINEAR_SOLVER_ILU_GPU_SWEEPS must be at least 1, 0 sweeps never factorizes the matrix.", - CURRENT_FUNCTION); - } - - if (Linear_Solver_ILU_GPU_Fwd_Sweeps == 0) { - SU2_MPI::Error( - "LINEAR_SOLVER_ILU_GPU_FWD_SWEEPS must be at least 1, 0 sweeps never solves the forward triangular system.", - CURRENT_FUNCTION); + for (auto n : Linear_Solver_ILU_GPU_Sweeps) { + if (n == 0) { + SU2_MPI::Error( + "LINEAR_SOLVER_ILU_GPU_SWEEPS entries must all be at least 1 (build, forward, backward); " + "0 sweeps never factorizes/solves.", + CURRENT_FUNCTION); + } } Radiation = (Kind_Radiation != RADIATION_MODEL::NONE); diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index e58ae302281..72d0047b00f 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -139,8 +139,8 @@ CSysMatrix::~CSysMatrix() { GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_l); GPUMemoryAllocation::gpu_free(gpu_ilu.row_ptr_u); GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_u); - GPUMemoryAllocation::gpu_free(d_ilu_level_idx); GPUMemoryAllocation::gpu_free(d_ilu_color_idx); + GPUMemoryAllocation::gpu_free(d_ilu_backward_rhs); if (ilu_build_graph_exec != nullptr) cudaGraphExecDestroy(ilu_build_graph_exec); if (ilu_apply_graph_exec != nullptr) cudaGraphExecDestroy(ilu_apply_graph_exec); if (ilu_stream != nullptr) cudaStreamDestroy(ilu_stream); @@ -277,8 +277,7 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi if (ilu_needed) { ilu_fill_in = config->GetLinear_Solver_ILU_n(); - ilu_gpu_color_sweeps = config->GetLinear_Solver_ILU_GPU_Sweeps(); - ilu_gpu_fwd_sweeps = config->GetLinear_Solver_ILU_GPU_Fwd_Sweeps(); + ilu_gpu_sweeps = config->GetLinear_Solver_ILU_GPU_Sweeps(); const auto& pat_ilu = geometry->GetSparsePattern(type, ilu_fill_in); ilu.row_ptr_l = pat_ilu.l.outerPtr(); @@ -355,20 +354,8 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi gpu_ilu.row_ptr_u = GPUMemoryAllocation::gpu_alloc_cpy(ilu.row_ptr_u, (nPointDomain + 1) * sizeof(su2uint)); gpu_ilu.col_ind_u = GPUMemoryAllocation::gpu_alloc_cpy(ilu.col_ind_u, ilu.nnz_u * sizeof(su2uint)); - /*--- Flatten the level structure, the index type differs from the one of the pattern. ---*/ - std::vector level_idx; - level_idx.reserve(nPointDomain); - ilu_level_ptr.clear(); - ilu_level_ptr.push_back(0); - for (auto level = 0ul; level < levels_ilu.getOuterSize(); ++level) { - for (auto k = 0ul; k < levels_ilu.getNumNonZeros(level); ++k) { - level_idx.push_back(static_cast(levels_ilu.getInnerIdx(level, k))); - } - ilu_level_ptr.push_back(static_cast(level_idx.size())); - } - d_ilu_level_idx = GPUMemoryAllocation::gpu_alloc_cpy(level_idx.data(), level_idx.size() * sizeof(su2uint)); - - /*--- Flatten the coloring the same way. ---*/ + /*--- Flatten the coloring, the index type differs from the one of the pattern. It drives + * the factorization and both triangular solves on the device. ---*/ std::vector color_idx; color_idx.reserve(nPointDomain); ilu_color_ptr.clear(); @@ -380,6 +367,8 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi ilu_color_ptr.push_back(static_cast(color_idx.size())); } d_ilu_color_idx = GPUMemoryAllocation::gpu_alloc_cpy(color_idx.data(), color_idx.size() * sizeof(su2uint)); + + d_ilu_backward_rhs = GPUMemoryAllocation::gpu_alloc(nPointDomain * nVar * sizeof(ScalarType)); } /*--- Thread parallel initialization. ---*/ diff --git a/Common/src/linear_algebra/CSysMatrixGPU.cu b/Common/src/linear_algebra/CSysMatrixGPU.cu index 53774aa6e4a..601ae385cf4 100644 --- a/Common/src/linear_algebra/CSysMatrixGPU.cu +++ b/Common/src/linear_algebra/CSysMatrixGPU.cu @@ -52,11 +52,14 @@ __global__ void ApplyJacobiPreconditionerKernel(const ScalarType* invM, const Sc } } -/*--- ILU. The factorization and both triangular solves are level scheduled: the rows of a - * level are independent of each other and only depend on rows of previous levels, so each - * level is one kernel launch and the launch boundaries provide the synchronization. The rows - * of a level are scattered through the matrix, hence the indirection through the level table. - * Throughout, one CUDA block works on one row. ---*/ +/*--- ILU. The factorization and both triangular solves are all scheduled by coloring: colors + * are true independent sets of the ILU dependency graph (no two same-colored rows depend on each + * other in either direction), so a color's rows can be processed with zero races in one kernel + * launch, but since a color is wider and less ordered than a level, one pass over all colors is + * only an approximation, not an exact result — several sweeps (repeated passes) are needed to + * converge to the same fixed point an exact level-scheduled algorithm would give in one pass. The + * rows of a color are scattered through the matrix, hence the indirection through the color + * table. Throughout, one CUDA block works on one row. ---*/ /*! * \brief The pointers of an LDU-partitioned matrix, all in device memory. This mirrors the @@ -218,23 +221,29 @@ __global__ void IluFactorColorKernel(const su2uint* __restrict__ color_idx, unsi } /*! - * \brief Forward substitution for the rows of one level, (L+I).prod = vec. - * \note One thread per block entry (like the factorization kernel), so the inner dot product - * over a neighbor block is spread across nVar threads instead of done serially by one. - * Unlike the factorization, forward/backward only ever *read* already-finalized prod - * values (no row-to-row write-through during the loop), so each thread can accumulate - * its own (iVar,jVar) partial product across every neighbor with no synchronization at - * all, and only the final nVar-way reduction (summing over jVar for each iVar) needs one - * __syncthreads() — not one per neighbor. Grid: one block per row of the level, + * \brief One colored-Jacobi sweep of forward substitution for the rows of one color, + * (L+I).prod = vec. + * \note Same idea as IluFactorColorKernel/IluBackwardColorKernel: colors are true independent + * sets, so a color can be processed with zero races, but an L-neighbor in a different, + * not-yet-processed color this sweep is read at whatever value it currently holds (a + * previous sweep's, or the zero-initialized buffer on the very first sweep). Repeated full + * passes over all colors (see ComputeILUPreconditionerGPU) converge this to the same + * result the exact level-scheduled solve would give, because the update is a genuine + * residual reduction, not an incremental accumulation. One thread per block entry (like + * the factorization kernel), so the inner dot product over a neighbor block is spread + * across nVar threads instead of done serially by one; each thread accumulates its own + * (iVar,jVar) partial product across every neighbor with no synchronization at all, and + * only the final nVar-way reduction (summing over jVar for each iVar) needs one + * __syncthreads() — not one per neighbor. Grid: one block per row of the color, * blockDim.x == nVar*nVar. Dynamic shared memory: nVar*nVar scalars. */ template -__global__ void IluForwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nVar, DeviceLDU M, +__global__ void IluForwardColorKernel(const su2uint* __restrict__ color_idx, unsigned long color_begin, + unsigned long color_size, unsigned long nVar, DeviceLDU M, const ScalarType* __restrict__ vec, ScalarType* __restrict__ prod) { - if (blockIdx.x >= level_size) return; + if (blockIdx.x >= color_size) return; - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const unsigned long iRow = color_idx[color_begin + blockIdx.x]; const unsigned long tid = threadIdx.x; const auto iVar = tid / nVar, jVar = tid % nVar; @@ -259,21 +268,32 @@ __global__ void IluForwardLevelKernel(const su2uint* __restrict__ level_idx, uns } /*! - * \brief Backward substitution for the rows of one level, U.prod = prod. - * \note Same idea as IluForwardLevelKernel: each thread accumulates its own (iVar,jVar) - * partial product across every neighbor with no synchronization, then one - * __syncthreads() to reduce over jVar and get the elimination result per iVar, then a - * second __syncthreads() before the diagonal multiply (which needs every iVar's result). - * Grid: one block per row of the level, blockDim.x == nVar*nVar. Dynamic shared memory: + * \brief One colored-Jacobi sweep of backward substitution for the rows of one color, + * U.prod = rhs. + * \note The right-hand side is read from a separate fixed array (\p rhs), not from \p prod + * itself: a color visits every row once per sweep, so after the first sweep prod[iRow] + * holds a solution *estimate*, not the right-hand side. Reading the right-hand side back + * out of prod past the first sweep would silently solve a different (and divergent) + * recurrence — see CSysMatrix::d_ilu_backward_rhs. Colors are true independent sets, so a color is race-free + * within a sweep; a U-neighbor in a different, not-yet-processed color this sweep is read + * at whatever value it currently holds (a previous sweep's, or the forward-solve result on + * the very first sweep, since prod is not reset before backward starts). Repeated full + * passes over all colors converge to the same result the exact level-scheduled solve would + * give (validated on the host: same fixed point, ~4.4x error reduction per sweep). Each + * thread accumulates its own (iVar,jVar) partial product across every neighbor with no + * synchronization, then one __syncthreads() to reduce over jVar, then a second + * __syncthreads() before the diagonal multiply (which needs every iVar's result). Grid: + * one block per row of the color, blockDim.x == nVar*nVar. Dynamic shared memory: * nVar*nVar + nVar scalars. */ template -__global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, unsigned long level_begin, - unsigned long level_size, unsigned long nRows, unsigned long nVar, - DeviceLDU M, ScalarType* __restrict__ prod) { - if (blockIdx.x >= level_size) return; +__global__ void IluBackwardColorKernel(const su2uint* __restrict__ color_idx, unsigned long color_begin, + unsigned long color_size, unsigned long nRows, unsigned long nVar, + DeviceLDU M, const ScalarType* __restrict__ rhs, + ScalarType* __restrict__ prod) { + if (blockIdx.x >= color_size) return; - const unsigned long iRow = level_idx[level_begin + blockIdx.x]; + const unsigned long iRow = color_idx[color_begin + blockIdx.x]; const auto blockSize = nVar * nVar; const unsigned long tid = threadIdx.x; const auto iVar = tid / nVar, jVar = tid % nVar; @@ -283,7 +303,6 @@ __global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, un auto* aux = partial + blockSize; ScalarType acc = 0; - /*--- The columns of U are rows of later levels, already updated by this sweep. ---*/ for (auto ku = M.row_ptr_u[iRow]; ku < M.row_ptr_u[iRow + 1]; ++ku) { const unsigned long jPoint = M.col_ind_u[ku]; if (jPoint >= nRows) break; @@ -294,7 +313,7 @@ __global__ void IluBackwardLevelKernel(const su2uint* __restrict__ level_idx, un __syncthreads(); if (jVar == 0) { - ScalarType sum = prod[iRow * nVar + iVar]; + ScalarType sum = rhs[iRow * nVar + iVar]; for (auto j = 0ul; j < nVar; ++j) sum -= partial[iVar * nVar + j]; /*--- The diagonal blocks are stored inverted by the factorization. ---*/ aux[iVar] = sum; @@ -428,7 +447,7 @@ void CSysMatrix::BuildILUPreconditionerGPU() { * change execution order relative to the rest of the (single-stream) solver. ---*/ if (ilu_stream == nullptr) gpuErrChk(cudaStreamCreate(&ilu_stream)); - /*--- The launch sequence (ilu_gpu_color_sweeps passes over all colors) is identical on every + /*--- The launch sequence (ilu_gpu_build_sweeps passes over all colors) is identical on every * call: the grid and block sizes only depend on the (fixed) sparsity pattern/coloring and the * device pointers are fixed members, allocated once. Capture it into a CUDA graph the first * time and replay that from then on, which removes the per-launch host-side overhead without @@ -443,7 +462,7 @@ void CSysMatrix::BuildILUPreconditionerGPU() { cudaGraph_t graph; gpuErrChk(cudaStreamBeginCapture(ilu_stream, cudaStreamCaptureModeThreadLocal)); - for (unsigned short sweep = 0; sweep < ilu_gpu_color_sweeps; ++sweep) { + for (unsigned short sweep = 0; sweep < ilu_gpu_sweeps[0]; ++sweep) { for (auto color = 0ul; color + 1 < ilu_color_ptr.size(); ++color) { const auto begin = ilu_color_ptr[color]; const auto size = ilu_color_ptr[color + 1] - begin; @@ -479,7 +498,7 @@ void CSysMatrix::ComputeILUPreconditionerGPU(const CSysVector::ComputeILUPreconditionerGPU(const CSysVector::ComputeILUPreconditionerGPU(const CSysVector + IluForwardColorKernel <<>>(d_ilu_color_idx, begin, size, nVar, M, d_vec, d_prod); } } - /*--- Backward substitution, levels in decreasing order. ---*/ - for (auto level = nLevels; level > 0;) { - --level; // unsigned type - const auto begin = ilu_level_ptr[level]; - const auto size = ilu_level_ptr[level + 1] - begin; - if (size == 0) continue; - IluBackwardLevelKernel<<>>( - d_ilu_level_idx, begin, size, nPointDomain, nVar, M, d_prod); + /*--- Backward substitution: also colored-iterative, see IluBackwardColorKernel for why the + * right-hand side must be copied out to a fixed buffer first (unlike forward, which already + * had one in d_vec). Starting prod as its own initial guess (the forward-solve result) is as + * good a starting point as zero and saves a second memset. ---*/ + gpuErrChk(cudaMemcpyAsync(d_ilu_backward_rhs, d_prod, nPointDomain * nVar * sizeof(ScalarType), + cudaMemcpyDeviceToDevice, ilu_stream)); + for (unsigned short sweep = 0; sweep < ilu_gpu_sweeps[2]; ++sweep) { + for (auto color = 0ul; color < nColors; ++color) { + const auto begin = ilu_color_ptr[color]; + const auto size = ilu_color_ptr[color + 1] - begin; + if (size == 0) continue; + IluBackwardColorKernel<<>>( + d_ilu_color_idx, begin, size, nPointDomain, nVar, M, d_ilu_backward_rhs, d_prod); + } } gpuErrChk(cudaStreamEndCapture(ilu_stream, &graph)); diff --git a/TestCases/euler/oneram6/inv_ONERAM6.cfg b/TestCases/euler/oneram6/inv_ONERAM6.cfg index a0351dbfbb5..e3a24291f40 100644 --- a/TestCases/euler/oneram6/inv_ONERAM6.cfg +++ b/TestCases/euler/oneram6/inv_ONERAM6.cfg @@ -56,6 +56,7 @@ RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) ITER= 50 LINEAR_SOLVER= FGMRES LINEAR_SOLVER_PREC= ILU +LINEAR_SOLVER_ILU_GPU_SWEEPS= (1, 2, 2) LINEAR_SOLVER_ERROR= 1E-3 LINEAR_SOLVER_ITER= 10 ENABLE_CUDA= YES diff --git a/config_template.cfg b/config_template.cfg index ab95850e5bf..b12ed3ccd0f 100644 --- a/config_template.cfg +++ b/config_template.cfg @@ -1661,12 +1661,14 @@ DISCADJ_LIN_PREC= ILU % Linear solver ILU preconditioner fill-in level (0 by default) LINEAR_SOLVER_ILU_FILL_IN= 0 % -% Colored Gauss-Seidel sweeps used to build the ILU preconditioner on the GPU (1 by default). -% The factorization is not reset between calls, so with the matrix changing little between -% outer/pseudo-time iterations, each call refines the previous one's result instead of -% reconverging from scratch. For cases that do few outer iterations (e.g. elasticity problems) -% it may useful to increase this number to 3-5. -LINEAR_SOLVER_ILU_GPU_SWEEPS= 1 +% Colored-iterative sweep counts for the GPU ILU preconditioner: (build, forward, backward), +% default (1, 2, 2). The build (factorization) is not reset between calls, so with the matrix +% changing little between outer/pseudo-time iterations, each call refines the previous one's +% result instead of reconverging from scratch, for cases that do few outer iterations (e.g. +% elasticity problems) it may be useful to increase it to 3-5. The forward/backward triangular +% solves have no such warm start (a new right-hand side every Krylov iteration), so more than +% one sweep is necessary, harder problems (e.g. higher CFL) may benefit from more than 3 sweeps. +LINEAR_SOLVER_ILU_GPU_SWEEPS= (1, 2, 2) % % Minimum error of the linear solver for implicit formulations LINEAR_SOLVER_ERROR= 1E-6 From 24a60ba96618fa7e4151701e886ffae68ef93dc9 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 22:17:10 -0700 Subject: [PATCH 11/14] revert test --- TestCases/euler/oneram6/inv_ONERAM6.cfg | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/TestCases/euler/oneram6/inv_ONERAM6.cfg b/TestCases/euler/oneram6/inv_ONERAM6.cfg index e3a24291f40..6517b957871 100644 --- a/TestCases/euler/oneram6/inv_ONERAM6.cfg +++ b/TestCases/euler/oneram6/inv_ONERAM6.cfg @@ -53,13 +53,11 @@ CFL_NUMBER= 5.0 CFL_ADAPT= NO CFL_ADAPT_PARAM= ( 1.5, 0.5, 1.0, 100.0 ) RK_ALPHA_COEFF= ( 0.66667, 0.66667, 1.000000 ) -ITER= 50 +ITER= 99999 LINEAR_SOLVER= FGMRES -LINEAR_SOLVER_PREC= ILU -LINEAR_SOLVER_ILU_GPU_SWEEPS= (1, 2, 2) -LINEAR_SOLVER_ERROR= 1E-3 -LINEAR_SOLVER_ITER= 10 -ENABLE_CUDA= YES +LINEAR_SOLVER_PREC= LU_SGS +LINEAR_SOLVER_ERROR= 1E-6 +LINEAR_SOLVER_ITER= 2 % ----------------------- SLOPE LIMITER DEFINITION ----------------------------% % @@ -70,7 +68,7 @@ SENS_REMOVE_SHARP= YES % -------------------------- MULTIGRID PARAMETERS -----------------------------% % -MGLEVEL= 0 +MGLEVEL= 3 MGCYCLE= W_CYCLE MG_PRE_SMOOTH= ( 1, 2, 3, 3 ) MG_POST_SMOOTH= ( 0, 0, 0, 0 ) @@ -93,7 +91,7 @@ TIME_DISCRE_ADJFLOW= EULER_IMPLICIT % --------------------------- CONVERGENCE PARAMETERS --------------------------& % -CONV_RESIDUAL_MINVAL= -14 +CONV_RESIDUAL_MINVAL= -12 CONV_STARTITER= 25 CONV_CAUCHY_ELEMS= 100 CONV_CAUCHY_EPS= 1E-10 @@ -115,7 +113,7 @@ GRAD_OBJFUNC_FILENAME= of_grad SURFACE_FILENAME= surface_flow SURFACE_ADJ_FILENAME= surface_adjoint OUTPUT_WRT_FREQ= 100 -SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG, LINSOL) +SCREEN_OUTPUT = (INNER_ITER, RMS_DENSITY, RMS_ENERGY, LIFT, DRAG) OUTPUT_FILES= (RESTART_ASCII, CGNS, SURFACE_CGNS) % ----------------------- DESIGN VARIABLE PARAMETERS --------------------------% From b0b21485cc09d96de10a3442892d3228899812c6 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 22:19:38 -0700 Subject: [PATCH 12/14] fix --- Common/include/CConfig.hpp | 2 +- Common/include/linear_algebra/CSysMatrix.hpp | 1567 +++++++++--------- 2 files changed, 784 insertions(+), 785 deletions(-) diff --git a/Common/include/CConfig.hpp b/Common/include/CConfig.hpp index fc0dac4edb2..446a158000d 100644 --- a/Common/include/CConfig.hpp +++ b/Common/include/CConfig.hpp @@ -648,7 +648,7 @@ class CConfig { bool Linear_Solver_ILU_levels; /*!< \brief Use level scheduling for OMP parallelization of ILU. */ /*!< \brief Colored-iterative sweep counts for the GPU ILU preconditioner: [0] builds the * factorization (Gauss-Seidel), [1]/[2] apply it (Jacobi, forward/backward triangular solve). */ - array Linear_Solver_ILU_GPU_Sweeps{(1, 2, 2}}; + array Linear_Solver_ILU_GPU_Sweeps{{1, 2, 2}}; su2double SemiSpan; /*!< \brief Wing Semi span. */ su2double MSW_Alpha; /*!< \brief Coefficient for blending states in the MSW scheme. */ su2double Roe_Kappa; /*!< \brief Relaxation of the Roe scheme. */ diff --git a/Common/include/linear_algebra/CSysMatrix.hpp b/Common/include/linear_algebra/CSysMatrix.hpp index a306e2ac5dd..cf25c264e24 100644 --- a/Common/include/linear_algebra/CSysMatrix.hpp +++ b/Common/include/linear_algebra/CSysMatrix.hpp @@ -304,382 +304,381 @@ class CSysMatrix { /*!< \brief Number of colored Gauss-Seidel sweeps used to build and apply the ILU factorization * on the device, see IluFactorColorKernel. Fixed per solve (not adaptive) so the result is * reproducible; set from config in Initialize(). */ - array ilu_gpu_sweeps {(1, 2, 2 - } -}; - -/*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent - * set (no dependency between same-colored rows in either direction), so far fewer, wider - * colors are needed than levels, but a color launch is only exact as one step of an iterative - * refinement (see BuildILUPreconditionerGPU and ComputeILUPreconditionerGPU), not a single - * pass — this does not change the elimination order/pattern, so the factorization and both - * triangular solves converge to the exact same result levels_ilu would give, just reached by - * iterating instead of substituting. ---*/ -vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ -su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ - -/*!< \brief Fixed right-hand side of the backward triangular solve (the forward-solve result), - * kept separate from the evolving prod buffer. A color visits every row once per sweep, so - * after the first sweep prod[iRow] holds a solution *estimate*, not the right-hand side - * anymore; reading the right-hand side back out of prod past the first sweep would silently - * solve the wrong equation. Size nPointDomain*nVar, allocated once in Initialize(). */ -ScalarType* d_ilu_backward_rhs = nullptr; - -/*--- The per-color kernel launch sequence (ilu_gpu_sweeps[0] passes for the factorization, - * ilu_gpu_sweeps[1] / ilu_gpu_sweeps[2] passes for the forward/backward triangular solves) - * is identical on every call: same grid/block sizes, same device pointers (all fixed members, - * allocated once). It is captured once into a CUDA graph and replayed, which removes - * host-side launch overhead without changing the parallelization (unlike a persistent - * cooperative-groups kernel, this does not cap per-color parallelism to the occupancy-resident - * block count). ---*/ -/*--- Types are forward-declared as opaque structs (matching the real cudaGraphExec_t / - * cudaStream_t typedefs) so this header does not need to include the CUDA runtime. ---*/ -mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; -mutable struct CUgraphExec_st* ilu_apply_graph_exec = nullptr; -mutable const ScalarType* ilu_apply_graph_vec = nullptr; /*!< \brief Pointers the apply graph - * was captured with, to detect when - * it must be recaptured. */ -mutable ScalarType* ilu_apply_graph_prod = nullptr; -/*--- The legacy default stream cannot be captured into a graph, so the ILU graphs are - * captured and replayed on this dedicated stream instead; every launch on it is followed by - * a sync back to the host before control returns to the rest of the (single-stream) solver, - * so this does not change execution order relative to everything else, which stays on the - * default stream. ---*/ -mutable struct CUstream_st* ilu_stream = nullptr; - -ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ - -/*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ -mutable vector> - LineletUpper; /*!< \brief Pointers to the upper blocks of the tri-diag system (working memory). */ -mutable vector> - LineletInvDiag; /*!< \brief Inverse of the diagonal blocks of the tri-diag system (working memory). */ -mutable vector> - LineletVector; /*!< \brief Solution and RHS of the tri-diag system (working memory). */ + array ilu_gpu_sweeps{{1, 2, 2}}; + + /*--- Coloring of the ILU dependency graph: unlike levels_ilu, a color is a true independent + * set (no dependency between same-colored rows in either direction), so far fewer, wider + * colors are needed than levels, but a color launch is only exact as one step of an iterative + * refinement (see BuildILUPreconditionerGPU and ComputeILUPreconditionerGPU), not a single + * pass — this does not change the elimination order/pattern, so the factorization and both + * triangular solves converge to the exact same result levels_ilu would give, just reached by + * iterating instead of substituting. ---*/ + vector ilu_color_ptr; /*!< \brief Start of each color in d_ilu_color_idx, size nColors+1. */ + su2uint* d_ilu_color_idx = nullptr; /*!< \brief Row indices, grouped by color. */ + + /*!< \brief Fixed right-hand side of the backward triangular solve (the forward-solve result), + * kept separate from the evolving prod buffer. A color visits every row once per sweep, so + * after the first sweep prod[iRow] holds a solution *estimate*, not the right-hand side + * anymore; reading the right-hand side back out of prod past the first sweep would silently + * solve the wrong equation. Size nPointDomain*nVar, allocated once in Initialize(). */ + ScalarType* d_ilu_backward_rhs = nullptr; + + /*--- The per-color kernel launch sequence (ilu_gpu_sweeps[0] passes for the factorization, + * ilu_gpu_sweeps[1] / ilu_gpu_sweeps[2] passes for the forward/backward triangular solves) + * is identical on every call: same grid/block sizes, same device pointers (all fixed members, + * allocated once). It is captured once into a CUDA graph and replayed, which removes + * host-side launch overhead without changing the parallelization (unlike a persistent + * cooperative-groups kernel, this does not cap per-color parallelism to the occupancy-resident + * block count). ---*/ + /*--- Types are forward-declared as opaque structs (matching the real cudaGraphExec_t / + * cudaStream_t typedefs) so this header does not need to include the CUDA runtime. ---*/ + mutable struct CUgraphExec_st* ilu_build_graph_exec = nullptr; + mutable struct CUgraphExec_st* ilu_apply_graph_exec = nullptr; + mutable const ScalarType* ilu_apply_graph_vec = nullptr; /*!< \brief Pointers the apply graph + * was captured with, to detect when + * it must be recaptured. */ + mutable ScalarType* ilu_apply_graph_prod = nullptr; + /*--- The legacy default stream cannot be captured into a graph, so the ILU graphs are + * captured and replayed on this dedicated stream instead; every launch on it is followed by + * a sync back to the host before control returns to the rest of the (single-stream) solver, + * so this does not change execution order relative to everything else, which stays on the + * default stream. ---*/ + mutable struct CUstream_st* ilu_stream = nullptr; + + ScalarType* invM; /*!< \brief Inverse of (Jacobi) preconditioner. */ + + /*--- Temporary (hence mutable) working memory used in the Linelet preconditioner, outer vector is for threads ---*/ + mutable vector> + LineletUpper; /*!< \brief Pointers to the upper blocks of the tri-diag system (working memory). */ + mutable vector> + LineletInvDiag; /*!< \brief Inverse of the diagonal blocks of the tri-diag system (working memory). */ + mutable vector> + LineletVector; /*!< \brief Solution and RHS of the tri-diag system (working memory). */ #ifdef USE_MKL -using gemm_t = typename mkl_jit_wrapper::gemm_t; -void* MatrixMatrixProductJitter; /*!< \brief Jitter handle for MKL JIT based GEMM. */ -gemm_t MatrixMatrixProductKernel; /*!< \brief MKL JIT based GEMM kernel. */ -void* MatrixVectorProductJitterBetaZero; /*!< \brief Jitter handle for MKL JIT based GEMV. */ -gemm_t MatrixVectorProductKernelBetaZero; /*!< \brief MKL JIT based GEMV kernel. */ -void* MatrixVectorProductJitterBetaOne; /*!< \brief Jitter handle for MKL JIT based GEMV with BETA=1.0. */ -gemm_t MatrixVectorProductKernelBetaOne; /*!< \brief MKL JIT based GEMV kernel with BETA=1.0. */ -void* MatrixVectorProductJitterAlphaMinusOne; /*!< \brief Jitter handle for MKL JIT based GEMV with ALPHA=-1.0 and - BETA=1.0. */ -gemm_t MatrixVectorProductKernelAlphaMinusOne; /*!< \brief MKL JIT based GEMV kernel with ALPHA=-1.0 and BETA=1.0. */ + using gemm_t = typename mkl_jit_wrapper::gemm_t; + void* MatrixMatrixProductJitter; /*!< \brief Jitter handle for MKL JIT based GEMM. */ + gemm_t MatrixMatrixProductKernel; /*!< \brief MKL JIT based GEMM kernel. */ + void* MatrixVectorProductJitterBetaZero; /*!< \brief Jitter handle for MKL JIT based GEMV. */ + gemm_t MatrixVectorProductKernelBetaZero; /*!< \brief MKL JIT based GEMV kernel. */ + void* MatrixVectorProductJitterBetaOne; /*!< \brief Jitter handle for MKL JIT based GEMV with BETA=1.0. */ + gemm_t MatrixVectorProductKernelBetaOne; /*!< \brief MKL JIT based GEMV kernel with BETA=1.0. */ + void* MatrixVectorProductJitterAlphaMinusOne; /*!< \brief Jitter handle for MKL JIT based GEMV with ALPHA=-1.0 and + BETA=1.0. */ + gemm_t MatrixVectorProductKernelAlphaMinusOne; /*!< \brief MKL JIT based GEMV kernel with ALPHA=-1.0 and BETA=1.0. */ #endif #ifdef HAVE_PASTIX -mutable CPastixWrapper pastix_wrapper; + mutable CPastixWrapper pastix_wrapper; #endif -/*! - * \brief Handle type conversion for when we Set, Add, etc. blocks, preserving derivative information (if supported by - * types). - */ -template ::value> = 0> -FORCEINLINE static DstType ActiveAssign(const SrcType& val) { - return SU2_TYPE::GetValue(val); -} + /*! + * \brief Handle type conversion for when we Set, Add, etc. blocks, preserving derivative information (if supported by + * types). + */ + template ::value> = 0> + FORCEINLINE static DstType ActiveAssign(const SrcType& val) { + return SU2_TYPE::GetValue(val); + } -template ::value> = 0> -FORCEINLINE static DstType ActiveAssign(const SrcType& val) { - return val; -} + template ::value> = 0> + FORCEINLINE static DstType ActiveAssign(const SrcType& val) { + return val; + } -/*! - * \brief Handle type conversion for when we Set, Add, etc. blocks, discarding derivative information. - */ -template -FORCEINLINE static ScalarType PassiveAssign(const SrcType& val) { - return SU2_TYPE::GetValue(val); -} + /*! + * \brief Handle type conversion for when we Set, Add, etc. blocks, discarding derivative information. + */ + template + FORCEINLINE static ScalarType PassiveAssign(const SrcType& val) { + return SU2_TYPE::GetValue(val); + } -/*! - * \brief Calculates the matrix-vector product: product = matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[out] product - */ -void MatrixVectorProduct(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; + /*! + * \brief Calculates the matrix-vector product: product = matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[out] product + */ + void MatrixVectorProduct(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; -/*! - * \brief Calculates the matrix-vector product: product += matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[in,out] product - */ -void MatrixVectorProductAdd(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; + /*! + * \brief Calculates the matrix-vector product: product += matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[in,out] product + */ + void MatrixVectorProductAdd(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; -/*! - * \brief Calculates the matrix-vector product: product -= matrix*vector - * \param[in] matrix - * \param[in] vector - * \param[in,out] product - */ -void MatrixVectorProductSub(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; + /*! + * \brief Calculates the matrix-vector product: product -= matrix*vector + * \param[in] matrix + * \param[in] vector + * \param[in,out] product + */ + void MatrixVectorProductSub(const ScalarType* matrix, const ScalarType* vector, ScalarType* product) const; -/*! - * \brief Calculates the matrix-matrix product - */ -void MatrixMatrixProduct(const ScalarType* matrix_a, const ScalarType* matrix_b, ScalarType* product) const; + /*! + * \brief Calculates the matrix-matrix product + */ + void MatrixMatrixProduct(const ScalarType* matrix_a, const ScalarType* matrix_b, ScalarType* product) const; -/*! - * \brief Subtract b from a and store the result in c. - */ -FORCEINLINE void VectorSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { - for (unsigned long iVar = 0; iVar < nVar; iVar++) c[iVar] = a[iVar] - b[iVar]; -} + /*! + * \brief Subtract b from a and store the result in c. + */ + FORCEINLINE void VectorSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { + for (unsigned long iVar = 0; iVar < nVar; iVar++) c[iVar] = a[iVar] - b[iVar]; + } -/*! - * \brief Subtract b from a and store the result in c. - */ -FORCEINLINE void MatrixSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { - SU2_OMP_SIMD - for (unsigned long iVar = 0; iVar < nVar * nEqn; iVar++) c[iVar] = a[iVar] - b[iVar]; -} + /*! + * \brief Subtract b from a and store the result in c. + */ + FORCEINLINE void MatrixSubtraction(const ScalarType* a, const ScalarType* b, ScalarType* c) const { + SU2_OMP_SIMD + for (unsigned long iVar = 0; iVar < nVar * nEqn; iVar++) c[iVar] = a[iVar] - b[iVar]; + } -/*! - * \brief Copy matrix src into dst, transpose if required. - */ -FORCEINLINE void MatrixCopy(const ScalarType* src, ScalarType* dst) const { - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) dst[iVar] = src[iVar]; -} + /*! + * \brief Copy matrix src into dst, transpose if required. + */ + FORCEINLINE void MatrixCopy(const ScalarType* src, ScalarType* dst) const { + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) dst[iVar] = src[iVar]; + } -/*! - * \brief Zero a matrix. - */ -FORCEINLINE void ZeroMatrix(ScalarType* mat) const { - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) mat[iVar] = 0; -} + /*! + * \brief Zero a matrix. + */ + FORCEINLINE void ZeroMatrix(ScalarType* mat) const { + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nEqn; ++iVar) mat[iVar] = 0; + } -/*! - * \brief Solve a small (nVar x nVar) linear system using Gaussian elimination. - * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. - * \param[in,out] vec - On entry the rhs, on exit the solution. - */ -void GaussElimination(ScalarType* matrix, ScalarType* vec) const; + /*! + * \brief Solve a small (nVar x nVar) linear system using Gaussian elimination. + * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. + * \param[in,out] vec - On entry the rhs, on exit the solution. + */ + void GaussElimination(ScalarType* matrix, ScalarType* vec) const; -/*! - * \brief Invert a small dense matrix. - * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. - * \param[out] inverse - the matrix inverse. - */ -void MatrixInverse(ScalarType* matrix, ScalarType* inverse) const; + /*! + * \brief Invert a small dense matrix. + * \param[in,out] matrix - On entry the system matrix, on exit the factorized matrix. + * \param[out] inverse - the matrix inverse. + */ + void MatrixInverse(ScalarType* matrix, ScalarType* inverse) const; -/*! - * \brief Performs the Gauss Elimination algorithm to solve the linear subsystem of the (i,i) subblock and rhs. - * \param[in] block_i - Index of the (i,i) diagonal block. - * \param[in] rhs - Right-hand-side of the linear system. - * \return Solution of the linear system (overwritten on rhs). - */ -inline void GaussElimination(unsigned long block_i, ScalarType* rhs) const; + /*! + * \brief Performs the Gauss Elimination algorithm to solve the linear subsystem of the (i,i) subblock and rhs. + * \param[in] block_i - Index of the (i,i) diagonal block. + * \param[in] rhs - Right-hand-side of the linear system. + * \return Solution of the linear system (overwritten on rhs). + */ + inline void GaussElimination(unsigned long block_i, ScalarType* rhs) const; -/*! - * \brief Inverse diagonal block. - * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. - * \param[out] invBlock - Inverse block. - */ -inline void InverseDiagonalBlock(unsigned long block_i, ScalarType* invBlock) const; + /*! + * \brief Inverse diagonal block. + * \param[in] block_i - Indexes of the block in the matrix-by-blocks structure. + * \param[out] invBlock - Inverse block. + */ + inline void InverseDiagonalBlock(unsigned long block_i, ScalarType* invBlock) const; -/*! - * \brief Invert diagonal block (Uii) of the ILU matrix in place. - * \param[in] block_i - Index of the block to invert. - * \return Inverted block. - */ -inline const ScalarType* InvertDiagonalBlockILUMatrix(unsigned long block_i); + /*! + * \brief Invert diagonal block (Uii) of the ILU matrix in place. + * \param[in] block_i - Index of the block to invert. + * \return Inverted block. + */ + inline const ScalarType* InvertDiagonalBlockILUMatrix(unsigned long block_i); -/*! - * \brief Returns the start of the ILU block or nullptr if (i,j) is not a nonzero. - * \param[in] block_i/j - Indexes of the block in the matrix-by-blocks structure. - */ -inline ScalarType* GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j); + /*! + * \brief Returns the start of the ILU block or nullptr if (i,j) is not a nonzero. + * \param[in] block_i/j - Indexes of the block in the matrix-by-blocks structure. + */ + inline ScalarType* GetBlock_ILUMatrix(unsigned long block_i, unsigned long block_j); -/*! - * \brief Performs the product of i-th row of the upper part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the upper part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \param[in] col_ub - Exclusive upper bound for column indices considered in multiplication. - * \param[out] prod - Result of the product U(A)*vec. - */ -inline void UpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, - ScalarType* prod) const; + /*! + * \brief Performs the product of i-th row of the upper part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the upper part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \param[in] col_ub - Exclusive upper bound for column indices considered in multiplication. + * \param[out] prod - Result of the product U(A)*vec. + */ + inline void UpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, + ScalarType* prod) const; -/*! - * \brief Performs the product of i-th row of the lower part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the lower part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \param[in] col_lb - Inclusive lower bound for column indices considered in multiplication. - * \param[out] prod - Result of the product L(A)*vec. - */ -inline void LowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, - ScalarType* prod) const; + /*! + * \brief Performs the product of i-th row of the lower part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the lower part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \param[in] col_lb - Inclusive lower bound for column indices considered in multiplication. + * \param[out] prod - Result of the product L(A)*vec. + */ + inline void LowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, + ScalarType* prod) const; -/*! - * \brief Performs the product of i-th row of the diagonal part of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the diagonal part of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \return prod Result of the product D(A)*vec (stored at *prod_row_vector). - */ -inline void DiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + /*! + * \brief Performs the product of i-th row of the diagonal part of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the diagonal part of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \return prod Result of the product D(A)*vec (stored at *prod_row_vector). + */ + inline void DiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; -/*! - * \brief Performs the product of i-th row of a sparse matrix by a vector. - * \param[in] vec - Vector to be multiplied by the row of the sparse matrix A. - * \param[in] row_i - Row of the matrix to be multiplied by vector vec. - * \return Result of the product (stored at *prod_row_vector). - */ -void RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + /*! + * \brief Performs the product of i-th row of a sparse matrix by a vector. + * \param[in] vec - Vector to be multiplied by the row of the sparse matrix A. + * \param[in] row_i - Row of the matrix to be multiplied by vector vec. + * \return Result of the product (stored at *prod_row_vector). + */ + void RowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; -/*! - * \brief Computes product += A_k * vec using the quantized representation of block k. - * \note Only valid after QuantizeDiagonalBlocks() has been called. - * \param[in] k - Block index in the CSR flat storage. - * \param[in] vec - Input vector (nEqn entries). - * \param[in,out] prod - Accumulation output (nVar entries). - */ -inline void QuantizedMatVecAdd(const QuantType* qs, const QuantType* qv, const ScalarType* vec, ScalarType* prod) const; + /*! + * \brief Computes product += A_k * vec using the quantized representation of block k. + * \note Only valid after QuantizeDiagonalBlocks() has been called. + * \param[in] k - Block index in the CSR flat storage. + * \param[in] vec - Input vector (nEqn entries). + * \param[in,out] prod - Accumulation output (nVar entries). + */ + inline void QuantizedMatVecAdd(const QuantType* qs, const QuantType* qv, const ScalarType* vec, + ScalarType* prod) const; -/*! \brief Quantize one nVar×nVar block (row-major) into the int8 scale+value arrays. - * Called on the hot assembly path (SetBlocks/UpdateBlocks in Q_LU_SGS mode). */ -void QuantizeBlock(const ScalarType* blk, QuantType* qs, QuantType* qv) const; + /*! \brief Quantize one nVar×nVar block (row-major) into the int8 scale+value arrays. + * Called on the hot assembly path (SetBlocks/UpdateBlocks in Q_LU_SGS mode). */ + void QuantizeBlock(const ScalarType* blk, QuantType* qs, QuantType* qv) const; -/*! \brief Full-row product using quantized L/D/U (Q_LU_SGS SpMV path). */ -inline void QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + /*! \brief Full-row product using quantized L/D/U (Q_LU_SGS SpMV path). */ + inline void QuantizedRowProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; -/*! \brief Upper-triangle product using quantized U (Q_LU_SGS backward sweep). */ -inline void QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, - ScalarType* prod) const; + /*! \brief Upper-triangle product using quantized U (Q_LU_SGS backward sweep). */ + inline void QuantizedUpperProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_ub, + ScalarType* prod) const; -/*! \brief Lower-triangle product using quantized L (Q_LU_SGS forward sweep). */ -inline void QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, - ScalarType* prod) const; + /*! \brief Lower-triangle product using quantized L (Q_LU_SGS forward sweep). */ + inline void QuantizedLowerProduct(const CSysVector& vec, unsigned long row_i, unsigned long col_lb, + ScalarType* prod) const; -/*! \brief Diagonal product using quantized D (Q_LU_SGS backward sweep). */ -inline void QuantizedDiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; + /*! \brief Diagonal product using quantized D (Q_LU_SGS backward sweep). */ + inline void QuantizedDiagonalProduct(const CSysVector& vec, unsigned long row_i, ScalarType* prod) const; -/*! \brief Gauss elimination on the quantized diagonal block: decodes q_blocks_d into a local - * ScalarType buffer and delegates to the scalar GaussElimination overload. */ -inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; + /*! \brief Gauss elimination on the quantized diagonal block: decodes q_blocks_d into a local + * ScalarType buffer and delegates to the scalar GaussElimination overload. */ + inline void QuantizedGaussElimination(unsigned long block_i, ScalarType* rhs) const; -/*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ + /*--- Hooks for GPU versions (implemented is in CSysMatrixGPU.cu). ---*/ -/*! - * \brief Performs the product of a sparse matrix by a CSysVector on the device. - */ -void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Performs the product of a sparse matrix by a CSysVector on the device. + */ + void MatrixVectorProductGPU(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; -/*! - * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. - * \note Requires the device matrix to be up to date, see HtDTransfer. - */ -void BuildJacobiPreconditionerGPU(); + /*! + * \brief Build the Jacobi preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildJacobiPreconditionerGPU(); -/*! - * \brief Apply the Jacobi preconditioner on the GPU/device side. - */ -void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, - CGeometry* geometry, const CConfig* config) const; + /*! + * \brief Apply the Jacobi preconditioner on the GPU/device side. + */ + void ComputeJacobiPreconditionerGPU(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const; -/*! - * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. - * \note Requires the device matrix to be up to date, see HtDTransfer. - */ -void BuildILUPreconditionerGPU(); + /*! + * \brief Build the ILU preconditioner on the device, from the device copy of the matrix. + * \note Requires the device matrix to be up to date, see HtDTransfer. + */ + void BuildILUPreconditionerGPU(); -/*! - * \brief Apply the ILU preconditioner on the device. - */ -void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; + /*! + * \brief Apply the ILU preconditioner on the device. + */ + void ComputeILUPreconditionerGPU(const CSysVector& vec, CSysVector& prod) const; -public: -/*! - * \brief Constructor of the class. - */ -CSysMatrix(); + public: + /*! + * \brief Constructor of the class. + */ + CSysMatrix(); -/*! - * \brief Destructor of the class. - */ -~CSysMatrix(); + /*! + * \brief Destructor of the class. + */ + ~CSysMatrix(); -/*! - * \brief Initializes the sparse matrix. - * \note The preconditioners require nVar == nEqn (square blocks). - * \param[in] npoint - Number of points including halos. - * \param[in] npointdomain - Number of points excluding halos. - * \param[in] nvar - Number of variables (and rows of the blocks). - * \param[in] neqn - Number of equations (and columns of the blocks). - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[in] needTranspPtr - If the L/U transpose maps should be built, used for "SetDiagonalAsColumnSum". - * \param[in] grad_mode - Gradient smoothing mode, only used to detect the right preconditioner type. - * \param[in] allow_quant - Quantization is only possible with solvers that "set and forget" the off-diagonal - * blocks of the matrix. Solvers that perform multiple updates would lose too much information, so - * that pattern is not supported with quantization (the code will hit null pointers). It is up to - * the solver to declare whether it will "set and forget". - */ -void Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, - bool EdgeConnect, CGeometry* geometry, const CConfig* config, bool needTranspPtr = false, - bool grad_mode = false, bool allow_quant = false); + /*! + * \brief Initializes the sparse matrix. + * \note The preconditioners require nVar == nEqn (square blocks). + * \param[in] npoint - Number of points including halos. + * \param[in] npointdomain - Number of points excluding halos. + * \param[in] nvar - Number of variables (and rows of the blocks). + * \param[in] neqn - Number of equations (and columns of the blocks). + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[in] needTranspPtr - If the L/U transpose maps should be built, used for "SetDiagonalAsColumnSum". + * \param[in] grad_mode - Gradient smoothing mode, only used to detect the right preconditioner type. + * \param[in] allow_quant - Quantization is only possible with solvers that "set and forget" the off-diagonal + * blocks of the matrix. Solvers that perform multiple updates would lose too much information, so + * that pattern is not supported with quantization (the code will hit null pointers). It is up to + * the solver to declare whether it will "set and forget". + */ + void Initialize(unsigned long npoint, unsigned long npointdomain, unsigned short nvar, unsigned short neqn, + bool EdgeConnect, CGeometry* geometry, const CConfig* config, bool needTranspPtr = false, + bool grad_mode = false, bool allow_quant = false); -/*! - * \brief Compresses off-diagonal blocks into quantized form for use with USE_QUANTIZATION. - */ -void QuantizeDiagonalBlocks(); + /*! + * \brief Compresses off-diagonal blocks into quantized form for use with USE_QUANTIZATION. + */ + void QuantizeDiagonalBlocks(); -/*! - * \brief Sets to zero all the entries of the sparse matrix. - */ -void SetValZero(); + /*! + * \brief Sets to zero all the entries of the sparse matrix. + */ + void SetValZero(); -/*! - * \brief Sets to zero all the block diagonal entries of the sparse matrix. - */ -void SetValDiagonalZero(); + /*! + * \brief Sets to zero all the block diagonal entries of the sparse matrix. + */ + void SetValDiagonalZero(); -/*! - * \brief Performs the memory copy from host to device. - * \param[in] trigger - boolean value that decides whether to conduct the transfer or not. True by default. - */ -void HtDTransfer(bool trigger = true) const; + /*! + * \brief Performs the memory copy from host to device. + * \param[in] trigger - boolean value that decides whether to conduct the transfer or not. True by default. + */ + void HtDTransfer(bool trigger = true) const; -/*! - * \brief Get a pointer to the start of block "ij" - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \return Pointer to location in memory where the block starts. - */ -FORCEINLINE const ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) const { - if (block_i == block_j) return &mat.d[block_i * nVar * nEqn]; - if (block_j < block_i) { - for (auto index = mat.row_ptr_l[block_i]; index < mat.row_ptr_l[block_i + 1]; ++index) - if (mat.col_ind_l[index] == block_j) return &mat.l[index * nVar * nEqn]; + /*! + * \brief Get a pointer to the start of block "ij" + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \return Pointer to location in memory where the block starts. + */ + FORCEINLINE const ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) const { + if (block_i == block_j) return &mat.d[block_i * nVar * nEqn]; + if (block_j < block_i) { + for (auto index = mat.row_ptr_l[block_i]; index < mat.row_ptr_l[block_i + 1]; ++index) + if (mat.col_ind_l[index] == block_j) return &mat.l[index * nVar * nEqn]; + return nullptr; + } + for (auto index = mat.row_ptr_u[block_i]; index < mat.row_ptr_u[block_i + 1]; ++index) + if (mat.col_ind_u[index] == block_j) return &mat.u[index * nVar * nEqn]; return nullptr; } - for (auto index = mat.row_ptr_u[block_i]; index < mat.row_ptr_u[block_i + 1]; ++index) - if (mat.col_ind_u[index] == block_j) return &mat.u[index * nVar * nEqn]; - return nullptr; -} -/*! - * \brief Get a pointer to the start of block "ij", non-const version. - */ -FORCEINLINE ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) { - const CSysMatrix& const_this = *this; - return const_cast(const_this.GetBlock(block_i, block_j)); -} + /*! + * \brief Get a pointer to the start of block "ij", non-const version. + */ + FORCEINLINE ScalarType* GetBlock(unsigned long block_i, unsigned long block_j) { + const CSysMatrix& const_this = *this; + return const_cast(const_this.GetBlock(block_i, block_j)); + } -/*! - * \brief Read-only view of block (block_i, block_j). In Q_LU_SGS mode values are decoded - * on access inside CBlockView::operator()(i,j); no temporary copy is made. - * \return A CBlockView that evaluates to false if the block is absent. - */ -FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) const { + /*! + * \brief Read-only view of block (block_i, block_j). In Q_LU_SGS mode values are decoded + * on access inside CBlockView::operator()(i,j); no temporary copy is made. + * \return A CBlockView that evaluates to false if the block is absent. + */ + FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) const { #define GET_BLOCK_VIEW_IMPL \ if (!quantized_mode || block_i == block_j) { \ return {GetBlock(block_i, block_j), nullptr, nullptr, nVar}; \ @@ -692,524 +691,524 @@ FORCEINLINE CBlockView GetBlockView(unsigned long block_i, uns if (mat.col_ind_u[k] == block_j) return {nullptr, &q_scale_u[k * nVar], &q_blocks_u[k * nVar * nVar], nVar}; \ } \ return {} - GET_BLOCK_VIEW_IMPL; -} + GET_BLOCK_VIEW_IMPL; + } -/*! - * \overload Non const version of GetBlockView. - */ -FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) { - GET_BLOCK_VIEW_IMPL; + /*! + * \overload Non const version of GetBlockView. + */ + FORCEINLINE CBlockView GetBlockView(unsigned long block_i, unsigned long block_j) { + GET_BLOCK_VIEW_IMPL; #undef GET_BLOCK_VIEW_IMPL -} + } -/*! - * \brief Set the value of a scaled block in the sparse matrix. - * \note This is an templated overload for C2Dcontainer specialization su2matrix. - * It assumes that MatrixType supports a member type Scalar and access operator(i, j). - * If the template param Overwrite is false we add to the block (bij += alpha*b). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. - */ -template -inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block(i, j)); }); -} + /*! + * \brief Set the value of a scaled block in the sparse matrix. + * \note This is an templated overload for C2Dcontainer specialization su2matrix. + * It assumes that MatrixType supports a member type Scalar and access operator(i, j). + * If the template param Overwrite is false we add to the block (bij += alpha*b). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to set to A(i, j). + * \param[in] alpha - Scale factor. + */ + template + inline void SetBlock(unsigned long block_i, unsigned long block_j, MatrixType& val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block(i, j)); }); + } -/*! - * \overload val_block is a pointer instead of a matrix type. - */ -template ::value> = 0> -inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i * nEqn + j]); }); -} + /*! + * \overload val_block is a pointer instead of a matrix type. + */ + template ::value> = 0> + inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i * nEqn + j]); }); + } -/*! - * \overload val_block is a double pointer instead of matrix type. - */ -template -inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, - std::decay_t alpha = 1.0) { - auto view = GetBlockView(block_i, block_j); - if (!view) return; - view.template apply( - [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i][j]); }); -} + /*! + * \overload val_block is a double pointer instead of matrix type. + */ + template + inline void SetBlock(unsigned long block_i, unsigned long block_j, const OtherType* const* val_block, + std::decay_t alpha = 1.0) { + auto view = GetBlockView(block_i, block_j); + if (!view) return; + view.template apply( + [&](unsigned long i, unsigned long j) { return PassiveAssign(alpha * val_block[i][j]); }); + } -/*! - * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to set to A(i, j). - * \param[in] alpha - Scale factor. - */ -template -inline void AddBlock(unsigned long block_i, unsigned long block_j, const T& val_block, OtherType alpha = 1.0) { - SetBlock(block_i, block_j, val_block, alpha); -} + /*! + * \brief Add a scaled block (in flat format) to the sparse matrix (see SetBlock). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to set to A(i, j). + * \param[in] alpha - Scale factor. + */ + template + inline void AddBlock(unsigned long block_i, unsigned long block_j, const T& val_block, OtherType alpha = 1.0) { + SetBlock(block_i, block_j, val_block, alpha); + } -/*! - * \brief Subtracts the specified block to the sparse matrix (see AddBlock). - * \param[in] block_i - Row index. - * \param[in] block_j - Column index. - * \param[in] val_block - Block to subtract to A(i, j). - */ -template -inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const T& val_block) { - AddBlock(block_i, block_j, val_block, -1); -} + /*! + * \brief Subtracts the specified block to the sparse matrix (see AddBlock). + * \param[in] block_i - Row index. + * \param[in] block_j - Column index. + * \param[in] val_block - Block to subtract to A(i, j). + */ + template + inline void SubtractBlock(unsigned long block_i, unsigned long block_j, const T& val_block) { + AddBlock(block_i, block_j, val_block, -1); + } -/*! - * \brief Returns the 4 blocks ii, ij, ji, jj used by "UpdateBlocks". - * \note This method assumes an FVM-type sparse pattern. - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] iPoint - Row to which we add the blocks. - * \param[in] jPoint - Row from which we subtract the blocks. - * \param[out] bii, bij, bji, bjj - Blocks of the matrix. - */ -inline void GetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, ScalarType*& bii, - ScalarType*& bij, ScalarType*& bji, ScalarType*& bjj) { - const auto blkSz = nVar * nEqn; - bii = &mat.d[iPoint * blkSz]; - bjj = &mat.d[jPoint * blkSz]; - bij = &mat.u[iEdge * blkSz]; - bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; -} + /*! + * \brief Returns the 4 blocks ii, ij, ji, jj used by "UpdateBlocks". + * \note This method assumes an FVM-type sparse pattern. + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] iPoint - Row to which we add the blocks. + * \param[in] jPoint - Row from which we subtract the blocks. + * \param[out] bii, bij, bji, bjj - Blocks of the matrix. + */ + inline void GetBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, ScalarType*& bii, + ScalarType*& bij, ScalarType*& bji, ScalarType*& bjj) { + const auto blkSz = nVar * nEqn; + bii = &mat.d[iPoint * blkSz]; + bjj = &mat.d[jPoint * blkSz]; + bij = &mat.u[iEdge * blkSz]; + bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + } -/*! - * \brief Update 4 blocks ii, ij, ji, jj (add to i* sub from j*). - * \note This method assumes an FVM-type sparse pattern. - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] iPoint - Row to which we add the blocks. - * \param[in] jPoint - Row from which we subtract the blocks. - * \param[in] block_i - Adds to ii, subs from ji. - * \param[in] block_j - Adds to ij, subs from jj. - * \param[in] scale - Scale blocks during update (axpy type op). - */ -template -inline void UpdateBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, - const MatrixType& block_j, OtherType scale = 1) { - const auto blkSz = nVar * nEqn; - auto* bii = &mat.d[iPoint * blkSz]; - auto* bjj = &mat.d[jPoint * blkSz]; - - unsigned long iVar, jVar, offset = 0; - - if (quantized_mode) { - assert(OverwriteOffDiag); - /*--- Diagonal: full-precision accumulation. Off-diagonal: quantize on the fly. ---*/ - ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; - for (iVar = 0; iVar < nVar; iVar++) - for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + /*! + * \brief Update 4 blocks ii, ij, ji, jj (add to i* sub from j*). + * \note This method assumes an FVM-type sparse pattern. + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] iPoint - Row to which we add the blocks. + * \param[in] jPoint - Row from which we subtract the blocks. + * \param[in] block_i - Adds to ii, subs from ji. + * \param[in] block_j - Adds to ij, subs from jj. + * \param[in] scale - Scale blocks during update (axpy type op). + */ + template + inline void UpdateBlocks(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, + const MatrixType& block_j, OtherType scale = 1) { + const auto blkSz = nVar * nEqn; + auto* bii = &mat.d[iPoint * blkSz]; + auto* bjj = &mat.d[jPoint * blkSz]; + + unsigned long iVar, jVar, offset = 0; + + if (quantized_mode) { + assert(OverwriteOffDiag); + /*--- Diagonal: full-precision accumulation. Off-diagonal: quantize on the fly. ---*/ + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); + bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } + QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + return; + } + + auto* bij = &mat.u[iEdge * blkSz]; + auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); - bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + if constexpr (OverwriteOffDiag) { + bij[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } else { + bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); + } + ++offset; } - QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); - const auto k_l = edge_ptr_l[iEdge]; - QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - return; + } } - auto* bij = &mat.u[iEdge * blkSz]; - auto* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; - for (iVar = 0; iVar < nVar; iVar++) { - for (jVar = 0; jVar < nEqn; jVar++) { - bii[offset] += PassiveAssign(block_i[iVar][jVar] * scale); - bjj[offset] -= PassiveAssign(block_j[iVar][jVar] * scale); - if constexpr (OverwriteOffDiag) { - bij[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); - } else { - bij[offset] += PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] -= PassiveAssign(block_i[iVar][jVar] * scale); - } - ++offset; - } + /*! + * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of UpdateBlocks. + */ + template + inline void UpdateBlocksSub(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, + const MatrixType& block_i, const MatrixType& block_j) { + UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); } -} -/*! - * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of UpdateBlocks. - */ -template -inline void UpdateBlocksSub(unsigned long iEdge, unsigned long iPoint, unsigned long jPoint, const MatrixType& block_i, - const MatrixType& block_j) { - UpdateBlocks(iEdge, iPoint, jPoint, block_i, block_j, -1); -} + /*! + * \brief SIMD version, does the update for multiple edges and points. + * \note Nothing is updated if the mask is 0. + */ + template + FORCEINLINE void SetBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, + const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { + static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); + static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); + constexpr size_t blkSz = MatTypeSIMD::StaticSize; + assert(blkSz == nVar * nEqn); + + /*--- "Transpose" the blocks, scale, and possibly convert types, + * giving the compiler the chance to vectorize all of these. ---*/ + ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; + + for (size_t i = 0; i < blkSz; ++i) { + SU2_OMP_SIMD_IF_NOT_AD + for (size_t k = 0; k < N; ++k) { + blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); + blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); + } + } -/*! - * \brief SIMD version, does the update for multiple edges and points. - * \note Nothing is updated if the mask is 0. - */ -template -FORCEINLINE void SetBlocks(simd::Array iEdge, simd::Array iPoint, simd::Array jPoint, - const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, simd::Array mask = 1) { - static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); - static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); - constexpr size_t blkSz = MatTypeSIMD::StaticSize; - assert(blkSz == nVar * nEqn); - - /*--- "Transpose" the blocks, scale, and possibly convert types, - * giving the compiler the chance to vectorize all of these. ---*/ - ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; - - for (size_t i = 0; i < blkSz; ++i) { - SU2_OMP_SIMD_IF_NOT_AD + /*--- Update one by one skipping if mask is 0. ---*/ for (size_t k = 0; k < N; ++k) { - blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); - blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); + if (mask[k] == 0) continue; + + auto bii = &mat.d[iPoint[k] * blkSz]; + auto bjj = &mat.d[jPoint[k] * blkSz]; + + if (quantized_mode) { + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + } + QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); + const auto k_l = edge_ptr_l[iEdge[k]]; + QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + } else { + auto bij = &mat.u[iEdge[k] * blkSz]; + auto bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bii[i] -= blk_i[k][i]; + bjj[i] -= blk_j[k][i]; + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; + } + } } } - /*--- Update one by one skipping if mask is 0. ---*/ - for (size_t k = 0; k < N; ++k) { - if (mask[k] == 0) continue; - - auto bii = &mat.d[iPoint[k] * blkSz]; - auto bjj = &mat.d[jPoint[k] * blkSz]; + /*! + * \brief Sets 2 blocks ij and ji (add to i* sub from j*) associated with + * one edge of an FVM-type sparse pattern. + * \note The parameter Overwrite allows completely writing over the + * current values held by the matrix (true), or updating them (false). + * \param[in] edge - Index of edge that connects iPoint and jPoint. + * \param[in] block_i - Subs from ji. + * \param[in] block_j - Adds to ij. + * \param[in] scale - Scale blocks during update (axpy type op). + */ + template + inline void SetBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, + OtherType scale = 1) { + const auto blkSz = nVar * nEqn; + unsigned long iVar, jVar, offset = 0; if (quantized_mode) { - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bii[i] -= blk_i[k][i]; - bjj[i] -= blk_j[k][i]; - } - QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); - const auto k_l = edge_ptr_l[iEdge[k]]; - QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - } else { - auto bij = &mat.u[iEdge[k] * blkSz]; - auto bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bii[i] -= blk_i[k][i]; - bjj[i] -= blk_j[k][i]; - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; - } + assert(Overwrite); + ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; + for (iVar = 0; iVar < nVar; iVar++) + for (jVar = 0; jVar < nEqn; jVar++, ++offset) { + bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); + bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + } + QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); + const auto k_l = edge_ptr_l[iEdge]; + QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + return; } - } -} -/*! - * \brief Sets 2 blocks ij and ji (add to i* sub from j*) associated with - * one edge of an FVM-type sparse pattern. - * \note The parameter Overwrite allows completely writing over the - * current values held by the matrix (true), or updating them (false). - * \param[in] edge - Index of edge that connects iPoint and jPoint. - * \param[in] block_i - Subs from ji. - * \param[in] block_j - Adds to ij. - * \param[in] scale - Scale blocks during update (axpy type op). - */ -template -inline void SetBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, OtherType scale = 1) { - const auto blkSz = nVar * nEqn; - unsigned long iVar, jVar, offset = 0; - - if (quantized_mode) { - assert(Overwrite); - ScalarType bij_buf[MAXNVAR * MAXNVAR], bji_buf[MAXNVAR * MAXNVAR]; - for (iVar = 0; iVar < nVar; iVar++) - for (jVar = 0; jVar < nEqn; jVar++, ++offset) { - bij_buf[offset] = PassiveAssign(block_j[iVar][jVar] * scale); - bji_buf[offset] = -PassiveAssign(block_i[iVar][jVar] * scale); + ScalarType* bij = &mat.u[iEdge * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; + for (iVar = 0; iVar < nVar; iVar++) { + for (jVar = 0; jVar < nEqn; jVar++) { + bij[offset] = (Overwrite ? ScalarType(0) : bij[offset]) + PassiveAssign(block_j[iVar][jVar] * scale); + bji[offset] = (Overwrite ? ScalarType(0) : bji[offset]) - PassiveAssign(block_i[iVar][jVar] * scale); + ++offset; } - QuantizeBlock(bij_buf, &q_scale_u[iEdge * nVar], &q_blocks_u[iEdge * blkSz]); - const auto k_l = edge_ptr_l[iEdge]; - QuantizeBlock(bji_buf, &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - return; - } - - ScalarType* bij = &mat.u[iEdge * blkSz]; - ScalarType* bji = &mat.l[edge_ptr_l[iEdge] * blkSz]; - for (iVar = 0; iVar < nVar; iVar++) { - for (jVar = 0; jVar < nEqn; jVar++) { - bij[offset] = (Overwrite ? ScalarType(0) : bij[offset]) + PassiveAssign(block_j[iVar][jVar] * scale); - bji[offset] = (Overwrite ? ScalarType(0) : bji[offset]) - PassiveAssign(block_i[iVar][jVar] * scale); - ++offset; } } -} -/*! - * \brief Short-hand for the "additive overwrite" version of SetBlocks. - */ -template -inline void UpdateBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, - OtherType scale = 1) { - SetBlocks(iEdge, block_i, block_j, scale); -} + /*! + * \brief Short-hand for the "additive overwrite" version of SetBlocks. + */ + template + inline void UpdateBlocks(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j, + OtherType scale = 1) { + SetBlocks(iEdge, block_i, block_j, scale); + } -/*! - * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of SetBlocks. - */ -template -inline void UpdateBlocksSub(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j) { - SetBlocks(iEdge, block_i, block_j, -1); -} + /*! + * \brief Short-hand for the "subtractive" version (sub from i* add to j*) of SetBlocks. + */ + template + inline void UpdateBlocksSub(unsigned long iEdge, const MatrixType& block_i, const MatrixType& block_j) { + SetBlocks(iEdge, block_i, block_j, -1); + } -/*! - * \brief SIMD version, does the update for multiple edges. - * \note Nothing is updated if the mask is 0. - */ -template -FORCEINLINE void SetBlocks(simd::Array iEdge, const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, - simd::Array mask = 1) { - static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); - static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); - constexpr size_t blkSz = MatTypeSIMD::StaticSize; - assert(blkSz == nVar * nEqn); - - /*--- "Transpose" the blocks, scale, and possibly convert types, - * giving the compiler the chance to vectorize all of these. ---*/ - ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; - - for (size_t i = 0; i < blkSz; ++i) { - SU2_OMP_SIMD_IF_NOT_AD - for (size_t k = 0; k < N; ++k) { - blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); - blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); + /*! + * \brief SIMD version, does the update for multiple edges. + * \note Nothing is updated if the mask is 0. + */ + template + FORCEINLINE void SetBlocks(simd::Array iEdge, const MatTypeSIMD& block_i, const MatTypeSIMD& block_j, + simd::Array mask = 1) { + static_assert(MatTypeSIMD::StaticSize, "This method requires static size blocks."); + static_assert(MatTypeSIMD::IsRowMajor, "Block storage is not compatible with matrix."); + constexpr size_t blkSz = MatTypeSIMD::StaticSize; + assert(blkSz == nVar * nEqn); + + /*--- "Transpose" the blocks, scale, and possibly convert types, + * giving the compiler the chance to vectorize all of these. ---*/ + ScalarType blk_i[N][blkSz], blk_j[N][blkSz]; + + for (size_t i = 0; i < blkSz; ++i) { + SU2_OMP_SIMD_IF_NOT_AD + for (size_t k = 0; k < N; ++k) { + blk_i[k][i] = PassiveAssign(-mask[k] * block_i.data()[i][k]); + blk_j[k][i] = PassiveAssign(mask[k] * block_j.data()[i][k]); + } } - } - /*--- Update one by one skipping if mask is 0. ---*/ - for (size_t k = 0; k < N; ++k) { - if (mask[k] == 0) continue; + /*--- Update one by one skipping if mask is 0. ---*/ + for (size_t k = 0; k < N; ++k) { + if (mask[k] == 0) continue; - if (quantized_mode) { - QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); - const auto k_l = edge_ptr_l[iEdge[k]]; - QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); - } else { - ScalarType* bij = &mat.u[iEdge[k] * blkSz]; - ScalarType* bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; - /*--- Update, block i was negated during transpose in the - * hope the assignments below become non-temporal stores. ---*/ - SU2_OMP_SIMD - for (size_t i = 0; i < blkSz; ++i) { - bij[i] = blk_j[k][i]; - bji[i] = blk_i[k][i]; + if (quantized_mode) { + QuantizeBlock(blk_j[k], &q_scale_u[iEdge[k] * nVar], &q_blocks_u[iEdge[k] * blkSz]); + const auto k_l = edge_ptr_l[iEdge[k]]; + QuantizeBlock(blk_i[k], &q_scale_l[k_l * nVar], &q_blocks_l[k_l * blkSz]); + } else { + ScalarType* bij = &mat.u[iEdge[k] * blkSz]; + ScalarType* bji = &mat.l[edge_ptr_l[iEdge[k]] * blkSz]; + /*--- Update, block i was negated during transpose in the + * hope the assignments below become non-temporal stores. ---*/ + SU2_OMP_SIMD + for (size_t i = 0; i < blkSz; ++i) { + bij[i] = blk_j[k][i]; + bji[i] = blk_i[k][i]; + } } } } -} -/*! - * \brief Sets the specified block to the (i, i) subblock of the sparse matrix. - * Scales the input block by factor alpha. If the Overwrite parameter is - * false we update instead (bii += alpha*b). - * \param[in] block_i - Diagonal index. - * \param[in] val_block - Block to add to the diagonal of the matrix. - * \param[in] alpha - Scale factor. - */ -template -inline void SetBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { - auto mat_ii = &mat.d[block_i * nVar * nEqn]; - - for (auto iVar = 0ul; iVar < nVar; iVar++) - for (auto jVar = 0ul; jVar < nEqn; jVar++) { - *mat_ii = (Overwrite ? ScalarType(0) : *mat_ii) + PassiveAssign(alpha * val_block[iVar][jVar]); - ++mat_ii; - } -} + /*! + * \brief Sets the specified block to the (i, i) subblock of the sparse matrix. + * Scales the input block by factor alpha. If the Overwrite parameter is + * false we update instead (bii += alpha*b). + * \param[in] block_i - Diagonal index. + * \param[in] val_block - Block to add to the diagonal of the matrix. + * \param[in] alpha - Scale factor. + */ + template + inline void SetBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { + auto mat_ii = &mat.d[block_i * nVar * nEqn]; + + for (auto iVar = 0ul; iVar < nVar; iVar++) + for (auto jVar = 0ul; jVar < nEqn; jVar++) { + *mat_ii = (Overwrite ? ScalarType(0) : *mat_ii) + PassiveAssign(alpha * val_block[iVar][jVar]); + ++mat_ii; + } + } -/*! - * \brief Non overwrite version of SetBlock2Diag, also with scaling. - */ -template -inline void AddBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { - SetBlock2Diag(block_i, val_block, alpha); -} + /*! + * \brief Non overwrite version of SetBlock2Diag, also with scaling. + */ + template + inline void AddBlock2Diag(unsigned long block_i, const OtherType& val_block, T alpha = 1.0) { + SetBlock2Diag(block_i, val_block, alpha); + } -/*! - * \brief Short-hand to AddBlock2Diag with alpha = -1, i.e. subtracts from the current diagonal. - */ -template -inline void SubtractBlock2Diag(unsigned long block_i, const OtherType& val_block) { - AddBlock2Diag(block_i, val_block, -1.0); -} + /*! + * \brief Short-hand to AddBlock2Diag with alpha = -1, i.e. subtracts from the current diagonal. + */ + template + inline void SubtractBlock2Diag(unsigned long block_i, const OtherType& val_block) { + AddBlock2Diag(block_i, val_block, -1.0); + } -/*! - * \brief Adds the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). - */ -template -inline void AddVal2Diag(unsigned long block_i, OtherType val_matrix) { - auto d = &mat.d[block_i * nVar * nVar]; - for (auto iVar = 0ul; iVar < nVar; iVar++) d[iVar * (nVar + 1)] += PassiveAssign(val_matrix); -} + /*! + * \brief Adds the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). + */ + template + inline void AddVal2Diag(unsigned long block_i, OtherType val_matrix) { + auto d = &mat.d[block_i * nVar * nVar]; + for (auto iVar = 0ul; iVar < nVar; iVar++) d[iVar * (nVar + 1)] += PassiveAssign(val_matrix); + } -/*! - * \brief Adds the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] iVar - Variable index. - * \param[in] val - Value to add to the diagonal elements of A(i, i). - */ -template -inline void AddVal2Diag(unsigned long block_i, unsigned long iVar, OtherType val) { - mat.d[block_i * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); -} + /*! + * \brief Adds the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] iVar - Variable index. + * \param[in] val - Value to add to the diagonal elements of A(i, i). + */ + template + inline void AddVal2Diag(unsigned long block_i, unsigned long iVar, OtherType val) { + mat.d[block_i * nVar * nVar + iVar * (nVar + 1)] += PassiveAssign(val); + } -/*! - * \brief Sets the specified value to the diagonal of the (i, i) subblock - * of the matrix-by-blocks structure. - * \param[in] block_i - Diagonal index. - * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). - */ -template -inline void SetVal2Diag(unsigned long block_i, OtherType val_matrix) { - /*--- Clear entire block before setting its diagonal. ---*/ - SU2_OMP_SIMD - for (auto iVar = 0ul; iVar < nVar * nVar; iVar++) mat.d[block_i * nVar * nVar + iVar] = 0.0; + /*! + * \brief Sets the specified value to the diagonal of the (i, i) subblock + * of the matrix-by-blocks structure. + * \param[in] block_i - Diagonal index. + * \param[in] val_matrix - Value to add to the diagonal elements of A(i, i). + */ + template + inline void SetVal2Diag(unsigned long block_i, OtherType val_matrix) { + /*--- Clear entire block before setting its diagonal. ---*/ + SU2_OMP_SIMD + for (auto iVar = 0ul; iVar < nVar * nVar; iVar++) mat.d[block_i * nVar * nVar + iVar] = 0.0; - AddVal2Diag(block_i, val_matrix); -} + AddVal2Diag(block_i, val_matrix); + } -/*! - * \brief Deletes the values of a row of the sparse matrix. - * \param[in] block_i - Index of the block. - * \param[in] row - Row within the block. - */ -void DeleteValsRowi(unsigned long block_i, unsigned long row); + /*! + * \brief Deletes the values of a row of the sparse matrix. + * \param[in] block_i - Index of the block. + * \param[in] row - Row within the block. + */ + void DeleteValsRowi(unsigned long block_i, unsigned long row); -/*! - * \brief Modifies this matrix (A) and a rhs vector (b) such that (A^-1 * b)_i = x_i. - * \param[in] node_i - Index of the node for which to enforce the solution of all DOF's. - * \param[in] x_i - Values to enforce (nVar sized). - * \param[in,out] b - The rhs vector (b := b - A_{*,i} * x_i; b_i = x_i). - */ -template -void EnforceSolutionAtNode(unsigned long node_i, const OtherType* x_i, CSysVector& b); + /*! + * \brief Modifies this matrix (A) and a rhs vector (b) such that (A^-1 * b)_i = x_i. + * \param[in] node_i - Index of the node for which to enforce the solution of all DOF's. + * \param[in] x_i - Values to enforce (nVar sized). + * \param[in,out] b - The rhs vector (b := b - A_{*,i} * x_i; b_i = x_i). + */ + template + void EnforceSolutionAtNode(unsigned long node_i, const OtherType* x_i, CSysVector& b); -/*! - * \brief Similar to EnforceSolutionAtNode, but for 0 projection in a given direction. - */ -template -void EnforceZeroProjection(unsigned long node_i, const OtherType* n, CSysVector& b); + /*! + * \brief Similar to EnforceSolutionAtNode, but for 0 projection in a given direction. + */ + template + void EnforceZeroProjection(unsigned long node_i, const OtherType* n, CSysVector& b); -/*! - * \brief Sets the diagonal entries of the matrix as the sum of the blocks in the corresponding column. - */ -void SetDiagonalAsColumnSum(); + /*! + * \brief Sets the diagonal entries of the matrix as the sum of the blocks in the corresponding column. + */ + void SetDiagonalAsColumnSum(); -/*! - * \brief Transposes the matrix, any preconditioner that was computed may be invalid. - */ -void TransposeInPlace(); + /*! + * \brief Transposes the matrix, any preconditioner that was computed may be invalid. + */ + void TransposeInPlace(); -/*! - * \brief Add a scaled sparse matrix to "this" (axpy-type operation, A = A+alpha*B). - * \note Matrices must have the same sparse pattern. - * \param[in] alpha - The scaling constant. - * \param[in] B - Matrix being. - */ -void MatrixMatrixAddition(ScalarType alpha, const CSysMatrix& B); + /*! + * \brief Add a scaled sparse matrix to "this" (axpy-type operation, A = A+alpha*B). + * \note Matrices must have the same sparse pattern. + * \param[in] alpha - The scaling constant. + * \param[in] B - Matrix being. + */ + void MatrixMatrixAddition(ScalarType alpha, const CSysMatrix& B); -/*! - * \brief Performs the product of a sparse matrix by a CSysVector. - * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[out] prod - Result of the product. - */ -void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Performs the product of a sparse matrix by a CSysVector. + * \param[in] vec - CSysVector to be multiplied by the sparse matrix A. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[out] prod - Result of the product. + */ + void MatrixVectorProduct(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; -/*! - * \brief Build the Jacobi preconditioner. - */ -void BuildJacobiPreconditioner(); + /*! + * \brief Build the Jacobi preconditioner. + */ + void BuildJacobiPreconditioner(); -/*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ -void ComputeJacobiPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ + void ComputeJacobiPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; -/*! - * \brief Build the ILU preconditioner. - */ -void BuildILUPreconditioner(); + /*! + * \brief Build the ILU preconditioner. + */ + void BuildILUPreconditioner(); -/*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ -void ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ + void ComputeILUPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; -/*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ -void ComputeLU_SGSPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + */ + void ComputeLU_SGSPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; -/*! - * \brief Build the Linelet preconditioner. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ -void BuildLineletPreconditioner(const CGeometry* geometry, const CConfig* config); + /*! + * \brief Build the Linelet preconditioner. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ + void BuildLineletPreconditioner(const CGeometry* geometry, const CConfig* config); -/*! - * \brief Multiply CSysVector by the preconditioner - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product A*vec. - */ -void ComputeLineletPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; + /*! + * \brief Multiply CSysVector by the preconditioner + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product A*vec. + */ + void ComputeLineletPreconditioner(const CSysVector& vec, CSysVector& prod, + CGeometry* geometry, const CConfig* config) const; -/*! - * \brief Compute the linear residual. - * \param[in] sol - Solution (x). - * \param[in] f - Right hand side (b). - * \param[out] res - Residual (Ax-b). - */ -void ComputeResidual(const CSysVector& sol, const CSysVector& f, - CSysVector& res) const; + /*! + * \brief Compute the linear residual. + * \param[in] sol - Solution (x). + * \param[in] f - Right hand side (b). + * \param[out] res - Residual (Ax-b). + */ + void ComputeResidual(const CSysVector& sol, const CSysVector& f, + CSysVector& res) const; -/*! - * \brief Factorize matrix using PaStiX. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - * \param[in] kind_fact - Type of factorization. - */ -void BuildPastixPreconditioner(CGeometry* geometry, const CConfig* config, unsigned short kind_fact); + /*! + * \brief Factorize matrix using PaStiX. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + * \param[in] kind_fact - Type of factorization. + */ + void BuildPastixPreconditioner(CGeometry* geometry, const CConfig* config, unsigned short kind_fact); -/*! - * \brief Apply the PaStiX factorization to CSysVec. - * \param[in] vec - CSysVector to be multiplied by the preconditioner. - * \param[out] prod - Result of the product M*vec. - * \param[in] geometry - Geometrical definition of the problem. - * \param[in] config - Definition of the particular problem. - */ -void ComputePastixPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, - const CConfig* config) const; -} -; + /*! + * \brief Apply the PaStiX factorization to CSysVec. + * \param[in] vec - CSysVector to be multiplied by the preconditioner. + * \param[out] prod - Result of the product M*vec. + * \param[in] geometry - Geometrical definition of the problem. + * \param[in] config - Definition of the particular problem. + */ + void ComputePastixPreconditioner(const CSysVector& vec, CSysVector& prod, CGeometry* geometry, + const CConfig* config) const; +}; From 8d03d76b3e96b57c7bc12f9dd784e27123aa549e Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 22:27:32 -0700 Subject: [PATCH 13/14] fix and clean --- Common/src/linear_algebra/CSysMatrix.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index 72d0047b00f..56ddd6088b2 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -141,9 +141,11 @@ CSysMatrix::~CSysMatrix() { GPUMemoryAllocation::gpu_free(gpu_ilu.col_ind_u); GPUMemoryAllocation::gpu_free(d_ilu_color_idx); GPUMemoryAllocation::gpu_free(d_ilu_backward_rhs); +#ifdef SU2_ENABLE_CUDA_KERNELS if (ilu_build_graph_exec != nullptr) cudaGraphExecDestroy(ilu_build_graph_exec); if (ilu_apply_graph_exec != nullptr) cudaGraphExecDestroy(ilu_apply_graph_exec); if (ilu_stream != nullptr) cudaStreamDestroy(ilu_stream); +#endif } #ifdef USE_MKL @@ -287,8 +289,9 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi ilu.col_ind_u = pat_ilu.u.innerIdx(); ilu.nnz_u = pat_ilu.u.getNumNonZeros(); - /*--- The GPU implementation is only level-scheduled, so the levels are not optional there. ---*/ - if (useCuda || (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels())) { + /*--- Only the host/OMP path uses levels; the GPU path is colored-iterative and never + * touches levels_ilu (see color_ilu below). ---*/ + if (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels()) { /*--- The pattern spans all points but only the domain rows are factorized, so drop the * halo rows. This cannot change the levels of the domain rows: a row can only depend on * rows with a lower index, and every halo row has a higher index than every domain row. ---*/ From ca7dfc72bc3cb9aa47581d46b87fa49449cb76db Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Sun, 2 Aug 2026 22:31:14 -0700 Subject: [PATCH 14/14] clean --- Common/src/linear_algebra/CSysMatrix.cpp | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/Common/src/linear_algebra/CSysMatrix.cpp b/Common/src/linear_algebra/CSysMatrix.cpp index 56ddd6088b2..35eb70ea641 100644 --- a/Common/src/linear_algebra/CSysMatrix.cpp +++ b/Common/src/linear_algebra/CSysMatrix.cpp @@ -289,23 +289,8 @@ void CSysMatrix::Initialize(unsigned long npoint, unsigned long npoi ilu.col_ind_u = pat_ilu.u.innerIdx(); ilu.nnz_u = pat_ilu.u.getNumNonZeros(); - /*--- Only the host/OMP path uses levels; the GPU path is colored-iterative and never - * touches levels_ilu (see color_ilu below). ---*/ if (omp_get_max_threads() > 1 && config->GetLinear_Solver_ILU_levels()) { - /*--- The pattern spans all points but only the domain rows are factorized, so drop the - * halo rows. This cannot change the levels of the domain rows: a row can only depend on - * rows with a lower index, and every halo row has a higher index than every domain row. ---*/ - const auto all_levels = computeLevels(pat_ilu.l); - std::vector> levels; - for (auto level = 0ul; level < all_levels.getOuterSize(); ++level) { - std::vector rows; - for (auto k = 0ul; k < all_levels.getNumNonZeros(level); ++k) { - const auto iPoint = all_levels.getInnerIdx(level, k); - if (iPoint < nPointDomain) rows.push_back(iPoint); - } - if (!rows.empty()) levels.push_back(std::move(rows)); - } - levels_ilu = CCompressedSparsePatternUL(levels); + levels_ilu = computeLevels(pat_ilu.l); } /*--- Coloring for the GPU iterative factorization, see IluFactorColorKernel. Colors are