From f561e846c7cc48242e1510207527dada1ee34e98 Mon Sep 17 00:00:00 2001 From: Chris Richardson Date: Wed, 15 Jul 2026 13:20:34 +0100 Subject: [PATCH 1/6] First draft --- cpp/dolfinx/fem/Function.h | 4 +--- cpp/dolfinx/fem/FunctionSpace.h | 25 +++++++++++++++++++++++-- cpp/dolfinx/la/Vector.h | 21 +++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/cpp/dolfinx/fem/Function.h b/cpp/dolfinx/fem/Function.h index bc8e2eb58f5..bb5c508430f 100644 --- a/cpp/dolfinx/fem/Function.h +++ b/cpp/dolfinx/fem/Function.h @@ -55,9 +55,7 @@ class Function /// @brief Create function on given function space. /// @param[in] V The function space explicit Function(std::shared_ptr> V) - : _function_space(V), _x(std::make_shared>( - V->dofmaps().front()->index_map, - V->dofmaps().front()->index_map_bs())) + : _function_space(V), _x(std::make_shared>(*V)) { if (!V->component().empty()) { diff --git a/cpp/dolfinx/fem/FunctionSpace.h b/cpp/dolfinx/fem/FunctionSpace.h index 81d30e3f6d8..bc665459a79 100644 --- a/cpp/dolfinx/fem/FunctionSpace.h +++ b/cpp/dolfinx/fem/FunctionSpace.h @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -45,7 +46,10 @@ class FunctionSpace std::shared_ptr> element, std::shared_ptr dofmap) : _mesh(mesh), _elements{element}, _dofmaps{std::move(dofmap)}, - _id(boost::uuids::random_generator()()), _root_space_id(_id) + _id(boost::uuids::random_generator()()), _root_space_id(_id), + _scatterer( + std::make_shared>>( + *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())) { // Do nothing } @@ -63,7 +67,10 @@ class FunctionSpace std::vector>> elements, std::vector> dofmaps) : _mesh(mesh), _elements(elements), _dofmaps(std::move(dofmaps)), - _id(boost::uuids::random_generator()()), _root_space_id(_id) + _id(boost::uuids::random_generator()()), _root_space_id(_id), + _scatterer( + std::make_shared>>( + *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())) { std::vector cell_types = mesh->topology()->cell_types(); std::size_t num_cell_types = cell_types.size(); @@ -400,6 +407,16 @@ class FunctionSpace return _dofmaps; } + /// @brief The scatterer associated with the function space. + /// + /// Used to scatter data between the local (owned + ghost) dof layout + /// and the global layout. + std::shared_ptr>> + scatterer() const + { + return _scatterer; + } + private: // The mesh std::shared_ptr> _mesh; @@ -410,6 +427,10 @@ class FunctionSpace // The dofmap std::vector> _dofmaps; + // An associated scatterer for the function space. This is used to scatter + // data from the local dof layout to the global dof layout and vice versa. + std::shared_ptr>> _scatterer; + // The component w.r.t. to root space std::vector _component; diff --git a/cpp/dolfinx/la/Vector.h b/cpp/dolfinx/la/Vector.h index afbdb10cd23..5a6ff13cf63 100644 --- a/cpp/dolfinx/la/Vector.h +++ b/cpp/dolfinx/la/Vector.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,12 @@ #include #include +namespace dolfinx::fem +{ +template +class FunctionSpace; +} // namespace dolfinx::fem + namespace dolfinx::la { /// @brief la::Vector scatter pack/unpack function concept. @@ -137,6 +144,20 @@ class Vector { } + /// @brief Create a distributed vector from a FunctionSpace, inheriting a + /// shared Scatterer. + /// @param V FunctionSpace defining the block size and dofmap layout of the + /// vector, and sharing a common Scatterer. + template + Vector(const fem::FunctionSpace& V) + : _map(V.dofmap()->index_map), _bs(V.dofmap()->bs()), + _x(_bs * (_map->size_local() + _map->num_ghosts())), + _scatterer(V.scatterer()), + _buffer_local(_scatterer->local_indices().size()), + _buffer_remote(_scatterer->remote_indices().size()) + { + } + /// Copy constructor Vector(const Vector& x) = default; From 5e8dced3866da9fb2d1defc8d6bb77f688deba38 Mon Sep 17 00:00:00 2001 From: Chris Richardson Date: Wed, 15 Jul 2026 14:58:45 +0100 Subject: [PATCH 2/6] Template ScatterT --- cpp/dolfinx/fem/FunctionSpace.h | 21 ++++++++------------- cpp/dolfinx/fem/assembler.h | 2 +- cpp/dolfinx/io/vtk_utils.h | 2 +- cpp/dolfinx/la/Vector.h | 4 ++-- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/cpp/dolfinx/fem/FunctionSpace.h b/cpp/dolfinx/fem/FunctionSpace.h index bc665459a79..e98e548235a 100644 --- a/cpp/dolfinx/fem/FunctionSpace.h +++ b/cpp/dolfinx/fem/FunctionSpace.h @@ -30,7 +30,8 @@ namespace dolfinx::fem /// degrees-of-freedom. /// @tparam T The floating point (real) type of the mesh geometry and /// the finite element basis. -template +template >> class FunctionSpace { public: @@ -47,9 +48,8 @@ class FunctionSpace std::shared_ptr dofmap) : _mesh(mesh), _elements{element}, _dofmaps{std::move(dofmap)}, _id(boost::uuids::random_generator()()), _root_space_id(_id), - _scatterer( - std::make_shared>>( - *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())) + _scatterer(std::make_shared(*_dofmaps.front()->index_map, + _dofmaps.front()->index_map_bs())) { // Do nothing } @@ -68,9 +68,8 @@ class FunctionSpace std::vector> dofmaps) : _mesh(mesh), _elements(elements), _dofmaps(std::move(dofmaps)), _id(boost::uuids::random_generator()()), _root_space_id(_id), - _scatterer( - std::make_shared>>( - *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())) + _scatterer(std::make_shared(*_dofmaps.front()->index_map, + _dofmaps.front()->index_map_bs())) { std::vector cell_types = mesh->topology()->cell_types(); std::size_t num_cell_types = cell_types.size(); @@ -411,11 +410,7 @@ class FunctionSpace /// /// Used to scatter data between the local (owned + ghost) dof layout /// and the global layout. - std::shared_ptr>> - scatterer() const - { - return _scatterer; - } + std::shared_ptr scatterer() const { return _scatterer; } private: // The mesh @@ -429,7 +424,7 @@ class FunctionSpace // An associated scatterer for the function space. This is used to scatter // data from the local dof layout to the global dof layout and vice versa. - std::shared_ptr>> _scatterer; + std::shared_ptr _scatterer; // The component w.r.t. to root space std::vector _component; diff --git a/cpp/dolfinx/fem/assembler.h b/cpp/dolfinx/fem/assembler.h index 0654e108e0d..8fec0919a6f 100644 --- a/cpp/dolfinx/fem/assembler.h +++ b/cpp/dolfinx/fem/assembler.h @@ -37,7 +37,7 @@ template class Expression; template class Form; -template +template class FunctionSpace; /// @brief Evaluate an Expression on cells or facets. diff --git a/cpp/dolfinx/io/vtk_utils.h b/cpp/dolfinx/io/vtk_utils.h index 001c2b6ed59..5e61938b38a 100644 --- a/cpp/dolfinx/io/vtk_utils.h +++ b/cpp/dolfinx/io/vtk_utils.h @@ -28,7 +28,7 @@ namespace dolfinx { namespace fem { -template +template class FunctionSpace; } diff --git a/cpp/dolfinx/la/Vector.h b/cpp/dolfinx/la/Vector.h index 5a6ff13cf63..71993b36d27 100644 --- a/cpp/dolfinx/la/Vector.h +++ b/cpp/dolfinx/la/Vector.h @@ -23,7 +23,7 @@ namespace dolfinx::fem { -template +template class FunctionSpace; } // namespace dolfinx::fem @@ -149,7 +149,7 @@ class Vector /// @param V FunctionSpace defining the block size and dofmap layout of the /// vector, and sharing a common Scatterer. template - Vector(const fem::FunctionSpace& V) + Vector(const fem::FunctionSpace>& V) : _map(V.dofmap()->index_map), _bs(V.dofmap()->bs()), _x(_bs * (_map->size_local() + _map->num_ghosts())), _scatterer(V.scatterer()), From 57b0feccca69996ada1b18debfda0706fd24bc5e Mon Sep 17 00:00:00 2001 From: Chris Richardson Date: Wed, 15 Jul 2026 15:15:53 +0100 Subject: [PATCH 3/6] order --- cpp/dolfinx/fem/FunctionSpace.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/dolfinx/fem/FunctionSpace.h b/cpp/dolfinx/fem/FunctionSpace.h index e98e548235a..1b2c2402fc2 100644 --- a/cpp/dolfinx/fem/FunctionSpace.h +++ b/cpp/dolfinx/fem/FunctionSpace.h @@ -47,9 +47,10 @@ class FunctionSpace std::shared_ptr> element, std::shared_ptr dofmap) : _mesh(mesh), _elements{element}, _dofmaps{std::move(dofmap)}, - _id(boost::uuids::random_generator()()), _root_space_id(_id), - _scatterer(std::make_shared(*_dofmaps.front()->index_map, - _dofmaps.front()->index_map_bs())) + _scatterer(std::make_shared( + *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())), + _id(boost::uuids::random_generator()()), _root_space_id(_id) + { // Do nothing } @@ -67,9 +68,9 @@ class FunctionSpace std::vector>> elements, std::vector> dofmaps) : _mesh(mesh), _elements(elements), _dofmaps(std::move(dofmaps)), - _id(boost::uuids::random_generator()()), _root_space_id(_id), - _scatterer(std::make_shared(*_dofmaps.front()->index_map, - _dofmaps.front()->index_map_bs())) + _scatterer(std::make_shared( + *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())), + _id(boost::uuids::random_generator()()), _root_space_id(_id) { std::vector cell_types = mesh->topology()->cell_types(); std::size_t num_cell_types = cell_types.size(); From f071e863a15f78bb993256dd436a27d609ee4dae Mon Sep 17 00:00:00 2001 From: Chris Richardson Date: Tue, 21 Jul 2026 14:35:05 +0100 Subject: [PATCH 4/6] fixes --- cpp/dolfinx/fem/FunctionSpace.h | 2 ++ cpp/dolfinx/fem/assembler.h | 2 -- cpp/dolfinx/io/vtk_utils.h | 6 ------ cpp/dolfinx/la/Vector.h | 3 ++- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cpp/dolfinx/fem/FunctionSpace.h b/cpp/dolfinx/fem/FunctionSpace.h index 1b2c2402fc2..a0796be8b05 100644 --- a/cpp/dolfinx/fem/FunctionSpace.h +++ b/cpp/dolfinx/fem/FunctionSpace.h @@ -30,6 +30,8 @@ namespace dolfinx::fem /// degrees-of-freedom. /// @tparam T The floating point (real) type of the mesh geometry and /// the finite element basis. +/// @tparam ScatterT Scatterer type used to scatter/gather dof data +/// between the local (owned + ghost) and global dof layouts. template >> class FunctionSpace diff --git a/cpp/dolfinx/fem/assembler.h b/cpp/dolfinx/fem/assembler.h index 2e3aa80f541..0e6199562b5 100644 --- a/cpp/dolfinx/fem/assembler.h +++ b/cpp/dolfinx/fem/assembler.h @@ -37,8 +37,6 @@ template class Expression; template class Form; -template -class FunctionSpace; /// @brief Evaluate an Expression on cells or facets. /// diff --git a/cpp/dolfinx/io/vtk_utils.h b/cpp/dolfinx/io/vtk_utils.h index 5e61938b38a..b4578408eb8 100644 --- a/cpp/dolfinx/io/vtk_utils.h +++ b/cpp/dolfinx/io/vtk_utils.h @@ -26,12 +26,6 @@ namespace dolfinx { -namespace fem -{ -template -class FunctionSpace; -} - namespace mesh { enum class CellType : std::int8_t; diff --git a/cpp/dolfinx/la/Vector.h b/cpp/dolfinx/la/Vector.h index 729dbeb0723..e7e0511a1dc 100644 --- a/cpp/dolfinx/la/Vector.h +++ b/cpp/dolfinx/la/Vector.h @@ -146,11 +146,12 @@ class Vector /// @brief Create a distributed vector from a FunctionSpace, inheriting a /// shared Scatterer. + /// @tparam U Floating point (geometry) type of the FunctionSpace. /// @param V FunctionSpace defining the block size and dofmap layout of the /// vector, and sharing a common Scatterer. template Vector(const fem::FunctionSpace>& V) - : _map(V.dofmap()->index_map), _bs(V.dofmap()->bs()), + : _map(V.dofmap()->index_map), _bs(V.dofmap()->index_map_bs()), _x(_bs * (_map->size_local() + _map->num_ghosts())), _scatterer(V.scatterer()), _buffer_local(_scatterer->local_indices().size()), From 9b54d90a09b869f1999696851a05ae7dafde4604 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Wed, 29 Jul 2026 22:24:07 +0100 Subject: [PATCH 5/6] Address review feedback on shared-Scatterer changes - Function::collapse() built its new Vector via the two-argument (map, bs) constructor, which constructs a fresh Scatterer instead of reusing the one already built by the collapsed FunctionSpace. Use the FunctionSpace-based Vector constructor instead. - Mark the new FunctionSpace-based Vector constructor explicit to match the existing converting-constructor convention and avoid accidental implicit conversions. - Remove a stray blank line before the first FunctionSpace constructor's body. Co-Authored-By: Claude Sonnet 5 --- cpp/dolfinx/fem/Function.h | 3 +-- cpp/dolfinx/fem/FunctionSpace.h | 1 - cpp/dolfinx/la/Vector.h | 3 ++- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cpp/dolfinx/fem/Function.h b/cpp/dolfinx/fem/Function.h index bb5c508430f..e84702d181d 100644 --- a/cpp/dolfinx/fem/Function.h +++ b/cpp/dolfinx/fem/Function.h @@ -120,8 +120,7 @@ class Function auto [V, map] = _function_space->collapse(); // Create new vector - auto x = std::make_shared>( - V.dofmap()->index_map, V.dofmap()->index_map_bs()); + auto x = std::make_shared>(V); // Copy values into new vector std::span x_old = _x->array(); diff --git a/cpp/dolfinx/fem/FunctionSpace.h b/cpp/dolfinx/fem/FunctionSpace.h index a0796be8b05..f84d86d5951 100644 --- a/cpp/dolfinx/fem/FunctionSpace.h +++ b/cpp/dolfinx/fem/FunctionSpace.h @@ -52,7 +52,6 @@ class FunctionSpace _scatterer(std::make_shared( *_dofmaps.front()->index_map, _dofmaps.front()->index_map_bs())), _id(boost::uuids::random_generator()()), _root_space_id(_id) - { // Do nothing } diff --git a/cpp/dolfinx/la/Vector.h b/cpp/dolfinx/la/Vector.h index e7e0511a1dc..2dce824fcdf 100644 --- a/cpp/dolfinx/la/Vector.h +++ b/cpp/dolfinx/la/Vector.h @@ -150,7 +150,8 @@ class Vector /// @param V FunctionSpace defining the block size and dofmap layout of the /// vector, and sharing a common Scatterer. template - Vector(const fem::FunctionSpace>& V) + explicit Vector( + const fem::FunctionSpace>& V) : _map(V.dofmap()->index_map), _bs(V.dofmap()->index_map_bs()), _x(_bs * (_map->size_local() + _map->num_ghosts())), _scatterer(V.scatterer()), From 65d086d30a35401918a2170c4d197297fbe07cd4 Mon Sep 17 00:00:00 2001 From: "Garth N. Wells" Date: Wed, 29 Jul 2026 22:46:24 +0100 Subject: [PATCH 6/6] Fix tag collision in point-to-point Scatterer scatter The point-to-point scatter_fwd_begin/scatter_rev_begin overloads share communicator _comm0 for both directions, but used a fixed send tag (0) paired with MPI_ANY_TAG on the matching receive. That relies on messages being posted in the same relative order on every rank to avoid a forward message being matched against a reverse receive (or vice versa) - an assumption that no longer holds now that a Scatterer can be shared across sibling Vectors on the same FunctionSpace, which allows independent scatters to be interleaved in either order. Give forward and reverse each a fixed, distinct tag instead, so message matching no longer depends on posting order. Co-Authored-By: Claude Sonnet 5 --- cpp/dolfinx/common/Scatterer.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/cpp/dolfinx/common/Scatterer.h b/cpp/dolfinx/common/Scatterer.h index 2efb61f6c32..f1af073b213 100644 --- a/cpp/dolfinx/common/Scatterer.h +++ b/cpp/dolfinx/common/Scatterer.h @@ -294,7 +294,7 @@ class Scatterer for (std::size_t i = 0; i < _src.size(); ++i) { int ierr = MPI_Irecv(recv_buffer + _displs_remote[i], _sizes_remote[i], - dolfinx::MPI::mpi_t, _src[i], MPI_ANY_TAG, + dolfinx::MPI::mpi_t, _src[i], p2p_tag_fwd, _comm0.comm(), &requests[i]); dolfinx::MPI::check_error(_comm0.comm(), ierr); } @@ -302,8 +302,8 @@ class Scatterer for (std::size_t i = 0; i < _dest.size(); ++i) { int ierr = MPI_Isend(send_buffer + _displs_local[i], _sizes_local[i], - dolfinx::MPI::mpi_t, _dest[i], 0, _comm0.comm(), - &requests[i + _src.size()]); + dolfinx::MPI::mpi_t, _dest[i], p2p_tag_fwd, + _comm0.comm(), &requests[i + _src.size()]); dolfinx::MPI::check_error(_comm0.comm(), ierr); } } @@ -380,7 +380,7 @@ class Scatterer for (std::size_t i = 0; i < _dest.size(); i++) { int ierr = MPI_Irecv(recv_buffer + _displs_local[i], _sizes_local[i], - dolfinx::MPI::mpi_t, _dest[i], MPI_ANY_TAG, + dolfinx::MPI::mpi_t, _dest[i], p2p_tag_rev, _comm0.comm(), &requests[i]); dolfinx::MPI::check_error(_comm0.comm(), ierr); } @@ -390,8 +390,8 @@ class Scatterer for (std::size_t i = 0; i < _src.size(); i++) { int ierr = MPI_Isend(send_buffer + _displs_remote[i], _sizes_remote[i], - dolfinx::MPI::mpi_t, _src[i], 0, _comm0.comm(), - &requests[i + _dest.size()]); + dolfinx::MPI::mpi_t, _src[i], p2p_tag_rev, + _comm0.comm(), &requests[i + _dest.size()]); dolfinx::MPI::check_error(_comm0.comm(), ierr); } } @@ -495,6 +495,14 @@ class Scatterer } private: + // MPI tags for point-to-point forward/reverse scatter messages. + // Both directions share communicator _comm0, so a fixed, distinct + // tag per direction is required to stop a forward message being + // matched against a reverse receive (or vice versa) irrespective of + // the relative order in which the two directions are posted. + static constexpr int p2p_tag_fwd = 0; + static constexpr int p2p_tag_rev = 1; + // Communicator where the source ranks own the indices in the callers // halo, and the destination ranks 'ghost' indices owned by the // caller. I.e.,