start of MPI inclusion

lanl · Jul 30, 2024 · 77e1aa4 · 77e1aa4
1 parent a07cfaf
commit 77e1aa4
Show file tree

Hide file tree

Showing 29 changed files with 149 additions and 83 deletions.
diff --git a/lib/Elements b/lib/Elements
diff --git a/single-node-refactor/CMakeLists.txt b/single-node-refactor/CMakeLists.txt
@@ -33,9 +33,20 @@ if(NOT CMAKE_BUILD_TYPE)
  # set(CMAKE_BUILD_TYPE Release)
 endif(NOT CMAKE_BUILD_TYPE)
 
-include_directories(../lib/Elements/matar)
-include_directories(../lib/Elements/matar/src/include)
-add_subdirectory(../lib/Elements/matar cbin)
+###DANinclude_directories(../lib/Elements/matar)
+###DANinclude_directories(../lib/Elements/matar/src/include)
+
+###DANadd_subdirectory(../lib/Elements/matar cbin)
+if (MPI)
+    find_package(MPI REQUIRED)
+    add_definitions(-DHAVE_MPI=1)
+    list(APPEND LINKING_LIBRARIES MPI::MPI_CXX)
+    if (GPU_AWARE_MPI)
+        add_definitions(-DHAVE_GPU_AWARE_MPI)
+#    elseif (GPU_SHARED_MEM)
+#        add_definitions(-DHAVE_GPU_SHARED_MEM)
+    endif()
+endif()
 
 # include_directories(Mesh-Builder)
 # add_subdirectory(Mesh-Builder)

diff --git a/single-node-refactor/scripts/cmake_build.sh b/single-node-refactor/scripts/cmake_build.sh
@@ -19,6 +19,9 @@ if [ "$solver" = "SGH" ]; then
 else
     echo "Error: Solver not supported."
 fi
+cmake_options+=(
+    -D MPI=ON
+)
 
 # Print CMake options for reference
 echo "CMake Options: ${cmake_options[@]}"

diff --git a/single-node-refactor/scripts/matar-install.sh b/single-node-refactor/scripts/matar-install.sh
@@ -19,6 +19,9 @@ else
         -D Matar_ENABLE_KOKKOS=ON
     )
 fi
+cmake_options+=(
+    -D Matar_ENABLE_MPI=ON
+)
 
 # Print CMake options for reference
 echo "CMake Options: ${cmake_options[@]}"

diff --git a/single-node-refactor/src/CMakeLists.txt b/single-node-refactor/src/CMakeLists.txt
@@ -37,6 +37,10 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${VECTOR_CXX_FLAGS}")
 
 find_package(Kokkos REQUIRED)
 find_package(Matar REQUIRED)
+if (MPI)
+    find_package(MPI REQUIRED)
+    add_definitions(-DHAVE_MPI=1)
+endif()
 
 
 # if(BUILD_KOKKOS_SGH)
@@ -77,3 +81,6 @@ include_directories(common)
 
 add_executable(Fierro main.cpp solver.cpp )
 target_link_libraries(Fierro PRIVATE matar parse_yaml sgh_solver Kokkos::kokkos)
+if (MPI)
+    target_link_libraries(Fierro PRIVATE MPI::MPI_CXX)
+endif()
diff --git a/single-node-refactor/src/Solvers/SGH_solver/include/sgh_solver.h b/single-node-refactor/src/Solvers/SGH_solver/include/sgh_solver.h
@@ -136,21 +136,21 @@ class SGH : public Solver
     void boundary_velocity(
         const mesh_t& mesh,
         const BoundaryCondition_t& Boundary,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         const double time_value) const;
 
     void boundary_contact(
         const mesh_t& mesh,
         const BoundaryCondition_t& Boundary,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         const double time_value) const;
 
     // **** Functions defined in energy_sgh.cpp **** //
     void update_energy(
         double rk_alpha,
         double dt,
         const mesh_t& mesh,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& node_coords,
         DCArrayKokkos<double>& MaterialPoints_sie,
         const DCArrayKokkos<double>& MaterialPoints_mass,
@@ -161,7 +161,7 @@ class SGH : public Solver
         const Material_t& Materials,
         const mesh_t& mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& MaterialPoints_den,
         const DCArrayKokkos<double>& MaterialPoints_sie,
         const DCArrayKokkos<double>& MaterialPoints_pres,
@@ -182,7 +182,7 @@ class SGH : public Solver
         const Material_t& Materials,
         const mesh_t& mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& MaterialPoints_den,
         const DCArrayKokkos<double>& MaterialPoints_sie,
         const DCArrayKokkos<double>& MaterialPoints_pres,
@@ -205,22 +205,22 @@ class SGH : public Solver
         const size_t num_dims,
         const size_t num_nodes,
         DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel) const;
+        const MPIArrayKokkos<double>& node_vel) const;
 
     // **** Functions defined in momentum.cpp **** //
     void update_velocity(
         double rk_alpha,
         double dt,
         const mesh_t& mesh,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& node_mass,
         const DCArrayKokkos<double>& corner_force) const;
 
     KOKKOS_FUNCTION
     void get_velgrad(
         ViewCArrayKokkos<double>& vel_grad,
         const ViewCArrayKokkos<size_t>& elem_node_gids,
-        const DCArrayKokkos<double>&    node_vel,
+        const MPIArrayKokkos<double>&    node_vel,
         const ViewCArrayKokkos<double>& b_matrix,
         const double GaussPoints_vol,
         const size_t elem_gid) const;
@@ -229,7 +229,7 @@ class SGH : public Solver
     void get_velgrad2D(
         ViewCArrayKokkos<double>& vel_grad,
         const ViewCArrayKokkos<size_t>& elem_node_gids,
-        const DCArrayKokkos<double>&    node_vel,
+        const MPIArrayKokkos<double>&    node_vel,
         const ViewCArrayKokkos<double>& b_matrix,
         const double GaussPoints_vol,
         const double elem_area,
@@ -239,14 +239,14 @@ class SGH : public Solver
         DCArrayKokkos<double>& GaussPoints_div,
         const mesh_t mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& GaussPoints_vol) const;
 
     void get_divergence2D(
         DCArrayKokkos<double>& GaussPoints_div,
         const mesh_t mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         const DCArrayKokkos<double>& GaussPoints_vol) const;
 
     KOKKOS_FUNCTION
@@ -260,7 +260,7 @@ class SGH : public Solver
         const Material_t& Materials,
         const mesh_t& mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         DCArrayKokkos<double>& MaterialPoints_den,
         DCArrayKokkos<double>& MaterialPoints_pres,
         DCArrayKokkos<double>& MaterialPoints_stress,
@@ -278,7 +278,7 @@ class SGH : public Solver
         const Material_t& Materials,
         const mesh_t& mesh,
         const DCArrayKokkos<double>& node_coords,
-        const DCArrayKokkos<double>& node_vel,
+        const MPIArrayKokkos<double>& node_vel,
         DCArrayKokkos<double>& MaterialPoints_den,
         DCArrayKokkos<double>& MaterialPoints_pres,
         DCArrayKokkos<double>& MaterialPoints_stress,
@@ -295,7 +295,7 @@ class SGH : public Solver
     // NOTE: Consider pulling up
     void rk_init(
         DCArrayKokkos<double>& node_coords,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         DCArrayKokkos<double>& MaterialPoints_sie,
         DCArrayKokkos<double>& MaterialPoints_stress,
         const size_t num_dims,
@@ -305,7 +305,7 @@ class SGH : public Solver
     void get_timestep(
         mesh_t& mesh,
         DCArrayKokkos<double>& node_coords,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         DCArrayKokkos<double>& MaterialPoints_sspd,
         DCArrayKokkos<double>& GaussPoints_vol,
         double time_value,
@@ -320,7 +320,7 @@ class SGH : public Solver
     void get_timestep2D(
         mesh_t& mesh,
         DCArrayKokkos<double>& node_coords,
-        DCArrayKokkos<double>& node_vel,
+        MPIArrayKokkos<double>& node_vel,
         DCArrayKokkos<double>& MaterialPoints_sspd,
         DCArrayKokkos<double>& GaussPoints_vol,
         double time_value,
@@ -358,7 +358,7 @@ class SGH : public Solver
         const ViewCArrayKokkos<double>& vel_grad,
         const ViewCArrayKokkos<size_t>& elem_node_gids,
         const DCArrayKokkos<double>&    node_coords,
-        const DCArrayKokkos<double>&    node_vel,
+        const MPIArrayKokkos<double>&    node_vel,
         const double vol,
         const double dt,
         const double rk_alpha);

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/boundary.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/boundary.cpp
@@ -48,7 +48,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 /////////////////////////////////////////////////////////////////////////////
 void SGH::boundary_velocity(const mesh_t&      mesh,
                             const BoundaryCondition_t& BoundaryConditions,
-                            DCArrayKokkos<double>& node_vel,
+                            MPIArrayKokkos<double>& node_vel,
                             const double time_value) const
 {
 
@@ -94,7 +94,7 @@ void SGH::boundary_velocity(const mesh_t&      mesh,
 /////////////////////////////////////////////////////////////////////////////
 void SGH::boundary_contact(const mesh_t& mesh,
                            const BoundaryCondition_t& BoundaryConditions,
-                           DCArrayKokkos<double>& node_vel,
+                           MPIArrayKokkos<double>& node_vel,
                            const double time_value) const
 {
     return;

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/energy_sgh.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/energy_sgh.cpp
@@ -53,7 +53,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 void SGH::update_energy(double rk_alpha,
     double dt,
     const mesh_t& mesh,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& node_coords,
     DCArrayKokkos<double>& MaterialPoints_sie,
     const DCArrayKokkos<double>& MaterialPoints_mass,

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/force_sgh.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/force_sgh.cpp
@@ -62,7 +62,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 void SGH::get_force(const Material_t& Materials,
     const mesh_t& mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& MaterialPoints_den,
     const DCArrayKokkos<double>& MaterialPoints_sie,
     const DCArrayKokkos<double>& MaterialPoints_pres,
@@ -436,7 +436,7 @@ void SGH::get_force(const Material_t& Materials,
 void SGH::get_force_2D(const Material_t& Materials,
     const mesh_t& mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& MaterialPoints_den,
     const DCArrayKokkos<double>& MaterialPoints_sie,
     const DCArrayKokkos<double>& MaterialPoints_pres,

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/momentum.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/momentum.cpp
@@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 void SGH::update_velocity(double rk_alpha,
     double dt,
     const mesh_t& mesh,
-    DCArrayKokkos<double>& node_vel,
+    MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& node_mass,
     const DCArrayKokkos<double>& corner_force) const
 {
@@ -101,7 +101,7 @@ void SGH::update_velocity(double rk_alpha,
 KOKKOS_FUNCTION
 void SGH::get_velgrad(ViewCArrayKokkos<double>& vel_grad,
     const ViewCArrayKokkos<size_t>& elem_node_gids,
-    const DCArrayKokkos<double>&    node_vel,
+    const MPIArrayKokkos<double>&    node_vel,
     const ViewCArrayKokkos<double>& b_matrix,
     const double elem_vol,
     const size_t elem_gid) const
@@ -196,7 +196,7 @@ void SGH::get_velgrad(ViewCArrayKokkos<double>& vel_grad,
 KOKKOS_FUNCTION
 void SGH::get_velgrad2D(ViewCArrayKokkos<double>& vel_grad,
     const ViewCArrayKokkos<size_t>& elem_node_gids,
-    const DCArrayKokkos<double>&    node_vel,
+    const MPIArrayKokkos<double>&    node_vel,
     const ViewCArrayKokkos<double>& b_matrix,
     const double elem_vol,
     const double elem_area,
@@ -266,7 +266,7 @@ void SGH::get_velgrad2D(ViewCArrayKokkos<double>& vel_grad,
 void SGH::get_divergence(DCArrayKokkos<double>& elem_div,
     const mesh_t mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& elem_vol) const
 {
     // --- calculate the forces acting on the nodes from the element ---
@@ -342,7 +342,7 @@ void SGH::get_divergence(DCArrayKokkos<double>& elem_div,
 void SGH::get_divergence2D(DCArrayKokkos<double>& elem_div,
     const mesh_t mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     const DCArrayKokkos<double>& elem_vol) const
 {
     // --- calculate the forces acting on the nodes from the element ---

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/position.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/position.cpp
@@ -52,7 +52,7 @@ void SGH::update_position(double rk_alpha,
     const size_t num_dims,
     const size_t num_nodes,
     DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel) const
+    const MPIArrayKokkos<double>& node_vel) const
 {
     // loop over all the nodes in the mesh
     FOR_ALL(node_gid, 0, num_nodes, {
@@ -61,4 +61,4 @@ void SGH::update_position(double rk_alpha,
             node_coords(1, node_gid, dim) = node_coords(0, node_gid, dim) + rk_alpha * dt * half_vel;
         }
     }); // end parallel for over nodes
-} // end subroutine
+} // end subroutine
diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/properties.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/properties.cpp
@@ -60,7 +60,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 void SGH::update_state(const Material_t& Materials,
     const mesh_t& mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     DCArrayKokkos<double>& MaterialPoints_den,
     DCArrayKokkos<double>& MaterialPoints_pres,
     DCArrayKokkos<double>& MaterialPoints_stress,
@@ -220,7 +220,7 @@ void SGH::update_state(const Material_t& Materials,
 void SGH::update_state2D(const Material_t& Materials,
     const mesh_t& mesh,
     const DCArrayKokkos<double>& node_coords,
-    const DCArrayKokkos<double>& node_vel,
+    const MPIArrayKokkos<double>& node_vel,
     DCArrayKokkos<double>& MaterialPoints_den,
     DCArrayKokkos<double>& MaterialPoints_pres,
     DCArrayKokkos<double>& MaterialPoints_stress,

diff --git a/single-node-refactor/src/Solvers/SGH_solver/src/time_integration.cpp b/single-node-refactor/src/Solvers/SGH_solver/src/time_integration.cpp
@@ -50,7 +50,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ///
 /////////////////////////////////////////////////////////////////////////////
 void SGH::rk_init(DCArrayKokkos<double>& node_coords,
-    DCArrayKokkos<double>& node_vel,
+    MPIArrayKokkos<double>& node_vel,
     DCArrayKokkos<double>& MaterialPoints_sie,
     DCArrayKokkos<double>& MaterialPoints_stress,
     const size_t num_dims,
@@ -101,7 +101,7 @@ void SGH::rk_init(DCArrayKokkos<double>& node_coords,
 /////////////////////////////////////////////////////////////////////////////
 void SGH::get_timestep(mesh_t& mesh,
     DCArrayKokkos<double>&     node_coords,
-    DCArrayKokkos<double>&     node_vel,
+    MPIArrayKokkos<double>&     node_vel,
     DCArrayKokkos<double>&     MaterialPoints_sspd,
     DCArrayKokkos<double>&     GaussPoints_vol,
     double time_value,
@@ -211,7 +211,7 @@ void SGH::get_timestep(mesh_t& mesh,
 /////////////////////////////////////////////////////////////////////////////
 void SGH::get_timestep2D(mesh_t& mesh,
     DCArrayKokkos<double>& node_coords,
-    DCArrayKokkos<double>& node_vel,
+    MPIArrayKokkos<double>& node_vel,
     DCArrayKokkos<double>& MaterialPoints_sspd,
     DCArrayKokkos<double>& GaussPoints_vol,
     double time_value,

diff --git a/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/constant_velocity_bc.h
@@ -65,7 +65,7 @@ namespace ConstantVelocityBC {
                          const DCArrayKokkos <BoundaryConditionEnums_t>& BoundaryConditionEnums,
                          const DCArrayKokkos<double>& bc_global_vars,
                          const DCArrayKokkos<double>& bc_state_vars,
-                         const DCArrayKokkos<double>& node_vel,
+                         const MPIArrayKokkos<double>& node_vel,
                          const double time_value,
                          const size_t bdy_node_gid,
                          const size_t bdy_set)
@@ -82,4 +82,4 @@ namespace ConstantVelocityBC {
 
 } // end namespace
 
-#endif // end Header Guard
+#endif // end Header Guard
diff --git a/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h b/single-node-refactor/src/boundary_conditions/velocity/no_velocity_bc.h
@@ -63,7 +63,7 @@ namespace NoVelocityBC {
                          const DCArrayKokkos <BoundaryConditionEnums_t>& BoundaryConditionEnums,
                          const DCArrayKokkos<double>& bc_global_vars,
                          const DCArrayKokkos<double>& bc_state_vars,
-                         const DCArrayKokkos<double>& node_vel,
+                         const MPIArrayKokkos<double>& node_vel,
                          const double time_value,
                          const size_t bdy_node_gid,
                          const size_t bdy_set)
@@ -75,4 +75,4 @@ namespace NoVelocityBC {
 
 } // end namespace
 
-#endif // end Header Guard
+#endif // end Header Guard