diff --git a/README.md b/README.md index 7dceace..9e8b036 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ Installing Branson: - MPI 3.0+ ([OpenMPI 1.10+](https://www.open-mpi.org/software/ompi/), [mpich](http://www.mpich.org), etc.) - [Metis](http://glaros.dtc.umn.edu/gkhome/metis/metis/overview) - - [Parmetis](http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview) - Optional dependencies (needed for visualization) - [HDF5](https://support.hdfgroup.org/HDF5/) - [Silo](http://wci.llnl.gov/simulation/computer-codes/silo) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 104628c..a057d2e 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -17,7 +17,7 @@ MAINTAINER KineticTheory "https://github.com/KineticTheory" ENV DEBIAN_FRONTEND noninteractive ENV DEBCONF_NONINTERACTIVE_SEEN true ENV SPACK_ROOT=/vendors/spack -ENV DRACO_TPL="cmake@3.11.4 random123@1.09 openmpi@3.1.1 netlib-lapack@3.8.0 metis@5.1.0 parmetis@4.0.3" +ENV DRACO_TPL="cmake@3.11.4 random123@1.09 openmpi@3.1.1 netlib-lapack@3.8.0 metis@5.1.0" ENV FORCE_UNSAFE_CONFIGURE=1 ENV DISTRO=bionic ENV CLANG_FORMAT_VER=6.0 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4ac838a..259c410 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ ################################################################################ # Load the modules for each libary and the environment variables will be set -# correctly (load cmake, compiler, mpi, metis, parmetis and optionally +# correctly (load cmake, compiler, mpi, metis, and optionally # hdf5 and silo) # Enbable debug mode by passing -DCMAKE_BUILD_TYPE=Debug to CMake, default is @@ -89,7 +89,7 @@ elseif( CMAKE_CXX_COMPILER_ID STREQUAL "Cray" ) endif() #------------------------------------------------------------------------------# -# Look for Third Party Libraries (metis, parmetis, etc.) +# Look for Third Party Libraries (metis, etc.) add_subdirectory(pugixml) include(find_tpls) @@ -143,10 +143,9 @@ add_executable(BRANSON main.cc) target_include_directories( BRANSON PRIVATE $ ${PROJECT_SOURCE_DIR}/pugixml/src/) set( branson_deps - ParMETIS::parmetis MPI::MPI_CXX MPI::MPI_C - METIS::metis + METIS::metis pugixml) if( VIZ_LIBRARIES_FOUND ) set( branson_deps "Silo::silo;${HDF5_LIBRARIES};${branson_deps}" ) diff --git a/src/config/FindParMETIS.cmake b/src/config/FindParMETIS.cmake deleted file mode 100644 index d3d8a9a..0000000 --- a/src/config/FindParMETIS.cmake +++ /dev/null @@ -1,200 +0,0 @@ -#.rst: -# FindParMetis -# -------- -# -# Find the ParMETIS and METIS includes and libraries. -# -# ParMETIS is an MPI-based parallel library that implements a variety of -# algorithms for partitioning unstructured graphs, meshes, and for computing -# fill-reducing orderings of sparse matrices. ParMETIS extends the functionality -# provided by METIS and includes routines that are especially suited for -# parallel AMR computations and large scale numerical simulations. The -# algorithms implemented in ParMETIS are based on the parallel multilevel k-way -# graph-partitioning, adaptive repartitioning, and parallel multi- constrained -# partitioning schemes developed in our lab. -# http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview -# -# Imported Targets -# ^^^^^^^^^^^^^^^^ -# -# If ParMETIS is found, this module defines the following :prop_tgt:`IMPORTED` -# targetts:: -# -# ParMETIS::parmetis - The main ParMETIS library. -# -# Result Variables -# ^^^^^^^^^^^^^^^^ -# -# This module will set the following variables in your project:: -# -# ParMETIS_FOUND - True if ParMETIS found on the local system -# ParMETIS_INCLUDE_DIRS - Location of ParMETIS header files. -# ParMETIS_LIBRARIES - The ParMETIS libraries. -# ParMETIS_VERSION - The version of the discovered ParMETIS install. -# -# Hints -# ^^^^^ -# -# Set ``ParMETIS_ROOT_DIR`` to a directory that contains a ParMETIS -# installation. -# -# This script expects to find libraries at ``$ParMETIS_ROOT_DIR/lib`` and the -# ParMETIS headers at ``$ParMETIS_ROOT_DIR/include``. The library directory may -# optionally provide Release and Debug folders. -# -# Cache Variables -# ^^^^^^^^^^^^^^^ -# -# This module may set the following variables depending on platform and type of -# ParMETIS installation discovered. These variables may optionally be set to -# help this module find the correct files:: -# -# ParMETIS_LIBRARY - Location of the ParMETIS library. -# ParMETIS_LIBRARY_DEBUG - Location of the debug ParMETIS library -# (if any). -# - -#============================================================================= -# Copyright 2016 Kelly Thompson -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - -# Include these modules to handle the QUIETLY and REQUIRED arguments. -include(FindPackageHandleStandardArgs) - -#============================================================================= -# Prerequisite: -# ParMETIS requires that METIS be found. -find_package( METIS QUIET ) - -#============================================================================= -# If the user has provided ``ParMETIS_ROOT_DIR``, use it! Choose items found -# at this location over system locations. -if( EXISTS "$ENV{ParMETIS_ROOT_DIR}" ) - file( TO_CMAKE_PATH "$ENV{ParMETIS_ROOT_DIR}" ParMETIS_ROOT_DIR ) - set( ParMETIS_ROOT_DIR "${ParMETIS_ROOT_DIR}" CACHE PATH - "Prefix for ParMETIS installation." ) -endif() - -#============================================================================= -# Set ParMETIS_INCLUDE_DIRS and ParMETIS_LIBRARIES. Try to find the libraries -# at $ParMETIS_ROOT_DIR (if provided) or in standard system locations. These -# find_library and find_path calls will prefer custom locations over standard -# locations (HINTS). If the requested file is not found at the HINTS -# location, standard system locations will be still be searched (/usr/lib64 -# (Redhat), lib/i386-linux-gnu (Debian)). - -find_path( ParMETIS_INCLUDE_DIR - NAMES parmetis.h - HINTS ${ParMETIS_ROOT_DIR}/include ${ParMETIS_INCLUDEDIR} - PATH_SUFFIXES Release Debug -) -find_library( ParMETIS_LIBRARY - NAMES parmetis - HINTS ${ParMETIS_ROOT_DIR}/lib ${ParMETIS_LIBDIR} - PATH_SUFFIXES Release Debug -) -# Do we also have debug versions? -find_library( ParMETIS_LIBRARY_DEBUG - NAMES parmetis - HINTS ${ParMETIS_ROOT_DIR}/lib ${ParMETIS_LIBDIR} - PATH_SUFFIXES Debug -) -set( ParMETIS_INCLUDE_DIRS ${ParMETIS_INCLUDE_DIR} ) -set( ParMETIS_LIBRARIES ${ParMETIS_LIBRARY} ) - -# Try to find the version. -if( NOT ParMETIS_VERSION ) - if( EXISTS "${ParMETIS_INCLUDE_DIRS}/parmetis.h" ) - file( STRINGS "${ParMETIS_INCLUDE_DIRS}/parmetis.h" parmetis_h_major - REGEX "define PARMETIS_MAJOR" ) - file( STRINGS "${ParMETIS_INCLUDE_DIRS}/parmetis.h" parmetis_h_minor - REGEX "define PARMETIS_MINOR" ) - file( STRINGS "${ParMETIS_INCLUDE_DIRS}/parmetis.h" parmetis_h_subminor - REGEX "define PARMETIS_SUBMINOR" ) - string( REGEX REPLACE ".*([0-9]+)" "\\1" ParMETIS_MAJOR ${parmetis_h_major} ) - string( REGEX REPLACE ".*([0-9]+)" "\\1" ParMETIS_MINOR ${parmetis_h_minor} ) - string( REGEX REPLACE ".*([0-9]+)" "\\1" ParMETIS_SUBMINOR - ${parmetis_h_subminor} ) - endif() - # We might also try scraping the directory name for a regex match - # "parmetis-X.X.X" -endif() - -#============================================================================= -# handle the QUIETLY and REQUIRED arguments and set ParMETIS_FOUND to TRUE if -# all listed variables are TRUE. -find_package_handle_standard_args( ParMETIS - FOUND_VAR - ParMETIS_FOUND - REQUIRED_VARS - ParMETIS_INCLUDE_DIR - ParMETIS_LIBRARY - METIS_FOUND - VERSION_VAR - ParMETIS_VERSION - ) - -mark_as_advanced( ParMETIS_ROOT_DIR ParMETIS_VERSION ParMETIS_LIBRARY - ParMETIS_INCLUDE_DIR ParMETIS_LIBRARY_DEBUG ParMETIS_USE_PKGCONFIG - ParMETIS_CONFIG ) - -#============================================================================= -# Register imported libraries: -# 1. If we can find a Windows .dll file (or if we can find both Debug and -# Release libraries), we will set appropriate target properties for these. -# 2. However, for most systems, we will only register the import location and -# include directory. - -# Look for dlls, or Release and Debug libraries. -if(WIN32) - string( REPLACE ".lib" ".dll" ParMETIS_LIBRARY_DLL - "${ParMETIS_LIBRARY}" ) - string( REPLACE ".lib" ".dll" ParMETIS_LIBRARY_DEBUG_DLL - "${ParMETIS_LIBRARY_DEBUG}" ) -endif() - -if( ParMETIS_FOUND AND NOT TARGET ParMETIS::parmetis ) - if( EXISTS "${ParMETIS_LIBRARY_DLL}" ) - - # Windows systems with dll libraries. - add_library( ParMETIS::parmetis SHARED IMPORTED ) - - # Windows with dlls, but only Release libraries. - set_target_properties( ParMETIS::parmetis PROPERTIES - IMPORTED_LOCATION_RELEASE "${ParMETIS_LIBRARY_DLL}" - IMPORTED_IMPLIB "${ParMETIS_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${ParMETIS_INCLUDE_DIRS}" - IMPORTED_CONFIGURATIONS Release - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - INTERFACE_LINK_LIBRARIES METIS::metis ) - - # If we have both Debug and Release libraries - if( EXISTS "${ParMETIS_LIBRARY_DEBUG_DLL}" ) - set_property( TARGET ParMETIS::parmetis APPEND PROPERTY - IMPORTED_CONFIGURATIONS Debug ) - set_target_properties( ParMETIS::parmetis PROPERTIES - IMPORTED_LOCATION_DEBUG "${ParMETIS_LIBRARY_DEBUG_DLL}" - IMPORTED_IMPLIB_DEBUG "${ParMETIS_LIBRARY_DEBUG}" ) - endif() - - else() - - # For all other environments (ones without dll libraries), create the - # imported library targets. - add_library( ParMETIS::parmetis UNKNOWN IMPORTED ) - set_target_properties( ParMETIS::parmetis PROPERTIES - IMPORTED_LOCATION "${ParMETIS_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${ParMETIS_INCLUDE_DIRS}" - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - INTERFACE_LINK_LIBRARIES METIS::metis ) - endif() -endif() diff --git a/src/config/find_tpls.cmake b/src/config/find_tpls.cmake index 38e711b..4c2618d 100644 --- a/src/config/find_tpls.cmake +++ b/src/config/find_tpls.cmake @@ -42,8 +42,8 @@ macro(setupTPLs) #link_directories(${CALIPER_INSTALL_DIR}/lib) ############################################################################## - # metis and parmetis - # Load modules for metis and parmetis to get correct environment variables + # metis + # Load modules for metis to get correct environment variables ############################################################################## if( NOT TARGET METIS::metis ) @@ -65,26 +65,6 @@ macro(setupTPLs) endif() - if( NOT TARGET ParMETIS::parmetis ) - - message( STATUS "Looking for ParMETIS..." ) - find_package( ParMETIS QUIET REQUIRED ) - if( ParMETIS_FOUND ) - message( STATUS "Looking for ParMETIS..found ${ParMETIS_LIBRARY}" ) - else() - message( STATUS "Looking for ParMETIS..not found" ) - endif() - - set_package_properties( ParMETIS PROPERTIES - DESCRIPTION "MPI Parallel METIS" - TYPE REQUIRED - URL "http://glaros.dtc.umn.edu/gkhome/metis/parmetis/overview" - PURPOSE "ParMETIS is an MPI-based parallel library that implements a - variety of algorithms for partitioning unstructured graphs, meshes, and for - computing fill-reducing orderings of sparse matrices." ) - -endif() - ############################################################################## # Silo and HDF5 libraries # Load modules for hdf5 and solo to get correct environment variables diff --git a/src/constants.h b/src/constants.h index 46f06c8..d162b21 100644 --- a/src/constants.h +++ b/src/constants.h @@ -31,7 +31,7 @@ enum { CELL_PASS_RMA, REPLICATED }; //!< Parallel types -enum { PARMETIS, CUBE }; //!< Mesh decomposition method +enum { METIS, CUBE }; //!< Mesh decomposition method enum { EMISSION, INITIAL_CENSUS }; //!< Particle type for work packets const int grip_id_tag(1); //!< MPI tag for grip ID messages const int cell_id_tag(2); //!< MPI tag for requested cell ID messages diff --git a/src/decompose_mesh.h b/src/decompose_mesh.h index 58e1835..863e7e3 100644 --- a/src/decompose_mesh.h +++ b/src/decompose_mesh.h @@ -3,7 +3,7 @@ * \file decompose_mesh.h * \author Alex Long * \date June 17 2015 - * \brief Functions to decompose mesh with ParMetis + * \brief Functions to decompose mesh with Metis or cubes * \note Copyright (C) 2017 Los Alamos National Security, LLC. * All rights reserved */ @@ -14,9 +14,9 @@ #include #include +#include #include #include -#include #include #include #include @@ -48,9 +48,9 @@ void print_MPI_out(const Proto_Mesh &mesh, const uint32_t rank, //----------------------------------------------------------------------------// //----------------------------------------------------------------------------// -//! partition a mesh with parmetis -std::vector parmetis_partition(Proto_Mesh &mesh, int &edgecut, - const int rank, const int n_rank) { +//! partition a mesh with metis +std::vector metis_partition(Proto_Mesh &mesh, int &edgecut, const int rank, + const int n_rank, const MPI_Types &mpi_types) { using Constants::X_NEG; using Constants::X_POS; @@ -63,6 +63,10 @@ std::vector parmetis_partition(Proto_Mesh &mesh, int &edgecut, MPI_Comm comm; MPI_Comm_dup(MPI_COMM_WORLD, &comm); + constexpr int cell_tag = 10100; + constexpr int part_tag = 21023; + MPI_Datatype MPI_Proto_Cell = mpi_types.get_proto_cell_type(); + // get the number of cells on each processor // vtxdist has number of vertices on each rank, same for all ranks vector start_ncells(n_rank, 0); @@ -76,93 +80,128 @@ std::vector parmetis_partition(Proto_Mesh &mesh, int &edgecut, partial_sum(start_ncells.begin(), start_ncells.end(), vtxdist.begin()); vtxdist.insert(vtxdist.begin(), 0); - // build adjacency list needed for ParMetis call for each rank also get - // coordinates of cell centers for geometry based partitioning - float *xyz = new float[ncell_on_rank * 3]; - vector xadj; - vector adjncy; - int adjncy_ctr = 0; - Proto_Cell cell; - uint32_t g_ID; //! Global ID - for (uint32_t i = 0; i < ncell_on_rank; ++i) { - cell = mesh.get_pre_window_allocation_cell(i); - g_ID = cell.get_ID(); - cell.get_center(&xyz[i * 3]); - uint32_t xm_neighbor = cell.get_next_cell(X_NEG); - uint32_t xp_neighbor = cell.get_next_cell(X_POS); - uint32_t ym_neighbor = cell.get_next_cell(Y_NEG); - uint32_t yp_neighbor = cell.get_next_cell(Y_POS); - uint32_t zm_neighbor = cell.get_next_cell(Z_NEG); - uint32_t zp_neighbor = cell.get_next_cell(Z_POS); + uint32_t n_global_cells = vtxdist.back(); - xadj.push_back(adjncy_ctr); // starting index in xadj for this cell's nodes - if (xm_neighbor != g_ID) { - adjncy.push_back(xm_neighbor); - adjncy_ctr++; - } - if (xp_neighbor != g_ID) { - adjncy.push_back(xp_neighbor); - adjncy_ctr++; - } - if (ym_neighbor != g_ID) { - adjncy.push_back(ym_neighbor); - adjncy_ctr++; - } - if (yp_neighbor != g_ID) { - adjncy.push_back(yp_neighbor); - adjncy_ctr++; - } - if (zm_neighbor != g_ID) { - adjncy.push_back(zm_neighbor); - adjncy_ctr++; + // return partition of each cell on rank + std::vector part(ncell_on_rank); + + // non-root ranks gather send cells to root + if (rank != 0) { + const std::vector send_cells = + mesh.get_pre_window_allocation_cells(); + MPI_Send(&send_cells[0], ncell_on_rank, MPI_Proto_Cell, 0, cell_tag, + MPI_COMM_WORLD); + MPI_Recv(&part[0], ncell_on_rank, MPI_INT, 0, part_tag, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + edgecut = 1; + } + // root rank gathers all cells and uses METIS to setup partitions + else { + std::vector all_cells(n_global_cells); + const std::vector send_cells = + mesh.get_pre_window_allocation_cells(); + std::copy(send_cells.begin(), send_cells.end(), all_cells.begin()); + for (int irank = 1; irank < n_rank; ++irank) { + MPI_Recv(&all_cells[vtxdist[irank]], start_ncells[irank], MPI_Proto_Cell, + irank, cell_tag, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } - if (zp_neighbor != g_ID) { - adjncy.push_back(zp_neighbor); - adjncy_ctr++; + + // do partitioning + // build adjacency list needed for Metis call for each rank + vector xadj; + vector adjncy; + int adjncy_ctr = 0; + uint32_t g_ID; //! Global ID + for (auto &icell : all_cells) { + g_ID = icell.get_ID(); + uint32_t xm_neighbor = icell.get_next_cell(X_NEG); + uint32_t xp_neighbor = icell.get_next_cell(X_POS); + uint32_t ym_neighbor = icell.get_next_cell(Y_NEG); + uint32_t yp_neighbor = icell.get_next_cell(Y_POS); + uint32_t zm_neighbor = icell.get_next_cell(Z_NEG); + uint32_t zp_neighbor = icell.get_next_cell(Z_POS); + + xadj.push_back( + adjncy_ctr); // starting index in xadj for this cell's nodes + if (xm_neighbor != g_ID) { + adjncy.push_back(xm_neighbor); + adjncy_ctr++; + } + if (xp_neighbor != g_ID) { + adjncy.push_back(xp_neighbor); + adjncy_ctr++; + } + if (ym_neighbor != g_ID) { + adjncy.push_back(ym_neighbor); + adjncy_ctr++; + } + if (yp_neighbor != g_ID) { + adjncy.push_back(yp_neighbor); + adjncy_ctr++; + } + if (zm_neighbor != g_ID) { + adjncy.push_back(zm_neighbor); + adjncy_ctr++; + } + if (zp_neighbor != g_ID) { + adjncy.push_back(zp_neighbor); + adjncy_ctr++; + } } - } - xadj.push_back(adjncy_ctr); + xadj.push_back(adjncy_ctr); - int wgtflag = 0; // no weights for cells - int numflag = 0; // C-style numbering - int ncon = 1; - int nparts = n_rank; // sub-domains = n_rank + int ncon = 1; + int n_parts = n_rank; // sub-domains = n_rank - float *tpwgts = new float[nparts]; - for (int i = 0; i < nparts; ++i) - tpwgts[i] = 1.0 / float(nparts); + int rank_options[METIS_NOPTIONS]; - float *ubvec = new float[ncon]; - for (int i = 0; i < ncon; ++i) - ubvec[i] = 1.05; + METIS_SetDefaultOptions(rank_options); + rank_options[METIS_OPTION_NUMBERING] = 0; // C-style numbering + //rank_options[1] = 3; // output level + //rank_options[2] = 1242; // random number seed - int options[3]; - options[0] = 1; // 0--use default values, 1--use the values in 1 and 2 - options[1] = 3; // output level - options[2] = 1242; // random number seed + int signed_n_global_cells = n_global_cells; - std::vector part(ncell_on_rank); + std::vector global_part(n_global_cells); - ParMETIS_V3_PartKway( - &vtxdist[0], // array describing how cells are distributed - &xadj[0], // how cells are stored locally - &adjncy[0], // how cells are stored loccaly - NULL, // weight of vertices - NULL, // weight of edges - &wgtflag, // 0 means no weights for node or edges - &numflag, // numbering style, 0 for C-style - &ncon, // weights per vertex - &nparts, // number of sub-domains - tpwgts, // weight per sub-domain - ubvec, // unbalance in vertex weight - options, // options array - &edgecut, // OUTPUT: Number of edgecuts - &part[0], // OUTPUT: partition of each vertex - &comm); // MPI communicator - - delete[] ubvec; - delete[] tpwgts; - delete[] xyz; + int metis_return = METIS_PartGraphKway( + &signed_n_global_cells, // number of on-rank vertices + &ncon, // weight of vertices + &xadj[0], // how cells are stored locally + &adjncy[0], // how cells are stored loccaly + NULL, // weight of vertices + NULL, // size of vertices for comm volume + NULL, // weight of the edges + &n_parts, // number of ranks (partitions) + NULL, // tpwgts (NULL = equal weight domains) + NULL, // unbalance in v-weight (NULL=1.001) + rank_options, // options array + &edgecut, // OUTPUT: Number of edgecuts + &global_part[0]); // OUTPUT: rank of each cell + + if (metis_return == METIS_ERROR_INPUT) + std::cout << "METIS: Input error" << std::endl; + else if (metis_return == METIS_ERROR_MEMORY) + std::cout << "METIS: Memory error" << std::endl; + else if (metis_return == METIS_ERROR) + std::cout << "METIS: Input error" << std::endl; + + if (!edgecut) { + std::cout << "ERROR: No partitioning occured, run with more cells or in"; + std::cout << "replicated mode. Exiting..." << std::endl; + exit(EXIT_FAILURE); + } + + // send partitioning to other ranks + for (int irank = 1; irank < n_rank; ++irank) { + MPI_Send(&global_part[vtxdist[irank]], start_ncells[irank], MPI_INT, + irank, part_tag, MPI_COMM_WORLD); + } + + // copy out root ranks partitioning + std::copy(global_part.begin(), global_part.begin() + ncell_on_rank, + part.begin()); + } return part; } @@ -765,13 +804,13 @@ void remap_cell_and_grip_indices_allreduce(Proto_Mesh &mesh, const int rank, } //----------------------------------------------------------------------------// -//! Generate new partitioning with ParMetis, send and receive cells, renumber -// mesh and communicate renumbering +//! Generate new partitioning with Metis, replications or cubes, send and +// receive cells, renumber mesh and communicate renumbering void decompose_mesh(Proto_Mesh &mesh, const MPI_Types &mpi_types, const Info &mpi_info, const uint32_t grip_size, const int decomposition_type) { using Constants::CUBE; - using Constants::PARMETIS; + using Constants::METIS; using std::unordered_map; using std::unordered_set; Timer t_partition; @@ -781,7 +820,7 @@ void decompose_mesh(Proto_Mesh &mesh, const MPI_Types &mpi_types, int rank = mpi_info.get_rank(); int n_rank = mpi_info.get_n_rank(); - // parmetis sets this, if it's zero no changes were made to the default + // metis sets this, if it's zero no changes were made to the default // partition (cube decomposition always changes paritioning) int edgecut = 0; if (rank == 0) @@ -793,8 +832,8 @@ void decompose_mesh(Proto_Mesh &mesh, const MPI_Types &mpi_types, if (decomposition_type == CUBE) { part = cube_partition(mesh, rank, n_rank); edgecut = 1; - } else if (decomposition_type == PARMETIS) - part = parmetis_partition(mesh, edgecut, rank, n_rank); + } else if (decomposition_type == METIS) + part = metis_partition(mesh, edgecut, rank, n_rank, mpi_types); else { if (rank == 0) { std::cout << "Decomposition type not recognized."; diff --git a/src/input.h b/src/input.h index 498bc45..570a02e 100644 --- a/src/input.h +++ b/src/input.h @@ -54,7 +54,7 @@ class Input { using Constants::CELL_PASS; using Constants::CELL_PASS_RMA; using Constants::CUBE; - using Constants::PARMETIS; + using Constants::METIS; using Constants::PARTICLE_PASS; using Constants::REPLICATED; using std::cout; @@ -182,14 +182,14 @@ class Input { // domain decomposition method tempString = settings_node.child_value("mesh_decomposition"); - if (tempString == "PARMETIS") - decomp_mode = PARMETIS; + if (tempString == "METIS") + decomp_mode = METIS; else if (tempString == "CUBE") decomp_mode = CUBE; else { cout << "WARNING: Mesh decomposition method not recognized... "; - cout << "setting to PARMETIS method" << endl; - decomp_mode = PARMETIS; + cout << "setting to METIS method" << endl; + decomp_mode = METIS; } if (dd_mode == REPLICATED) { std::cout << "Replicated transport mode, mesh decomposition method"; @@ -580,7 +580,7 @@ class Input { using Constants::REPLICATED; // mesh decomposition using Constants::CUBE; - using Constants::PARMETIS; + using Constants::METIS; cout << "Problem Specifications:"; cout << "Constants -- c: " << c << " (cm/sh) , a: " << a << endl; @@ -664,8 +664,8 @@ class Input { } cout << "Mesh decomposition: "; - if (decomp_mode == PARMETIS && dd_mode != REPLICATED) - cout << "PARMETIS" << endl; + if (decomp_mode == METIS && dd_mode != REPLICATED) + cout << "METIS" << endl; else if (decomp_mode == CUBE && dd_mode != REPLICATED) cout << "CUBE" << endl; else if (dd_mode == REPLICATED) diff --git a/src/mesh.h b/src/mesh.h index 550024e..ceba5eb 100644 --- a/src/mesh.h +++ b/src/mesh.h @@ -39,7 +39,7 @@ * * Using an Input class, make the mesh with the correct material properties * for each region. The mesh numbering and mapping between global IDs and local - * indices are all determined with the aid of Parmetis in the decompose_mesh + * indices are all determined with the aid of Metis in the decompose_mesh * function. The mesh class also manages two-sided messaging in the mesh- * passing method. * @@ -62,7 +62,7 @@ class Mesh { using Constants::bc_type; using Constants::CUBE; using Constants::ELEMENT; - using Constants::PARMETIS; + using Constants::METIS; using Constants::REPLICATED; using Constants::X_NEG; using Constants::X_POS; @@ -75,16 +75,16 @@ class Mesh { Proto_Mesh proto_mesh(input, mpi_types, mpi_info); // if mode is replicated ignore decomposition options, otherwise use - // parmetis or a simple cube + // metis or a simple cube if (input.get_dd_mode() == REPLICATED) { replicate_mesh(proto_mesh, mpi_types, mpi_info, imc_p.get_grip_size()); // get decomposition information from proto mesh off_rank_bounds = proto_mesh.get_off_rank_bounds(); on_rank_start = off_rank_bounds.front(); on_rank_end = off_rank_bounds.back() - 1; - } else if (input.get_decomposition_mode() == PARMETIS) { + } else if (input.get_decomposition_mode() == METIS) { decompose_mesh(proto_mesh, mpi_types, mpi_info, imc_p.get_grip_size(), - PARMETIS); + METIS); // get decomposition information from proto mesh off_rank_bounds = proto_mesh.get_off_rank_bounds(); on_rank_start = off_rank_bounds[rank]; diff --git a/src/mesh_request_manager.h b/src/mesh_request_manager.h index 8b5b326..e61c2cb 100644 --- a/src/mesh_request_manager.h +++ b/src/mesh_request_manager.h @@ -403,8 +403,6 @@ class Mesh_Request_Manager { void request_cell(const uint32_t &g_index, Message_Counter &mctr) { typedef std::pair rpair; // store cell for requesting if not already stored - if (g_index > rank_bounds.back()) - std::cout << "this is bad: g index > bounds" << std::endl; if (!mesh_is_requested(g_index)) { // get the rank of the global index, add to set if there are fewer than // max_ids for that rank @@ -436,6 +434,9 @@ class Mesh_Request_Manager { s_cell_count = 0; r_cell_max_index = 0; r_cell_count = 0; + s_id_in_use.clear(); + r_cell_in_use.clear(); + s_cell_in_use.clear(); } //! End simulation by cancelling pending receives requests diff --git a/src/proto_mesh.h b/src/proto_mesh.h index e4ccf03..8ddc1ce 100644 --- a/src/proto_mesh.h +++ b/src/proto_mesh.h @@ -35,7 +35,7 @@ * * Using an Input class, make the mesh with the correct material properties * for each region. The mesh numbering and mapping between global IDs and local - * indices are all determined with the aid of Parmetis in the decompose_mesh + * indices are all determined with the aid of Metis in the decompose_mesh * function. The Proto_Mesh does not hold any physical data (i.e. opacity, heat * capacity, temperature). * @@ -326,6 +326,11 @@ class Proto_Mesh { return cell_list[local_ID]; } + //! Gets contant reference to cell vector before it's deleted + const std::vector &get_pre_window_allocation_cells() const { + return cell_list; + } + uint32_t get_off_rank_id(const uint32_t &index) const { // find rank of index bool found = false; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 74778d9..261daf9 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -76,7 +76,6 @@ endforeach() # Other tests add_branson_test( SOURCE test_load_balance.cc PE_LIST "4" ) -add_branson_test( SOURCE test_parmetis.cc PE_LIST "1;4" ) add_branson_test( SOURCE test_imc_state.cc PE_LIST "2" ) # Don't run 8 proc tests in CI mode diff --git a/src/test/test_parmetis.cc b/src/test/test_parmetis.cc deleted file mode 100644 index 039523f..0000000 --- a/src/test/test_parmetis.cc +++ /dev/null @@ -1,177 +0,0 @@ -//----------------------------------*-C++-*----------------------------------// -/*! - * \file test_parmetis.cc - * \author Alex Long - * \date February 11 2016 - * \brief Make sure ParMetis can be called with a simple graph - * \note Copyright (C) 2017 Los Alamos National Security, LLC. - * All rights reserved - */ -//---------------------------------------------------------------------------// - -#include -#include -#include -#include -#include - -int main (int argc, char *argv[]) { - - using std::cout; - using std::endl; - using std::vector; - using std::partial_sum; - - MPI_Init(&argc, &argv); - - int rank, n_rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &n_rank); - - int nfail = 0; - - // Try to decompose a simple graph with Parmetis - { - - bool simple_partitioning_pass=true; - - MPI_Comm comm; - MPI_Comm_dup(MPI_COMM_WORLD, &comm); - - //get the number of cells on each processor - //vtxdist has number of vertices on each rank, same for all ranks - const int n_cell_on_rank = 100; - vector start_ncells(n_rank, n_cell_on_rank); //cells on each rank - vector vtxdist(n_rank, 0); //prefix sum for cells on rank - - partial_sum(start_ncells.begin(), - start_ncells.end(), - vtxdist.begin()); - vtxdist.insert(vtxdist.begin(), 0); - - //build adjacency list needed for ParMetis call for each rank - vector xadj; - vector adjncy; - int adjncy_counter = 0; - uint32_t g_ID; //! Global ID - for (uint32_t i=0; i n_on_rank(n_rank); - for (uint32_t i =0; i