CMake: Integrate librepa and kdpart

Co-authored-by: Jean-Noël Grad <[email protected]>

.gitlab-ci.yml

@@ -106,6 +106,7 @@ maxset:
      with_coverage: 'true'
      with_scafacos: 'true'
      with_stokesian_dynamics: 'true'
+     with_load_balancing: 'true'
      check_skip_long: 'true'
      cmake_params: '-DTEST_NP=8'
   script:

CMakeLists.txt

@@ -115,6 +115,7 @@ option(WARNINGS_ARE_ERRORS "Treat warnings as errors during compilation" OFF)
 option(WITH_CCACHE "Use ccache compiler invocation." OFF)
 option(WITH_PROFILER "Enable profiler annotations." OFF)
 option(INSIDE_DOCKER "Enable when running inside Docker." OFF)
+option(WITH_LOAD_BALANCING "Build with load balancing." OFF)
 set(TEST_TIMEOUT "300" CACHE STRING
                              "Timeout in seconds for each testsuite test")
 
@@ -138,6 +139,10 @@ set(MYCONFIG_NAME "myconfig.hpp" CACHE STRING
 # Check which config file to use
 include(MyConfig)
 
+if(WITH_LOAD_BALANCING)
+  set(CMAKE_CXX_STANDARD 17)
+endif()
+
 #
 # Pretty function
 #
@@ -207,6 +212,10 @@ set(ESPRESSO_INSTALL_BINDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}")
 # folder for C++ and CUDA shared objects
 set(ESPRESSO_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 
+# folder for header files
+set(ESPRESSO_INSTALL_INCDIR
+    "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}")
+
 # python site-packages, can be overriden with CMake options
 if(WITH_PYTHON)
   if(NOT ESPRESSO_INSTALL_PYTHON)
@@ -302,6 +311,55 @@ if(WITH_STOKESIAN_DYNAMICS)
   endif()
 endif(WITH_STOKESIAN_DYNAMICS)
 
+# librepa and kdpart
+if(WITH_LOAD_BALANCING)
+  include(FetchContent)
+
+  # workaround for CMake FetchContent_Declare() PATCH_COMMAND bug, see details
+  # in https://gitlab.kitware.com/cmake/cmake/-/issues/21146
+  FetchContent_Declare(
+    kdpart
+    GIT_REPOSITORY https://github.com/hirschsn/kdpart.git
+    GIT_TAG afec3688008e0519410fdb58a3a88230e31ebd5e # v1.2.1
+    PATCH_COMMAND
+      patch -p1 -sN --ignore-whitespace -i
+      ${PROJECT_SOURCE_DIR}/cmake/kdpart.patch || echo "Patch not applied!")
+  FetchContent_GetProperties(kdpart)
+  if(NOT kdpart_POPULATED)
+    FetchContent_Populate(kdpart)
+
+    # build kdpart
+    execute_process(COMMAND ${CMAKE_COMMAND} -E env CXX=${CMAKE_CXX_COMPILER}
+                            make -C ${kdpart_SOURCE_DIR})
+    # install kdpart
+    execute_process(COMMAND make -C ${kdpart_SOURCE_DIR}
+                            PREFIX=${kdpart_BINARY_DIR} install)
+  endif()
+
+  # workaround for CMake FetchContent_Declare() PATCH_COMMAND bug, see details
+  # in https://gitlab.kitware.com/cmake/cmake/-/issues/21146
+  FetchContent_Declare(
+    librepa
+    GIT_REPOSITORY https://github.com/SC-SGS/repa.git
+    GIT_TAG ff050adf2b5ba4822a6b87416f35e8dec29d6e18
+    PATCH_COMMAND
+      patch -p1 -sN --ignore-whitespace -i
+      ${PROJECT_SOURCE_DIR}/cmake/librepa.patch || echo "Patch not applied!")
+  FetchContent_GetProperties(librepa)
+  if(NOT librepa_POPULATED)
+    FetchContent_Populate(librepa)
+    set(REPA_WITH_KDPART on CACHE BOOL "")
+    set(REPA_WITH_P4EST off CACHE BOOL "")
+    set(REPA_WITH_PARMETIS off CACHE BOOL "")
+    set(REPA_WITH_TESTS off CACHE BOOL "")
+    set(REPA_WITH_COVERAGE off CACHE BOOL "")
+    set(KDPART_DIR "${kdpart_BINARY_DIR}" CACHE PATH "")
+    add_subdirectory(${librepa_SOURCE_DIR} ${librepa_BINARY_DIR})
+  endif()
+
+  set(LOAD_BALANCING 1)
+endif(WITH_LOAD_BALANCING)
+
 if(WITH_VALGRIND_INSTRUMENTATION)
   find_package(PkgConfig REQUIRED)
   pkg_check_modules(VALGRIND valgrind REQUIRED)

cmake/cmake_config.cmakein

@@ -19,6 +19,8 @@
 
 #cmakedefine STOKESIAN_DYNAMICS
 
+#cmakedefine LOAD_BALANCING
+
 #cmakedefine VALGRIND_INSTRUMENTATION
 
 #define PACKAGE_NAME "${PROJECT_NAME}"

cmake/kdpart.patch

@@ -0,0 +1,21 @@
+--- a/kdpart.h
++++ b/kdpart.h
+@@ -140,7 +140,7 @@ struct NodeAccess {
+             std::abort();
+         }
+
+-        assert(false);
++        throw std::logic_error("[libkdpart] Fatal exception in local repartitioning.");
+     }
+
+     inline NodeAccess find_root_of_subtree(int depth) const
+--- a/Makefile
++++ b/Makefile
+@@ -5,5 +5,6 @@
+ CXX = mpic++
+ CXXFLAGS = -std=c++14 -O3 -march=native
++# flag -Wno-cast-function-type needed for GCC, see https://github.com/open-mpi/ompi/issues/5157
+-CXXFLAGS += -Wall -Wextra -pedantic
++CXXFLAGS += -Wall -Wextra -pedantic -Wno-cast-function-type
+
+ AR = ar

cmake/librepa.patch

@@ -0,0 +1,29 @@
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -54,10 +54,6 @@ if(REPA_WITH_TESTS)
+ endif(REPA_WITH_TESTS)
+ find_package(Boost "1.67.0" REQUIRED ${REQ_BOOST_PACKAGES})
+
+-if (Boost_VERSION VERSION_GREATER_EQUAL "1.69" AND Boost_VERSION VERSION_LESS_EQUAL "1.71")
+-  message(FATAL_ERROR "Boost 1.69 - 1.71 have a bug and are not supported.")
+-endif()
+-
+ include_directories(${Boost_INCLUDE_DIRS})
+
+ # Used for librepa itself as well as all tests
+--- a/repa/CMakeLists.txt
++++ b/repa/CMakeLists.txt
+@@ -95,11 +95,5 @@ target_compile_options(repa
+                        PRIVATE ${REPA_DEFAULT_COMPILE_OPTIONS})
+
+ install(TARGETS repa
+-        LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib
+-        ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+-# Keep folder structure, so no install for all repa_HDR
+-foreach(hdr ${repa_HDR})
+-    get_filename_component(dir ${hdr} DIRECTORY)
+-    install(FILES ${hdr}
+-            DESTINATION ${CMAKE_INSTALL_PREFIX}/include/repa/${dir})
+-endforeach()
++        LIBRARY DESTINATION ${ESPRESSO_INSTALL_LIBDIR}
++        ARCHIVE DESTINATION ${ESPRESSO_INSTALL_LIBDIR})

doc/sphinx/installation.rst

@@ -575,6 +575,9 @@ using a CMake flag (see :ref:`Options and Variables`).
 - ``STOKESIAN_DYNAMICS`` Enables the Stokesian Dynamics feature
   (see :ref:`Stokesian Dynamics`). Requires BLAS and LAPACK.
 
+- ``LOAD_BALANCING`` Enables the MD dynamic load balancing feature
+  (not yet implemented). Builds and install the kdpart and librepa libraries.
+
 
 
 .. _Configuring:

maintainer/CI/build_cmake.sh

@@ -109,6 +109,7 @@ set_default_value with_ccache false
 set_default_value with_hdf5 true
 set_default_value with_scafacos false
 set_default_value with_stokesian_dynamics false
+set_default_value with_load_balancing false
 set_default_value test_timeout 300
 set_default_value hide_gpu false
 
@@ -159,6 +160,12 @@ else
     cmake_params="${cmake_params} -DWITH_STOKESIAN_DYNAMICS=OFF"
 fi
 
+if [ "${with_load_balancing}" = true ]; then
+    cmake_params="${cmake_params} -DWITH_LOAD_BALANCING=ON"
+else
+    cmake_params="${cmake_params} -DWITH_LOAD_BALANCING=OFF"
+fi
+
 if [ "${with_coverage}" = true ]; then
     cmake_params="-DWITH_COVERAGE=ON ${cmake_params}"
 fi