Fetch and squash FEniCS#3324

cpp/demo/checkpointing/CMakeLists.txt

@@ -0,0 +1,40 @@
+# This file was generated by running
+#
+# python cmake/scripts/generate-cmakefiles.py from dolfinx/cpp
+#
+cmake_minimum_required(VERSION 3.19)
+
+set(PROJECT_NAME demo_checkpointing)
+project(${PROJECT_NAME} LANGUAGES C CXX)
+
+# Set C++20 standard
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT TARGET dolfinx)
+  find_package(DOLFINX REQUIRED)
+endif()
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+
+add_executable(${PROJECT_NAME} main.cpp)
+target_link_libraries(${PROJECT_NAME} dolfinx)
+
+# Do not throw error for 'multi-line comments' (these are typical in rst which
+# includes LaTeX)
+include(CheckCXXCompilerFlag)
+check_cxx_compiler_flag("-Wno-comment" HAVE_NO_MULTLINE)
+set_source_files_properties(
+  main.cpp
+  PROPERTIES
+    COMPILE_FLAGS
+    "$<$<BOOL:${HAVE_NO_MULTLINE}>:-Wno-comment -Wall -Wextra -pedantic -Werror>"
+)
+
+# Test targets (used by DOLFINx testing system)
+set(TEST_PARAMETERS2 -np 2 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}")
+set(TEST_PARAMETERS3 -np 3 ${MPIEXEC_PARAMS} "./${PROJECT_NAME}")
+add_test(NAME ${PROJECT_NAME}_mpi_2 COMMAND "mpirun" ${TEST_PARAMETERS2})
+add_test(NAME ${PROJECT_NAME}_mpi_3 COMMAND "mpirun" ${TEST_PARAMETERS3})
+add_test(NAME ${PROJECT_NAME}_serial COMMAND ${PROJECT_NAME})

cpp/demo/checkpointing/checkpointing.yml

@@ -0,0 +1,10 @@
+---
+# adios2 config.yaml
+# IO YAML Sequence (-) Nodes to allow for multiple IO nodes
+# IO name referred in code with DeclareIO is mandatory
+
+- IO: "mesh-write"
+
+  Engine:
+     # If Type is missing or commented out, default Engine is picked up
+     Type: "BP5"

cpp/demo/checkpointing/main.cpp

@@ -0,0 +1,146 @@
+// ```text
+// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken, Jack S. Hale
+// This file is part of DOLFINx (https://www.fenicsproject.org)
+// SPDX-License-Identifier:    LGPL-3.0-or-later
+// ```
+
+// # Checkpointing
+//
+
+#include <adios2.h>
+#include <dolfinx.h>
+#include <dolfinx/common/version.h>
+#include <dolfinx/io/ADIOS2_utils.h>
+#include <dolfinx/io/checkpointing.h>
+#include <mpi.h>
+#include <typeinfo>
+#include <variant>
+
+using namespace dolfinx;
+using namespace dolfinx::io;
+
+// Create cell meshtags
+
+template <std::floating_point T>
+std::shared_ptr<mesh::MeshTags<std::int32_t>>
+create_meshtags(dolfinx::mesh::Mesh<T>& mesh)
+{
+  // Create cell meshtags
+  auto geometry = mesh.geometry();
+  auto topology = mesh.topology();
+
+  int dim = geometry.dim();
+  topology->create_entities(dim);
+  const std::shared_ptr<const dolfinx::common::IndexMap> topo_imap
+      = topology->index_map(dim);
+
+  std::int32_t num_entities = topo_imap->size_local();
+
+  auto cmap = geometry.cmap();
+  auto geom_layout = cmap.create_dof_layout();
+  std::uint32_t num_dofs_per_entity = geom_layout.num_entity_closure_dofs(dim);
+
+  std::vector<int32_t> entities_array(num_entities * num_dofs_per_entity);
+  std::vector<int32_t> entities_offsets(num_entities + 1);
+  std::uint64_t offset = topo_imap->local_range()[0];
+  std::vector<std::int32_t> values(num_entities);
+
+  for (std::int32_t i = 0; i < num_entities; ++i)
+  {
+    values[i] = i + offset;
+  }
+
+  auto entities = topology->connectivity(dim, 0);
+
+  for (int i = 0; i < (int)num_entities + 1; ++i)
+    entities_offsets[i] = entities->offsets()[i];
+
+  for (int i = 0; i < (int)(num_entities * num_dofs_per_entity); ++i)
+    entities_array[i] = entities->array()[i];
+
+  graph::AdjacencyList<std::int32_t> entities_local(entities_array,
+                                                    entities_offsets);
+
+  auto meshtags = std::make_shared<mesh::MeshTags<std::int32_t>>(
+      mesh::create_meshtags<std::int32_t>(topology, dim, entities_local,
+                                          values));
+
+  return meshtags;
+}
+
+int main(int argc, char* argv[])
+{
+  dolfinx::init_logging(argc, argv);
+  MPI_Init(&argc, &argv);
+
+  // Create mesh and function space
+  auto part = mesh::create_cell_partitioner(mesh::GhostMode::shared_facet);
+  auto mesh = std::make_shared<mesh::Mesh<float>>(mesh::create_rectangle<float>(
+      MPI_COMM_WORLD, {{{0.0, 0.0}, {1.0, 1.0}}}, {4, 4},
+      mesh::CellType::quadrilateral, part));
+
+  auto meshtags = create_meshtags<float>(*mesh);
+
+  try
+  {
+    // Set up ADIOS2 IO and Engine
+    adios2::ADIOS adios(mesh->comm());
+
+    adios2::IO io = adios.DeclareIO("mesh-write");
+    io.SetEngine("BP5");
+    adios2::Engine engine = io.Open("mesh.bp", adios2::Mode::Append);
+
+    io::native::write_mesh(io, engine, *mesh);
+    io::native::write_meshtags<float, std::int32_t>(io, engine, *mesh,
+                                                    *meshtags);
+
+    engine.Close();
+  }
+  catch (std::exception& e)
+  {
+    std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n";
+    MPI_Abort(MPI_COMM_WORLD, -1);
+  }
+
+  try
+  {
+    // Set up ADIOS2 IO and Engine
+    adios2::ADIOS adios_read(MPI_COMM_WORLD);
+    adios2::IO io_read = adios_read.DeclareIO("mesh-read");
+    io_read.SetEngine("BP5");
+    adios2::Engine engine_read = io_read.Open("mesh.bp", adios2::Mode::Read);
+
+    engine_read.BeginStep();
+    auto mesh_read
+        = io::native::read_mesh<float>(io_read, engine_read, MPI_COMM_WORLD);
+
+    mesh::MeshTags<std::int32_t> mt
+        = io::native::read_meshtags<float, std::int32_t>(
+            io_read, engine_read, mesh_read, "mesh_tags");
+
+    if (engine_read.BetweenStepPairs())
+    {
+      engine_read.EndStep();
+    }
+
+    engine_read.Close();
+
+    adios2::IO io_write = adios_read.DeclareIO("mesh-write");
+    io_write.SetEngine("BP5");
+    adios2::Engine engine_write
+        = io_write.Open("mesh2.bp", adios2::Mode::Write);
+
+    io::native::write_mesh(io_write, engine_write, mesh_read);
+    io::native::write_meshtags<float, std::int32_t>(io_write, engine_write,
+                                                    mesh_read, mt);
+    engine_write.Close();
+  }
+  catch (std::exception& e)
+  {
+    std::cout << "ERROR: ADIOS2 exception: " << e.what() << "\n";
+    MPI_Abort(MPI_COMM_WORLD, -1);
+  }
+
+  MPI_Finalize();
+  return 0;
+}

cpp/doc/source/demo.rst

@@ -42,3 +42,4 @@ Experimental
    :maxdepth: 1
 
    demos/demo_mixed_topology.md
+   demos/demo_checkpointing.md

cpp/dolfinx/io/ADIOS2_utils.h

@@ -0,0 +1,104 @@
+// Copyright (C) 2024 Abdullah Mujahid, Jørgen S. Dokken and Garth N. Wells
+//
+// This file is part of DOLFINX (https://www.fenicsproject.org)
+//
+// SPDX-License-Identifier:    LGPL-3.0-or-later
+
+#pragma once
+
+#ifdef HAS_ADIOS2
+
+#include <adios2.h>
+#include <cassert>
+#include <filesystem>
+#include <mpi.h>
+
+/// @file ADIOS2_utils.h
+/// @brief Utils for ADIOS2
+
+namespace
+{
+std::map<std::string, adios2::Mode> string_to_mode{
+    {"write", adios2::Mode::Write},
+    {"read", adios2::Mode::Read},
+    {"append", adios2::Mode::Append},
+};
+} // namespace
+
+namespace dolfinx::io
+{
+
+/// ADIOS2-based writers/readers
+class ADIOS2Wrapper
+{
+public:
+  /// @brief Create an ADIOS2-based engine writer/reader
+  /// @param[in] comm The MPI communicator
+  /// @param[in] filename Name of output file
+  /// @param[in] tag The ADIOS2 IO name
+  /// @param[in] engine_type ADIOS2 engine type. See
+  /// https://adios2.readthedocs.io/en/latest/engines/engines.html.
+  /// @param[in] mode ADIOS2 mode, default is Write or Read
+  ADIOS2Wrapper(MPI_Comm comm, std::string filename, std::string tag,
+                std::string engine_type = "BP5", std::string mode = "write")
+      : _adios(std::make_shared<adios2::ADIOS>(comm)),
+        _io(std::make_shared<adios2::IO>(_adios->DeclareIO(tag)))
+  {
+    _io->SetEngine(engine_type);
+    _engine = std::make_shared<adios2::Engine>(
+        _io->Open(filename, string_to_mode[mode]));
+  }
+
+  /// @brief Create an ADIOS2-based engine writer/reader
+  /// @param[in] config_file Path to config file to set up ADIOS2 engine from
+  /// @param[in] comm The MPI communicator
+  /// @param[in] filename Name of output file
+  /// @param[in] tag The ADIOS2 IO name
+  /// @param[in] mode ADIOS2 mode, default is Write or Read
+  ADIOS2Wrapper(std::string config_file, MPI_Comm comm, std::string filename,
+                std::string tag, std::string mode = "append")
+  {
+    _adios = std::make_shared<adios2::ADIOS>(config_file, comm);
+    _io = std::make_shared<adios2::IO>(_adios->DeclareIO(tag));
+    _engine = std::make_shared<adios2::Engine>(
+        _io->Open(filename, string_to_mode[mode]));
+  }
+
+  /// @brief Move constructor
+  ADIOS2Wrapper(ADIOS2Wrapper&& ADIOS2) = default;
+
+  /// @brief Copy constructor
+  ADIOS2Wrapper(const ADIOS2Wrapper&) = delete;
+
+  /// @brief Destructor
+  ~ADIOS2Wrapper() { close(); }
+
+  /// @brief Move assignment
+  ADIOS2Wrapper& operator=(ADIOS2Wrapper&& ADIOS2) = default;
+
+  // Copy assignment
+  ADIOS2Wrapper& operator=(const ADIOS2Wrapper&) = delete;
+
+  /// @brief  Close the file
+  void close()
+  {
+    assert(_engine);
+    if (*_engine)
+      _engine->Close();
+  }
+
+  /// @brief  Get the IO object
+  std::shared_ptr<adios2::IO> io() { return _io; }
+
+  /// @brief  Get the Engine object
+  std::shared_ptr<adios2::Engine> engine() { return _engine; }
+
+private:
+  std::shared_ptr<adios2::ADIOS> _adios;
+  std::shared_ptr<adios2::IO> _io;
+  std::shared_ptr<adios2::Engine> _engine;
+};
+
+} // namespace dolfinx::io
+
+#endif

cpp/dolfinx/io/CMakeLists.txt

@@ -1,7 +1,9 @@
 set(HEADERS_io
     ${CMAKE_CURRENT_SOURCE_DIR}/dolfinx_io.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2_utils.h
     ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.h
     ${CMAKE_CURRENT_SOURCE_DIR}/cells.h
+    ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.h
     ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.h
     ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.h
     ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.h
@@ -16,6 +18,7 @@ target_sources(
   dolfinx
   PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/ADIOS2Writers.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/cells.cpp
+          ${CMAKE_CURRENT_SOURCE_DIR}/checkpointing.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/HDF5Interface.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/VTKFile.cpp
           ${CMAKE_CURRENT_SOURCE_DIR}/vtk_utils.cpp