From e9b8e6adb0f2d1bb2f148629e7d9c57d4f70bb22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Jul 2023 16:03:28 +0200 Subject: [PATCH 01/10] Plumberwork for MPI communicator in JSON backend --- include/openPMD/IO/JSON/JSONIOHandler.hpp | 15 ++++++++++++++- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 16 ++++++++++++++++ src/IO/AbstractIOHandlerHelper.cpp | 19 +++++++++++++++++-- src/IO/JSON/JSONIOHandler.cpp | 18 ++++++++++++++++-- src/IO/JSON/JSONIOHandlerImpl.cpp | 17 +++++++++++++++++ 5 files changed, 80 insertions(+), 5 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 7fdea5b6f0..7cb6870f5b 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -24,17 +24,30 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" +#if openPMD_HAVE_MPI +#include +#endif + namespace openPMD { class JSONIOHandler : public AbstractIOHandler { public: JSONIOHandler( - std::string const &path, + std::string path, + Access at, + openPMD::json::TracingJSON config, + JSONIOHandlerImpl::FileFormat, + std::string originalExtension); +#if openPMD_HAVE_MPI + JSONIOHandler( + std::string path, Access at, + MPI_Comm, openPMD::json::TracingJSON config, JSONIOHandlerImpl::FileFormat, std::string originalExtension); +#endif ~JSONIOHandler() override; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 5ce9d057c3..81dc9c39f3 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -31,6 +31,9 @@ #include #include +#if openPMD_HAVE_MPI +#include +#endif #include #include @@ -167,6 +170,15 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl FileFormat, std::string originalExtension); +#if openPMD_HAVE_MPI + JSONIOHandlerImpl( + AbstractIOHandler *, + MPI_Comm, + openPMD::json::TracingJSON config, + FileFormat, + std::string originalExtension); +#endif + ~JSONIOHandlerImpl() override; void @@ -230,6 +242,10 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl std::future flush(); private: +#if openPMD_HAVE_MPI + std::optional m_communicator; +#endif + using FILEHANDLE = std::fstream; // map each Writable to its associated file diff --git a/src/IO/AbstractIOHandlerHelper.cpp b/src/IO/AbstractIOHandlerHelper.cpp index 699dfd3619..8576343e5d 100644 --- a/src/IO/AbstractIOHandlerHelper.cpp +++ b/src/IO/AbstractIOHandlerHelper.cpp @@ -125,8 +125,23 @@ std::unique_ptr createIOHandler( "ssc", std::move(originalExtension)); case Format::JSON: - throw error::WrongAPIUsage( - "JSON backend not available in parallel openPMD."); + return constructIOHandler( + "JSON", + path, + access, + comm, + std::move(options), + JSONIOHandlerImpl::FileFormat::Json, + std::move(originalExtension)); + case Format::TOML: + return constructIOHandler( + "JSON", + path, + access, + comm, + std::move(options), + JSONIOHandlerImpl::FileFormat::Toml, + std::move(originalExtension)); default: throw error::WrongAPIUsage( "Unknown file format! Did you specify a file ending? Specified " diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 041b236340..d2a6217eb5 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -26,15 +26,29 @@ namespace openPMD JSONIOHandler::~JSONIOHandler() = default; JSONIOHandler::JSONIOHandler( - std::string const &path, + std::string path, Access at, openPMD::json::TracingJSON jsonCfg, JSONIOHandlerImpl::FileFormat format, std::string originalExtension) - : AbstractIOHandler{path, at} + : AbstractIOHandler{std::move(path), at} , m_impl{this, std::move(jsonCfg), format, std::move(originalExtension)} {} +#if openPMD_HAVE_MPI +JSONIOHandler::JSONIOHandler( + std::string path, + Access at, + MPI_Comm comm, + openPMD::json::TracingJSON jsonCfg, + JSONIOHandlerImpl::FileFormat format, + std::string originalExtension) + : AbstractIOHandler{std::move(path), at} + , m_impl{JSONIOHandlerImpl{ + this, comm, std::move(jsonCfg), format, std::move(originalExtension)}} +{} +#endif + std::future JSONIOHandler::flush(internal::ParsedFlushParams &) { return m_impl.flush(); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index a4e1bb39ab..5112b0bb2b 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -23,6 +23,8 @@ #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/Error.hpp" +#include "openPMD/IO/AbstractIOHandler.hpp" +#include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -133,6 +135,21 @@ JSONIOHandlerImpl::JSONIOHandlerImpl( , m_originalExtension{std::move(originalExtension)} {} +#if openPMD_HAVE_MPI +JSONIOHandlerImpl::JSONIOHandlerImpl( + AbstractIOHandler *handler, + MPI_Comm comm, + // NOLINTNEXTLINE(performance-unnecessary-value-param) + [[maybe_unused]] openPMD::json::TracingJSON config, + FileFormat format, + std::string originalExtension) + : AbstractIOHandlerImpl(handler) + , m_communicator{comm} + , m_fileFormat{format} + , m_originalExtension{std::move(originalExtension)} +{} +#endif + JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; std::future JSONIOHandlerImpl::flush() From 2c987960d62fa8cc70009b0676b19f1f92fd158a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 7 Jul 2023 16:12:58 +0200 Subject: [PATCH 02/10] Parallel reading --- src/IO/JSON/JSONIOHandlerImpl.cpp | 70 +++++++++++++++++++++++++------ src/auxiliary/Filesystem.cpp | 3 +- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 5112b0bb2b..f6887dc34c 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -26,11 +26,13 @@ #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/backend/Writable.hpp" +#include #include #include @@ -1260,21 +1262,65 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) { return it->second; } + auto serialImplementation = [&file, this]() { + auto [fh, fh_with_precision, _] = + getFilehandle(file, Access::READ_ONLY); + (void)_; + std::shared_ptr res = + std::make_shared(); + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision >> *res; + break; + case FileFormat::Toml: + *res = openPMD::json::tomlToJson( + toml::parse(*fh_with_precision, *file)); + break; + } + VERIFY(fh->good(), "[JSON] Failed reading from a file."); + return res; + }; + auto parallelImplementation = [&file, this](MPI_Comm comm) { + auto path = fullPath(*file); + std::string collectivelyReadRawData = + auxiliary::collective_file_read(path, comm); + std::shared_ptr res = + std::make_shared(); + switch (m_fileFormat) + { + case FileFormat::Json: + *res = nlohmann::json::parse(collectivelyReadRawData); + break; + case FileFormat::Toml: + std::istringstream istream( + collectivelyReadRawData.c_str(), + std::ios_base::binary | std::ios_base::in); + auto as_toml = toml::parse( + istream >> std::setprecision( + std::numeric_limits::digits10 + 1), + *file); + *res = openPMD::json::tomlToJson(as_toml); + break; + } + return res; + }; // read from file - auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); - (void)_; - std::shared_ptr res = std::make_shared(); - switch (m_fileFormat) +#if openPMD_HAVE_MPI + std::shared_ptr res; + if (m_communicator.has_value()) { - case FileFormat::Json: - *fh_with_precision >> *res; - break; - case FileFormat::Toml: - *res = - openPMD::json::tomlToJson(toml::parse(*fh_with_precision, *file)); - break; + res = parallelImplementation(m_communicator.value()); } - VERIFY(fh->good(), "[JSON] Failed reading from a file."); + else + { + res = serialImplementation(); + } + +#else + auto res = serialImplementation(); +#endif + m_jsonVals.emplace(file, res); return res; } diff --git a/src/auxiliary/Filesystem.cpp b/src/auxiliary/Filesystem.cpp index cce80b9d17..564d266ee3 100644 --- a/src/auxiliary/Filesystem.cpp +++ b/src/auxiliary/Filesystem.cpp @@ -195,7 +195,8 @@ std::string collective_file_read(std::string const &path, MPI_Comm comm) if (!handle.good()) { throw std::runtime_error( - "Failed reading JSON config from file " + path + "."); + "[collective_file_read] Failed acessing file '" + path + + "' on MPI rank 0."); } stringLength = res.size() + 1; } From 18c42e6e5f8fad1eaaf0f0ca62b86a2365a8e876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jul 2023 14:46:09 +0200 Subject: [PATCH 03/10] ... and writing --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 3 +- src/IO/JSON/JSONIOHandlerImpl.cpp | 81 ++++++++++++++----- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 81dc9c39f3..9598602869 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -339,7 +339,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl // write to disk the json contents associated with the file // remove from m_dirty if unsetDirty == true - void putJsonContents(File const &, bool unsetDirty = true); + auto putJsonContents(File const &, bool unsetDirty = true) + -> decltype(m_jsonVals)::iterator; // figure out the file position of the writable // (preferring the parent's file position) and extend it diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index f6887dc34c..fba3fd7d8a 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -33,6 +33,7 @@ #include "openPMD/backend/Writable.hpp" #include +#include #include #include @@ -631,7 +632,11 @@ void JSONIOHandlerImpl::closeFile( auto fileIterator = m_files.find(writable); if (fileIterator != m_files.end()) { - putJsonContents(fileIterator->second); + auto it = putJsonContents(fileIterator->second); + if (it != m_jsonVals.end()) + { + m_jsonVals.erase(it); + } m_dirty.erase(fileIterator->second); // do not invalidate the file // it still exists, it is just not open @@ -1262,6 +1267,7 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) { return it->second; } + // read from file auto serialImplementation = [&file, this]() { auto [fh, fh_with_precision, _] = getFilehandle(file, Access::READ_ONLY); @@ -1281,6 +1287,7 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) VERIFY(fh->good(), "[JSON] Failed reading from a file."); return res; }; +#if openPMD_HAVE_MPI auto parallelImplementation = [&file, this](MPI_Comm comm) { auto path = fullPath(*file); std::string collectivelyReadRawData = @@ -1305,8 +1312,6 @@ JSONIOHandlerImpl::obtainJsonContents(File const &file) } return res; }; - // read from file -#if openPMD_HAVE_MPI std::shared_ptr res; if (m_communicator.has_value()) { @@ -1332,10 +1337,10 @@ nlohmann::json &JSONIOHandlerImpl::obtainJsonContents(Writable *writable) return (*obtainJsonContents(file))[filePosition->id]; } -void JSONIOHandlerImpl::putJsonContents( +auto JSONIOHandlerImpl::putJsonContents( File const &filename, bool unsetDirty // = true -) + ) -> decltype(m_jsonVals)::iterator { VERIFY_ALWAYS( filename.valid(), @@ -1343,29 +1348,69 @@ void JSONIOHandlerImpl::putJsonContents( auto it = m_jsonVals.find(filename); if (it != m_jsonVals.end()) { - auto [fh, _, fh_with_precision] = - getFilehandle(filename, Access::CREATE); - (void)_; (*it->second)["platform_byte_widths"] = platformSpecifics(); - switch (m_fileFormat) + auto writeSingleFile = [this, &it](std::string const &writeThisFile) { + auto [fh, _, fh_with_precision] = + getFilehandle(File(writeThisFile), Access::CREATE); + (void)_; + + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } + + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") + }; + + auto serialImplementation = [&filename, &writeSingleFile]() { + writeSingleFile(*filename); + }; + +#if openPMD_HAVE_MPI + auto parallelImplementation = + [this, &filename, &writeSingleFile](MPI_Comm comm) { + auto path = fullPath(*filename); + auto dirpath = path + ".parallel"; + if (!auxiliary::create_directories(dirpath)) + { + throw std::runtime_error( + "Failed creating directory '" + dirpath + + "' for parallel JSON output"); + } + int rank = 0; + MPI_Comm_rank(comm, &rank); + std::stringstream subfilePath; + subfilePath << dirpath << "/mpi_rank_" << std::setw(6) + << std::setfill('0') << rank << ".json"; + writeSingleFile(subfilePath.str()); + }; + + std::shared_ptr res; + if (m_communicator.has_value()) { - case FileFormat::Json: - *fh_with_precision << *it->second << std::endl; - break; - case FileFormat::Toml: - *fh_with_precision << openPMD::json::jsonToToml(*it->second) - << std::endl; - break; + parallelImplementation(m_communicator.value()); + } + else + { + serialImplementation(); } - VERIFY(fh->good(), "[JSON] Failed writing data to disk.") - m_jsonVals.erase(it); +#else + serialImplementation(); +#endif if (unsetDirty) { m_dirty.erase(filename); } } + return it; } std::shared_ptr JSONIOHandlerImpl::setAndGetFilePosition( From 8725e800921ddff1c1fa3eeac6121fd9a8ee3a29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jul 2023 15:18:26 +0200 Subject: [PATCH 04/10] Set padding according to MPI rank --- src/IO/JSON/JSONIOHandlerImpl.cpp | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index fba3fd7d8a..91e9eac6fb 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1374,8 +1374,24 @@ auto JSONIOHandlerImpl::putJsonContents( }; #if openPMD_HAVE_MPI + auto num_digits = [](unsigned n) -> unsigned { + constexpr auto max = std::numeric_limits::max(); + unsigned base_10 = 1; + unsigned res = 1; + while (base_10 < max) + { + base_10 *= 10; + if (n / base_10 == 0) + { + return res; + } + ++res; + } + return res; + }; + auto parallelImplementation = - [this, &filename, &writeSingleFile](MPI_Comm comm) { + [this, &filename, &writeSingleFile, &num_digits](MPI_Comm comm) { auto path = fullPath(*filename); auto dirpath = path + ".parallel"; if (!auxiliary::create_directories(dirpath)) @@ -1384,10 +1400,12 @@ auto JSONIOHandlerImpl::putJsonContents( "Failed creating directory '" + dirpath + "' for parallel JSON output"); } - int rank = 0; + int rank = 0, size = 0; MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); std::stringstream subfilePath; - subfilePath << dirpath << "/mpi_rank_" << std::setw(6) + subfilePath << dirpath << "/mpi_rank_" + << std::setw(num_digits(size - 1)) << std::setfill('0') << rank << ".json"; writeSingleFile(subfilePath.str()); }; From db251298d1457c9f04681c1fe9b7e62b6c26d2a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 10 Jul 2023 15:49:34 +0200 Subject: [PATCH 05/10] Write README.txt file --- include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp | 1 + src/IO/JSON/JSONIOHandlerImpl.cpp | 67 ++++++++++++++----- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 9598602869..da7e296d59 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -73,6 +73,7 @@ struct File std::string name; bool valid = true; + bool printedReadmeWarningAlready = false; }; std::shared_ptr fileState; diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 91e9eac6fb..8f775832be 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1390,25 +1390,58 @@ auto JSONIOHandlerImpl::putJsonContents( return res; }; - auto parallelImplementation = - [this, &filename, &writeSingleFile, &num_digits](MPI_Comm comm) { - auto path = fullPath(*filename); - auto dirpath = path + ".parallel"; - if (!auxiliary::create_directories(dirpath)) + auto parallelImplementation = [this, + &filename, + &writeSingleFile, + &num_digits](MPI_Comm comm) { + auto path = fullPath(*filename); + auto dirpath = path + ".parallel"; + if (!auxiliary::create_directories(dirpath)) + { + throw std::runtime_error( + "Failed creating directory '" + dirpath + + "' for parallel JSON output"); + } + int rank = 0, size = 0; + MPI_Comm_rank(comm, &rank); + MPI_Comm_size(comm, &size); + std::stringstream subfilePath; + subfilePath << dirpath << "/mpi_rank_" + << std::setw(num_digits(size - 1)) << std::setfill('0') + << rank << ".json"; + writeSingleFile(subfilePath.str()); + if (rank == 0) + { + constexpr char const *readme_msg = R"( +This folder has been created by a parallel instance of the JSON backend in +openPMD. There is one JSON file for each parallel writer MPI rank. +The parallel JSON backend performs no metadata or data aggregation at all. + +This functionality is intended mainly for debugging and prototyping workflows. +There is no support in the openPMD-api for reading this folder as a single +dataset. For reading purposes, either pick a single .json file and read that, or +merge the .json files somehow (no tooling provided for this (yet)). +)"; + std::fstream readme_file; + readme_file.open( + dirpath + "/README.txt", + std::ios_base::out | std::ios_base::trunc); + readme_file << readme_msg + 1; + readme_file.close(); + if (!readme_file.good() && + !filename.fileState->printedReadmeWarningAlready) { - throw std::runtime_error( - "Failed creating directory '" + dirpath + - "' for parallel JSON output"); + std::cerr + << "[Warning] Something went wrong in trying to create " + "README file at '" + << dirpath + << "/README.txt'. Will ignore and continue. The README " + "message would have been:\n----------\n" + << readme_msg + 1 << "----------" << std::endl; + filename.fileState->printedReadmeWarningAlready = true; } - int rank = 0, size = 0; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); - std::stringstream subfilePath; - subfilePath << dirpath << "/mpi_rank_" - << std::setw(num_digits(size - 1)) - << std::setfill('0') << rank << ".json"; - writeSingleFile(subfilePath.str()); - }; + } + }; std::shared_ptr res; if (m_communicator.has_value()) From 2c67158a1f5847898c85b0027f4a6df111302503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 22 Sep 2023 17:55:53 +0200 Subject: [PATCH 06/10] Bug fix: don't double prepend base dir --- src/IO/JSON/JSONIOHandlerImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 8f775832be..8a70d4ca97 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1406,7 +1406,8 @@ auto JSONIOHandlerImpl::putJsonContents( MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); std::stringstream subfilePath; - subfilePath << dirpath << "/mpi_rank_" + // writeSingleFile will prepend the base dir + subfilePath << *filename << ".parallel/mpi_rank_" << std::setw(num_digits(size - 1)) << std::setfill('0') << rank << ".json"; writeSingleFile(subfilePath.str()); From ebaccade1b6b866b00e27e03d2b218cb1545504b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 09:41:24 +0200 Subject: [PATCH 07/10] Test parallel output in openpmd-pipe test --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c21690f0c9..ef3f77bf9d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1370,7 +1370,7 @@ if(openPMD_BUILD_TESTING) --outfile \ ../samples/git-sample/thetaMode/data_%T.bp && \ \ - ${Python_EXECUTABLE} \ + ${MPI_TEST_EXE} ${Python_EXECUTABLE} \ ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe \ --infile ../samples/git-sample/thetaMode/data_%T.bp \ --outfile ../samples/git-sample/thetaMode/data%T.json \ From c6746f9009b764648a1a0ef1621ad870e4bb7e76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 12 Oct 2023 11:01:22 +0200 Subject: [PATCH 08/10] Bug fix: use mpi_rank_%i.toml when writing to TOML --- src/IO/JSON/JSONIOHandlerImpl.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 8a70d4ca97..5518b1b98f 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1409,7 +1409,16 @@ auto JSONIOHandlerImpl::putJsonContents( // writeSingleFile will prepend the base dir subfilePath << *filename << ".parallel/mpi_rank_" << std::setw(num_digits(size - 1)) << std::setfill('0') - << rank << ".json"; + << rank << [&]() { + switch (m_fileFormat) + { + case FileFormat::Json: + return ".json"; + case FileFormat::Toml: + return ".toml"; + } + throw std::runtime_error("Unreachable!"); + }(); writeSingleFile(subfilePath.str()); if (rank == 0) { From d2fb0703d50a4627951262f88f32df3648fe26f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 13:37:12 +0100 Subject: [PATCH 09/10] Refactor `if` statement --- src/IO/JSON/JSONIOHandlerImpl.cpp | 104 +++++++++++++++--------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 5518b1b98f..9910714c4b 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1346,54 +1346,55 @@ auto JSONIOHandlerImpl::putJsonContents( filename.valid(), "[JSON] File has been overwritten/deleted before writing"); auto it = m_jsonVals.find(filename); - if (it != m_jsonVals.end()) + if (it == m_jsonVals.end()) { - (*it->second)["platform_byte_widths"] = platformSpecifics(); + return it; + } - auto writeSingleFile = [this, &it](std::string const &writeThisFile) { - auto [fh, _, fh_with_precision] = - getFilehandle(File(writeThisFile), Access::CREATE); - (void)_; + (*it->second)["platform_byte_widths"] = platformSpecifics(); - switch (m_fileFormat) - { - case FileFormat::Json: - *fh_with_precision << *it->second << std::endl; - break; - case FileFormat::Toml: - *fh_with_precision << openPMD::json::jsonToToml(*it->second) - << std::endl; - break; - } + auto writeSingleFile = [this, &it](std::string const &writeThisFile) { + auto [fh, _, fh_with_precision] = + getFilehandle(File(writeThisFile), Access::CREATE); + (void)_; - VERIFY(fh->good(), "[JSON] Failed writing data to disk.") - }; + switch (m_fileFormat) + { + case FileFormat::Json: + *fh_with_precision << *it->second << std::endl; + break; + case FileFormat::Toml: + *fh_with_precision << openPMD::json::jsonToToml(*it->second) + << std::endl; + break; + } - auto serialImplementation = [&filename, &writeSingleFile]() { - writeSingleFile(*filename); - }; + VERIFY(fh->good(), "[JSON] Failed writing data to disk.") + }; + + auto serialImplementation = [&filename, &writeSingleFile]() { + writeSingleFile(*filename); + }; #if openPMD_HAVE_MPI - auto num_digits = [](unsigned n) -> unsigned { - constexpr auto max = std::numeric_limits::max(); - unsigned base_10 = 1; - unsigned res = 1; - while (base_10 < max) + auto num_digits = [](unsigned n) -> unsigned { + constexpr auto max = std::numeric_limits::max(); + unsigned base_10 = 1; + unsigned res = 1; + while (base_10 < max) + { + base_10 *= 10; + if (n / base_10 == 0) { - base_10 *= 10; - if (n / base_10 == 0) - { - return res; - } - ++res; + return res; } - return res; - }; + ++res; + } + return res; + }; - auto parallelImplementation = [this, - &filename, - &writeSingleFile, - &num_digits](MPI_Comm comm) { + auto parallelImplementation = + [this, &filename, &writeSingleFile, &num_digits](MPI_Comm comm) { auto path = fullPath(*filename); auto dirpath = path + ".parallel"; if (!auxiliary::create_directories(dirpath)) @@ -1453,23 +1454,22 @@ merge the .json files somehow (no tooling provided for this (yet)). } }; - std::shared_ptr res; - if (m_communicator.has_value()) - { - parallelImplementation(m_communicator.value()); - } - else - { - serialImplementation(); - } + std::shared_ptr res; + if (m_communicator.has_value()) + { + parallelImplementation(m_communicator.value()); + } + else + { + serialImplementation(); + } #else - serialImplementation(); + serialImplementation(); #endif - if (unsetDirty) - { - m_dirty.erase(filename); - } + if (unsetDirty) + { + m_dirty.erase(filename); } return it; } From b92b9aa40df70fdcf5183ac949aaad119e09805d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 24 Nov 2023 14:33:37 +0100 Subject: [PATCH 10/10] Add documentation --- docs/source/backends/json.rst | 37 +++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/source/backends/json.rst b/docs/source/backends/json.rst index 48ec6b1f44..bbae92aaf6 100644 --- a/docs/source/backends/json.rst +++ b/docs/source/backends/json.rst @@ -92,7 +92,6 @@ propagate the exception thrown by Niels Lohmann's library. The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. -A parallel (i.e. MPI) implementation is *not* available. TOML Restrictions ----------------- @@ -106,7 +105,41 @@ TOML does not support null values. The (keys) names ``"attributes"``, ``"data"`` and ``"datatype"`` are reserved and must not be used for base/mesh/particles path, records and their components. -A parallel (i.e. MPI) implementation is *not* available. + +Using in parallel (MPI) +----------------------- + +Parallel I/O is not a first-class citizen in the JSON and TOML backends, and neither backend will "go out of its way" to support parallel workflows. + +However there is a rudimentary form of read and write support in parallel: + +Parallel reading +................ + +In order not to overload the parallel filesystem with parallel reads, read access to JSON datasets is done by rank 0 and then broadcast to all other ranks. +Note that there is no granularity whatsoever in reading a JSON file. +A JSON file is always read into memory and broadcast to all other ranks in its entirety. + +Parallel writing +................ + +When executed in an MPI context, the JSON/TOML backends will not directly output a single text file, but instead a folder containing one file per MPI rank. +Neither backend will perform any data aggregation at all. + +.. note:: + + The parallel write support of the JSON/TOML backends is intended mainly for debugging and prototyping workflows. + +The folder will use the specified Series name, but append the postfix ``.parallel``. +(This is a deliberate indication that this folder cannot directly be opened again by the openPMD-api as a JSON/TOML dataset.) +This folder contains for each MPI rank *i* a file ``mpi_rank_.json`` (resp. ``mpi_rank_.toml``), containing the serial output of that rank. +A ``README.txt`` with basic usage instructions is also written. + +.. note:: + + There is no direct support in the openPMD-api to read a JSON/TOML dataset written in this parallel fashion. The single files (e.g. ``data.json.parallel/mpi_rank_0.json``) are each valid openPMD files and can be read separately, however. + + Note that the auxiliary function ``json::merge()`` (or in Python ``openpmd_api.merge_json()``) is not adequate for merging the single JSON/TOML files back into one, since it does not merge anything below the array level. Example