From 314a0a6c26552ab55a3c54c57be70be734d69fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 Jan 2024 18:29:18 +0100 Subject: [PATCH 1/4] Fix availableChunks for READ_LINEAR in ADIOS2 --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 708fbbdef0..618bae3002 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -33,6 +33,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" +#include #include #include // std::tolower #include @@ -1455,8 +1456,9 @@ void ADIOS2IOHandlerImpl::availableChunks( std::string varName = nameOfVariable(writable); auto engine = ba.getEngine(); // make sure that data are present auto datatype = detail::fromADIOS2Type(ba.m_IO.VariableType(varName)); - bool allSteps = ba.streamStatus == - detail::BufferedActions::StreamStatus::ReadWithoutStream; + bool allSteps = ba.m_mode != adios2::Mode::Read && + ba.streamStatus == + detail::BufferedActions::StreamStatus::ReadWithoutStream; switchAdios2VariableType( datatype, parameters, From ceb29925e22c46ebc0d1c8bbd46a13f2481f0dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 Jan 2024 18:51:14 +0100 Subject: [PATCH 2/4] Update src/IO/ADIOS/ADIOS2IOHandler.cpp --- src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 618bae3002..82cc2210e6 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -33,7 +33,6 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" -#include #include #include // std::tolower #include From d39e8bc9699223427802001dbe0b8546c5cd46cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 12 Jan 2024 13:24:35 +0100 Subject: [PATCH 3/4] Add testing --- include/openPMD/backend/Attributable.hpp | 2 + src/backend/Attributable.cpp | 21 ++++++ test/SerialIOTest.cpp | 93 ++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index c7b92b8b44..3621a22369 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -229,6 +229,8 @@ class Attributable /** Reconstructs a path that can be passed to a Series constructor */ std::string filePath() const; + /** Return the path ob the object within the openPMD file */ + std::string openPMDPath() const; }; /** diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 7eaf47dd07..357914d8b9 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace openPMD { @@ -187,6 +188,26 @@ std::string Attributable::MyPath::filePath() const return directory + seriesName + seriesExtension; } +std::string Attributable::MyPath::openPMDPath() const +{ + if (group.empty()) + { + return std::string(); + } + else + { + std::stringstream res; + auto it = group.begin(); + auto end = group.end(); + res << *it++; + for (; it != end; ++it) + { + res << '/' << *it; + } + return res.str(); + } +} + auto Attributable::myPath() const -> MyPath { MyPath res; diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 7f126e104f..94f2fe65a3 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5096,6 +5096,39 @@ bool areEqual(T a, T b) } // namespace epsilon #if openPMD_HAVE_ADIOS2 + +#define openPMD_VERBOSE_CHUNKS 0 + +#if openPMD_VERBOSE_CHUNKS +static std::string format_chunk(ChunkInfo const &chunk_info) +{ + std::stringstream result; + auto print_vector = [&result](auto const &vec) { + if (vec.empty()) + { + result << "[]"; + } + else + { + auto it = vec.begin(); + result << '[' << *it++; + auto end = vec.end(); + for (; it != end; ++it) + { + result << ',' << *it; + } + result << ']'; + } + }; + result << '('; + print_vector(chunk_info.offset); + result << '|'; + print_vector(chunk_info.extent); + result << ')'; + return result.str(); +} +#endif + TEST_CASE("git_adios2_sample_test", "[serial][adios2]") { using namespace epsilon; @@ -5105,11 +5138,71 @@ TEST_CASE("git_adios2_sample_test", "[serial][adios2]") std::string const samplePath = "../samples/git-sample/3d-bp4/example-3d-bp4.bp"; + std::string const samplePathFilebased = + "../samples/git-sample/3d-bp4/example-3d-bp4_%T.bp"; if (!auxiliary::directory_exists(samplePath)) { std::cerr << "git sample '" << samplePath << "' not accessible \n"; return; } + + /* + * This checks a regression introduced by + * https://github.com/openPMD/openPMD-api/pull/1498 and fixed by + * https://github.com/openPMD/openPMD-api/pull/1586 + */ + for (auto const &[filepath, access] : + {std::make_pair(samplePath, Access::READ_ONLY), + std::make_pair(samplePathFilebased, Access::READ_ONLY), + std::make_pair(samplePath, Access::READ_LINEAR), + std::make_pair(samplePathFilebased, Access::READ_LINEAR)}) + { + Series read(filepath, access); + + for (auto iteration : read.readIterations()) + { + for (auto &[mesh_name, mesh] : iteration.meshes) + { + for (auto &[component_name, component] : mesh) + { +#if openPMD_VERBOSE_CHUNKS + std::cout << "Chunks for '" + << component.myPath().openPMDPath() + << "':" << std::endl; + for (auto const &chunk : component.availableChunks()) + { + std::cout << "\t" << format_chunk(chunk) << std::endl; + } +#else + component.availableChunks(); +#endif + } + } + for (auto &[particle_species_name, particle_species] : + iteration.particles) + { + for (auto &[record_name, record] : particle_species) + { + for (auto &[component_name, component] : record) + { +#if openPMD_VERBOSE_CHUNKS + std::cout << "Chunks for '" + << component.myPath().openPMDPath() + << "':" << std::endl; + for (auto const &chunk : component.availableChunks()) + { + std::cout << "\t" << format_chunk(chunk) + << std::endl; + } +#else + component.availableChunks(); +#endif + } + } + } + } + } + Series o(samplePath, Access::READ_ONLY, R"({"backend": "adios2"})"); REQUIRE(o.openPMD() == "1.1.0"); REQUIRE(o.openPMDextension() == 0); From 230323e47879d1ab0fee2d1a09be66db73757e28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 12 Jan 2024 17:12:27 +0100 Subject: [PATCH 4/4] CI fixes --- test/SerialIOTest.cpp | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 94f2fe65a3..e0d4c75348 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -5159,43 +5159,45 @@ TEST_CASE("git_adios2_sample_test", "[serial][adios2]") { Series read(filepath, access); + // false positive by clang-tidy? + // NOLINTNEXTLINE(performance-for-range-copy) for (auto iteration : read.readIterations()) { - for (auto &[mesh_name, mesh] : iteration.meshes) + for (auto &mesh : iteration.meshes) { - for (auto &[component_name, component] : mesh) + for (auto &component : mesh.second) { #if openPMD_VERBOSE_CHUNKS std::cout << "Chunks for '" - << component.myPath().openPMDPath() + << component.second.myPath().openPMDPath() << "':" << std::endl; - for (auto const &chunk : component.availableChunks()) + for (auto const &chunk : component.second.availableChunks()) { std::cout << "\t" << format_chunk(chunk) << std::endl; } #else - component.availableChunks(); + component.second.availableChunks(); #endif } } - for (auto &[particle_species_name, particle_species] : - iteration.particles) + for (auto &particle_species : iteration.particles) { - for (auto &[record_name, record] : particle_species) + for (auto &record : particle_species.second) { - for (auto &[component_name, component] : record) + for (auto &component : record.second) { #if openPMD_VERBOSE_CHUNKS std::cout << "Chunks for '" - << component.myPath().openPMDPath() + << component.second.myPath().openPMDPath() << "':" << std::endl; - for (auto const &chunk : component.availableChunks()) + for (auto const &chunk : + component.second.availableChunks()) { std::cout << "\t" << format_chunk(chunk) << std::endl; } #else - component.availableChunks(); + component.second.availableChunks(); #endif } }