diff --git a/CMakeLists.txt b/CMakeLists.txt index c52373361e..b7a3425633 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -540,14 +540,12 @@ set(IO_SOURCE src/IO/ADIOS/ADIOS2PreloadAttributes.cpp src/IO/InvalidatableFile.cpp) set(IO_ADIOS1_SEQUENTIAL_SOURCE - src/Error.cpp src/auxiliary/Filesystem.cpp src/ChunkInfo.cpp src/IO/ADIOS/CommonADIOS1IOHandler.cpp src/IO/ADIOS/ADIOS1IOHandler.cpp src/IO/IOTask.cpp) set(IO_ADIOS1_SOURCE - src/Error.cpp src/auxiliary/Filesystem.cpp src/ChunkInfo.cpp src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -674,6 +672,9 @@ if(openPMD_HAVE_ADIOS1) target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_HAVE_ADIOS1=1) target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE openPMD_HAVE_MPI=0) target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE _NOMPI) # ADIOS header + # This ensures that the ADIOS1 targets don't ever include Error.hpp + # To avoid incompatible error types in weird compile configurations + target_compile_definitions(openPMD.ADIOS1.Serial PRIVATE OPENPMD_ADIOS1_IMPLEMENTATION) if(openPMD_HAVE_MPI) set_target_properties(openPMD.ADIOS1.Parallel PROPERTIES @@ -711,6 +712,9 @@ if(openPMD_HAVE_ADIOS1) target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE openPMD_HAVE_MPI=0) target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE _NOMPI) # ADIOS header endif() + # This ensures that the ADIOS1 targets don't ever include Error.hpp + # To avoid incompatible error types in weird compile configurations + target_compile_definitions(openPMD.ADIOS1.Parallel PRIVATE OPENPMD_ADIOS1_IMPLEMENTATION) # Runtime parameter and API status checks ("asserts") if(openPMD_USE_VERIFY) @@ -909,6 +913,7 @@ if(openPMD_USE_INVASIVE_TESTS) message(WARNING "Invasive tests that redefine class signatures are " "known to fail on Windows!") endif() + target_compile_definitions(openPMD PRIVATE openPMD_USE_INVASIVE_TESTS=1) endif() if(openPMD_BUILD_TESTING) diff --git a/include/openPMD/Error.hpp b/include/openPMD/Error.hpp index e172670bcc..9845cdcdf0 100644 --- a/include/openPMD/Error.hpp +++ b/include/openPMD/Error.hpp @@ -1,10 +1,17 @@ #pragma once +#include "openPMD/ThrowError.hpp" + #include +#include #include #include #include +#if defined(OPENPMD_ADIOS1_IMPLEMENTATION) +static_assert(false, "ADIOS1 implementation must not include Error.hpp"); +#endif + namespace openPMD { /** @@ -80,5 +87,39 @@ namespace error public: Internal(std::string const &what); }; + + /* + * Read error concerning a specific object. + */ + class ReadError : public Error + { + public: + AffectedObject affectedObject; + Reason reason; + // If empty, then the error is thrown by the frontend + std::optional backend; + std::string description; // object path, further details, ... + + ReadError( + AffectedObject, + Reason, + std::optional backend_in, + std::string description_in); + }; + + /* + * Inrecoverable parse error from the frontend. + */ + class ParseError : public Error + { + public: + ParseError(std::string what); + }; } // namespace error + +/** + * @brief Backward-compatibility alias for no_such_file_error. + * + */ +using no_such_file_error = error::ReadError; } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index bc8ea80ad5..7c0dd1a44e 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -1070,6 +1070,9 @@ namespace detail template void enqueue(BA &&ba, decltype(m_buffer) &); + template + void flush(Args &&...args); + struct ADIOS2FlushParams { /* @@ -1103,7 +1106,7 @@ namespace detail * deferred IO tasks had been queued. */ template - void flush( + void flush_impl( ADIOS2FlushParams flushParams, F &&performPutsGets, bool writeAttributes, @@ -1114,7 +1117,7 @@ namespace detail * and does not flush unconditionally. * */ - void flush(ADIOS2FlushParams, bool writeAttributes = false); + void flush_impl(ADIOS2FlushParams, bool writeAttributes = false); /** * @brief Begin or end an ADIOS step. diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index c3ac9d91e5..f444996c66 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -34,18 +34,10 @@ #include #include #include +#include namespace openPMD { -class no_such_file_error : public std::runtime_error -{ -public: - no_such_file_error(std::string const &what_arg) - : std::runtime_error(what_arg) - {} - virtual ~no_such_file_error() - {} -}; class unsupported_data_error : public std::runtime_error { @@ -143,6 +135,48 @@ namespace internal ///< Special state only active while internal routines are ///< running. }; + + // @todo put this somewhere else + template + auto withRWAccess(SeriesStatus &status, Functor &&functor, Args &&...args) + -> decltype(std::forward(functor)(std::forward(args)...)) + { + using Res = decltype(std::forward(functor)( + std::forward(args)...)); + if constexpr (std::is_void_v) + { + auto oldStatus = status; + status = internal::SeriesStatus::Parsing; + try + { + std::forward(functor)(); + } + catch (...) + { + status = oldStatus; + throw; + } + status = oldStatus; + return; + } + else + { + auto oldStatus = status; + status = internal::SeriesStatus::Parsing; + Res res; + try + { + res = std::forward(functor)(); + } + catch (...) + { + status = oldStatus; + throw; + } + status = oldStatus; + return res; + } + } } // namespace internal /** Interface for communicating between logical and physically persistent data. diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 04820a28d4..79e9b35739 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -208,10 +208,13 @@ class AbstractIOHandlerImpl { std::cerr << "[AbstractIOHandlerImpl] IO Task " << internal::operationAsString(i.operation) - << " failed with exception. Removing task" - << " from IO queue and passing on the exception." + << " failed with exception. Clearing IO queue and " + "passing on the exception." << std::endl; - (*m_handler).m_work.pop(); + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } throw; } (*m_handler).m_work.pop(); diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 2393b93322..179ade29fe 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -286,6 +286,8 @@ class Iteration : public Attributable std::string filePath, std::string const &groupPath, bool beginStep); void readGorVBased(std::string const &groupPath, bool beginStep); void read_impl(std::string const &groupPath); + void readMeshes(std::string const &meshesPath); + void readParticles(std::string const &particlesPath); /** * Status after beginning an IO step. Currently includes: diff --git a/include/openPMD/ReadIterations.hpp b/include/openPMD/ReadIterations.hpp index c5b2720dce..7d6266e4f0 100644 --- a/include/openPMD/ReadIterations.hpp +++ b/include/openPMD/ReadIterations.hpp @@ -104,9 +104,21 @@ class SeriesIterator std::optional nextIterationInStep(); - std::optional nextStep(); + /* + * When a step cannot successfully be opened, the method nextStep() calls + * itself again recursively. + * (Recursion massively simplifies the logic here, and it only happens + * in case of error.) + * After successfully beginning a step, this methods needs to remember, how + * many broken steps have been skipped. In case the Series does not use + * the /data/snapshot attribute, this helps figuring out which iteration + * is now active. Hence, recursion_depth. + */ + std::optional nextStep(size_t recursion_depth); std::optional loopBody(); + + void deactivateDeadIteration(iteration_index_t); }; /** diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index f3c40582fb..30b618db40 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -611,9 +611,15 @@ OPENPMD_private * Iterations/Records/Record Components etc. * If series.iterations contains the attribute `snapshot`, returns its * value. + * If do_always_throw_errors is false, this method will try to handle errors + * and turn them into a warning (useful when parsing a Series, since parsing + * should succeed without issue). + * If true, the error will always be re-thrown (useful when using + * ReadIterations since those methods should be aware when the current step + * is broken). */ std::optional > - readGorVBased(bool init = true); + readGorVBased(bool do_always_throw_errors, bool init); void readBase(); std::string iterationFilename(IterationIndex_t i); diff --git a/include/openPMD/ThrowError.hpp b/include/openPMD/ThrowError.hpp new file mode 100644 index 0000000000..eae561aff7 --- /dev/null +++ b/include/openPMD/ThrowError.hpp @@ -0,0 +1,70 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +/* + * For objects that must not include Error.hpp but still need to throw errors. + * In some exotic compiler configurations (clang-6 with libc++), + * including Error.hpp into the ADIOS1 backend leads to incompatible error type + * symbols. + * So, use only the functions defined in here in the ADIOS1 backend. + * Definitions are in Error.cpp. + */ + +#pragma once + +#include "openPMD/auxiliary/Export.hpp" + +#include +#include +#include + +namespace openPMD::error +{ +enum class AffectedObject +{ + Attribute, + Dataset, + File, + Group, + Other +}; + +enum class Reason +{ + NotFound, + CannotRead, + UnexpectedContent, + Inaccessible, + Other +}; + +[[noreturn]] OPENPMDAPI_EXPORT void +throwBackendConfigSchema(std::vector jsonPath, std::string what); + +[[noreturn]] OPENPMDAPI_EXPORT void +throwOperationUnsupportedInBackend(std::string backend, std::string what); + +[[noreturn]] OPENPMDAPI_EXPORT void throwReadError( + AffectedObject affectedObject, + Reason reason_in, + std::optional backend, + std::string description_in); +} // namespace openPMD::error diff --git a/include/openPMD/auxiliary/JSON_internal.hpp b/include/openPMD/auxiliary/JSON_internal.hpp index e2dc838423..e865ddc7de 100644 --- a/include/openPMD/auxiliary/JSON_internal.hpp +++ b/include/openPMD/auxiliary/JSON_internal.hpp @@ -23,8 +23,6 @@ #include "openPMD/config.hpp" -#include "openPMD/Error.hpp" - #include #include diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 8db82c69f0..c697593a12 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -141,6 +141,7 @@ class Container : public Attributable friend class Series; template friend class internal::EraseStaleEntries; + friend class SeriesIterator; protected: using ContainerData = internal::ContainerData; diff --git a/src/Error.cpp b/src/Error.cpp index a0331948c8..99096bd54e 100644 --- a/src/Error.cpp +++ b/src/Error.cpp @@ -17,6 +17,13 @@ namespace error , backend{std::move(backend_in)} {} + void + throwOperationUnsupportedInBackend(std::string backend, std::string what) + { + throw OperationUnsupportedInBackend( + std::move(backend), std::move(what)); + } + WrongAPIUsage::WrongAPIUsage(std::string what) : Error("Wrong API usage: " + what) {} @@ -46,11 +53,87 @@ namespace error , errorLocation(std::move(errorLocation_in)) {} + void throwBackendConfigSchema( + std::vector jsonPath, std::string what) + { + throw BackendConfigSchema(std::move(jsonPath), std::move(what)); + } + Internal::Internal(std::string const &what) : Error( "Internal error: " + what + "\nThis is a bug. Please report at ' " "https://github.com/openPMD/openPMD-api/issues'.") {} + + namespace + { + std::string asString(AffectedObject obj) + { + switch (obj) + { + using AO = AffectedObject; + case AO::Attribute: + return "Attribute"; + case AO::Dataset: + return "Dataset"; + case AO::File: + return "File"; + case AO::Group: + return "Group"; + case AO::Other: + return "Other"; + } + return "Unreachable"; + } + std::string asString(Reason obj) + { + switch (obj) + { + using Re = Reason; + case Re::NotFound: + return "NotFound"; + case Re::CannotRead: + return "CannotRead"; + case Re::UnexpectedContent: + return "UnexpectedContent"; + case Re::Inaccessible: + return "Inaccessible"; + case Re::Other: + return "Other"; + } + return "Unreachable"; + } + } // namespace + + ReadError::ReadError( + AffectedObject affectedObject_in, + Reason reason_in, + std::optional backend_in, + std::string description_in) + : Error( + (backend_in ? ("Read Error in backend " + *backend_in) + : "Read Error in frontend ") + + "\nObject type:\t" + asString(affectedObject_in) + + "\nError type:\t" + asString(reason_in) + + "\nFurther description:\t" + description_in) + , affectedObject(affectedObject_in) + , reason(reason_in) + , backend(std::move(backend_in)) + , description(std::move(description_in)) + {} + + void throwReadError( + AffectedObject affectedObject, + Reason reason, + std::optional backend, + std::string description) + { + throw ReadError( + affectedObject, reason, std::move(backend), std::move(description)); + } + + ParseError::ParseError(std::string what) : Error("Parse Error: " + what) + {} } // namespace error } // namespace openPMD diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 019e5a8078..662bf1ec69 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -18,6 +18,7 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ + #include "openPMD/IO/ADIOS/ADIOS1IOHandler.hpp" #include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp" @@ -157,10 +158,13 @@ std::future ADIOS1IOHandlerImpl::flush() { std::cerr << "[AbstractIOHandlerImpl] IO Task " << internal::operationAsString(i.operation) - << " failed with exception. Removing task" - << " from IO queue and passing on the exception." + << " failed with exception. Clearing IO queue and " + "passing on the exception." << std::endl; - handler->m_setup.pop(); + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } throw; } handler->m_setup.pop(); @@ -289,10 +293,13 @@ std::future ADIOS1IOHandlerImpl::flush() { std::cerr << "[AbstractIOHandlerImpl] IO Task " << internal::operationAsString(i.operation) - << " failed with exception. Removing task" - << " from IO queue and passing on the exception." + << " failed with exception. Clearing IO queue and " + "passing on the exception." << std::endl; - m_handler->m_work.pop(); + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } throw; } handler->m_work.pop(); diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 95362b2687..d768ec5b6b 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -112,7 +112,17 @@ ADIOS2IOHandlerImpl::~ADIOS2IOHandlerImpl() sorted.push_back(std::move(pair.second)); } m_fileData.clear(); - std::sort( + /* + * Technically, std::sort() is sufficient here, since file names are unique. + * Use std::stable_sort() for two reasons: + * 1) On some systems (clang 13.0.1, libc++ 13.0.1), std::sort() leads to + * weird inconsistent segfaults here. + * 2) Robustness against future changes. stable_sort() might become needed + * in future, and debugging this can be hard. + * 3) It does not really matter, this is just the destructor, so we can take + * the extra time. + */ + std::stable_sort( sorted.begin(), sorted.end(), [](auto const &left, auto const &right) { return left->m_file <= right->m_file; }); @@ -685,9 +695,11 @@ void ADIOS2IOHandlerImpl::openFile( { if (!auxiliary::directory_exists(m_handler->directory)) { - throw no_such_file_error( - "[ADIOS2] Supplied directory is not valid: " + - m_handler->directory); + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + "ADIOS2", + "Supplied directory is not valid: " + m_handler->directory); } std::string name = parameters.name + fileSuffix(); @@ -2305,9 +2317,11 @@ namespace detail if (type == Datatype::UNDEFINED) { - throw std::runtime_error( - "[ADIOS2] Requested attribute (" + name + - ") not found in backend."); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "ADIOS2", + name); } Datatype ret = switchType( @@ -2325,9 +2339,11 @@ namespace detail if (type == Datatype::UNDEFINED) { - throw std::runtime_error( - "[ADIOS2] Requested attribute (" + name + - ") not found in backend."); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "ADIOS2", + name); } Datatype ret = switchType( @@ -2844,8 +2860,49 @@ namespace detail std::unique_ptr(new _BA(std::forward(ba)))); } + template + void BufferedActions::flush(Args &&...args) + { + try + { + flush_impl(std::forward(args)...); + } + catch (error::ReadError const &) + { + /* + * We need to take actions out of the buffer, since an exception + * should reset everything from the current IOHandler->flush() call. + * However, we cannot simply clear the buffer, since tasks may have + * been enqueued to ADIOS2 already and we cannot undo that. + * So, we need to keep the memory alive for the benefit of ADIOS2. + * Luckily, we have m_alreadyEnqueued for exactly that purpose. + */ + for (auto &task : m_buffer) + { + m_alreadyEnqueued.emplace_back(std::move(task)); + } + m_buffer.clear(); + + // m_attributeWrites and m_attributeReads are for implementing the + // 2021 ADIOS2 schema which will go anyway. + // So, this ugliness here is temporary. + for (auto &task : m_attributeWrites) + { + m_alreadyEnqueued.emplace_back(std::unique_ptr{ + new BufferedAttributeWrite{std::move(task.second)}}); + } + m_attributeWrites.clear(); + /* + * An AttributeRead is not a deferred action, so we can clear it + * immediately. + */ + m_attributeReads.clear(); + throw; + } + } + template - void BufferedActions::flush( + void BufferedActions::flush_impl( ADIOS2FlushParams flushParams, F &&performPutGets, bool writeAttributes, @@ -2947,8 +3004,8 @@ namespace detail } } - void - BufferedActions::flush(ADIOS2FlushParams flushParams, bool writeAttributes) + void BufferedActions::flush_impl( + ADIOS2FlushParams flushParams, bool writeAttributes) { auto decideFlushAPICall = [this, flushTarget = flushParams.flushTarget]( adios2::Engine &engine) { @@ -2984,7 +3041,7 @@ namespace detail #endif }; - flush( + flush_impl( flushParams, [decideFlushAPICall = std::move(decideFlushAPICall)]( BufferedActions &ba, adios2::Engine &eng) { @@ -3019,7 +3076,9 @@ namespace detail { m_IO.DefineAttribute( ADIOS2Defaults::str_usesstepsAttribute, 0); - flush({FlushLevel::UserFlush}, /* writeAttributes = */ false); + flush( + ADIOS2FlushParams{FlushLevel::UserFlush}, + /* writeAttributes = */ false); return AdvanceStatus::RANDOMACCESS; } @@ -3059,7 +3118,7 @@ namespace detail } } flush( - {FlushLevel::UserFlush}, + ADIOS2FlushParams{FlushLevel::UserFlush}, [](BufferedActions &, adios2::Engine &eng) { eng.EndStep(); }, /* writeAttributes = */ true, /* flushUnconditionally = */ true); diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 3a13dc7fc8..dded98d94c 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -20,11 +20,10 @@ */ #include "openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp" -#include "openPMD/Error.hpp" +#include "openPMD/ThrowError.hpp" #if openPMD_HAVE_ADIOS1 -#include "openPMD/Error.hpp" #include "openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp" #include "openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" @@ -425,7 +424,7 @@ void CommonADIOS1IOHandlerImpl::createFile( if (m_handler->m_backendAccess == Access::APPEND && auxiliary::file_exists(name)) { - throw error::OperationUnsupportedInBackend( + error::throwOperationUnsupportedInBackend( "ADIOS1", "Appending to existing file on disk (use Access::CREATE to " "overwrite)"); @@ -515,7 +514,7 @@ static std::optional datasetTransform(json::TracingJSON config) } else { - throw error::BackendConfigSchema( + error::throwBackendConfigSchema( {"adios1", "dataset", "transform"}, "Key must convertible to type string."); } @@ -658,9 +657,11 @@ void CommonADIOS1IOHandlerImpl::openFile( Writable *writable, Parameter const ¶meters) { if (!auxiliary::directory_exists(m_handler->directory)) - throw no_such_file_error( - "[ADIOS1] Supplied directory is not valid: " + - m_handler->directory); + error::throwReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + "ADIOS1", + "Supplied directory is not valid: " + m_handler->directory); std::string name = m_handler->directory + parameters.name; if (!auxiliary::ends_with(name, ".bp")) @@ -1248,15 +1249,22 @@ void CommonADIOS1IOHandlerImpl::readAttribute( int status; status = adios_get_attr(f, attrname.c_str(), &datatype, &size, &data); - VERIFY( - status == 0, - "[ADIOS1] Internal error: Failed to get ADIOS1 attribute during " - "attribute read"); - VERIFY( - datatype != adios_unknown, - "[ADIOS1] Internal error: Read unknown ADIOS1 datatype during " - "attribute read"); - VERIFY(size != 0, "[ADIOS1] Internal error: ADIOS1 read 0-size attribute"); + if (status != 0) + { + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "ADIOS1", + attrname); + } + if (datatype == adios_unknown) + { + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "ADIOS1", + "Unknown datatype: " + attrname); + } // size is returned in number of allocated bytes // note the ill-named fixed-byte adios_... types @@ -1307,9 +1315,11 @@ void CommonADIOS1IOHandlerImpl::readAttribute( break; default: - throw unsupported_data_error( - "[ADIOS1] readAttribute: Unsupported ADIOS1 attribute datatype '" + - std::to_string(datatype) + "' in size check"); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "ADIOS1", + "Unsupported datatype: " + attrname); } Datatype dtype; @@ -1345,9 +1355,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_short " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_short: " + + attrname); break; case adios_integer: if (sizeof(short) == 4u) @@ -1371,9 +1384,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_integer " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_integer: " + + attrname); break; case adios_long: if (sizeof(short) == 8u) @@ -1397,9 +1413,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_long " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_long: " + + attrname); break; case adios_unsigned_byte: dtype = DT::UCHAR; @@ -1427,9 +1446,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_short found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_short: " + + attrname); break; case adios_unsigned_integer: if (sizeof(unsigned short) == 4u) @@ -1453,9 +1476,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_integer found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_integer: " + + attrname); break; case adios_unsigned_long: if (sizeof(unsigned short) == 8u) @@ -1479,9 +1506,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( a = Attribute(*reinterpret_cast(data)); } else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_long found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_long: " + + attrname); break; case adios_real: dtype = DT::FLOAT; @@ -1527,10 +1558,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( break; } default: - throw unsupported_data_error( - "[ADIOS1] readAttribute: Unsupported ADIOS1 attribute datatype " - "'" + - std::to_string(datatype) + "' in scalar branch"); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "Unsupported ADIOS1 attribute datatype '" + + std::to_string(datatype) + + "' in scalar branch: " + attrname); } } else @@ -1565,9 +1599,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_LONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_short " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_short: " + + attrname); break; } case adios_integer: { @@ -1587,9 +1624,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_LONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_integer " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_integer: " + + attrname); break; } case adios_long: { @@ -1609,9 +1649,12 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_LONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype adios_long " - "found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type adios_long: " + + attrname); break; } case adios_unsigned_byte: { @@ -1642,9 +1685,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_ULONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_short found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_short: " + + attrname); break; } case adios_unsigned_integer: { @@ -1665,9 +1712,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_ULONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_integer found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_integer: " + + attrname); break; } case adios_unsigned_long: { @@ -1688,9 +1739,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( readVectorAttributeInternal(data, size), DT::VEC_ULONGLONG); else - throw unsupported_data_error( - "[ADIOS1] No native equivalent for Datatype " - "adios_unsigned_long found."); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "No native equivalent found for type " + "adios_unsigned_long: " + + attrname); break; } case adios_real: { @@ -1750,10 +1805,13 @@ void CommonADIOS1IOHandlerImpl::readAttribute( } default: - throw unsupported_data_error( - "[ADIOS1] readAttribute: Unsupported ADIOS1 attribute datatype " - "'" + - std::to_string(datatype) + "' in vector branch"); + error::throwReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + "ADIOS1", + "Unsupported ADIOS1 attribute datatype '" + + std::to_string(datatype) + + "' in vector branch: " + attrname); } } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index 20f571e980..fe628e48aa 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -179,10 +179,13 @@ std::future ParallelADIOS1IOHandlerImpl::flush() { std::cerr << "[AbstractIOHandlerImpl] IO Task " << internal::operationAsString(i.operation) - << " failed with exception. Removing task" - << " from IO queue and passing on the exception." + << " failed with exception. Clearing IO queue and " + "passing on the exception." << std::endl; - handler->m_setup.pop(); + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } throw; } handler->m_setup.pop(); @@ -309,10 +312,13 @@ std::future ParallelADIOS1IOHandlerImpl::flush() { std::cerr << "[AbstractIOHandlerImpl] IO Task " << internal::operationAsString(i.operation) - << " failed with exception. Removing task" - << " from IO queue and passing on the exception." + << " failed with exception. Clearing IO queue and " + "passing on the exception." << std::endl; - m_handler->m_work.pop(); + while (!m_handler->m_work.empty()) + { + m_handler->m_work.pop(); + } throw; } handler->m_work.pop(); diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 01979d8071..db26b2098d 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -774,8 +774,11 @@ void HDF5IOHandlerImpl::openFile( Writable *writable, Parameter const ¶meters) { if (!auxiliary::directory_exists(m_handler->directory)) - throw no_such_file_error( - "[HDF5] Supplied directory is not valid: " + m_handler->directory); + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + "HDF5", + "Supplied directory is not valid: " + m_handler->directory); std::string name = m_handler->directory + parameters.name; if (!auxiliary::ends_with(name, ".h5")) @@ -809,7 +812,11 @@ void HDF5IOHandlerImpl::openFile( hid_t file_id; file_id = H5Fopen(name.c_str(), flags, m_fileAccessProperty); if (file_id < 0) - throw no_such_file_error("[HDF5] Failed to open HDF5 file " + name); + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + "HDF5", + "Failed to open HDF5 file " + name); writable->written = true; writable->abstractFilePosition = std::make_shared("/"); @@ -853,9 +860,15 @@ void HDF5IOHandlerImpl::openPath( node_id = H5Gopen( file.id, concrete_h5_file_position(writable->parent).c_str(), gapl); - VERIFY( - node_id >= 0, - "[HDF5] Internal error: Failed to open HDF5 group during path opening"); + if (node_id < 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "HDF5", + "[HDF5] Internal error: Failed to open HDF5 group during path " + "opening"); + } /* Sanitize path */ std::string path = parameters.path; @@ -866,30 +879,50 @@ void HDF5IOHandlerImpl::openPath( if (!auxiliary::ends_with(path, '/')) path += '/'; path_id = H5Gopen(node_id, path.c_str(), gapl); - VERIFY( - path_id >= 0, - "[HDF5] Internal error: Failed to open HDF5 group during path " - "opening"); + if (path_id < 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::NotFound, + "HDF5", + "[HDF5] Internal error: Failed to open HDF5 group during path " + "opening"); + } herr_t status; status = H5Gclose(path_id); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close HDF5 group during path " - "opening"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::Other, + "HDF5", + "[HDF5] Internal error: Failed to close HDF5 group during path " + "opening"); + } } herr_t status; status = H5Gclose(node_id); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close HDF5 group during path " - "opening"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::Other, + "HDF5", + "[HDF5] Internal error: Failed to close HDF5 group during path " + "opening"); + } status = H5Pclose(gapl); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close HDF5 property during path " - "opening"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Group, + error::Reason::Other, + "HDF5", + "[HDF5] Internal error: Failed to close HDF5 property during path " + "opening"); + } writable->written = true; writable->abstractFilePosition = std::make_shared(path); @@ -1771,18 +1804,30 @@ void HDF5IOHandlerImpl::readAttribute( obj_id = H5Oopen(file.id, concrete_h5_file_position(writable).c_str(), fapl); - VERIFY( - obj_id >= 0, - std::string("[HDF5] Internal error: Failed to open HDF5 object '") + - concrete_h5_file_position(writable).c_str() + - "' during attribute read"); + if (obj_id < 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "HDF5", + std::string("[HDF5] Internal error: Failed to open HDF5 object '") + + concrete_h5_file_position(writable).c_str() + + "' during attribute read"); + } std::string const &attr_name = parameters.name; attr_id = H5Aopen(obj_id, attr_name.c_str(), H5P_DEFAULT); - VERIFY( - attr_id >= 0, - std::string("[HDF5] Internal error: Failed to open HDF5 attribute '") + - attr_name + "' (" + concrete_h5_file_position(writable).c_str() + - ") during attribute read"); + if (attr_id < 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "HDF5", + std::string( + "[HDF5] Internal error: Failed to open HDF5 attribute '") + + attr_name + "' (" + + concrete_h5_file_position(writable).c_str() + + ") during attribute read"); + } hid_t attr_type, attr_space; attr_type = H5Aget_type(attr_id); @@ -1793,10 +1838,15 @@ void HDF5IOHandlerImpl::readAttribute( std::vector maxdims(ndims, 0); status = H5Sget_simple_extent_dims(attr_space, dims.data(), maxdims.data()); - VERIFY( - status == ndims, - "[HDF5] Internal error: Failed to get dimensions during attribute " - "read"); + if (status != ndims) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to get dimensions during attribute " + "read"); + } H5S_class_t attr_class = H5Sget_simple_extent_type(attr_space); Attribute a(0); @@ -1929,7 +1979,10 @@ void HDF5IOHandlerImpl::readAttribute( a = Attribute(static_cast(enumVal)); } else - throw unsupported_data_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Unsupported attribute enumeration"); } else if (H5Tget_class(attr_type) == H5T_COMPOUND) @@ -1997,21 +2050,33 @@ void HDF5IOHandlerImpl::readAttribute( a = Attribute(cld); } else - throw unsupported_data_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Unknown complex type representation"); } else - throw unsupported_data_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Compound attribute type not supported"); } else - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Unsupported scalar attribute type"); } else if (attr_class == H5S_SIMPLE) { if (ndims != 1) - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Unsupported attribute (array with ndims != 1)"); if (H5Tequal(attr_type, H5T_NATIVE_CHAR)) @@ -2126,11 +2191,16 @@ void HDF5IOHandlerImpl::readAttribute( { std::vector vc(dims[0]); status = H5Aread(attr_id, attr_type, vc.data()); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to read attribute " + - attr_name + " at " + - concrete_h5_file_position(writable)); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to read attribute " + + attr_name + " at " + + concrete_h5_file_position(writable)); + } for (auto const &val : vc) vs.push_back(auxiliary::strip(std::string(val), {'\0'})); status = H5Dvlen_reclaim( @@ -2148,26 +2218,45 @@ void HDF5IOHandlerImpl::readAttribute( a = Attribute(vs); } else - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "HDF5", "[HDF5] Unsupported simple attribute type"); } else throw std::runtime_error("[HDF5] Unsupported attribute class"); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to read attribute " + attr_name + - " at " + concrete_h5_file_position(writable)); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to read attribute " + attr_name + + " at " + concrete_h5_file_position(writable)); + } status = H5Tclose(attr_type); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close attribute datatype during " - "attribute read"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to close attribute datatype during " + "attribute read"); + } status = H5Sclose(attr_space); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close attribute file space during " - "attribute read"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to close attribute file space " + "during " + "attribute read"); + } auto dtype = parameters.dtype; *dtype = a.dtype; @@ -2175,21 +2264,36 @@ void HDF5IOHandlerImpl::readAttribute( *resource = a.getResource(); status = H5Aclose(attr_id); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close attribute " + attr_name + - " at " + concrete_h5_file_position(writable) + - " during attribute read"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to close attribute " + attr_name + + " at " + concrete_h5_file_position(writable) + + " during attribute read"); + } status = H5Oclose(obj_id); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close " + - concrete_h5_file_position(writable) + " during attribute read"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to close " + + concrete_h5_file_position(writable) + " during attribute read"); + } status = H5Pclose(fapl); - VERIFY( - status == 0, - "[HDF5] Internal error: Failed to close HDF5 attribute during " - "attribute read"); + if (status != 0) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::CannotRead, + "HDF5", + "[HDF5] Internal error: Failed to close HDF5 attribute during " + "attribute read"); + } } void HDF5IOHandlerImpl::listPaths( diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 272478789b..8f25b56584 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -22,6 +22,7 @@ #include "openPMD/IO/JSON/JSONIOHandlerImpl.hpp" #include "openPMD/Datatype.hpp" #include "openPMD/DatatypeHelpers.hpp" +#include "openPMD/Error.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -529,8 +530,11 @@ void JSONIOHandlerImpl::openFile( { if (!auxiliary::directory_exists(m_handler->directory)) { - throw no_such_file_error( - "[JSON] Supplied directory is not valid: " + m_handler->directory); + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + "JSON", + "Supplied directory is not valid: " + m_handler->directory); } std::string name = parameter.name; @@ -857,11 +861,15 @@ void JSONIOHandlerImpl::readAttribute( auto &jsonLoc = obtainJsonContents(writable)["attributes"]; setAndGetFilePosition(writable); std::string error_msg("[JSON] No such attribute '"); - error_msg.append(name) - .append("' in the given location '") - .append(jsonLoc.dump()) - .append("'."); - VERIFY_ALWAYS(hasKey(jsonLoc, name), error_msg) + if (!hasKey(jsonLoc, name)) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::NotFound, + "JSON", + "Tried looking up attribute '" + name + + "' in object: " + jsonLoc.dump()); + } auto &j = jsonLoc[name]; try { @@ -871,9 +879,12 @@ void JSONIOHandlerImpl::readAttribute( } catch (json::type_error &) { - throw std::runtime_error( - "[JSON] The given location does not contain a properly formatted " - "attribute"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + "JSON", + "No properly formatted attribute with name '" + name + + "' found in object: " + jsonLoc.dump()); } } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 179bf0f97d..54d23589f7 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -430,7 +430,12 @@ void Iteration::read_impl(std::string const &groupPath) val.has_value()) setDt(val.value()); else - throw std::runtime_error("Unexpected Attribute datatype for 'dt'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'dt' (expected double, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "time"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -446,7 +451,13 @@ void Iteration::read_impl(std::string const &groupPath) val.has_value()) setTime(val.value()); else - throw std::runtime_error("Unexpected Attribute datatype for 'time'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'time' (expected double, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -455,8 +466,13 @@ void Iteration::read_impl(std::string const &groupPath) val.has_value()) setTimeUnitSI(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'timeUnitSI'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'timeUnitSI' (expected double, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ @@ -489,97 +505,177 @@ void Iteration::read_impl(std::string const &groupPath) if (hasMeshes) { - pOpen.path = s.meshesPath(); - IOHandler()->enqueue(IOTask(&meshes, pOpen)); + try + { + readMeshes(s.meshesPath()); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read meshes in iteration " << groupPath + << " and will skip them due to read error:\n" + << err.what() << std::endl; + meshes = {}; + meshes.dirty() = false; + } + } + else + { + meshes.dirty() = false; + } + + if (hasParticles) + { + try + { + readParticles(s.particlesPath()); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particles in iteration " << groupPath + << " and will skip them due to read error:\n" + << err.what() << std::endl; + particles = {}; + particles.dirty() = false; + } + } + else + { + particles.dirty() = false; + } - meshes.readAttributes(ReadMode::FullyReread); + readAttributes(ReadMode::FullyReread); +#ifdef openPMD_USE_INVASIVE_TESTS + if (containsAttribute("__openPMD_internal_fail")) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::Other, + {}, + "Deliberately failing this iteration for testing purposes"); + } +#endif +} - internal::EraseStaleEntries map{meshes}; +void Iteration::readMeshes(std::string const &meshesPath) +{ + Parameter pOpen; + Parameter pList; - /* obtain all non-scalar meshes */ - IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(internal::defaultFlushParams); + pOpen.path = meshesPath; + IOHandler()->enqueue(IOTask(&meshes, pOpen)); - Parameter aList; - for (auto const &mesh_name : *pList.paths) - { - Mesh &m = map[mesh_name]; - pOpen.path = mesh_name; - aList.attributes->clear(); - IOHandler()->enqueue(IOTask(&m, pOpen)); - IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(internal::defaultFlushParams); + meshes.readAttributes(ReadMode::FullyReread); - auto att_begin = aList.attributes->begin(); - auto att_end = aList.attributes->end(); - auto value = std::find(att_begin, att_end, "value"); - auto shape = std::find(att_begin, att_end, "shape"); - if (value != att_end && shape != att_end) - { - MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; - mrc.parent() = m.parent(); - IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); - mrc.get().m_isConstant = true; - } - m.read(); - } + internal::EraseStaleEntries map{meshes}; + + /* obtain all non-scalar meshes */ + IOHandler()->enqueue(IOTask(&meshes, pList)); + IOHandler()->flush(internal::defaultFlushParams); - /* obtain all scalar meshes */ - Parameter dList; - IOHandler()->enqueue(IOTask(&meshes, dList)); + Parameter aList; + for (auto const &mesh_name : *pList.paths) + { + Mesh &m = map[mesh_name]; + pOpen.path = mesh_name; + aList.attributes->clear(); + IOHandler()->enqueue(IOTask(&m, pOpen)); + IOHandler()->enqueue(IOTask(&m, aList)); IOHandler()->flush(internal::defaultFlushParams); - Parameter dOpen; - for (auto const &mesh_name : *dList.datasets) + auto att_begin = aList.attributes->begin(); + auto att_end = aList.attributes->end(); + auto value = std::find(att_begin, att_end, "value"); + auto shape = std::find(att_begin, att_end, "shape"); + if (value != att_end && shape != att_end) { - Mesh &m = map[mesh_name]; - dOpen.name = mesh_name; - IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(internal::defaultFlushParams); MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); - IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->enqueue(IOTask(&mrc, pOpen)); IOHandler()->flush(internal::defaultFlushParams); - mrc.written() = false; - mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); - mrc.written() = true; + mrc.get().m_isConstant = true; + } + m.read(); + try + { m.read(); } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } } - else + + /* obtain all scalar meshes */ + Parameter dList; + IOHandler()->enqueue(IOTask(&meshes, dList)); + IOHandler()->flush(internal::defaultFlushParams); + + Parameter dOpen; + for (auto const &mesh_name : *dList.datasets) { - meshes.dirty() = false; + Mesh &m = map[mesh_name]; + dOpen.name = mesh_name; + IOHandler()->enqueue(IOTask(&m, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; + mrc.parent() = m.parent(); + IOHandler()->enqueue(IOTask(&mrc, dOpen)); + IOHandler()->flush(internal::defaultFlushParams); + mrc.written() = false; + mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); + mrc.written() = true; + try + { + m.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read mesh with name '" << mesh_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(mesh_name); + } } +} - if (hasParticles) - { - pOpen.path = s.particlesPath(); - IOHandler()->enqueue(IOTask(&particles, pOpen)); +void Iteration::readParticles(std::string const &particlesPath) +{ + Parameter pOpen; + Parameter pList; - particles.readAttributes(ReadMode::FullyReread); + pOpen.path = particlesPath; + IOHandler()->enqueue(IOTask(&particles, pOpen)); - /* obtain all particle species */ - pList.paths->clear(); - IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(internal::defaultFlushParams); + particles.readAttributes(ReadMode::FullyReread); + + /* obtain all particle species */ + pList.paths->clear(); + IOHandler()->enqueue(IOTask(&particles, pList)); + IOHandler()->flush(internal::defaultFlushParams); - internal::EraseStaleEntries map{particles}; - for (auto const &species_name : *pList.paths) + internal::EraseStaleEntries map{particles}; + for (auto const &species_name : *pList.paths) + { + ParticleSpecies &p = map[species_name]; + pOpen.path = species_name; + IOHandler()->enqueue(IOTask(&p, pOpen)); + IOHandler()->flush(internal::defaultFlushParams); + try { - ParticleSpecies &p = map[species_name]; - pOpen.path = species_name; - IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(internal::defaultFlushParams); p.read(); } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle species with name '" + << species_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(species_name); + } } - else - { - particles.dirty() = false; - } - - readAttributes(ReadMode::FullyReread); } auto Iteration::beginStep(bool reread) -> BeginStepStatus @@ -660,7 +756,8 @@ auto Iteration::beginStep( IOHandl->m_seriesStatus = internal::SeriesStatus::Parsing; try { - res.iterationsInOpenedStep = series.readGorVBased(false); + res.iterationsInOpenedStep = series.readGorVBased( + /* do_always_throw_errors = */ true, /* init = */ false); } catch (...) { diff --git a/src/Mesh.cpp b/src/Mesh.cpp index 037e2ca1de..5b91dd26dc 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/Mesh.hpp" +#include "openPMD/Error.hpp" #include "openPMD/Series.hpp" #include "openPMD/auxiliary/DerefDynamicCast.hpp" #include "openPMD/auxiliary/StringManip.hpp" @@ -297,8 +298,13 @@ void Mesh::read() setGeometry(tmpGeometry); } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'geometry'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'geometry' (expected a string, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "dataOrder"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -313,12 +319,20 @@ void Mesh::read() if (tmpDataOrder.size() == 1) setDataOrder(static_cast(tmpDataOrder[0])); else - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, "Unexpected Attribute value for 'dataOrder': " + tmpDataOrder); } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'dataOrder'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'dataOrder' (expected char or " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "axisLabels"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -327,8 +341,13 @@ void Mesh::read() setAxisLabels( Attribute(*aRead.resource).get >()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'axisLabels'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'axisLabels' (expected a vector " + "of string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "gridSpacing"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -345,8 +364,13 @@ void Mesh::read() else if (auto val = a.getOptional >(); val.has_value()) setGridSpacing(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'gridSpacing'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'gridSpacing' (expected a " + "vector of double, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -356,8 +380,13 @@ void Mesh::read() val.has_value()) setGridGlobalOffset(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'gridGlobalOffset'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'gridGlobalOffset' (expected a " + "vector of double, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -366,8 +395,13 @@ void Mesh::read() val.has_value()) setGridUnitSI(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'gridUnitSI'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'gridUnitSI' (expected double, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); if (scalar()) { @@ -387,7 +421,17 @@ void Mesh::read() pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); rc.get().m_isConstant = true; - rc.read(); + try + { + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read record component '" << component + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(component); + } } Parameter dList; @@ -404,7 +448,17 @@ void Mesh::read() rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; - rc.read(); + try + { + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read record component '" << component + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + map.forget(component); + } } } diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 76017bbf94..8cd48e212f 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -18,7 +18,11 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ + #include "openPMD/ParticlePatches.hpp" +#include "openPMD/Error.hpp" + +#include namespace openPMD { @@ -43,7 +47,17 @@ void ParticlePatches::read() PatchRecord &pr = (*this)[record_name]; pOpen.path = record_name; IOHandler()->enqueue(IOTask(&pr, pOpen)); - pr.read(); + try + { + pr.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read patch record '" << record_name + << "' due to read error and will skip it:" << err.what() + << std::endl; + this->container().erase(record_name); + } } Parameter dList; @@ -55,9 +69,13 @@ void ParticlePatches::read() { if (!("numParticles" == component_name || "numParticlesOffset" == component_name)) - throw std::runtime_error( - "Unexpected record component" + component_name + - "in particlePatch"); + { + + std::cerr << "Unexpected record component" + component_name + + "in particlePatch. Will ignore it." + << std::endl; + continue; + } PatchRecord &pr = Container::operator[](component_name); PatchRecordComponent &prc = pr[RecordComponent::SCALAR]; @@ -68,8 +86,13 @@ void ParticlePatches::read() IOHandler()->flush(internal::defaultFlushParams); if (determineDatatype() != *dOpen.dtype) - throw std::runtime_error( - "Unexpected datatype for " + component_name); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected datatype for " + component_name + + "(expected uint64, found " + + datatypeToString(*dOpen.dtype) + ")"); /* allow all attributes to be set */ prc.written() = false; @@ -77,7 +100,18 @@ void ParticlePatches::read() prc.written() = true; pr.dirty() = false; - prc.read(); + try + { + prc.read(); + } + catch (error::ReadError const &err) + { + std::cerr + << "Cannot read record component '" << component_name + << "' in particle patch and will skip it due to read error:\n" + << err.what() << std::endl; + Container::container().erase(component_name); + } } } } // namespace openPMD diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 24ebadadec..998c6c6317 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -52,7 +52,17 @@ void ParticleSpecies::read() hasParticlePatches = true; pOpen.path = "particlePatches"; IOHandler()->enqueue(IOTask(&particlePatches, pOpen)); - particlePatches.read(); + try + { + particlePatches.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle patches and will skip them " + "due to read error:\n" + << err.what() << std::endl; + hasParticlePatches = false; + } } else { @@ -76,7 +86,18 @@ void ParticleSpecies::read() IOHandler()->flush(internal::defaultFlushParams); rc.get().m_isConstant = true; } - r.read(); + try + { + r.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read particle record '" << record_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + + map.forget(record_name); + } } } @@ -111,12 +132,11 @@ void ParticleSpecies::read() rc.written() = true; r.read(); } - catch (std::runtime_error const &) + catch (error::ReadError const &err) { - std::cerr << "WARNING: Skipping invalid openPMD record '" - << record_name << "'" << std::endl; - while (!IOHandler()->m_work.empty()) - IOHandler()->m_work.pop(); + std::cerr << "Cannot read particle record '" << record_name + << "' and will skip it due to read error:\n" + << err.what() << std::endl; map.forget(record_name); //(*this)[record_name].erase(RecordComponent::SCALAR); diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index b567aa0ff1..679146e896 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -23,6 +23,8 @@ #include "openPMD/Series.hpp" +#include + namespace openPMD { @@ -167,25 +169,66 @@ std::optional SeriesIterator::nextIterationInStep() {FlushLevel::UserFlush}, /* flushIOHandler = */ true); - series.iterations[m_currentIteration].open(); + try + { + series.iterations[m_currentIteration].open(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read iteration '" << m_currentIteration + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + return nextIterationInStep(); + } + return {this}; } case IterationEncoding::fileBased: - series.iterations[m_currentIteration].open(); - series.iterations[m_currentIteration].beginStep(/* reread = */ true); + try + { + /* + * Errors in here might appear due to deferred iteration parsing. + */ + series.iterations[m_currentIteration].open(); + /* + * Errors in here might appear due to reparsing after opening a + * new step. + */ + series.iterations[m_currentIteration].beginStep( + /* reread = */ true); + } + catch (error::ReadError const &err) + { + std::cerr << "[SeriesIterator] Cannot read iteration due to error " + "below, will skip it.\n" + << err.what() << std::endl; + return nextIterationInStep(); + } + return {this}; } throw std::runtime_error("Unreachable!"); } -std::optional SeriesIterator::nextStep() +std::optional SeriesIterator::nextStep(size_t recursion_depth) { // since we are in group-based iteration layout, it does not // matter which iteration we begin a step upon - AdvanceStatus status; + AdvanceStatus status{}; Iteration::BeginStepStatus::AvailableIterations_t availableIterations; - std::tie(status, availableIterations) = - Iteration::beginStep({}, *m_series, /* reread = */ true); + try + { + std::tie(status, availableIterations) = + Iteration::beginStep({}, *m_series, /* reread = */ true); + } + catch (error::ReadError const &err) + { + std::cerr << "[SeriesIterator] Cannot read iteration due to error " + "below, will skip it.\n" + << err.what() << std::endl; + m_series->advance(AdvanceMode::ENDSTEP); + return nextStep(recursion_depth + 1); + } if (availableIterations.has_value() && status != AdvanceStatus::RANDOMACCESS) @@ -224,7 +267,10 @@ std::optional SeriesIterator::nextStep() } else { - ++it; + for (size_t i = 0; i < recursion_depth && it != itEnd; ++i) + { + ++it; + } if (it == itEnd) { @@ -295,9 +341,22 @@ std::optional SeriesIterator::loopBody() auto iteration = iterations.at(currentIterationIndex.value()); if (iteration.get().m_closed != internal::CloseStatus::ClosedInBackend) { - iteration.open(); - option.value()->setCurrentIteration(); - return option; + try + { + iteration.open(); + option.value()->setCurrentIteration(); + return option; + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read iteration '" + << currentIterationIndex.value() + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + option.value()->deactivateDeadIteration( + currentIterationIndex.value()); + return std::nullopt; + } } else { @@ -325,10 +384,34 @@ std::optional SeriesIterator::loopBody() return {this}; } - auto option = nextStep(); + auto option = nextStep(/*recursion_depth = */ 1); return guardReturn(option); } +void SeriesIterator::deactivateDeadIteration(iteration_index_t index) +{ + switch (m_series->iterationEncoding()) + { + case IterationEncoding::fileBased: { + Parameter param; + m_series->IOHandler()->enqueue( + IOTask(&m_series->iterations[index], std::move(param))); + m_series->IOHandler()->flush({FlushLevel::UserFlush}); + } + break; + case IterationEncoding::variableBased: + case IterationEncoding::groupBased: { + Parameter param; + param.mode = AdvanceMode::ENDSTEP; + m_series->IOHandler()->enqueue( + IOTask(&m_series->iterations[index], std::move(param))); + m_series->IOHandler()->flush({FlushLevel::UserFlush}); + } + break; + } + m_series->iterations.container().erase(index); +} + SeriesIterator &SeriesIterator::operator++() { if (!m_series.has_value()) @@ -340,6 +423,9 @@ SeriesIterator &SeriesIterator::operator++() /* * loopBody() might return an empty option to indicate a skipped iteration. * Loop until it returns something real for us. + * Note that this is not an infinite loop: + * Upon end of the Series, loopBody() does not return an empty option, + * but the end iterator. */ do { diff --git a/src/Record.cpp b/src/Record.cpp index 717d709072..485e817e14 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -109,7 +109,18 @@ void Record::read() if (scalar()) { /* using operator[] will incorrectly update parent */ - this->at(RecordComponent::SCALAR).read(); + auto &scalarComponent = this->at(RecordComponent::SCALAR); + try + { + scalarComponent.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read scalar record component and will skip it " + "due to read error:\n" + << err.what() << std::endl; + this->container().erase(RecordComponent::SCALAR); + } } else { @@ -124,7 +135,17 @@ void Record::read() pOpen.path = component; IOHandler()->enqueue(IOTask(&rc, pOpen)); rc.get().m_isConstant = true; - rc.read(); + try + { + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read record component '" << component + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + this->container().erase(component); + } } Parameter dList; @@ -141,7 +162,17 @@ void Record::read() rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; - rc.read(); + try + { + rc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read record component '" << component + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + this->container().erase(component); + } } } diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 83d5ac688e..b7679ca598 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -297,7 +297,15 @@ namespace rc.makeConstant(attr.get()); } - static constexpr char const *errorMsg = "Unexpected constant datatype"; + template + static void call(Args &&...) + { + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Undefined constant datatype."); + } }; } // namespace @@ -335,7 +343,11 @@ void RecordComponent::readBase() oss << "Unexpected datatype (" << *aRead.dtype << ") for attribute 'shape' (" << determineDatatype() << " aka uint64_t)"; - throw std::runtime_error(oss.str()); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + oss.str()); } written() = false; @@ -350,7 +362,13 @@ void RecordComponent::readBase() val.has_value()) setUnitSI(val.value()); else - throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'unitSI' (expected double, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); readAttributes(ReadMode::FullyReread); } diff --git a/src/Series.cpp b/src/Series.cpp index c25f0f173c..5b17c24642 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -587,7 +587,8 @@ Given file pattern: ')END" if (input->iterationEncoding == IterationEncoding::fileBased) readFileBased(); else - readGorVBased(); + readGorVBased( + /* do_always_throw_errors = */ false, /* init = */ true); if (series.iterations.empty()) { @@ -989,7 +990,10 @@ void Series::readFileBased() fOpen.encoding = iterationEncoding(); if (!auxiliary::directory_exists(IOHandler()->directory)) - throw no_such_file_error( + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + {}, "Supplied directory is not valid: " + IOHandler()->directory); auto isPartOfSeries = matcher( @@ -1017,19 +1021,36 @@ void Series::readFileBased() * parameter modification. Backend access type stays unchanged for the * lifetime of a Series. */ if (IOHandler()->m_backendAccess == Access::READ_ONLY) - throw no_such_file_error("No matching iterations found: " + name()); + throw error::ReadError( + error::AffectedObject::File, + error::Reason::Inaccessible, + {}, + "No matching iterations found: " + name()); else std::cerr << "No matching iterations found: " << name() << std::endl; } - auto readIterationEagerly = [](Iteration &iteration) { - iteration.runDeferredParseAccess(); + /* + * Return true if parsing was successful + */ + auto readIterationEagerly = + [](Iteration &iteration) -> std::optional { + try + { + iteration.runDeferredParseAccess(); + } + catch (error::ReadError const &err) + { + return err; + } Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); iteration.IOHandler()->flush(internal::defaultFlushParams); iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; + return {}; }; + std::vector unparseableIterations; if (series.m_parseLazily) { for (auto &iteration : series.iterations) @@ -1037,18 +1058,102 @@ void Series::readFileBased() iteration.second.get().m_closed = internal::CloseStatus::ParseAccessDeferred; } - // open the last iteration, just to parse Series attributes - auto getLastIteration = series.iterations.end(); - getLastIteration--; - auto &lastIteration = getLastIteration->second; - readIterationEagerly(lastIteration); + // open the first iteration, just to parse Series attributes + bool atLeastOneIterationSuccessful = false; + std::optional forwardFirstError; + for (auto &pair : series.iterations) + { + if (auto error = readIterationEagerly(pair.second); error) + { + std::cerr << "Cannot read iteration '" << pair.first + << "' and will skip it due to read error:\n" + << error->what() << std::endl; + unparseableIterations.push_back(pair.first); + if (!forwardFirstError.has_value()) + { + forwardFirstError = std::move(error); + } + } + else + { + atLeastOneIterationSuccessful = true; + break; + } + } + if (!atLeastOneIterationSuccessful) + { + if (forwardFirstError.has_value()) + { + auto &firstError = forwardFirstError.value(); + firstError.description.append( + "\n[Note] Not a single iteration can be successfully " + "parsed (see above errors). Returning the first observed " + "error, for better recoverability in user code. Need to " + "access at least one iteration even in deferred parsing " + "mode in order to read global Series attributes."); + throw firstError; + } + else + { + throw error::ReadError( + error::AffectedObject::Other, + error::Reason::Other, + {}, + "Not a single iteration can be successfully parsed (see " + "above errors). Need to access at least one iteration even " + "in deferred parsing mode in order to read global Series " + "attributes."); + } + } } else { + bool atLeastOneIterationSuccessful = false; + std::optional forwardFirstError; for (auto &iteration : series.iterations) { - readIterationEagerly(iteration.second); + if (auto error = readIterationEagerly(iteration.second); error) + { + std::cerr << "Cannot read iteration '" << iteration.first + << "' and will skip it due to read error:\n" + << error->what() << std::endl; + unparseableIterations.push_back(iteration.first); + if (!forwardFirstError.has_value()) + { + forwardFirstError = std::move(error); + } + } + else + { + atLeastOneIterationSuccessful = true; + } } + if (!atLeastOneIterationSuccessful) + { + if (forwardFirstError.has_value()) + { + auto &firstError = forwardFirstError.value(); + firstError.description.append( + "\n[Note] Not a single iteration can be successfully " + "parsed (see above errors). Returning the first observed " + "error, for better recoverability in user code."); + throw firstError; + } + else + { + throw error::ReadError( + error::AffectedObject::Other, + error::Reason::Other, + {}, + "Not a single iteration can be successfully parsed (see " + "above warnings)."); + } + } + } + + for (auto index : unparseableIterations) + { + series.iterations.container().erase(index); } if (padding > 0) @@ -1102,20 +1207,28 @@ void Series::readOneIterationFileBased(std::string const &filePath) * Unlike if the file were group-based, this one doesn't work * at all since the group paths are different. */ - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Other, + error::Reason::Other, + {}, "Series constructor called with iteration " "regex '%T' suggests loading a " "time series with fileBased iteration " "encoding. Loaded file is variableBased."); } else - throw std::runtime_error("Unknown iterationEncoding: " + encoding); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unknown iterationEncoding: " + encoding); setAttribute("iterationEncoding", encoding); } else throw std::runtime_error( - "Unexpected Attribute datatype " - "for 'iterationEncoding'"); + "Unexpected Attribute datatype for 'iterationEncoding' (expected " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -1127,8 +1240,13 @@ void Series::readOneIterationFileBased(std::string const &filePath) written() = true; } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'iterationFormat'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'iterationFormat' (expected " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); Parameter pOpen; std::string version = openPMD(); @@ -1142,7 +1260,32 @@ void Series::readOneIterationFileBased(std::string const &filePath) series.iterations.readAttributes(ReadMode::OverrideExisting); } -auto Series::readGorVBased(bool do_init) +namespace +{ + /* + * This function is efficient if subtract is empty and inefficient + * otherwise. Use only where an empty subtract vector is the + * common case. + */ + template + void + vectorDifference(std::vector &baseVector, std::vector const &subtract) + { + for (auto const &elem : subtract) + { + for (auto it = baseVector.begin(); it != baseVector.end(); ++it) + { + if (*it == elem) + { + baseVector.erase(it); + break; + } + } + } + } +} // namespace + +auto Series::readGorVBased(bool do_always_throw_errors, bool do_init) -> std::optional> { auto &series = get(); @@ -1184,13 +1327,21 @@ auto Series::readGorVBased(bool do_init) series.m_overrideFilebasedFilename = series.m_name; } else - throw std::runtime_error( + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, "Unknown iterationEncoding: " + encoding); setAttribute("iterationEncoding", encoding); } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'iterationEncoding'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'iterationEncoding' " + "(expected string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -1202,8 +1353,13 @@ auto Series::readGorVBased(bool do_init) written() = true; } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'iterationFormat'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'iterationFormat' (expected " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } Parameter pOpen; @@ -1211,30 +1367,19 @@ auto Series::readGorVBased(bool do_init) if (version == "1.0.0" || version == "1.0.1" || version == "1.1.0") pOpen.path = auxiliary::replace_first(basePath(), "/%T/", ""); else - throw std::runtime_error("Unknown openPMD version - " + version); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unknown openPMD version - " + version); IOHandler()->enqueue(IOTask(&series.iterations, pOpen)); readAttributes(ReadMode::IgnoreExisting); - auto withRWAccess = [this](auto &&functor) { - auto oldStatus = IOHandler()->m_seriesStatus; - IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing; - try - { - std::forward(functor)(); - } - catch (...) - { - IOHandler()->m_seriesStatus = oldStatus; - throw; - } - IOHandler()->m_seriesStatus = oldStatus; - }; - /* * 'snapshot' changes over steps, so reread that. */ - withRWAccess([&series]() { + internal::withRWAccess(IOHandler()->m_seriesStatus, [&series]() { series.iterations.readAttributes(ReadMode::OverrideExisting); }); @@ -1243,11 +1388,15 @@ auto Series::readGorVBased(bool do_init) IOHandler()->enqueue(IOTask(&series.iterations, pList)); IOHandler()->flush(internal::defaultFlushParams); - auto readSingleIteration = [&series, &pOpen, this, withRWAccess]( - IterationIndex_t index, - std::string path, - bool guardAgainstRereading, - bool beginStep) { + /* + * Return error if one is caught. + */ + auto readSingleIteration = + [&series, &pOpen, this]( + IterationIndex_t index, + std::string path, + bool guardAgainstRereading, + bool beginStep) -> std::optional { if (series.iterations.contains(index)) { // maybe re-read @@ -1256,13 +1405,16 @@ auto Series::readGorVBased(bool do_init) // reparsing is not needed if (guardAgainstRereading && i.written()) { - return; + return {}; } if (i.get().m_closed != internal::CloseStatus::ParseAccessDeferred) { pOpen.path = path; IOHandler()->enqueue(IOTask(&i, pOpen)); - withRWAccess([&i, &path]() { i.reread(path); }); + // @todo catch stuff from here too + internal::withRWAccess( + IOHandler()->m_seriesStatus, + [&i, &path]() { i.reread(path); }); } } else @@ -1272,7 +1424,18 @@ auto Series::readGorVBased(bool do_init) i.deferParseAccess({path, index, false, "", beginStep}); if (!series.m_parseLazily) { - i.runDeferredParseAccess(); + try + { + i.runDeferredParseAccess(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read iteration '" << index + << "' and will skip it due to read error:\n" + << err.what() << std::endl; + series.iterations.container().erase(index); + return {err}; + } i.get().m_closed = internal::CloseStatus::Open; } else @@ -1280,6 +1443,7 @@ auto Series::readGorVBased(bool do_init) i.get().m_closed = internal::CloseStatus::ParseAccessDeferred; } } + return std::nullopt; }; /* @@ -1294,20 +1458,31 @@ auto Series::readGorVBased(bool do_init) * Sic! This happens when a file-based Series is opened in group-based mode. */ case IterationEncoding::fileBased: { + std::vector unreadableIterations; for (auto const &it : *pList.paths) { IterationIndex_t index = std::stoull(it); - /* - * For now: parse a Series in RandomAccess mode. - * (beginStep = false) - * A streaming read mode might come in a future API addition. - */ - withRWAccess( - [&]() { readSingleIteration(index, it, true, false); }); + if (auto err = internal::withRWAccess( + IOHandler()->m_seriesStatus, + [&]() { + return readSingleIteration(index, it, true, false); + }); + err) + { + std::cerr << "Cannot read iteration " << index + << " and will skip it due to read error:\n" + << err.value().what() << std::endl; + if (do_always_throw_errors) + { + throw *err; + } + unreadableIterations.push_back(index); + } } if (currentSteps.has_value()) { - auto const &vec = currentSteps.value(); + auto &vec = currentSteps.value(); + vectorDifference(vec, unreadableIterations); return std::deque{vec.begin(), vec.end()}; } else @@ -1327,7 +1502,21 @@ auto Series::readGorVBased(bool do_init) * Variable-based iteration encoding relies on steps, so parsing * must happen after opening the first step. */ - withRWAccess([&]() { readSingleIteration(it, "", false, true); }); + if (auto err = internal::withRWAccess( + IOHandler()->m_seriesStatus, + [&readSingleIteration, it]() { + return readSingleIteration(it, "", false, true); + }); + err) + { + /* + * Cannot recover from errors in this place. + * If there is an error in the first iteration, the Series + * cannot be read in variable-based encoding. The read API will + * try to skip other iterations that have errors. + */ + throw *err; + } } return res; } @@ -1347,7 +1536,13 @@ void Series::readBase() val.has_value()) setOpenPMD(val.value()); else - throw std::runtime_error("Unexpected Attribute datatype for 'openPMD'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'openPMD' (expected string, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -1356,8 +1551,13 @@ void Series::readBase() val.has_value()) setOpenPMDextension(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'openPMDextension'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'openPMDextension' (expected " + "uint32, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -1366,8 +1566,13 @@ void Series::readBase() val.has_value()) setAttribute("basePath", val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'basePath'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'basePath' (expected string, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); @@ -1392,8 +1597,13 @@ void Series::readBase() it.second.meshes.written() = true; } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'meshesPath'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'meshesPath' (expected " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } if (std::count( @@ -1417,8 +1627,13 @@ void Series::readBase() it.second.particles.written() = true; } else - throw std::runtime_error( - "Unexpected Attribute datatype for 'particlesPath'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'particlesPath' (expected " + "string, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); } } @@ -2100,8 +2315,14 @@ auto Series::currentSnapshot() const default: { std::stringstream s; s << "Unexpected datatype for '/data/snapshot': " << attribute.dtype + << " (expected a vector of integer, found " + + datatypeToString(attribute.dtype) + ")" << std::endl; - throw std::runtime_error(s.str()); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + s.str()); } } } diff --git a/src/auxiliary/JSON.cpp b/src/auxiliary/JSON.cpp index 0b277e0580..14ab0f3cb5 100644 --- a/src/auxiliary/JSON.cpp +++ b/src/auxiliary/JSON.cpp @@ -22,6 +22,7 @@ #include "openPMD/auxiliary/JSON.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/Error.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/StringManip.hpp" diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 0c0b715e28..9010614c75 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -299,9 +299,14 @@ void Attributable::readAttributes(ReadMode mode) // Some backends may report the wrong type when reading if (vector.size() != 7) { - throw std::runtime_error( - "[Attributable] " - "Unexpected datatype for unitDimension."); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "[Attributable] Unexpected datatype for unitDimension " + "(supplied vector has " + + std::to_string(vector.size()) + + " entries, but 7 are expected)."); } std::array arr; std::copy_n(vector.begin(), 7, arr.begin()); @@ -541,7 +546,11 @@ void Attributable::readAttributes(ReadMode mode) internal::SetAttributeMode::WhileReadingAttributes); break; case DT::UNDEFINED: - throw std::runtime_error("Invalid Attribute datatype during read"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Undefined Attribute datatype during read"); } } diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index 49f2a99d64..e7eef6e047 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -47,8 +47,13 @@ void MeshRecordComponent::read() else if (auto val = a.getOptional >(); val.has_value()) setPosition(val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'position'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'position' (expected a vector " + "of any floating point type, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); readBase(); } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 5b9b708311..8f9e64012c 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -21,6 +21,8 @@ #include "openPMD/backend/PatchRecord.hpp" #include "openPMD/auxiliary/Memory.hpp" +#include + namespace openPMD { PatchRecord & @@ -68,8 +70,13 @@ void PatchRecord::read() val.has_value()) this->setAttribute("unitDimension", val.value()); else - throw std::runtime_error( - "Unexpected Attribute datatype for 'unitDimension'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'unitDimension' (expected an " + "array of seven floating point numbers, found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); @@ -86,7 +93,18 @@ void PatchRecord::read() prc.written() = false; prc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); prc.written() = true; - prc.read(); + try + { + prc.read(); + } + catch (error::ReadError const &err) + { + std::cerr << "Cannot read patch record component '" + << component_name + << "' and will skip it due to read error:" << err.what() + << std::endl; + this->container().erase(component_name); + } } dirty() = false; } diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 4cac01424b..3db0545d40 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -130,7 +130,13 @@ void PatchRecordComponent::read() val.has_value()) setUnitSI(val.value()); else - throw std::runtime_error("Unexpected Attribute datatype for 'unitSI'"); + throw error::ReadError( + error::AffectedObject::Attribute, + error::Reason::UnexpectedContent, + {}, + "Unexpected Attribute datatype for 'unitSI' (expected double, " + "found " + + datatypeToString(Attribute(*aRead.resource).dtype) + ")"); readAttributes(ReadMode::FullyReread); // this will set dirty() = false } diff --git a/test/JSONTest.cpp b/test/JSONTest.cpp index 485689acb4..ec193b8246 100644 --- a/test/JSONTest.cpp +++ b/test/JSONTest.cpp @@ -1,4 +1,5 @@ #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/Error.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/openPMD.hpp" diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index b21bcc2e97..1dd599cf17 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -283,10 +283,14 @@ TEST_CASE("git_hdf5_sample_content_test", "[parallel][hdf5]") REQUIRE(raw_ptr[i] == constant_value); } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -619,10 +623,14 @@ TEST_CASE("hzdr_adios_sample_content_test", "[parallel][adios1]") REQUIRE(raw_ptr[j * 3 + k] == actual[rank][j][k]); } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 0ded4eaa17..f4b1b0ebe0 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -2379,9 +2379,16 @@ inline void optional_paths_110_test(const std::string &backend) REQUIRE(s.iterations[400].particles.empty()); } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; + } + else + { + throw; + } } { @@ -2450,10 +2457,14 @@ void git_early_chunk_query( } } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -2490,9 +2501,16 @@ TEST_CASE("empty_alternate_fbpic", "[serial][hdf5]") helper::listSeries(list); } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "issue sample not accessible. (" << e.what() << ")\n"; + } + else + { + throw; + } } } @@ -2679,10 +2697,14 @@ TEST_CASE("git_hdf5_sample_structure_test", "[serial][hdf5]") int32_t i32 = 32; REQUIRE_THROWS(o.setAttribute("setAttributeFail", i32)); } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } #else std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n"; @@ -2936,10 +2958,14 @@ TEST_CASE("git_hdf5_sample_attribute_test", "[serial][hdf5]") REQUIRE(weighting_scalar.getDimensionality() == 1); REQUIRE(weighting_scalar.getExtent() == e); } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -3010,10 +3036,14 @@ TEST_CASE("git_hdf5_sample_content_test", "[serial][hdf5]") REQUIRE(raw_ptr[i] == constant_value); } } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -3034,10 +3064,15 @@ TEST_CASE("git_hdf5_sample_fileBased_read_test", "[serial][hdf5]") REQUIRE(o.get().m_filenamePadding == 8); #endif } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible && + e.affectedObject == error::AffectedObject::File) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } try @@ -3056,15 +3091,19 @@ TEST_CASE("git_hdf5_sample_fileBased_read_test", "[serial][hdf5]") REQUIRE(o.get().m_filenamePadding == 8); #endif } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } - REQUIRE_THROWS_WITH( + REQUIRE_THROWS_AS( Series("../samples/git-sample/data%07T.h5", Access::READ_ONLY), - Catch::Equals("No matching iterations found: data%07T")); + error::ReadError); try { @@ -3101,10 +3140,16 @@ TEST_CASE("git_hdf5_sample_fileBased_read_test", "[serial][hdf5]") auxiliary::remove_file(file); } } + // use no_such_file_error here to check that the backward-compatibility + // alias works catch (no_such_file_error &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -3181,10 +3226,14 @@ TEST_CASE("git_hdf5_sample_read_thetaMode", "[serial][hdf5][thetaMode]") auto data = B_z.loadChunk(offset, extent); o.flush(); } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "git sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "git sample not accessible. (" << e.what() << ")\n"; + return; + } + throw; } } @@ -3654,10 +3703,13 @@ TEST_CASE("hzdr_hdf5_sample_content_test", "[serial][hdf5]") REQUIRE( isSame(e_offset_z.getDatatype(), determineDatatype())); } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; + return; + } } } @@ -3848,10 +3900,13 @@ TEST_CASE("hzdr_adios1_sample_content_test", "[serial][adios1]") for (int c = 0; c < 3; ++c) REQUIRE(raw_ptr[((a * 3) + b) * 3 + c] == actual[a][b][c]); } - catch (no_such_file_error &e) + catch (error::ReadError &e) { - std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; - return; + if (e.reason == error::Reason::Inaccessible) + { + std::cerr << "HZDR sample not accessible. (" << e.what() << ")\n"; + return; + } } } @@ -6052,7 +6107,7 @@ void deferred_parsing(std::string const &extension) Series series(basename + "%06T." + extension, Access::CREATE); std::vector buffer(20); std::iota(buffer.begin(), buffer.end(), 0.f); - auto dataset = series.iterations[1000].meshes["E"]["x"]; + auto dataset = series.iterations[0].meshes["E"]["x"]; dataset.resetDataset({Datatype::FLOAT, {20}}); dataset.storeChunk(buffer, {0}, {20}); series.flush(); @@ -6060,7 +6115,7 @@ void deferred_parsing(std::string const &extension) // create some empty pseudo files // if the reader tries accessing them it's game over { - for (size_t i = 0; i < 1000; i += 100) + for (size_t i = 1; i < 1000; i += 100) { std::string infix = std::to_string(i); std::string padding; @@ -6080,7 +6135,7 @@ void deferred_parsing(std::string const &extension) Access::READ_ONLY, "{\"defer_iteration_parsing\": true}"); auto dataset = - series.iterations[1000].open().meshes["E"]["x"].loadChunk( + series.iterations[0].open().meshes["E"]["x"].loadChunk( {0}, {20}); series.flush(); for (size_t i = 0; i < 20; ++i) @@ -6096,7 +6151,7 @@ void deferred_parsing(std::string const &extension) Access::READ_WRITE, "{\"defer_iteration_parsing\": true}"); auto dataset = - series.iterations[1000].open().meshes["E"]["x"].loadChunk( + series.iterations[0].open().meshes["E"]["x"].loadChunk( {0}, {20}); series.flush(); for (size_t i = 0; i < 20; ++i) @@ -6253,6 +6308,116 @@ TEST_CASE("chaotic_stream", "[serial]") } } +#ifdef openPMD_USE_INVASIVE_TESTS +void unfinished_iteration_test( + std::string const &ext, bool filebased, std::string const &config = "{}") +{ + std::cout << "\n\nTESTING " << ext << "\n\n" << std::endl; + std::string file = std::string("../samples/unfinished_iteration") + + (filebased ? "_%T." : ".") + ext; + { + Series write(file, Access::CREATE, config); + auto it0 = write.writeIterations()[0]; + auto it5 = write.writeIterations()[5]; + /* + * With enabled invasive tests, this attribute will let the Iteration + * fail parsing. + */ + it5.setAttribute("__openPMD_internal_fail", "asking for trouble"); + auto it10 = write.writeIterations()[10]; + auto E_x = it10.meshes["E"]["x"]; + auto e_density = it10.meshes["e_density"][RecordComponent::SCALAR]; + auto electron_x = it10.particles["e"]["position"]["x"]; + auto electron_mass = + it10.particles["e"]["mass"][RecordComponent::SCALAR]; + } + auto tryReading = [&config, file, filebased]( + std::string const &additionalConfig = "{}") { + { + Series read( + file, Access::READ_ONLY, json::merge(config, additionalConfig)); + + std::vector iterations; + std::cout << "Going to list iterations in " << file << ":" + << std::endl; + for (auto iteration : read.readIterations()) + { + std::cout << "Seeing iteration " << iteration.iterationIndex + << std::endl; + iterations.push_back(iteration.iterationIndex); + + Parameter readAttribute; + readAttribute.name = "this_does_definitely_not_exist"; + read.IOHandler()->enqueue(IOTask(&iteration, readAttribute)); + // enqueue a second time to check that the queue is cleared upon + // exception + read.IOHandler()->enqueue(IOTask(&iteration, readAttribute)); + + REQUIRE_THROWS_AS( + read.IOHandler()->flush({FlushLevel::InternalFlush}), + error::ReadError); + REQUIRE(read.IOHandler()->m_work.empty()); + } + REQUIRE( + (iterations == + std::vector{0, 10})); + } + + if (filebased) + { + Series read( + file, Access::READ_ONLY, json::merge(config, additionalConfig)); + if (additionalConfig == "{}") + { + // Eager parsing, defective iteration has already been removed + REQUIRE(!read.iterations.contains(5)); + read.iterations[0].open(); + read.iterations[10].open(); + } + else + { + REQUIRE_THROWS_AS(read.iterations[5].open(), error::ReadError); + read.iterations[0].open(); + read.iterations[10].open(); + } + } + }; + + tryReading(); + tryReading(R"({"defer_iteration_parsing": true})"); +} + +TEST_CASE("unfinished_iteration_test", "[serial]") +{ +#if openPMD_HAVE_ADIOS2 + unfinished_iteration_test("bp", false, R"({"backend": "adios2"})"); + unfinished_iteration_test( + "bp", + false, + R"( +{ + "backend": "adios2", + "iteration_encoding": "variable_based", + "adios2": { + "schema": 20210209 + } +} +)"); + unfinished_iteration_test("bp", true, R"({"backend": "adios2"})"); +#endif +#if openPMD_HAVE_ADIOS1 + unfinished_iteration_test("adios1.bp", false, R"({"backend": "adios1"})"); + unfinished_iteration_test("adios1.bp", true, R"({"backend": "adios1"})"); +#endif +#if openPMD_HAVE_HDF5 + unfinished_iteration_test("h5", false); + unfinished_iteration_test("h5", true); +#endif + unfinished_iteration_test("json", false); + unfinished_iteration_test("json", true); +} +#endif + TEST_CASE("late_setting_of_iterationencoding", "[serial]") { {