From a89a6a47b1e382043788b92b1c24dd7507b0bbe7 Mon Sep 17 00:00:00 2001 From: C0nsultant Date: Sun, 12 Aug 2018 17:10:06 +0200 Subject: [PATCH] Avoid ADIOS1 overwriting files during READ_WRITE operations --- include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp | 2 +- include/openPMD/IO/HDF5/HDF5Auxiliary.hpp | 4 +- include/openPMD/auxiliary/Memory.hpp | 2 +- src/IO/ADIOS/CommonADIOS1IOHandler.cpp | 18 ++++---- src/IO/HDF5/HDF5IOHandler.cpp | 4 +- src/RecordComponent.cpp | 2 + src/Series.cpp | 26 ++++++++++- src/backend/PatchRecordComponent.cpp | 2 + test/CoreTest.cpp | 2 + test/SerialIOTest.cpp | 47 +++++++++++++++----- 10 files changed, 81 insertions(+), 28 deletions(-) diff --git a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp index 2225b79410..02fc5e5995 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp @@ -122,7 +122,7 @@ getBP1DataType(Datatype dtype) case DT::DATATYPE: throw std::runtime_error("Meta-Datatype leaked into IO"); case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (ADIOS datatype)"); default: throw std::runtime_error("Datatype not implemented in ADIOS IO"); } diff --git a/include/openPMD/IO/HDF5/HDF5Auxiliary.hpp b/include/openPMD/IO/HDF5/HDF5Auxiliary.hpp index a305ec4903..ce7553543e 100644 --- a/include/openPMD/IO/HDF5/HDF5Auxiliary.hpp +++ b/include/openPMD/IO/HDF5/HDF5Auxiliary.hpp @@ -92,7 +92,7 @@ getH5DataType(Attribute const& att) case DT::DATATYPE: throw std::runtime_error("Meta-Datatype leaked into IO"); case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (HDF5 datatype)"); default: throw std::runtime_error("Datatype not implemented in HDF5 IO"); } @@ -210,7 +210,7 @@ getH5DataSpace(Attribute const& att) return array_t_id; } case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (HDF5 dataspace)"); default: throw std::runtime_error("Datatype not implemented in HDF5 IO"); } diff --git a/include/openPMD/auxiliary/Memory.hpp b/include/openPMD/auxiliary/Memory.hpp index 08f8c69b90..49a02078df 100644 --- a/include/openPMD/auxiliary/Memory.hpp +++ b/include/openPMD/auxiliary/Memory.hpp @@ -124,7 +124,7 @@ allocatePtr(Datatype dtype, uint64_t numPoints) break; case DT::UNDEFINED: default: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (Pointer allocation)"); } return std::unique_ptr< void, std::function< void(void*) > >(data, del); diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 2686f49dcb..d1343f0420 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -18,6 +18,7 @@ * and the GNU Lesser General Public License along with openPMD-api. * If not, see . */ +#include void CommonADIOS1IOHandlerImpl::close(int64_t fd) @@ -83,7 +84,7 @@ CommonADIOS1IOHandlerImpl::flush_attribute(int64_t group, std::string const& nam break; case DT::UNDEFINED: case DT::DATATYPE: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Attribute flush)"); default: nelems = 1; } @@ -281,7 +282,7 @@ CommonADIOS1IOHandlerImpl::flush_attribute(int64_t group, std::string const& nam } case DT::UNDEFINED: case DT::DATATYPE: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Attribute flush)"); default: throw std::runtime_error("Datatype not implemented in ADIOS IO"); } @@ -462,10 +463,12 @@ CommonADIOS1IOHandlerImpl::openFile(Writable* writable, name += ".bp"; std::shared_ptr< std::string > filePath; - if( m_filePaths.find(writable) == m_filePaths.end() ) + auto it = std::find_if(m_filePaths.begin(), m_filePaths.end(), + [name](std::unordered_map< Writable*, std::shared_ptr< std::string > >::value_type const& entry){ return *entry.second == name; }); + if( it == m_filePaths.end() ) filePath = std::make_shared< std::string >(name); else - filePath = m_filePaths[writable]; + filePath = it->second; /* close the handle that corresponds to the file we want to open */ if( m_openWriteFileHandles.find(filePath) != m_openWriteFileHandles.end() ) @@ -484,6 +487,7 @@ CommonADIOS1IOHandlerImpl::openFile(Writable* writable, m_openReadFileHandles[filePath] = f; m_filePaths[writable] = filePath; + m_existsOnDisk[filePath] = true; } void @@ -500,9 +504,7 @@ CommonADIOS1IOHandlerImpl::openPath(Writable* writable, writable->written = true; writable->abstractFilePosition = std::make_shared< ADIOS1FilePosition >(path); - auto res = m_filePaths.find(writable); - if( res == m_filePaths.end() ) - res = m_filePaths.find(writable->parent); + auto res = writable->parent ? m_filePaths.find(writable->parent) : m_filePaths.find(writable); m_filePaths[writable] = res->second; } @@ -732,7 +734,7 @@ CommonADIOS1IOHandlerImpl::readDataset(Writable* writable, case DT::BOOL: break; case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Dataset read)"); case DT::DATATYPE: throw std::runtime_error("Meta-Datatype leaked into IO"); default: diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index c19e2d7fd4..b8b6f76cc7 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -934,7 +934,7 @@ HDF5IOHandlerImpl::writeAttribute(Writable* writable, } case DT::UNDEFINED: case DT::DATATYPE: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (HDF5 Attribute write)"); default: throw std::runtime_error("Datatype not implemented in HDF5 IO"); } @@ -1006,7 +1006,7 @@ HDF5IOHandlerImpl::readDataset(Writable* writable, case DT::BOOL: break; case DT::UNDEFINED: - throw std::runtime_error("Unknown Attribute datatype"); + throw std::runtime_error("Unknown Attribute datatype (HDF5 Dataset read)"); case DT::DATATYPE: throw std::runtime_error("Meta-Datatype leaked into IO"); default: diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ca91212d70..d8eddedb02 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -113,6 +113,8 @@ RecordComponent::flush(std::string const& name) std::shared_ptr< void > data{std::move(uptr)}; dWrite.data = data; dWrite.dtype = dCreate.dtype; + if( dWrite.dtype == Datatype::UNDEFINED ) + throw std::runtime_error("Dataset has not been defined for Record Component " + name); dWrite.extent = Extent(getDimensionality(), 1); dWrite.offset = Offset(getDimensionality(), 0); m_chunks->push(IOTask(this, dWrite)); diff --git a/src/Series.cpp b/src/Series.cpp index 9d37ddf155..b30fe8e4e9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -53,6 +53,13 @@ namespace openPMD */ Format determineFormat(std::string const& filename); +/** Determine the default filename suffix for a given storage format. + * + * @param f File format to determine suffix for. + * @return String containing the default filename suffix + */ +std::string suffix(Format f); + /** Remove the filename extension of a given storage format. * * @param filename String containing the filename, possibly with filename extension. @@ -633,6 +640,7 @@ Series::readFileBased(AccessType actualAccessType) fOpen.name = entry; IOHandler->enqueue(IOTask(this, fOpen)); IOHandler->flush(); + iterations.m_writable->parent = getWritable(this); iterations.parent = getWritable(this); /* allow all attributes to be set */ @@ -856,16 +864,30 @@ determineFormat(std::string const& filename) return Format::DUMMY; } +std::string +suffix(Format f) +{ + switch( f ) + { + case Format::HDF5: + return ".h5"; + case Format::ADIOS1: + case Format::ADIOS2: + return ".bp"; + default: + return ""; + } +} + std::string cleanFilename(std::string const& filename, Format f) { switch( f ) { case Format::HDF5: - return auxiliary::replace_last(filename, ".h5", ""); case Format::ADIOS1: case Format::ADIOS2: - return auxiliary::replace_last(filename, ".bp", ""); + return auxiliary::replace_last(filename, suffix(f), ""); default: return filename; } diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 3a61191623..91c6448941 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -91,6 +91,8 @@ PatchRecordComponent::flush(std::string const& name) std::shared_ptr< void > data{std::move(uptr)}; dWrite.data = data; dWrite.dtype = dCreate.dtype; + if( dWrite.dtype == Datatype::UNDEFINED ) + throw std::runtime_error("Dataset has not been defined for ParticlePatch RecordComponent " + name); dWrite.extent = Extent(getDimensionality(), 1); dWrite.offset = Offset(getDimensionality(), 0); m_chunks->push(IOTask(this, dWrite)); diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index 6d723c21d7..f159b6e5a6 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -203,6 +203,7 @@ TEST_CASE( "particleSpecies_modification_test", "[core]" ) REQUIRE(zeros == offset.unitDimension()); auto& off_x = offset["x"]; + off_x.resetDataset(dset); REQUIRE(1 == off_x.unitSI()); } @@ -468,6 +469,7 @@ TEST_CASE( "structure_test", "[core]" ) REQUIRE(o.iterations[1].particles["P"].particlePatches["extent"]["x"].IOHandler); REQUIRE(prc.parent == getWritable(&o.iterations[1].particles["P"].particlePatches["extent"])); REQUIRE(o.iterations[1].particles["P"].particlePatches["extent"]["x"].parent == getWritable(&o.iterations[1].particles["P"].particlePatches["extent"])); + prc.resetDataset(dset); #else std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n"; #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 22c98c94c7..6c694854d0 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -1161,14 +1161,13 @@ TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" ) o.setOpenPMDextension(1); o.iterations[3].setTime(static_cast< double >(3)); - o.flush(); } REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001.h5")); REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000002.h5")); REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000003.h5")); { - Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE); + Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY); REQUIRE(o.iterations.size() == 3); REQUIRE(o.iterations.count(1) == 1); @@ -1189,28 +1188,42 @@ TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" ) REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); std::array< double, 7 > udim{{1, 0, 0, 0, 0, 0, 0}}; Extent ext{4}; - for( auto const& entry : o.iterations ) + for( auto& entry : o.iterations ) { - auto const& it = entry.second; + auto& it = entry.second; REQUIRE(it.dt< double >() == 1.); REQUIRE(it.time< double >() == static_cast< double >(entry.first)); REQUIRE(it.timeUnitSI() == 1.); - auto const& pos = it.particles.at("e").at("position"); + auto& pos = it.particles.at("e").at("position"); REQUIRE(pos.timeOffset< float >() == 0.f); REQUIRE(pos.unitDimension() == udim); - auto const& pos_x = pos.at("x"); + auto& pos_x = pos.at("x"); REQUIRE(pos_x.unitSI() == 1.); REQUIRE(pos_x.getExtent() == ext); REQUIRE(pos_x.getDatatype() == Datatype::DOUBLE); - auto const& posOff = it.particles.at("e").at("positionOffset"); + auto& posOff = it.particles.at("e").at("positionOffset"); REQUIRE(posOff.timeOffset< float >() == 0.f); REQUIRE(posOff.unitDimension() == udim); - auto const& posOff_x = posOff.at("x"); + auto& posOff_x = posOff.at("x"); REQUIRE(posOff_x.unitSI() == 1.); REQUIRE(posOff_x.getExtent() == ext); REQUIRE(posOff_x.getDatatype() == Datatype::UINT64); + + auto position = pos_x.loadChunk< double >({0}, {4}); + auto position_raw = position.get(); + auto positionOffset = posOff_x.loadChunk< uint64_t >({0}, {4}); + auto positionOffset_raw = positionOffset.get(); + o.flush(); + for( uint64_t j = 0; j < 4; ++j ) + { + REQUIRE(position_raw[j] == static_cast< double >(j + (entry.first-1)*4)); + REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4); + } } + } + { + Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE); o.iterations[4]; } REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004.h5")); @@ -1698,12 +1711,13 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" ) o.setOpenPMDextension(1); o.iterations[3].setTime(3.f); } + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write1.bp")); + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write2.bp")); + REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write3.bp")); { Series o = Series("../samples/serial_fileBased_write%T.bp", AccessType::READ_ONLY); - //REQUIRE(o.openPMDextension() == 1); - REQUIRE(o.iterations.size() == 3); REQUIRE(o.iterations.count(1) == 1); REQUIRE(o.iterations.count(2) == 1); @@ -1713,6 +1727,15 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" ) REQUIRE(o.iterations.at(2).time< float >() == 2.f); REQUIRE(o.iterations.at(3).time< float >() == 3.f); + REQUIRE(o.basePath() == "/data/%T/"); + REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased); + REQUIRE(o.iterationFormat() == "serial_fileBased_write%T"); + REQUIRE(o.openPMD() == "1.1.0"); + REQUIRE(o.openPMDextension() == 1u); + REQUIRE(o.particlesPath() == "particles/"); + REQUIRE_FALSE(o.containsAttribute("meshesPath")); + REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error); + for( uint64_t i = 1; i <= 3; ++i ) { Iteration iteration = o.iterations.at(i); @@ -1744,8 +1767,8 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" ) o.flush(); for( uint64_t j = 0; j < 4; ++j ) { - REQUIRE(position_raw[j] == static_cast< double >(j)); - REQUIRE(positionOffset_raw[j] == j); + REQUIRE(position_raw[j] == static_cast< double >(j + (i-1)*4)); + REQUIRE(positionOffset_raw[j] == j + (i-1)*4); } } }