Avoid ADIOS1 overwriting files during READ_WRITE operations

include/openPMD/IO/ADIOS/ADIOS1Auxiliary.hpp

@@ -122,7 +122,7 @@ getBP1DataType(Datatype dtype)
         case DT::DATATYPE:
             throw std::runtime_error("Meta-Datatype leaked into IO");
         case DT::UNDEFINED:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (ADIOS datatype)");
         default:
             throw std::runtime_error("Datatype not implemented in ADIOS IO");
     }

include/openPMD/IO/HDF5/HDF5Auxiliary.hpp

@@ -92,7 +92,7 @@ getH5DataType(Attribute const& att)
         case DT::DATATYPE:
             throw std::runtime_error("Meta-Datatype leaked into IO");
         case DT::UNDEFINED:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (HDF5 datatype)");
         default:
             throw std::runtime_error("Datatype not implemented in HDF5 IO");
     }
@@ -210,7 +210,7 @@ getH5DataSpace(Attribute const& att)
             return array_t_id;
         }
         case DT::UNDEFINED:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (HDF5 dataspace)");
         default:
             throw std::runtime_error("Datatype not implemented in HDF5 IO");
     }

include/openPMD/auxiliary/Memory.hpp

@@ -124,7 +124,7 @@ allocatePtr(Datatype dtype, uint64_t numPoints)
             break;
         case DT::UNDEFINED:
         default:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (Pointer allocation)");
     }
 
     return std::unique_ptr< void, std::function< void(void*) > >(data, del);

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -18,6 +18,7 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
+#include <algorithm>
 
 void
 CommonADIOS1IOHandlerImpl::close(int64_t fd)
@@ -83,7 +84,7 @@ CommonADIOS1IOHandlerImpl::flush_attribute(int64_t group, std::string const& nam
         break;
         case DT::UNDEFINED:
         case DT::DATATYPE:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Attribute flush)");
         default:
             nelems = 1;
     }
@@ -281,7 +282,7 @@ CommonADIOS1IOHandlerImpl::flush_attribute(int64_t group, std::string const& nam
         }
         case DT::UNDEFINED:
         case DT::DATATYPE:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Attribute flush)");
         default:
             throw std::runtime_error("Datatype not implemented in ADIOS IO");
     }
@@ -462,10 +463,12 @@ CommonADIOS1IOHandlerImpl::openFile(Writable* writable,
         name += ".bp";
 
     std::shared_ptr< std::string > filePath;
-    if( m_filePaths.find(writable) == m_filePaths.end() )
+    auto it = std::find_if(m_filePaths.begin(), m_filePaths.end(),
+                           [name](std::unordered_map< Writable*, std::shared_ptr< std::string > >::value_type const& entry){ return *entry.second == name; });
+    if( it == m_filePaths.end() )
         filePath = std::make_shared< std::string >(name);
     else
-        filePath = m_filePaths[writable];
+        filePath = it->second;
 
     /* close the handle that corresponds to the file we want to open */
     if( m_openWriteFileHandles.find(filePath) != m_openWriteFileHandles.end() )
@@ -484,6 +487,7 @@ CommonADIOS1IOHandlerImpl::openFile(Writable* writable,
 
     m_openReadFileHandles[filePath] = f;
     m_filePaths[writable] = filePath;
+    m_existsOnDisk[filePath] = true;
 }
 
 void
@@ -500,9 +504,7 @@ CommonADIOS1IOHandlerImpl::openPath(Writable* writable,
     writable->written = true;
     writable->abstractFilePosition = std::make_shared< ADIOS1FilePosition >(path);
 
-    auto res = m_filePaths.find(writable);
-    if( res == m_filePaths.end() )
-        res = m_filePaths.find(writable->parent);
+    auto res = writable->parent ? m_filePaths.find(writable->parent) : m_filePaths.find(writable);
 
     m_filePaths[writable] = res->second;
 }
@@ -732,7 +734,7 @@ CommonADIOS1IOHandlerImpl::readDataset(Writable* writable,
         case DT::BOOL:
             break;
         case DT::UNDEFINED:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (ADIOS1 Dataset read)");
         case DT::DATATYPE:
             throw std::runtime_error("Meta-Datatype leaked into IO");
         default:

src/IO/HDF5/HDF5IOHandler.cpp

@@ -934,7 +934,7 @@ HDF5IOHandlerImpl::writeAttribute(Writable* writable,
         }
         case DT::UNDEFINED:
         case DT::DATATYPE:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (HDF5 Attribute write)");
         default:
             throw std::runtime_error("Datatype not implemented in HDF5 IO");
     }
@@ -1006,7 +1006,7 @@ HDF5IOHandlerImpl::readDataset(Writable* writable,
         case DT::BOOL:
             break;
         case DT::UNDEFINED:
-            throw std::runtime_error("Unknown Attribute datatype");
+            throw std::runtime_error("Unknown Attribute datatype (HDF5 Dataset read)");
         case DT::DATATYPE:
             throw std::runtime_error("Meta-Datatype leaked into IO");
         default:

src/RecordComponent.cpp

@@ -113,6 +113,8 @@ RecordComponent::flush(std::string const& name)
                     std::shared_ptr< void > data{std::move(uptr)};
                     dWrite.data = data;
                     dWrite.dtype = dCreate.dtype;
+                    if( dWrite.dtype == Datatype::UNDEFINED )
+                        throw std::runtime_error("Dataset has not been defined for Record Component " + name);
                     dWrite.extent = Extent(getDimensionality(), 1);
                     dWrite.offset = Offset(getDimensionality(), 0);
                     m_chunks->push(IOTask(this, dWrite));

src/Series.cpp

@@ -53,6 +53,13 @@ namespace openPMD
  */
 Format determineFormat(std::string const& filename);
 
+/** Determine the default filename suffix for a given storage format.
+ *
+ * @param   f   File format to determine suffix for.
+ * @return  String containing the default filename suffix
+ */
+std::string suffix(Format f);
+
 /** Remove the filename extension of a given storage format.
  *
  * @param   filename    String containing the filename, possibly with filename extension.
@@ -633,6 +640,7 @@ Series::readFileBased(AccessType actualAccessType)
             fOpen.name = entry;
             IOHandler->enqueue(IOTask(this, fOpen));
             IOHandler->flush();
+            iterations.m_writable->parent = getWritable(this);
             iterations.parent = getWritable(this);
 
             /* allow all attributes to be set */
@@ -856,16 +864,30 @@ determineFormat(std::string const& filename)
     return Format::DUMMY;
 }
 
+std::string
+suffix(Format f)
+{
+    switch( f )
+    {
+        case Format::HDF5:
+            return ".h5";
+        case Format::ADIOS1:
+        case Format::ADIOS2:
+            return ".bp";
+        default:
+            return "";
+    }
+}
+
 std::string
 cleanFilename(std::string const& filename, Format f)
 {
     switch( f )
     {
         case Format::HDF5:
-            return auxiliary::replace_last(filename, ".h5", "");
         case Format::ADIOS1:
         case Format::ADIOS2:
-            return auxiliary::replace_last(filename, ".bp", "");
+            return auxiliary::replace_last(filename, suffix(f), "");
         default:
             return filename;
     }

src/backend/PatchRecordComponent.cpp

@@ -91,6 +91,8 @@ PatchRecordComponent::flush(std::string const& name)
                 std::shared_ptr< void > data{std::move(uptr)};
                 dWrite.data = data;
                 dWrite.dtype = dCreate.dtype;
+                if( dWrite.dtype == Datatype::UNDEFINED )
+                    throw std::runtime_error("Dataset has not been defined for ParticlePatch RecordComponent " + name);
                 dWrite.extent = Extent(getDimensionality(), 1);
                 dWrite.offset = Offset(getDimensionality(), 0);
                 m_chunks->push(IOTask(this, dWrite));

test/CoreTest.cpp

@@ -203,6 +203,7 @@ TEST_CASE( "particleSpecies_modification_test", "[core]" )
     REQUIRE(zeros == offset.unitDimension());
 
     auto& off_x = offset["x"];
+    off_x.resetDataset(dset);
     REQUIRE(1 == off_x.unitSI());
 }
 
@@ -468,6 +469,7 @@ TEST_CASE( "structure_test", "[core]" )
     REQUIRE(o.iterations[1].particles["P"].particlePatches["extent"]["x"].IOHandler);
     REQUIRE(prc.parent == getWritable(&o.iterations[1].particles["P"].particlePatches["extent"]));
     REQUIRE(o.iterations[1].particles["P"].particlePatches["extent"]["x"].parent == getWritable(&o.iterations[1].particles["P"].particlePatches["extent"]));
+    prc.resetDataset(dset);
 #else
     std::cerr << "Invasive tests not enabled. Hierarchy is not visible.\n";
 #endif

test/SerialIOTest.cpp

@@ -1161,14 +1161,13 @@ TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" )
 
         o.setOpenPMDextension(1);
         o.iterations[3].setTime(static_cast< double >(3));
-        o.flush();
     }
     REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000001.h5"));
     REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000002.h5"));
     REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000003.h5"));
 
     {
-        Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE);
+        Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_ONLY);
 
         REQUIRE(o.iterations.size() == 3);
         REQUIRE(o.iterations.count(1) == 1);
@@ -1189,28 +1188,42 @@ TEST_CASE( "hdf5_fileBased_write_test", "[serial][hdf5]" )
         REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error);
         std::array< double, 7 > udim{{1, 0, 0, 0, 0, 0, 0}};
         Extent ext{4};
-        for( auto const& entry : o.iterations )
+        for( auto& entry : o.iterations )
         {
-            auto const& it = entry.second;
+            auto& it = entry.second;
             REQUIRE(it.dt< double >() == 1.);
             REQUIRE(it.time< double >() == static_cast< double >(entry.first));
             REQUIRE(it.timeUnitSI() == 1.);
-            auto const& pos = it.particles.at("e").at("position");
+            auto& pos = it.particles.at("e").at("position");
             REQUIRE(pos.timeOffset< float >() == 0.f);
             REQUIRE(pos.unitDimension() == udim);
-            auto const& pos_x = pos.at("x");
+            auto& pos_x = pos.at("x");
             REQUIRE(pos_x.unitSI() == 1.);
             REQUIRE(pos_x.getExtent() == ext);
             REQUIRE(pos_x.getDatatype() == Datatype::DOUBLE);
-            auto const& posOff = it.particles.at("e").at("positionOffset");
+            auto& posOff = it.particles.at("e").at("positionOffset");
             REQUIRE(posOff.timeOffset< float >() == 0.f);
             REQUIRE(posOff.unitDimension() == udim);
-            auto const& posOff_x = posOff.at("x");
+            auto& posOff_x = posOff.at("x");
             REQUIRE(posOff_x.unitSI() == 1.);
             REQUIRE(posOff_x.getExtent() == ext);
             REQUIRE(posOff_x.getDatatype() == Datatype::UINT64);
+
+            auto position = pos_x.loadChunk< double >({0}, {4});
+            auto position_raw = position.get();
+            auto positionOffset = posOff_x.loadChunk< uint64_t >({0}, {4});
+            auto positionOffset_raw = positionOffset.get();
+            o.flush();
+            for( uint64_t j = 0; j < 4; ++j )
+            {
+              REQUIRE(position_raw[j] == static_cast< double >(j + (entry.first-1)*4));
+              REQUIRE(positionOffset_raw[j] == j + (entry.first-1)*4);
+            }
         }
+    }
 
+    {
+        Series o = Series("../samples/subdir/serial_fileBased_write%T.h5", AccessType::READ_WRITE);
         o.iterations[4];
     }
     REQUIRE(auxiliary::file_exists("../samples/subdir/serial_fileBased_write00000004.h5"));
@@ -1698,12 +1711,13 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" )
         o.setOpenPMDextension(1);
         o.iterations[3].setTime(3.f);
     }
+    REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write1.bp"));
+    REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write2.bp"));
+    REQUIRE(auxiliary::file_exists("../samples/serial_fileBased_write3.bp"));
 
     {
         Series o = Series("../samples/serial_fileBased_write%T.bp", AccessType::READ_ONLY);
 
-        //REQUIRE(o.openPMDextension() == 1);
-
         REQUIRE(o.iterations.size() == 3);
         REQUIRE(o.iterations.count(1) == 1);
         REQUIRE(o.iterations.count(2) == 1);
@@ -1713,6 +1727,15 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" )
         REQUIRE(o.iterations.at(2).time< float >() == 2.f);
         REQUIRE(o.iterations.at(3).time< float >() == 3.f);
 
+        REQUIRE(o.basePath() == "/data/%T/");
+        REQUIRE(o.iterationEncoding() == IterationEncoding::fileBased);
+        REQUIRE(o.iterationFormat() == "serial_fileBased_write%T");
+        REQUIRE(o.openPMD() == "1.1.0");
+        REQUIRE(o.openPMDextension() == 1u);
+        REQUIRE(o.particlesPath() == "particles/");
+        REQUIRE_FALSE(o.containsAttribute("meshesPath"));
+        REQUIRE_THROWS_AS(o.meshesPath(), no_such_attribute_error);
+
         for( uint64_t i = 1; i <= 3; ++i )
         {
             Iteration iteration = o.iterations.at(i);
@@ -1744,8 +1767,8 @@ TEST_CASE( "adios1_fileBased_write_test", "[serial][adios1]" )
             o.flush();
             for( uint64_t j = 0; j < 4; ++j )
             {
-                REQUIRE(position_raw[j] == static_cast< double >(j));
-                REQUIRE(positionOffset_raw[j] == j);
+                REQUIRE(position_raw[j] == static_cast< double >(j + (i-1)*4));
+                REQUIRE(positionOffset_raw[j] == j + (i-1)*4);
             }
         }
     }