Iterate through a broken Series

openPMD · Mar 29, 2022 · 0795431 · 0795431
1 parent b41a44b
commit 0795431
Show file tree

Hide file tree

Showing 4 changed files with 62 additions and 27 deletions.
diff --git a/src/Iteration.cpp b/src/Iteration.cpp
@@ -595,7 +595,15 @@ AdvanceStatus Iteration::beginStep(bool reread)
         auto newType =
             const_cast<Access *>(&this->IOHandler()->m_frontendAccess);
         *newType = Access::READ_WRITE;
-        series.readGorVBased(false);
+        try
+        {
+            series.readGorVBased(false);
+        }
+        catch (...)
+        {
+            *newType = oldType;
+            throw;
+        }
         *newType = oldType;
         series.iterations.written() = previous;
     }

diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp
@@ -117,6 +117,12 @@ SeriesIterator &SeriesIterator::operator++()
             std::cerr << "[SeriesIterator] Cannot read iteration due to error "
                          "below, will skip it.\n"
                       << err.what() << std::endl;
+            // Need to close the current step manually because there is no
+            // iteration to close
+            Parameter<Operation::ADVANCE> param;
+            param.mode = AdvanceMode::ENDSTEP;
+            series.IOHandler()->enqueue(IOTask(&series, std::move(param)));
+            series.IOHandler()->flush({FlushLevel::UserFlush});
             return operator++();
         }
         if (status == AdvanceStatus::OVER)

diff --git a/src/Series.cpp b/src/Series.cpp
@@ -510,21 +510,30 @@ Given file pattern: ')END"
         auto newType = const_cast<Access *>(&IOHandler()->m_frontendAccess);
         *newType = Access::READ_WRITE;
 
-        if (input->iterationEncoding == IterationEncoding::fileBased)
-            readFileBased();
-        else
-            readGorVBased();
-
-        if (series.iterations.empty())
+        try
         {
-            /* Access::READ_WRITE can be used to create a new Series
-             * allow setting attributes in that case */
-            written() = false;
 
-            initDefaults(input->iterationEncoding);
-            setIterationEncoding(input->iterationEncoding);
+            if (input->iterationEncoding == IterationEncoding::fileBased)
+                readFileBased();
+            else
+                readGorVBased();
 
-            written() = true;
+            if (series.iterations.empty())
+            {
+                /* Access::READ_WRITE can be used to create a new Series
+                 * allow setting attributes in that case */
+                written() = false;
+
+                initDefaults(input->iterationEncoding);
+                setIterationEncoding(input->iterationEncoding);
+
+                written() = true;
+            }
+        }
+        catch (...)
+        {
+            *newType = oldType;
+            throw;
         }
 
         *newType = oldType;
@@ -1100,13 +1109,14 @@ void Series::readGorVBased(bool do_init)
     IOHandler()->flush(internal::defaultFlushParams);
 
     /*
-     * Return true when iteration is parsed successfully.
+     * Return error if one is caught.
      */
-    auto readSingleIteration = [&series, &pOpen, this](
-                                   uint64_t index,
-                                   std::string path,
-                                   bool guardAgainstRereading,
-                                   bool beginStep) -> bool {
+    auto readSingleIteration =
+        [&series, &pOpen, this](
+            uint64_t index,
+            std::string path,
+            bool guardAgainstRereading,
+            bool beginStep) -> std::optional<error::ReadError> {
         if (series.iterations.contains(index))
         {
             // maybe re-read
@@ -1115,7 +1125,7 @@ void Series::readGorVBased(bool do_init)
             // reparsing is not needed
             if (guardAgainstRereading && i.written())
             {
-                return true;
+                return {};
             }
             if (i.get().m_closed != internal::CloseStatus::ParseAccessDeferred)
             {
@@ -1142,7 +1152,7 @@ void Series::readGorVBased(bool do_init)
                               << "' and will skip it due to read error:\n"
                               << err.what() << std::endl;
                     series.iterations.container().erase(index);
-                    return false;
+                    return {err};
                 }
                 i.get().m_closed = internal::CloseStatus::Open;
             }
@@ -1151,7 +1161,7 @@ void Series::readGorVBased(bool do_init)
                 i.get().m_closed = internal::CloseStatus::ParseAccessDeferred;
             }
         }
-        return true;
+        return {};
     };
 
     switch (iterationEncoding())
@@ -1173,11 +1183,15 @@ void Series::readGorVBased(bool do_init)
         {
             index = series.iterations.getAttribute("snapshot").get<uint64_t>();
         }
-        if (!readSingleIteration(index, "", false, true))
+        if (auto err = readSingleIteration(index, "", false, true); err)
         {
-            throw error::ParseError(
-                "In variable-based iteration encoding, read errors in the "
-                "first iteration are not recoverable (see warnings above).");
+            /*
+             * Cannot recover from errors in this place.
+             * If there is an error in the first iteration, the Series cannot
+             * be read in variable-based encoding.
+             * The read API will try to skip other iterations that have errors.
+             */
+            throw *err;
         }
         break;
     }

diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp
@@ -5493,11 +5493,12 @@ void unfinished_iteration_test(
     {
         Series write(file, Access::CREATE, config);
         auto it0 = write.writeIterations()[0];
+        auto it5 = write.writeIterations()[5];
         /*
          * With enabled invasive tests, this attribute will let the Iteration
          * fail parsing.
          */
-        it0.setAttribute("__openPMD_internal_fail", "asking for trouble");
+        it5.setAttribute("__openPMD_internal_fail", "asking for trouble");
         auto it10 = write.writeIterations()[10];
         auto E_x = it10.meshes["E"]["x"];
         auto e_density = it10.meshes["e_density"][RecordComponent::SCALAR];
@@ -5507,6 +5508,12 @@ void unfinished_iteration_test(
     }
     {
         Series read(file, Access::READ_ONLY);
+        std::cout << "Going to list iterations in " << file << ":" << std::endl;
+        for (auto const &iteration : read.readIterations())
+        {
+            std::cout << "Seeing iteration " << iteration.iterationIndex
+                      << std::endl;
+        }
     }
 }