From 3236c0e935be7aa7fb3a9f46b38a72cf53cddc0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 30 Sep 2021 12:05:05 +0200 Subject: [PATCH] Apply redesign to Iteration class --- include/openPMD/Iteration.hpp | 155 +++++++++++++++++++--------------- src/Iteration.cpp | 39 +++++---- src/ReadIterations.cpp | 6 +- src/Series.cpp | 75 ++++++++-------- src/backend/Attributable.cpp | 2 +- 5 files changed, 150 insertions(+), 127 deletions(-) diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 6c086ea7f6..b642aaece3 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -32,11 +32,78 @@ namespace openPMD { +namespace internal +{ + /** + * @brief Whether an iteration has been closed yet. + * + */ + enum class CloseStatus + { + ParseAccessDeferred, //!< The reader has not yet parsed this iteration + Open, //!< Iteration has not been closed + ClosedInFrontend, /*!< Iteration has been closed, but task has not yet + been propagated to the backend */ + ClosedInBackend, /*!< Iteration has been closed and task has been + propagated to the backend */ + ClosedTemporarily /*!< Iteration has been closed internally and may + be reopened later */ + }; + + struct DeferredParseAccess + { + /** + * The group path within /data containing this iteration. + * Example: "1" for iteration 1, "" in variable-based iteration + * encoding. + */ + std::string path; + /** + * The iteration index as accessed by the user in series.iterations[i] + */ + uint64_t iteration = 0; + /** + * If this iteration is part of a Series with file-based layout. + * (Group- and variable-based parsing shares the same code logic.) + */ + bool fileBased = false; + /** + * If fileBased == true, the file name (without file path) of the file + * containing this iteration. + */ + std::string filename; + }; + + class IterationData : public AttributableData + { + public: + /* + * An iteration may be logically closed in the frontend, + * but not necessarily yet in the backend. + * Will be propagated to the backend upon next flush. + * Store the current status. + * Once an iteration has been closed, no further flushes shall be performed. + * If flushing a closed file, the old file may otherwise be overwritten. + */ + CloseStatus m_closed = CloseStatus::Open; + + /** + * Whether a step is currently active for this iteration. + * Used for file-based iteration layout, see Series.hpp for + * group-based layout. + * Access via stepStatus() method to automatically select the correct + * one among both flags. + */ + StepStatus m_stepStatus = StepStatus::NoStep; + + auxiliary::Option< DeferredParseAccess > m_deferredParseAccess{}; + }; +} /** @brief Logical compilation of data from one snapshot (e.g. a single simulation cycle). * * @see https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public LegacyAttributable +class Iteration : public Attributable { template< typename T, @@ -150,42 +217,32 @@ class Iteration : public LegacyAttributable bool closedByWriter() const; - Container< Mesh > meshes; - Container< ParticleSpecies > particles; //particleSpecies? + Container< Mesh > meshes{}; + Container< ParticleSpecies > particles{}; //particleSpecies? virtual ~Iteration() = default; private: Iteration(); - struct DeferredParseAccess + std::shared_ptr< internal::IterationData > m_iterationData{ + new internal::IterationData }; + + inline internal::IterationData const & get() const { - /** - * The group path within /data containing this iteration. - * Example: "1" for iteration 1, "" in variable-based iteration - * encoding. - */ - std::string path; - /** - * The iteration index as accessed by the user in series.iterations[i] - */ - uint64_t iteration = 0; - /** - * If this iteration is part of a Series with file-based layout. - * (Group- and variable-based parsing shares the same code logic.) - */ - bool fileBased = false; - /** - * If fileBased == true, the file name (without file path) of the file - * containing this iteration. - */ - std::string filename; - }; + return *m_iterationData; + } + + inline internal::IterationData & get() + { + return const_cast< internal::IterationData & >( + static_cast< Iteration const * >( this )->get() ); + } void flushFileBased(std::string const&, uint64_t); void flushGroupBased(uint64_t); void flushVariableBased(uint64_t); void flush(); - void deferParseAccess( DeferredParseAccess ); + void deferParseAccess( internal::DeferredParseAccess ); /* * Control flow for read(), readFileBased(), readGroupBased() and * read_impl(): @@ -198,7 +255,7 @@ class Iteration : public LegacyAttributable * allow for those different control flows. * Finally, read_impl() is called which contains the common parsing * logic for an iteration. - * + * * reread() reads again an Iteration that has been previously read. * Calling it on an Iteration not yet parsed is an error. * @@ -209,47 +266,7 @@ class Iteration : public LegacyAttributable void readGorVBased( std::string const & groupPath ); void read_impl( std::string const & groupPath ); - /** - * @brief Whether an iteration has been closed yet. - * - */ - enum class CloseStatus - { - ParseAccessDeferred, //!< The reader has not yet parsed this iteration - Open, //!< Iteration has not been closed - ClosedInFrontend, /*!< Iteration has been closed, but task has not yet - been propagated to the backend */ - ClosedInBackend, /*!< Iteration has been closed and task has been - propagated to the backend */ - ClosedTemporarily /*!< Iteration has been closed internally and may - be reopened later */ - }; - - /* - * An iteration may be logically closed in the frontend, - * but not necessarily yet in the backend. - * Will be propagated to the backend upon next flush. - * Store the current status. - * Once an iteration has been closed, no further flushes shall be performed. - * If flushing a closed file, the old file may otherwise be overwritten. - */ - std::shared_ptr< CloseStatus > m_closed = - std::make_shared< CloseStatus >( CloseStatus::Open ); - - /** - * Whether a step is currently active for this iteration. - * Used for file-based iteration layout, see Series.hpp for - * group-based layout. - * Access via stepStatus() method to automatically select the correct - * one among both flags. - */ - std::shared_ptr< StepStatus > m_stepStatus = - std::make_shared< StepStatus >( StepStatus::NoStep ); - std::shared_ptr< auxiliary::Option< DeferredParseAccess > > - m_deferredParseAccess = - std::make_shared< auxiliary::Option< DeferredParseAccess > >( - auxiliary::Option< DeferredParseAccess >() ); /** * @brief Begin an IO step on the IO file (or file-like object) @@ -305,7 +322,7 @@ class Iteration : public LegacyAttributable /** * @brief Link with parent. - * + * * @param w The Writable representing the parent. */ virtual void linkHierarchy(Writable& w); @@ -313,7 +330,7 @@ class Iteration : public LegacyAttributable /** * @brief Access an iteration in read mode that has potentially not been * parsed yet. - * + * */ void runDeferredParseAccess(); }; // Iteration diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 8bb3f1d2c2..c4e23f1bd7 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -33,10 +33,12 @@ namespace openPMD { -Iteration::Iteration() - : meshes{Container< Mesh >()}, - particles{Container< ParticleSpecies >()} +using internal::CloseStatus; +using internal::DeferredParseAccess; + +Iteration::Iteration() : Attributable{ nullptr } { + Attributable::setData( m_iterationData ); setTime(static_cast< double >(0)); setDt(static_cast< double >(1)); setTimeUnitSI(1); @@ -83,29 +85,30 @@ Iteration & Iteration::close( bool _flush ) { using bool_type = unsigned char; + auto & it = get(); if( this->IOHandler()->m_frontendAccess != Access::READ_ONLY ) { setAttribute< bool_type >( "closed", 1u ); } StepStatus flag = getStepStatus(); // update close status - switch( *m_closed ) + switch( it.m_closed ) { case CloseStatus::Open: case CloseStatus::ClosedInFrontend: - *m_closed = CloseStatus::ClosedInFrontend; + it.m_closed = CloseStatus::ClosedInFrontend; break; case CloseStatus::ClosedTemporarily: // should we bother to reopen? if( dirtyRecursive() ) { // let's reopen - *m_closed = CloseStatus::ClosedInFrontend; + it.m_closed = CloseStatus::ClosedInFrontend; } else { // don't reopen - *m_closed = CloseStatus::ClosedInBackend; + it.m_closed = CloseStatus::ClosedInBackend; } break; case CloseStatus::ParseAccessDeferred: @@ -148,9 +151,10 @@ Iteration::close( bool _flush ) Iteration & Iteration::open() { - if( *m_closed == CloseStatus::ParseAccessDeferred ) + auto & it = get(); + if( it.m_closed == CloseStatus::ParseAccessDeferred ) { - *m_closed = CloseStatus::Open; + it.m_closed = CloseStatus::Open; } runDeferredParseAccess(); Series s = retrieveSeries(); @@ -164,7 +168,7 @@ Iteration::open() bool Iteration::closed() const { - switch( *m_closed ) + switch( get().m_closed ) { case CloseStatus::ParseAccessDeferred: case CloseStatus::Open: @@ -323,17 +327,18 @@ Iteration::flush() void Iteration::deferParseAccess( DeferredParseAccess dr ) { - *m_deferredParseAccess = + get().m_deferredParseAccess = auxiliary::makeOption< DeferredParseAccess >( std::move( dr ) ); } void Iteration::read() { - if( !m_deferredParseAccess->has_value() ) + auto & it = get(); + if( !it.m_deferredParseAccess.has_value() ) { return; } - auto const & deferred = m_deferredParseAccess->get(); + auto const & deferred = it.m_deferredParseAccess.get(); if( deferred.fileBased ) { readFileBased( deferred.filename, deferred.path ); @@ -343,12 +348,12 @@ void Iteration::read() readGorVBased( deferred.path ); } // reset this thing - *m_deferredParseAccess = auxiliary::Option< DeferredParseAccess >(); + it.m_deferredParseAccess = auxiliary::Option< DeferredParseAccess >(); } void Iteration::reread( std::string const & path ) { - if( m_deferredParseAccess->has_value() ) + if( get().m_deferredParseAccess.has_value() ) { throw std::runtime_error( "[Iteration] Internal control flow error: Trying to reread an " @@ -614,7 +619,7 @@ Iteration::getStepStatus() { using IE = IterationEncoding; case IE::fileBased: - return *this->m_stepStatus; + return get().m_stepStatus; case IE::groupBased: case IE::variableBased: return s.get().m_stepStatus; @@ -631,7 +636,7 @@ Iteration::setStepStatus( StepStatus status ) { using IE = IterationEncoding; case IE::fileBased: - *this->m_stepStatus = status; + get().m_stepStatus = status; break; case IE::groupBased: case IE::variableBased: diff --git a/src/ReadIterations.cpp b/src/ReadIterations.cpp index 14329abf5d..6c88ccd2be 100644 --- a/src/ReadIterations.cpp +++ b/src/ReadIterations.cpp @@ -49,8 +49,8 @@ SeriesIterator::SeriesIterator( Series series ) * Use case: See Python ApiTest testListSeries: * Call listSeries twice. */ - if( *it->second.m_closed != - Iteration::CloseStatus::ClosedInBackend ) + if( it->second.get().m_closed != + internal::CloseStatus::ClosedInBackend ) { it->second.open(); } @@ -136,7 +136,7 @@ SeriesIterator & SeriesIterator::operator++() return *this; } m_currentIteration = it->first; - if( *it->second.m_closed != Iteration::CloseStatus::ClosedInBackend ) + if( it->second.get().m_closed != internal::CloseStatus::ClosedInBackend ) { it->second.open(); } diff --git a/src/Series.cpp b/src/Series.cpp index abc1f8a797..a8c18e70ef 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -606,13 +606,13 @@ Series::flushFileBased( iterations_iterator begin, iterations_iterator end ) } // Phase 2 - if( *it->second.m_closed == - Iteration::CloseStatus::ClosedInFrontend ) + if( it->second.get().m_closed == + internal::CloseStatus::ClosedInFrontend ) { Parameter< Operation::CLOSE_FILE > fClose; IOHandler()->enqueue( IOTask( &it->second, std::move( fClose ) ) ); - *it->second.m_closed = Iteration::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::ClosedInBackend; } // Phase 3 @@ -650,13 +650,13 @@ Series::flushFileBased( iterations_iterator begin, iterations_iterator end ) } // Phase 2 - if( *it->second.m_closed == - Iteration::CloseStatus::ClosedInFrontend ) + if( it->second.get().m_closed == + internal::CloseStatus::ClosedInFrontend ) { Parameter< Operation::CLOSE_FILE > fClose; IOHandler()->enqueue( IOTask( &it->second, std::move( fClose ) ) ); - *it->second.m_closed = Iteration::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::ClosedInBackend; } // Phase 3 @@ -689,11 +689,11 @@ Series::flushGorVBased( iterations_iterator begin, iterations_iterator end ) } // Phase 2 - if( *it->second.m_closed == - Iteration::CloseStatus::ClosedInFrontend ) + if( it->second.get().m_closed == + internal::CloseStatus::ClosedInFrontend ) { // the iteration has no dedicated file in group-based mode - *it->second.m_closed = Iteration::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::ClosedInBackend; } // Phase 3 @@ -742,11 +742,11 @@ Series::flushGorVBased( iterations_iterator begin, iterations_iterator end ) } // Phase 2 - if( *it->second.m_closed == - Iteration::CloseStatus::ClosedInFrontend ) + if( it->second.get().m_closed == + internal::CloseStatus::ClosedInFrontend ) { // the iteration has no dedicated file in group-based mode - *it->second.m_closed = Iteration::CloseStatus::ClosedInBackend; + it->second.get().m_closed = internal::CloseStatus::ClosedInBackend; } } @@ -827,14 +827,14 @@ Series::readFileBased( ) Parameter< Operation::CLOSE_FILE > fClose; iteration.IOHandler()->enqueue( IOTask( &iteration, fClose ) ); iteration.IOHandler()->flush(); - *iteration.m_closed = Iteration::CloseStatus::ClosedTemporarily; + iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; }; if( series.m_parseLazily ) { for( auto & iteration : series.iterations ) { - *iteration.second.m_closed = - Iteration::CloseStatus::ParseAccessDeferred; + iteration.second.get().m_closed = + internal::CloseStatus::ParseAccessDeferred; } // open the last iteration, just to parse Series attributes auto getLastIteration = series.iterations.end(); @@ -1028,7 +1028,7 @@ Series::readGorVBased( bool do_init ) { return; } - if( *i.m_closed != Iteration::CloseStatus::ParseAccessDeferred ) + if( i.get().m_closed != internal::CloseStatus::ParseAccessDeferred ) { pOpen.path = path; IOHandler()->enqueue( IOTask( &i, pOpen ) ); @@ -1043,11 +1043,11 @@ Series::readGorVBased( bool do_init ) if( !series.m_parseLazily ) { i.runDeferredParseAccess(); - *i.m_closed = Iteration::CloseStatus::Open; + i.get().m_closed = internal::CloseStatus::Open; } else { - *i.m_closed = Iteration::CloseStatus::ParseAccessDeferred; + i.get().m_closed = internal::CloseStatus::ParseAccessDeferred; } } }; @@ -1205,23 +1205,24 @@ Series::advance( * In order to avoid having those tasks automatically appended by * flush_impl(), set CloseStatus to Open for now. */ - Iteration::CloseStatus oldCloseStatus = *iteration.m_closed; - if( oldCloseStatus == Iteration::CloseStatus::ClosedInFrontend ) + auto & itData = iteration.get(); + internal::CloseStatus oldCloseStatus = itData.m_closed; + if( oldCloseStatus == internal::CloseStatus::ClosedInFrontend ) { - *iteration.m_closed = Iteration::CloseStatus::Open; + itData.m_closed = internal::CloseStatus::Open; } flush_impl( begin, end, FlushLevel::UserFlush, /* flushIOHandler = */ false ); - if( oldCloseStatus == Iteration::CloseStatus::ClosedInFrontend ) + if( oldCloseStatus == internal::CloseStatus::ClosedInFrontend ) { // Series::flush() would normally turn a `ClosedInFrontend` into // a `ClosedInBackend`. Do that manually. - *iteration.m_closed = Iteration::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::ClosedInBackend; } else if( - oldCloseStatus == Iteration::CloseStatus::ClosedInBackend && + oldCloseStatus == internal::CloseStatus::ClosedInBackend && series.m_iterationEncoding == IterationEncoding::fileBased ) { /* @@ -1233,7 +1234,7 @@ Series::advance( } Parameter< Operation::ADVANCE > param; - if( *iteration.m_closed == Iteration::CloseStatus::ClosedTemporarily && + if( itData.m_closed == internal::CloseStatus::ClosedTemporarily && series.m_iterationEncoding == IterationEncoding::fileBased ) { /* @@ -1251,7 +1252,7 @@ Series::advance( } - if( oldCloseStatus == Iteration::CloseStatus::ClosedInFrontend && + if( oldCloseStatus == internal::CloseStatus::ClosedInFrontend && mode == AdvanceMode::ENDSTEP ) { using IE = IterationEncoding; @@ -1259,14 +1260,14 @@ Series::advance( { case IE::fileBased: { - if( *iteration.m_closed != - Iteration::CloseStatus::ClosedTemporarily ) + if( itData.m_closed != + internal::CloseStatus::ClosedTemporarily ) { Parameter< Operation::CLOSE_FILE > fClose; IOHandler()->enqueue( IOTask( &iteration, std::move( fClose ) ) ); } - *iteration.m_closed = Iteration::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::ClosedInBackend; break; } case IE::groupBased: @@ -1277,7 +1278,7 @@ Series::advance( // In group-based iteration layout, files are // not closed on a per-iteration basis // We will treat it as such nonetheless - *iteration.m_closed = Iteration::CloseStatus::ClosedInBackend; + itData.m_closed = internal::CloseStatus::ClosedInBackend; break; } case IE::variableBased: // no action necessary @@ -1310,12 +1311,12 @@ auto Series::openIterationIfDirty( uint64_t index, Iteration iteration ) * Check side conditions on accessing iterations, and if they are fulfilled, * forward function params to openIteration(). */ - if( *iteration.m_closed == Iteration::CloseStatus::ParseAccessDeferred ) + if( iteration.get().m_closed == internal::CloseStatus::ParseAccessDeferred ) { return IterationOpened::RemainsClosed; } bool const dirtyRecursive = iteration.dirtyRecursive(); - if( *iteration.m_closed == Iteration::CloseStatus::ClosedInBackend ) + if( iteration.get().m_closed == internal::CloseStatus::ClosedInBackend ) { // file corresponding with the iteration has previously been // closed and fully flushed @@ -1367,10 +1368,10 @@ auto Series::openIterationIfDirty( uint64_t index, Iteration iteration ) void Series::openIteration( uint64_t index, Iteration iteration ) { - auto oldStatus = *iteration.m_closed; - switch( *iteration.m_closed ) + auto oldStatus = iteration.get().m_closed; + switch( oldStatus ) { - using CL = Iteration::CloseStatus; + using CL = internal::CloseStatus; case CL::ClosedInBackend: throw std::runtime_error( "[Series] Detected illegal access to iteration that " @@ -1378,7 +1379,7 @@ void Series::openIteration( uint64_t index, Iteration iteration ) case CL::ParseAccessDeferred: case CL::Open: case CL::ClosedTemporarily: - *iteration.m_closed = CL::Open; + iteration.get().m_closed = CL::Open; break; case CL::ClosedInFrontend: // just keep it like it is @@ -1406,7 +1407,7 @@ void Series::openIteration( uint64_t index, Iteration iteration ) */ if( !iteration.written() && ( IOHandler()->m_frontendAccess == Access::CREATE || - oldStatus != Iteration::CloseStatus::ParseAccessDeferred ) ) + oldStatus != internal::CloseStatus::ParseAccessDeferred ) ) { // nothing to do, file will be opened by writing routines break; diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 043a6922a2..d1e5a221ef 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -166,7 +166,7 @@ Iteration const & Attributable::containingIteration() const ( *searchQueue.rbegin() )->attributable ); for( auto const & pair : series.iterations ) { - if( &pair.second.get() == attr ) + if( &static_cast< Attributable const & >( pair.second ).get() == attr ) { return pair.second; }