From bf1ddeff9f62a83bfd3d1f800d15438d96b453ff Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 29 Nov 2023 17:00:42 -0800 Subject: [PATCH 1/5] #2209: runtime: add dependency on objgroup for LM --- src/vt/runtime/runtime.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vt/runtime/runtime.cc b/src/vt/runtime/runtime.cc index 51ace6b1c9..bc42fe2725 100644 --- a/src/vt/runtime/runtime.cc +++ b/src/vt/runtime/runtime.cc @@ -824,7 +824,8 @@ void Runtime::initializeComponents() { p_->registerComponent( &theLocMan, Deps< ctx::Context, // Everything depends on theContext - messaging::ActiveMessenger // Depends on active messenger for sending + messaging::ActiveMessenger, // Depends on active messenger for sending + objgroup::ObjGroupManager // Depends on objgroup since it creates them >{} ); From 00f494e85621a4ae331109b1ca3ada4c49ecb119 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 29 Nov 2023 17:02:05 -0800 Subject: [PATCH 2/5] #2209: location: rewrite using objgroups to reduce complexity greatly --- src/vt/topos/location/location.h | 88 ++--- src/vt/topos/location/location.impl.h | 337 ++++-------------- src/vt/topos/location/manager.cc | 37 +- src/vt/topos/location/manager.h | 100 ++---- src/vt/topos/location/manager.impl.h | 105 ++---- src/vt/topos/location/message/msg.h | 37 -- .../vrt/collection/collection_builder.impl.h | 6 +- src/vt/vrt/collection/manager.impl.h | 11 +- 8 files changed, 170 insertions(+), 551 deletions(-) diff --git a/src/vt/topos/location/location.h b/src/vt/topos/location/location.h index 1e465e0e34..fd30232023 100644 --- a/src/vt/topos/location/location.h +++ b/src/vt/topos/location/location.h @@ -57,6 +57,7 @@ #include "vt/context/context.h" #include "vt/activefn/activefn.h" #include "vt/vrt/vrt_common.h" +#include "vt/objgroup/manager.h" #include #include @@ -93,16 +94,16 @@ struct collection_lm_tag_t {}; */ template struct EntityLocationCoord : LocationCoord { + using ThisType = EntityLocationCoord; using LocRecType = LocRecord; using LocCacheType = LocLookup; using LocEntityMsg = LocEntity; using LocalRegisteredContType = std::unordered_set; using LocalRegisteredMsgContType = std::unordered_map; - using ActionListType = std::list; + using ActionListType = std::vector; using PendingType = PendingLocationLookup; using PendingLocLookupsType = std::unordered_map; using ActionContainerType = std::unordered_map; - using LocMsgType = LocationMsg; using LocAsksType = std::unordered_map>; template @@ -111,25 +112,11 @@ struct EntityLocationCoord : LocationCoord { /** * \internal \brief System call to construct a new entity coordinator */ - EntityLocationCoord(); + EntityLocationCoord() + : recs_(default_max_cache_size, theContext()->getNode()) + { } - /** - * \internal \brief System call to construct a new entity coordinator for - * collections - * - * \param[in] collection_lm_tag_t tag - * \param[in] identifier the entity class identifier - */ - EntityLocationCoord(collection_lm_tag_t, LocInstType identifier); - - /** - * \internal \brief System call to construct a new entity coordinator - * - * \param[in] identifier the entity class identifier - */ - explicit EntityLocationCoord(LocInstType const identifier); - - virtual ~EntityLocationCoord(); + virtual ~EntityLocationCoord() {} /** * \brief Register a new entity @@ -296,14 +283,12 @@ struct EntityLocationCoord : LocationCoord { /** * \internal \brief Update location * - * \param[in] event_id the event ID waiting on the location * \param[in] id the entity ID * \param[in] resolved_node the node reported to have the entity * \param[in] home_node the home node for the entity */ void updatePendingRequest( - LocEventID const& event_id, EntityID const& id, - NodeType const& resolved_node, NodeType const& home_node + EntityID const& id, NodeType const& resolved_node, NodeType const& home_node ); /** @@ -365,6 +350,15 @@ struct EntityLocationCoord : LocationCoord { template bool useEagerProtocol(MsgSharedPtr const& msg) const; + /** + * \brief Set the proxy for the objgroup + * + * \param[in] proxy proxy to set + */ + void setProxy(objgroup::proxy::Proxy> proxy) { + proxy_ = proxy; + } + private: /** * \internal \brief Handle relocation on different node. @@ -372,28 +366,27 @@ struct EntityLocationCoord : LocationCoord { * \param[in] msg the message */ template - static void routedHandler(MessageT *msg); + void routedHandler(MessageT *msg); /** * \internal \brief Request location handler from this node * - * \param[in] msg the location request message + * \param[in] id entity id + * \param[in] home_node the home node for the entity + * \param[in] cb the callback to trigger */ - static void getLocationHandler(LocMsgType *msg); + void getLocationRequest( + EntityID id, NodeType home_node, Callback cb + ); /** * \internal \brief Update the location on this node * - * \param[in] msg the location update message - */ - static void updateLocation(LocMsgType *msg); - - /** - * \internal \brief Receive an eager location update - * - * \param[in] msg the location update message + * \param[in] id entity id + * \param[in] answer the answer of where the entity is location + * \param[in] home_node the home node for the entity */ - static void recvEagerUpdate(LocMsgType *msg); + void updateLocation(EntityID const& id, NodeType answer, NodeType home_node); /** * \internal \brief Route a message to destination with eager protocol @@ -433,34 +426,27 @@ struct EntityLocationCoord : LocationCoord { */ void insertPendingEntityAction(EntityID const& id, NodeActionType action); -public: - /** - * \internal \brief Get the instance identifier for this location manager - * - * \return the instance ID - */ - LocInstType getInst() const; - private: - LocInstType this_inst = no_loc_inst; - - // message handlers for local registrations + /// message handlers for local registrations LocalRegisteredMsgContType local_registered_msg_han_; - // registered entities + /// registered entities LocalRegisteredContType local_registered_; - // the cached location records + /// the cached location records LocCacheType recs_; - // hold pending actions that this location manager is waiting on + /// hold pending actions that this location manager is waiting on ActionContainerType pending_actions_; - // pending lookup requests where this manager is the home node + /// pending lookup requests where this manager is the home node PendingLocLookupsType pending_lookups_; - // List of nodes that inquire about an entity that require an update + /// List of nodes that inquire about an entity that require an update LocAsksType loc_asks_; + + /// the location manager's objgroup proxy + objgroup::proxy::Proxy> proxy_; }; }} // end namespace vt::location diff --git a/src/vt/topos/location/location.impl.h b/src/vt/topos/location/location.impl.h index d547a16b35..101d125566 100644 --- a/src/vt/topos/location/location.impl.h +++ b/src/vt/topos/location/location.impl.h @@ -60,38 +60,6 @@ namespace vt { namespace location { -template -EntityLocationCoord::EntityLocationCoord() - : EntityLocationCoord( LocationManager::cur_loc_inst++ ) -{ } - -template -EntityLocationCoord::EntityLocationCoord( - collection_lm_tag_t, LocInstType identifier -) : EntityLocationCoord(identifier) -{ } - -template -EntityLocationCoord::EntityLocationCoord(LocInstType const identifier) - : LocationCoord(), - this_inst(identifier), - recs_(default_max_cache_size, theContext()->getNode()) -{ - vt_debug_print( - normal, location, - "EntityLocationCoord constructor: inst={}, this={}\n", - this_inst, print_ptr(this) - ); - - LocationManager::insertInstance>( - this_inst, this - ); -} - -template -/*virtual*/ EntityLocationCoord::~EntityLocationCoord() { -} - template void EntityLocationCoord::registerEntity( EntityID const& id, NodeType const& home, LocMsgActionType msg_action, @@ -107,9 +75,9 @@ void EntityLocationCoord::registerEntity( vt_debug_print( terse, location, - "EntityLocationCoord: registerEntity: inst={}, home={}, migrated={}, " + "EntityLocationCoord: registerEntity: home={}, migrated={}, " "id={}\n", - this_inst, home, migrated, id + home, migrated, id ); local_registered_.insert(id); @@ -121,16 +89,9 @@ void EntityLocationCoord::registerEntity( // local_registered_msg_han_.find(id) == local_registered_msg_han_.end(), // "Entity should not exist in local registered msg handler" // ); - local_registered_msg_han_.emplace( - std::piecewise_construct, - std::forward_as_tuple(id), - std::forward_as_tuple(LocEntityMsg{id, msg_action}) - ); + local_registered_msg_han_.emplace(id, LocEntityMsg{id, msg_action}); } - // trigger any pending actions upon registration - auto pending_lookup_iter = pending_lookups_.find(id); - vt_debug_print( normal, location, "EntityLocationCoord: registerEntity: pending lookups size={}, this={}, " @@ -138,19 +99,21 @@ void EntityLocationCoord::registerEntity( pending_lookups_.size(), print_ptr(this), id ); - if (pending_lookup_iter != pending_lookups_.end()) { + // trigger any pending actions upon registration + { auto const& node = theContext()->getNode(); int action = 0; - for (auto&& pending_action : pending_lookup_iter->second) { - vt_debug_print( - verbose, location, - "EntityLocationCoord: registerEntity: running pending action {}\n", - action - ); - action++; - pending_action(node); + if (auto lookups = pending_lookups_.extract(id); lookups) { + for (auto&& pending_action : lookups.mapped()) { + vt_debug_print( + verbose, location, + "EntityLocationCoord: registerEntity: running pending action {}\n", + action + ); + action++; + pending_action(node); + } } - pending_lookups_.erase(pending_lookup_iter); } /* @@ -168,13 +131,10 @@ void EntityLocationCoord::registerEntity( id, home, migrated ); - auto const& ask_node = uninitialized_destination; - auto msg = makeMessage( - this_inst, id, no_location_event_id, ask_node, home - ); - msg->setResolvedNode(this_node); - theMsg()->markAsLocationMessage(msg); - theMsg()->sendMsg<&EntityLocationCoord::updateLocation>(home, msg); + proxy_[home].template send<&ThisType::updateLocation>( + MsgProps().asLocationMsg(), id, this_node, home + ); + } } } @@ -191,9 +151,9 @@ void EntityLocationCoord::registerEntityRemote( vt_debug_print( normal, location, - "EntityLocationCoord: registerEntityRemote: inst={}, home={}, " + "EntityLocationCoord: registerEntityRemote: home={}, " "create_node={}, id={}\n", - this_inst, home, create_node, id + home, create_node, id ); auto const this_node = theContext()->getNode(); @@ -202,11 +162,7 @@ void EntityLocationCoord::registerEntityRemote( recs_.insert(id, home, LocRecType{id, eLocState::Remote, create_node}); if (msg_action != nullptr) { - local_registered_msg_han_.emplace( - std::piecewise_construct, - std::forward_as_tuple(id), - std::forward_as_tuple(LocEntityMsg{id, msg_action}) - ); + local_registered_msg_han_.emplace(id, LocEntityMsg{id, msg_action}); } } @@ -319,17 +275,7 @@ void EntityLocationCoord::insertPendingEntityAction( print_ptr(this), id ); - // this is the home node and there is no record on this entity - auto pending_iter = pending_lookups_.find(id); - if (pending_iter != pending_lookups_.end()) { - pending_iter->second.push_back(action); - } else { - pending_lookups_.emplace( - std::piecewise_construct, - std::forward_as_tuple(id), - std::forward_as_tuple(ActionListType{action}) - ); - } + pending_lookups_[id].push_back(action); } template @@ -389,6 +335,24 @@ void EntityLocationCoord::routeMsgEager( return routeMsgNode(id, home_node, route_to_node, msg); } +template +void EntityLocationCoord::getLocationRequest( + EntityID id, NodeType home_node, + Callback cb +) { + getLocation(id, home_node, [=](NodeType n) mutable { + auto props = MsgProps().asLocationMsg(); + cb.send(props, id, n, home_node); + }); +} + +template +void EntityLocationCoord::updateLocation( + EntityID const& id, NodeType answer, NodeType home_node +) { + updatePendingRequest(id, answer, home_node); +} + template void EntityLocationCoord::getLocation( EntityID const& id, NodeType const& home_node, NodeActionType const& action @@ -407,38 +371,28 @@ void EntityLocationCoord::getLocation( action(this_node); return; } else { - bool const& rec_exists = recs_.exists(id); + bool const rec_exists = recs_.exists(id); vt_debug_print( normal, location, - "EntityLocationCoord: getLocation: home_node={}, rec_exists={}, " - "msg size={}\n", - home_node, print_bool(rec_exists), sizeof(LocMsgType) + "EntityLocationCoord: getLocation: home_node={}, rec_exists={}\n", + home_node, print_bool(rec_exists) ); if (not rec_exists) { if (home_node != this_node) { - auto const& event_id = fst_location_event_id++; - auto msg = makeMessage( - this_inst, id, event_id, this_node, home_node + auto cb = theCB()->makeSend<&ThisType::updateLocation>(proxy_[this_node]); + proxy_[home_node].template send<&ThisType::getLocationRequest>( + MsgProps().asLocationMsg(), id, home_node, cb ); - theMsg()->markAsLocationMessage(msg); - theMsg()->sendMsg<&EntityLocationCoord::getLocationHandler>(home_node, msg); - // save a pending action when information about location arrives - pending_actions_.emplace( - std::piecewise_construct, - std::forward_as_tuple(event_id), - std::forward_as_tuple(PendingType{id, action}) - ); - } else { - // this is the home node and there is no record on this entity - insertPendingEntityAction(id, action); } + + pending_lookups_[id].push_back(action); } else { auto const& rec = recs_.get(id); if (rec.isLocal()) { - vtAssert(0, "Should be registered if this is the case!"); + vtAssert(false, "Should be registered if this is the case!"); action(this_node); } else if (rec.isRemote()) { vt_debug_print( @@ -493,10 +447,9 @@ void EntityLocationCoord::sendEagerUpdate( auto this_node = theContext()->getNode(); if (ask_node != this_node) { vtAssert(ask_node != uninitialized_destination, "Ask node must be valid"); - auto msg = makeMessage( - this_inst, id, ask_node, home_node, deliver_node + proxy_[ask_node].template send<&ThisType::handleEagerUpdate>( + id, home_node, deliver_node ); - theMsg()->sendMsg<&EntityLocationCoord::recvEagerUpdate>(ask_node, msg); } } @@ -511,9 +464,9 @@ void EntityLocationCoord::routeMsgNode( vt_debug_print( normal, location, - "EntityLocationCoord: routeMsgNode: to_node={}, this_node={}: inst={}, " + "EntityLocationCoord: routeMsgNode: to_node={}, this_node={}: " "home_node={}, id={}, ref={}, from={}, msg={}, epoch={:x}\n", - to_node, this_node, this_inst, home_node, id, + to_node, this_node, home_node, id, envelopeGetRef(msg->env), msg->getLocFromNode(), print_ptr(msg.get()), epoch ); @@ -531,12 +484,9 @@ void EntityLocationCoord::routeMsgNode( // Update the new asking node, as this node is will be the next to ask msg->setAskNode(this_node); - // set the instance on the message to deliver to the correct manager - msg->setLocInst(this_inst); - auto m = msg; //copy for msg thief // send to the node discovered by the location manager - theMsg()->sendMsg::routedHandler>(to_node, m); + proxy_[to_node].template sendMsg<&ThisType::routedHandler>(m); } else { vt_debug_print( normal, location, @@ -693,7 +643,6 @@ void EntityLocationCoord::setupMessageForRouting( msg->setEntity(id); msg->setHomeNode(home_node); msg->setLocFromNode(theContext()->getNode()); - msg->setLocInst(this_inst); } template @@ -717,9 +666,9 @@ void EntityLocationCoord::routePreparedMsg( vt_debug_print( verbose, location, - "routeMsg: inst={}, home={}, msg_size={}, is_large_msg={}, eager={}, " + "routeMsg: home={}, msg_size={}, is_large_msg={}, eager={}, " "msg={}, from={}, epoch={:x}\n", - this_inst, msg->getHomeNode(), msg_size, msg_size > small_msg_max_size, use_eager, + msg->getHomeNode(), msg_size, msg_size > small_msg_max_size, use_eager, print_ptr(msg.get()), msg->getLocFromNode(), epoch ); @@ -762,46 +711,21 @@ void EntityLocationCoord::routeMsg( template void EntityLocationCoord::updatePendingRequest( - LocEventID const& event_id, EntityID const& id, - NodeType const& node, NodeType const& home_node + EntityID const& id, NodeType const& node, NodeType const& home_node ) { vt_debug_print( normal, location, - "EntityLocationCoord: updatePendingRequest: event_id={}, node={}\n", - event_id, node + "EntityLocationCoord: updatePendingRequest: node={}\n", + node ); - if (event_id != no_location_event_id) { - auto pending_iter = pending_actions_.find(event_id); - - vtAssert( - pending_iter != pending_actions_.end(), "Event must exist in pending" - ); + recs_.insert(id, home_node, LocRecType{id, eLocState::Remote, node}); - auto const& entity = pending_iter->second.entity_; - - recs_.insert(entity, home_node, LocRecType{entity, eLocState::Remote, node}); - - pending_iter->second.applyNodeAction(node); - - pending_actions_.erase(pending_iter); - } else { - recs_.insert(id, home_node, LocRecType{id, eLocState::Remote, node}); - - // trigger any pending actions upon registration - auto pending_lookup_iter = pending_lookups_.find(id); - - vt_debug_print( - normal, location, - "EntityLocationCoord: updatePendingRequest: node={}\n", node - ); - - if (pending_lookup_iter != pending_lookups_.end()) { - for (auto&& pending_action : pending_lookup_iter->second) { - pending_action(node); - } - pending_lookups_.erase(pending_lookup_iter); + // trigger any pending actions upon registration + if (auto lookups = pending_lookups_.extract(id); lookups) { + for (auto&& action : lookups.mapped()) { + action(node); } } } @@ -811,21 +735,15 @@ void EntityLocationCoord::printCurrentCache() const { recs_.printCache(); } -template -LocInstType EntityLocationCoord::getInst() const { - return this_inst; -} - template template -/*static*/ void EntityLocationCoord::routedHandler(MessageT *raw_msg) { +void EntityLocationCoord::routedHandler(MessageT *raw_msg) { // Message may be re-routed (and sent) again from subsequent routeMsg. envelopeUnlockForForwarding(raw_msg->env); auto msg = promoteMsg(raw_msg); auto const entity_id = msg->getEntity(); auto const home_node = msg->getHomeNode(); - auto const inst = msg->getLocInst(); auto const from_node = msg->getLocFromNode(); auto const epoch = theMsg()->getEpochContextMsg(msg); @@ -833,130 +751,13 @@ template vt_debug_print( verbose, location, - "routedHandler: msg={}, ref={}, loc_inst={}, id={}, from={}, " + "routedHandler: msg={}, ref={}, id={}, from={}, " "epoch={:x}, hops={}, ask={}\n", - print_ptr(msg.get()), envelopeGetRef(msg->env), inst, entity_id, + print_ptr(msg.get()), envelopeGetRef(msg->env), entity_id, from_node, epoch, msg->getHops(), msg->getAskNode() ); - theTerm()->produce(epoch); - LocationManager::applyInstance>( - inst, [=](EntityLocationCoord* loc) { - theMsg()->pushEpoch(epoch); - loc->routeMsg(entity_id, home_node, msg, from_node); - theMsg()->popEpoch(epoch); - theTerm()->consume(epoch); - } - ); -} - -template -/*static*/ void EntityLocationCoord::getLocationHandler( - LocMsgType* raw_msg -) { - auto msg = promoteMsg(raw_msg); - auto const& event_id = msg->loc_event; - auto const& inst = msg->loc_man_inst; - auto const& entity = msg->entity; - auto const& home_node = msg->home_node; - auto const& ask_node = msg->ask_node; - auto const epoch = theMsg()->getEpochContextMsg(msg); - - vt_debug_print( - normal, location, - "getLocationHandler: event_id={}, home={}, ask={}, epoch={:x}\n", - event_id, home_node, ask_node, epoch - ); - - theTerm()->produce(epoch); - LocationManager::applyInstance>( - inst, [=](EntityLocationCoord* loc) { - theMsg()->pushEpoch(epoch); - - vt_debug_print( - verbose, location, - "getLocationHandler: calling getLocation event_id={}, epoch={:x}\n", - event_id, epoch - ); - - loc->getLocation(entity, home_node, [=](NodeType node) { - vt_debug_print( - verbose, location, - "getLocation: (action) event_id={}, epoch={:x}\n", - event_id, epoch - ); - - auto msg2 = makeMessage( - inst, entity, event_id, ask_node, home_node - ); - msg2->setResolvedNode(node); - theMsg()->markAsLocationMessage(msg2); - theMsg()->sendMsg<&EntityLocationCoord::updateLocation>(ask_node, msg2); - }); - theMsg()->popEpoch(epoch); - theTerm()->consume(epoch); - } - ); -} - -template -/*static*/ void EntityLocationCoord::updateLocation( - LocMsgType *raw_msg -) { - auto msg = promoteMsg(raw_msg); - auto const& event_id = msg->loc_event; - auto const& inst = msg->loc_man_inst; - auto const& entity = msg->entity; - auto const epoch = theMsg()->getEpochContextMsg(msg); - - vt_debug_print( - verbose, location, - "updateLocation: event_id={}, resolved={}, id={}, epoch={:x}\n", - event_id, msg->resolved_node, entity, epoch - ); - - theTerm()->produce(epoch); - LocationManager::applyInstance>( - inst, [=](EntityLocationCoord* loc) { - theMsg()->pushEpoch(epoch); - vt_debug_print( - verbose, location, - "updateLocation: event_id={}, running pending: resolved={}, id={}\n", - event_id, msg->resolved_node, entity - ); - loc->updatePendingRequest( - event_id, entity, msg->resolved_node, msg->home_node - ); - theMsg()->popEpoch(epoch); - theTerm()->consume(epoch); - } - ); -} - -template -/*static*/ void EntityLocationCoord::recvEagerUpdate( - LocMsgType *raw_msg -) { - auto msg = promoteMsg(raw_msg); - auto const& inst = msg->loc_man_inst; - auto const& entity = msg->entity; - auto const epoch = theMsg()->getEpochContextMsg(msg); - - vt_debug_print( - verbose, location, - "recvEagerUpdate: resolved={}, id={}, epoch={:x}\n", - msg->resolved_node, entity, epoch - ); - - theTerm()->produce(epoch); - LocationManager::applyInstance>( - inst, [=](EntityLocationCoord* loc) { - theMsg()->pushEpoch(epoch); - loc->handleEagerUpdate(entity, msg->home_node, msg->resolved_node); - theMsg()->popEpoch(epoch); - theTerm()->consume(epoch); - } - ); + routeMsg(entity_id, home_node, msg, from_node); } }} // end namespace vt::location diff --git a/src/vt/topos/location/manager.cc b/src/vt/topos/location/manager.cc index c18c3c9d12..ad153bcebd 100644 --- a/src/vt/topos/location/manager.cc +++ b/src/vt/topos/location/manager.cc @@ -49,30 +49,21 @@ namespace vt { namespace location { -/*static*/ LocationManager::LocInstContainerType LocationManager::loc_insts; +/*virtual*/ LocationManager::~LocationManager() { } -/*static*/ LocationManager::LocCoordPtrType LocationManager::getInstance( - LocInstType const inst -) { - auto inst_iter = loc_insts.find(inst); - - vtAssert( - inst_iter != loc_insts.end(), - "LocationManager instance must exist in container" - ); - - return inst_iter->second; +void LocationManager::initialize() { + { + auto lm_proxy = theObjGroup()->makeCollective("VrtLoc"); + lm_proxy.get()->setProxy(lm_proxy); + virtual_loc = lm_proxy.get(); + } + { + auto lm_proxy = theObjGroup()->makeCollective( + "VrtContextLoc" + ); + lm_proxy.get()->setProxy(lm_proxy); + vrtContextLoc = lm_proxy.get(); + } } -/*virtual*/ LocationManager::~LocationManager() { - virtual_loc = nullptr; - vrtContextLoc = nullptr; - collectionLoc.clear(); - loc_insts.clear(); - //pending_inst_.clear(); - cur_loc_inst = 0xFF00000000000000; -} - -/*static*/ LocInstType LocationManager::cur_loc_inst = 0xFF00000000000000; - }} // end namespace vt::location diff --git a/src/vt/topos/location/manager.h b/src/vt/topos/location/manager.h index 8f555c1de7..3dd7252e27 100644 --- a/src/vt/topos/location/manager.h +++ b/src/vt/topos/location/manager.h @@ -51,6 +51,7 @@ #include "vt/topos/location/utility/coord.h" #include "vt/vrt/vrt_common.h" #include "vt/vrt/collection/proxy.h" +#include "vt/objgroup/proxy/proxy_objgroup.h" #include "vt/runtime/component/component_pack.h" #include @@ -76,7 +77,7 @@ namespace vt { namespace location { */ struct LocationManager : runtime::component::Component { template - using PtrType = std::unique_ptr; + using PtrType = LocType*; using LocCoordPtrType = LocationCoord*; using LocInstContainerType = std::unordered_map; using VrtLocType = EntityLocationCoord; @@ -88,18 +89,9 @@ struct LocationManager : runtime::component::Component { template using CollectionLocType = PtrType; - using LocErasureType = LocationCoord; - using LocDeleterType = std::function; - using CollectionLocErasedType = std::unique_ptr< - LocErasureType, LocDeleterType - >; using CollectionContainerType = std::unordered_map< - VirtualProxyType, CollectionLocErasedType + VirtualProxyType, ObjGroupProxyType >; - template - using ActionLocInstType = std::function; - template - using PendingContainerType = std::vector>; /** * \internal \brief System call to construct a location manager @@ -110,100 +102,54 @@ struct LocationManager : runtime::component::Component { std::string name() override { return "LocationManager"; } - /** - * \internal \brief Next instance identifier - */ - static LocInstType cur_loc_inst; - - PtrType virtual_loc = std::make_unique(); - PtrType vrtContextLoc = std::make_unique(); + void initialize() override; /** - * \internal \brief Get the location coordinator for a collection + * \internal \brief Make a new location coordinator for a collection * * \param[in] proxy the collection proxy bits * - * \return pointer to the coordinator, typed on the collection/index + * \return proxy to location manager */ template - IndexedElmType* getCollectionLM(VirtualProxyType const& proxy); + objgroup::proxy::Proxy> makeCollectionLM( + VirtualProxyType proxy + ); /** - * \internal \brief Insert a new location coordinator for a collection + * \internal \brief Get the new location coordinator for a collection * * \param[in] proxy the collection proxy bits - */ - template - void insertCollectionLM(VirtualProxyType const& proxy); - -public: - // Manage different instances of individually managed entities - - /** - * \internal \brief Insert a new coordinator instance - * - * \param[in] i the instance ID - * \param[in] ptr pointer to the coordinator - */ - template - static void insertInstance(LocInstType const i, LocType* ptr); - - /** - * \internal \brief Get a coordinator instance - * - * \param[in] inst the instance ID * - * \return the location coordinator + * \return the location manager */ - static LocCoordPtrType getInstance(LocInstType const inst); + template + IndexedElmType* getCollectionLM(VirtualProxyType proxy); /** - * \internal \brief Perform an action on a coordinator + * \brief Destroy a location manager * - * \param[in] inst the instance ID - * \param[in] action action to perform + * \param[in] proxy the collection proxy bits */ - template - static void applyInstance( - LocInstType const inst, ActionLocInstType action - ); + template + void destroyCollectionLM(VirtualProxyType proxy); +public: template void serialize(SerializerT& s) { - s | collectionLoc - | cur_loc_inst - | loc_insts + s | collection_lms | virtual_loc | vrtContextLoc; } -protected: - CollectionContainerType collectionLoc; +public: + PtrType virtual_loc; + PtrType vrtContextLoc; private: - static LocInstContainerType loc_insts; + CollectionContainerType collection_lms; }; -namespace details { - -/** - * \struct PendingInst - * - * \brief Hold templated static instances for location coordinators - * - * This is split out into a separate class to address the inability of - * Intel 18 to process static member variable templates - */ -template -struct PendingInst -{ - static std::unordered_map< - LocInstType, LocationManager::PendingContainerType - > pending_inst_; -}; - -} /* end namespace details */ - }} /* end namespace vt::location */ #include "vt/topos/location/manager.impl.h" diff --git a/src/vt/topos/location/manager.impl.h b/src/vt/topos/location/manager.impl.h index d6597ccca7..8b1d908cc3 100644 --- a/src/vt/topos/location/manager.impl.h +++ b/src/vt/topos/location/manager.impl.h @@ -57,101 +57,36 @@ namespace vt { namespace location { template -void LocationManager::insertCollectionLM(VirtualProxyType const& proxy) { - using LocType = IndexedElmType; - auto loc_man_typed = new LocType( - collection_lm_tag_t{}, static_cast(proxy) - ); - auto loc_ptr = std::unique_ptr( - static_cast(loc_man_typed), - [](LocErasureType* elm) { - auto const& typed_elm = static_cast(elm); - delete typed_elm; - } - ); - collectionLoc.emplace( - std::piecewise_construct, - std::forward_as_tuple(proxy), - std::forward_as_tuple(std::move(loc_ptr)) - ); +typename LocationManager::IndexedElmType* +LocationManager::getCollectionLM(VirtualProxyType proxy) { + if (auto it = collection_lms.find(proxy); it != collection_lms.end()) { + return objgroup::proxy::Proxy>(it->second).get(); + } else { + vtAbort("Could not find location manager for proxy"); + return nullptr; + } } template -LocationManager::IndexedElmType* -LocationManager::getCollectionLM(VirtualProxyType const& proxy) { +objgroup::proxy::Proxy> +LocationManager::makeCollectionLM(VirtualProxyType proxy) { using LocType = IndexedElmType; - - auto loc_iter = collectionLoc.find(proxy); - auto const& found = loc_iter != collectionLoc.end(); - - vt_debug_print( - normal, location, - "getCollectionLM: proxy={}, found={}\n", - proxy, print_bool(found) - ); - - if (!found) { - LocationManager::insertCollectionLM(proxy); - loc_iter = collectionLoc.find(proxy); - } else if (!found) { - return nullptr; - } - vtAssert( - loc_iter != collectionLoc.end() && loc_iter->second != nullptr, - "Location manager must exist now for this collection" - ); - auto manager = loc_iter->second.get(); - auto const& typed_loc_ptr = static_cast(manager); - return typed_loc_ptr; + auto lm_proxy = theObjGroup()->makeCollective("LocationManager"); + lm_proxy.get()->setProxy(lm_proxy); + collection_lms[proxy] = lm_proxy.getProxy(); + return lm_proxy; } -template -/*static*/ void LocationManager::applyInstance( - LocInstType const inst, ActionLocInstType action -) { - auto inst_iter = loc_insts.find(inst); - if (inst_iter == loc_insts.end()) { - auto pending_iter = details::PendingInst::pending_inst_.find(inst); - if (pending_iter == details::PendingInst::pending_inst_.end()) { - details::PendingInst::pending_inst_.emplace( - std::piecewise_construct, - std::forward_as_tuple(inst), - std::forward_as_tuple(PendingContainerType{action}) - ); - } else { - pending_iter->second.push_back(action); - } +template +void LocationManager::destroyCollectionLM(VirtualProxyType proxy) { + if (auto it = collection_lms.find(proxy); it != collection_lms.end()) { + objgroup::proxy::Proxy> lm_proxy(it->second); + lm_proxy.destroyCollective(); } else { - auto const& inst_ret = LocationManager::getInstance(inst); - action(static_cast(inst_ret)); - } -} - -template -/*static*/ void LocationManager::insertInstance(LocInstType const inst, LocType* ptr) { - loc_insts.emplace( - std::piecewise_construct, - std::forward_as_tuple(inst), - std::forward_as_tuple(static_cast(ptr)) - ); - - auto iter = details::PendingInst::pending_inst_.find(inst); - if (iter != details::PendingInst::pending_inst_.end()) { - for (auto&& elm : iter->second) { - elm(ptr); - } - details::PendingInst::pending_inst_.erase(iter); + vtAbort("Could not find location manager for proxy"); } } - -namespace details { -template -/*static*/ std::unordered_map< - LocInstType, LocationManager::PendingContainerType -> PendingInst::pending_inst_; -} - }} /* end namespace vt::location */ #endif /*INCLUDED_VT_TOPOS_LOCATION_MANAGER_IMPL_H*/ diff --git a/src/vt/topos/location/message/msg.h b/src/vt/topos/location/message/msg.h index d6e53825c8..4e091d8c3f 100644 --- a/src/vt/topos/location/message/msg.h +++ b/src/vt/topos/location/message/msg.h @@ -50,39 +50,6 @@ namespace vt { namespace location { -template -struct LocationMsg : vt::Message { - using MessageParentType = vt::Message; - vt_msg_serialize_prohibited(); - - LocInstType loc_man_inst = 0; - EntityID entity{}; - LocEventID loc_event = no_location_event_id; - NodeType ask_node = uninitialized_destination; - NodeType home_node = uninitialized_destination; - NodeType resolved_node = uninitialized_destination; - - LocationMsg( - LocInstType const& in_loc_man_inst, EntityID const& in_entity, - LocEventID const& in_loc_event, NodeType const& in_ask_node, - NodeType in_home_node - ) : loc_man_inst(in_loc_man_inst), entity(in_entity), loc_event(in_loc_event), - ask_node(in_ask_node), home_node(in_home_node) - { } - - LocationMsg( - LocInstType const& in_loc_man_inst, EntityID const& in_entity, - NodeType const& in_ask_node, NodeType const& in_home_node, - NodeType in_resolved - ) : loc_man_inst(in_loc_man_inst), entity(in_entity), ask_node(in_ask_node), - home_node(in_home_node), resolved_node(in_resolved) - { } - - void setResolvedNode(NodeType const& node) { - resolved_node = node; - } -}; - template struct EntityMsg : ActiveMessageT { using MessageParentType = ActiveMessageT; @@ -99,8 +66,6 @@ struct EntityMsg : ActiveMessageT { NodeType getHomeNode() const { return home_node_; } void setLocFromNode(NodeType const& node) { loc_from_node_ = node; } NodeType getLocFromNode() const { return loc_from_node_; } - void setLocInst(LocInstType const& inst) { loc_man_inst_ = inst; } - LocInstType getLocInst() const { return loc_man_inst_; } bool hasHandler() const { return handler_ != uninitialized_handler; } void setHandler(HandlerType const han) { handler_ = han; } HandlerType getHandler() const { return handler_; } @@ -115,7 +80,6 @@ struct EntityMsg : ActiveMessageT { s | entity_id_; s | home_node_; s | loc_from_node_; - s | loc_man_inst_; s | handler_; s | hops_; s | ask_node_; @@ -125,7 +89,6 @@ struct EntityMsg : ActiveMessageT { EntityID entity_id_{}; NodeType home_node_ = uninitialized_destination; NodeType loc_from_node_ = uninitialized_destination; - LocInstType loc_man_inst_ = no_loc_inst; HandlerType handler_ = uninitialized_handler; int16_t hops_ = 0; NodeType ask_node_ = uninitialized_destination; diff --git a/src/vt/vrt/collection/collection_builder.impl.h b/src/vt/vrt/collection/collection_builder.impl.h index 9c4e94b744..e9115497af 100644 --- a/src/vt/vrt/collection/collection_builder.impl.h +++ b/src/vt/vrt/collection/collection_builder.impl.h @@ -129,9 +129,9 @@ void CollectionManager::makeCollectionImpl(param::ConstructParams& po) { auto const map_han = po.map_han_; auto const map_object = po.map_object_; - // Invoke getCollectionLM() to create a new location manager instance for - // this collection - theLocMan()->getCollectionLM(proxy); + // makeCollectionLM() to create a new location manager instance for this + // collection + theLocMan()->makeCollectionLM(proxy); // Insert action on cleanup for this collection addCleanupFn(proxy); diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index bf1ecd151e..eea8d65a87 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1137,7 +1137,7 @@ messaging::PendingSend CollectionManager::sendMsgUntypedHandler( return messaging::PendingSend{ msg, [](MsgSharedPtr& inner_msg){ auto typed_msg = inner_msg.template to(); - auto lm2 = theLocMan()->getCollectionLM(typed_msg->getLocInst()); + auto lm2 = theLocMan()->getCollectionLM(typed_msg->getProxy().getCollectionProxy()); lm2->template routePreparedMsgHandler(typed_msg); } }; @@ -1326,12 +1326,6 @@ void CollectionManager::insertMetaCollection( theCollection()->constructGroup(proxy); }, label); - /* - * This is to ensure that the collection LM instance gets created so that - * messages can be forwarded properly - */ - theLocMan()->getCollectionLM(proxy); - /** * Type-erase some lambdas for doing the collective broadcast that collects up * the LB data on each node for each collection element @@ -1983,6 +1977,9 @@ void CollectionManager::destroyMatching( EntireHolder::remove(untyped_proxy); + // destroy the location manager + theLocMan()->destroyCollectionLM(proxy.getProxy()); + auto iter = cleanup_fns_.find(untyped_proxy); if (iter != cleanup_fns_.end()) { cleanup_fns_.erase(iter); From 073a97432d855c2be031d94f6822dab1428112e1 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Wed, 29 Nov 2023 22:30:31 -0800 Subject: [PATCH 3/5] #2209: collection: order rooted constructions with new objgroup location manager --- .../vrt/collection/collection_builder.impl.h | 34 ++++++-- src/vt/vrt/collection/manager.h | 15 +++- .../collection/param/construct_params_msg.h | 82 ------------------- 3 files changed, 41 insertions(+), 90 deletions(-) delete mode 100644 src/vt/vrt/collection/param/construct_params_msg.h diff --git a/src/vt/vrt/collection/collection_builder.impl.h b/src/vt/vrt/collection/collection_builder.impl.h index e9115497af..7d66df9061 100644 --- a/src/vt/vrt/collection/collection_builder.impl.h +++ b/src/vt/vrt/collection/collection_builder.impl.h @@ -66,9 +66,7 @@ std::tuple CollectionManager::makeCollection( "collection construction", term::UseDS{false} ); theMsg()->pushEpoch(ep); - using MsgType = param::ConstructParamMsg; - auto m = makeMessage(po); - theMsg()->broadcastMsg(m); + theMsg()->send>(vt::Node(0), po, true); theMsg()->popEpoch(ep); theTerm()->finishedEpoch(ep); return std::make_tuple(ep, proxy_bits); @@ -82,11 +80,37 @@ std::tuple CollectionManager::makeCollection( } } +/*static*/ inline void CollectionManager::finishedRootedConstruction() { + theCollection()->has_pending_construction_ = false; + if (theCollection()->pending_rooted_constructions_.size() > 0) { + auto action = theCollection()->pending_rooted_constructions_.back(); + theCollection()->pending_rooted_constructions_.pop_back(); + action(); + } +} + template /*static*/ void CollectionManager::makeCollectionHandler( - param::ConstructParamMsg* msg + param::ConstructParams po, bool is_root ) { - theCollection()->makeCollectionImpl(*msg->po); + if (is_root) { + if (theCollection()->has_pending_construction_) { + auto ep = theMsg()->getEpoch(); + theTerm()->produce(ep); + theCollection()->pending_rooted_constructions_.push_back([=]{ + theTerm()->pushEpoch(ep); + makeCollectionHandler(po, true); + theTerm()->consume(ep); + theTerm()->popEpoch(ep); + }); + } else { + theMsg()->broadcast>(po, false); + } + } else { + theCollection()->makeCollectionImpl(po); + auto r = theCollection()->reducer(); + r->reduce(vt::Node(0)); + } } namespace detail { diff --git a/src/vt/vrt/collection/manager.h b/src/vt/vrt/collection/manager.h index 9ef6540206..000dbd18b8 100644 --- a/src/vt/vrt/collection/manager.h +++ b/src/vt/vrt/collection/manager.h @@ -72,7 +72,6 @@ #include "vt/runtime/component/component_pack.h" #include "vt/context/runnable_context/lb_data.fwd.h" #include "vt/vrt/collection/param/construct_params.h" -#include "vt/vrt/collection/param/construct_params_msg.h" #include "vt/utils/fntraits/fntraits.h" #include @@ -1697,10 +1696,18 @@ struct CollectionManager * \internal \brief Handler for receiving a new collection configuration to * construct on this node * - * \param[in] msg the configuration message + * \param[in] po construct parameters + * \param[in] is_root whether if it's boucing off the root first */ template - static void makeCollectionHandler(param::ConstructParamMsg* msg); + static void makeCollectionHandler( + param::ConstructParams po, bool is_root + ); + + /** + * \brief Finished a rooted construction--do the next if needed + */ + static void finishedRootedConstruction(); /** * \internal \brief System function to actually constructing the collection @@ -1766,6 +1773,8 @@ struct CollectionManager VirtualIDType next_rooted_id_ = 0; TypelessHolder typeless_holder_; std::unordered_map reduce_stamp_; + bool has_pending_construction_ = false; + std::list pending_rooted_constructions_; }; }}} /* end namespace vt::vrt::collection */ diff --git a/src/vt/vrt/collection/param/construct_params_msg.h b/src/vt/vrt/collection/param/construct_params_msg.h deleted file mode 100644 index 63086db3cf..0000000000 --- a/src/vt/vrt/collection/param/construct_params_msg.h +++ /dev/null @@ -1,82 +0,0 @@ -/* -//@HEADER -// ***************************************************************************** -// -// construct_params_msg.h -// DARMA/vt => Virtual Transport -// -// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC -// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. -// Government retains certain rights in this software. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// * Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * Neither the name of the copyright holder nor the names of its -// contributors may be used to endorse or promote products derived from this -// software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -// POSSIBILITY OF SUCH DAMAGE. -// -// Questions? Contact darma@sandia.gov -// -// ***************************************************************************** -//@HEADER -*/ - -#if !defined INCLUDED_VT_VRT_COLLECTION_PARAM_CONSTRUCT_PARAMS_MSG_H -#define INCLUDED_VT_VRT_COLLECTION_PARAM_CONSTRUCT_PARAMS_MSG_H - -#include "vt/vrt/collection/param/construct_params.h" - -namespace vt { namespace vrt { namespace collection { namespace param { - -/** - * \struct ConstructParamMsg - * - * \brief Construct PO configuration message for distributed construction - */ -template -struct ConstructParamMsg : vt::Message { - using MessageParentType = ::vt::Message; - vt_msg_serialize_required(); // po - - ConstructParamMsg() = default; - explicit ConstructParamMsg(param::ConstructParams& in_po) - : po(std::make_unique>(in_po)) - { } - - template - void serialize(SerializerT& s) { - MessageParentType::serialize(s); - s | po; - } - - /// Must use \c std::unique_ptr here because without the indirection, - /// AppleClang generates invalid alignment that causes a segfault when \c new - /// is called on this message type. The only other work around is some - /// seemingly arbitrary value to alignas (alignas(1024) seems to do the - /// trick). - std::unique_ptr> po = nullptr; -}; - -}}}} /* end namespace vt::vrt::collection::param */ - -#endif /*INCLUDED_VT_VRT_COLLECTION_PARAM_CONSTRUCT_PARAMS_MSG_H*/ From 5c50e61bf23bbf3ff225b4b2d26b302fc78a5cc7 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Thu, 30 Nov 2023 09:08:18 -0800 Subject: [PATCH 4/5] #2209: location: properly erase from map when destroyed --- src/vt/topos/location/manager.impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vt/topos/location/manager.impl.h b/src/vt/topos/location/manager.impl.h index 8b1d908cc3..43d17748dc 100644 --- a/src/vt/topos/location/manager.impl.h +++ b/src/vt/topos/location/manager.impl.h @@ -79,8 +79,8 @@ LocationManager::makeCollectionLM(VirtualProxyType proxy) { template void LocationManager::destroyCollectionLM(VirtualProxyType proxy) { - if (auto it = collection_lms.find(proxy); it != collection_lms.end()) { - objgroup::proxy::Proxy> lm_proxy(it->second); + if (auto elm = collection_lms.extract(proxy); elm) { + objgroup::proxy::Proxy> lm_proxy(elm.mapped()); lm_proxy.destroyCollective(); } else { vtAbort("Could not find location manager for proxy"); From adeef2ea46dfe7f72397ef49473516cc00e46d72 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Tue, 30 Apr 2024 15:53:46 -0700 Subject: [PATCH 5/5] #2209: location: fix compile bug --- src/vt/topos/location/location.impl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/vt/topos/location/location.impl.h b/src/vt/topos/location/location.impl.h index 101d125566..b42233738a 100644 --- a/src/vt/topos/location/location.impl.h +++ b/src/vt/topos/location/location.impl.h @@ -131,10 +131,9 @@ void EntityLocationCoord::registerEntity( id, home, migrated ); - proxy_[home].template send<&ThisType::updateLocation>( - MsgProps().asLocationMsg(), id, this_node, home - ); - } + proxy_[home].template send<&ThisType::updateLocation>( + MsgProps().asLocationMsg(), id, this_node, home + ); } }