added check for pending get requests after put data

Signed-off-by: arun696 <[email protected]>
modin-project · Mar 7, 2023 · 51718b5 · 51718b5
1 parent 7045e94
commit 51718b5
Show file tree

Hide file tree

Showing 16 changed files with 628 additions and 3 deletions.
diff --git a/api.log b/api.log
@@ -0,0 +1,10 @@
+GET ['rank_4_id_0'] ids
+GET ['rank_6_id_0'] ids
+GET ['rank_9_id_0'] ids
+GET ['rank_2_id_0'] ids
+GET ['rank_3_id_0'] ids
+GET ['rank_4_id_1'] ids
+GET ['rank_5_id_0'] ids
+GET ['rank_6_id_1'] ids
+GET ['rank_7_id_0'] ids
+GET ['rank_9_id_1'] ids
diff --git a/common.log b/common.log
@@ -0,0 +1,37 @@
+RoundRobin init for 4 rank
+RoundRobin reserve rank 3 for actor on worker with rank 4
+GET rank_4_id_0 id from 3 rank
+RoundRobin release rank 3 reserved for actor on worker with rank 4
+RoundRobin init for 6 rank
+RoundRobin reserve rank 5 for actor on worker with rank 6
+GET rank_6_id_0 id from 5 rank
+RoundRobin release rank 5 reserved for actor on worker with rank 6
+RoundRobin init for 9 rank
+RoundRobin reserve rank 8 for actor on worker with rank 9
+GET rank_9_id_0 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 9
+RoundRobin init for 2 rank
+RoundRobin reserve rank 8 for actor on worker with rank 2
+GET rank_2_id_0 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 2
+RoundRobin init for 3 rank
+RoundRobin reserve rank 8 for actor on worker with rank 3
+GET rank_3_id_0 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 3
+RoundRobin reserve rank 8 for actor on worker with rank 4
+GET rank_4_id_1 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 4
+RoundRobin init for 5 rank
+RoundRobin reserve rank 8 for actor on worker with rank 5
+GET rank_5_id_0 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 5
+RoundRobin reserve rank 8 for actor on worker with rank 6
+GET rank_6_id_1 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 6
+RoundRobin init for 7 rank
+RoundRobin reserve rank 8 for actor on worker with rank 7
+GET rank_7_id_0 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 7
+RoundRobin reserve rank 8 for actor on worker with rank 9
+GET rank_9_id_1 id from 8 rank
+RoundRobin release rank 8 reserved for actor on worker with rank 9
diff --git a/communication.log b/communication.log
@@ -0,0 +1,140 @@
+#               0	1	2	3	4	5	6	7	8	9	
+ACTOR_CREATE:   .---.--->   .   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.--->   .   .   .   .   .   .   .
+GET:            .---.--->   .   .   .   .   .   .   .
+TASK_DONE:      .   <---.   .   .   .   .   .   .   .
+ACTOR_CREATE:   .---.---.--->   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.---.--->   .   .   .   .   .   .
+GET:            .---.---.--->   .   .   .   .   .   .
+TASK_DONE:      .   <---.---.   .   .   .   .   .   .
+PUT_DATA:       .---.---.---.--->   .   .   .   .   .
+ACTOR_CREATE:   .---.---.---.---.--->   .   .   .   .
+PUT_OWNER:      .---.---.---.---.--->   .   .   .   .
+ACTOR_EXECUTE:  .---.---.---.---.--->   .   .   .   .
+GET:            .   .   .   .   <---.   .   .   .   .
+#               0	1	2	3	4	5	6	7	8	9	
+ACTOR_CREATE:   .---.--->   .   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.--->   .   .   .   .   .   .   .
+GET:            .---.--->   .   .   .   .   .   .   .
+TASK_DONE:      .   <---.   .   .   .   .   .   .   .
+ACTOR_CREATE:   .---.---.--->   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.---.--->   .   .   .   .   .   .
+GET:            .---.---.--->   .   .   .   .   .   .
+TASK_DONE:      .   <---.---.   .   .   .   .   .   .
+PUT_DATA:       .---.---.---.--->   .   .   .   .   .
+ACTOR_CREATE:   .---.---.---.---.--->   .   .   .   .
+PUT_OWNER:      .---.---.---.---.--->   .   .   .   .
+ACTOR_EXECUTE:  .---.---.---.---.--->   .   .   .   .
+GET:            .   .   .   .   <---.   .   .   .   .
+GET:            .---.---.---.---.--->   .   .   .   .
+PUT_DATA:       .   .   .   .   .--->   .   .   .   .
+TASK_DONE:      .   <---.---.---.---.   .   .   .   .
+ACTOR_CREATE:   .---.---.---.---.---.--->   .   .   .
+ACTOR_EXECUTE:  .---.---.---.---.---.--->   .   .   .
+TASK_DONE:      .   <---.---.---.---.---.   .   .   .
+GET:            .---.---.---.---.---.--->   .   .   .
+GET:            .---.---.---.---.---.--->   .   .   .
+ACTOR_CREATE:   .---.---.---.---.---.---.--->   .   .
+ACTOR_CREATE:   .---.---.---.---.---.---.---.--->   .
+ACTOR_EXECUTE:  .---.---.---.---.---.---.--->   .   .
+ACTOR_EXECUTE:  .---.---.---.---.---.---.---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.   .   .
+GET:            .---.---.---.---.---.---.--->   .   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .---.---.---.---.---.---.---.--->   .
+ACTOR_CREATE:   .---.---.---.---.---.---.---.---.--->
+ACTOR_CREATE:   .---.--->   .   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.--->   .   .   .   .   .   .   .
+ACTOR_EXECUTE:  .---.---.---.---.---.---.---.---.--->
+TASK_DONE:      .   <---.   .   .   .   .   .   .   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.---.
+GET:            .---.---.---.---.---.---.---.---.--->
+GET:            .---.--->   .   .   .   .   .   .   .
+ACTOR_CREATE:   .---.---.--->   .   .   .   .   .   .
+EXECUTE:        .---.---.---.--->   .   .   .   .   .
+ACTOR_EXECUTE:  .   .   .   <---.   .   .   .   .   .
+TASK_DONE:      .   <---.---.   .   .   .   .   .   .
+GET:            .   .   .   <---.   .   .   .   .   .
+GET:            .---.---.---.--->   .   .   .   .   .
+TASK_DONE:      .   <---.---.---.   .   .   .   .   .
+ACTOR_CREATE:   .---.---.---.---.--->   .   .   .   .
+EXECUTE:        .---.---.---.---.---.--->   .   .   .
+ACTOR_EXECUTE:  .   .   .   .   .   <---.   .   .   .
+TASK_DONE:      .   <---.---.---.---.   .   .   .   .
+GET:            .   .   .   .   .   <---.   .   .   .
+TASK_DONE:      .   <---.---.---.---.---.   .   .   .
+GET:            .---.---.---.---.---.--->   .   .   .
+ACTOR_CREATE:   .---.---.---.---.---.---.--->   .   .
+ACTOR_EXECUTE:  .---.---.---.---.---.---.--->   .   .
+TASK_DONE:      .   <---.---.---.---.---.---.   .   .
+GET:            .---.---.---.---.---.---.--->   .   .
+ACTOR_CREATE:   .---.---.---.---.---.---.---.--->   .
+EXECUTE:        .---.---.---.---.---.---.---.---.--->
+ACTOR_EXECUTE:  .   .   .   .   .   .   .   .   <---.
+GET:            .   .   .   .   .   .   .   .   <---.
+GET:            .---.---.---.---.---.---.---.---.--->
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.---.
+EXECUTE:        .---.--->   .   .   .   .   .   .   .
+ACTOR_EXECUTE:  .   .   .---.---.---.---.---.--->   .
+GET:            .   .   .---.---.---.---.---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .---.--->   .   .   .   .   .   .   .
+TASK_DONE:      .   <---.   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.--->   .   .   .   .   .   .
+ACTOR_EXECUTE:  .   .   .   .---.---.---.---.--->   .
+GET:            .   .   .   .---.---.---.---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .---.---.--->   .   .   .   .   .   .
+TASK_DONE:      .   <---.---.   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.---.--->   .   .   .   .   .
+ACTOR_EXECUTE:  .   .   .   .   .---.---.---.--->   .
+GET:            .   .   .   .   .---.---.---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+TASK_DONE:      .   <---.---.---.   .   .   .   .   .
+GET:            .---.---.---.--->   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.---.---.--->   .   .   .   .
+ACTOR_EXECUTE:  .   .   .   .   .   .---.---.--->   .
+GET:            .   .   .   .   .   .---.---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .---.---.---.---.--->   .   .   .   .
+TASK_DONE:      .   <---.---.---.---.   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.---.---.---.--->   .   .   .
+ACTOR_EXECUTE:  .   .   .   .   .   .   .---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .   .   .   .   .   .   .---.--->   .
+TASK_DONE:      .   <---.---.---.---.---.   .   .   .
+GET:            .---.---.---.---.---.--->   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.---.---.---.---.--->   .   .
+ACTOR_EXECUTE:  .   .   .   .   .   .   .   .--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+GET:            .   .   .   .   .   .   .   .--->   .
+TASK_DONE:      .   <---.---.---.---.---.---.   .   .
+GET:            .---.---.---.---.---.---.--->   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+EXECUTE:        .---.---.---.---.---.---.---.---.--->
+ACTOR_EXECUTE:  .   .   .   .   .   .   .   .   <---.
+GET:            .   .   .   .   .   .   .   .   <---.
+GET:            .---.---.---.---.---.---.---.---.--->
+TASK_DONE:      .   <---.---.---.---.---.---.---.   .
+TASK_DONE:      .   <---.---.---.---.---.---.---.---.
+GET_TASK_COUNT: .--->   .   .   .   .   .   .   .   .
+CANCEL:         .--->   .   .   .   .   .   .   .   .
+CANCEL:         .---.--->   .   .   .   .   .   .   .
+CANCEL:         .---.---.--->   .   .   .   .   .   .
+CANCEL:         .---.---.---.--->   .   .   .   .   .
+CANCEL:         .---.---.---.---.--->   .   .   .   .
+CANCEL:         .---.---.---.---.---.--->   .   .   .
+CANCEL:         .---.---.---.---.---.---.--->   .   .
+CANCEL:         .---.---.---.---.---.---.---.---.--->
+CANCEL:         .---.---.---.---.---.---.---.--->   .
diff --git a/unidist/core/backends/mpi/core/common.py b/unidist/core/backends/mpi/core/common.py
@@ -118,7 +118,8 @@ def __del__(self):
     def __getstate__(self):
         """Remove a reference to garbage collector for correct `pickle` serialization."""
         attributes = self.__dict__.copy()
-        del attributes["_gc"]
+        if hasattr(self, "_gc"):
+            del attributes["_gc"]
         return attributes
 
     def base_data_id(self):

diff --git a/unidist/core/backends/mpi/core/worker/loop.py b/unidist/core/backends/mpi/core/worker/loop.py
@@ -4,7 +4,6 @@
 
 """Worker MPI process task processing functionality."""
 import asyncio
-import sys
 from functools import wraps, partial
 
 try:
@@ -126,6 +125,8 @@ async def worker_loop():
             # Check pending actor requests also.
             task_store.check_pending_actor_tasks()
 
+            RequestStore.get_instance().check_pending_get_requests(request["id"])
+
         elif operation_type == common.Operation.PUT_OWNER:
             request = communication.recv_simple_operation(mpi_state.comm, source_rank)
             object_store.put_data_owner(request["id"], request["owner"])

diff --git a/unidist/core/backends/mpi/core/worker/task_store.py b/unidist/core/backends/mpi/core/worker/task_store.py
@@ -180,7 +180,7 @@ def unwrap_local_data_id(self, arg):
                         self.request_worker_data(owner_rank, arg)
                 return arg, True
             else:
-                raise ValueError("DataID is missing!")
+                raise ValueError("DataID is missing! {}".format(arg))
         else:
             return arg, False
 
@@ -357,6 +357,12 @@ def process_task_request(self, request):
         w_logger.debug("Is pending - {}".format(is_pending))
 
         if is_pending or is_kw_pending:
+            current_rank = communication.MPIState.get_instance().rank
+            if isinstance(output_ids, (list, tuple)):
+                for output_id in output_ids:
+                    ObjectStore.get_instance().put_data_owner(output_id, current_rank)
+            else:
+                ObjectStore.get_instance().put_data_owner(output_ids, current_rank)
             request["args"] = args
             request["kwargs"] = kwargs
             return request

diff --git a/utils.log b/utils.log
@@ -0,0 +1,20 @@
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
+Cleanup list len - 0
+Cleanup counter 1, threshold reached - False
diff --git a/worker_0.log b/worker_0.log
@@ -0,0 +1,38 @@
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f4d75c6d800> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f4d75c6d8c0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f4d75c6d940> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fcdbd37c540> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fcdbd37cd00> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddd40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f41743461c0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddc80> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f4174014e40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddfc0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddf40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff326ef9cc0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff32726f680> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff326ef96c0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711c13ca00> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711c13cb00> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711481d080> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f164879f3c0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f164877e240> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f1647440a40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fee4df30f00> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fee4e244a00> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fee4df37500> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddf40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbddcc0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f416cbef600> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fcdc6bf3580> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fcdbe3faac0> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7fcdbd38b200> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff326ef9740> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff326ef9d40> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7ff326f0db80> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f0aa2f03d80> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f0aa2f0a380> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f0aa2f0a300> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711480e700> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711480e400> - ready
+CHECK ASYNC HANDLER <mpi4py.MPI.Request object at 0x7f711481d940> - ready
diff --git a/worker_2.log b/worker_2.log
@@ -0,0 +1,45 @@
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_1']
+Is pending - False
+- Start task execution -
+Task evaluation time: 3.4999975468963385e-06
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_0_id_1 id to 0 rank - PROCESSED
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_1']
+Is pending - False
+- Start task execution -
+Task evaluation time: 3.6999990697950125e-06
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_0_id_1 id to 0 rank - PROCESSED
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_17']
+Is pending - False
+- Start task execution -
+Task evaluation time: 1.6999983927235007e-06
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_0_id_17 id to 0 rank - PROCESSED
+common.Operation processing - 1
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_26']
+Is pending - False
+- Start task execution -
+Task evaluation time: 0.0020414000027813017
+- End task execution -
+Check pending tasks
+common.Operation processing - 2
+Send requested rank_0_id_26 id to 0 rank - PROCESSED
+common.Operation processing - 11
+Exit worker event loop
diff --git a/worker_3.log b/worker_3.log
@@ -0,0 +1,45 @@
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_3']
+Is pending - False
+- Start task execution -
+Task evaluation time: 7.999915396794677e-07
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_0_id_3 id to 0 rank - PROCESSED
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_3']
+Is pending - False
+- Start task execution -
+Task evaluation time: 2.4000037228688598e-06
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_0_id_3 id to 0 rank - PROCESSED
+common.Operation processing - 6
+common.Operation processing - 7
+REMOTE args: []
+REMOTE outputs: ['rank_4_id_0']
+Is pending - False
+- Start task execution -
+Task evaluation time: 1.7999991541728377e-06
+- End task execution -
+Check pending actor tasks
+common.Operation processing - 2
+Send requested rank_4_id_0 id to 4 rank - PROCESSED
+common.Operation processing - 1
+REMOTE args: []
+REMOTE outputs: ['rank_0_id_27']
+Is pending - False
+- Start task execution -
+Task evaluation time: 0.0022320999996736646
+- End task execution -
+Check pending tasks
+common.Operation processing - 2
+Send requested rank_0_id_27 id to 0 rank - PROCESSED
+common.Operation processing - 11
+Exit worker event loop