From 51718b5e93fffdd5bd78c5a2aa9d930807ab8d23 Mon Sep 17 00:00:00 2001 From: arun696 Date: Tue, 7 Mar 2023 16:09:19 +0100 Subject: [PATCH] added check for pending get requests after put data Signed-off-by: arun696 --- api.log | 10 ++ common.log | 37 +++++ communication.log | 140 ++++++++++++++++++ unidist/core/backends/mpi/core/common.py | 3 +- unidist/core/backends/mpi/core/worker/loop.py | 3 +- .../backends/mpi/core/worker/task_store.py | 8 +- utils.log | 20 +++ worker_0.log | 38 +++++ worker_2.log | 45 ++++++ worker_3.log | 45 ++++++ worker_4.log | 34 +++++ worker_5.log | 52 +++++++ worker_6.log | 35 +++++ worker_7.log | 34 +++++ worker_8.log | 94 ++++++++++++ worker_9.log | 33 +++++ 16 files changed, 628 insertions(+), 3 deletions(-) create mode 100644 api.log create mode 100644 common.log create mode 100644 communication.log create mode 100644 utils.log create mode 100644 worker_0.log create mode 100644 worker_2.log create mode 100644 worker_3.log create mode 100644 worker_4.log create mode 100644 worker_5.log create mode 100644 worker_6.log create mode 100644 worker_7.log create mode 100644 worker_8.log create mode 100644 worker_9.log diff --git a/api.log b/api.log new file mode 100644 index 00000000..b54cb2e1 --- /dev/null +++ b/api.log @@ -0,0 +1,10 @@ +GET ['rank_4_id_0'] ids +GET ['rank_6_id_0'] ids +GET ['rank_9_id_0'] ids +GET ['rank_2_id_0'] ids +GET ['rank_3_id_0'] ids +GET ['rank_4_id_1'] ids +GET ['rank_5_id_0'] ids +GET ['rank_6_id_1'] ids +GET ['rank_7_id_0'] ids +GET ['rank_9_id_1'] ids diff --git a/common.log b/common.log new file mode 100644 index 00000000..dae1c50f --- /dev/null +++ b/common.log @@ -0,0 +1,37 @@ +RoundRobin init for 4 rank +RoundRobin reserve rank 3 for actor on worker with rank 4 +GET rank_4_id_0 id from 3 rank +RoundRobin release rank 3 reserved for actor on worker with rank 4 +RoundRobin init for 6 rank +RoundRobin reserve rank 5 for actor on worker with rank 6 +GET rank_6_id_0 id from 5 rank +RoundRobin release rank 5 reserved for actor on worker with rank 6 +RoundRobin init for 9 rank +RoundRobin reserve rank 8 for actor on worker with rank 9 +GET rank_9_id_0 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 9 +RoundRobin init for 2 rank +RoundRobin reserve rank 8 for actor on worker with rank 2 +GET rank_2_id_0 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 2 +RoundRobin init for 3 rank +RoundRobin reserve rank 8 for actor on worker with rank 3 +GET rank_3_id_0 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 3 +RoundRobin reserve rank 8 for actor on worker with rank 4 +GET rank_4_id_1 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 4 +RoundRobin init for 5 rank +RoundRobin reserve rank 8 for actor on worker with rank 5 +GET rank_5_id_0 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 5 +RoundRobin reserve rank 8 for actor on worker with rank 6 +GET rank_6_id_1 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 6 +RoundRobin init for 7 rank +RoundRobin reserve rank 8 for actor on worker with rank 7 +GET rank_7_id_0 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 7 +RoundRobin reserve rank 8 for actor on worker with rank 9 +GET rank_9_id_1 id from 8 rank +RoundRobin release rank 8 reserved for actor on worker with rank 9 diff --git a/communication.log b/communication.log new file mode 100644 index 00000000..55977d85 --- /dev/null +++ b/communication.log @@ -0,0 +1,140 @@ +# 0 1 2 3 4 5 6 7 8 9 +ACTOR_CREATE: .---.---> . . . . . . . +ACTOR_EXECUTE: .---.---> . . . . . . . +GET: .---.---> . . . . . . . +TASK_DONE: . <---. . . . . . . . +ACTOR_CREATE: .---.---.---> . . . . . . +ACTOR_EXECUTE: .---.---.---> . . . . . . +GET: .---.---.---> . . . . . . +TASK_DONE: . <---.---. . . . . . . +PUT_DATA: .---.---.---.---> . . . . . +ACTOR_CREATE: .---.---.---.---.---> . . . . +PUT_OWNER: .---.---.---.---.---> . . . . +ACTOR_EXECUTE: .---.---.---.---.---> . . . . +GET: . . . . <---. . . . . +# 0 1 2 3 4 5 6 7 8 9 +ACTOR_CREATE: .---.---> . . . . . . . +ACTOR_EXECUTE: .---.---> . . . . . . . +GET: .---.---> . . . . . . . +TASK_DONE: . <---. . . . . . . . +ACTOR_CREATE: .---.---.---> . . . . . . +ACTOR_EXECUTE: .---.---.---> . . . . . . +GET: .---.---.---> . . . . . . +TASK_DONE: . <---.---. . . . . . . +PUT_DATA: .---.---.---.---> . . . . . +ACTOR_CREATE: .---.---.---.---.---> . . . . +PUT_OWNER: .---.---.---.---.---> . . . . +ACTOR_EXECUTE: .---.---.---.---.---> . . . . +GET: . . . . <---. . . . . +GET: .---.---.---.---.---> . . . . +PUT_DATA: . . . . .---> . . . . +TASK_DONE: . <---.---.---.---. . . . . +ACTOR_CREATE: .---.---.---.---.---.---> . . . +ACTOR_EXECUTE: .---.---.---.---.---.---> . . . +TASK_DONE: . <---.---.---.---.---. . . . +GET: .---.---.---.---.---.---> . . . +GET: .---.---.---.---.---.---> . . . +ACTOR_CREATE: .---.---.---.---.---.---.---> . . +ACTOR_CREATE: .---.---.---.---.---.---.---.---> . +ACTOR_EXECUTE: .---.---.---.---.---.---.---> . . +ACTOR_EXECUTE: .---.---.---.---.---.---.---.---> . +TASK_DONE: . <---.---.---.---.---.---. . . +GET: .---.---.---.---.---.---.---> . . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: .---.---.---.---.---.---.---.---> . +ACTOR_CREATE: .---.---.---.---.---.---.---.---.---> +ACTOR_CREATE: .---.---> . . . . . . . +ACTOR_EXECUTE: .---.---> . . . . . . . +ACTOR_EXECUTE: .---.---.---.---.---.---.---.---.---> +TASK_DONE: . <---. . . . . . . . +TASK_DONE: . <---.---.---.---.---.---.---.---. +GET: .---.---.---.---.---.---.---.---.---> +GET: .---.---> . . . . . . . +ACTOR_CREATE: .---.---.---> . . . . . . +EXECUTE: .---.---.---.---> . . . . . +ACTOR_EXECUTE: . . . <---. . . . . . +TASK_DONE: . <---.---. . . . . . . +GET: . . . <---. . . . . . +GET: .---.---.---.---> . . . . . +TASK_DONE: . <---.---.---. . . . . . +ACTOR_CREATE: .---.---.---.---.---> . . . . +EXECUTE: .---.---.---.---.---.---> . . . +ACTOR_EXECUTE: . . . . . <---. . . . +TASK_DONE: . <---.---.---.---. . . . . +GET: . . . . . <---. . . . +TASK_DONE: . <---.---.---.---.---. . . . +GET: .---.---.---.---.---.---> . . . +ACTOR_CREATE: .---.---.---.---.---.---.---> . . +ACTOR_EXECUTE: .---.---.---.---.---.---.---> . . +TASK_DONE: . <---.---.---.---.---.---. . . +GET: .---.---.---.---.---.---.---> . . +ACTOR_CREATE: .---.---.---.---.---.---.---.---> . +EXECUTE: .---.---.---.---.---.---.---.---.---> +ACTOR_EXECUTE: . . . . . . . . <---. +GET: . . . . . . . . <---. +GET: .---.---.---.---.---.---.---.---.---> +TASK_DONE: . <---.---.---.---.---.---.---. . +TASK_DONE: . <---.---.---.---.---.---.---.---. +EXECUTE: .---.---> . . . . . . . +ACTOR_EXECUTE: . . .---.---.---.---.---.---> . +GET: . . .---.---.---.---.---.---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: .---.---> . . . . . . . +TASK_DONE: . <---. . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---> . . . . . . +ACTOR_EXECUTE: . . . .---.---.---.---.---> . +GET: . . . .---.---.---.---.---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: .---.---.---> . . . . . . +TASK_DONE: . <---.---. . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---.---> . . . . . +ACTOR_EXECUTE: . . . . .---.---.---.---> . +GET: . . . . .---.---.---.---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +TASK_DONE: . <---.---.---. . . . . . +GET: .---.---.---.---> . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---.---.---> . . . . +ACTOR_EXECUTE: . . . . . .---.---.---> . +GET: . . . . . .---.---.---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: .---.---.---.---.---> . . . . +TASK_DONE: . <---.---.---.---. . . . . +GET_TASK_COUNT: .---> . . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---.---.---.---> . . . +ACTOR_EXECUTE: . . . . . . .---.---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: . . . . . . .---.---> . +TASK_DONE: . <---.---.---.---.---. . . . +GET: .---.---.---.---.---.---> . . . +GET_TASK_COUNT: .---> . . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---.---.---.---.---> . . +ACTOR_EXECUTE: . . . . . . . .---> . +TASK_DONE: . <---.---.---.---.---.---.---. . +GET: . . . . . . . .---> . +TASK_DONE: . <---.---.---.---.---.---. . . +GET: .---.---.---.---.---.---.---> . . +GET_TASK_COUNT: .---> . . . . . . . . +GET_TASK_COUNT: .---> . . . . . . . . +EXECUTE: .---.---.---.---.---.---.---.---.---> +ACTOR_EXECUTE: . . . . . . . . <---. +GET: . . . . . . . . <---. +GET: .---.---.---.---.---.---.---.---.---> +TASK_DONE: . <---.---.---.---.---.---.---. . +TASK_DONE: . <---.---.---.---.---.---.---.---. +GET_TASK_COUNT: .---> . . . . . . . . +CANCEL: .---> . . . . . . . . +CANCEL: .---.---> . . . . . . . +CANCEL: .---.---.---> . . . . . . +CANCEL: .---.---.---.---> . . . . . +CANCEL: .---.---.---.---.---> . . . . +CANCEL: .---.---.---.---.---.---> . . . +CANCEL: .---.---.---.---.---.---.---> . . +CANCEL: .---.---.---.---.---.---.---.---.---> +CANCEL: .---.---.---.---.---.---.---.---> . diff --git a/unidist/core/backends/mpi/core/common.py b/unidist/core/backends/mpi/core/common.py index 195a5b5d..3f6ab4bb 100755 --- a/unidist/core/backends/mpi/core/common.py +++ b/unidist/core/backends/mpi/core/common.py @@ -118,7 +118,8 @@ def __del__(self): def __getstate__(self): """Remove a reference to garbage collector for correct `pickle` serialization.""" attributes = self.__dict__.copy() - del attributes["_gc"] + if hasattr(self, "_gc"): + del attributes["_gc"] return attributes def base_data_id(self): diff --git a/unidist/core/backends/mpi/core/worker/loop.py b/unidist/core/backends/mpi/core/worker/loop.py index af1e5853..19df5bd5 100644 --- a/unidist/core/backends/mpi/core/worker/loop.py +++ b/unidist/core/backends/mpi/core/worker/loop.py @@ -4,7 +4,6 @@ """Worker MPI process task processing functionality.""" import asyncio -import sys from functools import wraps, partial try: @@ -126,6 +125,8 @@ async def worker_loop(): # Check pending actor requests also. task_store.check_pending_actor_tasks() + RequestStore.get_instance().check_pending_get_requests(request["id"]) + elif operation_type == common.Operation.PUT_OWNER: request = communication.recv_simple_operation(mpi_state.comm, source_rank) object_store.put_data_owner(request["id"], request["owner"]) diff --git a/unidist/core/backends/mpi/core/worker/task_store.py b/unidist/core/backends/mpi/core/worker/task_store.py index 86242125..e2105134 100644 --- a/unidist/core/backends/mpi/core/worker/task_store.py +++ b/unidist/core/backends/mpi/core/worker/task_store.py @@ -180,7 +180,7 @@ def unwrap_local_data_id(self, arg): self.request_worker_data(owner_rank, arg) return arg, True else: - raise ValueError("DataID is missing!") + raise ValueError("DataID is missing! {}".format(arg)) else: return arg, False @@ -357,6 +357,12 @@ def process_task_request(self, request): w_logger.debug("Is pending - {}".format(is_pending)) if is_pending or is_kw_pending: + current_rank = communication.MPIState.get_instance().rank + if isinstance(output_ids, (list, tuple)): + for output_id in output_ids: + ObjectStore.get_instance().put_data_owner(output_id, current_rank) + else: + ObjectStore.get_instance().put_data_owner(output_ids, current_rank) request["args"] = args request["kwargs"] = kwargs return request diff --git a/utils.log b/utils.log new file mode 100644 index 00000000..7212facb --- /dev/null +++ b/utils.log @@ -0,0 +1,20 @@ +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False +Cleanup list len - 0 +Cleanup counter 1, threshold reached - False diff --git a/worker_0.log b/worker_0.log new file mode 100644 index 00000000..73a6bdd4 --- /dev/null +++ b/worker_0.log @@ -0,0 +1,38 @@ +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready +CHECK ASYNC HANDLER - ready diff --git a/worker_2.log b/worker_2.log new file mode 100644 index 00000000..4e9f1035 --- /dev/null +++ b/worker_2.log @@ -0,0 +1,45 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_1'] +Is pending - False +- Start task execution - +Task evaluation time: 3.4999975468963385e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_1 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_1'] +Is pending - False +- Start task execution - +Task evaluation time: 3.6999990697950125e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_1 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_17'] +Is pending - False +- Start task execution - +Task evaluation time: 1.6999983927235007e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_17 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_26'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0020414000027813017 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_26 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_3.log b/worker_3.log new file mode 100644 index 00000000..c915a3ce --- /dev/null +++ b/worker_3.log @@ -0,0 +1,45 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_3'] +Is pending - False +- Start task execution - +Task evaluation time: 7.999915396794677e-07 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_3 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_3'] +Is pending - False +- Start task execution - +Task evaluation time: 2.4000037228688598e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_3 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_4_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 1.7999991541728377e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_4_id_0 id to 4 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_27'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0022320999996736646 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_27 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_4.log b/worker_4.log new file mode 100644 index 00000000..60187a8b --- /dev/null +++ b/worker_4.log @@ -0,0 +1,34 @@ +common.Operation processing - 3 +PUT (RECV) rank_0_id_4 id from 0 rank +Check pending tasks +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_4 id to 5 rank - PROCESSED +common.Operation processing - 3 +PUT (RECV) rank_0_id_4 id from 0 rank +Check pending tasks +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_4 id to 5 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_19'] +Is pending - False +- Start task execution - +Task evaluation time: 0.002019899999140762 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_19 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_28'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0029266999918036163 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_28 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_5.log b/worker_5.log new file mode 100644 index 00000000..9cf131b6 --- /dev/null +++ b/worker_5.log @@ -0,0 +1,52 @@ +common.Operation processing - 6 +common.Operation processing - 4 +PUT_OWNER rank_0_id_4 id is owned by 4 rank +common.Operation processing - 7 +REMOTE args: ['rank_0_id_4'] +REMOTE outputs: ['rank_0_id_6'] +Request rank_0_id_4 id from 4 worker rank +Is pending - True +common.Operation processing - 6 +common.Operation processing - 4 +PUT_OWNER rank_0_id_4 id is owned by 4 rank +common.Operation processing - 7 +REMOTE args: ['rank_0_id_4'] +REMOTE outputs: ['rank_0_id_6'] +Request rank_0_id_4 id from 4 worker rank +Is pending - True +common.Operation processing - 2 +Pending request rank_0_id_6 id to 0 rank +common.Operation processing - 3 +PUT (RECV) rank_0_id_4 id from 4 rank +Check pending tasks +Check pending actor tasks +REMOTE args: ['rank_0_id_4'] +REMOTE outputs: ['rank_0_id_6'] +Is pending - False +- Start task execution - +Task evaluation time: 5.199995939619839e-06 +- End task execution - +Send requested rank_0_id_6 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_6_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 2.7000060072168708e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_6_id_0 id to 6 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_29'] +Is pending - False +- Start task execution - +Task evaluation time: 0.002048599999397993 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_29 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_6.log b/worker_6.log new file mode 100644 index 00000000..cb48b33c --- /dev/null +++ b/worker_6.log @@ -0,0 +1,35 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [None] +REMOTE outputs: ['rank_0_id_8', 'rank_0_id_9'] +Is pending - False +- Start task execution - +Task evaluation time: 1.8999999156221747e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_8 id to 0 rank - PROCESSED +common.Operation processing - 2 +Send requested rank_0_id_9 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [None] +REMOTE outputs: ['rank_0_id_21'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0025775999965844676 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_21 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_30'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0032587000023340806 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_30 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_7.log b/worker_7.log new file mode 100644 index 00000000..e0977356 --- /dev/null +++ b/worker_7.log @@ -0,0 +1,34 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_12'] +Is pending - False +- Start task execution - +Task evaluation time: 3.40001133736223e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_12 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_23'] +Is pending - False +- Start task execution - +Task evaluation time: 2.0000006770715117e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_23 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_31'] +Is pending - False +- Start task execution - +Task evaluation time: 0.0037706000002799556 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_31 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_8.log b/worker_8.log new file mode 100644 index 00000000..1f54b4d3 --- /dev/null +++ b/worker_8.log @@ -0,0 +1,94 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_13'] +Is pending - False +- Start task execution - +Task evaluation time: 4.0000013541430235e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_13 id to 0 rank - PROCESSED +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_9_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 3.3999967854470015e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_9_id_0 id to 9 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_2_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 2.2000021999701858e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_2_id_0 id to 2 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_3_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 2.0000006770715117e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_3_id_0 id to 3 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_4_id_1'] +Is pending - False +- Start task execution - +Task evaluation time: 3.6999990697950125e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_4_id_1 id to 4 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_5_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 2.4000037228688598e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_5_id_0 id to 5 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_6_id_1'] +Is pending - False +- Start task execution - +Task evaluation time: 2.5000044843181968e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_6_id_1 id to 6 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_7_id_0'] +Is pending - False +- Start task execution - +Task evaluation time: 3.1999952625483274e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_7_id_0 id to 7 rank - PROCESSED +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_9_id_1'] +Is pending - False +- Start task execution - +Task evaluation time: 2.0000006770715117e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_9_id_1 id to 9 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop diff --git a/worker_9.log b/worker_9.log new file mode 100644 index 00000000..e2a7d285 --- /dev/null +++ b/worker_9.log @@ -0,0 +1,33 @@ +common.Operation processing - 6 +common.Operation processing - 7 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_16'] +Is pending - False +- Start task execution - +Task evaluation time: 1.7999991541728377e-06 +- End task execution - +Check pending actor tasks +common.Operation processing - 2 +Send requested rank_0_id_16 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_25'] +Is pending - False +- Start task execution - +Task evaluation time: 0.002860699998564087 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_25 id to 0 rank - PROCESSED +common.Operation processing - 1 +REMOTE args: [] +REMOTE outputs: ['rank_0_id_32'] +Is pending - False +- Start task execution - +Task evaluation time: 0.001774400006979704 +- End task execution - +Check pending tasks +common.Operation processing - 2 +Send requested rank_0_id_32 id to 0 rank - PROCESSED +common.Operation processing - 11 +Exit worker event loop