From 3d093cba47e842c99c5747098c54dfce5bc83ad4 Mon Sep 17 00:00:00 2001 From: arunjose696 Date: Mon, 6 Mar 2023 09:52:05 -0600 Subject: [PATCH] using isend instead of send for operation --- unidist/core/backends/mpi/core/common.py | 2 +- .../core/backends/mpi/core/controller/api.py | 1 + .../backends/mpi/core/controller/common.py | 2 +- worker_2.log | 828 ------------------ worker_3.log | 773 ---------------- 5 files changed, 3 insertions(+), 1603 deletions(-) delete mode 100644 worker_2.log delete mode 100644 worker_3.log diff --git a/unidist/core/backends/mpi/core/common.py b/unidist/core/backends/mpi/core/common.py index 28610e915..195a5b5d2 100755 --- a/unidist/core/backends/mpi/core/common.py +++ b/unidist/core/backends/mpi/core/common.py @@ -133,7 +133,7 @@ def base_data_id(self): return DataID(self._id) -def get_logger(logger_name, file_name, activate=False): +def get_logger(logger_name, file_name, activate=True): """ Configure logger and get it's instance. diff --git a/unidist/core/backends/mpi/core/controller/api.py b/unidist/core/backends/mpi/core/controller/api.py index 48c37b619..302200831 100644 --- a/unidist/core/backends/mpi/core/controller/api.py +++ b/unidist/core/backends/mpi/core/controller/api.py @@ -237,6 +237,7 @@ def put(data): data_id = object_store.generate_data_id(garbage_collector) dest_rank = RoundRobin.get_instance().schedule_rank() object_store.put_data_owner(data_id, dest_rank) + object_store.put_data_size(data_id, data) push_data_directly_to_worker(dest_rank, data_id, data) logger.debug("PUT {} id".format(data_id._id)) diff --git a/unidist/core/backends/mpi/core/controller/common.py b/unidist/core/backends/mpi/core/controller/common.py index edcda42fb..74c29db32 100644 --- a/unidist/core/backends/mpi/core/controller/common.py +++ b/unidist/core/backends/mpi/core/controller/common.py @@ -228,7 +228,7 @@ def push_data_directly_to_worker(dest_rank, data_id, value): "id": data_id, "data": value, } - communication.send_operation( + communication.isend_complex_operation( mpi_state.comm, operation_type, operation_data, diff --git a/worker_2.log b/worker_2.log deleted file mode 100644 index 384c11b59..000000000 --- a/worker_2.log +++ /dev/null @@ -1,828 +0,0 @@ -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 8.610077202320099e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_1 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_4 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_4 id to 3 rank - PROCESSED -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_8', 'rank_0_id_9'] -Is pending - False -- Start task execution - -Task evaluation time: 9.541399776935577e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_8 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_9 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_13'] -Is pending - False -- Start task execution - -Task evaluation time: 8.502975106239319e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_13 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_17'] -Is pending - False -- Start task execution - -Task evaluation time: 8.353963494300842e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_17 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_19'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007361192256212234 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_19 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_21'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0005735033191740513 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_21 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 9.75094735622406e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_0 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 8.312053978443146e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_1 id to 3 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_28'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.1280720829963684e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_28 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_31 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_31 id to 3 rank - PROCESSED -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_35', 'rank_0_id_36'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.1816231310367584e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_35 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_36 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_40'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.952052116394043e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_40 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_44'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.0260922610759735e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_44 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_46'] -Is pending - False -- Start task execution - -Task evaluation time: 0.000543511938303709 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_46 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_48'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0005778721533715725 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_48 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_2'] -Is pending - False -Check pending actor tasks -- Start task execution - -common.Operation processing - 2 -Pending request rank_3_id_2 id to 3 rank -Task evaluation time: 5.0054347068071365 -- End task execution - -Send requested rank_3_id_2 id to 3 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_54'] -Is pending - False -- Start task execution - -Task evaluation time: 2.7958303689956665e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_54 id to 0 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_3'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 3.655906766653061e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_3_id_3 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 1.7476268112659454e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_4 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_5'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.7050264179706573e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_3_id_5 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_6'] -Is pending - False -- Start task execution - -Task evaluation time: 1.6940757632255554e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_6 id to 3 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_57 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_60'] -Is pending - False -- Start task execution - -Task evaluation time: 1.6689300537109375e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_62'] -Is pending - False -- Start task execution - -Task evaluation time: 3.003092106897384 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_62 id -Wait data rank_0_id_62 id is ready -common.Operation processing - 3 -PUT (RECV) rank_0_id_64 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_65'] -Is pending - False -- Start task execution - -Task evaluation time: 9.159557521343231e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_66'] -Is pending - False -- Start task execution - -Task evaluation time: 7.897615432739258e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_67'] -Is pending - False -- Start task execution - -Task evaluation time: 8.00006091594696e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_68'] -Is pending - False -- Start task execution - -Task evaluation time: 7.846392691135406e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_65 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_66 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_67 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_68 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_71', 'rank_0_id_72'] -Is pending - False -- Start task execution - -Task evaluation time: 9.075738489627838e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_71 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_72 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_75'] -Is pending - False -- Start task execution - -Task evaluation time: 3.00309204030782 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_75'] -REMOTE outputs: ['rank_0_id_76'] -Is pending - False -- Start task execution - -Task evaluation time: 8.898787200450897e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_76 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_77'] -Is pending - False -- Start task execution - -Exception - division by zero -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_77 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_79'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007768003270030022 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_79 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_81'] -Is pending - False -- Start task execution - -Task evaluation time: 7.557682693004608e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_81 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 3 -PUT (RECV) rank_0_id_0 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_3'] -Is pending - False -- Start task execution - -Task evaluation time: 1.6987323760986328e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_5'] -Is pending - False -- Start task execution - -Task evaluation time: 3.003088278695941 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_5 id -Wait data rank_0_id_5 id is ready -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 1.0039657354354858e-06 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_0 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_6', 'rank_0_id_7'] -Is pending - False -- Start task execution - -Task evaluation time: 8.01868736743927e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_6 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_7 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_10 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_10'] -REMOTE outputs: ['rank_0_id_11'] -Is pending - False -- Start task execution - -Task evaluation time: 1.0221265256404877e-06 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_11 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: [None] -Is pending - False -- Start task execution - -Task evaluation time: 1.2777745723724365e-06 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_15'] -Is pending - False -- Start task execution - -Task evaluation time: 7.469207048416138e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_15 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 9.168870747089386e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_3_id_0 id to 3 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_17'] -Is pending - False -- Start task execution - -Task evaluation time: 8.847564458847046e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_17 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 1.0277144610881805e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_1 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_4 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_4 id to 3 rank - PROCESSED -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_8', 'rank_0_id_9'] -Is pending - False -- Start task execution - -Task evaluation time: 7.650814950466156e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_8 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_9 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_13'] -Is pending - False -- Start task execution - -Task evaluation time: 8.530914783477783e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_13 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_17'] -Is pending - False -- Start task execution - -Task evaluation time: 8.051283657550812e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_17 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_19'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0005851546302437782 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_19 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_21'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0005111629143357277 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_21 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 7.627531886100769e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_0 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 7.511116564273834e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_1 id to 3 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_28'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.676846295595169e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_28 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_31 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_31 id to 3 rank - PROCESSED -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -CHECK ASYNC HANDLER - ready -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_35', 'rank_0_id_36'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.007000148296356e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_35 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_36 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_40'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.7350539565086365e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_40 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_44'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.7327256500720978e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_44 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_46'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007281741127371788 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_46 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_48'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0006989450193941593 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_48 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_3_id_2'] -Is pending - False -Check pending actor tasks -- Start task execution - -common.Operation processing - 2 -Pending request rank_3_id_2 id to 3 rank -Task evaluation time: 5.005361226852983 -- End task execution - -Send requested rank_3_id_2 id to 3 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_54'] -Is pending - False -- Start task execution - -Task evaluation time: 2.5099143385887146e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_54 id to 0 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_3'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 3.552064299583435e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_3_id_3 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 1.8007121980190277e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_4 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_5'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.5993213057518005e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_3_id_5 id to 3 rank - PROCESSED -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_3_id_6'] -Is pending - False -- Start task execution - -Task evaluation time: 1.701992005109787e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_3_id_6 id to 3 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_57 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_60'] -Is pending - False -- Start task execution - -Task evaluation time: 1.5641562640666962e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_62'] -Is pending - False -- Start task execution - -Task evaluation time: 3.003092911094427 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_62 id -Wait data rank_0_id_62 id is ready -common.Operation processing - 3 -PUT (RECV) rank_0_id_64 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_65'] -Is pending - False -- Start task execution - -Task evaluation time: 8.330680429935455e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_66'] -Is pending - False -- Start task execution - -Task evaluation time: 6.943009793758392e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_67'] -Is pending - False -- Start task execution - -Task evaluation time: 6.849877536296844e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_64'] -REMOTE outputs: ['rank_0_id_68'] -Is pending - False -- Start task execution - -Task evaluation time: 6.803311407566071e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_65 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_66 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_67 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_68 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_71', 'rank_0_id_72'] -Is pending - False -- Start task execution - -Task evaluation time: 7.632188498973846e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_71 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_72 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_75'] -Is pending - False -- Start task execution - -Task evaluation time: 3.0031022960320115 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_75'] -REMOTE outputs: ['rank_0_id_76'] -Is pending - False -- Start task execution - -Task evaluation time: 7.953494787216187e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_76 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_77'] -Is pending - False -- Start task execution - -Exception - division by zero -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_77 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_79'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007683439180254936 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_79 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_81'] -Is pending - False -- Start task execution - -Task evaluation time: 6.319023668766022e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_81 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop diff --git a/worker_3.log b/worker_3.log deleted file mode 100644 index 1f6203826..000000000 --- a/worker_3.log +++ /dev/null @@ -1,773 +0,0 @@ -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_3'] -Is pending - False -- Start task execution - -Task evaluation time: 1.091044396162033e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_3 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 4 -PUT_OWNER rank_0_id_4 id is owned by 2 rank -common.Operation processing - 7 -REMOTE args: ['rank_0_id_4'] -REMOTE outputs: ['rank_0_id_6'] -Request rank_0_id_4 id from 2 worker rank -Is pending - True -common.Operation processing - 2 -Pending request rank_0_id_6 id to 0 rank -common.Operation processing - 3 -PUT (RECV) rank_0_id_4 id from 2 rank -Check pending tasks -Check pending actor tasks -REMOTE args: ['rank_0_id_4'] -REMOTE outputs: ['rank_0_id_6'] -Is pending - False -- Start task execution - -Task evaluation time: 1.4333054423332214e-05 -- End task execution - -Send requested rank_0_id_6 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_12'] -Is pending - False -- Start task execution - -Task evaluation time: 8.577480912208557e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_12 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_16'] -Is pending - False -- Start task execution - -Task evaluation time: 7.641501724720001e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_16 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 7.851049304008484e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_2_id_0 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 7.287599146366119e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_2_id_1 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_23'] -Is pending - False -- Start task execution - -Task evaluation time: 6.933696568012238e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_23 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_25'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0006418293341994286 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_25 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_26'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007013240829110146 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_26 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_30'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.700129359960556e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_30 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 4 -PUT_OWNER rank_0_id_31 id is owned by 2 rank -common.Operation processing - 7 -REMOTE args: ['rank_0_id_31'] -REMOTE outputs: ['rank_0_id_33'] -Request rank_0_id_31 id from 2 worker rank -Is pending - True -common.Operation processing - 2 -Pending request rank_0_id_33 id to 0 rank -common.Operation processing - 3 -PUT (RECV) rank_0_id_31 id from 2 rank -Check pending tasks -Check pending actor tasks -REMOTE args: ['rank_0_id_31'] -REMOTE outputs: ['rank_0_id_33'] -Is pending - False -- Start task execution - -Task evaluation time: 3.0351802706718445e-06 -- End task execution - -Send requested rank_0_id_33 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_39'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.7960555851459503e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_39 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_43'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.7760321497917175e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_43 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_2'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.6689300537109375e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_2_id_2 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_3'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.6880221664905548e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_2_id_3 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_50'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.5213154256343842e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_50 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_52'] -Is pending - False -- Start task execution - -Task evaluation time: 5.006451622117311 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_52 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_55'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0013913167640566826 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_56'] -Is pending - False -- Start task execution - -Task evaluation time: 0.000949472188949585 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_55 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_56 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_58'] -Is pending - False -- Start task execution - -Task evaluation time: 6.570480763912201e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_61'] -Is pending - False -- Start task execution - -Task evaluation time: 6.309710443019867e-07 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_61 id -Wait data rank_0_id_61 id is ready -common.Operation processing - 5 -WAIT for rank_0_id_61 id -Wait data rank_0_id_61 id is ready -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_63'] -Is pending - False -- Start task execution - -Task evaluation time: 9.038485586643219e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_63 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_69', 'rank_0_id_70'] -Is pending - False -- Start task execution - -Task evaluation time: 7.60890543460846e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_69 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_70 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_73 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_73'] -REMOTE outputs: ['rank_0_id_74'] -Is pending - False -- Start task execution - -Task evaluation time: 1.1469237506389618e-06 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_74 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: [None] -Is pending - False -- Start task execution - -Task evaluation time: 8.870847523212433e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_78'] -Is pending - False -- Start task execution - -Task evaluation time: 6.332993507385254e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_78 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_2_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 6.868503987789154e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_2_id_4 id to 2 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_80'] -Is pending - False -- Start task execution - -Task evaluation time: 6.658956408500671e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_80 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 8.470378816127777e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 7.557682693004608e-07 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_4 id -Wait data rank_0_id_4 id is ready -common.Operation processing - 5 -WAIT for rank_0_id_4 id -Wait data rank_0_id_4 id is ready -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 3 -PUT (RECV) rank_0_id_1 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_1'] -REMOTE outputs: ['rank_0_id_2'] -Is pending - False -- Start task execution - -Task evaluation time: 1.0356307029724121e-06 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_1'] -REMOTE outputs: ['rank_0_id_3'] -Is pending - False -- Start task execution - -Task evaluation time: 7.511116564273834e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_1'] -REMOTE outputs: ['rank_0_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 6.472691893577576e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_1'] -REMOTE outputs: ['rank_0_id_5'] -Is pending - False -- Start task execution - -Task evaluation time: 6.700865924358368e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_2 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_3 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_4 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_5 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_8', 'rank_0_id_9'] -Is pending - False -- Start task execution - -Task evaluation time: 7.417984306812286e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_8 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_9 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_12'] -Is pending - False -- Start task execution - -Task evaluation time: 3.003110804129392 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_12'] -REMOTE outputs: ['rank_0_id_13'] -Is pending - False -- Start task execution - -Task evaluation time: 9.457580745220184e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_13 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_14'] -Is pending - False -- Start task execution - -Exception - division by zero -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_14 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_16'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0009354320354759693 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_16 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_18'] -Is pending - False -- Start task execution - -Task evaluation time: 7.59027898311615e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_18 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_3'] -Is pending - False -- Start task execution - -Task evaluation time: 1.2302771210670471e-06 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_3 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 4 -PUT_OWNER rank_0_id_4 id is owned by 2 rank -common.Operation processing - 7 -REMOTE args: ['rank_0_id_4'] -REMOTE outputs: ['rank_0_id_6'] -Request rank_0_id_4 id from 2 worker rank -Is pending - True -common.Operation processing - 2 -Pending request rank_0_id_6 id to 0 rank -common.Operation processing - 3 -PUT (RECV) rank_0_id_4 id from 2 rank -Check pending tasks -Check pending actor tasks -REMOTE args: ['rank_0_id_4'] -REMOTE outputs: ['rank_0_id_6'] -Is pending - False -- Start task execution - -Task evaluation time: 1.7071142792701721e-06 -- End task execution - -Send requested rank_0_id_6 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_12'] -Is pending - False -- Start task execution - -Task evaluation time: 8.279457688331604e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_12 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_16'] -Is pending - False -- Start task execution - -Task evaluation time: 7.9302117228508e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_16 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_0'] -Is pending - False -- Start task execution - -Task evaluation time: 7.9302117228508e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_2_id_0 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_1'] -Is pending - False -- Start task execution - -Task evaluation time: 8.540228009223938e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_2_id_1 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_23'] -Is pending - False -- Start task execution - -Task evaluation time: 7.627531886100769e-07 -- End task execution - -Check pending actor tasks -common.Operation processing - 2 -Send requested rank_0_id_23 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_25'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0005705156363546848 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_25 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_26'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0007081101648509502 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_26 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_30'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.7923302948474884e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_30 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 4 -PUT_OWNER rank_0_id_31 id is owned by 2 rank -common.Operation processing - 7 -REMOTE args: ['rank_0_id_31'] -REMOTE outputs: ['rank_0_id_33'] -Request rank_0_id_31 id from 2 worker rank -Is pending - True -common.Operation processing - 2 -Pending request rank_0_id_33 id to 0 rank -common.Operation processing - 3 -PUT (RECV) rank_0_id_31 id from 2 rank -Check pending tasks -Check pending actor tasks -REMOTE args: ['rank_0_id_31'] -REMOTE outputs: ['rank_0_id_33'] -Is pending - False -- Start task execution - -Task evaluation time: 2.522021532058716e-06 -- End task execution - -Send requested rank_0_id_33 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_39'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.8463470041751862e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_39 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_43'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 1.8528662621974945e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_43 id to 0 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_2'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.5150366127490997e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_2_id_2 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_2_id_3'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.5480985641479492e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_2_id_3 id to 2 rank - PROCESSED -common.Operation processing - 6 -common.Operation processing - 7 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_50'] -Is pending - False -Check pending actor tasks -- Start task execution - -Task evaluation time: 2.3460015654563904e-06 -- End task execution - -common.Operation processing - 2 -Send requested rank_0_id_50 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_52'] -Is pending - False -- Start task execution - -Task evaluation time: 5.0064789592288435 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_52 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_55'] -Is pending - False -- Start task execution - -Task evaluation time: 0.001236163079738617 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_56'] -Is pending - False -- Start task execution - -Task evaluation time: 0.0010133758187294006 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_55 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_56 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_58'] -Is pending - False -- Start task execution - -Task evaluation time: 7.059425115585327e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_61'] -Is pending - False -- Start task execution - -Task evaluation time: 7.580965757369995e-07 -- End task execution - -Check pending tasks -common.Operation processing - 5 -WAIT for rank_0_id_61 id -Wait data rank_0_id_61 id is ready -common.Operation processing - 5 -WAIT for rank_0_id_61 id -Wait data rank_0_id_61 id is ready -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_63'] -Is pending - False -- Start task execution - -Task evaluation time: 8.381903171539307e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_63 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_69', 'rank_0_id_70'] -Is pending - False -- Start task execution - -Task evaluation time: 7.436610758304596e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_69 id to 0 rank - PROCESSED -common.Operation processing - 2 -Send requested rank_0_id_70 id to 0 rank - PROCESSED -common.Operation processing - 3 -PUT (RECV) rank_0_id_73 id from 0 rank -Check pending tasks -Check pending actor tasks -common.Operation processing - 1 -REMOTE args: ['rank_0_id_73'] -REMOTE outputs: ['rank_0_id_74'] -Is pending - False -- Start task execution - -Task evaluation time: 2.328772097826004e-06 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_74 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: [None] -Is pending - False -- Start task execution - -Task evaluation time: 9.220093488693237e-07 -- End task execution - -Check pending tasks -common.Operation processing - 1 -REMOTE args: [] -REMOTE outputs: ['rank_0_id_78'] -Is pending - False -- Start task execution - -Task evaluation time: 6.416812539100647e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_78 id to 0 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_2_id_4'] -Is pending - False -- Start task execution - -Task evaluation time: 7.711350917816162e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_2_id_4 id to 2 rank - PROCESSED -common.Operation processing - 1 -REMOTE args: [None] -REMOTE outputs: ['rank_0_id_80'] -Is pending - False -- Start task execution - -Task evaluation time: 6.752088665962219e-07 -- End task execution - -Check pending tasks -common.Operation processing - 2 -Send requested rank_0_id_80 id to 0 rank - PROCESSED -common.Operation processing - 11 -Exit worker event loop