Skip to content

Commit

Permalink
fix lint issues
Browse files Browse the repository at this point in the history
  • Loading branch information
tphung3 committed Dec 16, 2024
1 parent 6057360 commit c9547d3
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 10 deletions.
22 changes: 13 additions & 9 deletions parsl/executors/taskvine/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,12 +178,13 @@ def __init__(self,
# Path to directory that holds all tasks' data and results.
self._function_data_dir = ""

# Mapping of function names to function details
# Currently the values include function objects, path to serialized functions, path to serialized function contexts, and whether functions are serialized
# Helpful to detect inconsistencies in serverless functions
# Helpful to deduplicate the same function
# Mapping of function names to function details.
# Currently the values include function objects, path to serialized functions,
# path to serialized function contexts, and whether functions are serialized.
# Helpful to detect inconsistencies in serverless functions.
# Helpful to deduplicate the same function.
self._map_func_names_to_func_details = {}

# Helper scripts to prepare package tarballs for Parsl apps
self._package_analyze_script = shutil.which("poncho_package_analyze")
self._package_create_script = shutil.which("poncho_package_create")
Expand Down Expand Up @@ -350,11 +351,14 @@ def submit(self, func, resource_specification, *args, **kwargs):
exec_mode = 'regular'

if exec_mode == 'serverless':
if func.__name__ not in self._map_func_names_to_func_details or 'func_obj' not in self._map_func_names_to_func_details[func.__name__]:
if func.__name__ not in self._map_func_names_to_func_details or\
'func_obj' not in self._map_func_names_to_func_details[func.__name__]:
self._map_func_names_to_func_details[func.__name__] = {'func_obj': func}
else:
if id(func) != id(self._map_func_names_to_func_details[func.__name__]['func_obj']):
logger.warning('Inconsistency in a serverless function call detected. A function name cannot point to two different function objects. Falling back to executing it as a regular task.')
logger.warning('Inconsistency in a serverless function call detected.\
A function name cannot point to two different function objects.\
Falling back to executing it as a regular task.')
exec_mode = 'regular'

# Detect resources and features of a submitted Parsl app
Expand Down Expand Up @@ -425,7 +429,8 @@ def submit(self, func, resource_specification, *args, **kwargs):
# Get path to files that will contain the pickled function,
# arguments, result, and map of input and output files
if exec_mode == 'serverless':
if func.__name__ not in self._map_func_names_to_func_details or 'function_file' not in self._map_func_names_to_func_details[func.__name__]:
if func.__name__ not in self._map_func_names_to_func_details or\
'function_file' not in self._map_func_names_to_func_details[func.__name__]:
function_file = os.path.join(self._function_data_dir.name, func.__name__, 'function')
self._map_func_names_to_func_details[func.__name__] = {'function_file': function_file, 'is_serialized': False}
os.makedirs(os.path.join(self._function_data_dir.name, func.__name__))
Expand All @@ -449,7 +454,6 @@ def submit(self, func, resource_specification, *args, **kwargs):
else:
function_context_file = self._map_func_names_to_func_details[func.__name__]['function_context_file']


logger.debug("Creating executor task {} with function at: {}, argument at: {}, \
and result to be found at: {}".format(executor_task_id, function_file, argument_file, result_file))

Expand Down
2 changes: 1 addition & 1 deletion parsl/executors/taskvine/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
from parsl.executors.taskvine import exec_parsl_function
from parsl.executors.taskvine.utils import VineTaskToParsl, run_parsl_function
from parsl.process_loggers import wrap_with_logs
from parsl.utils import setproctitle
from parsl.serialize import deserialize, serialize
from parsl.utils import setproctitle

try:
from ndcctools.taskvine import FunctionCall, Manager, Task, cvine
Expand Down

0 comments on commit c9547d3

Please sign in to comment.