Merge pull request #732 from lanl/issue-741/query-updates

Make task manager background time interval configurable

beeflow/common/config_driver.py

@@ -250,6 +250,9 @@ def filepath_completion_input(*pargs, **kwargs):
                  info='container runtime to use for configuration')
 VALIDATOR.option('task_manager', 'runner_opts', attrs={'default': ''},
                  info='special runner options to pass to the runner opts')
+VALIDATOR.option('task_manager', 'background_interval', attrs={'default': 10},
+                 validator=int,
+                 info='interval at which the task manager processes queues and updates states')
 
 # Note: The special attrs keyword can include anything. One use case is for
 # storing a special 'init' function that can be used to initialize the

beeflow/task_manager.py

@@ -232,7 +232,8 @@ def process_queues():
 
 if "pytest" not in sys.modules:
     scheduler = BackgroundScheduler({'apscheduler.timezone': 'UTC'})
-    scheduler.add_job(func=process_queues, trigger="interval", seconds=30)
+    scheduler.add_job(func=process_queues, trigger="interval",
+                      seconds=bc.get('task_manager', 'background_interval'))
     scheduler.start()
 
     # This kills the scheduler when the process terminates

ci/bee_config.sh

@@ -17,6 +17,7 @@ max_restarts = 2
 [task_manager]
 container_runtime = Charliecloud
 runner_opts =
+background_interval = 2
 
 [charliecloud]
 image_mntdir = /tmp