Issue877/Add config option for archive directory (#965)

Add config option for archive directory

README.rst

@@ -56,6 +56,7 @@ Contributors:
 * Patricia Grubel - `pagrubel <https://github.com/pagrubel>`_
 * Qiang Guan - `guanxyz <https://github.com/guanxyz>`_
 * Ragini Gupta - `raginigupta6 <https://github.com/raginigupta6>`_
+* Aaron R Hall - `arhall0 <https://github.com/arhall0>`_
 * Leah Howell - `Leahh02 <https://github.com/Leahh02>`_
 * Andres Quan - `aquan9 <https://github.com/aquan9>`_
 * Quincy Wofford - `qwofford <https://github.com/qwofford>`_

beeflow/common/config_driver.py

@@ -222,6 +222,7 @@ def filepath_completion_input(*pargs, **kwargs):
 DEFAULT_REDIS_IMAGE = join_path('/usr/projects/BEE/redis.tar.gz')
 
 DEFAULT_BEE_WORKDIR = join_path(HOME_DIR, '.beeflow')
+DEFAULT_BEE_ARCHIVE_DIR = join_path(DEFAULT_BEE_WORKDIR, 'archives')
 DEFAULT_BEE_DROPPOINT = join_path(HOME_DIR, '.beeflow/droppoint')
 USER = getpass.getuser()
 
@@ -242,6 +243,9 @@ def filepath_completion_input(*pargs, **kwargs):
 VALIDATOR.option('DEFAULT', 'bee_workdir', info='main BEE workdir',
                  default=DEFAULT_BEE_WORKDIR, validator=validation.make_dir)
 
+VALIDATOR.option('DEFAULT', 'bee_archive_dir', info='directory to store workflow archives',
+                 default=DEFAULT_BEE_ARCHIVE_DIR, validator=validation.make_dir)
+
 VALIDATOR.option('DEFAULT', 'bee_droppoint', info='BEE remote workflow drop point',
                  default=DEFAULT_BEE_DROPPOINT, validator=validation.make_dir)
 

beeflow/wf_manager/resources/wf_update.py

@@ -37,10 +37,9 @@ def archive_workflow(db, wf_id, final_state=None):
     db.workflows.update_workflow_state(wf_id, wf_state)
     wf_utils.update_wf_status(wf_id, wf_state)
 
-    bee_workdir = wf_utils.get_bee_workdir()
-    archive_dir = os.path.join(bee_workdir, 'archives')
+    archive_dir = bc.get('DEFAULT', 'bee_archive_dir')
     os.makedirs(archive_dir, exist_ok=True)
-    archive_path = f'../archives/{wf_id}.tgz'
+    archive_path = os.path.join(archive_dir, f'{wf_id}.tgz')
     # We use tar directly since tarfile is apparently very slow
     workflows_dir = wf_utils.get_workflows_dir()
     subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir)

ci/bee_config.sh

@@ -15,6 +15,7 @@ cat >> $BEE_CONFIG <<EOF
 # BEE CONFIGURATION FILE #
 [DEFAULT]
 bee_workdir = $BEE_WORKDIR
+bee_archive_dir = $BEE_WORKDIR/archives
 workload_scheduler = $WORKLOAD_SCHEDULER
 neo4j_image = $NEO4J_CONTAINER
 redis_image = $REDIS_CONTAINER