From fbb15e91b2df6a26ed360bef8f4232e7b0337873 Mon Sep 17 00:00:00 2001 From: Leah Howell Date: Fri, 20 Sep 2024 14:07:19 -0600 Subject: [PATCH 01/24] add support for custom DAG locations and versioning --- beeflow/client/bee_client.py | 5 +++-- beeflow/common/deps/neo4j_manager.py | 4 +--- beeflow/common/gdb/generate_graph.py | 20 +++++++++++++++----- beeflow/wf_manager/resources/wf_utils.py | 4 ++-- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 1dc03a9b..978cf3c9 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -625,9 +625,10 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'), @app.command() -def dag(wf_id: str = typer.Argument(..., callback=match_short_id)): +def dag(wf_id: str = typer.Argument(..., callback=match_short_id), + output_dir: pathlib.Path = typer.Argument(..., help='Path to the where the dag directory will be')): """Export a DAG of the workflow to a GraphML file.""" - wf_utils.export_dag(wf_id) + wf_utils.export_dag(wf_id, output_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) diff --git a/beeflow/common/deps/neo4j_manager.py b/beeflow/common/deps/neo4j_manager.py index 298710cd..9a8b6556 100644 --- a/beeflow/common/deps/neo4j_manager.py +++ b/beeflow/common/deps/neo4j_manager.py @@ -23,8 +23,7 @@ run_dir = mount_dir + '/run' certs_dir = mount_dir + '/certificates' confs_dir = mount_dir + "/conf" -dags_dir = os.path.join(bee_workdir, 'dags') -graphmls_dir = dags_dir + "/graphmls" +graphmls_dir = os.path.join(bee_workdir, 'graphmls') container_path = container_manager.get_container_dir('neo4j') log = bee_logging.setup('neo4j') @@ -61,7 +60,6 @@ def setup_mounts(): os.makedirs(mount_dir, exist_ok=True) os.makedirs(certs_dir, exist_ok=True) os.makedirs(run_dir, exist_ok=True) - os.makedirs(dags_dir, exist_ok=True) os.makedirs(graphmls_dir, exist_ok=True) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index aa0cb73f..a065ad17 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -1,21 +1,31 @@ """Module to make a png of a graph from a graphml file.""" import os +import shutil import networkx as nx import graphviz from beeflow.common import paths bee_workdir = paths.workdir() -dags_dir = os.path.join(bee_workdir, 'dags') -graphmls_dir = dags_dir + "/graphmls" +graphmls_dir = os.path.join(bee_workdir, 'graphmls') -def generate_viz(wf_id): +def generate_viz(wf_id, output_dir): """Generate a PNG of a workflow graph from a GraphML file.""" short_id = wf_id[:6] graphml_path = graphmls_dir + "/" + short_id + ".graphml" - output_path = dags_dir + "/" + short_id + dags_dir = output_dir + "/" + short_id + "_dags" + os.makedirs(dags_dir, exist_ok=True) + + output_path = dags_dir + "/" + short_id + ".png" + if os.path.exists(output_path): + i = 1 + backup_path = f'{dags_dir}/{short_id}_v{i}.png' + while os.path.exists(backup_path): + i += 1 + backup_path = f'{dags_dir}/{short_id}_v{i}.png' + shutil.copy(output_path, backup_path) # Load the GraphML file using NetworkX graph = nx.read_graphml(graphml_path) @@ -29,7 +39,7 @@ def generate_viz(wf_id): # Render the graph and save as PNG png_data = dot.pipe(format='png') - with open(output_path + ".png", "wb") as png_file: + with open(output_path, "wb") as png_file: png_file.write(png_data) diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index efccb2ba..a654789b 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -294,12 +294,12 @@ def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, start_workflow(wf_id) -def export_dag(wf_id): +def export_dag(wf_id, output_dir): """Export the DAG of the workflow.""" wfi = get_workflow_interface(wf_id) wfi.export_graphml() update_graphml(wf_id) - generate_viz(wf_id) + generate_viz(wf_id, output_dir) def start_workflow(wf_id): From e01710707cc34b723dc1ab2650fe5ca6bf07d675 Mon Sep 17 00:00:00 2001 From: Leah Howell Date: Fri, 20 Sep 2024 14:50:48 -0600 Subject: [PATCH 02/24] added support for saving final dag png in workflow archive --- beeflow/wf_manager/resources/wf_update.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 6202e32f..04ee183d 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -15,6 +15,7 @@ from beeflow.common.db import wfm_db from beeflow.common.db.bdb import connect_db from beeflow.common.config_driver import BeeConfig as bc +from beeflow.wf_manager.resources import wf_utils log = bee_logging.setup(__name__) db_path = wf_utils.get_db_path() @@ -35,6 +36,7 @@ def archive_workflow(db, wf_id, final_state=None): archive_dir = os.path.join(bee_workdir, 'archives') os.makedirs(archive_dir, exist_ok=True) archive_path = f'../archives/{wf_id}.tgz' + wf_utils.export_dag(wf_id, archive_dir) # We use tar directly since tarfile is apparently very slow workflows_dir = wf_utils.get_workflows_dir() subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir) From aeea7af688403fc26cafcaa0223b47e66f3812a0 Mon Sep 17 00:00:00 2001 From: Leah Howell Date: Fri, 20 Sep 2024 15:03:00 -0600 Subject: [PATCH 03/24] updated documentation to include information about paths --- docs/sphinx/commands.rst | 5 +++-- docs/sphinx/visualization.rst | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/sphinx/commands.rst b/docs/sphinx/commands.rst index 07ff348b..8fdc3950 100644 --- a/docs/sphinx/commands.rst +++ b/docs/sphinx/commands.rst @@ -103,9 +103,10 @@ Arguments: ``beeflow reexecute``: Reexecute an archived workflow. Arguments: - WF_ID [required] + - WF_ID [required] + - OUTPUT_DIR, Path to the where the DAG directory will be [required] -``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to ~/.beeflow/dags. See :ref:`workflow-visualization` for more information. +``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID_dags. See :ref:`workflow-visualization` for more information. Arguments: WF_ID [required] diff --git a/docs/sphinx/visualization.rst b/docs/sphinx/visualization.rst index 80cb4998..a105bd39 100644 --- a/docs/sphinx/visualization.rst +++ b/docs/sphinx/visualization.rst @@ -4,7 +4,7 @@ Workflow Visualization ********************** BEE includes a simple command for viewing BEE workflows. By using the ``beeflow -dag $ID`` command, you can view the directed acyclic graph (DAG) of any submitted +dag $ID $OUTPUT_DIR`` command, you can view the directed acyclic graph (DAG) of any submitted workflow. Creating DAGs @@ -13,8 +13,7 @@ Creating DAGs The dag command can be run at any point of the workflow, and can be run multiple times. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The -DAGs are exported in PNG format to ~/.beeflow/dags. They follow the naming -convention ``$ID.png`` +DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID_dags. Example DAG =========== From 65a37a9611fb30ca1d4aee580f65bfec04274fc3 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 10:03:16 -0600 Subject: [PATCH 04/24] fixed input type for export_dag --- beeflow/client/bee_client.py | 10 ++++++++++ beeflow/common/gdb/graphml_key_updater.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 978cf3c9..3cef3975 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -628,6 +628,16 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'), def dag(wf_id: str = typer.Argument(..., callback=match_short_id), output_dir: pathlib.Path = typer.Argument(..., help='Path to the where the dag directory will be')): """Export a DAG of the workflow to a GraphML file.""" + output_dir = output_dir.resolve() + + # Make sure output_dir is an absolute path and exists + output_dir = os.path.expanduser(output_dir) + output_dir = os.path.abspath(output_dir) + if not os.path.exists(output_dir): + error_exit(f"Path for dag directory \"{output_dir}\" doesn't exist") + + # output_dir must be a string + output_dir = str(output_dir) wf_utils.export_dag(wf_id, output_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) diff --git a/beeflow/common/gdb/graphml_key_updater.py b/beeflow/common/gdb/graphml_key_updater.py index d18b9548..4eed142a 100644 --- a/beeflow/common/gdb/graphml_key_updater.py +++ b/beeflow/common/gdb/graphml_key_updater.py @@ -6,8 +6,8 @@ from beeflow.common import paths bee_workdir = paths.workdir() -dags_dir = os.path.join(bee_workdir, 'dags') -graphmls_dir = dags_dir + "/graphmls" +graphmls_dir = os.path.join(bee_workdir, 'graphmls') + expected_keys = {"id", "name", "state", "class", "type", "value", "source", "workflow_id", "base_command", "stdout", "stderr", "default", From db228d61d8d64a46980f74ebf09dc89cbf7a6270 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 10:03:43 -0600 Subject: [PATCH 05/24] fixed final dag location --- beeflow/wf_manager/resources/wf_update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 04ee183d..94bb295d 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -36,9 +36,9 @@ def archive_workflow(db, wf_id, final_state=None): archive_dir = os.path.join(bee_workdir, 'archives') os.makedirs(archive_dir, exist_ok=True) archive_path = f'../archives/{wf_id}.tgz' - wf_utils.export_dag(wf_id, archive_dir) # We use tar directly since tarfile is apparently very slow workflows_dir = wf_utils.get_workflows_dir() + wf_utils.export_dag(wf_id, workflow_dir) subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir) remove_wf_dir = bc.get('DEFAULT', 'delete_completed_workflow_dirs') if remove_wf_dir: From 605234e9cf172953385ba599bf964d4890985a39 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 10:13:37 -0600 Subject: [PATCH 06/24] fixed all my linting errors for my files so far --- beeflow/client/bee_client.py | 3 ++- beeflow/common/gdb/generate_graph.py | 2 +- beeflow/wf_manager/resources/wf_update.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 3cef3975..7cfd7d08 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -626,7 +626,8 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'), @app.command() def dag(wf_id: str = typer.Argument(..., callback=match_short_id), - output_dir: pathlib.Path = typer.Argument(..., help='Path to the where the dag directory will be')): + output_dir: pathlib.Path = typer.Argument(..., + help='Path to the where the dag directory will be')): """Export a DAG of the workflow to a GraphML file.""" output_dir = output_dir.resolve() diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index a065ad17..2b421aa3 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -15,7 +15,7 @@ def generate_viz(wf_id, output_dir): """Generate a PNG of a workflow graph from a GraphML file.""" short_id = wf_id[:6] graphml_path = graphmls_dir + "/" + short_id + ".graphml" - dags_dir = output_dir + "/" + short_id + "_dags" + dags_dir = output_dir + "/" + short_id + "_dags" os.makedirs(dags_dir, exist_ok=True) output_path = dags_dir + "/" + short_id + ".png" diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 94bb295d..335057cf 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -15,7 +15,6 @@ from beeflow.common.db import wfm_db from beeflow.common.db.bdb import connect_db from beeflow.common.config_driver import BeeConfig as bc -from beeflow.wf_manager.resources import wf_utils log = bee_logging.setup(__name__) db_path = wf_utils.get_db_path() From e72e1be3b037a76d661a90c8674c6bac42cb34d5 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 11:10:10 -0600 Subject: [PATCH 07/24] added an option to not make the dags dir, set to false for archive --- beeflow/client/bee_client.py | 6 ++++-- beeflow/common/gdb/generate_graph.py | 10 +++++++--- beeflow/wf_manager/resources/wf_update.py | 2 +- beeflow/wf_manager/resources/wf_utils.py | 4 ++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 7cfd7d08..258491c7 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -627,7 +627,9 @@ def reexecute(wf_name: str = typer.Argument(..., help='The workflow name'), @app.command() def dag(wf_id: str = typer.Argument(..., callback=match_short_id), output_dir: pathlib.Path = typer.Argument(..., - help='Path to the where the dag directory will be')): + help='Path to the where the dag output will be'), + no_dag_dir: bool = typer.Option(False, '--no-dag-dir', + help='do not make a subdirectory within ouput_dir for the dags')): """Export a DAG of the workflow to a GraphML file.""" output_dir = output_dir.resolve() @@ -639,7 +641,7 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # output_dir must be a string output_dir = str(output_dir) - wf_utils.export_dag(wf_id, output_dir) + wf_utils.export_dag(wf_id, output_dir, no_dag_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index 2b421aa3..75e87a4a 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -11,12 +11,16 @@ graphmls_dir = os.path.join(bee_workdir, 'graphmls') -def generate_viz(wf_id, output_dir): +def generate_viz(wf_id, output_dir, no_dag_dir): """Generate a PNG of a workflow graph from a GraphML file.""" short_id = wf_id[:6] graphml_path = graphmls_dir + "/" + short_id + ".graphml" - dags_dir = output_dir + "/" + short_id + "_dags" - os.makedirs(dags_dir, exist_ok=True) + + if no_dag_dir: + dags_dir = output_dir + else: + dags_dir = output_dir + "/" + short_id + "_dags" + os.makedirs(dags_dir, exist_ok=True) output_path = dags_dir + "/" + short_id + ".png" if os.path.exists(output_path): diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 335057cf..4772655e 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -37,7 +37,7 @@ def archive_workflow(db, wf_id, final_state=None): archive_path = f'../archives/{wf_id}.tgz' # We use tar directly since tarfile is apparently very slow workflows_dir = wf_utils.get_workflows_dir() - wf_utils.export_dag(wf_id, workflow_dir) + wf_utils.export_dag(wf_id, workflow_dir, no_dag_dir=True) subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir) remove_wf_dir = bc.get('DEFAULT', 'delete_completed_workflow_dirs') if remove_wf_dir: diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index a654789b..e75211c5 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -294,12 +294,12 @@ def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, start_workflow(wf_id) -def export_dag(wf_id, output_dir): +def export_dag(wf_id, output_dir, no_dag_dir): """Export the DAG of the workflow.""" wfi = get_workflow_interface(wf_id) wfi.export_graphml() update_graphml(wf_id) - generate_viz(wf_id, output_dir) + generate_viz(wf_id, output_dir, no_dag_dir) def start_workflow(wf_id): From b9cafbeb4c94510bcf109fd93fd1d1cd94fd49d3 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 11:26:29 -0600 Subject: [PATCH 08/24] updated command documentation and visualization documentation --- docs/sphinx/commands.rst | 9 ++++++--- docs/sphinx/visualization.rst | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/commands.rst b/docs/sphinx/commands.rst index 8fdc3950..b95bd5d6 100644 --- a/docs/sphinx/commands.rst +++ b/docs/sphinx/commands.rst @@ -103,13 +103,16 @@ Arguments: ``beeflow reexecute``: Reexecute an archived workflow. Arguments: - - WF_ID [required] - - OUTPUT_DIR, Path to the where the DAG directory will be [required] + WF_ID [required] ``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID_dags. See :ref:`workflow-visualization` for more information. Arguments: - WF_ID [required] + - WF_ID [required] + - OUTPUT_DIR, Directory for the output [required] + +Options: + ``no-dag-dir``: Do not make a subdirectory within the output_dir for the DAGs. Generating and Managing Configuration Files =========================================== diff --git a/docs/sphinx/visualization.rst b/docs/sphinx/visualization.rst index a105bd39..e1cd5440 100644 --- a/docs/sphinx/visualization.rst +++ b/docs/sphinx/visualization.rst @@ -13,7 +13,10 @@ Creating DAGs The dag command can be run at any point of the workflow, and can be run multiple times. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The -DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID_dags. +DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID_dags by default. If the +``no-dag-dir`` flag is specified when the dag command is run, the DAG will be +exported to $OUTPUT_DIR. Running the dag command will make new DAGs for the workflow +without overriding older versions. Example DAG =========== From f8867ed6f7b31a172542115793f92b6fec76d707 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 23 Sep 2024 11:30:13 -0600 Subject: [PATCH 09/24] added export location information to command docs --- docs/sphinx/commands.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sphinx/commands.rst b/docs/sphinx/commands.rst index b95bd5d6..4300dad5 100644 --- a/docs/sphinx/commands.rst +++ b/docs/sphinx/commands.rst @@ -105,7 +105,7 @@ Arguments: Arguments: WF_ID [required] -``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID_dags. See :ref:`workflow-visualization` for more information. +``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID_dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. See :ref:`workflow-visualization` for more information. Arguments: - WF_ID [required] From ece517b7c8c2d143a2070f76dc4918ce7e55b881 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 24 Sep 2024 14:34:54 -0600 Subject: [PATCH 10/24] changed the underscore in the dags dir to a hyphen --- beeflow/common/gdb/generate_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index 75e87a4a..2412dc9b 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -19,7 +19,7 @@ def generate_viz(wf_id, output_dir, no_dag_dir): if no_dag_dir: dags_dir = output_dir else: - dags_dir = output_dir + "/" + short_id + "_dags" + dags_dir = output_dir + "/" + short_id + "-dags" os.makedirs(dags_dir, exist_ok=True) output_path = dags_dir + "/" + short_id + ".png" From 1f804b25a756474a960e4405b4100ea8da1b252f Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 24 Sep 2024 14:36:27 -0600 Subject: [PATCH 11/24] added the apoc install recipe and updated bee_install.sh to use it --- ci/bee_install.sh | 4 ++-- ci/install_apoc_docker | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 ci/install_apoc_docker diff --git a/ci/bee_install.sh b/ci/bee_install.sh index c044e0b7..8d42ebd5 100755 --- a/ci/bee_install.sh +++ b/ci/bee_install.sh @@ -9,8 +9,8 @@ printf "**Setting up BEE containers**\n" printf "\n\n" mkdir -p $HOME/img # Pull the Neo4j container -ch-image pull neo4j:5.17 || exit 1 -ch-convert -i ch-image -o tar neo4j:5.17 $NEO4J_CONTAINER || exit 1 +ch-image build -t apoc_neo4j -f install_apoc_docker . || exit 1 +ch-convert -i ch-image -o tar apoc_neo4j $NEO4J_CONTAINER || exit 1 # Pull the Redis container ch-image pull redis || exit 1 ch-convert -i ch-image -o tar redis $REDIS_CONTAINER || exit 1 diff --git a/ci/install_apoc_docker b/ci/install_apoc_docker new file mode 100644 index 00000000..da185352 --- /dev/null +++ b/ci/install_apoc_docker @@ -0,0 +1,2 @@ +FROM neo4j:5.17-community +RUN cp /var/lib/neo4j/labs/apoc-5.17.0-core.jar /var/lib/neo4j/plugins/apoc-5.17.8-core.jar From 56479a37c439c85eeaa826eab48fed51a5b822a1 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 24 Sep 2024 14:49:08 -0600 Subject: [PATCH 12/24] modified permissions for recipe and added location of recipe --- ci/bee_install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/bee_install.sh b/ci/bee_install.sh index 8d42ebd5..581e7660 100755 --- a/ci/bee_install.sh +++ b/ci/bee_install.sh @@ -9,7 +9,8 @@ printf "**Setting up BEE containers**\n" printf "\n\n" mkdir -p $HOME/img # Pull the Neo4j container -ch-image build -t apoc_neo4j -f install_apoc_docker . || exit 1 +chmod +x ./ci/install_apoc_docker +ch-image build -t apoc_neo4j -f ./ci/install_apoc_docker . || exit 1 ch-convert -i ch-image -o tar apoc_neo4j $NEO4J_CONTAINER || exit 1 # Pull the Redis container ch-image pull redis || exit 1 From d95039ec35cb7f93e749271a79cced603729abd3 Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 24 Sep 2024 15:32:34 -0600 Subject: [PATCH 13/24] add gaphviz and libgraphviz-dev to ci deps_install script --- ci/deps_install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/deps_install.sh b/ci/deps_install.sh index 056b6a34..391bc585 100755 --- a/ci/deps_install.sh +++ b/ci/deps_install.sh @@ -7,7 +7,8 @@ sudo apt-get install -y slurmctld slurmd slurmrestd munge python3 python3-venv \ curl build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev \ libssl-dev libsqlite3-dev libreadline-dev libffi-dev libbz2-dev \ libmunge-dev \ - libyaml-dev # needed for PyYAML + libyaml-dev # needed for PyYAML \ + graphviz libgraphviz-dev # Install most recent Charliecloud curl -O -L https://github.com/hpc/charliecloud/releases/download/v${CHARLIECLOUD_VERSION}/charliecloud-${CHARLIECLOUD_VERSION}.tar.gz From c7ff910fa67af49417fe2b18db433fe97c026d1e Mon Sep 17 00:00:00 2001 From: leahh Date: Tue, 24 Sep 2024 15:38:21 -0600 Subject: [PATCH 14/24] search for graphviz in install --- ci/deps_install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/deps_install.sh b/ci/deps_install.sh index 391bc585..dcdbf3f8 100755 --- a/ci/deps_install.sh +++ b/ci/deps_install.sh @@ -7,8 +7,9 @@ sudo apt-get install -y slurmctld slurmd slurmrestd munge python3 python3-venv \ curl build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev \ libssl-dev libsqlite3-dev libreadline-dev libffi-dev libbz2-dev \ libmunge-dev \ - libyaml-dev # needed for PyYAML \ - graphviz libgraphviz-dev + libyaml-dev # needed for PyYAML + +sudo apt-get install -y graphviz libgraphviz-dev # Install most recent Charliecloud curl -O -L https://github.com/hpc/charliecloud/releases/download/v${CHARLIECLOUD_VERSION}/charliecloud-${CHARLIECLOUD_VERSION}.tar.gz From f8a4775e2a3916efb976b00053e8255cd9e730e5 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 07:27:57 -0600 Subject: [PATCH 15/24] added information about the naming for multiple dag versions to docs --- docs/sphinx/commands.rst | 2 +- docs/sphinx/visualization.rst | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/sphinx/commands.rst b/docs/sphinx/commands.rst index 4300dad5..7b44db5a 100644 --- a/docs/sphinx/commands.rst +++ b/docs/sphinx/commands.rst @@ -105,7 +105,7 @@ Arguments: Arguments: WF_ID [required] -``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID_dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. See :ref:`workflow-visualization` for more information. +``beeflow dag``: Export a directed acyclic graph (DAG) of a submitted workflow. This command can be run at any point of the workflow. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The DAGs are exported to $OUTPUT_DIR/$WD_ID-dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. The dag command makes multiple versions of the DAGs. The most recent version is $WF_ID.png and the others are $WD_ID_v1.png, $WF_ID_v2.png ... where v1 is the oldest. See :ref:`workflow-visualization` for more information. Arguments: - WF_ID [required] diff --git a/docs/sphinx/visualization.rst b/docs/sphinx/visualization.rst index e1cd5440..64fe3649 100644 --- a/docs/sphinx/visualization.rst +++ b/docs/sphinx/visualization.rst @@ -13,10 +13,11 @@ Creating DAGs The dag command can be run at any point of the workflow, and can be run multiple times. To see the DAG of a workflow before it runs, submit the workflow with the ``--no-start`` flag and then use the dag command. The -DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID_dags by default. If the +DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID-dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be -exported to $OUTPUT_DIR. Running the dag command will make new DAGs for the workflow -without overriding older versions. +exported to $OUTPUT_DIR. The dag command makes multiple versions of the DAGs. The +most recent version is $WF_ID.png and the others are $WD_ID_v1.png, +$WF_ID_v2.png ... where v1 is the oldest. Example DAG =========== From 105ac74b5af6eb206a189e55c06d673ea3971bd2 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 09:47:48 -0600 Subject: [PATCH 16/24] initial working commit for graphml location in workflow archive --- beeflow/client/bee_client.py | 4 +++- beeflow/common/deps/neo4j_manager.py | 2 +- beeflow/common/gdb/generate_graph.py | 7 +------ beeflow/common/gdb/graphml_key_updater.py | 15 ++++++++------- beeflow/wf_manager/resources/wf_update.py | 2 +- beeflow/wf_manager/resources/wf_utils.py | 8 +++++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 258491c7..dd7847d7 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -641,7 +641,9 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # output_dir must be a string output_dir = str(output_dir) - wf_utils.export_dag(wf_id, output_dir, no_dag_dir) + wf_dir = wf_utils.get_workflow_dir(wf_id) + graphmls_dir = wf_dir + "/graphmls" + wf_utils.export_dag(wf_id, output_dir, wf_dir, no_dag_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) diff --git a/beeflow/common/deps/neo4j_manager.py b/beeflow/common/deps/neo4j_manager.py index 9a8b6556..f7ed2dc5 100644 --- a/beeflow/common/deps/neo4j_manager.py +++ b/beeflow/common/deps/neo4j_manager.py @@ -23,7 +23,7 @@ run_dir = mount_dir + '/run' certs_dir = mount_dir + '/certificates' confs_dir = mount_dir + "/conf" -graphmls_dir = os.path.join(bee_workdir, 'graphmls') +graphmls_dir = mount_dir + '/graphmls' container_path = container_manager.get_container_dir('neo4j') log = bee_logging.setup('neo4j') diff --git a/beeflow/common/gdb/generate_graph.py b/beeflow/common/gdb/generate_graph.py index 95ca7936..d823f5fc 100644 --- a/beeflow/common/gdb/generate_graph.py +++ b/beeflow/common/gdb/generate_graph.py @@ -5,13 +5,8 @@ import networkx as nx import graphviz -from beeflow.common import paths -bee_workdir = paths.workdir() -graphmls_dir = os.path.join(bee_workdir, 'graphmls') - - -def generate_viz(wf_id, output_dir, no_dag_dir): +def generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir): """Generate a PNG of a workflow graph from a GraphML file.""" short_id = wf_id[:6] graphml_path = graphmls_dir + "/" + short_id + ".graphml" diff --git a/beeflow/common/gdb/graphml_key_updater.py b/beeflow/common/gdb/graphml_key_updater.py index 4eed142a..1e49b716 100644 --- a/beeflow/common/gdb/graphml_key_updater.py +++ b/beeflow/common/gdb/graphml_key_updater.py @@ -6,8 +6,8 @@ from beeflow.common import paths bee_workdir = paths.workdir() -graphmls_dir = os.path.join(bee_workdir, 'graphmls') - +mount_dir = os.path.join(bee_workdir, 'gdb_mount') +gdb_graphmls_dir = mount_dir + '/graphmls' expected_keys = {"id", "name", "state", "class", "type", "value", "source", "workflow_id", "base_command", "stdout", "stderr", "default", @@ -33,12 +33,13 @@ } -def update_graphml(wf_id): +def update_graphml(wf_id, graphmls_dir): """Update GraphML file by ensuring required keys are present and updating its structure.""" short_id = wf_id[:6] - graphml_path = graphmls_dir + "/" + short_id + ".graphml" + gdb_graphml_path = gdb_graphmls_dir + "/" + short_id + ".graphml" + output_graphml_path = graphmls_dir + "/" + short_id + ".graphml" # Parse the GraphML file and preserve namespaces - tree = ET.parse(graphml_path) + tree = ET.parse(gdb_graphml_path) root = tree.getroot() name_space = {'graphml': 'http://graphml.graphdrawing.org/xmlns'} @@ -56,5 +57,5 @@ def update_graphml(wf_id): **default_def) root.insert(0, key_element) - # Save the updated GraphML file by overwriting the original one - tree.write(graphml_path, encoding='UTF-8', xml_declaration=True) + # Save the updated GraphML file + tree.write(output_graphml_path, encoding='UTF-8', xml_declaration=True) diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 4772655e..7b074c19 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -37,7 +37,7 @@ def archive_workflow(db, wf_id, final_state=None): archive_path = f'../archives/{wf_id}.tgz' # We use tar directly since tarfile is apparently very slow workflows_dir = wf_utils.get_workflows_dir() - wf_utils.export_dag(wf_id, workflow_dir, no_dag_dir=True) + wf_utils.export_dag(wf_id, workflow_dir, workflow_dir, no_dag_dir=True) subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir) remove_wf_dir = bc.get('DEFAULT', 'delete_completed_workflow_dirs') if remove_wf_dir: diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index e75211c5..46411b56 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -294,12 +294,14 @@ def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, start_workflow(wf_id) -def export_dag(wf_id, output_dir, no_dag_dir): +def export_dag(wf_id, output_dir, wf_dir, no_dag_dir): """Export the DAG of the workflow.""" wfi = get_workflow_interface(wf_id) wfi.export_graphml() - update_graphml(wf_id) - generate_viz(wf_id, output_dir, no_dag_dir) + graphmls_dir = wf_dir + "/graphmls" + os.makedirs(graphmls_dir, exist_ok=True) + update_graphml(wf_id, graphmls_dir) + generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) def start_workflow(wf_id): From ec32455c38095c9b407c683916e8dbad067c4996 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 10:48:31 -0600 Subject: [PATCH 17/24] make multiple versions of graphmls without overriding older ones --- beeflow/common/gdb/graphml_key_updater.py | 9 +++++++++ beeflow/wf_manager/resources/wf_update.py | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/beeflow/common/gdb/graphml_key_updater.py b/beeflow/common/gdb/graphml_key_updater.py index 1e49b716..023b6937 100644 --- a/beeflow/common/gdb/graphml_key_updater.py +++ b/beeflow/common/gdb/graphml_key_updater.py @@ -2,6 +2,7 @@ import xml.etree.ElementTree as ET import os +import shutil from beeflow.common import paths @@ -38,6 +39,14 @@ def update_graphml(wf_id, graphmls_dir): short_id = wf_id[:6] gdb_graphml_path = gdb_graphmls_dir + "/" + short_id + ".graphml" output_graphml_path = graphmls_dir + "/" + short_id + ".graphml" + # Handle making multiple versions of the graphmls without overriding old ones + if os.path.exists(output_graphml_path): + i = 1 + backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml' + while os.path.exists(backup_path): + i += 1 + backup_path = f'{graphmls_dir}/{short_id}_v{i}.graphml' + shutil.copy(output_graphml_path, backup_path) # Parse the GraphML file and preserve namespaces tree = ET.parse(gdb_graphml_path) root = tree.getroot() diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 7b074c19..90f34962 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -26,6 +26,8 @@ def archive_workflow(db, wf_id, final_state=None): workflow_dir = wf_utils.get_workflow_dir(wf_id) shutil.copyfile(os.path.expanduser("~") + '/.config/beeflow/bee.conf', workflow_dir + '/' + 'bee.conf') + # Archive Completed DAG + wf_utils.export_dag(wf_id, workflow_dir, workflow_dir, no_dag_dir=True) wf_state = f'Archived/{final_state}' if final_state is not None else 'Archived' db.workflows.update_workflow_state(wf_id, wf_state) @@ -37,7 +39,6 @@ def archive_workflow(db, wf_id, final_state=None): archive_path = f'../archives/{wf_id}.tgz' # We use tar directly since tarfile is apparently very slow workflows_dir = wf_utils.get_workflows_dir() - wf_utils.export_dag(wf_id, workflow_dir, workflow_dir, no_dag_dir=True) subprocess.call(['tar', '-czf', archive_path, wf_id], cwd=workflows_dir) remove_wf_dir = bc.get('DEFAULT', 'delete_completed_workflow_dirs') if remove_wf_dir: From 68fb16135121f7ef8194d8900c17e464f7fff87b Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 11:22:07 -0600 Subject: [PATCH 18/24] edited bee_client to not let people make DAGs after the workflow has been archived --- beeflow/client/bee_client.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index dd7847d7..ed61ec27 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -631,8 +631,11 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), no_dag_dir: bool = typer.Option(False, '--no-dag-dir', help='do not make a subdirectory within ouput_dir for the dags')): """Export a DAG of the workflow to a GraphML file.""" + # Check if the workflow is archived + wf_status = get_wf_status(wf_id) + if wf_status == 'Archived': + error_exit('Workflow has been archived. Check the workflow archive for the final DAG.') output_dir = output_dir.resolve() - # Make sure output_dir is an absolute path and exists output_dir = os.path.expanduser(output_dir) output_dir = os.path.abspath(output_dir) From 56181c8900b2961ec6b44586efa621722c90c99e Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 12:36:35 -0600 Subject: [PATCH 19/24] edited bee_client and wf_update and wf_utils so now multiple versions of the completed dags can be made --- beeflow/client/bee_client.py | 20 +++++++++++++------- beeflow/wf_manager/resources/wf_update.py | 4 +++- beeflow/wf_manager/resources/wf_utils.py | 4 +--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index ed61ec27..7b435d58 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -631,10 +631,6 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), no_dag_dir: bool = typer.Option(False, '--no-dag-dir', help='do not make a subdirectory within ouput_dir for the dags')): """Export a DAG of the workflow to a GraphML file.""" - # Check if the workflow is archived - wf_status = get_wf_status(wf_id) - if wf_status == 'Archived': - error_exit('Workflow has been archived. Check the workflow archive for the final DAG.') output_dir = output_dir.resolve() # Make sure output_dir is an absolute path and exists output_dir = os.path.expanduser(output_dir) @@ -644,9 +640,19 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), # output_dir must be a string output_dir = str(output_dir) - wf_dir = wf_utils.get_workflow_dir(wf_id) - graphmls_dir = wf_dir + "/graphmls" - wf_utils.export_dag(wf_id, output_dir, wf_dir, no_dag_dir) + # Check if the workflow is archived + wf_status = get_wf_status(wf_id) + if wf_status == 'Archived': + bee_workdir = wf_utils.get_bee_workdir() + mount_dir = os.path.join(bee_workdir, 'gdb_mount') + graphmls_dir = mount_dir + '/graphmls' + typer.secho(f"Workflow has been archived. All new DAGs will look the same as the one in the archive directory.", + fg=typer.colors.MAGENTA) + else: + wf_dir = wf_utils.get_workflow_dir(wf_id) + graphmls_dir = wf_dir + '/graphmls' + os.makedirs(graphmls_dir, exist_ok=True) + wf_utils.export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir) typer.secho(f"DAG for workflow {_short_id(wf_id)} has been exported successfully.", fg=typer.colors.GREEN) diff --git a/beeflow/wf_manager/resources/wf_update.py b/beeflow/wf_manager/resources/wf_update.py index 90f34962..47dbea2d 100644 --- a/beeflow/wf_manager/resources/wf_update.py +++ b/beeflow/wf_manager/resources/wf_update.py @@ -27,7 +27,9 @@ def archive_workflow(db, wf_id, final_state=None): shutil.copyfile(os.path.expanduser("~") + '/.config/beeflow/bee.conf', workflow_dir + '/' + 'bee.conf') # Archive Completed DAG - wf_utils.export_dag(wf_id, workflow_dir, workflow_dir, no_dag_dir=True) + graphmls_dir = workflow_dir + "/graphmls" + os.makedirs(graphmls_dir, exist_ok=True) + wf_utils.export_dag(wf_id, workflow_dir, graphmls_dir, no_dag_dir=True) wf_state = f'Archived/{final_state}' if final_state is not None else 'Archived' db.workflows.update_workflow_state(wf_id, wf_state) diff --git a/beeflow/wf_manager/resources/wf_utils.py b/beeflow/wf_manager/resources/wf_utils.py index 46411b56..4dde69b8 100644 --- a/beeflow/wf_manager/resources/wf_utils.py +++ b/beeflow/wf_manager/resources/wf_utils.py @@ -294,12 +294,10 @@ def setup_workflow(wf_id, wf_name, wf_dir, wf_workdir, no_start, workflow=None, start_workflow(wf_id) -def export_dag(wf_id, output_dir, wf_dir, no_dag_dir): +def export_dag(wf_id, output_dir, graphmls_dir, no_dag_dir): """Export the DAG of the workflow.""" wfi = get_workflow_interface(wf_id) wfi.export_graphml() - graphmls_dir = wf_dir + "/graphmls" - os.makedirs(graphmls_dir, exist_ok=True) update_graphml(wf_id, graphmls_dir) generate_viz(wf_id, output_dir, graphmls_dir, no_dag_dir) From 2d7707dbfc58d2afdde808632fd06536866272f6 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 12:58:32 -0600 Subject: [PATCH 20/24] fixed linting error in bee_client and added to docs about saving graphmls to archie --- beeflow/client/bee_client.py | 3 ++- docs/sphinx/visualization.rst | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 7b435d58..139583da 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -646,7 +646,8 @@ def dag(wf_id: str = typer.Argument(..., callback=match_short_id), bee_workdir = wf_utils.get_bee_workdir() mount_dir = os.path.join(bee_workdir, 'gdb_mount') graphmls_dir = mount_dir + '/graphmls' - typer.secho(f"Workflow has been archived. All new DAGs will look the same as the one in the archive directory.", + typer.secho("Workflow has been archived. All new DAGs will look the same as the one " + "in the archive directory.", fg=typer.colors.MAGENTA) else: wf_dir = wf_utils.get_workflow_dir(wf_id) diff --git a/docs/sphinx/visualization.rst b/docs/sphinx/visualization.rst index 550f9d5a..f4c63e91 100644 --- a/docs/sphinx/visualization.rst +++ b/docs/sphinx/visualization.rst @@ -17,7 +17,9 @@ DAGs are exported in PNG format to $OUTPUT_DIR/$WD_ID-dags by default. If the ``no-dag-dir`` flag is specified when the dag command is run, the DAG will be exported to $OUTPUT_DIR. The dag command makes multiple versions of the DAGs. The most recent version is $WF_ID.png and the others are $WD_ID_v1.png, -$WF_ID_v2.png ... where v1 is the oldest. +$WF_ID_v2.png ... where v1 is the oldest. The graphmls used to make the DAGs are saved +in the workflow archive and are saved with their version number. These graphmls can +be useful for debugging when there are errors creating the DAGs. Example DAG =========== From 7a19c9fd48076520bf5b2a8fa8d4c6fb8884a32e Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 30 Sep 2024 14:29:25 -0600 Subject: [PATCH 21/24] change context directory for ci neo4j image --- ci/bee_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/bee_install.sh b/ci/bee_install.sh index 581e7660..b8e73ea6 100755 --- a/ci/bee_install.sh +++ b/ci/bee_install.sh @@ -10,7 +10,7 @@ printf "\n\n" mkdir -p $HOME/img # Pull the Neo4j container chmod +x ./ci/install_apoc_docker -ch-image build -t apoc_neo4j -f ./ci/install_apoc_docker . || exit 1 +ch-image build -t apoc_neo4j -f ./ci/install_apoc_docker ./ci || exit 1 ch-convert -i ch-image -o tar apoc_neo4j $NEO4J_CONTAINER || exit 1 # Pull the Redis container ch-image pull redis || exit 1 From 7736bcd7a0651f765a1abb5d06ffd5b0f1901b83 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 7 Oct 2024 10:05:48 -0600 Subject: [PATCH 22/24] change neo4j path variable in ci --- ci/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/env.sh b/ci/env.sh index 8a725fab..4eea8db9 100644 --- a/ci/env.sh +++ b/ci/env.sh @@ -18,7 +18,7 @@ export MUNGE_PID=/tmp/munge.pid mkdir -p /tmp/munge export MUNGE_KEY=/tmp/munge/munge.key export BEE_WORKDIR=$HOME/.beeflow -export NEO4J_CONTAINER=$HOME/img/neo4j.tar.gz +export NEO4J_CONTAINER=$HOME/img/apoc_neo4j.tar.gz export REDIS_CONTAINER=$HOME/img/redis.tar.gz mkdir -p $BEE_WORKDIR export SLURM_CONF=~/slurm.conf From 085ddbd7f209e0f5aa59bb6602bda55954526cb5 Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 7 Oct 2024 12:30:37 -0600 Subject: [PATCH 23/24] add sleep to beginning of main --- beeflow/common/integration_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beeflow/common/integration_test.py b/beeflow/common/integration_test.py index 71fac912..f26cb96b 100644 --- a/beeflow/common/integration_test.py +++ b/beeflow/common/integration_test.py @@ -4,6 +4,7 @@ import os import shutil import sys +import time import uuid import typer @@ -293,6 +294,7 @@ def main(tests = typer.Option(None, '--tests', '-t', # noqa (conflict on '=' si timeout: int = typer.Option(utils.TIMEOUT, '--timeout', help='workflow timeout in seconds')): """Launch the integration tests.""" + time.sleep(60) if show_tests: print('INTEGRATION TEST CASES:') for test_name, ignore in TEST_RUNNER.test_details(): From 0512fa1aa18cdd8936cb375cf160528f3ea5443d Mon Sep 17 00:00:00 2001 From: leahh Date: Mon, 7 Oct 2024 12:45:35 -0600 Subject: [PATCH 24/24] change neo4j path variable back to original --- ci/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/env.sh b/ci/env.sh index 4eea8db9..8a725fab 100644 --- a/ci/env.sh +++ b/ci/env.sh @@ -18,7 +18,7 @@ export MUNGE_PID=/tmp/munge.pid mkdir -p /tmp/munge export MUNGE_KEY=/tmp/munge/munge.key export BEE_WORKDIR=$HOME/.beeflow -export NEO4J_CONTAINER=$HOME/img/apoc_neo4j.tar.gz +export NEO4J_CONTAINER=$HOME/img/neo4j.tar.gz export REDIS_CONTAINER=$HOME/img/redis.tar.gz mkdir -p $BEE_WORKDIR export SLURM_CONF=~/slurm.conf