From 28ef9560aa45c1df446ecac82cb391b5760cd6c1 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Mon, 16 Sep 2024 14:43:15 -0600 Subject: [PATCH 01/17] create client database and populate --- beeflow/client/bee_client.py | 23 ++++++++++++++- beeflow/client/core.py | 1 + beeflow/common/db/client_db.py | 51 +++++++++++++++++++++++++++++++++ beeflow/tests/test_db_client.py | 31 ++++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 beeflow/common/db/client_db.py create mode 100644 beeflow/tests/test_db_client.py diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 7bc1b53e3..f9be5bcad 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """bee-client. -This script provides an client interface to the user to manage workflows. +This script provides a client interface to the user to manage workflows. Capablities include submitting, starting, listing, pausing and cancelling workflows. """ import os @@ -29,6 +29,8 @@ from beeflow.common.wf_data import generate_workflow_id from beeflow.client import core from beeflow.wf_manager.resources import wf_utils +from beeflow.common.db import client_db +from beeflow.common.db import bdb # Length of a shortened workflow ID short_id_len = 6 #noqa: Not a constant @@ -52,6 +54,25 @@ def __init__(self, *args): self.args = args +def db_path(): + """Return the client database path.""" + bee_workdir = config_driver.BeeConfig.get('DEFAULT', 'bee_workdir') + return os.path.join(bee_workdir, 'client.db') + + +def connect_db(): + """Connect to the client database.""" + return bdb.connect_db(client_db, db_path()) + + +def setup_hostname(): + """Set up front end name when beeflow core start is returned.""" + db = connect_db(client_db, db_path()) + # hard coding front end name for now + new_hostname = 'front_end_name' + db.info.set_hostname(new_hostname) + + def error_exit(msg, include_caller=True): """Print a message and exit or raise an error with that message.""" if include_caller: diff --git a/beeflow/client/core.py b/beeflow/client/core.py index 2ea24d912..ca17f6051 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -323,6 +323,7 @@ def __init__(self, mgr, base_components): def loop(self): """Run the main loop.""" print(f'Running on {socket.gethostname()}') + # add {socket.gethostname} to client database here?? self.mgr.run(self.base_components) with cli_connection.server(paths.beeflow_socket()) as server: while not self.quit: diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py new file mode 100644 index 000000000..33fc85278 --- /dev/null +++ b/beeflow/common/db/client_db.py @@ -0,0 +1,51 @@ +"""Client database code.""" + +from collections import namedtuple + +from beeflow.common.db import bdb + + +class ClientInfo: + """Client Info object.""" + + def __init__(self, db_file): + """Initialize info and db file.""" + self.Info = namedtuple("Info", "id hostname") # noqa Snake Case + self.db_file = db_file + + def set_hostname(self, new_hostname): + """Set hostname for current front end.""" + stmt = f"UPDATE info set hostname=?" + bdb.run(self.db_file, stmt, [new_hostname]) + + def get_hostname(self): + """Return hostname for current front end.""" + stmt = f"SELECT hostname FROM info" + result = bdb.getone(self.db_file, stmt)[0] + hostname = result + return hostname + +class ClientDB: + """Client database.""" + + def __init__(self, db_file): + """Construct a new client database connection.""" + self.db_file = db_file + self._init_tables() + + def _init_tables(self): + """Initialize the client table if it doesn't exist.""" + info_stmt = """CREATE TABLE IF NOT EXISTS info( + id INTEGER PRIMARY KEY ASC, + hostname TEXT);""" + bdb.create_table(self.db_file, info_stmt) + + @property + def info(self): + """Get info from the database.""" + return ClientInfo(self.db_file) + + +def open_db(db_file): + """Open and return a new database.""" + return ClientDB(db_file) diff --git a/beeflow/tests/test_db_client.py b/beeflow/tests/test_db_client.py new file mode 100644 index 000000000..7fd930f58 --- /dev/null +++ b/beeflow/tests/test_db_client.py @@ -0,0 +1,31 @@ +"""Tests of the client database.""" +import tempfile +import os + +import pytest + +from beeflow.common.db import client_db + + +@pytest.fixture +def temp_db(): + """Create a fixture for making a temporary database.""" + fname = tempfile.mktemp() + db = client_db.open_db(fname) + yield db + os.remove(fname) + + +def test_empty(temp_db): + """Test the empty database.""" + db = temp_db + + assert len(list(db.info)) == 0 + + +def test_info(temp_db): + """Test setting the info.""" + db = temp_db + + db.info.set_hostname('front_end_name') + assert db.info.get_hostname == 'front_end_name' From e40d8e6e056b3c05b96f2407694b68f466a4e825 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Mon, 30 Sep 2024 17:25:39 -0600 Subject: [PATCH 02/17] set up hostname and keep track of it - wip --- beeflow/client/bee_client.py | 29 ++++++++++++++++++++--------- beeflow/client/core.py | 10 ++++++++++ beeflow/common/db/client_db.py | 23 ++++++++++++++++++----- beeflow/tests/test_db_client.py | 8 ++++++-- 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index f9be5bcad..0dc9636f7 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -54,23 +54,34 @@ def __init__(self, *args): self.args = args +def warn(*pargs): + """Print a red warning message.""" + typer.secho(' '.join(pargs), fg=typer.colors.RED, file=sys.stderr) + + def db_path(): """Return the client database path.""" bee_workdir = config_driver.BeeConfig.get('DEFAULT', 'bee_workdir') return os.path.join(bee_workdir, 'client.db') -def connect_db(): - """Connect to the client database.""" - return bdb.connect_db(client_db, db_path()) +def setup_hostname(start_hn): + """Set up front end name when beeflow core start is returned.""" + db = bdb.connect_db(client_db, db_path()) + db.info.set_hostname(start_hn) -def setup_hostname(): - """Set up front end name when beeflow core start is returned.""" - db = connect_db(client_db, db_path()) - # hard coding front end name for now - new_hostname = 'front_end_name' - db.info.set_hostname(new_hostname) +def check_hostname(curr_hn, stop = False): + """Check current front end name matches the one beeflow was started on.""" + db = bdb.connect_db(client_db, db_path()) + start_hn = db.info.get_hostname() + if start_hn != "": + if curr_hn != start_hn: + warn(f'beeflow was started on "{start_hn}" and you are trying to run a command on "{curr_hn}".') + else: + warn('beeflow has not been started!') + if stop: + db.info.set_hostname("") def error_exit(msg, include_caller=True): diff --git a/beeflow/client/core.py b/beeflow/client/core.py index ca17f6051..c3684369f 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -409,6 +409,9 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', version = importlib.metadata.version("hpc-beeflow") print(f'Starting beeflow {version}...') + start_hn = socket.gethostname() # hostname when beeflow starts + print(f'Running beeflow on {start_hn}') + bee_client.setup_hostname(start_hn) # add to client db if not foreground: print('Run `beeflow core status` for more information.') # Create the log path if it doesn't exist yet @@ -427,11 +430,13 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', @app.command() def status(): """Check the status of beeflow and the components.""" + status_hn = socket.gethostname() # hostname when beeflow core status returned resp = cli_connection.send(paths.beeflow_socket(), {'type': 'status'}) if resp is None: beeflow_log = paths.log_fname('beeflow') warn('Cannot connect to the beeflow daemon, is it running? Check the ' f'log at "{beeflow_log}".') + bee_client.check_hostname(status_hn) sys.exit(1) print('beeflow components:') for comp, stat in resp['components'].items(): @@ -441,6 +446,8 @@ def status(): @app.command() def info(): """Get information about beeflow's installation.""" + info_hn = socket.gethostname() # hostname when beeflow core info returned + bee_client.check_hostname(info_hn) version = importlib.metadata.version("hpc-beeflow") print(f"Beeflow version: {version}") print(f"bee_workflow directory: {paths.workdir()}") @@ -450,6 +457,7 @@ def info(): @app.command() def stop(query='yes'): """Stop the current running beeflow daemon.""" + stop_hn = socket.gethostname() # hostname when beeflow core stop returned # Check workflow states; warn if there are active states, pause running workflows workflow_list = bee_client.get_wf_list() concern_states = {'Running', 'Initializing', 'Waiting'} @@ -476,11 +484,13 @@ def stop(query='yes'): warn('Error: beeflow is not running on this system. It could be ' 'running on a different front end.\n' f' Check the beeflow log: "{beeflow_log}".') + #bee_client.check_hostname(stop_hn, stop = True) sys.exit(1) # As long as it returned something, we should be good beeflow_log = paths.log_fname('beeflow') if query == "yes": print(f'Beeflow has stopped. Check the log at "{beeflow_log}".') + bee_client.check_hostname(stop_hn, stop = True) def archive_dir(dir_to_archive): diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index 33fc85278..e41ed7641 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -9,9 +9,9 @@ class ClientInfo: """Client Info object.""" def __init__(self, db_file): - """Initialize info and db file.""" - self.Info = namedtuple("Info", "id hostname") # noqa Snake Case - self.db_file = db_file + """Initialize info and db file.""" + self.Info = namedtuple("Info", "id hostname") # noqa Snake Case + self.db_file = db_file def set_hostname(self, new_hostname): """Set hostname for current front end.""" @@ -25,6 +25,13 @@ def get_hostname(self): hostname = result return hostname + def get_info(self): + """Return an info object containing port information.""" + stmt = "SELECT * FROM info" + result = bdb.getone(self.db_file, stmt) + info = self.Info(*result) + return info + class ClientDB: """Client database.""" @@ -35,10 +42,16 @@ def __init__(self, db_file): def _init_tables(self): """Initialize the client table if it doesn't exist.""" - info_stmt = """CREATE TABLE IF NOT EXISTS info( + info_stmt = """CREATE TABLE IF NOT EXISTS info ( id INTEGER PRIMARY KEY ASC, hostname TEXT);""" - bdb.create_table(self.db_file, info_stmt) +# bdb.create_table(self.db_file, info_stmt) + if not bdb.table_exists(self.db_file, 'info'): + bdb.create_table(self.db_file, info_stmt) + # insert a new workflow into the database + stmt = """INSERT INTO info (hostname) VALUES(?);""" + tmp = "" + bdb.run(self.db_file, stmt, [tmp]) @property def info(self): diff --git a/beeflow/tests/test_db_client.py b/beeflow/tests/test_db_client.py index 7fd930f58..7c37719b1 100644 --- a/beeflow/tests/test_db_client.py +++ b/beeflow/tests/test_db_client.py @@ -20,7 +20,8 @@ def test_empty(temp_db): """Test the empty database.""" db = temp_db - assert len(list(db.info)) == 0 + hn = db.info.get_hostname() + assert hn == "" def test_info(temp_db): @@ -28,4 +29,7 @@ def test_info(temp_db): db = temp_db db.info.set_hostname('front_end_name') - assert db.info.get_hostname == 'front_end_name' + hn = db.info.get_hostname() + + print("testing get_info: ", db.info.get_info()) + assert hn == 'front_end_name' From ee011ee2049ba6b12d3b267f160f983cbbda9e03 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 10:18:54 -0600 Subject: [PATCH 03/17] fix pylama issue --- beeflow/common/db/client_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index e41ed7641..20a5d7e46 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -32,6 +32,7 @@ def get_info(self): info = self.Info(*result) return info + class ClientDB: """Client database.""" From 2e63d34b7d3df22ee2474670d7f8ea45f35e5a7e Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 13:34:00 -0600 Subject: [PATCH 04/17] fix pylama issue --- beeflow/common/db/client_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index 20a5d7e46..3b1686d94 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -20,7 +20,7 @@ def set_hostname(self, new_hostname): def get_hostname(self): """Return hostname for current front end.""" - stmt = f"SELECT hostname FROM info" + stmt = "SELECT hostname FROM info" result = bdb.getone(self.db_file, stmt)[0] hostname = result return hostname From 4fb76aba58e34aafaf79cdb851d390de7ee39879 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 13:40:18 -0600 Subject: [PATCH 05/17] fix pylama issue --- beeflow/common/db/client_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index 3b1686d94..3cfe6b0a4 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -15,7 +15,7 @@ def __init__(self, db_file): def set_hostname(self, new_hostname): """Set hostname for current front end.""" - stmt = f"UPDATE info set hostname=?" + stmt = "UPDATE info set hostname=?" bdb.run(self.db_file, stmt, [new_hostname]) def get_hostname(self): From 8a19ad704190333b6e351ee346f6bd0d1de1bf9e Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 13:48:05 -0600 Subject: [PATCH 06/17] remove beeflow core info front end check --- beeflow/client/bee_client.py | 3 ++- beeflow/client/core.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index d2395921a..e48cad6c3 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -78,7 +78,8 @@ def check_hostname(curr_hn, stop = False): start_hn = db.info.get_hostname() if start_hn != "": if curr_hn != start_hn: - warn(f'beeflow was started on "{start_hn}" and you are trying to run a command on "{curr_hn}".') + warn(f'beeflow was started on "{start_hn}" and you are trying to ' + f'run a command on "{curr_hn}".') else: warn('beeflow has not been started!') if stop: diff --git a/beeflow/client/core.py b/beeflow/client/core.py index c3684369f..0056c67f8 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -446,8 +446,6 @@ def status(): @app.command() def info(): """Get information about beeflow's installation.""" - info_hn = socket.gethostname() # hostname when beeflow core info returned - bee_client.check_hostname(info_hn) version = importlib.metadata.version("hpc-beeflow") print(f"Beeflow version: {version}") print(f"bee_workflow directory: {paths.workdir()}") From bb162ab6914e881ea3735cf320d67c506fc27274 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:00:52 -0600 Subject: [PATCH 07/17] fix pylama issue --- beeflow/client/bee_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index e48cad6c3..40775fdfa 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -79,7 +79,7 @@ def check_hostname(curr_hn, stop = False): if start_hn != "": if curr_hn != start_hn: warn(f'beeflow was started on "{start_hn}" and you are trying to ' - f'run a command on "{curr_hn}".') + f'run a command on "{curr_hn}".') else: warn('beeflow has not been started!') if stop: From 8eb03f5e5ed7d68d8931d46a568cb4a9c9241642 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:04:02 -0600 Subject: [PATCH 08/17] more pylama issues ahh --- beeflow/client/bee_client.py | 2 +- beeflow/client/core.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 40775fdfa..0fb1a40d7 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -72,7 +72,7 @@ def setup_hostname(start_hn): db.info.set_hostname(start_hn) -def check_hostname(curr_hn, stop = False): +def check_hostname(curr_hn, stop=False): """Check current front end name matches the one beeflow was started on.""" db = bdb.connect_db(client_db, db_path()) start_hn = db.info.get_hostname() diff --git a/beeflow/client/core.py b/beeflow/client/core.py index 0056c67f8..514eb0f79 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -323,7 +323,6 @@ def __init__(self, mgr, base_components): def loop(self): """Run the main loop.""" print(f'Running on {socket.gethostname()}') - # add {socket.gethostname} to client database here?? self.mgr.run(self.base_components) with cli_connection.server(paths.beeflow_socket()) as server: while not self.quit: @@ -488,7 +487,7 @@ def stop(query='yes'): beeflow_log = paths.log_fname('beeflow') if query == "yes": print(f'Beeflow has stopped. Check the log at "{beeflow_log}".') - bee_client.check_hostname(stop_hn, stop = True) + bee_client.check_hostname(stop_hn, stop=True) def archive_dir(dir_to_archive): From f813f9ba80098a1d78fd0fd787079321b2d95b61 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:06:30 -0600 Subject: [PATCH 09/17] fix pylama issue --- beeflow/client/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/client/core.py b/beeflow/client/core.py index 514eb0f79..a3a484ff7 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -481,7 +481,7 @@ def stop(query='yes'): warn('Error: beeflow is not running on this system. It could be ' 'running on a different front end.\n' f' Check the beeflow log: "{beeflow_log}".') - #bee_client.check_hostname(stop_hn, stop = True) + # bee_client.check_hostname(stop_hn, stop = True) sys.exit(1) # As long as it returned something, we should be good beeflow_log = paths.log_fname('beeflow') From 2d400dae596cd678f71e6fc8456c391845f00bd8 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:10:29 -0600 Subject: [PATCH 10/17] more pylama fixes --- beeflow/client/core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/beeflow/client/core.py b/beeflow/client/core.py index a3a484ff7..691cce13f 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -408,9 +408,9 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', version = importlib.metadata.version("hpc-beeflow") print(f'Starting beeflow {version}...') - start_hn = socket.gethostname() # hostname when beeflow starts + start_hn = socket.gethostname() # hostname when beeflow starts print(f'Running beeflow on {start_hn}') - bee_client.setup_hostname(start_hn) # add to client db + bee_client.setup_hostname(start_hn) # add to client db if not foreground: print('Run `beeflow core status` for more information.') # Create the log path if it doesn't exist yet @@ -429,7 +429,7 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', @app.command() def status(): """Check the status of beeflow and the components.""" - status_hn = socket.gethostname() # hostname when beeflow core status returned + status_hn = socket.gethostname() # hostname when beeflow core status returned resp = cli_connection.send(paths.beeflow_socket(), {'type': 'status'}) if resp is None: beeflow_log = paths.log_fname('beeflow') @@ -454,7 +454,7 @@ def info(): @app.command() def stop(query='yes'): """Stop the current running beeflow daemon.""" - stop_hn = socket.gethostname() # hostname when beeflow core stop returned + stop_hn = socket.gethostname() # hostname when beeflow core stop returned # Check workflow states; warn if there are active states, pause running workflows workflow_list = bee_client.get_wf_list() concern_states = {'Running', 'Initializing', 'Waiting'} From 37d6e57b4d2216216ee08c824fd0d64afd1c6e74 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:15:43 -0600 Subject: [PATCH 11/17] pylama fixx --- beeflow/tests/test_db_client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/beeflow/tests/test_db_client.py b/beeflow/tests/test_db_client.py index 7c37719b1..e543d306d 100644 --- a/beeflow/tests/test_db_client.py +++ b/beeflow/tests/test_db_client.py @@ -20,8 +20,8 @@ def test_empty(temp_db): """Test the empty database.""" db = temp_db - hn = db.info.get_hostname() - assert hn == "" + host_name = db.info.get_hostname() + assert host_name == "" def test_info(temp_db): @@ -29,7 +29,10 @@ def test_info(temp_db): db = temp_db db.info.set_hostname('front_end_name') - hn = db.info.get_hostname() + host_name = db.info.get_hostname() print("testing get_info: ", db.info.get_info()) - assert hn == 'front_end_name' + assert host_name == 'front_end_name' +# Ignore W0621: PyLama complains about redefining 'temp_db' from the outer +# scope. This is how pytest fixtures work. +# pylama:ignore=W0621 \ No newline at end of file From 82e3f5611307c4b685434736753f3a6160bb77ce Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 1 Oct 2024 14:20:05 -0600 Subject: [PATCH 12/17] pylama fixx --- beeflow/tests/test_db_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeflow/tests/test_db_client.py b/beeflow/tests/test_db_client.py index e543d306d..e36f123a3 100644 --- a/beeflow/tests/test_db_client.py +++ b/beeflow/tests/test_db_client.py @@ -35,4 +35,4 @@ def test_info(temp_db): assert host_name == 'front_end_name' # Ignore W0621: PyLama complains about redefining 'temp_db' from the outer # scope. This is how pytest fixtures work. -# pylama:ignore=W0621 \ No newline at end of file +# pylama:ignore=W0621 From 45c9cdd8606aca4da45f328b4d6c1421cacb8b97 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Mon, 7 Oct 2024 12:30:22 -0600 Subject: [PATCH 13/17] tidy up --- beeflow/client/bee_client.py | 9 ++++----- beeflow/client/core.py | 2 +- beeflow/common/db/client_db.py | 1 - beeflow/tests/test_db_client.py | 1 - 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 0fb1a40d7..cd3563aa6 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -76,11 +76,10 @@ def check_hostname(curr_hn, stop=False): """Check current front end name matches the one beeflow was started on.""" db = bdb.connect_db(client_db, db_path()) start_hn = db.info.get_hostname() - if start_hn != "": - if curr_hn != start_hn: - warn(f'beeflow was started on "{start_hn}" and you are trying to ' - f'run a command on "{curr_hn}".') - else: + if start_hn != "" and curr_hn != start_hn: + warn(f'beeflow was started on "{start_hn}" and you are trying to ' + f'run a command on "{curr_hn}".') + if start_hn == "": warn('beeflow has not been started!') if stop: db.info.set_hostname("") diff --git a/beeflow/client/core.py b/beeflow/client/core.py index 691cce13f..e0173e455 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -481,7 +481,7 @@ def stop(query='yes'): warn('Error: beeflow is not running on this system. It could be ' 'running on a different front end.\n' f' Check the beeflow log: "{beeflow_log}".') - # bee_client.check_hostname(stop_hn, stop = True) + bee_client.check_hostname(stop_hn, stop=True) sys.exit(1) # As long as it returned something, we should be good beeflow_log = paths.log_fname('beeflow') diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index 3cfe6b0a4..d7b7ca5f5 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -46,7 +46,6 @@ def _init_tables(self): info_stmt = """CREATE TABLE IF NOT EXISTS info ( id INTEGER PRIMARY KEY ASC, hostname TEXT);""" -# bdb.create_table(self.db_file, info_stmt) if not bdb.table_exists(self.db_file, 'info'): bdb.create_table(self.db_file, info_stmt) # insert a new workflow into the database diff --git a/beeflow/tests/test_db_client.py b/beeflow/tests/test_db_client.py index e36f123a3..50220446e 100644 --- a/beeflow/tests/test_db_client.py +++ b/beeflow/tests/test_db_client.py @@ -31,7 +31,6 @@ def test_info(temp_db): db.info.set_hostname('front_end_name') host_name = db.info.get_hostname() - print("testing get_info: ", db.info.get_info()) assert host_name == 'front_end_name' # Ignore W0621: PyLama complains about redefining 'temp_db' from the outer # scope. This is how pytest fixtures work. From 700ed6a6ba4c3b8795092e34c7ff5cff67295021 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Mon, 7 Oct 2024 12:39:01 -0600 Subject: [PATCH 14/17] ignore pylint error --- beeflow/client/bee_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index cd3563aa6..9f9566fb3 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -76,7 +76,7 @@ def check_hostname(curr_hn, stop=False): """Check current front end name matches the one beeflow was started on.""" db = bdb.connect_db(client_db, db_path()) start_hn = db.info.get_hostname() - if start_hn != "" and curr_hn != start_hn: + if start_hn != "" and curr_hn != start_hn: # noqa: don't use set instead warn(f'beeflow was started on "{start_hn}" and you are trying to ' f'run a command on "{curr_hn}".') if start_hn == "": @@ -686,4 +686,5 @@ def main(): # Ignore W0511: This allows us to have TODOs in the code # Ignore R1732: Significant code restructuring required to fix +# Ignore R1714: Not using a set instead # pylama:ignore=W0511,R1732 From 15bc9bd117aaa3e24b21bbf895215b23be620874 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Tue, 8 Oct 2024 09:31:28 -0600 Subject: [PATCH 15/17] adding requested changes --- beeflow/client/bee_client.py | 2 +- beeflow/client/core.py | 3 --- beeflow/common/db/client_db.py | 7 ------- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 9f9566fb3..7191525e7 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -76,7 +76,7 @@ def check_hostname(curr_hn, stop=False): """Check current front end name matches the one beeflow was started on.""" db = bdb.connect_db(client_db, db_path()) start_hn = db.info.get_hostname() - if start_hn != "" and curr_hn != start_hn: # noqa: don't use set instead + if start_hn and curr_hn != start_hn: # noqa: don't use set instead warn(f'beeflow was started on "{start_hn}" and you are trying to ' f'run a command on "{curr_hn}".') if start_hn == "": diff --git a/beeflow/client/core.py b/beeflow/client/core.py index e0173e455..bb1cb2193 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -432,9 +432,6 @@ def status(): status_hn = socket.gethostname() # hostname when beeflow core status returned resp = cli_connection.send(paths.beeflow_socket(), {'type': 'status'}) if resp is None: - beeflow_log = paths.log_fname('beeflow') - warn('Cannot connect to the beeflow daemon, is it running? Check the ' - f'log at "{beeflow_log}".') bee_client.check_hostname(status_hn) sys.exit(1) print('beeflow components:') diff --git a/beeflow/common/db/client_db.py b/beeflow/common/db/client_db.py index d7b7ca5f5..2bb837bae 100644 --- a/beeflow/common/db/client_db.py +++ b/beeflow/common/db/client_db.py @@ -25,13 +25,6 @@ def get_hostname(self): hostname = result return hostname - def get_info(self): - """Return an info object containing port information.""" - stmt = "SELECT * FROM info" - result = bdb.getone(self.db_file, stmt) - info = self.Info(*result) - return info - class ClientDB: """Client database.""" From 18f23f97b0cd123674cb46349550b004ee73cb53 Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Fri, 1 Nov 2024 16:39:22 -0600 Subject: [PATCH 16/17] adding checks to start and reset methods and modifying check to stop method --- beeflow/client/bee_client.py | 17 +++++++++++++---- beeflow/client/core.py | 18 ++++++++++-------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index 1efef46b8..e616cd36e 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -72,7 +72,14 @@ def setup_hostname(start_hn): db.info.set_hostname(start_hn) -def check_hostname(curr_hn, stop=False): +def get_hostname(): + """Check if beeflow is running somewhere else.""" + db = bdb.connect_db(client_db, db_path()) + hn = db.info.get_hostname() + return hn + + +def check_hostname(curr_hn): """Check current front end name matches the one beeflow was started on.""" db = bdb.connect_db(client_db, db_path()) start_hn = db.info.get_hostname() @@ -81,8 +88,6 @@ def check_hostname(curr_hn, stop=False): f'run a command on "{curr_hn}".') if start_hn == "": warn('beeflow has not been started!') - if stop: - db.info.set_hostname("") def error_exit(msg, include_caller=True): @@ -147,7 +152,11 @@ def get_wf_list(): conn = _wfm_conn() resp = conn.get(_url(), timeout=60) except requests.exceptions.ConnectionError: - error_exit('Could not reach WF Manager.') + if get_hostname() == "": + warn('beeflow has not been started!') + sys.exit(1) + else: + error_exit('Could not reach WF Manager.') if resp.status_code != requests.codes.okay: # pylint: disable=no-member error_exit('WF Manager did not return workflow list') diff --git a/beeflow/client/core.py b/beeflow/client/core.py index 01508e2b8..a02b2e6fa 100644 --- a/beeflow/client/core.py +++ b/beeflow/client/core.py @@ -410,6 +410,12 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', help='run in the foreground'), backend: bool = typer.Option(False, '--backend', '-B', help='allow to run on a backend node')): """Start all BEE components.""" + start_hn = socket.gethostname() # hostname when beeflow starts + if bee_client.get_hostname() == "": + bee_client.setup_hostname(start_hn) # add to client db + elif bee_client.get_hostname() != start_hn: + warn(f'Error: beeflow is already running on "{bee_client.get_hostname()}."') + sys.exit(1) if backend: # allow beeflow to run on backend node check_dependencies(backend=True) else: @@ -439,9 +445,7 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F', version = importlib.metadata.version("hpc-beeflow") print(f'Starting beeflow {version}...') - start_hn = socket.gethostname() # hostname when beeflow starts print(f'Running beeflow on {start_hn}') - bee_client.setup_hostname(start_hn) # add to client db if not foreground: print('Run `beeflow core status` for more information.') # Create the log path if it doesn't exist yet @@ -505,17 +509,13 @@ def stop(query='yes'): bee_client.pause(wf_id) resp = cli_connection.send(paths.beeflow_socket(), {'type': 'quit'}) if resp is None: - beeflow_log = paths.log_fname('beeflow') - warn('Error: beeflow is not running on this system. It could be ' - 'running on a different front end.\n' - f' Check the beeflow log: "{beeflow_log}".') - bee_client.check_hostname(stop_hn, stop=True) + bee_client.check_hostname(stop_hn) sys.exit(1) # As long as it returned something, we should be good beeflow_log = paths.log_fname('beeflow') if query == "yes": print(f'Beeflow has stopped. Check the log at "{beeflow_log}".') - bee_client.check_hostname(stop_hn, stop=True) + bee_client.setup_hostname("") def archive_dir(dir_to_archive): @@ -552,6 +552,8 @@ def reset(archive: bool = typer.Option(False, '--archive', '-a', help='Archive bee_workdir before removal')): """Stop all components and delete the bee_workdir directory.""" # Check workflow states; warn if there are active states. + reset_hn = socket.gethostname() + bee_client.check_hostname(reset_hn) workflow_list = bee_client.get_wf_list() active_states = {'Running', 'Paused', 'Initializing', 'Waiting'} caution = "" From 9d18e84663628dbb0a1dec92ddc0b1d67a1be09e Mon Sep 17 00:00:00 2001 From: Krishna Chilleri Date: Fri, 1 Nov 2024 16:41:54 -0600 Subject: [PATCH 17/17] fix pylama --- beeflow/client/bee_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beeflow/client/bee_client.py b/beeflow/client/bee_client.py index e616cd36e..d7f6804a1 100644 --- a/beeflow/client/bee_client.py +++ b/beeflow/client/bee_client.py @@ -75,8 +75,8 @@ def setup_hostname(start_hn): def get_hostname(): """Check if beeflow is running somewhere else.""" db = bdb.connect_db(client_db, db_path()) - hn = db.info.get_hostname() - return hn + curr_hn = db.info.get_hostname() + return curr_hn def check_hostname(curr_hn):