Skip to content

Commit

Permalink
Issue963/Make beeflow config new non-interactive by default (#983)
Browse files Browse the repository at this point in the history 
* Make beeflow config new non-interactive by default

* Updated coverage.svg

* No config walkthrough messages when non-interactive
  • Loading branch information
@arhall0
arhall0 authored Jan 23, 2025
1 parent 8123777 commit 1c31558
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 63 deletions.
127 changes: 73 additions & 54 deletions beeflow/common/config_driver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ def init(cls, userconfig=None, **_kwargs):
break
except FileNotFoundError:
print("Configuration file is missing! Generating new config file.")
new(USERCONFIG_FILE)
new(USERCONFIG_FILE, interactive=False)
# remove default keys from the other sections
cls.CONFIG = config_utils.filter_and_validate(config, VALIDATOR)

Expand Down Expand Up @@ -241,36 +241,39 @@ def filepath_completion_input(*pargs, **kwargs):
VALIDATOR.section('DEFAULT', info='Default bee.conf configuration section.')

VALIDATOR.option('DEFAULT', 'bee_workdir', info='main BEE workdir',
default=DEFAULT_BEE_WORKDIR, validator=validation.make_dir)
default=DEFAULT_BEE_WORKDIR, validator=validation.make_dir,
prompt=True)

VALIDATOR.option('DEFAULT', 'bee_archive_dir', info='directory to store workflow archives',
default=DEFAULT_BEE_ARCHIVE_DIR, validator=validation.make_dir)
default=DEFAULT_BEE_ARCHIVE_DIR, validator=validation.make_dir,
prompt=True)

VALIDATOR.option('DEFAULT', 'bee_droppoint', info='BEE remote workflow drop point',
default=DEFAULT_BEE_DROPPOINT, validator=validation.make_dir)
default=DEFAULT_BEE_DROPPOINT, validator=validation.make_dir,
prompt=False)

VALIDATOR.option('DEFAULT', 'remote_api', info='BEE remote REST API activation',
default=False, validator=validation.bool_)
default=False, validator=validation.bool_, prompt=False)

VALIDATOR.option('DEFAULT', 'remote_api_port', info='BEE remote REST API port',
default=7777, validator=int)
default=7777, validator=int, prompt=False)

VALIDATOR.option('DEFAULT', 'workload_scheduler', choices=('Slurm', 'LSF', 'Flux', 'Simple'),
info='backend workload scheduler to interact with ')
default='Slurm', info='backend workload scheduler to interact with ',
prompt=True)

VALIDATOR.option('DEFAULT', 'delete_completed_workflow_dirs', validator=validation.bool_,
default=True, info='delete workflow directory for completed jobs')
default=True, info='delete workflow directory for completed jobs', prompt=False)

VALIDATOR.option('DEFAULT', 'neo4j_image', validator=validation.file_,
default=NEO4J_IMAGE, info='neo4j container image',
input_fn=filepath_completion_input)
input_fn=filepath_completion_input, prompt=True)

VALIDATOR.option('DEFAULT', 'redis_image', validator=validation.file_,
default=REDIS_IMAGE, info='redis container image',
input_fn=filepath_completion_input)
input_fn=filepath_completion_input, prompt=True)

VALIDATOR.option('DEFAULT', 'max_restarts', validator=int,
default=3,
VALIDATOR.option('DEFAULT', 'max_restarts', validator=int, default=3, prompt=False,
info='max number of times beeflow will restart a component on failure')

# Workflow Manager
Expand All @@ -279,27 +282,27 @@ def filepath_completion_input(*pargs, **kwargs):
VALIDATOR.section('task_manager',
info='Task manager configuration and config of container to use.')
VALIDATOR.option('task_manager', 'container_runtime', default='Charliecloud',
choices=('Charliecloud', 'Singularity'),
choices=('Charliecloud', 'Singularity'), prompt=False,
info='container runtime to use for configuration')
VALIDATOR.option('task_manager', 'runner_opts', default='',
VALIDATOR.option('task_manager', 'runner_opts', default='', prompt=False,
info='special runner options to pass to the runner opts')
VALIDATOR.option('task_manager', 'background_interval', default=5,
validator=int,
validator=int, prompt=False,
info='interval at which the task manager processes queues and updates states')

# Charliecloud (depends on task_manager::container_runtime == Charliecloud)
VALIDATOR.section('charliecloud', info='Charliecloud configuration section.',
depends_on=('task_manager', 'container_runtime', 'Charliecloud'))
VALIDATOR.option('charliecloud', 'image_mntdir', default=join_path('/tmp', USER),
VALIDATOR.option('charliecloud', 'image_mntdir', default=join_path('/tmp', USER), prompt=False,
info='Charliecloud mount directory', validator=validation.make_dir)
# General job requirements
VALIDATOR.section('job', info='General job requirements.')
VALIDATOR.option('job', 'default_account', validator=lambda val: val.strip(),
info='default account to launch jobs with (leave blank if none)')
VALIDATOR.option('job', 'default_time_limit', validator=validation.time_limit,
info='default account time limit (leave blank if none)')
VALIDATOR.option('job', 'default_partition', validator=lambda val: val.strip(),
info='default partition to run jobs on (leave blank if none)')
VALIDATOR.option('job', 'default_account', validator=lambda val: val.strip(), prompt=True,
default='', info='default account to launch jobs with (leave blank if none)')
VALIDATOR.option('job', 'default_time_limit', validator=validation.time_limit, prompt=True,
default='', info='default account time limit (leave blank if none)')
VALIDATOR.option('job', 'default_partition', validator=lambda val: val.strip(), prompt=True,
default='', info='default partition to run jobs on (leave blank if none)')


def validate_chrun_opts(opts):
Expand All @@ -313,48 +316,49 @@ def validate_chrun_opts(opts):


VALIDATOR.option('charliecloud', 'chrun_opts', default='--home',
validator=validate_chrun_opts,
validator=validate_chrun_opts, prompt=False,
info='extra options to pass to ch-run')
VALIDATOR.option('charliecloud', 'setup', default='',
VALIDATOR.option('charliecloud', 'setup', default='', prompt=False,
info='extra Charliecloud setup to put in a job script')
# Graph Database
VALIDATOR.section('graphdb', info='Main graph database configuration section.')
VALIDATOR.option('graphdb', 'hostname', default='localhost',
VALIDATOR.option('graphdb', 'hostname', default='localhost', prompt=False,
info='hostname of database')

VALIDATOR.option('graphdb', 'dbpass', default='password', info='password for database')
VALIDATOR.option('graphdb', 'dbpass', default='password', info='password for database',
prompt=False)

VALIDATOR.option('graphdb', 'gdb_image_mntdir', default=join_path('/tmp', USER),
VALIDATOR.option('graphdb', 'gdb_image_mntdir', default=join_path('/tmp', USER), prompt=False,
info='graph database image mount directory', validator=validation.make_dir)
VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=1,
VALIDATOR.option('graphdb', 'sleep_time', validator=int, default=1, prompt=False,
info='how long to wait for the graph database to come up (this can take a while, '
'depending on the system)')
# Builder
VALIDATOR.section('builder', info='General builder configuration section.')
VALIDATOR.option('builder', 'deployed_image_root', default='/tmp',
VALIDATOR.option('builder', 'deployed_image_root', default='/tmp', prompt=False,
info='where to deploy container images', validator=validation.make_dir)
VALIDATOR.option('builder', 'container_output_path', default='/tmp',
VALIDATOR.option('builder', 'container_output_path', default='/tmp', prompt=False,
info='container output path', validator=validation.make_dir)
VALIDATOR.option('builder', 'container_archive',
VALIDATOR.option('builder', 'container_archive', prompt=True,
default=join_path(DEFAULT_BEE_WORKDIR, 'container_archive'),
info='container archive location')
VALIDATOR.option('builder', 'container_type', default='charliecloud',
info='container type to use')
info='container type to use', prompt=False)
# Slurmrestd (depends on DEFAULT:workload_scheduler == Slurm)
VALIDATOR.section('slurm', info='Configuration section for Slurm.',
depends_on=('DEFAULT', 'workload_scheduler', 'Slurm'))
VALIDATOR.option('slurm', 'use_commands', validator=validation.bool_,
default=(shutil.which('slurmrestd') is None),
default=(shutil.which('slurmrestd') is None), prompt=False,
info='if set, use slurm cli commands instead of slurmrestd')
DEFAULT_SLURMRESTD_SOCK = join_path('/tmp', f'slurm_{USER}_{random.randint(1, 10000)}.sock')

# Scheduler
VALIDATOR.section('scheduler', info='Scheduler configuration section.')
SCHEDULER_ALGORITHMS = ('fcfs', 'backfill', 'sjf')
VALIDATOR.option('scheduler', 'algorithm', default='fcfs', choices=SCHEDULER_ALGORITHMS,
info='scheduling algorithm to use')
info='scheduling algorithm to use', prompt=False)
VALIDATOR.option('scheduler', 'default_algorithm', default='fcfs',
choices=SCHEDULER_ALGORITHMS,
choices=SCHEDULER_ALGORITHMS, prompt=False,
info=('default algorithm to use'))


Expand All @@ -373,19 +377,20 @@ def __init__(self, fname, validator):
self.validator = validator
self.sections = {}

def choose_values(self):
def choose_values(self, interactive=False, flux=False):
"""Choose configuration values based on user input."""
dirname = os.path.dirname(self.fname)
if dirname:
os.makedirs(dirname, exist_ok=True)
print(f'Creating a new config: "{self.fname}".')
print()
print_wrap('This will walk you through creating a new configuration for BEE. '
'Note that you will only be required to enter values for options '
'without defaults. Please take a look at the other options and '
'their defaults before running BEE.')
print()
print('Please enter values for the following sections and options:')
if interactive:
print_wrap('This will walk you through creating a new configuration for BEE. '
'Note that you will only be required to enter values for some '
'options. Please take a look at the other options and their '
'values before running BEE.')
print()
print('Please enter values for the following sections and options:')
# Let the user choose values for each required attribute
for sec_name, section in self.validator.sections:
# Determine if this section is valid under the current configuration
Expand All @@ -401,10 +406,13 @@ def choose_values(self):
print()
printed = True

this_default = option.default
if flux is True and opt_name == 'workload_scheduler':
this_default = "Flux"
option.prompt = False
# Check for a default value
if option.default is not None:
default = option.default
value = option.validate(default)
if (not interactive or option.prompt is False) and this_default is not None:
value = option.validate(this_default)
print(f'Setting option "{opt_name}" to default value "{value}".')
print()
self.sections[sec_name][opt_name] = value
Expand All @@ -427,7 +435,8 @@ def _input_loop(self, opt_name, option):
print_wrap(f'{opt_name} - {option.info}')
if option.choices is not None:
print(f'(allowed values: {",".join(option.choices)})')
value = input_fn(f'{opt_name}: ')
value = input_fn(f'Enter selection for {opt_name} or\n'
+ f'leave blank for default ({option.default}): ') or option.default
# Validate the input
try:
option.validate(value)
Expand All @@ -436,7 +445,7 @@ def _input_loop(self, opt_name, option):
value = None
return value

def save(self):
def save(self, interactive=False):
"""Save the config to a file."""
print()
print('The following configuration options were chosen:')
Expand All @@ -446,10 +455,11 @@ def save(self):
for opt_name in section:
print(f'{opt_name} = {section[opt_name]}')
print()
ans = input('Would you like to save this config? [y/n] ')
if ans.lower() != 'y':
print('Quitting without saving')
return
if interactive:
ans = input('Would you like to save this config? [y/n] ')
if ans.lower() != 'y':
print('Quitting without saving')
return
config_utils.write_config(self.fname, self.sections)
print(70 * '#')
print('Before running BEE, check defaults in the configuration file:',
Expand Down Expand Up @@ -568,12 +578,21 @@ def info():

@app.command()
def new(path: str = typer.Argument(default=USERCONFIG_FILE,
help='Path to new config file')):
help='Path to new config file'),
interactive: bool = typer.Option(False, '--interactive', '-i',
help='Whether or not to be prompted'
+ ' during config generation'),
flux: bool = typer.Option(False, '--flux', '-f',
help='Changes default scheduler to Flux')):
"""Create a new config file."""
if os.path.exists(path):
if check_yes(f'Path "{path}" already exists.\nWould you like to save a copy of it?'):
if not interactive or check_yes(f'Path "{path}" already exists.\n'
+ 'Would you like to save a copy of it?'):
config_utils.backup(path)
ConfigGenerator(path, VALIDATOR).choose_values().save()
ConfigGenerator(path, VALIDATOR).choose_values(
flux=flux,
interactive=interactive
).save(interactive=interactive)


@app.command()
Expand Down
4 changes: 3 additions & 1 deletion beeflow/common/config_validator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,8 @@ def __init__(self, info, depends_on=None):
class ConfigOption:
"""Config option/validation class."""

def __init__(self, info, validator=str, choices=None, default=None, input_fn=None):
def __init__(self, info, validator=str, choices=None, default=None, input_fn=None,
prompt=False):
"""Construct the config option class."""
self.info = info
self.choices = choices
Expand All @@ -169,6 +170,7 @@ def __init__(self, info, validator=str, choices=None, default=None, input_fn=Non
# Function used to set the option from user input at config generation time
self.input_fn = input_fn
self._validator = validator
self.prompt = prompt

def validate(self, value):
"""Validate the value and return it."""
Expand Down
63 changes: 62 additions & 1 deletion beeflow/tests/test_config_driver.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""Unit tests for the config driver."""

import os
import pytest
from beeflow.common.config_driver import AlterConfig
from beeflow.common.config_driver import AlterConfig, ConfigGenerator, new
from beeflow.common.config_validator import ConfigValidator


# AlterConfig tests
Expand Down Expand Up @@ -103,3 +105,62 @@ def test_change_value_multiple_times(mocker):
alter_config.change_value("DEFAULT", "bee_workdir", "/path/two")
assert alter_config.config["DEFAULT"]["bee_workdir"] == "/path/two"
assert alter_config.changes == {"DEFAULT": {"bee_workdir": "/path/two"}}


@pytest.mark.parametrize("interactive, flux, prompt, expected",
[
(True, True, False, 'Flux'),
(True, False, False, 'default'),
(True, True, True, 'Flux'),
(True, False, True, 'input'),
(False, True, False, 'Flux'),
(False, False, False, 'default'),
],
)
def test_choose_values(interactive, flux, prompt, expected):
"""Test running choose_values with different flags."""
validator = ConfigValidator('')
validator.section('sec', info='')
validator.option('sec', 'workload_scheduler', info='', default='default',
validator=lambda _: _, input_fn=lambda _: 'input', prompt=prompt)
config_generator = ConfigGenerator('', validator)
config = config_generator.choose_values(interactive=interactive,
flux=flux).sections
assert config['sec']['workload_scheduler'] == expected


@pytest.mark.parametrize("interactive, input_val, expected",
[
(True, 'y', 'bee.conf'),
(True, 'n', 'Quitting'),
(False, '', 'bee.conf'),
],
)
def test_config_generator_save(mocker, tmpdir, capsys, interactive, input_val, expected):
"""Test the ConfigGenerator save function with different flags/input options."""
mocker.patch('builtins.input', return_value=input_val)
alter_config = ConfigGenerator('bee.conf', lambda _: _)
with tmpdir.as_cwd():
alter_config.save(interactive=interactive)
captured = capsys.readouterr()
assert expected in captured.out


@pytest.mark.parametrize("interactive, input_val, expected",
[
(True, 'y', True),
(True, 'n', False),
(False, '', True),
],
)
def test_config_new(mocker, tmpdir, interactive, input_val, expected):
"""Test creation of new config with different settings."""
filename = 'bee.conf'
mocker.patch('builtins.input', return_value=input_val)
mocker.patch('beeflow.common.config_driver.ConfigGenerator.choose_values')
mocker.patch('beeflow.common.config_driver.ConfigGenerator.choose_values.save')
with tmpdir.as_cwd():
with open(filename, 'w', encoding='utf-8'):
pass
new("bee.conf", interactive=interactive)
assert os.path.exists('bee.conf.1') == expected
4 changes: 2 additions & 2 deletions coverage.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit 1c31558

Please sign in to comment.