From ecb1a6c670526e8595dba9f1c9918a80cbec97d9 Mon Sep 17 00:00:00 2001 From: Douglas Jacobsen Date: Wed, 29 Jan 2025 13:55:18 -0700 Subject: [PATCH 1/3] Add front-end commands for managing includes This commit adds front-end commands to help add, remove, and inspect includes within a workspace. Includes are not defined in a configuration section, and as a result are difficult to manage through `ramble config` commands. --- lib/ramble/ramble/cmd/workspace.py | 58 +++++++++++++++++++++++- lib/ramble/ramble/namespace.py | 1 + lib/ramble/ramble/workspace/workspace.py | 46 +++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/lib/ramble/ramble/cmd/workspace.py b/lib/ramble/ramble/cmd/workspace.py index 0b0914375..01f0b1bf5 100644 --- a/lib/ramble/ramble/cmd/workspace.py +++ b/lib/ramble/ramble/cmd/workspace.py @@ -34,6 +34,7 @@ import ramble.software_environments import ramble.util.colors as rucolor from ramble.util.logger import logger +from ramble.namespace import namespace description = "manage experiment workspaces" @@ -59,7 +60,7 @@ "manage", ] -manage_commands = ["experiments", "software"] +manage_commands = ["experiments", "software", "includes"] def workspace_activate_setup_parser(subparser): @@ -1315,6 +1316,61 @@ def workspace_manage_software(args): ws.print_config() +def workspace_manage_includes_setup_parser(subparser): + """manage workspace includes""" + actions = subparser.add_mutually_exclusive_group() + actions.add_argument( + "--list", "-l", action="store_true", help="whether to print existing includes" + ) + + actions.add_argument( + "--remove", + "-r", + dest="remove_pattern", + metavar="PATTERN", + help="whether to remove an existing include by name / pattern", + ) + + actions.add_argument( + "--remove-index", + dest="remove_index", + metavar="IDX", + help="whether to remove an existing include by index", + ) + + actions.add_argument( + "--add", "-a", dest="add_include", metavar="PATH", help="whether to add a new include" + ) + + +def workspace_manage_includes(args): + """Execute workspace manage include command""" + + ws = ramble.cmd.require_active_workspace(cmd_name="workspace manage includes") + + if args.list: + with ws.read_transaction(): + workspace_dict = ws._get_workspace_dict() + if namespace.include in workspace_dict[namespace.ramble]: + includes = workspace_dict[namespace.ramble][namespace.include] + if includes: + logger.msg("Workspace includes:") + for idx, include in enumerate(includes): + logger.msg(f"{idx}: {include}") + return + logger.msg("Workspace contains no includes.") + elif args.remove_index: + remove_index = int(args.remove_index) + with ws.write_transaction(): + ws.remove_include(index=remove_index) + elif args.remove_pattern: + with ws.write_transaction(): + ws.remove_include(pattern=args.remove_pattern) + elif args.add_include: + with ws.write_transaction(): + ws.add_include(args.add_include) + + def workspace_generate_config_setup_parser(subparser): """generate current workspace config""" workspace_manage_experiments_setup_parser(subparser) diff --git a/lib/ramble/ramble/namespace.py b/lib/ramble/ramble/namespace.py index 156ac06a9..9d6232c9f 100644 --- a/lib/ramble/ramble/namespace.py +++ b/lib/ramble/ramble/namespace.py @@ -65,3 +65,4 @@ class namespace: workflow_manager = "workflow_manager" metadata = "metadata" + include = "include" diff --git a/lib/ramble/ramble/workspace/workspace.py b/lib/ramble/ramble/workspace/workspace.py index 1728415c2..9c5979394 100644 --- a/lib/ramble/ramble/workspace/workspace.py +++ b/lib/ramble/ramble/workspace/workspace.py @@ -1891,6 +1891,52 @@ def workspace_paths(self): return self._workspace_path_replacements + def add_include(self, new_include): + """Add a new include to this workspace""" + + if namespace.include not in self.config_sections["workspace"]["yaml"][namespace.ramble]: + self.config_sections["workspace"]["yaml"][namespace.ramble][namespace.include] = [] + includes = self.config_sections["workspace"]["yaml"][namespace.ramble][namespace.include] + includes.append(new_include) + self._write_config(config_section) + + def remove_include(self, index=None, pattern=None): + """Remove one or more includes from this workspace. + + Args: + index (optional): Numerical position of include to remove + pattern (optional): String or pattern of include to remove. + Removes all matching includes. + """ + + if namespace.include not in self.config_sections["workspace"]["yaml"][namespace.ramble]: + return + + includes = self.config_sections["workspace"]["yaml"][namespace.ramble][namespace.include] + changed = False + + if index is not None: + if index < 0 or index >= len(includes): + logger.die( + f"Requested index {index} " "is outside of the range of existing includes." + ) + includes.pop(index) + changed = True + + if pattern is not None: + remove_indices = [] + for idx, include in enumerate(includes): + if fnmatch.fnmatch(include, pattern): + remove_indices.append(idx) + + for remove_idx in reversed(remove_indices): + if remove_idx >= 0 and remove_idx < len(includes): + includes.pop(remove_idx) + changed = True + + if changed: + self._write_config(config_section) + def included_config_scopes(self): """List of included configuration scopes from the environment. From 650481f238b2eeb7ea4e782fc17f65be72a44a28 Mon Sep 17 00:00:00 2001 From: Douglas Jacobsen Date: Wed, 29 Jan 2025 14:10:32 -0700 Subject: [PATCH 2/3] Add tests for managing workspace includes --- .../test/end_to_end/workspace_includes.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 lib/ramble/ramble/test/end_to_end/workspace_includes.py diff --git a/lib/ramble/ramble/test/end_to_end/workspace_includes.py b/lib/ramble/ramble/test/end_to_end/workspace_includes.py new file mode 100644 index 000000000..7e2f26743 --- /dev/null +++ b/lib/ramble/ramble/test/end_to_end/workspace_includes.py @@ -0,0 +1,141 @@ +# Copyright 2022-2025 The Ramble Authors +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +import os + +import pytest + +import ramble.workspace +import ramble.config +import ramble.software_environments +from ramble.main import RambleCommand + + +# everything here uses the mock_workspace_path +pytestmark = pytest.mark.usefixtures( + "mutable_config", + "mutable_mock_workspace_path", +) + +workspace = RambleCommand("workspace") + + +def test_workspace_add_includes(request): + workspace_name = request.node.name + ws = ramble.workspace.create(workspace_name) + global_args = ["-w", workspace_name] + + ws.write() + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "Workspace contains no includes." in output + + workspace( + "manage", + "includes", + "--add", + "$workspace_configs/auxiliary_software_files", + global_args=global_args, + ) + + ws._re_read() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + with open(config_path) as f: + data = f.read() + assert "- $workspace_configs/auxiliary_software_files" in data + + +def test_workspace_remove_includes_index(request): + workspace_name = request.node.name + ws = ramble.workspace.create(workspace_name) + global_args = ["-w", workspace_name] + + ws.write() + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "Workspace contains no includes." in output + + workspace( + "manage", + "includes", + "--add", + "$workspace_configs/auxiliary_software_files", + global_args=global_args, + ) + + ws._re_read() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "0: $workspace_configs/auxiliary_software_files" in output + + with open(config_path) as f: + data = f.read() + assert "- $workspace_configs/auxiliary_software_files" in data + + workspace("manage", "includes", "--remove-index", "0", global_args=global_args) + + ws._re_read() + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "Workspace contains no includes." in output + + with open(config_path) as f: + data = f.read() + assert "- $workspace_configs/auxiliary_software_files" not in data + + +def test_workspace_remove_includes_pattern(request): + workspace_name = request.node.name + ws = ramble.workspace.create(workspace_name) + global_args = ["-w", workspace_name] + + ws.write() + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "Workspace contains no includes." in output + + workspace( + "manage", + "includes", + "--add", + "$workspace_configs/auxiliary_software_files", + global_args=global_args, + ) + + ws._re_read() + + config_path = os.path.join(ws.config_dir, ramble.workspace.config_file_name) + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "0: $workspace_configs/auxiliary_software_files" in output + + with open(config_path) as f: + data = f.read() + assert "- $workspace_configs/auxiliary_software_files" in data + + workspace("manage", "includes", "--remove", "*aux*", global_args=global_args) + + ws._re_read() + + output = workspace("manage", "includes", "--list", global_args=global_args) + + assert "Workspace contains no includes." in output + + with open(config_path) as f: + data = f.read() + assert "- $workspace_configs/auxiliary_software_files" not in data From 8ff3d33fa25ebc5988c786b74b7160fd39a6f878 Mon Sep 17 00:00:00 2001 From: Douglas Jacobsen Date: Wed, 29 Jan 2025 14:33:05 -0700 Subject: [PATCH 3/3] Update bash completion --- share/ramble/ramble-completion.bash | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/share/ramble/ramble-completion.bash b/share/ramble/ramble-completion.bash index 9da6ceef9..7b45852b0 100755 --- a/share/ramble/ramble-completion.bash +++ b/share/ramble/ramble-completion.bash @@ -747,7 +747,7 @@ _ramble_workspace_manage() { then RAMBLE_COMPREPLY="-h --help" else - RAMBLE_COMPREPLY="experiments software" + RAMBLE_COMPREPLY="experiments software includes" fi } @@ -763,3 +763,7 @@ _ramble_workspace_manage_experiments() { _ramble_workspace_manage_software() { RAMBLE_COMPREPLY="-h --help --environment-name --env --environment-packages --external-env --package-name --pkg --package-spec --pkg-spec --spec --compiler-package --compiler-pkg --compiler --compiler-spec --package-manager-prefix --prefix --remove --delete --overwrite -o --dry-run --print" } + +_ramble_workspace_manage_includes() { + RAMBLE_COMPREPLY="-h --help --list -l --remove -r --remove-index --add -a" +}