From a9e0c66f5299ab30a63bb8c05a7fa4704e53a9ec Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Mon, 26 Feb 2024 00:04:03 +0100 Subject: [PATCH] First working version of the config parser This was the hard part. In particular, autocompletion works now in a pretty generic way. And the help info shows which arguments have values in the Qleverfile. TODO: Right now, the help info is still Python-style. At some point, we should format this in a way that is more consistent with the output of the rest of the script. --- .gitignore | 4 + src/qlever/__init__.py | 8 +- src/qlever/command.py | 20 ++- src/qlever/commands/index.py | 12 +- src/qlever/commands/sehr_doof.py | 12 +- src/qlever/config.py | 229 ++++++++++++++++++++++--------- src/qlever/log.py | 30 ++++ src/qlever/qlever_main.py | 18 ++- 8 files changed, 251 insertions(+), 82 deletions(-) create mode 100644 src/qlever/log.py diff --git a/.gitignore b/.gitignore index bee8a64b..762a39c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ __pycache__ +build/ +dist/ +*.egg-info +*.swp diff --git a/src/qlever/__init__.py b/src/qlever/__init__.py index 4aa10ff2..c084f6cd 100644 --- a/src/qlever/__init__.py +++ b/src/qlever/__init__.py @@ -21,7 +21,7 @@ def snake_to_camel(str): f"{module_path} for command {command}: {e}") command_classes[command_name] = getattr(module, class_name) -print(f"Package path: {package_path}") -print(f"Command names: {command_names}") -print(f"Command classes: {command_classes}") -print() +# print(f"Package path: {package_path}") +# print(f"Command names: {command_names}") +# print(f"Command classes: {command_classes}") +# print() diff --git a/src/qlever/command.py b/src/qlever/command.py index c6df9d4a..5713747c 100644 --- a/src/qlever/command.py +++ b/src/qlever/command.py @@ -9,18 +9,30 @@ class QleverCommand(ABC): @staticmethod @abstractmethod - def add_subparser(subparsers): + def help_text(): """ - Add a subparser for the command to the given `subparsers` object. + Return the help text that will be shown upon `qlever --help`. """ pass @staticmethod @abstractmethod - def arguments(): + def relevant_arguments(): """ Retun the arguments relevant for this command. This must be a subset of - the arguments defined in the `QleverConfig` object. + the names of `all_arguments` defined in `QleverConfig`. Only these + arguments can then be used in the `execute` method. + """ + pass + + @staticmethod + @abstractmethod + def should_have_qleverfile(): + """ + Return `True` if the command should have a Qleverfile, `False` + otherwise. If a command should have a Qleverfile, but none is + specified, the command can still be executed if all the required + arguments are specified on the command line, but there will be warning. """ pass diff --git a/src/qlever/commands/index.py b/src/qlever/commands/index.py index eafb2120..b3c25ca1 100644 --- a/src/qlever/commands/index.py +++ b/src/qlever/commands/index.py @@ -7,15 +7,17 @@ class IndexCommand(QleverCommand): """ @staticmethod - def add_subparser(subparsers): - subparsers.add_parser( - "index", - help="Building an index") + def help_text(): + return "Building an index" @staticmethod - def arguments(): + def relevant_arguments(): return {"index": ["cat_files", "settings_json"]} + @staticmethod + def should_have_qleverfile(): + return True + @staticmethod def execute(args): print(f"Executing command `doof` with args: {args}") diff --git a/src/qlever/commands/sehr_doof.py b/src/qlever/commands/sehr_doof.py index 44013b47..772228f1 100644 --- a/src/qlever/commands/sehr_doof.py +++ b/src/qlever/commands/sehr_doof.py @@ -7,15 +7,17 @@ class SehrDoofCommand(QleverCommand): """ @staticmethod - def add_subparser(subparsers): - subparsers.add_parser( - "sehr_doof", - help="Sehr doofes command") + def help_text(): + return "Sehr doofes command" @staticmethod - def arguments(): + def relevant_arguments(): return {"data": ["name"], "server": ["port"]} + @staticmethod + def should_have_qleverfile(): + return False + @staticmethod def execute(args): print(f"Executing command `sehr doof` with args: {args}") diff --git a/src/qlever/config.py b/src/qlever/config.py index a8f9249c..6bb14993 100644 --- a/src/qlever/config.py +++ b/src/qlever/config.py @@ -1,24 +1,40 @@ import argparse import argcomplete from qlever import command_classes +from configparser import ConfigParser, ExtendedInterpolation from pathlib import Path +from qlever.log import log import os import sys import shlex +import traceback # Simple exception class for configuration errors (the class need not do # anything, we just want a distinct exception type). class ConfigException(Exception): - pass + def __init__(self, message): + stack = traceback.extract_stack()[-2] # Caller's frame. + self.filename = stack.filename + self.lineno = stack.lineno + full_message = f"{message} [in {self.filename}:{self.lineno}]" + super().__init__(full_message) -# Class that manages all config parameters, and overwrites them with the -# settings from a Qleverfile. class QleverConfig: + """ + Class that manages all config parameters, and overwrites them with the + settings from a Qleverfile. - @staticmethod - def all_arguments(): + IMPORTANT: An instance of this class is created for each execution of + the `qlever` script, in particular, each time the user triggers + autocompletion. It's initialization should therefore be lightweight. In + particular, the functions `all_arguments()` and `parse_qleverfile()` should + not be called in the constructor, but only when needed (which is, when the + user has already committed to a command). + """ + + def all_arguments(self): """ Define all possible parameters. A value of `None` means that there is no default value, and the parameter is mandatory in the Qleverfile (or @@ -32,12 +48,12 @@ def all_arguments(): def arg(*args, **kwargs): return (args, kwargs) - arguments = {} - data_args = arguments["data"] = {} - index_args = arguments["index"] = {} - server_args = arguments["server"] = {} - runtime_args = arguments["runtime"] = {} - ui_args = arguments["ui"] = {} + all_args = {} + data_args = all_args["data"] = {} + index_args = all_args["index"] = {} + server_args = all_args["server"] = {} + runtime_args = all_args["runtime"] = {} + ui_args = all_args["ui"] = {} data_args["name"] = arg( "--name", type=str, required=True, @@ -66,61 +82,150 @@ def arg(*args, **kwargs): "--memory-for-queries", type=str, default="1G", help="The maximal memory allowed for query processing") - return arguments + runtime_args["environment"] = arg( + "--environment", type=str, + choices=["docker", "podman", "native"], + default="docker", + help="The runtime environment for the server") + + ui_args["port"] = arg( + "--port", type=int, default=7000, + help="The port of the Qlever UI web app") + + return all_args + + def parse_qleverfile(self, qleverfile_path): + """ + Parse the given Qleverfile (the function assumes that it exists) and + return a `ConfigParser` object with all the options and their values. + + NOTE: The keys have the same hierarchical structure as the keys in + `all_arguments()`. The Qleverfile may contain options that are not + defined in `all_arguments()`. They can be used as temporary variables + to define other options, but cannot be accessed by the commands later. + """ + + config = ConfigParser(interpolation=ExtendedInterpolation()) + try: + config.read(qleverfile_path) + return config + except Exception as e: + raise ConfigException(f"Error parsing {qleverfile_path}: {e}") + + def add_arguments_to_subparser(self, subparser, arg_names, all_args, + qleverfile_config): + """ + Add the specified arguments to the given subparser. Take the default + values from `all_arguments()` and overwrite them with the values from + the `Qleverfile`, in case it exists. + + IMPORTANT: Don't call this function repeatedly (in particular, not for + every command, but only for the command for which it is needed), it's + not cheap. + """ - @staticmethod - def parse_args(): - # Add subparser for each command via the `add_subparser` method of the - # corresponding command class (the classes are loaded dynamically in - # `__init__.py`). + for section in arg_names: + if section not in all_args: + raise ConfigException(f"Section '{section}' not found") + for arg_name in arg_names[section]: + if arg_name not in all_args[section]: + raise ConfigException(f"Argument '{arg_name}' not found " + f"in section '{section}'") + args, kwargs = all_args[section][arg_name] + # If `qleverfile_config` is given, add info about default + # values to the help string. + if qleverfile_config is not None: + default_value = kwargs.get("default", None) + qleverfile_value = qleverfile_config.get( + section, arg_name, fallback=None) + if qleverfile_value is not None: + kwargs["default"] = qleverfile_value + kwargs["required"] = False + kwargs["help"] += (f" [default, from Qleverfile:" + f" {qleverfile_value}]") + else: + kwargs["help"] += f" [default: {default_value}]" + subparser.add_argument(*args, **kwargs) + + def get_command_line_arguments(self): + """ + Get the current command line arguments. + + NOTE: This should work both when the script is called "normally" (in + which case, we can just use `sys.argv`), as well as when it is called + by the shell's autocompletion mechanism (in which case, the command + line is in the environment variable `COMP_LINE`). + """ + if "COMP_LINE" in os.environ: + # Note: `COMP_LINE` is a string, with spaces used to separate the + # arguments and spaces within arguments escaped with a backslash. + return shlex.split(os.environ["COMP_LINE"]) + else: + return sys.argv + + def parse_args(self): + # Create a temporary parser only to parse the `--qleverfile` option, in + # case it is given. This is because in the actual parser below we want + # the values from the Qleverfile to be shown in the help strings. + def add_qleverfile_option(parser): + parser.add_argument( + "--qleverfile", "-q", type=str, default="Qleverfile", + help="The Qleverfile to use (default: Qleverfile)") + qleverfile_parser = argparse.ArgumentParser(add_help=False) + add_qleverfile_option(qleverfile_parser) + qleverfile_args, _ = qleverfile_parser.parse_known_args() + qleverfile_path_name = qleverfile_args.qleverfile + + # If this is the normal execution of the script (and not a call invoked + # by the shell's autocompletion mechanism), parse the Qleverfile. + qleverfile_path = Path(qleverfile_path_name) + qleverfile_exists = qleverfile_path.is_file() + qleverfile_is_default = qleverfile_path_name \ + == qleverfile_parser.get_default("qleverfile") + # If a Qleverfile with a non-default name was specified, but it does + # not exist, that's an error. + if not qleverfile_exists and not qleverfile_is_default: + raise ConfigException(f"Qleverfile with non-default name " + f"`{qleverfile_path_name}` specified, " + f"but it does not exist") + # If it exists and we arenot in the autocompletion mode, parse it. + if qleverfile_exists and "COMP_LINE" not in os.environ: + qleverfile_config = self.parse_qleverfile(qleverfile_path) + else: + qleverfile_config = None + + # Now the regular parser with commands and a subparser for each + # command. We have a dedicated class for each command, these classes + # are defined in the modules in the `qlever/commands` directory and + # dynamically imported in `__init__.py`. parser = argparse.ArgumentParser() + add_qleverfile_option(parser) subparsers = parser.add_subparsers(dest='command') subparsers.required = True - for command in command_classes: - command_classes[command].add_subparser(subparsers) + all_args = self.all_arguments() + for command_name, command_class in command_classes.items(): + help_text = command_class.help_text() + subparser = subparsers.add_parser(command_name, help=help_text) + arg_names = command_class.relevant_arguments() + self.add_arguments_to_subparser(subparser, arg_names, all_args, + qleverfile_config) - # Only if the user has typed a command, do we add the arguments for - # the respective subparser (which entails parsing the Qleverfile). - # - # NOTE 1: We don't want to do this for every command because this has to - # be done whenever the user triggers autocompletion. Also, this parses - # the Qleverfile, and there is no need for that before a command has - # been chosen. - # - # NOTE 2: This code can be reached in two different ways: either in the - # "normal" way, e.g. when the user types `qlever index --help`, or in - # the "autocompletion" way, e.g. when the user types `qlever index - # --`, and the shell's completion function is called. In the - # latter case, the command line is stored in the environment variable - # `COMP_LINE`. - try: - argv = shlex.split(os.environ["COMP_LINE"]) - except KeyError: - argv = sys.argv - if len(argv) > 1: - command_name = argv[1] - if command_name in command_classes: - command_class = command_classes[command_name] - subparser = subparsers.choices[command_name] - command_arguments = command_class.arguments() - all_arguments = QleverConfig.all_arguments() - for section in command_arguments: - if section not in all_arguments: - raise ConfigException(f"Section '{section}' not found") - for arg_name in command_arguments[section]: - if arg_name not in all_arguments[section]: - raise ConfigException( - f"Argument '{arg_name}' not found in section '{section}'") - args, kwargs = all_arguments[section][arg_name] - subparser.add_argument(*args, **kwargs) - - # Enable autocompletion for the commands as well as for the command - # options. + # Enable autocompletion for the commands and their options. # - # NOTE: It is important that all code that is executed before this line - # is relatively cheap because it is executed whenever the user presses - # the key (usually TAB) that invokes autocompletion. + # NOTE: All code executed before this line should be relatively cheap + # because it is executed whenever the user triggers the autocompletion. argcomplete.autocomplete(parser, always_complete_options="long") - - # Parse the command line arguments and return them. - return parser.parse_args() + + # Parse the command line arguments. + args = parser.parse_args() + + # If the command says that we should have a Qleverfile, but we don't, + # issue a warning. + if command_classes[args.command].should_have_qleverfile(): + if not qleverfile_exists: + log.warning(f"Invoking command `{args.command}` without a " + "Qleverfile. You have to specify all required " + "arguments on the command line. This is possible, " + "but not recommended.") + + return args diff --git a/src/qlever/log.py b/src/qlever/log.py new file mode 100644 index 00000000..f65fc809 --- /dev/null +++ b/src/qlever/log.py @@ -0,0 +1,30 @@ +# Copyright 2024, University of Freiburg, +# Chair of Algorithms and Data Structures +# Author: Hannah Bast + +import logging +from termcolor import colored + + +class CustomFormatter(logging.Formatter): + """ + Custom formatter for logging. + """ + def format(self, record): + message = record.getMessage() + if record.levelno == logging.DEBUG: + return colored(f"DEBUG: {message}", "magenta") + elif record.levelno == logging.WARNING: + return colored(f"WARNING: {message}", "yellow") + elif record.levelno in [logging.CRITICAL, logging.ERROR]: + return colored(f"ERROR: {message}", "red") + else: + return message + + +# Custom logger. +log = logging.getLogger("qlever") +log.setLevel(logging.INFO) +handler = logging.StreamHandler() +handler.setFormatter(CustomFormatter()) +log.addHandler(handler) diff --git a/src/qlever/qlever_main.py b/src/qlever/qlever_main.py index 1f2767ac..07195a68 100644 --- a/src/qlever/qlever_main.py +++ b/src/qlever/qlever_main.py @@ -1,9 +1,23 @@ #!/usr/bin/env python3 # PYTHON_ARGCOMPLETE_OK -from qlever.config import QleverConfig +# Copyright 2024, University of Freiburg, +# Chair of Algorithms and Data Structures +# Author: Hannah Bast + +from qlever.config import QleverConfig, ConfigException from qlever.command import execute_command +from termcolor import colored + def main(): - args = QleverConfig.parse_args() + # Parse the command line arguments and read the Qleverfile. + try: + qlever_config = QleverConfig() + args = qlever_config.parse_args() + except ConfigException as e: + print(colored(e, "red")) + exit(1) + + # Execute the command. execute_command(args.command, args)