First working version of the config parser

This was the hard part. In particular, autocompletion works now in a
pretty generic way. And the help info shows which arguments have values
in the Qleverfile.

TODO: Right now, the help info is still Python-style. At some point,
we should format this in a way that is more consistent with the output
of the rest of the script.

.gitignore

@@ -1 +1,5 @@
 __pycache__
+build/
+dist/
+*.egg-info
+*.swp

src/qlever/__init__.py

@@ -21,7 +21,7 @@ def snake_to_camel(str):
                         f"{module_path} for command {command}: {e}")
     command_classes[command_name] = getattr(module, class_name)
 
-print(f"Package path: {package_path}")
-print(f"Command names: {command_names}")
-print(f"Command classes: {command_classes}")
-print()
+# print(f"Package path: {package_path}")
+# print(f"Command names: {command_names}")
+# print(f"Command classes: {command_classes}")
+# print()

src/qlever/command.py

@@ -9,18 +9,30 @@ class QleverCommand(ABC):
 
     @staticmethod
     @abstractmethod
-    def add_subparser(subparsers):
+    def help_text():
         """
-        Add a subparser for the command to the given `subparsers` object.
+        Return the help text that will be shown upon `qlever <command> --help`.
         """
         pass
 
     @staticmethod
     @abstractmethod
-    def arguments():
+    def relevant_arguments():
         """
         Retun the arguments relevant for this command. This must be a subset of
-        the arguments defined in the `QleverConfig` object.
+        the names of `all_arguments` defined in `QleverConfig`. Only these
+        arguments can then be used in the `execute` method.
+        """
+        pass
+
+    @staticmethod
+    @abstractmethod
+    def should_have_qleverfile():
+        """
+        Return `True` if the command should have a Qleverfile, `False`
+        otherwise. If a command should have a Qleverfile, but none is
+        specified, the command can still be executed if all the required
+        arguments are specified on the command line, but there will be warning.
         """
         pass
 

src/qlever/commands/index.py

@@ -7,15 +7,17 @@ class IndexCommand(QleverCommand):
     """
 
     @staticmethod
-    def add_subparser(subparsers):
-        subparsers.add_parser(
-                "index",
-                help="Building an index")
+    def help_text():
+        return "Building an index"
 
     @staticmethod
-    def arguments():
+    def relevant_arguments():
         return {"index": ["cat_files", "settings_json"]}
 
+    @staticmethod
+    def should_have_qleverfile():
+        return True
+
     @staticmethod
     def execute(args):
         print(f"Executing command `doof` with args: {args}")

src/qlever/commands/sehr_doof.py

@@ -7,15 +7,17 @@ class SehrDoofCommand(QleverCommand):
     """
 
     @staticmethod
-    def add_subparser(subparsers):
-        subparsers.add_parser(
-                "sehr_doof",
-                help="Sehr doofes command")
+    def help_text():
+        return "Sehr doofes command"
 
     @staticmethod
-    def arguments():
+    def relevant_arguments():
         return {"data": ["name"], "server": ["port"]}
 
+    @staticmethod
+    def should_have_qleverfile():
+        return False
+
     @staticmethod
     def execute(args):
         print(f"Executing command `sehr doof` with args: {args}")

src/qlever/config.py

@@ -1,24 +1,40 @@
 import argparse
 import argcomplete
 from qlever import command_classes
+from configparser import ConfigParser, ExtendedInterpolation
 from pathlib import Path
+from qlever.log import log
 import os
 import sys
 import shlex
+import traceback
 
 
 # Simple exception class for configuration errors (the class need not do
 # anything, we just want a distinct exception type).
 class ConfigException(Exception):
-    pass
+    def __init__(self, message):
+        stack = traceback.extract_stack()[-2]  # Caller's frame.
+        self.filename = stack.filename
+        self.lineno = stack.lineno
+        full_message = f"{message} [in {self.filename}:{self.lineno}]"
+        super().__init__(full_message)
 
 
-# Class that manages all config parameters, and overwrites them with the
-# settings from a Qleverfile.
 class QleverConfig:
+    """
+    Class that manages all config parameters, and overwrites them with the
+    settings from a Qleverfile.
 
-    @staticmethod
-    def all_arguments():
+    IMPORTANT: An instance of this class is created for each execution of
+    the `qlever` script, in particular, each time the user triggers
+    autocompletion. It's initialization should therefore be lightweight. In
+    particular, the functions `all_arguments()` and `parse_qleverfile()` should
+    not be called in the constructor, but only when needed (which is, when the
+    user has already committed to a command).
+    """
+
+    def all_arguments(self):
         """
         Define all possible parameters. A value of `None` means that there is
         no default value, and the parameter is mandatory in the Qleverfile (or
@@ -32,12 +48,12 @@ def all_arguments():
         def arg(*args, **kwargs):
             return (args, kwargs)
 
-        arguments = {}
-        data_args = arguments["data"] = {}
-        index_args = arguments["index"] = {}
-        server_args = arguments["server"] = {}
-        runtime_args = arguments["runtime"] = {}
-        ui_args = arguments["ui"] = {}
+        all_args = {}
+        data_args = all_args["data"] = {}
+        index_args = all_args["index"] = {}
+        server_args = all_args["server"] = {}
+        runtime_args = all_args["runtime"] = {}
+        ui_args = all_args["ui"] = {}
 
         data_args["name"] = arg(
                 "--name", type=str, required=True,
@@ -66,61 +82,150 @@ def arg(*args, **kwargs):
                 "--memory-for-queries", type=str, default="1G",
                 help="The maximal memory allowed for query processing")
 
-        return arguments
+        runtime_args["environment"] = arg(
+                "--environment", type=str,
+                choices=["docker", "podman", "native"],
+                default="docker",
+                help="The runtime environment for the server")
+
+        ui_args["port"] = arg(
+                "--port", type=int, default=7000,
+                help="The port of the Qlever UI web app")
+
+        return all_args
+
+    def parse_qleverfile(self, qleverfile_path):
+        """
+        Parse the given Qleverfile (the function assumes that it exists) and
+        return a `ConfigParser` object with all the options and their values.
+
+        NOTE: The keys have the same hierarchical structure as the keys in
+        `all_arguments()`. The Qleverfile may contain options that are not
+        defined in `all_arguments()`. They can be used as temporary variables
+        to define other options, but cannot be accessed by the commands later.
+        """
+
+        config = ConfigParser(interpolation=ExtendedInterpolation())
+        try:
+            config.read(qleverfile_path)
+            return config
+        except Exception as e:
+            raise ConfigException(f"Error parsing {qleverfile_path}: {e}")
+
+    def add_arguments_to_subparser(self, subparser, arg_names, all_args,
+                                   qleverfile_config):
+        """
+        Add the specified arguments to the given subparser. Take the default
+        values from `all_arguments()` and overwrite them with the values from
+        the `Qleverfile`, in case it exists.
+
+        IMPORTANT: Don't call this function repeatedly (in particular, not for
+        every command, but only for the command for which it is needed), it's
+        not cheap.
+        """
 
-    @staticmethod
-    def parse_args():
-        # Add subparser for each command via the `add_subparser` method of the
-        # corresponding command class (the classes are loaded dynamically in
-        # `__init__.py`).
+        for section in arg_names:
+            if section not in all_args:
+                raise ConfigException(f"Section '{section}' not found")
+            for arg_name in arg_names[section]:
+                if arg_name not in all_args[section]:
+                    raise ConfigException(f"Argument '{arg_name}' not found "
+                                          f"in section '{section}'")
+                args, kwargs = all_args[section][arg_name]
+                # If `qleverfile_config` is given, add info about default
+                # values to the help string.
+                if qleverfile_config is not None:
+                    default_value = kwargs.get("default", None)
+                    qleverfile_value = qleverfile_config.get(
+                            section, arg_name, fallback=None)
+                    if qleverfile_value is not None:
+                        kwargs["default"] = qleverfile_value
+                        kwargs["required"] = False
+                        kwargs["help"] += (f" [default, from Qleverfile:"
+                                           f" {qleverfile_value}]")
+                    else:
+                        kwargs["help"] += f" [default: {default_value}]"
+                subparser.add_argument(*args, **kwargs)
+
+    def get_command_line_arguments(self):
+        """
+        Get the current command line arguments.
+
+        NOTE: This should work both when the script is called "normally" (in
+        which case, we can just use `sys.argv`), as well as when it is called
+        by the shell's autocompletion mechanism (in which case, the command
+        line is in the environment variable `COMP_LINE`).
+        """
+        if "COMP_LINE" in os.environ:
+            # Note: `COMP_LINE` is a string, with spaces used to separate the
+            # arguments and spaces within arguments escaped with a backslash.
+            return shlex.split(os.environ["COMP_LINE"])
+        else:
+            return sys.argv
+
+    def parse_args(self):
+        # Create a temporary parser only to parse the `--qleverfile` option, in
+        # case it is given. This is because in the actual parser below we want
+        # the values from the Qleverfile to be shown in the help strings.
+        def add_qleverfile_option(parser):
+            parser.add_argument(
+                    "--qleverfile", "-q", type=str, default="Qleverfile",
+                    help="The Qleverfile to use (default: Qleverfile)")
+        qleverfile_parser = argparse.ArgumentParser(add_help=False)
+        add_qleverfile_option(qleverfile_parser)
+        qleverfile_args, _ = qleverfile_parser.parse_known_args()
+        qleverfile_path_name = qleverfile_args.qleverfile
+
+        # If this is the normal execution of the script (and not a call invoked
+        # by the shell's autocompletion mechanism), parse the Qleverfile.
+        qleverfile_path = Path(qleverfile_path_name)
+        qleverfile_exists = qleverfile_path.is_file()
+        qleverfile_is_default = qleverfile_path_name \
+            == qleverfile_parser.get_default("qleverfile")
+        # If a Qleverfile with a non-default name was specified, but it does
+        # not exist, that's an error.
+        if not qleverfile_exists and not qleverfile_is_default:
+            raise ConfigException(f"Qleverfile with non-default name "
+                                  f"`{qleverfile_path_name}` specified, "
+                                  f"but it does not exist")
+        # If it exists and we arenot in the autocompletion mode, parse it.
+        if qleverfile_exists and "COMP_LINE" not in os.environ:
+            qleverfile_config = self.parse_qleverfile(qleverfile_path)
+        else:
+            qleverfile_config = None
+
+        # Now the regular parser with commands and a subparser for each
+        # command. We have a dedicated class for each command, these classes
+        # are defined in the modules in the `qlever/commands` directory and
+        # dynamically imported in `__init__.py`.
         parser = argparse.ArgumentParser()
+        add_qleverfile_option(parser)
         subparsers = parser.add_subparsers(dest='command')
         subparsers.required = True
-        for command in command_classes:
-            command_classes[command].add_subparser(subparsers)
+        all_args = self.all_arguments()
+        for command_name, command_class in command_classes.items():
+            help_text = command_class.help_text()
+            subparser = subparsers.add_parser(command_name, help=help_text)
+            arg_names = command_class.relevant_arguments()
+            self.add_arguments_to_subparser(subparser, arg_names, all_args,
+                                            qleverfile_config)
 
-        # Only if the user has typed a command, do we add the arguments for
-        # the respective subparser (which entails parsing the Qleverfile).
-        #
-        # NOTE 1: We don't want to do this for every command because this has to
-        # be done whenever the user triggers autocompletion. Also, this parses
-        # the Qleverfile, and there is no need for that before a command has
-        # been chosen.
-        # 
-        # NOTE 2: This code can be reached in two different ways: either in the
-        # "normal" way, e.g. when the user types `qlever index --help`, or in
-        # the "autocompletion" way, e.g. when the user types `qlever index
-        # --<TAB>`, and the shell's completion function is called. In the
-        # latter case, the command line is stored in the environment variable
-        # `COMP_LINE`.
-        try:
-            argv = shlex.split(os.environ["COMP_LINE"])
-        except KeyError:
-            argv = sys.argv
-        if len(argv) > 1:
-            command_name = argv[1]
-            if command_name in command_classes:
-                command_class = command_classes[command_name]
-                subparser = subparsers.choices[command_name]
-                command_arguments = command_class.arguments()
-                all_arguments = QleverConfig.all_arguments()
-                for section in command_arguments:
-                    if section not in all_arguments:
-                        raise ConfigException(f"Section '{section}' not found")
-                    for arg_name in command_arguments[section]:
-                        if arg_name not in all_arguments[section]:
-                            raise ConfigException(
-                                    f"Argument '{arg_name}' not found in section '{section}'")
-                        args, kwargs = all_arguments[section][arg_name]
-                        subparser.add_argument(*args, **kwargs)
-
-        # Enable autocompletion for the commands as well as for the command
-        # options.
+        # Enable autocompletion for the commands and their options.
         #
-        # NOTE: It is important that all code that is executed before this line
-        # is relatively cheap because it is executed whenever the user presses
-        # the key (usually TAB) that invokes autocompletion.
+        # NOTE: All code executed before this line should be relatively cheap
+        # because it is executed whenever the user triggers the autocompletion.
         argcomplete.autocomplete(parser, always_complete_options="long")
-
-        # Parse the command line arguments and return them.
-        return parser.parse_args()
+
+        # Parse the command line arguments.
+        args = parser.parse_args()
+
+        # If the command says that we should have a Qleverfile, but we don't,
+        # issue a warning.
+        if command_classes[args.command].should_have_qleverfile():
+            if not qleverfile_exists:
+                log.warning(f"Invoking command `{args.command}` without a "
+                            "Qleverfile. You have to specify all required "
+                            "arguments on the command line. This is possible, "
+                            "but not recommended.")
+
+        return args

src/qlever/log.py

@@ -0,0 +1,30 @@
+# Copyright 2024, University of Freiburg,
+# Chair of Algorithms and Data Structures
+# Author: Hannah Bast <[email protected]>
+
+import logging
+from termcolor import colored
+
+
+class CustomFormatter(logging.Formatter):
+    """
+    Custom formatter for logging.
+    """
+    def format(self, record):
+        message = record.getMessage()
+        if record.levelno == logging.DEBUG:
+            return colored(f"DEBUG: {message}", "magenta")
+        elif record.levelno == logging.WARNING:
+            return colored(f"WARNING: {message}", "yellow")
+        elif record.levelno in [logging.CRITICAL, logging.ERROR]:
+            return colored(f"ERROR: {message}", "red")
+        else:
+            return message
+
+
+# Custom logger.
+log = logging.getLogger("qlever")
+log.setLevel(logging.INFO)
+handler = logging.StreamHandler()
+handler.setFormatter(CustomFormatter())
+log.addHandler(handler)