From eb59a0c6182d436b34f23dc28356d08697d93a23 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Wed, 14 Aug 2024 04:40:12 +0200 Subject: [PATCH 1/3] Allow configuration via environment variables --- src/qlever/commands/config.py | 195 ++++++++++++++++++++++++++++++++++ src/qlever/config.py | 66 +++++++++--- src/qlever/envvars.py | 52 +++++++++ src/qlever/globals.py | 15 +++ src/qlever/qlever_main.py | 27 ++++- src/qlever/qleverfile.py | 3 +- 6 files changed, 339 insertions(+), 19 deletions(-) create mode 100644 src/qlever/commands/config.py create mode 100644 src/qlever/envvars.py create mode 100644 src/qlever/globals.py diff --git a/src/qlever/commands/config.py b/src/qlever/commands/config.py new file mode 100644 index 00000000..e995250a --- /dev/null +++ b/src/qlever/commands/config.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import shlex +import subprocess +from pathlib import Path + +import qlever.globals +from qlever.command import QleverCommand +from qlever.envvars import Envvars +from qlever.log import log +from qlever.util import get_random_string + + +class ConfigCommand(QleverCommand): + """ + Class for executing the `config` command. + """ + + def __init__(self): + self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles" + self.qleverfile_names = \ + [p.name.split(".")[1] + for p in self.qleverfiles_path.glob("Qleverfile.*")] + + def description(self) -> str: + return "Set up a Qleverfile or show the current configuration" + + def should_have_qleverfile(self) -> bool: + return False + + def relevant_qleverfile_arguments(self) -> dict[str: list[str]]: + return {} + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--get-preconfigured-qleverfile", type=str, + choices=self.qleverfile_names, + help="Get one the many pre-configured Qleverfiles") + subparser.add_argument( + "--show-qleverfile", action="store_true", + default=False, + help="Show the configuration from the Qleverfile " + "(if it exists)") + subparser.add_argument( + "--show-envvars", action="store_true", default=False, + help="Show all existing environment variables of the form " + "QLEVER_SECTION_VARIABLE") + subparser.add_argument( + "--varname-width", type=int, default=25, + help="Width for variable names in the output") + subparser.add_argument( + "--set-envvars-from-qleverfile", action="store_true", + default=False, + help="Set the environment variables that correspond to the " + "Qleverfile configuration (for copying and pasting)") + subparser.add_argument( + "--unset-envvars", action="store_true", default=False, + help="Unset all environment variables of the form " + "QLEVER_SECTION_VARIABLE (for copying and pasting)") + + def execute(self, args) -> bool: + # Show the configuration from the Qleverfile. + if args.show_qleverfile: + if qlever.globals.qleverfile_path is None: + log.error("No Qleverfile found") + return False + if qlever.globals.qleverfile_config is None: + log.error("Qleverfile found, but contains no configuration") + return False + self.show(f"Show the configuration from " + f"{qlever.globals.qleverfile_path} (with any variables " + f"on the right-hand side already substituted)", + only_show=args.show) + if args.show: + return False + else: + print_empty_line_before_section = False + for section, varname_and_values in \ + qlever.globals.qleverfile_config.items(): + if section == "DEFAULT": + continue + if print_empty_line_before_section: + log.info("") + print_empty_line_before_section = True + log.info(f"[{section}]") + for varname, value in varname_and_values.items(): + log.info(f"{varname.upper():{args.varname_width}} = " + f"{value}") + return True + + # Show all environment variables of the form QLEVER_SECTION_VARIABLE. + if args.show_envvars: + self.show("Show all environment variables of the form " + "QLEVER_SECTION_VARIABLE", only_show=args.show) + if args.show: + return False + if qlever.globals.envvars_config is None: + log.info("No environment variables found") + else: + for section, varname_and_values in \ + qlever.globals.envvars_config.items(): + for varname, value in varname_and_values.items(): + var = Envvars.envvar_name(section, varname) + log.info(f"{var:{args.varname_width+7}}" + f" = {shlex.quote(value)}") + return True + + # Show the environment variables that correspond to the Qleverfile. + if args.set_envvars_from_qleverfile: + if qlever.globals.qleverfile_path is None: + log.error("No Qleverfile found") + return False + if qlever.globals.qleverfile_config is None: + log.error("Qleverfile found, but contains no configuration") + return False + self.show("Show the environment variables that correspond to the " + "Qleverfile configuration (for copying and pasting)", + only_show=args.show) + if args.show: + return False + else: + for section, varname_and_values in \ + qlever.globals.qleverfile_config.items(): + if section == "DEFAULT": + continue + for varname, value in varname_and_values.items(): + var = Envvars.envvar_name(section, varname) + log.info(f"export {var}={shlex.quote(value)}") + return True + + # Unset all environment variables of the form QLEVER_SECTION_VARIABLE. + # Note that this cannot be done in this script because it would not + # affect the shell calling this script. Instead, show the commands for + # unsetting the environment variables to copy and paste. + if args.unset_envvars: + self.show("Unset all environment variables of the form " + "QLEVER_SECTION_VARIABLE (for copying and pasting, " + "this command cannot affect the shell from which you " + " are calling it)", only_show=args.show) + if args.show: + return False + if qlever.globals.envvars_config is None: + log.info("No environment variables found") + else: + envvar_names = [] + for section, varname_and_values in \ + qlever.globals.envvars_config.items(): + for varname, value in varname_and_values.items(): + envvar_name = Envvars.envvar_name(section, varname) + envvar_names.append(envvar_name) + log.info(f"unset {' '.join(envvar_names)}") + return True + + # Get one of the pre-configured Qleverfiles. + if args.get_preconfigured_qleverfile: + preconfigured_qleverfile_path = \ + self.qleverfiles_path / f"Qleverfile.{args.config_name}" + random_string = get_random_string(12) + setup_config_cmd = ( + f"cat {preconfigured_qleverfile_path}" + f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{random_string}/'" + f" > Qleverfile") + self.show(setup_config_cmd, only_show=args.show) + if args.show: + return False + + # If there is already a Qleverfile in the current directory, exit. + existing_qleverfile_path = Path("Qleverfile") + if existing_qleverfile_path.exists(): + log.error("`Qleverfile` already exists in current directory") + log.info("") + log.info("If you want to create a new Qleverfile using " + "`qlever setup-config`, delete the existing " + "Qleverfile first") + return False + + # Copy the Qleverfile to the current directory. + try: + subprocess.run(setup_config_cmd, shell=True, check=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) + except Exception as e: + log.error(f"Could not copy \"{preconfigured_qleverfile_path}\"" + f" to current directory: {e}") + return False + + # If we get here, everything went well. + log.info(f"Created Qleverfile for config \"{args.config_name}\"" + f" in current directory") + return True + + # Calling `qlever config` without arguments is an error. Show the help. + log.error("`qlever config` requires at least one argument, " + "see `qlever config --help`") + return False diff --git a/src/qlever/config.py b/src/qlever/config.py index 35e0bb94..f79e6700 100644 --- a/src/qlever/config.py +++ b/src/qlever/config.py @@ -10,6 +10,7 @@ from termcolor import colored from qlever import command_objects, script_name +from qlever.envvars import Envvars from qlever.log import log, log_levels from qlever.qleverfile import Qleverfile @@ -40,17 +41,26 @@ class QleverConfig: def add_subparser_for_command(self, subparsers, command_name, command_object, all_qleverfile_args, - qleverfile_config=None): + qleverfile_config=None, + envvars_config=None): """ Add subparser for the given command. Take the arguments from `command_object.relevant_qleverfile_arguments()` and report an error if one of them is not contained in `all_qleverfile_args`. Overwrite the - default values with the values from `qleverfile_config` if specified. + default values with the values from `qleverfile_config` or + `envvars_config` if they are given. + + NOTE: For now, throw an exception if both `qleverfile_config` and + `envvars_config` are given. It would be easy to let one override the + other, but that might lead to unexpected behavior for a user. For + example, the user might write or create `Qleverfile` and is unaware + that they also some environment variables set. """ + # All argument names for this command. arg_names = command_object.relevant_qleverfile_arguments() - # Helper function that shows a detailed error messahe when an argument + # Helper function that shows a detailed error message when an argument # from `relevant_qleverfile_arguments` is not contained in # `all_qleverfile_args`. def argument_error(prefix): @@ -81,8 +91,8 @@ def argument_error(prefix): f"`{section}` not found") args, kwargs = all_qleverfile_args[section][arg_name] kwargs_copy = kwargs.copy() - # If `qleverfile_config` is given, add info about default - # values to the help string. + # If `qleverfile_config` is given, add the corresponding + # default values. if qleverfile_config is not None: default_value = kwargs.get("default", None) qleverfile_value = qleverfile_config.get( @@ -94,6 +104,21 @@ def argument_error(prefix): f" {qleverfile_value}]") else: kwargs_copy["help"] += f" [default: {default_value}]" + # If `envvars_config` is given, add the corresponding default + # values. + if envvars_config is not None: + default_value = kwargs.get("default", None) + envvar_name = Envvars.envvar_name(section, arg_name) + envvars_value = envvars_config[section].get(arg_name, None) + if envvars_value is not None: + kwargs_copy["default"] = envvars_value + kwargs_copy["required"] = False + kwargs_copy["help"] += (f" [default, from environment " + f"variable `{envvar_name}`: " + f"{envvars_value}]") + else: + kwargs_copy["help"] += f" [default: {default_value}]" + # Now add the argument to the subparser. subparser.add_argument(*args, **kwargs_copy) # Additional arguments that are shared by all commands. @@ -170,6 +195,24 @@ def add_qleverfile_option(parser): exit(1) else: qleverfile_config = None + qleverfile_path = None + + # Now also check if the user has set any environment variables. + envvars_config = Envvars.read() + + # Check that at most one of `qleverfile_config` and `envvars_config` is + # not `None`, unless the command is `config` (which can be used to + # produce a `Qleverfile` or unset all environment variables). + if qleverfile_args.command != "config" \ + and qleverfile_config is not None \ + and envvars_config is not None: + raise ConfigException( + "You both have a `Qleverfile` and environment variables " + "of the QLEVER_SECTION_VARIABLE. This is not supported " + "because it is bound to lead to unexpected behavior. " + "Either remove the `Qleverfile` (just delete it), or the " + "environment variables (use `qlever config " + "--unset-envvars`).") # Now the regular parser with commands and a subparser for each # command. We have a dedicated class for each command. These classes @@ -188,7 +231,7 @@ def add_qleverfile_option(parser): for command_name, command_object in command_objects.items(): self.add_subparser_for_command( subparsers, command_name, command_object, - all_args, qleverfile_config) + all_args, qleverfile_config, envvars_config) # Enable autocompletion for the commands and their options. # @@ -204,13 +247,4 @@ def add_qleverfile_option(parser): # Parse the command line arguments. args = parser.parse_args() - # If the command says that we should have a Qleverfile, but we don't, - # issue a warning. - if command_objects[args.command].should_have_qleverfile(): - if not qleverfile_exists: - log.warning(f"Invoking command `{args.command}` without a " - "Qleverfile. You have to specify all required " - "arguments on the command line. This is possible, " - "but not recommended.") - - return args + return args, qleverfile_path, qleverfile_config, envvars_config diff --git a/src/qlever/envvars.py b/src/qlever/envvars.py new file mode 100644 index 00000000..068c1136 --- /dev/null +++ b/src/qlever/envvars.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import os + +from qlever.qleverfile import Qleverfile + + +class EnvvarsException(Exception): + pass + + +class Envvars: + """ + Class for parsing environment variables with analogous names to those in + the `Qleverfile` class, according to the schema `QLEVER_SECTION_VARIABLE`. + For example, variable `PORT` in section `server` corresponds to the + environment variable `QLEVER_SERVER_PORT`. + """ + + @staticmethod + def envvar_name(section: str, name: str) -> str: + """ + For a given section and variable name, return the environment variable + name according to the schema described above. + """ + return f"QLEVER_{section.upper()}_{name.upper()}" + + @staticmethod + def read(): + """ + Check all environment variables that correspond to an entry in + `Qleverfile.all_arguments()` according to the schema described above, + and return a dictionary `config` with all the values found that way. + For example, for `QLEVER_SERVER_PORT=8000`, there would be an entry + `config['server']['port'] = 8000`. + + NOTE: If no environment variables was found at all, the method will + return `None`. Otherwise, there will be an entry for each section, even + if it is empty. + """ + + all_args = Qleverfile.all_arguments() + config = {} + num_envvars_found = 0 + for section, args in all_args.items(): + config[section] = {} + for arg in args: + envvar = Envvars.envvar_name(section, arg) + if envvar in os.environ: + config[section][arg] = os.environ[envvar] + num_envvars_found += 1 + return config if num_envvars_found > 0 else None diff --git a/src/qlever/globals.py b/src/qlever/globals.py new file mode 100644 index 00000000..5c6cfba8 --- /dev/null +++ b/src/qlever/globals.py @@ -0,0 +1,15 @@ +# Global variables for the full configuration, set in `qlever_main.py`. +# For example, these are used by by `qlever config --show-qleverfile-config` or +# `qlever config --show-envvars`. +# +# NOTE 1: Most commands do not (and should not) use these: the `args` passed to +# the `execute` method of a command class is deliberately reduced to those +# arguments that are relevant for the command. +# +# NOTE 2: If we would define these in `config.py`, which seems like the natural +# place, we get a circular import error, because we need these in +# `qlever/commands/config.py`, which would have to import `config.py`, which +# imports `__init__.py`, which imports all the command modules. +qleverfile_path = None +qleverfile_config = None +envvars_config = None diff --git a/src/qlever/qlever_main.py b/src/qlever/qlever_main.py index a936abd5..95c5a3f5 100644 --- a/src/qlever/qlever_main.py +++ b/src/qlever/qlever_main.py @@ -12,6 +12,7 @@ from termcolor import colored +import qlever.globals from qlever import command_objects from qlever.config import ConfigException, QleverConfig from qlever.log import log, log_levels @@ -21,7 +22,9 @@ def main(): # Parse the command line arguments and read the Qleverfile. try: qlever_config = QleverConfig() - args = qlever_config.parse_args() + args, qlever.globals.qleverfile_path, \ + qlever.globals.qleverfile_config, \ + qlever.globals.envvars_config = qlever_config.parse_args() except ConfigException as e: log.error(e) log.info("") @@ -35,6 +38,28 @@ def main(): log.info("") log.info(colored(f"Command: {args.command}", attrs=["bold"])) log.info("") + # If the command says that we should have a Qleverfile, but we don't, + # issue a warning. + if command_objects[args.command].should_have_qleverfile(): + if qlever.globals.qleverfile_path: + log.info(f"Invoking command with config from " + f"`{qlever.globals.qleverfile_path}` " + f"(use `qlever config --show-qleverfile` " + f"to see the effective config, or " + f"edit the file to change it).") + else: + if qlever.globals.envvars_config: + log.info("Invoking command without a Qleverfile, but " + "with environment variables (use `qlever " + "config --show-envvars` to see them and `qlever " + "config --unset-envvars` to unset them).") + else: + log.warning("Invoking command with neither a Qleverfile " + "nor environment variables. You have to " + "specify all required arguments on the " + "command line. This is possible, but not " + "recommended.") + log.info("") command_object.execute(args) log.info("") except KeyboardInterrupt: diff --git a/src/qlever/qleverfile.py b/src/qlever/qleverfile.py index 6a353239..2a24c762 100644 --- a/src/qlever/qleverfile.py +++ b/src/qlever/qleverfile.py @@ -209,9 +209,8 @@ def read(qleverfile_path): """ # Read the Qleverfile. - defaults = {"random": "83724324hztz", "version": "01.01.01"} config = ConfigParser(interpolation=ExtendedInterpolation(), - defaults=defaults) + defaults=None) try: config.read(qleverfile_path) except Exception as e: From ca66e9227390d6a8f48a65cacf49ee7ad7d2a2f2 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Thu, 15 Aug 2024 17:03:15 +0200 Subject: [PATCH 2/3] Fix minor bug --- src/qlever/commands/config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/qlever/commands/config.py b/src/qlever/commands/config.py index e995250a..29172c2d 100644 --- a/src/qlever/commands/config.py +++ b/src/qlever/commands/config.py @@ -33,7 +33,7 @@ def relevant_qleverfile_arguments(self) -> dict[str: list[str]]: def additional_arguments(self, subparser) -> None: subparser.add_argument( - "--get-preconfigured-qleverfile", type=str, + "--get-qleverfile", type=str, choices=self.qleverfile_names, help="Get one the many pre-configured Qleverfiles") subparser.add_argument( @@ -152,9 +152,10 @@ def execute(self, args) -> bool: return True # Get one of the pre-configured Qleverfiles. - if args.get_preconfigured_qleverfile: + if args.get_qleverfile: + config_name = args.get_qleverfile preconfigured_qleverfile_path = \ - self.qleverfiles_path / f"Qleverfile.{args.config_name}" + self.qleverfiles_path / f"Qleverfile.{config_name}" random_string = get_random_string(12) setup_config_cmd = ( f"cat {preconfigured_qleverfile_path}" @@ -185,7 +186,7 @@ def execute(self, args) -> bool: return False # If we get here, everything went well. - log.info(f"Created Qleverfile for config \"{args.config_name}\"" + log.info(f"Created Qleverfile for config \"{config_name}\"" f" in current directory") return True From d55fdf17d2db935f51318c1f97a98a81f9425132 Mon Sep 17 00:00:00 2001 From: Hannah Bast Date: Sun, 15 Dec 2024 23:47:28 +0100 Subject: [PATCH 3/3] Split between commands `setup-config` and `show-config' The options `--set-envvars-from-qleverfile` and `--unset-envvars` are now part of the `setup-config` command (because they are about setting up the configuration). For showing the current configuration, a new command `show-config` has been added. Whether running `qlever` via environment variables works is now also tested as part of the `qleverfiles-check` workflow. --- .github/workflows/qleverfiles-check.yml | 24 +++ src/qlever/Qleverfiles/Qleverfile.olympics | 4 +- src/qlever/commands/config.py | 196 --------------------- src/qlever/commands/setup_config.py | 148 ++++++++++++++-- src/qlever/commands/show_config.py | 98 +++++++++++ src/qlever/config.py | 155 +++++++++------- src/qlever/envvars.py | 6 +- 7 files changed, 361 insertions(+), 270 deletions(-) delete mode 100644 src/qlever/commands/config.py create mode 100644 src/qlever/commands/show_config.py diff --git a/.github/workflows/qleverfiles-check.yml b/.github/workflows/qleverfiles-check.yml index 4cda5f8d..e68968b5 100644 --- a/.github/workflows/qleverfiles-check.yml +++ b/.github/workflows/qleverfiles-check.yml @@ -49,3 +49,27 @@ jobs: echo -e "\x1b[34mAll checks passed for ${QLEVERFILE}\x1b[0m" echo done + + - name: Repeat this check using environment variables (derived from the Qleverfile) + working-directory: ${{github.workspace}}/qlever-control + run: | + export QLEVER_ARGCOMPLETE_ENABLED=1 + for QLEVERFILE in src/qlever/Qleverfiles/Qleverfile.*; do + echo + echo -e "\x1b[1;34mChecking ${QLEVERFILE} via environment variables\x1b[0m" + echo + NAME=${QLEVERFILE##*.} + rm -f Qleverfile + qlever setup-config $NAME + qlever setup-config --set-envvars-from-qleverfile --file set-envvars.sh + source set-envvars.sh && rm -f Qleverfile + qlever get-data --show + qlever index --show + qlever start --show + qlever ui --show + qlever setup-config --unset-envvars --file unset-envvars.sh + source unset-envvars.sh + echo + echo -e "\x1b[34mAll checks passed for ${QLEVERFILE}\x1b[0m" + echo + done diff --git a/src/qlever/Qleverfiles/Qleverfile.olympics b/src/qlever/Qleverfiles/Qleverfile.olympics index 0887fcae..281f5a65 100644 --- a/src/qlever/Qleverfiles/Qleverfile.olympics +++ b/src/qlever/Qleverfiles/Qleverfile.olympics @@ -1,4 +1,4 @@ -# Qleverfile for Olympics, use with https://github.com/ad-freiburg/qlever-control +# Qleverfile for Olympics, use with qlever CLI (`pip install qlever`) # # qlever get-data # downloads .zip file of size 13 MB, uncompressed to 323 MB # qlever index # takes ~10 seconds and ~1 GB RAM (on an AMD Ryzen 9 5900X) @@ -18,7 +18,7 @@ SETTINGS_JSON = { "ascii-prefixes-only": false, "num-triples-per-batch": 10000 [server] PORT = 7019 -ACCESS_TOKEN = ${data:NAME}_7643543846 +ACCESS_TOKEN = ${data:NAME} MEMORY_FOR_QUERIES = 5G CACHE_MAX_SIZE = 2G TIMEOUT = 30s diff --git a/src/qlever/commands/config.py b/src/qlever/commands/config.py deleted file mode 100644 index 29172c2d..00000000 --- a/src/qlever/commands/config.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -import shlex -import subprocess -from pathlib import Path - -import qlever.globals -from qlever.command import QleverCommand -from qlever.envvars import Envvars -from qlever.log import log -from qlever.util import get_random_string - - -class ConfigCommand(QleverCommand): - """ - Class for executing the `config` command. - """ - - def __init__(self): - self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles" - self.qleverfile_names = \ - [p.name.split(".")[1] - for p in self.qleverfiles_path.glob("Qleverfile.*")] - - def description(self) -> str: - return "Set up a Qleverfile or show the current configuration" - - def should_have_qleverfile(self) -> bool: - return False - - def relevant_qleverfile_arguments(self) -> dict[str: list[str]]: - return {} - - def additional_arguments(self, subparser) -> None: - subparser.add_argument( - "--get-qleverfile", type=str, - choices=self.qleverfile_names, - help="Get one the many pre-configured Qleverfiles") - subparser.add_argument( - "--show-qleverfile", action="store_true", - default=False, - help="Show the configuration from the Qleverfile " - "(if it exists)") - subparser.add_argument( - "--show-envvars", action="store_true", default=False, - help="Show all existing environment variables of the form " - "QLEVER_SECTION_VARIABLE") - subparser.add_argument( - "--varname-width", type=int, default=25, - help="Width for variable names in the output") - subparser.add_argument( - "--set-envvars-from-qleverfile", action="store_true", - default=False, - help="Set the environment variables that correspond to the " - "Qleverfile configuration (for copying and pasting)") - subparser.add_argument( - "--unset-envvars", action="store_true", default=False, - help="Unset all environment variables of the form " - "QLEVER_SECTION_VARIABLE (for copying and pasting)") - - def execute(self, args) -> bool: - # Show the configuration from the Qleverfile. - if args.show_qleverfile: - if qlever.globals.qleverfile_path is None: - log.error("No Qleverfile found") - return False - if qlever.globals.qleverfile_config is None: - log.error("Qleverfile found, but contains no configuration") - return False - self.show(f"Show the configuration from " - f"{qlever.globals.qleverfile_path} (with any variables " - f"on the right-hand side already substituted)", - only_show=args.show) - if args.show: - return False - else: - print_empty_line_before_section = False - for section, varname_and_values in \ - qlever.globals.qleverfile_config.items(): - if section == "DEFAULT": - continue - if print_empty_line_before_section: - log.info("") - print_empty_line_before_section = True - log.info(f"[{section}]") - for varname, value in varname_and_values.items(): - log.info(f"{varname.upper():{args.varname_width}} = " - f"{value}") - return True - - # Show all environment variables of the form QLEVER_SECTION_VARIABLE. - if args.show_envvars: - self.show("Show all environment variables of the form " - "QLEVER_SECTION_VARIABLE", only_show=args.show) - if args.show: - return False - if qlever.globals.envvars_config is None: - log.info("No environment variables found") - else: - for section, varname_and_values in \ - qlever.globals.envvars_config.items(): - for varname, value in varname_and_values.items(): - var = Envvars.envvar_name(section, varname) - log.info(f"{var:{args.varname_width+7}}" - f" = {shlex.quote(value)}") - return True - - # Show the environment variables that correspond to the Qleverfile. - if args.set_envvars_from_qleverfile: - if qlever.globals.qleverfile_path is None: - log.error("No Qleverfile found") - return False - if qlever.globals.qleverfile_config is None: - log.error("Qleverfile found, but contains no configuration") - return False - self.show("Show the environment variables that correspond to the " - "Qleverfile configuration (for copying and pasting)", - only_show=args.show) - if args.show: - return False - else: - for section, varname_and_values in \ - qlever.globals.qleverfile_config.items(): - if section == "DEFAULT": - continue - for varname, value in varname_and_values.items(): - var = Envvars.envvar_name(section, varname) - log.info(f"export {var}={shlex.quote(value)}") - return True - - # Unset all environment variables of the form QLEVER_SECTION_VARIABLE. - # Note that this cannot be done in this script because it would not - # affect the shell calling this script. Instead, show the commands for - # unsetting the environment variables to copy and paste. - if args.unset_envvars: - self.show("Unset all environment variables of the form " - "QLEVER_SECTION_VARIABLE (for copying and pasting, " - "this command cannot affect the shell from which you " - " are calling it)", only_show=args.show) - if args.show: - return False - if qlever.globals.envvars_config is None: - log.info("No environment variables found") - else: - envvar_names = [] - for section, varname_and_values in \ - qlever.globals.envvars_config.items(): - for varname, value in varname_and_values.items(): - envvar_name = Envvars.envvar_name(section, varname) - envvar_names.append(envvar_name) - log.info(f"unset {' '.join(envvar_names)}") - return True - - # Get one of the pre-configured Qleverfiles. - if args.get_qleverfile: - config_name = args.get_qleverfile - preconfigured_qleverfile_path = \ - self.qleverfiles_path / f"Qleverfile.{config_name}" - random_string = get_random_string(12) - setup_config_cmd = ( - f"cat {preconfigured_qleverfile_path}" - f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{random_string}/'" - f" > Qleverfile") - self.show(setup_config_cmd, only_show=args.show) - if args.show: - return False - - # If there is already a Qleverfile in the current directory, exit. - existing_qleverfile_path = Path("Qleverfile") - if existing_qleverfile_path.exists(): - log.error("`Qleverfile` already exists in current directory") - log.info("") - log.info("If you want to create a new Qleverfile using " - "`qlever setup-config`, delete the existing " - "Qleverfile first") - return False - - # Copy the Qleverfile to the current directory. - try: - subprocess.run(setup_config_cmd, shell=True, check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL) - except Exception as e: - log.error(f"Could not copy \"{preconfigured_qleverfile_path}\"" - f" to current directory: {e}") - return False - - # If we get here, everything went well. - log.info(f"Created Qleverfile for config \"{config_name}\"" - f" in current directory") - return True - - # Calling `qlever config` without arguments is an error. Show the help. - log.error("`qlever config` requires at least one argument, " - "see `qlever config --help`") - return False diff --git a/src/qlever/commands/setup_config.py b/src/qlever/commands/setup_config.py index 0eff3b23..0434d9bc 100644 --- a/src/qlever/commands/setup_config.py +++ b/src/qlever/commands/setup_config.py @@ -1,12 +1,14 @@ from __future__ import annotations -import subprocess +import shlex from os import environ from pathlib import Path +import qlever.globals from qlever.command import QleverCommand +from qlever.envvars import Envvars from qlever.log import log -from qlever.util import get_random_string +from qlever.util import get_random_string, run_command class SetupConfigCommand(QleverCommand): @@ -32,12 +34,142 @@ def relevant_qleverfile_arguments(self) -> dict[str : list[str]]: def additional_arguments(self, subparser) -> None: subparser.add_argument( "config_name", + nargs="?", type=str, choices=self.qleverfile_names, help="The name of the pre-configured Qleverfile to create", ) + subparser.add_argument( + "--set-envvars-from-qleverfile", + action="store_true", + default=False, + help="Show the command line to set the environment variables " + "that correspond to the Qleverfile configuration (suitable for " + "copying and pasting)", + ) + subparser.add_argument( + "--unset-envvars", + action="store_true", + default=False, + help="Show the command line to unset all environment variables of " + "the form QLEVER_SECTION_VARIABLE (suitable for copying and pasting)", + ) + subparser.add_argument( + "--file", + type=str, + default=None, + help="File to which to write the commands from " + "`--set-envvars-from-qleverfile` or `--unset-envvars`", + ) def execute(self, args) -> bool: + # Either create a Qleverfile or set or unset environment variables. + if args.config_name and args.set_envvars_from_qleverfile: + log.error( + "If you want to set environment variables based on a " + "Qleverfile, first create a Qleverfile by running " + "`qlever setup-config CONFIG_NAME`, and then run " + "`qlever config --set-envvars-from-qleverfile`" + ) + return False + if args.config_name and args.unset_envvars: + log.error( + "You cannot create a Qleverfile and unset environment " + "variables at the same time" + ) + return False + if args.set_envvars_from_qleverfile and args.unset_envvars: + log.error("You cannot set and unset environment variables at the same time") + return False + + # Show the environment variables that correspond to the Qleverfile. + if args.set_envvars_from_qleverfile: + if qlever.globals.qleverfile_path is None: + log.error("No Qleverfile found") + return False + if qlever.globals.qleverfile_config is None: + log.error("Qleverfile found, but contains no configuration") + return False + self.show( + "Set the environment variables that correspond to the " + "configuration from the Qleverfile", + only_show=args.show, + ) + if args.show: + return False + else: + set_envvar_cmds = [] + for ( + section, + varname_and_values, + ) in qlever.globals.qleverfile_config.items(): + if section == "DEFAULT": + continue + for varname, value in varname_and_values.items(): + var = Envvars.envvar_name(section, varname) + set_envvar_cmd = f"export {var}={shlex.quote(value)}" + set_envvar_cmds.append(set_envvar_cmd) + log.info(set_envvar_cmd) + log.info("") + if args.file: + with open(args.file, "w") as f: + for cmd in set_envvar_cmds: + f.write(cmd + "\n") + log.info( + f"Commands written to file `{args.file}`, to set " + "the environment variables, run `source {args.file}` " + "(and if you want to use `qlever` based on these " + "environment variables, move or delete the Qleverfile)" + ) + else: + log.info( + "If you want to write these commands to a file, " + "rerun with `--file FILENAME`" + ) + return True + + # Unset all environment variables of the form QLEVER_SECTION_VARIABLE. + # Note that this cannot be done in this script because it would not + # affect the shell calling this script. Instead, show the commands for + # unsetting the environment variables to copy and paste. + if args.unset_envvars: + self.show( + "Unset all environment variables of the form " + "QLEVER_SECTION_VARIABLE (for copying and pasting, " + "this command cannot affect the shell from which you " + " are calling it)", + only_show=args.show, + ) + if args.show: + return False + if qlever.globals.envvars_config is None: + log.info("No environment variables found") + else: + envvar_names = [] + for ( + section, + varname_and_values, + ) in qlever.globals.envvars_config.items(): + for varname, value in varname_and_values.items(): + envvar_name = Envvars.envvar_name(section, varname) + envvar_names.append(envvar_name) + unset_cmd = f"unset {' '.join(envvar_names)}" + log.info(unset_cmd) + log.info("") + if args.file: + with open(args.file, "w") as f: + f.write(unset_cmd) + log.info( + f"Command written to file `{args.file}`, to unset " + "the environment variables, run `source {args.file}`" + ) + else: + log.info( + "If you want to write this command to a file, " + "rerun with `--file FILENAME`" + ) + return True + # Show a warning if `QLEVER_OVERRIDE_SYSTEM_NATIVE` is set. qlever_is_running_in_container = environ.get("QLEVER_IS_RUNNING_IN_CONTAINER") if qlever_is_running_in_container: @@ -47,11 +179,13 @@ def execute(self, args) -> bool: "(since inside the container, QLever should run natively)" ) log.info("") + # Construct the command line and show it. qleverfile_path = self.qleverfiles_path / f"Qleverfile.{args.config_name}" + random_string = get_random_string(12) setup_config_cmd = ( f"cat {qleverfile_path}" - f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{get_random_string(12)}/'" + f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{random_string}/'" ) if qlever_is_running_in_container: setup_config_cmd += ( @@ -76,13 +210,7 @@ def execute(self, args) -> bool: # Copy the Qleverfile to the current directory. try: - subprocess.run( - setup_config_cmd, - shell=True, - check=True, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - ) + run_command(setup_config_cmd) except Exception as e: log.error( f'Could not copy "{qleverfile_path}"' f" to current directory: {e}" diff --git a/src/qlever/commands/show_config.py b/src/qlever/commands/show_config.py new file mode 100644 index 00000000..bdb5bb1b --- /dev/null +++ b/src/qlever/commands/show_config.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import shlex +from pathlib import Path + +import qlever.globals +from qlever.command import QleverCommand +from qlever.envvars import Envvars +from qlever.log import log + + +class ShowConfigCommand(QleverCommand): + """ + Class for showing the current configuration (either via a Qleverfile or + via environment variables). + + """ + + def __init__(self): + self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles" + self.qleverfile_names = [ + p.name.split(".")[1] for p in self.qleverfiles_path.glob("Qleverfile.*") + ] + + def description(self) -> str: + return "Set up a Qleverfile or show the current configuration" + + def should_have_qleverfile(self) -> bool: + return False + + def relevant_qleverfile_arguments(self) -> dict[str : list[str]]: + return {} + + def additional_arguments(self, subparser) -> None: + subparser.add_argument( + "--varname-width", + type=int, + default=25, + help="Width for variable names in the output", + ) + + def execute(self, args) -> bool: + # Determine if there is a Qleverfile or environment variables (we need + # one or the other, but not both). + qleverfile_exists = qlever.globals.qleverfile_path is not None + envvars_exist = qlever.globals.envvars_config is not None + if qleverfile_exists and envvars_exist: + log.error( + "There are both a Qleverfile and environment variables, " + "this should not happen because it is bound to cause " + "confusion; either remove the Qleverfile or unset the " + "environment variables using `qlever config --unset-envvars`" + ) + return False + if not qleverfile_exists and not envvars_exist: + log.error("Neither a Qleverfile nor environment variables found") + return False + + # Show the configuration from the Qleverfile. + if qleverfile_exists: + if qlever.globals.qleverfile_config is None: + log.error("Qleverfile found, but contains no configuration") + return False + self.show( + f"Show the configuration from " + f"{qlever.globals.qleverfile_path} (with any variables " + f"on the right-hand side already substituted)", + only_show=args.show, + ) + if args.show: + return True + + is_first_section = True + for section, varname_and_values in qlever.globals.qleverfile_config.items(): + if section == "DEFAULT": + continue + if not is_first_section: + log.info("") + is_first_section = False + log.info(f"[{section}]") + for varname, value in varname_and_values.items(): + log.info(f"{varname.upper():{args.varname_width}} = " f"{value}") + return True + + # Show all environment variables of the form QLEVER_SECTION_VARIABLE. + if envvars_exist: + self.show( + "Show all environment variables of the form QLEVER_SECTION_VARIABLE", + only_show=args.show, + ) + if args.show: + return True + + for section, varname_and_values in qlever.globals.envvars_config.items(): + for varname, value in varname_and_values.items(): + var = Envvars.envvar_name(section, varname) + log.info(f"{var:{args.varname_width+7}}" f" = {shlex.quote(value)}") + return True diff --git a/src/qlever/config.py b/src/qlever/config.py index f79e6700..6b0f17c6 100644 --- a/src/qlever/config.py +++ b/src/qlever/config.py @@ -39,10 +39,15 @@ class QleverConfig: autocompletion. """ - def add_subparser_for_command(self, subparsers, command_name, - command_object, all_qleverfile_args, - qleverfile_config=None, - envvars_config=None): + def add_subparser_for_command( + self, + subparsers, + command_name, + command_object, + all_qleverfile_args, + qleverfile_config=None, + envvars_config=None, + ): """ Add subparser for the given command. Take the arguments from `command_object.relevant_qleverfile_arguments()` and report an error if @@ -65,11 +70,15 @@ def add_subparser_for_command(self, subparsers, command_name, # `all_qleverfile_args`. def argument_error(prefix): log.info("") - log.error(f"{prefix} in `Qleverfile.all_arguments()` for command " - f"`{command_name}`") + log.error( + f"{prefix} in `Qleverfile.all_arguments()` for command " + f"`{command_name}`" + ) log.info("") - log.info(f"Value of `relevant_qleverfile_arguments` for " - f"command `{command_name}`:") + log.info( + f"Value of `relevant_qleverfile_arguments` for " + f"command `{command_name}`:" + ) log.info("") log.info(f"{arg_names}") log.info("") @@ -77,9 +86,9 @@ def argument_error(prefix): # Add the subparser. description = command_object.description() - subparser = subparsers.add_parser(command_name, - description=description, - help=description) + subparser = subparsers.add_parser( + command_name, description=description, help=description + ) # Add the arguments relevant for the command. for section in arg_names: @@ -87,8 +96,9 @@ def argument_error(prefix): argument_error(f"Section `{section}` not found") for arg_name in arg_names[section]: if arg_name not in all_qleverfile_args[section]: - argument_error(f"Argument `{arg_name}` of section " - f"`{section}` not found") + argument_error( + f"Argument `{arg_name}` of section " f"`{section}` not found" + ) args, kwargs = all_qleverfile_args[section][arg_name] kwargs_copy = kwargs.copy() # If `qleverfile_config` is given, add the corresponding @@ -96,12 +106,14 @@ def argument_error(prefix): if qleverfile_config is not None: default_value = kwargs.get("default", None) qleverfile_value = qleverfile_config.get( - section, arg_name, fallback=None) + section, arg_name, fallback=None + ) if qleverfile_value is not None: kwargs_copy["default"] = qleverfile_value kwargs_copy["required"] = False - kwargs_copy["help"] += (f" [default, from Qleverfile:" - f" {qleverfile_value}]") + kwargs_copy["help"] += ( + f" [default, from Qleverfile:" f" {qleverfile_value}]" + ) else: kwargs_copy["help"] += f" [default: {default_value}]" # If `envvars_config` is given, add the corresponding default @@ -113,9 +125,11 @@ def argument_error(prefix): if envvars_value is not None: kwargs_copy["default"] = envvars_value kwargs_copy["required"] = False - kwargs_copy["help"] += (f" [default, from environment " - f"variable `{envvar_name}`: " - f"{envvars_value}]") + kwargs_copy["help"] += ( + f" [default, from environment " + f"variable `{envvar_name}`: " + f"{envvars_value}]" + ) else: kwargs_copy["help"] += f" [default: {default_value}]" # Now add the argument to the subparser. @@ -123,14 +137,18 @@ def argument_error(prefix): # Additional arguments that are shared by all commands. command_object.additional_arguments(subparser) - subparser.add_argument("--show", action="store_true", - default=False, - help="Only show what would be executed" - ", but don't execute it") - subparser.add_argument("--log-level", - choices=log_levels.keys(), - default="INFO", - help="Set the log level") + subparser.add_argument( + "--show", + action="store_true", + default=False, + help="Only show what would be executed" ", but don't execute it", + ) + subparser.add_argument( + "--log-level", + choices=log_levels.keys(), + default="INFO", + help="Set the log level", + ) def parse_args(self): # Determine whether we are in autocomplete mode or not. @@ -141,11 +159,13 @@ def parse_args(self): argcomplete_enabled = os.environ.get("QLEVER_ARGCOMPLETE_ENABLED") if not argcomplete_enabled and not argcomplete_check_off: log.info("") - log.warn(f"To enable autocompletion, run the following command, " - f"and consider adding it to your `.bashrc` or `.zshrc`:" - f"\n\n" - f"eval \"$(register-python-argcomplete {script_name})\"" - f" && export QLEVER_ARGCOMPLETE_ENABLED=1") + log.warn( + f"To enable autocompletion, run the following command, " + f"and consider adding it to your `.bashrc` or `.zshrc`:" + f"\n\n" + f'eval "$(register-python-argcomplete {script_name})"' + f" && export QLEVER_ARGCOMPLETE_ENABLED=1" + ) log.info("") # Create a temporary parser only to parse the `--qleverfile` option, in @@ -154,8 +174,8 @@ def parse_args(self): # want the values from the Qleverfile to be shown in the help strings, # but only if this is actually necessary. def add_qleverfile_option(parser): - parser.add_argument("--qleverfile", "-q", type=str, - default="Qleverfile") + parser.add_argument("--qleverfile", "-q", type=str, default="Qleverfile") + qleverfile_parser = argparse.ArgumentParser(add_help=False) add_qleverfile_option(qleverfile_parser) qleverfile_parser.add_argument("command", type=str, nargs="?") @@ -169,14 +189,17 @@ def add_qleverfile_option(parser): # We need this again further down in the code, so remember it. qleverfile_path = Path(qleverfile_path_name) qleverfile_exists = qleverfile_path.is_file() - qleverfile_is_default = qleverfile_path_name \ - == qleverfile_parser.get_default("qleverfile") + qleverfile_is_default = qleverfile_path_name == qleverfile_parser.get_default( + "qleverfile" + ) # If a Qleverfile with a non-default name was specified, but it does # not exist, that's an error. if not qleverfile_exists and not qleverfile_is_default: - raise ConfigException(f"Qleverfile with non-default name " - f"`{qleverfile_path_name}` specified, " - f"but it does not exist") + raise ConfigException( + f"Qleverfile with non-default name " + f"`{qleverfile_path_name}` specified, " + f"but it does not exist" + ) # If it exists and we are not in the autocompletion mode, parse it. # # IMPORTANT: No need to parse the Qleverfile in autocompletion mode and @@ -189,8 +212,7 @@ def add_qleverfile_option(parser): qleverfile_config = Qleverfile.read(qleverfile_path) except Exception as e: log.info("") - log.error(f"Error parsing Qleverfile `{qleverfile_path}`" - f": {e}") + log.error(f"Error parsing Qleverfile `{qleverfile_path}`" f": {e}") log.info("") exit(1) else: @@ -203,35 +225,50 @@ def add_qleverfile_option(parser): # Check that at most one of `qleverfile_config` and `envvars_config` is # not `None`, unless the command is `config` (which can be used to # produce a `Qleverfile` or unset all environment variables). - if qleverfile_args.command != "config" \ - and qleverfile_config is not None \ - and envvars_config is not None: - raise ConfigException( - "You both have a `Qleverfile` and environment variables " - "of the QLEVER_SECTION_VARIABLE. This is not supported " - "because it is bound to lead to unexpected behavior. " - "Either remove the `Qleverfile` (just delete it), or the " - "environment variables (use `qlever config " - "--unset-envvars`).") + if ( + qleverfile_args.command != "setup-config" + and qleverfile_config is not None + and envvars_config is not None + ): + log.info("") + log.error( + "You have a `Qleverfile` _and_ environment variables " + "of the form QLEVER_SECTION_VARIABLE. This is not supported " + "because it is bound to lead to unexpected behavior. " + "Either remove the `Qleverfile` (just delete it), or the " + "environment variables (use `qlever setup-config " + "--unset-envvars`)." + ) + log.info("") + exit(1) # Now the regular parser with commands and a subparser for each # command. We have a dedicated class for each command. These classes # are defined in the modules in `qlever/commands`. In `__init__.py` # an object of each class is created and stored in `command_objects`. parser = argparse.ArgumentParser( - description=colored("This is the qlever command line tool, " - "it's all you need to work with QLever", - attrs=["bold"])) - parser.add_argument("--version", action="version", - version=f"%(prog)s {version('qlever')}") + description=colored( + "This is the qlever command line tool, " + "it's all you need to work with QLever", + attrs=["bold"], + ) + ) + parser.add_argument( + "--version", action="version", version=f"%(prog)s {version('qlever')}" + ) add_qleverfile_option(parser) - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") subparsers.required = True all_args = Qleverfile.all_arguments() for command_name, command_object in command_objects.items(): self.add_subparser_for_command( - subparsers, command_name, command_object, - all_args, qleverfile_config, envvars_config) + subparsers, + command_name, + command_object, + all_args, + qleverfile_config, + envvars_config, + ) # Enable autocompletion for the commands and their options. # diff --git a/src/qlever/envvars.py b/src/qlever/envvars.py index 068c1136..2cbf0348 100644 --- a/src/qlever/envvars.py +++ b/src/qlever/envvars.py @@ -34,9 +34,9 @@ def read(): For example, for `QLEVER_SERVER_PORT=8000`, there would be an entry `config['server']['port'] = 8000`. - NOTE: If no environment variables was found at all, the method will - return `None`. Otherwise, there will be an entry for each section, even - if it is empty. + NOTE: If no environment variables are found at all, the method returns + `None`. Otherwise, there is an entry for each section, even if there + are no environment variables for that section. """ all_args = Qleverfile.all_arguments()