diff --git a/.gitignore b/.gitignore index d887f5e3..b3261e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ + +# this file will be generated by `setuptools_scm` +/nichtparasoup/VERSION + + # Distribution / packaging /nichtparasoup-*/ diff --git a/nichtparasoup/.gitignore b/nichtparasoup/.gitignore deleted file mode 100644 index aa286d4d..00000000 --- a/nichtparasoup/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ - -# this file will be generated by `setuptools_scm` -/VERSION diff --git a/nichtparasoup/__main__.py b/nichtparasoup/__main__.py deleted file mode 100644 index 7741326a..00000000 --- a/nichtparasoup/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -import sys - -from nichtparasoup.cli.main import main as _main - -sys.exit(_main()) diff --git a/nichtparasoup/_internals/__init__.py b/nichtparasoup/_internals/__init__.py index 881cff3b..5092a7dc 100644 --- a/nichtparasoup/_internals/__init__.py +++ b/nichtparasoup/_internals/__init__.py @@ -65,21 +65,5 @@ def _message_exception(exception: BaseException, file: Optional[TextIO] = None) _message('{}: {}'.format(exception_name, exception), file=file) -def _confirm(prompt: str, default: bool = False) -> Optional[bool]: - return_values = { - 'y': True, - 'yes': True, - '': default, - 'n': False, - 'no': False, - } - options = 'Y/n' if default else 'y/N' - try: - value = input('{} [{}]: '.format(prompt, options)).strip().lower() - return return_values[value] - except (KeyboardInterrupt, EOFError, KeyError): - return None - - def _type_module_name_str(t: Type[Any]) -> str: return '{}:{}'.format(t.__module__, t.__name__) diff --git a/nichtparasoup/cli/__init__.py b/nichtparasoup/cli/__init__.py index 13d491a1..a7506b4e 100644 --- a/nichtparasoup/cli/__init__.py +++ b/nichtparasoup/cli/__init__.py @@ -1,4 +1,63 @@ -"""Subpackage containing all of nichtparasoup's command line interface related code -""" +__all__ = ['main'] -# This file intentionally does not import submodules +from os.path import dirname +from sys import version_info + +from click import group, version_option + +import nichtparasoup +# from nichtparasoup.cli.completion import main as completion # @FIXME removed until fixed -- see module itself +from nichtparasoup.commands.imagecrawler_desc import main as imagecrawler_desc +from nichtparasoup.commands.imagecrawler_list import main as imagecrawler_list +from nichtparasoup.commands.server_config_check import main as server_config_check +from nichtparasoup.commands.server_config_dump_defaults import main as server_config_dump_defaults +from nichtparasoup.commands.server_run import main as server_run + +VERSION_STRING = '%(version)s from {location} (python {py_version})'.format( + location=dirname(nichtparasoup.__file__), + py_version='{}.{}'.format(version_info.major, version_info.minor) +) + + +@group(name='nichtparasoup') +@version_option(version=nichtparasoup.__version__, message=VERSION_STRING) +def main() -> None: # pragma: no cover + """Nichtparasoup + """ + pass + + +# @FIXME removed until fixed -- see module itself +# main.add_command(completion, name='completion') + + +@main.group(name='server') +def server() -> None: # pragma: no cover + """Manage server. + """ + pass + + +server.add_command(server_run, name='run') + + +@server.group(name='config') +def server_config() -> None: # pragma: no cover + """Manage server configs. + """ + pass + + +server_config.add_command(server_config_check, name='check') +server_config.add_command(server_config_dump_defaults, name='dump-defaults') + + +@main.group(name='imagecrawler') +def imagecrawler() -> None: # pragma: no cover + """Manage imagecrawlers. + """ + pass + + +imagecrawler.add_command(imagecrawler_list, name='list') +imagecrawler.add_command(imagecrawler_desc, name='desc') diff --git a/nichtparasoup/cli/__main__.py b/nichtparasoup/cli/__main__.py new file mode 100644 index 00000000..a91c8cf2 --- /dev/null +++ b/nichtparasoup/cli/__main__.py @@ -0,0 +1,4 @@ + +from . import main + +main() diff --git a/nichtparasoup/cli/commands/__init__.py b/nichtparasoup/cli/commands/__init__.py deleted file mode 100644 index 1db43061..00000000 --- a/nichtparasoup/cli/commands/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -__all__ = ["BaseCommand", "create_command"] - -import importlib -from abc import ABC, abstractmethod -from collections import namedtuple -from types import ModuleType -from typing import Any, Dict, Type - - -class BaseCommand(ABC): - - def __init__(self, debug: bool = False) -> None: - self._debug = debug - - @abstractmethod - def main(self, options: Dict[str, Any]) -> int: # pragma: no cover - raise NotImplementedError() - - -__THIS_PACKAGE = 'nichtparasoup.cli.commands' - -_Command = namedtuple('_Command', 'module_name, class_name') - -_COMMANDS = dict( - run=_Command('.run', 'RunCommand'), - config=_Command('.config', 'ConfigCommand'), - imagecrawler=_Command('.imagecrawler', 'ImagecrawlerCommand'), - completion=_Command('.completion', 'CompletionCommand'), -) # type: Dict[str, _Command] - - -def create_command(command_name: str, debug: bool = False) -> BaseCommand: - command = _COMMANDS[command_name] - module = importlib.import_module(command.module_name, package=__THIS_PACKAGE) # type: ModuleType - command_class = getattr(module, command.class_name) # type: Type[BaseCommand] - # No check if ABS is resolved here. In fact this is done via UnitTests. - return command_class(debug) diff --git a/nichtparasoup/cli/commands/completion.py b/nichtparasoup/cli/commands/completion.py deleted file mode 100644 index 7be1a30f..00000000 --- a/nichtparasoup/cli/commands/completion.py +++ /dev/null @@ -1,23 +0,0 @@ -__all__ = ["CompletionCommand"] - -from sys import stdout -from typing import Any, Dict - -from argcomplete import shellcode # type: ignore - -from nichtparasoup.cli.commands import BaseCommand - - -class CompletionCommand(BaseCommand): - - def main(self, options: Dict[str, Any]) -> int: # pragma: no cover - shell = options['shell'] - return self.run_completer(shell) - - @staticmethod - def run_completer(shell: str) -> int: - stdout.write(shellcode( - ['nichtparasoup'], shell=shell, - use_defaults=True, complete_arguments=None, - )) - return 0 diff --git a/nichtparasoup/cli/commands/config.py b/nichtparasoup/cli/commands/config.py deleted file mode 100644 index 02ed8adf..00000000 --- a/nichtparasoup/cli/commands/config.py +++ /dev/null @@ -1,51 +0,0 @@ -__all__ = ["ConfigCommand"] - -from logging import DEBUG as L_DEBUG, ERROR as L_ERROR -from os.path import abspath, isfile -from typing import Any, Dict - -from nichtparasoup._internals import _confirm, _log, _logging_init, _message, _message_exception -from nichtparasoup.cli.commands import BaseCommand -from nichtparasoup.config import dump_defaults -from nichtparasoup.testing.config import ConfigFileTest - - -class ConfigCommand(BaseCommand): - - def main(self, options: Dict[str, Any]) -> int: - active_actions = {k: v for k, v in options.items() if v} # type: Dict[str, Any] - if len(active_actions) != 1: - _message_exception(ValueError('exactly one action required')) - return 255 - action_name, action_value = active_actions.popitem() - action = getattr(self, 'run_{}'.format(action_name)) - return action(action_value) # type: ignore - - def run_dump(self, config_file: str) -> int: - _logging_init(L_DEBUG if self._debug else L_ERROR) - config_file = abspath(config_file) - _log('debug', 'ConfigFile: {}'.format(config_file)) - if isfile(config_file): - overwrite = _confirm('File already exists, overwrite?') - if overwrite is not True: - _message('Abort.') - return 1 - try: - dump_defaults(config_file) - return 0 - except Exception as e: - _message_exception(e) - return 1 - - def run_check(self, config_file: str) -> int: - _logging_init(L_DEBUG if self._debug else L_ERROR) - config_file = abspath(config_file) - _log('debug', 'ConfigFile: {}'.format(config_file)) - config_test = ConfigFileTest() - try: - config_test.validate(config_file) - config_test.probe(config_file) - except Exception as e: - _message_exception(e) - return 1 - return 0 diff --git a/nichtparasoup/cli/commands/imagecrawler.py b/nichtparasoup/cli/commands/imagecrawler.py deleted file mode 100644 index 036261c0..00000000 --- a/nichtparasoup/cli/commands/imagecrawler.py +++ /dev/null @@ -1,65 +0,0 @@ -__all__ = ["ImagecrawlerCommand"] - -from logging import DEBUG as L_DEBUG, ERROR as L_ERROR -from typing import Any, Dict, Optional, Type - -from nichtparasoup._internals import ( - _LINEBREAK, _log, _logging_init, _message, _message_exception, _type_module_name_str, -) -from nichtparasoup.cli.commands import BaseCommand -from nichtparasoup.imagecrawler import BaseImageCrawler, get_imagecrawlers - - -class ImagecrawlerCommand(BaseCommand): - - def main(self, options: Dict[str, Any]) -> int: - active_actions = {k: v for k, v in options.items() if v} # type: Dict[str, Any] - if len(active_actions) != 1: - _message_exception(ValueError('exactly one action required')) - return 255 - action_name, action_value = active_actions.popitem() - action = getattr(self, 'run_{}'.format(action_name)) - return action(action_value) # type: ignore - - def run_list(self, _: Optional[Any] = None) -> int: - _logging_init(L_DEBUG if self._debug else L_ERROR) - imagecrawlers = get_imagecrawlers() # may trigger debug output - _log('debug', '- List of loaded ImageCrawlers -') - if len(imagecrawlers) > 0: - _message(sorted(imagecrawlers.names())) - else: - _message_exception(Warning('no ImageCrawler found')) - return 0 - - def run_desc(self, imagecrawler: str) -> int: - imagecrawler_class = get_imagecrawlers().get_class(imagecrawler) - if imagecrawler_class: - self._print_imagecrawler_info(imagecrawler_class) - return 0 - _message_exception(ValueError('unknown ImageCrawler {!r}'.format(imagecrawler))) - return 1 - - def _print_imagecrawler_info(self, imagecrawler_class: Type[BaseImageCrawler]) -> None: - bull = ' * ' - imagecrawler_info = imagecrawler_class.info() - _message(imagecrawler_info.description) - _message('') - if imagecrawler_info.long_description: - _message(imagecrawler_info.long_description) - _message('') - if imagecrawler_info.config: - _message('CONFIG') - mlen = max(len(k) for k in imagecrawler_info.config.keys()) - _message(_LINEBREAK.join( - bull + '{key:{mlen}}: {desc}'.format(mlen=mlen, key=key, desc=desc) - for key, desc - in imagecrawler_info.config.items() - )) - _message('') - if self._debug: - _message(_LINEBREAK.join([ - 'DEBUG INFO', - bull + 'Icon : {}'.format(imagecrawler_info.icon_url), - bull + 'Class: {}'.format(_type_module_name_str(imagecrawler_class)), - ])) - _message('') diff --git a/nichtparasoup/cli/commands/run.py b/nichtparasoup/cli/commands/run.py deleted file mode 100644 index c96ffc07..00000000 --- a/nichtparasoup/cli/commands/run.py +++ /dev/null @@ -1,42 +0,0 @@ -__all__ = ["RunCommand"] - -import logging -from os.path import abspath -from typing import Any, Dict, Optional - -from nichtparasoup._internals import _log, _logging_init, _message_exception -from nichtparasoup.cli.commands import BaseCommand -from nichtparasoup.config import get_config, get_imagecrawler -from nichtparasoup.core import NPCore -from nichtparasoup.core.server import Server as ImageServer -from nichtparasoup.webserver import WebServer - - -class RunCommand(BaseCommand): - - def main(self, options: Dict[str, Any]) -> int: # pragma: no cover - config_file = options['config_file'] - return self.run_server(config_file) - - def run_server(self, config_file: Optional[str]) -> int: - config_file = abspath(config_file) if config_file else None - try: - config = get_config(config_file) - _logging_init(logging.DEBUG if self._debug else getattr(logging, config['logging']['level'])) - _log('debug', 'ConfigFile: {}'.format(config_file or 'builtin SystemDefaults')) - _log('debug', 'Config: {!r}'.format(config)) - webserver = self._create_webserver(config) - webserver.run() - return 0 - except Exception as e: - _message_exception(e) - return 1 - - @staticmethod - def _create_webserver(config: Dict[str, Any]) -> WebServer: - imageserver = ImageServer(NPCore(), **config['imageserver']) - for crawler_config in config['crawlers']: - imagecrawler = get_imagecrawler(crawler_config) - if not imageserver.core.has_imagecrawler(imagecrawler): - imageserver.core.add_imagecrawler(imagecrawler, crawler_config['weight']) - return WebServer(imageserver, **config['webserver']) diff --git a/nichtparasoup/cli/completion.py b/nichtparasoup/cli/completion.py new file mode 100644 index 00000000..4f581b00 --- /dev/null +++ b/nichtparasoup/cli/completion.py @@ -0,0 +1,26 @@ +__all__ = ["main"] + +from typing import Optional + +from click import Choice, command, echo, option +from click_completion import Shell, get_code, init # type: ignore + +# click's builtin completion is insufficient and partially broken. +# completes loaded sub commands as 'main' instead of the correct name +init() +# @FIXME unfortunately there is no proper completion for files +# @FIXME and since there is no support for manual `autocomplete` callbacks, i cannot patch it in ... +# - see https://github.com/click-contrib/click-completion/pull/27 + +ShellChoice = Choice(Shell.__members__.keys()) + + +@command(name='completion') +@option('--shell', type=ShellChoice, default=None, metavar='SHELL', + help='Override auto-detection. Values: {}.'.format(', '.join(ShellChoice.choices))) +def main(shell: Optional[Shell]) -> None: # pragma: no cover + """Emit completion code for the shell. + + Enables the shell to auto-complete nichtparasoup commands and options. + """ + echo(get_code(shell=shell, prog_name='nichtparasoup')) diff --git a/nichtparasoup/cli/main.py b/nichtparasoup/cli/main.py deleted file mode 100644 index 87de2d14..00000000 --- a/nichtparasoup/cli/main.py +++ /dev/null @@ -1,21 +0,0 @@ -# PYTHON_ARGCOMPLETE_OK - -__all__ = ["main"] - -from typing import List, Optional - -from argcomplete import autocomplete # type: ignore - -from nichtparasoup.cli.commands import create_command -from nichtparasoup.cli.parser import create_parser - - -def main(args: Optional[List[str]] = None) -> int: # pragma: no cover - parser = create_parser() - autocomplete(parser, always_complete_options='long') - options = dict(parser.parse_args(args=args).__dict__) # TODO don't use dict .. use Namespace ... - del parser - debug = options.pop('debug', False) - command_name = options.pop('command') - command = create_command(command_name, debug) - return command.main(options) diff --git a/nichtparasoup/cli/parser.py b/nichtparasoup/cli/parser.py deleted file mode 100644 index cfa11816..00000000 --- a/nichtparasoup/cli/parser.py +++ /dev/null @@ -1,131 +0,0 @@ -__all__ = ["create_parser"] - -from argparse import ArgumentParser -from typing import Any, Set - -from argcomplete import FilesCompleter # type: ignore - -from nichtparasoup import __version__ -from nichtparasoup.imagecrawler import get_imagecrawlers - - -def _imagecrawler_completion(*args: Any, **kwargs: Any) -> Set[str]: # pragma: no cover - """ImageCrawler completer. - see https://kislyuk.github.io/argcomplete/#specifying-completers - """ - del args - del kwargs - return set(get_imagecrawlers().names()) - - -_YAML_FILE_COMPLETION = FilesCompleter(allowednames=('yaml', 'yml'), directories=True) - - -def create_parser() -> ArgumentParser: # pragma: no cover - # used `__tmp_action` several times, to omit type-checkers warning ala 'Action has no attribute "completer"' - - debug = ArgumentParser(add_help=False) - debug.add_argument( - '--debug', - help='enable debug output', - action='store_true', dest="debug", - ) - - parser = ArgumentParser( - add_help=True, - allow_abbrev=False, - ) - - parser.add_argument( - '--version', - action='version', - version=__version__, - ) - - commands = parser.add_subparsers( - title='Commands', - metavar='', - dest='command', - ) - commands.required = True - - command_run = commands.add_parser( - 'run', - help='run a server', - description='Start a web-server to display random images.', - add_help=True, - allow_abbrev=False, - parents=[debug], - ) - __tmp_action = command_run.add_argument( - '-c', '--use-config', - help='use a YAML config file instead of the defaults.', - metavar='', - action='store', dest="config_file", type=str, - ) - __tmp_action.completer = _YAML_FILE_COMPLETION # type: ignore - del __tmp_action - - command_config = commands.add_parser( - 'config', - help='config related functions', - description='Get config related things done.', - add_help=True, - allow_abbrev=False, - parents=[debug], - ) - command_config_switches = command_config.add_mutually_exclusive_group(required=True) - __tmp_action = command_config_switches.add_argument( - '--check', - help='validate and probe a YAML config file', - metavar='', - action='store', dest='check', type=str, - ) - __tmp_action.completer = _YAML_FILE_COMPLETION # type: ignore - del __tmp_action - command_config_switches.add_argument( - '--dump', - help='dump YAML config into a file', - metavar='', - action='store', dest='dump', type=str, - ) - - command_imagecrawler = commands.add_parser( - 'imagecrawler', - help='get info for several topics', - description='Get info for several topics.', - add_help=True, - allow_abbrev=False, - parents=[debug], - ) - command_imagecrawler_switches = command_imagecrawler.add_mutually_exclusive_group(required=True) - command_imagecrawler_switches.add_argument( - '--list', - help='list available image crawler types', - action='store_true', dest='list', - ) - __tmp_action = command_imagecrawler_switches.add_argument( - '--desc', - help='describe an image crawler type and its config', - metavar='', - action='store', dest='desc', type=str, - ) - __tmp_action.completer = _imagecrawler_completion # type: ignore - del __tmp_action - - command_completion = commands.add_parser( - 'completion', - help='helper command to be used for command completion', - description='Helper command used for command completion.', - epilog='Completion is powered by https://pypi.org/project/argcomplete/', - add_help=True, - allow_abbrev=False, - ) - command_completion.add_argument( - '-s', '--shell', - help='emit completion code for the specified shell', - action='store', dest='shell', type=str, required=True, - choices=('bash', 'tcsh', 'fish'), - ) - - return parser diff --git a/nichtparasoup/commands/__init__.py b/nichtparasoup/commands/__init__.py new file mode 100644 index 00000000..918d8db6 --- /dev/null +++ b/nichtparasoup/commands/__init__.py @@ -0,0 +1,4 @@ +"""Subpackage containing all of nichtparasoup's commands +""" + +# This file intentionally does not import submodules diff --git a/nichtparasoup/commands/imagecrawler_desc.py b/nichtparasoup/commands/imagecrawler_desc.py new file mode 100644 index 00000000..58ac2c5b --- /dev/null +++ b/nichtparasoup/commands/imagecrawler_desc.py @@ -0,0 +1,50 @@ +__all__ = ['main'] + +from typing import Type + +from click import BadParameter, Choice, argument, command, option + +from nichtparasoup._internals import _LINEBREAK, _message, _type_module_name_str +from nichtparasoup.imagecrawler import BaseImageCrawler, get_imagecrawlers + + +@command(name='imagecrawler-desc') +@argument('name', type=Choice(tuple(get_imagecrawlers().names())), metavar='NAME') +@option('--debug', is_flag=True, help='Enable debug output.') +def main(name: str, *, debug: bool = False) -> None: # pragma: no cover + """Describe an imagecrawler and its configuration. + """ + imagecrawler_class = get_imagecrawlers().get_class(name) + if not imagecrawler_class: + raise BadParameter(name, param_hint='name') + _print_imagecrawler_info(imagecrawler_class, debug=debug) + + +def _print_imagecrawler_info(imagecrawler_class: Type[BaseImageCrawler], *, debug: bool) -> None: + bull = ' * ' + imagecrawler_info = imagecrawler_class.info() + _message(imagecrawler_info.description) + _message('') + if imagecrawler_info.long_description: + _message(imagecrawler_info.long_description) + _message('') + if imagecrawler_info.config: + _message('CONFIG') + mlen = max(len(k) for k in imagecrawler_info.config.keys()) + _message(_LINEBREAK.join( + bull + '{key:{mlen}}: {desc}'.format(mlen=mlen, key=key, desc=desc) + for key, desc + in imagecrawler_info.config.items() + )) + _message('') + if debug: + _message(_LINEBREAK.join([ + 'DEBUG INFO', + bull + 'Icon : {}'.format(imagecrawler_info.icon_url), + bull + 'Class: {}'.format(_type_module_name_str(imagecrawler_class)), + ])) + _message('') + + +if __name__ == '__main__': + main() diff --git a/nichtparasoup/commands/imagecrawler_list.py b/nichtparasoup/commands/imagecrawler_list.py new file mode 100644 index 00000000..81aa806d --- /dev/null +++ b/nichtparasoup/commands/imagecrawler_list.py @@ -0,0 +1,26 @@ +__all__ = ['main'] + +from logging import DEBUG as L_DEBUG, ERROR as L_ERROR + +from click import command, option + +from nichtparasoup._internals import _log, _logging_init, _message, _message_exception +from nichtparasoup.imagecrawler import get_imagecrawlers + + +@command(name='imagecrawler-list') +@option('--debug', is_flag=True, help='Enable debug output.') +def main(*, debug: bool = False) -> None: # pragma: no cover + """List available imagecrawlers. + """ + _logging_init(L_DEBUG if debug else L_ERROR) + imagecrawlers = get_imagecrawlers() # may trigger debug output + _log('debug', '- List of loaded ImageCrawlers -') + if len(imagecrawlers) > 0: + _message(sorted(imagecrawlers.names())) + else: + _message_exception(Warning('no ImageCrawler found.')) + + +if __name__ == '__main__': + main() diff --git a/nichtparasoup/commands/server_config_check.py b/nichtparasoup/commands/server_config_check.py new file mode 100644 index 00000000..ea010e87 --- /dev/null +++ b/nichtparasoup/commands/server_config_check.py @@ -0,0 +1,49 @@ +__all__ = ['main'] + +from logging import DEBUG as L_DEBUG, ERROR as L_ERROR + +from click import FloatRange, IntRange, Path, argument, command, option +from click.exceptions import ClickException + +from nichtparasoup._internals import _log, _logging_init +from nichtparasoup.config import Config +from nichtparasoup.testing.config import PROBE_DELAY_DEFAULT, PROBE_RETRIES_DEFAULT, ConfigFileTest, ConfigTest + + +@command(name='server-config-check') +@argument('file', type=Path(exists=True, dir_okay=False, resolve_path=True)) +@option('--probe/--no-probe', default=True, help='Enable/disable probe crawls.', show_default=True) +@option('--probe-retries', metavar='retries', type=IntRange(min=0), default=PROBE_RETRIES_DEFAULT, + help='Set number of probe retries in case of errors.', show_default=True) +@option('--probe-delay', metavar='seconds', type=FloatRange(min=0), default=PROBE_DELAY_DEFAULT, + help='Set probe delay in seconds.', show_default=True) +@option('--debug', is_flag=True, help='Enable debug output.') +def main(file: str, *, + probe: bool = True, probe_retries: int = PROBE_RETRIES_DEFAULT, probe_delay: float = PROBE_DELAY_DEFAULT, + debug: bool = False) -> None: # pragma: no cover + """Validate (and probe) a YAML config file. + """ + _logging_init(L_DEBUG if debug else L_ERROR) + _log('debug', 'ConfigFile: {}'.format(file)) + config = _validate_file(file) + if probe: + _probe_config(config, retries=probe_retries, delay=probe_delay) + + +def _validate_file(file: str) -> Config: # pragma: no cover + try: + return ConfigFileTest().validate(file) + except Exception as e: + raise ClickException('ValidateError: {!s}'.format(e)) from e + + +def _probe_config(config: Config, *, retries: int, delay: float) -> None: # pragma: no cover + config_test = ConfigTest() + try: + config_test.probe(config, delay=delay, retries=retries) + except Exception as e: + raise ClickException('ProbeError: {!s}'.format(e)) from e + + +if __name__ == '__main__': + main() diff --git a/nichtparasoup/commands/server_config_dump_defaults.py b/nichtparasoup/commands/server_config_dump_defaults.py new file mode 100644 index 00000000..702584cd --- /dev/null +++ b/nichtparasoup/commands/server_config_dump_defaults.py @@ -0,0 +1,33 @@ +__all__ = ['main'] + +from logging import DEBUG as L_DEBUG, ERROR as L_ERROR +from os.path import isfile +from typing import Optional + +from click import BadParameter, Path, argument, command, confirm, option + +from nichtparasoup._internals import _log, _logging_init +from nichtparasoup.config import dump_defaults + + +@command(name='server-config-dump-defaults') +@argument('file', type=Path(writable=True, dir_okay=False, resolve_path=True)) +@option('--overwrite/--no-overwrite', default=None, help='Overwrite file target.', show_default=True) +@option('--debug', is_flag=True, help='Enable debug output.') +def main(file: str, *, overwrite: Optional[bool] = None, debug: bool = False) -> None: # pragma: no cover + """Dump the builtin default YAML config. + + The dumped defaults give a quick start when writing an own config. + """ + _logging_init(L_DEBUG if debug else L_ERROR) + _log('debug', 'ConfigFile: {}'.format(file)) + if isfile(file): + if overwrite is None: + confirm('File already exists, overwrite?', default=False, abort=True) + elif not overwrite: + raise BadParameter('File already exists.', param_hint='file') + dump_defaults(file) + + +if __name__ == '__main__': + main() diff --git a/nichtparasoup/commands/server_run.py b/nichtparasoup/commands/server_run.py new file mode 100644 index 00000000..e7d3abc3 --- /dev/null +++ b/nichtparasoup/commands/server_run.py @@ -0,0 +1,47 @@ +__all__ = ['main'] + +import logging +from typing import Optional + +from click import BadParameter, Context, Parameter, Path, command, option + +from nichtparasoup._internals import _LINEBREAK, _log, _logging_init +from nichtparasoup.config import Config, get_config, get_imagecrawler +from nichtparasoup.core import NPCore +from nichtparasoup.core.server import Server as ImageServer +from nichtparasoup.webserver import WebServer + + +def _param_get_config(_: Context, param: Parameter, config_file: Optional[str]) -> Config: # pragma: no cover + try: + return get_config(config_file) + except Exception as e: + raise BadParameter( + '{}{}Use the "server config check" command for an analyse.'.format(e, _LINEBREAK), + param=param + ) from e + + +@command(name='server-run') +@option('--config', type=Path(exists=True, dir_okay=False, resolve_path=True), + required=False, default=None, callback=_param_get_config, + help='Use custom YAML config file instead of the defaults.') +@option('--debug', is_flag=True, help='Enable debug output.') +@option('--develop', is_flag=True, help='Start the server in frontend-developer mode.') +def main(config: Config, *, debug: bool = False, develop: bool = False) -> None: # pragma: no cover + """Start a web-server to display random images. + """ + del develop # @TODO implement develop mode - enable arbitrary CORS + _logging_init(logging.DEBUG if debug else getattr(logging, config['logging']['level'])) + _log('debug', 'Config: {!r}'.format(config)) + imageserver = ImageServer(NPCore(), **config['imageserver']) + for crawler_config in config['crawlers']: + imagecrawler = get_imagecrawler(crawler_config) + if not imageserver.core.has_imagecrawler(imagecrawler): + imageserver.core.add_imagecrawler(imagecrawler, crawler_config['weight']) + webserver = WebServer(imageserver, **config['webserver']) + webserver.run() + + +if __name__ == '__main__': + main() diff --git a/nichtparasoup/config/__init__.py b/nichtparasoup/config/__init__.py index c8532cc1..9d0fdd4b 100644 --- a/nichtparasoup/config/__init__.py +++ b/nichtparasoup/config/__init__.py @@ -1,5 +1,5 @@ __all__ = ["get_config", "get_defaults", "dump_defaults", "get_imagecrawler", "parse_yaml_file", - "ImageCrawlerSetupError"] + "ImageCrawlerSetupError", "Config"] from copy import deepcopy from os.path import dirname, join as path_join, realpath @@ -17,6 +17,8 @@ _DEFAULTS_FILE = realpath(path_join(dirname(__file__), "defaults.yaml")) _defaults = None # type: Optional[Dict[str, Any]] +Config = Dict[str, Any] + class ImageCrawlerSetupError(Exception): @@ -41,7 +43,7 @@ def get_imagecrawler(config_crawler: Dict[str, Any]) -> BaseImageCrawler: raise ImageCrawlerSetupError(imagecrawler_name, imagecrawler_class, imagecrawler_config) from e -def parse_yaml_file(file_path: str) -> Dict[str, Any]: +def parse_yaml_file(file_path: str) -> Config: global _schema if not _schema: _schema = yamale.make_schema(_SCHEMA_FILE, parser='ruamel') @@ -59,14 +61,14 @@ def dump_defaults(file_path: str) -> None: copyfile(_DEFAULTS_FILE, file_path) -def get_defaults() -> Dict[str, Any]: +def get_defaults() -> Config: global _defaults if not _defaults: _defaults = parse_yaml_file(_DEFAULTS_FILE) return deepcopy(_defaults) -def get_config(config_file: Optional[str] = None) -> Dict[str, Any]: +def get_config(config_file: Optional[str] = None) -> Config: if not config_file: return get_defaults() try: diff --git a/nichtparasoup/core/imagecrawler.py b/nichtparasoup/core/imagecrawler.py index 8847a2e3..0988e2ba 100644 --- a/nichtparasoup/core/imagecrawler.py +++ b/nichtparasoup/core/imagecrawler.py @@ -1,4 +1,8 @@ -__all__ = ["ImageCrawlerConfig", "BaseImageCrawler", "ImageCrawlerInfo", "RemoteFetcher", "ImageRecognizer"] +__all__ = [ + "ImageCrawlerConfig", + "BaseImageCrawler", + "ImageCrawlerInfo", "RemoteFetcher", "ImageRecognizer" +] from abc import ABC, abstractmethod from http.client import HTTPResponse diff --git a/nichtparasoup/imagecrawler/__init__.py b/nichtparasoup/imagecrawler/__init__.py index 0ec75765..b974396c 100644 --- a/nichtparasoup/imagecrawler/__init__.py +++ b/nichtparasoup/imagecrawler/__init__.py @@ -1,7 +1,7 @@ __all__ = [ "get_imagecrawlers", # for convenience, all classes that are needed to implement an ImageCrawler are exported, here - "BaseImageCrawler", "ImageCrawlerConfig", "ImageCrawlerInfo", + "BaseImageCrawler", "ImageCrawler", "ImageCrawlerConfig", "ImageCrawlerInfo", "Image", "ImageCollection", "RemoteFetcher", "ImageRecognizer", ] diff --git a/nichtparasoup/testing/config.py b/nichtparasoup/testing/config.py index e7888bc7..195efe50 100644 --- a/nichtparasoup/testing/config.py +++ b/nichtparasoup/testing/config.py @@ -4,18 +4,50 @@ from typing import List from unittest import TestCase -from nichtparasoup.config import get_imagecrawler, parse_yaml_file +from nichtparasoup.config import Config, get_imagecrawler, parse_yaml_file from nichtparasoup.core.imagecrawler import BaseImageCrawler +PROBE_DELAY_DEFAULT = 0.05 # type: float +PROBE_RETRIES_DEFAULT = 2 # type: int + class ConfigFileTest(TestCase): - def validate(self, file: str) -> None: - """Validate a config file. - :param file: file path to the config to validate. + def parse(self, file: str) -> Config: + """Parse a config file. + :param file: file path to the config to parse. """ config = parse_yaml_file(file) self.assertIsInstance(config, dict) + return config + + def validate(self, file: str) -> Config: + """Validate a config file. + :param file: file path to the config to validate. + """ + config = self.parse(file) + ConfigTest().validate(config) + return config + + def probe(self, file: str, *, + delay: float = PROBE_DELAY_DEFAULT, retries: int = PROBE_RETRIES_DEFAULT + ) -> Config: # pragma: no cover + """Probe a config file. + :param file: config to probe. + :param delay: delay to wait between each crawler probes. + :param retries: number of retries in case an error occurred. + """ + config = self.parse(file) + ConfigTest().probe(config, delay=delay, retries=retries) + return config + + +class ConfigTest(TestCase): + + def validate(self, config: Config) -> None: # pragma: no cover + """Validate a config. + :param config: file path to the config to validate. + """ imagecrawlers = list() # type: List[BaseImageCrawler] for crawler_config in config['crawlers']: imagecrawler = get_imagecrawler(crawler_config) @@ -29,17 +61,30 @@ def _probe_crawl(imagecrawler: BaseImageCrawler) -> None: # pragma: no cover except Exception as e: raise ImageCrawlerProbeCrawlError(imagecrawler) from e - def probe(self, file: str, delay: float = 0.01) -> None: # pragma: no cover - """Probe a config file. - :param file: file path to the config to probe. + @classmethod + def _probe_crawl_retry(cls, imagecrawler: BaseImageCrawler, + retries: int, delay: float) -> None: # pragma: no cover + try: + imagecrawler._crawl() + except Exception as e: + if retries <= 0: + raise ImageCrawlerProbeCrawlError(imagecrawler) from e + del e + sleep(delay) + cls._probe_crawl_retry(imagecrawler, retries - 1, delay) + + def probe(self, config: Config, *, + delay: float = PROBE_DELAY_DEFAULT, retries: int = PROBE_RETRIES_DEFAULT + ) -> None: # pragma: no cover + """Probe a config. + :param config: config to probe. :param delay: delay to wait between each crawler probes. + :param retries: number of retries in case an error occurred. """ - config = parse_yaml_file(file) - self.assertIsInstance(config, dict) for crawler_config in config['crawlers']: imagecrawler = get_imagecrawler(crawler_config) - self._probe_crawl(imagecrawler) - sleep(delay) # do not be too greedy + self._probe_crawl_retry(imagecrawler, retries, delay) + sleep(delay) class ImageCrawlerProbeCrawlError(Exception): diff --git a/setup.cfg b/setup.cfg index 89de6a53..e99cc645 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,7 +48,8 @@ install_requires = werkzeug >= 1.0 mako >= 1.1 setuptools >= 40.0 - argcomplete >= 1.11 + click >= 7.1 + # click_completion >= 0.5 # currently unused; see nichtparasoup.cli.completion for details [options.packages.find] exclude = @@ -80,4 +81,4 @@ testing= [options.entry_points] console_scripts = - nichtparasoup = nichtparasoup.cli.main:main + nichtparasoup = nichtparasoup.cli:main diff --git a/tests/test_10_nichtparasoup/test_cli/test_commands.py b/tests/test_10_nichtparasoup/test_cli/test_commands.py deleted file mode 100644 index b66afa0a..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_commands.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest - -from ddt import ddt, idata as ddt_idata # type: ignore - -from nichtparasoup.cli.commands import _COMMANDS, BaseCommand, create_command - - -@ddt -class CreateCommandTest(unittest.TestCase): - - @ddt_idata(_COMMANDS.keys()) # type: ignore - def test_create_command(self, command_name: str) -> None: - command = create_command(command_name) - self.assertIsInstance(command, BaseCommand, command_name) diff --git a/tests/test_10_nichtparasoup/test_cli/test_commands/test_completion.py b/tests/test_10_nichtparasoup/test_cli/test_commands/test_completion.py deleted file mode 100644 index 2feb40d6..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_commands/test_completion.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from nichtparasoup.cli.commands import create_command -from nichtparasoup.cli.commands.completion import CompletionCommand - - -class CompletionCommandTest(unittest.TestCase): - - def test_create_command(self) -> None: - command = create_command('completion') - self.assertIsInstance(command, CompletionCommand) - - @unittest.skip("TODO: write the test") - def test_(self) -> None: - raise NotImplementedError() diff --git a/tests/test_10_nichtparasoup/test_cli/test_commands/test_config.py b/tests/test_10_nichtparasoup/test_cli/test_commands/test_config.py deleted file mode 100644 index c16b6a20..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_commands/test_config.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from nichtparasoup.cli.commands import create_command -from nichtparasoup.cli.commands.config import ConfigCommand - - -class ConfigCommandTest(unittest.TestCase): - - def test_create_command(self) -> None: - command = create_command('config') - self.assertIsInstance(command, ConfigCommand) - - @unittest.skip("TODO: write the test") - def test_(self) -> None: - raise NotImplementedError() diff --git a/tests/test_10_nichtparasoup/test_cli/test_commands/test_imagecrawler.py b/tests/test_10_nichtparasoup/test_cli/test_commands/test_imagecrawler.py deleted file mode 100644 index 182171a5..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_commands/test_imagecrawler.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from nichtparasoup.cli.commands import create_command -from nichtparasoup.cli.commands.imagecrawler import ImagecrawlerCommand - - -class RunCommandTest(unittest.TestCase): - - def test_create_command(self) -> None: - command = create_command('imagecrawler') - self.assertIsInstance(command, ImagecrawlerCommand) - - @unittest.skip("TODO: write the test") - def test_(self) -> None: - raise NotImplementedError() diff --git a/tests/test_10_nichtparasoup/test_cli/test_commands/test_run.py b/tests/test_10_nichtparasoup/test_cli/test_commands/test_run.py deleted file mode 100644 index 8f1b8b91..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_commands/test_run.py +++ /dev/null @@ -1,15 +0,0 @@ -import unittest - -from nichtparasoup.cli.commands import create_command -from nichtparasoup.cli.commands.run import RunCommand - - -class RunCommandTest(unittest.TestCase): - - def test_create_command(self) -> None: - command = create_command('run') - self.assertIsInstance(command, RunCommand) - - @unittest.skip("TODO: write the test") - def test_(self) -> None: - raise NotImplementedError() diff --git a/tests/test_10_nichtparasoup/test_cli/test_main.py b/tests/test_10_nichtparasoup/test_cli/test_main.py deleted file mode 100644 index aac656c0..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_main.py +++ /dev/null @@ -1,14 +0,0 @@ -import unittest - -from nichtparasoup.cli.main import main as main_ - - -class CommandsRunTest(unittest.TestCase): - """Current implementation is just a preparation for later. - """ - - @unittest.skip("nothing to test, yet") - def test_nothing(self) -> None: - """Current implementation is a placeholder, so the imported main_ stays active. - """ - main_() diff --git a/tests/test_10_nichtparasoup/test_cli/test_parser.py b/tests/test_10_nichtparasoup/test_cli/test_parser.py deleted file mode 100644 index de1459f5..00000000 --- a/tests/test_10_nichtparasoup/test_cli/test_parser.py +++ /dev/null @@ -1,18 +0,0 @@ -import unittest - -from nichtparasoup.cli.parser import create_parser - - -class CommandsRunTest(unittest.TestCase): - """Current implementation is just a preparation for later. - """ - - def setUp(self) -> None: - self.parser = create_parser() - - def tearDown(self) -> None: - del self.parser - - @unittest.skip("nothing to test, yet") - def test_nothing(self) -> None: - raise NotImplementedError()