diff --git a/CHANGELOG.md b/CHANGELOG.md index 321608e7..3b43478d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,162 +1,37 @@ # Changelog -## [V0.0.24a18](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a18) (2023-10-25) +## [0.0.26a4](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/0.0.26a4) (2024-01-03) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a17...V0.0.24a18) - -**Fixed bugs:** - -- STT class init with wrong config [\#132](https://github.com/OpenVoiceOS/ovos-plugin-manager/issues/132) -- Fix STT and TTS configuration handling [\#187](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/187) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a17](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a17) (2023-10-25) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a16...V0.0.24a17) - -**Merged pull requests:** - -- Update test dependencies [\#190](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/190) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a16](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a16) (2023-10-24) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a15...V0.0.24a16) - -**Fixed bugs:** - -- remove log spam [\#188](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/188) ([JarbasAl](https://github.com/JarbasAl)) - -**Closed issues:** - -- support ovos-translate-server [\#182](https://github.com/OpenVoiceOS/ovos-plugin-manager/issues/182) - -## [V0.0.24a15](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a15) (2023-10-12) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a14...V0.0.24a15) - -**Fixed bugs:** - -- Language Module Factory Tests/Fixes [\#184](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/184) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a14](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a14) (2023-10-11) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a13...V0.0.24a14) - -**Fixed bugs:** - -- No module named 'mycroft\_bus\_client' [\#146](https://github.com/OpenVoiceOS/ovos-plugin-manager/issues/146) -- Circular import while importing `StreamHandler` [\#130](https://github.com/OpenVoiceOS/ovos-plugin-manager/issues/130) - -**Merged pull requests:** - -- Add license notice and link to plugins from neon\_solvers package [\#185](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/185) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a13](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a13) (2023-10-10) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a12...V0.0.24a13) - -**Implemented enhancements:** - -- feat/translate\_plug\_as\_arg [\#183](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/183) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a12](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a12) (2023-10-08) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a11...V0.0.24a12) - -**Fixed bugs:** - -- fix/playback\_thread startup [\#181](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/181) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a11](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a11) (2023-10-08) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a10...V0.0.24a11) - -**Fixed bugs:** - -- fix/playback\_thread startup [\#180](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/180) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a10](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a10) (2023-10-07) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a9...V0.0.24a10) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.26a3...0.0.26a4) **Implemented enhancements:** -- feat/ovos\_dialog\_tts\_transformers [\#179](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/179) ([JarbasAl](https://github.com/JarbasAl)) +- feat/sentence\_tokenize\_before\_TTS [\#206](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/206) ([JarbasAl](https://github.com/JarbasAl)) -## [V0.0.24a9](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a9) (2023-09-22) +## [V0.0.26a3](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.26a3) (2023-12-30) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a8...V0.0.24a9) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.26a2...V0.0.26a3) **Fixed bugs:** -- fix/deprecation\_warnings [\#178](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/178) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a8](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a8) (2023-09-17) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a6...V0.0.24a8) - -**Merged pull requests:** - -- Don't swallow plugin load errors [\#176](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/176) ([strugee](https://github.com/strugee)) - -## [V0.0.24a6](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a6) (2023-09-08) +- fix phal plugins in ovos-utils \>= 0.1.0a2 [\#205](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/205) ([JarbasAl](https://github.com/JarbasAl)) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a5...V0.0.24a6) +## [V0.0.26a2](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.26a2) (2023-12-30) -**Implemented enhancements:** - -- feat/extract\_speech [\#139](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/139) ([JarbasAl](https://github.com/JarbasAl)) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.26a1...V0.0.26a2) **Fixed bugs:** -- fix/audio\_config [\#174](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/174) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a5](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a5) (2023-07-07) +- fix gui extensions in ovos-utils \>= 0.1.0a2 [\#204](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/204) ([JarbasAl](https://github.com/JarbasAl)) -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a4...V0.0.24a5) +## [V0.0.26a1](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.26a1) (2023-12-29) -**Merged pull requests:** - -- Updates logging around language plugin errors [\#171](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/171) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a4](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a4) (2023-07-07) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a3...V0.0.24a4) +[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.25...V0.0.26a1) **Merged pull requests:** -- Replace `sleep` with `Event.wait` [\#170](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/170) ([NeonDaniel](https://github.com/NeonDaniel)) - -## [V0.0.24a3](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a3) (2023-07-04) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a2...V0.0.24a3) - -**Fixed bugs:** - -- feat/optional\_g2p [\#169](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/169) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a2](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a2) (2023-07-04) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.24a1...V0.0.24a2) - -**Fixed bugs:** - -- fix/on\_mouth\_viseme\_list [\#168](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/168) ([JarbasAl](https://github.com/JarbasAl)) - -## [V0.0.24a1](https://github.com/OpenVoiceOS/ovos-plugin-manager/tree/V0.0.24a1) (2023-06-21) - -[Full Changelog](https://github.com/OpenVoiceOS/ovos-plugin-manager/compare/V0.0.23...V0.0.24a1) - -**Implemented enhancements:** - -- feat/tts\_session\_context [\#167](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/167) ([JarbasAl](https://github.com/JarbasAl)) - -**Fixed bugs:** - -- Automation fixes [\#164](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/164) ([NeonDaniel](https://github.com/NeonDaniel)) - -**Closed issues:** - -- ImportError: Wake Word marvin with module ovos-ww-plugin-precise-lite failed to load [\#166](https://github.com/OpenVoiceOS/ovos-plugin-manager/issues/166) +- packaging/update imports [\#203](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/203) ([JarbasAl](https://github.com/JarbasAl)) +- Update requirements.txt [\#201](https://github.com/OpenVoiceOS/ovos-plugin-manager/pull/201) ([JarbasAl](https://github.com/JarbasAl)) diff --git a/ovos_plugin_manager/skills.py b/ovos_plugin_manager/skills.py index ec868a3b..a38093dc 100644 --- a/ovos_plugin_manager/skills.py +++ b/ovos_plugin_manager/skills.py @@ -1,13 +1,152 @@ +from os import makedirs, listdir +from os.path import join, isdir, dirname, expanduser, isfile + +import importlib.util +from ovos_config.config import read_mycroft_config +from ovos_config.locations import get_xdg_data_save_path, get_xdg_data_dirs +from typing import List, Optional + from ovos_plugin_manager.utils import PluginTypes -from ovos_utils.log import LOG +from ovos_plugin_manager.utils import find_plugins +from ovos_utils.log import LOG, log_deprecation + + +def get_installed_skill_ids(conf: Optional[dict] = None) -> List[str]: + """ + Gets a list of `skill_id`s for all installed skills + Args: + conf: Configuration, else loads from ovos_config.config.Configuration + Returns: + list of `skill_id` strings for all installed skills + """ + _, skill_ids = get_plugin_skills() + for d in get_skill_directories(conf): + for skill_dir in listdir(d): + if isdir(join(d, skill_dir)) and isfile(join(d, skill_dir, + "__init__.py")): + if skill_dir in skill_ids: + LOG.info(f"{skill_dir} installed as plugin and local dir") + continue + skill_ids.append(skill_dir) + return skill_ids + + +def get_skill_directories(conf: Optional[dict] = None) -> List[str]: + """ returns list of skill directories ordered by expected loading order + This corresponds to: + - XDG_DATA_DIRS + - default directory (see get_default_skills_directory method for details) + - user defined extra directories + Each directory contains individual skill folders to be loaded + If a skill exists in more than one directory (same folder name) previous instances will be ignored + ie. directories at the end of the list have priority over earlier directories + NOTE: empty folders are interpreted as disabled skills + new directories can be defined in mycroft.conf by specifying a full path + each extra directory is expected to contain individual skill folders to be loaded + the xdg folder name can also be changed, it defaults to "skills" + eg. ~/.local/share/mycroft/FOLDER_NAME + { + "skills": { + "directory": "skills", + "extra_directories": ["path/to/extra/dir/to/scan/for/skills"] + } + } + Args: + conf: Configuration, else loads from ovos_config.config.Configuration + Returns: + list of fully-qualified directories containing non-plugin skills + """ + # the contents of each skills directory must be individual skill folders + # we are still dependent on the mycroft-core structure of skill_id/__init__.py + + conf = conf or read_mycroft_config() + folder = conf["skills"].get("directory") or "skills" + # load all valid XDG paths + # NOTE: skills are actually code, but treated as user data! + # they should be considered applets rather than full applications + skill_locations = list(reversed( + [join(p, folder) for p in get_xdg_data_dirs() if + isdir(join(p, folder))] + )) + + # load the default skills folder + # only meaningful if xdg support is disabled + default = get_default_skills_directory(conf) + if default not in skill_locations: + skill_locations.append(default) + # load additional explicitly configured directories + conf = conf.get("skills") or {} + # extra_directories is a list of directories containing skill subdirectories + # NOT a list of individual skill folders + # preserve order while removing any duplicate entries + extra_dirs = (expanduser(d) for d in conf.get("extra_directories") or []) + for d in extra_dirs: + if isdir(d) and d not in skill_locations: + skill_locations.append(d) + return skill_locations -def find_plugins(*args, **kwargs): - # TODO: Deprecate in 0.1.0 - LOG.warning("This reference is deprecated. " - "Import from ovos_plugin_manager.utils directly") - from ovos_plugin_manager.utils import find_plugins - return find_plugins(*args, **kwargs) + +def get_default_skills_directory(conf: Optional[dict] = None) -> str: + """ return default directory to scan for skills + This is only meaningful if xdg is disabled in ovos.conf + If xdg is enabled then data_dir is always XDG_DATA_DIR + If xdg is disabled then data_dir by default corresponds to /opt/mycroft + users can define the data directory in mycroft.conf + the skills folder name (relative to data_dir) can also be defined there + NOTE: folder name also impacts all XDG skill directories! + { + "data_dir": "/opt/mycroft", + "skills": { + "directory": "skills" + } + } + Args: + conf: Configuration, else loads from ovos_config.config.Configuration + Returns: + Absolute path to default skills directory + """ + conf = conf or read_mycroft_config() + path_override = conf["skills"].get("directory_override") + folder = conf["skills"].get("directory") or "skills" + + # if .conf wants to use a specific path, use it! + if path_override: + log_deprecation("'directory_override' is deprecated!" + "add the new path to 'extra_directories' instead", + "0.1.0") + skills_folder = expanduser(path_override) + elif conf["skills"].get("extra_directories") and \ + len(conf["skills"].get("extra_directories")) > 0: + skills_folder = expanduser(conf["skills"]["extra_directories"][0]) + else: + skills_folder = join(get_xdg_data_save_path(), folder) + # create folder if needed + try: + makedirs(skills_folder, exist_ok=True) + except PermissionError: # old style /opt/mycroft/skills not available + skills_folder = join(get_xdg_data_save_path(), folder) + makedirs(skills_folder, exist_ok=True) + + return skills_folder + + +def get_plugin_skills() -> (list, list): + """ + Get the package directories for any pip installed skill plugins + Returns: + lists of skill directories and plugin skill IDs + """ + + skill_dirs = list() + plugins = find_skill_plugins() + skill_ids = list(plugins.keys()) + for skill_class in plugins.values(): + skill_dir = dirname(importlib.util.find_spec( + skill_class.__module__).origin) + skill_dirs.append(skill_dir) + LOG.info(f"Located plugin skills: {skill_ids}") + return skill_dirs, skill_ids def find_skill_plugins() -> dict: @@ -15,7 +154,6 @@ def find_skill_plugins() -> dict: Find all installed plugins @return: dict plugin names to entrypoints """ - from ovos_plugin_manager.utils import find_plugins return find_plugins(PluginTypes.SKILL) diff --git a/ovos_plugin_manager/templates/gui.py b/ovos_plugin_manager/templates/gui.py index fad98e19..e3c020cc 100644 --- a/ovos_plugin_manager/templates/gui.py +++ b/ovos_plugin_manager/templates/gui.py @@ -1,6 +1,6 @@ from ovos_bus_client import Message from ovos_bus_client import MessageBusClient -from ovos_utils.gui import GUIInterface +from ovos_bus_client.apis.gui import GUIInterface from ovos_utils.log import LOG from ovos_config import Configuration diff --git a/ovos_plugin_manager/templates/phal.py b/ovos_plugin_manager/templates/phal.py index aed65f3a..8a8fb471 100644 --- a/ovos_plugin_manager/templates/phal.py +++ b/ovos_plugin_manager/templates/phal.py @@ -5,7 +5,7 @@ from ovos_utils import camel_case_split from ovos_utils import classproperty from ovos_utils.log import LOG -from ovos_utils.messagebus import get_mycroft_bus +from ovos_bus_client.util import get_mycroft_bus from ovos_utils.process_utils import RuntimeRequirements from ovos_plugin_manager.utils.config import get_plugin_config diff --git a/ovos_plugin_manager/templates/transformers.py b/ovos_plugin_manager/templates/transformers.py index 32a8f65f..d2e48005 100644 --- a/ovos_plugin_manager/templates/transformers.py +++ b/ovos_plugin_manager/templates/transformers.py @@ -1,7 +1,7 @@ from typing import List, Tuple from ovos_config.config import Configuration -from ovos_utils.messagebus import get_mycroft_bus +from ovos_bus_client.util import get_mycroft_bus from ovos_plugin_manager.utils import ReadWriteStream diff --git a/ovos_plugin_manager/templates/tts.py b/ovos_plugin_manager/templates/tts.py index 8f6edcb2..d90b99dd 100644 --- a/ovos_plugin_manager/templates/tts.py +++ b/ovos_plugin_manager/templates/tts.py @@ -25,6 +25,7 @@ import random import re import subprocess +import quebra_frases from os.path import isfile, join from pathlib import Path from queue import Queue @@ -42,7 +43,7 @@ from ovos_utils.file_utils import get_cache_directory from ovos_utils.lang.visimes import VISIMES from ovos_utils.log import LOG -from ovos_utils.messagebus import FakeBus as BUS +from ovos_utils.fakebus import FakeBus from ovos_utils.metrics import Stopwatch from ovos_utils.process_utils import RuntimeRequirements @@ -153,7 +154,7 @@ def __init__(self, lang="en-us", config=None, validator=None, self.stopwatch = Stopwatch() self.tts_name = self.__class__.__name__ - self.bus = BUS() # initialized in "init" step + self.bus = FakeBus() # initialized in "init" step self.lang = lang or self.config.get("lang") or 'en-us' self.validator = validator or TTSValidator(self) self.phonetic_spelling = phonetic_spelling @@ -161,6 +162,8 @@ def __init__(self, lang="en-us", config=None, validator=None, self.ssml_tags = ssml_tags or [] self.log_timestamps = self.config.get("log_timestamps", False) + self.enable_cache = self.config.get("enable_cache", True) + self.voice = self.config.get("voice") or "default" # TODO can self.filename be deprecated ? is it used anywhere at all? cache_dir = get_cache_directory(self.tts_name) @@ -312,7 +315,7 @@ def init(self, bus=None, playback=None): Arguments: bus: OpenVoiceOS messagebus connection """ - self.bus = bus or BUS() + self.bus = bus or FakeBus() if playback is None: LOG.warning("PlaybackThread should be inited by ovos-audio, initing via plugin has been deprecated, " "please pass playback=PlaybackThread() to TTS.init") @@ -454,7 +457,8 @@ def validate_ssml(self, utterance): return utterance.replace(" ", " ") def _preprocess_sentence(self, sentence): - """Default preprocessing is no preprocessing. + """Default preprocessing is a sentence_tokenizer, + ie. splits the utterance into sub-sentences using quebra_frases This method can be overridden to create chunks suitable to the TTS engine in question. @@ -465,6 +469,8 @@ def _preprocess_sentence(self, sentence): Returns: list: list of sentence parts """ + if self.config.get("sentence_tokenize"): # TODO default to True on next major release + return quebra_frases.sentence_tokenize(sentence) return [sentence] def execute(self, sentence, ident=None, listen=False, **kwargs): @@ -550,7 +556,7 @@ def synth(self, sentence, **kwargs): cache = self.get_cache(voice, lang) # cache per tts_id (lang/voice combo) # load from cache - if sentence_hash in cache: + if self.enable_cache and sentence_hash in cache: audio, phonemes = self.get_from_cache(sentence, **kwargs) self.add_metric({"metric_type": "tts.synth.finished", "cache": True}) return audio, phonemes @@ -571,8 +577,9 @@ def synth(self, sentence, **kwargs): self.add_metric({"metric_type": "tts.synth.finished"}) # cache sentence + phonemes - self._cache_sentence(sentence, audio, phonemes, sentence_hash, - voice=voice, lang=lang) + if self.enable_cache: + self._cache_sentence(sentence, audio, phonemes, sentence_hash, + voice=voice, lang=lang) return audio, phonemes def _cache_phonemes(self, sentence, phonemes=None, sentence_hash=None): diff --git a/ovos_plugin_manager/version.py b/ovos_plugin_manager/version.py index aacf6ad5..2d154dda 100644 --- a/ovos_plugin_manager/version.py +++ b/ovos_plugin_manager/version.py @@ -2,6 +2,6 @@ # START_VERSION_BLOCK VERSION_MAJOR = 0 VERSION_MINOR = 0 -VERSION_BUILD = 24 -VERSION_ALPHA = 0 +VERSION_BUILD = 26 +VERSION_ALPHA = 4 # END_VERSION_BLOCK diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6e39a978..b10d2770 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,10 +1,10 @@ -ovos-utils < 0.1.0, >=0.0.32 -ovos-bus-client < 0.1.0, >=0.0.3 -ovos-config < 0.1.0,>=0.0.8 +ovos-utils < 0.2.0, >=0.0.37 +ovos-bus-client < 0.2.0, >=0.0.8 +ovos-config < 0.2.0, >=0.0.12 combo_lock~=0.2 requests~=2.26 quebra_frases langcodes~=3.3.0 # see https://github.com/pypa/setuptools/issues/1471 -importlib_metadata \ No newline at end of file +importlib_metadata