From c991273e0d707c58906a6abe2c5128b206e7db26 Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:43:10 -0500 Subject: [PATCH 1/6] Initial napari specific classes for package installer related logic --- .../_tests/test_installer_process.py | 48 +++++----- .../base_qt_package_installer.py | 10 +++ .../base_qt_plugin_dialog.py | 3 +- napari_plugin_manager/qt_package_installer.py | 89 +++++++++++++------ napari_plugin_manager/qt_plugin_dialog.py | 2 + 5 files changed, 101 insertions(+), 51 deletions(-) create mode 100644 napari_plugin_manager/base_qt_package_installer.py diff --git a/napari_plugin_manager/_tests/test_installer_process.py b/napari_plugin_manager/_tests/test_installer_process.py index 87219f0..0c1712a 100644 --- a/napari_plugin_manager/_tests/test_installer_process.py +++ b/napari_plugin_manager/_tests/test_installer_process.py @@ -9,11 +9,11 @@ from napari_plugin_manager.qt_package_installer import ( AbstractInstallerTool, - CondaInstallerTool, InstallerActions, - InstallerQueue, InstallerTools, - PipInstallerTool, + NapariCondaInstallerTool, + NapariInstallerQueue, + NapariPipInstallerTool, ) if TYPE_CHECKING: @@ -71,9 +71,11 @@ def test_not_implemented_methods(): def test_pip_installer_tasks(qtbot, tmp_virtualenv: 'Session', monkeypatch): - installer = InstallerQueue() + installer = NapariInstallerQueue() monkeypatch.setattr( - PipInstallerTool, "executable", lambda *a: tmp_virtualenv.creator.exe + NapariPipInstallerTool, + "executable", + lambda *a: tmp_virtualenv.creator.exe, ) with qtbot.waitSignal(installer.allFinished, timeout=20000): installer.install( @@ -139,9 +141,11 @@ def test_pip_installer_tasks(qtbot, tmp_virtualenv: 'Session', monkeypatch): def test_installer_failures(qtbot, tmp_virtualenv: 'Session', monkeypatch): - installer = InstallerQueue() + installer = NapariInstallerQueue() monkeypatch.setattr( - PipInstallerTool, "executable", lambda *a: tmp_virtualenv.creator.exe + NapariPipInstallerTool, + "executable", + lambda *a: tmp_virtualenv.creator.exe, ) # CHECK 1) Errors should trigger finished and allFinished too @@ -181,7 +185,7 @@ def test_installer_failures(qtbot, tmp_virtualenv: 'Session', monkeypatch): def test_cancel_incorrect_job_id(qtbot, tmp_virtualenv: 'Session'): - installer = InstallerQueue() + installer = NapariInstallerQueue() with qtbot.waitSignal(installer.allFinished, timeout=20000): job_id = installer.install( tool=InstallerTools.PIP, @@ -192,13 +196,13 @@ def test_cancel_incorrect_job_id(qtbot, tmp_virtualenv: 'Session'): @pytest.mark.skipif( - not CondaInstallerTool.available(), reason="Conda is not available." + not NapariCondaInstallerTool.available(), reason="Conda is not available." ) def test_conda_installer(qtbot, tmp_conda_env: Path): conda_meta = tmp_conda_env / "conda-meta" glob_pat = "typing-extensions-*.json" glob_pat_2 = "pyzenhub-*.json" - installer = InstallerQueue() + installer = NapariInstallerQueue() with qtbot.waitSignal(installer.allFinished, timeout=600_000): installer.install( @@ -277,9 +281,11 @@ def test_conda_installer(qtbot, tmp_conda_env: Path): def test_installer_error(qtbot, tmp_virtualenv: 'Session', monkeypatch): - installer = InstallerQueue() + installer = NapariInstallerQueue() monkeypatch.setattr( - PipInstallerTool, "executable", lambda *a: 'not-a-real-executable' + NapariPipInstallerTool, + "executable", + lambda *a: 'not-a-real-executable', ) with qtbot.waitSignal(installer.allFinished, timeout=600_000): installer.install( @@ -289,10 +295,10 @@ def test_installer_error(qtbot, tmp_virtualenv: 'Session', monkeypatch): @pytest.mark.skipif( - not CondaInstallerTool.available(), reason="Conda is not available." + not NapariCondaInstallerTool.available(), reason="Conda is not available." ) def test_conda_installer_wait_for_finished(qtbot, tmp_conda_env: Path): - installer = InstallerQueue() + installer = NapariInstallerQueue() with qtbot.waitSignal(installer.allFinished, timeout=600_000): installer.install( @@ -309,8 +315,8 @@ def test_conda_installer_wait_for_finished(qtbot, tmp_conda_env: Path): def test_constraints_are_in_sync(): - conda_constraints = sorted(CondaInstallerTool.constraints()) - pip_constraints = sorted(PipInstallerTool.constraints()) + conda_constraints = sorted(NapariCondaInstallerTool.constraints()) + pip_constraints = sorted(NapariPipInstallerTool.constraints()) assert len(conda_constraints) == len(pip_constraints) @@ -324,15 +330,15 @@ def test_constraints_are_in_sync(): def test_executables(): - assert CondaInstallerTool.executable() - assert PipInstallerTool.executable() + assert NapariCondaInstallerTool.executable() + assert NapariPipInstallerTool.executable() def test_available(): - assert str(CondaInstallerTool.available()) - assert PipInstallerTool.available() + assert str(NapariCondaInstallerTool.available()) + assert NapariPipInstallerTool.available() def test_unrecognized_tool(): with pytest.raises(ValueError): - InstallerQueue().install(tool='shrug', pkgs=[]) + NapariInstallerQueue().install(tool='shrug', pkgs=[]) diff --git a/napari_plugin_manager/base_qt_package_installer.py b/napari_plugin_manager/base_qt_package_installer.py new file mode 100644 index 0000000..0b492c1 --- /dev/null +++ b/napari_plugin_manager/base_qt_package_installer.py @@ -0,0 +1,10 @@ +""" +A tool-agnostic installation logic for the plugin manager. + +The main object is `InstallerQueue`, a `QProcess` subclass +with the notion of a job queue. The queued jobs are represented +by a `deque` of `*InstallerTool` dataclasses that contain the +executable path, arguments and environment modifications. +Available actions for each tool are `install`, `uninstall` +and `cancel`. +""" diff --git a/napari_plugin_manager/base_qt_plugin_dialog.py b/napari_plugin_manager/base_qt_plugin_dialog.py index 7b9b09c..974aa7f 100644 --- a/napari_plugin_manager/base_qt_plugin_dialog.py +++ b/napari_plugin_manager/base_qt_plugin_dialog.py @@ -1025,6 +1025,7 @@ class BaseQtPluginDialog(QDialog): PACKAGE_METADATA_CLASS = BasePackageMetadata PROJECT_INFO_VERSION_CLASS = BaseProjectInfoVersions PLUGIN_LIST_CLASS = BaseQPluginList + INSTALLER_QUEUE_CLASS = InstallerQueue BASE_PACKAGE_NAME = '' MAX_PLUGIN_SEARCH_ITEMS = 35 @@ -1065,7 +1066,7 @@ def __init__(self, parent=None, prefix=None) -> None: self._add_items_timer.setInterval(61) # ms self._add_items_timer.timeout.connect(self._add_items) - self.installer = InstallerQueue(parent=self, prefix=prefix) + self.installer = self.INSTALLER_QUEUE_CLASS(parent=self, prefix=prefix) self.setWindowTitle(self._trans('Plugin Manager')) self._setup_ui() self.installer.set_output_widget(self.stdout_text) diff --git a/napari_plugin_manager/qt_package_installer.py b/napari_plugin_manager/qt_package_installer.py index 14a5d59..343ab3f 100644 --- a/napari_plugin_manager/qt_package_installer.py +++ b/napari_plugin_manager/qt_package_installer.py @@ -96,7 +96,7 @@ def constraints() -> Sequence[str]: """ Version constraints to limit unwanted changes in installation. """ - return [f"napari=={_napari_version}", "numpy<2"] + raise NotImplementedError @classmethod def available(cls) -> bool: @@ -107,10 +107,6 @@ def available(cls) -> bool: class PipInstallerTool(AbstractInstallerTool): - @classmethod - def executable(cls): - return str(_get_python_exe()) - @classmethod def available(cls): return call([cls.executable(), "-m", "pip", "--version"]) == 0 @@ -151,12 +147,7 @@ def environment( @classmethod @lru_cache(maxsize=0) def _constraints_file(cls) -> str: - with NamedTemporaryFile( - "w", suffix="-napari-constraints.txt", delete=False - ) as f: - f.write("\n".join(cls.constraints())) - atexit.register(os.unlink, f.name) - return f.name + raise NotImplementedError class CondaInstallerTool(AbstractInstallerTool): @@ -214,21 +205,6 @@ def environment( env.remove("PYTHONEXECUTABLE") return env - @staticmethod - def constraints() -> Sequence[str]: - # FIXME - # dev or rc versions might not be available in public channels - # but only installed locally - if we try to pin those, mamba - # will fail to pin it because there's no record of that version - # in the remote index, only locally; to work around this bug - # we will have to pin to e.g. 0.4.* instead of 0.4.17.* for now - version_lower = _napari_version.lower() - is_dev = "rc" in version_lower or "dev" in version_lower - pin_level = 2 if is_dev else 3 - version = ".".join([str(x) for x in _napari_version_tuple[:pin_level]]) - - return [f"napari={version}", "numpy<2.0a0"] - def _add_constraints_to_env( self, env: QProcessEnvironment ) -> QProcessEnvironment: @@ -263,6 +239,13 @@ class InstallerQueue(QObject): # emitted when each job starts started = Signal() + # classes to manage pip and conda installations + PIP_INSTALLER_TOOL_CLASS = PipInstallerTool + CONDA_INSTALLER_TOOL_CLASS = CondaInstallerTool + # This should be set to the name of package that handles plugins + # e.g `napari` for napari + BASE_PACKAGE_NAME = '' + def __init__( self, parent: Optional[QObject] = None, prefix: Optional[str] = None ) -> None: @@ -502,9 +485,9 @@ def _log(self, msg: str): def _get_tool(self, tool: InstallerTools): if tool == InstallerTools.PIP: - return PipInstallerTool + return self.PIP_INSTALLER_TOOL_CLASS if tool == InstallerTools.CONDA: - return CondaInstallerTool + return self.CONDA_INSTALLER_TOOL_CLASS raise ValueError(f"InstallerTool {tool} not recognized!") def _build_queue_item( @@ -590,7 +573,9 @@ def _on_process_finished( plugin_manager.unregister(pkg) else: log.warning( - 'Cannot unregister %s, not a known napari plugin.', pkg + 'Cannot unregister %s, not a known %s plugin.', + pkg, + self.BASE_PACKAGE_NAME, ) self._on_process_done(exit_code=exit_code, exit_status=exit_status) @@ -647,6 +632,52 @@ def _on_stderr_ready(self): self._log(text) +class NapariPipInstallerTool(PipInstallerTool): + @classmethod + def executable(cls): + return str(_get_python_exe()) + + @staticmethod + def constraints() -> Sequence[str]: + """ + Version constraints to limit unwanted changes in installation. + """ + return [f"napari=={_napari_version}", "numpy<2"] + + @classmethod + @lru_cache(maxsize=0) + def _constraints_file(cls) -> str: + with NamedTemporaryFile( + "w", suffix="-napari-constraints.txt", delete=False + ) as f: + f.write("\n".join(cls.constraints())) + atexit.register(os.unlink, f.name) + return f.name + + +class NapariCondaInstallerTool(CondaInstallerTool): + @staticmethod + def constraints() -> Sequence[str]: + # FIXME + # dev or rc versions might not be available in public channels + # but only installed locally - if we try to pin those, mamba + # will fail to pin it because there's no record of that version + # in the remote index, only locally; to work around this bug + # we will have to pin to e.g. 0.4.* instead of 0.4.17.* for now + version_lower = _napari_version.lower() + is_dev = "rc" in version_lower or "dev" in version_lower + pin_level = 2 if is_dev else 3 + version = ".".join([str(x) for x in _napari_version_tuple[:pin_level]]) + + return [f"napari={version}", "numpy<2.0a0"] + + +class NapariInstallerQueue(InstallerQueue): + PIP_INSTALLER_TOOL_CLASS = NapariPipInstallerTool + CONDA_INSTALLER_TOOL_CLASS = NapariCondaInstallerTool + BASE_PACKAGE_NAME = 'napari' + + def _get_python_exe(): # Note: is_bundled_app() returns False even if using a Briefcase bundle... # Workaround: see if sys.executable is set to something something napari on Mac diff --git a/napari_plugin_manager/qt_plugin_dialog.py b/napari_plugin_manager/qt_plugin_dialog.py index 9b5dd79..691f07e 100644 --- a/napari_plugin_manager/qt_plugin_dialog.py +++ b/napari_plugin_manager/qt_plugin_dialog.py @@ -34,6 +34,7 @@ from napari_plugin_manager.qt_package_installer import ( InstallerActions, InstallerTools, + NapariInstallerQueue, ) from napari_plugin_manager.utils import is_conda_package @@ -194,6 +195,7 @@ class QtPluginDialog(BaseQtPluginDialog): PACKAGE_METADATA_CLASS = npe2.PackageMetadata PROJECT_INFO_VERSION_CLASS = ProjectInfoVersions PLUGIN_LIST_CLASS = QPluginList + INSTALLER_QUEUE_CLASS = NapariInstallerQueue BASE_PACKAGE_NAME = 'napari' def _setup_theme_update(self): From 5dbd74b35a0b4054eff4d402cfa9132bd23ed150 Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Fri, 13 Dec 2024 11:00:58 -0500 Subject: [PATCH 2/6] Move base package installer classes definitions to their own module --- .../_tests/test_installer_process.py | 4 +- .../_tests/test_qt_plugin_dialog.py | 2 +- .../base_qt_package_installer.py | 619 +++++++++++++++++ .../base_qt_plugin_dialog.py | 2 +- napari_plugin_manager/qt_package_installer.py | 649 +----------------- napari_plugin_manager/qt_plugin_dialog.py | 10 +- 6 files changed, 650 insertions(+), 636 deletions(-) diff --git a/napari_plugin_manager/_tests/test_installer_process.py b/napari_plugin_manager/_tests/test_installer_process.py index 0c1712a..0f22a8f 100644 --- a/napari_plugin_manager/_tests/test_installer_process.py +++ b/napari_plugin_manager/_tests/test_installer_process.py @@ -7,10 +7,12 @@ import pytest from qtpy.QtCore import QProcessEnvironment -from napari_plugin_manager.qt_package_installer import ( +from napari_plugin_manager.base_qt_package_installer import ( AbstractInstallerTool, InstallerActions, InstallerTools, +) +from napari_plugin_manager.qt_package_installer import ( NapariCondaInstallerTool, NapariInstallerQueue, NapariPipInstallerTool, diff --git a/napari_plugin_manager/_tests/test_qt_plugin_dialog.py b/napari_plugin_manager/_tests/test_qt_plugin_dialog.py index 3c1a886..fc65727 100644 --- a/napari_plugin_manager/_tests/test_qt_plugin_dialog.py +++ b/napari_plugin_manager/_tests/test_qt_plugin_dialog.py @@ -22,7 +22,7 @@ ) from napari_plugin_manager import qt_plugin_dialog -from napari_plugin_manager.qt_package_installer import InstallerActions +from napari_plugin_manager.base_qt_package_installer import InstallerActions N_MOCKED_PLUGINS = 2 diff --git a/napari_plugin_manager/base_qt_package_installer.py b/napari_plugin_manager/base_qt_package_installer.py index 0b492c1..f9e7999 100644 --- a/napari_plugin_manager/base_qt_package_installer.py +++ b/napari_plugin_manager/base_qt_package_installer.py @@ -8,3 +8,622 @@ Available actions for each tool are `install`, `uninstall` and `cancel`. """ + +import contextlib +import os +import sys +from collections import deque +from dataclasses import dataclass +from enum import auto +from functools import lru_cache +from logging import getLogger +from pathlib import Path +from subprocess import call +from tempfile import gettempdir +from typing import Deque, Optional, Sequence, Tuple, TypedDict + +from napari.plugins import plugin_manager +from napari.plugins.npe2api import _user_agent +from napari.utils.misc import StringEnum +from napari.utils.translations import trans +from npe2 import PluginManager +from qtpy.QtCore import QObject, QProcess, QProcessEnvironment, Signal +from qtpy.QtWidgets import QTextEdit + +JobId = int +log = getLogger(__name__) + + +class InstallerActions(StringEnum): + "Available actions for the plugin manager" + INSTALL = auto() + UNINSTALL = auto() + CANCEL = auto() + CANCEL_ALL = auto() + UPGRADE = auto() + + +class ProcessFinishedData(TypedDict): + exit_code: int + exit_status: int + action: InstallerActions + pkgs: Tuple[str, ...] + + +class InstallerTools(StringEnum): + "Available tools for InstallerQueue jobs" + CONDA = auto() + PIP = auto() + + +@dataclass(frozen=True) +class AbstractInstallerTool: + action: InstallerActions + pkgs: Tuple[str, ...] + origins: Tuple[str, ...] = () + prefix: Optional[str] = None + process: QProcess = None + + @property + def ident(self): + return hash( + (self.action, *self.pkgs, *self.origins, self.prefix, self.process) + ) + + # abstract method + @classmethod + def executable(cls): + "Path to the executable that will run the task" + raise NotImplementedError + + # abstract method + def arguments(self): + "Arguments supplied to the executable" + raise NotImplementedError + + # abstract method + def environment( + self, env: QProcessEnvironment = None + ) -> QProcessEnvironment: + "Changes needed in the environment variables." + raise NotImplementedError + + @staticmethod + def constraints() -> Sequence[str]: + """ + Version constraints to limit unwanted changes in installation. + """ + raise NotImplementedError + + @classmethod + def available(cls) -> bool: + """ + Check if the tool is available by performing a little test + """ + raise NotImplementedError + + +class PipInstallerTool(AbstractInstallerTool): + @classmethod + def available(cls): + return call([cls.executable(), "-m", "pip", "--version"]) == 0 + + def arguments(self) -> Tuple[str, ...]: + args = ['-m', 'pip'] + if self.action == InstallerActions.INSTALL: + args += ['install', '-c', self._constraints_file()] + for origin in self.origins: + args += ['--extra-index-url', origin] + elif self.action == InstallerActions.UPGRADE: + args += [ + 'install', + '--upgrade', + '-c', + self._constraints_file(), + ] + for origin in self.origins: + args += ['--extra-index-url', origin] + elif self.action == InstallerActions.UNINSTALL: + args += ['uninstall', '-y'] + else: + raise ValueError(f"Action '{self.action}' not supported!") + if 10 <= log.getEffectiveLevel() < 30: # DEBUG level + args.append('-vvv') + if self.prefix is not None: + args.extend(['--prefix', str(self.prefix)]) + return (*args, *self.pkgs) + + def environment( + self, env: QProcessEnvironment = None + ) -> QProcessEnvironment: + if env is None: + env = QProcessEnvironment.systemEnvironment() + env.insert("PIP_USER_AGENT_USER_DATA", _user_agent()) + return env + + @classmethod + @lru_cache(maxsize=0) + def _constraints_file(cls) -> str: + raise NotImplementedError + + +class CondaInstallerTool(AbstractInstallerTool): + @classmethod + def executable(cls): + bat = ".bat" if os.name == "nt" else "" + for path in ( + Path(os.environ.get('MAMBA_EXE', '')), + Path(os.environ.get('CONDA_EXE', '')), + # $CONDA is usually only available on GitHub Actions + Path(os.environ.get('CONDA', '')) / 'condabin' / f'conda{bat}', + ): + if path.is_file(): + return str(path) + return f'conda{bat}' # cross our fingers 'conda' is in PATH + + @classmethod + def available(cls): + executable = cls.executable() + try: + return call([executable, "--version"]) == 0 + except FileNotFoundError: # pragma: no cover + return False + + def arguments(self) -> Tuple[str, ...]: + prefix = self.prefix or self._default_prefix() + if self.action == InstallerActions.UPGRADE: + args = ['update', '-y', '--prefix', prefix] + else: + args = [self.action.value, '-y', '--prefix', prefix] + args.append('--override-channels') + for channel in (*self.origins, *self._default_channels()): + args.extend(["-c", channel]) + return (*args, *self.pkgs) + + def environment( + self, env: QProcessEnvironment = None + ) -> QProcessEnvironment: + if env is None: + env = QProcessEnvironment.systemEnvironment() + self._add_constraints_to_env(env) + if 10 <= log.getEffectiveLevel() < 30: # DEBUG level + env.insert('CONDA_VERBOSITY', '3') + if os.name == "nt": + if not env.contains("TEMP"): + temp = gettempdir() + env.insert("TMP", temp) + env.insert("TEMP", temp) + if not env.contains("USERPROFILE"): + env.insert("HOME", os.path.expanduser("~")) + env.insert("USERPROFILE", os.path.expanduser("~")) + if sys.platform == 'darwin' and env.contains('PYTHONEXECUTABLE'): + # Fix for macOS when napari launched from terminal + # related to https://github.com/napari/napari/pull/5531 + env.remove("PYTHONEXECUTABLE") + return env + + def _add_constraints_to_env( + self, env: QProcessEnvironment + ) -> QProcessEnvironment: + PINNED = 'CONDA_PINNED_PACKAGES' + constraints = self.constraints() + if env.contains(PINNED): + constraints.append(env.value(PINNED)) + env.insert(PINNED, "&".join(constraints)) + return env + + def _default_channels(self): + return ('conda-forge',) + + def _default_prefix(self): + if (Path(sys.prefix) / "conda-meta").is_dir(): + return sys.prefix + raise ValueError("Prefix has not been specified!") + + +class InstallerQueue(QObject): + """Queue for installation and uninstallation tasks in the plugin manager.""" + + # emitted when all jobs are finished. Not to be confused with finished, + # which is emitted when each individual job is finished. + # Tuple of exit codes for each individual job + allFinished = Signal(tuple) + + # emitted when each job finishes + # dict: ProcessFinishedData + processFinished = Signal(dict) + + # emitted when each job starts + started = Signal() + + # classes to manage pip and conda installations + PIP_INSTALLER_TOOL_CLASS = PipInstallerTool + CONDA_INSTALLER_TOOL_CLASS = CondaInstallerTool + # This should be set to the name of package that handles plugins + # e.g `napari` for napari + BASE_PACKAGE_NAME = '' + + def __init__( + self, parent: Optional[QObject] = None, prefix: Optional[str] = None + ) -> None: + super().__init__(parent) + self._queue: Deque[AbstractInstallerTool] = deque() + self._current_process: QProcess = None + self._prefix = prefix + self._output_widget = None + self._exit_codes = [] + + # -------------------------- Public API ------------------------------ + def install( + self, + tool: InstallerTools, + pkgs: Sequence[str], + *, + prefix: Optional[str] = None, + origins: Sequence[str] = (), + **kwargs, + ) -> JobId: + """Install packages in `pkgs` into `prefix` using `tool` with additional + `origins` as source for `pkgs`. + + Parameters + ---------- + tool : InstallerTools + Which type of installation tool to use. + pkgs : Sequence[str] + List of packages to install. + prefix : Optional[str], optional + Optional prefix to install packages into. + origins : Optional[Sequence[str]], optional + Additional sources for packages to be downloaded from. + + Returns + ------- + JobId : int + ID that can be used to cancel the process. + """ + item = self._build_queue_item( + tool=tool, + action=InstallerActions.INSTALL, + pkgs=pkgs, + prefix=prefix, + origins=origins, + process=self._create_process(), + **kwargs, + ) + return self._queue_item(item) + + def upgrade( + self, + tool: InstallerTools, + pkgs: Sequence[str], + *, + prefix: Optional[str] = None, + origins: Sequence[str] = (), + **kwargs, + ) -> JobId: + """Upgrade packages in `pkgs` into `prefix` using `tool` with additional + `origins` as source for `pkgs`. + + Parameters + ---------- + tool : InstallerTools + Which type of installation tool to use. + pkgs : Sequence[str] + List of packages to install. + prefix : Optional[str], optional + Optional prefix to install packages into. + origins : Optional[Sequence[str]], optional + Additional sources for packages to be downloaded from. + + Returns + ------- + JobId : int + ID that can be used to cancel the process. + """ + item = self._build_queue_item( + tool=tool, + action=InstallerActions.UPGRADE, + pkgs=pkgs, + prefix=prefix, + origins=origins, + process=self._create_process(), + **kwargs, + ) + return self._queue_item(item) + + def uninstall( + self, + tool: InstallerTools, + pkgs: Sequence[str], + *, + prefix: Optional[str] = None, + **kwargs, + ) -> JobId: + """Uninstall packages in `pkgs` from `prefix` using `tool`. + + Parameters + ---------- + tool : InstallerTools + Which type of installation tool to use. + pkgs : Sequence[str] + List of packages to uninstall. + prefix : Optional[str], optional + Optional prefix from which to uninstall packages. + + Returns + ------- + JobId : int + ID that can be used to cancel the process. + """ + item = self._build_queue_item( + tool=tool, + action=InstallerActions.UNINSTALL, + pkgs=pkgs, + prefix=prefix, + process=self._create_process(), + **kwargs, + ) + return self._queue_item(item) + + def cancel(self, job_id: JobId): + """Cancel `job_id` if it is running. If `job_id` does not exist int the queue, + a ValueError is raised. + + Parameters + ---------- + job_id : JobId + Job ID to cancel. + """ + for i, item in enumerate(deque(self._queue)): + if item.ident == job_id: + if i == 0: + # first in queue, currently running + self._queue.remove(item) + + with contextlib.suppress(RuntimeError): + item.process.finished.disconnect( + self._on_process_finished + ) + item.process.errorOccurred.disconnect( + self._on_error_occurred + ) + + self._end_process(item.process) + else: + # still pending, just remove from queue + self._queue.remove(item) + + self.processFinished.emit( + { + 'exit_code': 1, + 'exit_status': 0, + 'action': InstallerActions.CANCEL, + 'pkgs': item.pkgs, + } + ) + self._process_queue() + return + + msg = f"No job with id {job_id}. Current queue:\n - " + msg += "\n - ".join( + [ + f"{item.ident} -> {item.executable()} {item.arguments()}" + for item in self._queue + ] + ) + raise ValueError(msg) + + def cancel_all(self): + """Terminate all process in the queue and emit the `processFinished` signal.""" + all_pkgs = [] + for item in deque(self._queue): + all_pkgs.extend(item.pkgs) + process = item.process + + with contextlib.suppress(RuntimeError): + process.finished.disconnect(self._on_process_finished) + process.errorOccurred.disconnect(self._on_error_occurred) + + self._end_process(process) + + self._queue.clear() + self._current_process = None + self.processFinished.emit( + { + 'exit_code': 1, + 'exit_status': 0, + 'action': InstallerActions.CANCEL_ALL, + 'pkgs': all_pkgs, + } + ) + self._process_queue() + return + + def waitForFinished(self, msecs: int = 10000) -> bool: + """Block and wait for all jobs to finish. + + Parameters + ---------- + msecs : int, optional + Time to wait, by default 10000 + """ + while self.hasJobs(): + if self._current_process is not None: + self._current_process.waitForFinished(msecs) + return True + + def hasJobs(self) -> bool: + """True if there are jobs remaining in the queue.""" + return bool(self._queue) + + def currentJobs(self) -> int: + """Return the number of running jobs in the queue.""" + return len(self._queue) + + def set_output_widget(self, output_widget: QTextEdit): + if output_widget: + self._output_widget = output_widget + + # -------------------------- Private methods ------------------------------ + def _create_process(self) -> QProcess: + process = QProcess(self) + process.setProcessChannelMode(QProcess.MergedChannels) + process.readyReadStandardOutput.connect(self._on_stdout_ready) + process.readyReadStandardError.connect(self._on_stderr_ready) + process.finished.connect(self._on_process_finished) + process.errorOccurred.connect(self._on_error_occurred) + return process + + def _log(self, msg: str): + log.debug(msg) + if self._output_widget: + self._output_widget.append(msg) + + def _get_tool(self, tool: InstallerTools): + if tool == InstallerTools.PIP: + return self.PIP_INSTALLER_TOOL_CLASS + if tool == InstallerTools.CONDA: + return self.CONDA_INSTALLER_TOOL_CLASS + raise ValueError(f"InstallerTool {tool} not recognized!") + + def _build_queue_item( + self, + tool: InstallerTools, + action: InstallerActions, + pkgs: Sequence[str], + prefix: Optional[str] = None, + origins: Sequence[str] = (), + **kwargs, + ) -> AbstractInstallerTool: + return self._get_tool(tool)( + pkgs=pkgs, + action=action, + origins=origins, + prefix=prefix or self._prefix, + **kwargs, + ) + + def _queue_item(self, item: AbstractInstallerTool) -> JobId: + self._queue.append(item) + self._process_queue() + return item.ident + + def _process_queue(self): + if not self._queue: + self.allFinished.emit(tuple(self._exit_codes)) + self._exit_codes = [] + return + + tool = self._queue[0] + process = tool.process + + if process.state() != QProcess.Running: + process.setProgram(str(tool.executable())) + process.setProcessEnvironment(tool.environment()) + process.setArguments([str(arg) for arg in tool.arguments()]) + process.started.connect(self.started) + + self._log( + trans._( + "Starting '{program}' with args {args}", + program=process.program(), + args=process.arguments(), + ) + ) + + process.start() + self._current_process = process + + def _end_process(self, process: QProcess): + if os.name == 'nt': + # TODO: this might be too agressive and won't allow rollbacks! + # investigate whether we can also do .terminate() + process.kill() + else: + process.terminate() + + if self._output_widget: + self._output_widget.append( + trans._("\nTask was cancelled by the user.") + ) + + def _on_process_finished( + self, exit_code: int, exit_status: QProcess.ExitStatus + ): + try: + current = self._queue[0] + except IndexError: + current = None + if ( + current + and current.action == InstallerActions.UNINSTALL + and exit_status == QProcess.ExitStatus.NormalExit + and exit_code == 0 + ): + pm2 = PluginManager.instance() + npe1_plugins = set(plugin_manager.iter_available()) + for pkg in current.pkgs: + if pkg in pm2: + pm2.unregister(pkg) + elif pkg in npe1_plugins: + plugin_manager.unregister(pkg) + else: + log.warning( + 'Cannot unregister %s, not a known %s plugin.', + pkg, + self.BASE_PACKAGE_NAME, + ) + self._on_process_done(exit_code=exit_code, exit_status=exit_status) + + def _on_error_occurred(self, error: QProcess.ProcessError): + self._on_process_done(error=error) + + def _on_process_done( + self, + exit_code: Optional[int] = None, + exit_status: Optional[QProcess.ExitStatus] = None, + error: Optional[QProcess.ProcessError] = None, + ): + item = None + with contextlib.suppress(IndexError): + item = self._queue.popleft() + + if error: + msg = trans._( + "Task finished with errors! Error: {error}.", error=error + ) + else: + msg = trans._( + "Task finished with exit code {exit_code} with status {exit_status}.", + exit_code=exit_code, + exit_status=exit_status, + ) + + if item is not None: + self.processFinished.emit( + { + 'exit_code': exit_code, + 'exit_status': exit_status, + 'action': item.action, + 'pkgs': item.pkgs, + } + ) + self._exit_codes.append(exit_code) + + self._log(msg) + self._process_queue() + + def _on_stdout_ready(self): + if self._current_process is not None: + text = ( + self._current_process.readAllStandardOutput().data().decode() + ) + if text: + self._log(text) + + def _on_stderr_ready(self): + if self._current_process is not None: + text = self._current_process.readAllStandardError().data().decode() + if text: + self._log(text) diff --git a/napari_plugin_manager/base_qt_plugin_dialog.py b/napari_plugin_manager/base_qt_plugin_dialog.py index 974aa7f..1ae8e39 100644 --- a/napari_plugin_manager/base_qt_plugin_dialog.py +++ b/napari_plugin_manager/base_qt_plugin_dialog.py @@ -48,7 +48,7 @@ ) from superqt import QCollapsible, QElidingLabel -from napari_plugin_manager.qt_package_installer import ( +from napari_plugin_manager.base_qt_package_installer import ( InstallerActions, InstallerQueue, InstallerTools, diff --git a/napari_plugin_manager/qt_package_installer.py b/napari_plugin_manager/qt_package_installer.py index 343ab3f..e3f7b15 100644 --- a/napari_plugin_manager/qt_package_installer.py +++ b/napari_plugin_manager/qt_package_installer.py @@ -1,635 +1,41 @@ """ -A tool-agnostic installation logic for the plugin manager. +The installation logic for the napari plugin manager. -The main object is `InstallerQueue`, a `QProcess` subclass +The main object is `NapariInstallerQueue`, a `InstallerQueue` subclass with the notion of a job queue. The queued jobs are represented -by a `deque` of `*InstallerTool` dataclasses that contain the -executable path, arguments and environment modifications. -Available actions for each tool are `install`, `uninstall` -and `cancel`. +by a `deque` of `*InstallerTool` dataclasses (`NapariPipInstallerTool` and +`NapariCondaInstallerTool`). """ import atexit -import contextlib import os import sys -from collections import deque -from dataclasses import dataclass -from enum import auto from functools import lru_cache -from logging import getLogger from pathlib import Path -from subprocess import call -from tempfile import NamedTemporaryFile, gettempdir -from typing import Deque, Optional, Sequence, Tuple, TypedDict +from tempfile import NamedTemporaryFile +from typing import Sequence from napari._version import version as _napari_version from napari._version import version_tuple as _napari_version_tuple -from napari.plugins import plugin_manager -from napari.plugins.npe2api import _user_agent -from napari.utils.misc import StringEnum -from napari.utils.translations import trans -from npe2 import PluginManager -from qtpy.QtCore import QObject, QProcess, QProcessEnvironment, Signal -from qtpy.QtWidgets import QTextEdit -JobId = int -log = getLogger(__name__) +from napari_plugin_manager.base_qt_package_installer import ( + CondaInstallerTool, + InstallerQueue, + PipInstallerTool, +) -class InstallerActions(StringEnum): - "Available actions for the plugin manager" - INSTALL = auto() - UNINSTALL = auto() - CANCEL = auto() - CANCEL_ALL = auto() - UPGRADE = auto() - - -class ProcessFinishedData(TypedDict): - exit_code: int - exit_status: int - action: InstallerActions - pkgs: Tuple[str, ...] - - -class InstallerTools(StringEnum): - "Available tools for InstallerQueue jobs" - CONDA = auto() - PIP = auto() - - -@dataclass(frozen=True) -class AbstractInstallerTool: - action: InstallerActions - pkgs: Tuple[str, ...] - origins: Tuple[str, ...] = () - prefix: Optional[str] = None - process: QProcess = None - - @property - def ident(self): - return hash( - (self.action, *self.pkgs, *self.origins, self.prefix, self.process) - ) - - # abstract method - @classmethod - def executable(cls): - "Path to the executable that will run the task" - raise NotImplementedError - - # abstract method - def arguments(self): - "Arguments supplied to the executable" - raise NotImplementedError - - # abstract method - def environment( - self, env: QProcessEnvironment = None - ) -> QProcessEnvironment: - "Changes needed in the environment variables." - raise NotImplementedError - - @staticmethod - def constraints() -> Sequence[str]: - """ - Version constraints to limit unwanted changes in installation. - """ - raise NotImplementedError - - @classmethod - def available(cls) -> bool: - """ - Check if the tool is available by performing a little test - """ - raise NotImplementedError - - -class PipInstallerTool(AbstractInstallerTool): - @classmethod - def available(cls): - return call([cls.executable(), "-m", "pip", "--version"]) == 0 - - def arguments(self) -> Tuple[str, ...]: - args = ['-m', 'pip'] - if self.action == InstallerActions.INSTALL: - args += ['install', '-c', self._constraints_file()] - for origin in self.origins: - args += ['--extra-index-url', origin] - elif self.action == InstallerActions.UPGRADE: - args += [ - 'install', - '--upgrade', - '-c', - self._constraints_file(), - ] - for origin in self.origins: - args += ['--extra-index-url', origin] - elif self.action == InstallerActions.UNINSTALL: - args += ['uninstall', '-y'] - else: - raise ValueError(f"Action '{self.action}' not supported!") - if 10 <= log.getEffectiveLevel() < 30: # DEBUG level - args.append('-vvv') - if self.prefix is not None: - args.extend(['--prefix', str(self.prefix)]) - return (*args, *self.pkgs) - - def environment( - self, env: QProcessEnvironment = None - ) -> QProcessEnvironment: - if env is None: - env = QProcessEnvironment.systemEnvironment() - env.insert("PIP_USER_AGENT_USER_DATA", _user_agent()) - return env - - @classmethod - @lru_cache(maxsize=0) - def _constraints_file(cls) -> str: - raise NotImplementedError - - -class CondaInstallerTool(AbstractInstallerTool): - @classmethod - def executable(cls): - bat = ".bat" if os.name == "nt" else "" - for path in ( - Path(os.environ.get('MAMBA_EXE', '')), - Path(os.environ.get('CONDA_EXE', '')), - # $CONDA is usually only available on GitHub Actions - Path(os.environ.get('CONDA', '')) / 'condabin' / f'conda{bat}', - ): - if path.is_file(): - return str(path) - return f'conda{bat}' # cross our fingers 'conda' is in PATH - - @classmethod - def available(cls): - executable = cls.executable() - try: - return call([executable, "--version"]) == 0 - except FileNotFoundError: # pragma: no cover - return False - - def arguments(self) -> Tuple[str, ...]: - prefix = self.prefix or self._default_prefix() - if self.action == InstallerActions.UPGRADE: - args = ['update', '-y', '--prefix', prefix] - else: - args = [self.action.value, '-y', '--prefix', prefix] - args.append('--override-channels') - for channel in (*self.origins, *self._default_channels()): - args.extend(["-c", channel]) - return (*args, *self.pkgs) - - def environment( - self, env: QProcessEnvironment = None - ) -> QProcessEnvironment: - if env is None: - env = QProcessEnvironment.systemEnvironment() - self._add_constraints_to_env(env) - if 10 <= log.getEffectiveLevel() < 30: # DEBUG level - env.insert('CONDA_VERBOSITY', '3') - if os.name == "nt": - if not env.contains("TEMP"): - temp = gettempdir() - env.insert("TMP", temp) - env.insert("TEMP", temp) - if not env.contains("USERPROFILE"): - env.insert("HOME", os.path.expanduser("~")) - env.insert("USERPROFILE", os.path.expanduser("~")) - if sys.platform == 'darwin' and env.contains('PYTHONEXECUTABLE'): - # Fix for macOS when napari launched from terminal - # related to https://github.com/napari/napari/pull/5531 - env.remove("PYTHONEXECUTABLE") - return env - - def _add_constraints_to_env( - self, env: QProcessEnvironment - ) -> QProcessEnvironment: - PINNED = 'CONDA_PINNED_PACKAGES' - constraints = self.constraints() - if env.contains(PINNED): - constraints.append(env.value(PINNED)) - env.insert(PINNED, "&".join(constraints)) - return env - - def _default_channels(self): - return ('conda-forge',) - - def _default_prefix(self): - if (Path(sys.prefix) / "conda-meta").is_dir(): - return sys.prefix - raise ValueError("Prefix has not been specified!") - - -class InstallerQueue(QObject): - """Queue for installation and uninstallation tasks in the plugin manager.""" - - # emitted when all jobs are finished. Not to be confused with finished, - # which is emitted when each individual job is finished. - # Tuple of exit codes for each individual job - allFinished = Signal(tuple) - - # emitted when each job finishes - # dict: ProcessFinishedData - processFinished = Signal(dict) - - # emitted when each job starts - started = Signal() - - # classes to manage pip and conda installations - PIP_INSTALLER_TOOL_CLASS = PipInstallerTool - CONDA_INSTALLER_TOOL_CLASS = CondaInstallerTool - # This should be set to the name of package that handles plugins - # e.g `napari` for napari - BASE_PACKAGE_NAME = '' - - def __init__( - self, parent: Optional[QObject] = None, prefix: Optional[str] = None - ) -> None: - super().__init__(parent) - self._queue: Deque[AbstractInstallerTool] = deque() - self._current_process: QProcess = None - self._prefix = prefix - self._output_widget = None - self._exit_codes = [] - - # -------------------------- Public API ------------------------------ - def install( - self, - tool: InstallerTools, - pkgs: Sequence[str], - *, - prefix: Optional[str] = None, - origins: Sequence[str] = (), - **kwargs, - ) -> JobId: - """Install packages in `pkgs` into `prefix` using `tool` with additional - `origins` as source for `pkgs`. - - Parameters - ---------- - tool : InstallerTools - Which type of installation tool to use. - pkgs : Sequence[str] - List of packages to install. - prefix : Optional[str], optional - Optional prefix to install packages into. - origins : Optional[Sequence[str]], optional - Additional sources for packages to be downloaded from. - - Returns - ------- - JobId : int - ID that can be used to cancel the process. - """ - item = self._build_queue_item( - tool=tool, - action=InstallerActions.INSTALL, - pkgs=pkgs, - prefix=prefix, - origins=origins, - process=self._create_process(), - **kwargs, - ) - return self._queue_item(item) - - def upgrade( - self, - tool: InstallerTools, - pkgs: Sequence[str], - *, - prefix: Optional[str] = None, - origins: Sequence[str] = (), - **kwargs, - ) -> JobId: - """Upgrade packages in `pkgs` into `prefix` using `tool` with additional - `origins` as source for `pkgs`. - - Parameters - ---------- - tool : InstallerTools - Which type of installation tool to use. - pkgs : Sequence[str] - List of packages to install. - prefix : Optional[str], optional - Optional prefix to install packages into. - origins : Optional[Sequence[str]], optional - Additional sources for packages to be downloaded from. - - Returns - ------- - JobId : int - ID that can be used to cancel the process. - """ - item = self._build_queue_item( - tool=tool, - action=InstallerActions.UPGRADE, - pkgs=pkgs, - prefix=prefix, - origins=origins, - process=self._create_process(), - **kwargs, - ) - return self._queue_item(item) - - def uninstall( - self, - tool: InstallerTools, - pkgs: Sequence[str], - *, - prefix: Optional[str] = None, - **kwargs, - ) -> JobId: - """Uninstall packages in `pkgs` from `prefix` using `tool`. - - Parameters - ---------- - tool : InstallerTools - Which type of installation tool to use. - pkgs : Sequence[str] - List of packages to uninstall. - prefix : Optional[str], optional - Optional prefix from which to uninstall packages. - - Returns - ------- - JobId : int - ID that can be used to cancel the process. - """ - item = self._build_queue_item( - tool=tool, - action=InstallerActions.UNINSTALL, - pkgs=pkgs, - prefix=prefix, - process=self._create_process(), - **kwargs, - ) - return self._queue_item(item) - - def cancel(self, job_id: JobId): - """Cancel `job_id` if it is running. If `job_id` does not exist int the queue, - a ValueError is raised. - - Parameters - ---------- - job_id : JobId - Job ID to cancel. - """ - for i, item in enumerate(deque(self._queue)): - if item.ident == job_id: - if i == 0: - # first in queue, currently running - self._queue.remove(item) - - with contextlib.suppress(RuntimeError): - item.process.finished.disconnect( - self._on_process_finished - ) - item.process.errorOccurred.disconnect( - self._on_error_occurred - ) - - self._end_process(item.process) - else: - # still pending, just remove from queue - self._queue.remove(item) - - self.processFinished.emit( - { - 'exit_code': 1, - 'exit_status': 0, - 'action': InstallerActions.CANCEL, - 'pkgs': item.pkgs, - } - ) - self._process_queue() - return - - msg = f"No job with id {job_id}. Current queue:\n - " - msg += "\n - ".join( - [ - f"{item.ident} -> {item.executable()} {item.arguments()}" - for item in self._queue - ] - ) - raise ValueError(msg) - - def cancel_all(self): - """Terminate all process in the queue and emit the `processFinished` signal.""" - all_pkgs = [] - for item in deque(self._queue): - all_pkgs.extend(item.pkgs) - process = item.process - - with contextlib.suppress(RuntimeError): - process.finished.disconnect(self._on_process_finished) - process.errorOccurred.disconnect(self._on_error_occurred) - - self._end_process(process) - - self._queue.clear() - self._current_process = None - self.processFinished.emit( - { - 'exit_code': 1, - 'exit_status': 0, - 'action': InstallerActions.CANCEL_ALL, - 'pkgs': all_pkgs, - } - ) - self._process_queue() - return - - def waitForFinished(self, msecs: int = 10000) -> bool: - """Block and wait for all jobs to finish. - - Parameters - ---------- - msecs : int, optional - Time to wait, by default 10000 - """ - while self.hasJobs(): - if self._current_process is not None: - self._current_process.waitForFinished(msecs) - return True - - def hasJobs(self) -> bool: - """True if there are jobs remaining in the queue.""" - return bool(self._queue) - - def currentJobs(self) -> int: - """Return the number of running jobs in the queue.""" - return len(self._queue) - - def set_output_widget(self, output_widget: QTextEdit): - if output_widget: - self._output_widget = output_widget - - # -------------------------- Private methods ------------------------------ - def _create_process(self) -> QProcess: - process = QProcess(self) - process.setProcessChannelMode(QProcess.MergedChannels) - process.readyReadStandardOutput.connect(self._on_stdout_ready) - process.readyReadStandardError.connect(self._on_stderr_ready) - process.finished.connect(self._on_process_finished) - process.errorOccurred.connect(self._on_error_occurred) - return process - - def _log(self, msg: str): - log.debug(msg) - if self._output_widget: - self._output_widget.append(msg) - - def _get_tool(self, tool: InstallerTools): - if tool == InstallerTools.PIP: - return self.PIP_INSTALLER_TOOL_CLASS - if tool == InstallerTools.CONDA: - return self.CONDA_INSTALLER_TOOL_CLASS - raise ValueError(f"InstallerTool {tool} not recognized!") - - def _build_queue_item( - self, - tool: InstallerTools, - action: InstallerActions, - pkgs: Sequence[str], - prefix: Optional[str] = None, - origins: Sequence[str] = (), - **kwargs, - ) -> AbstractInstallerTool: - return self._get_tool(tool)( - pkgs=pkgs, - action=action, - origins=origins, - prefix=prefix or self._prefix, - **kwargs, - ) - - def _queue_item(self, item: AbstractInstallerTool) -> JobId: - self._queue.append(item) - self._process_queue() - return item.ident - - def _process_queue(self): - if not self._queue: - self.allFinished.emit(tuple(self._exit_codes)) - self._exit_codes = [] - return - - tool = self._queue[0] - process = tool.process - - if process.state() != QProcess.Running: - process.setProgram(str(tool.executable())) - process.setProcessEnvironment(tool.environment()) - process.setArguments([str(arg) for arg in tool.arguments()]) - process.started.connect(self.started) - - self._log( - trans._( - "Starting '{program}' with args {args}", - program=process.program(), - args=process.arguments(), - ) - ) - - process.start() - self._current_process = process - - def _end_process(self, process: QProcess): - if os.name == 'nt': - # TODO: this might be too agressive and won't allow rollbacks! - # investigate whether we can also do .terminate() - process.kill() - else: - process.terminate() - - if self._output_widget: - self._output_widget.append( - trans._("\nTask was cancelled by the user.") - ) - - def _on_process_finished( - self, exit_code: int, exit_status: QProcess.ExitStatus - ): - try: - current = self._queue[0] - except IndexError: - current = None - if ( - current - and current.action == InstallerActions.UNINSTALL - and exit_status == QProcess.ExitStatus.NormalExit - and exit_code == 0 - ): - pm2 = PluginManager.instance() - npe1_plugins = set(plugin_manager.iter_available()) - for pkg in current.pkgs: - if pkg in pm2: - pm2.unregister(pkg) - elif pkg in npe1_plugins: - plugin_manager.unregister(pkg) - else: - log.warning( - 'Cannot unregister %s, not a known %s plugin.', - pkg, - self.BASE_PACKAGE_NAME, - ) - self._on_process_done(exit_code=exit_code, exit_status=exit_status) - - def _on_error_occurred(self, error: QProcess.ProcessError): - self._on_process_done(error=error) - - def _on_process_done( - self, - exit_code: Optional[int] = None, - exit_status: Optional[QProcess.ExitStatus] = None, - error: Optional[QProcess.ProcessError] = None, +def _get_python_exe(): + # Note: is_bundled_app() returns False even if using a Briefcase bundle... + # Workaround: see if sys.executable is set to something something napari on Mac + if ( + sys.executable.endswith("napari") + and sys.platform == 'darwin' + and (python := Path(sys.prefix) / "bin" / "python3").is_file() ): - item = None - with contextlib.suppress(IndexError): - item = self._queue.popleft() - - if error: - msg = trans._( - "Task finished with errors! Error: {error}.", error=error - ) - else: - msg = trans._( - "Task finished with exit code {exit_code} with status {exit_status}.", - exit_code=exit_code, - exit_status=exit_status, - ) - - if item is not None: - self.processFinished.emit( - { - 'exit_code': exit_code, - 'exit_status': exit_status, - 'action': item.action, - 'pkgs': item.pkgs, - } - ) - self._exit_codes.append(exit_code) - - self._log(msg) - self._process_queue() - - def _on_stdout_ready(self): - if self._current_process is not None: - text = ( - self._current_process.readAllStandardOutput().data().decode() - ) - if text: - self._log(text) - - def _on_stderr_ready(self): - if self._current_process is not None: - text = self._current_process.readAllStandardError().data().decode() - if text: - self._log(text) + # sys.prefix should be /Contents/Resources/Support/Python/Resources + return str(python) + return sys.executable class NapariPipInstallerTool(PipInstallerTool): @@ -676,16 +82,3 @@ class NapariInstallerQueue(InstallerQueue): PIP_INSTALLER_TOOL_CLASS = NapariPipInstallerTool CONDA_INSTALLER_TOOL_CLASS = NapariCondaInstallerTool BASE_PACKAGE_NAME = 'napari' - - -def _get_python_exe(): - # Note: is_bundled_app() returns False even if using a Briefcase bundle... - # Workaround: see if sys.executable is set to something something napari on Mac - if ( - sys.executable.endswith("napari") - and sys.platform == 'darwin' - and (python := Path(sys.prefix) / "bin" / "python3").is_file() - ): - # sys.prefix should be /Contents/Resources/Support/Python/Resources - return str(python) - return sys.executable diff --git a/napari_plugin_manager/qt_plugin_dialog.py b/napari_plugin_manager/qt_plugin_dialog.py index 691f07e..43d1821 100644 --- a/napari_plugin_manager/qt_plugin_dialog.py +++ b/napari_plugin_manager/qt_plugin_dialog.py @@ -21,6 +21,10 @@ ) from qtpy.QtWidgets import QCheckBox, QMessageBox +from napari_plugin_manager.base_qt_package_installer import ( + InstallerActions, + InstallerTools, +) from napari_plugin_manager.base_qt_plugin_dialog import ( BasePluginListItem, BaseProjectInfoVersions, @@ -31,11 +35,7 @@ cache_clear, iter_napari_plugin_info, ) -from napari_plugin_manager.qt_package_installer import ( - InstallerActions, - InstallerTools, - NapariInstallerQueue, -) +from napari_plugin_manager.qt_package_installer import NapariInstallerQueue from napari_plugin_manager.utils import is_conda_package # Scaling factor for each list widget item when expanding. From c0159ff8e4e917207aad9cc2b2a9634fbed97c63 Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Fri, 13 Dec 2024 13:14:46 -0500 Subject: [PATCH 3/6] Improve pip installer tool code coverage --- .../_tests/test_installer_process.py | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/napari_plugin_manager/_tests/test_installer_process.py b/napari_plugin_manager/_tests/test_installer_process.py index 0f22a8f..d0837e3 100644 --- a/napari_plugin_manager/_tests/test_installer_process.py +++ b/napari_plugin_manager/_tests/test_installer_process.py @@ -1,3 +1,4 @@ +import logging import re import time from pathlib import Path @@ -7,6 +8,7 @@ import pytest from qtpy.QtCore import QProcessEnvironment +import napari_plugin_manager.base_qt_package_installer as bqpi from napari_plugin_manager.base_qt_package_installer import ( AbstractInstallerTool, InstallerActions, @@ -68,17 +70,28 @@ def test_not_implemented_methods(): with pytest.raises(NotImplementedError): tool.environment() + with pytest.raises(NotImplementedError): + tool.constraints() + with pytest.raises(NotImplementedError): tool.available() -def test_pip_installer_tasks(qtbot, tmp_virtualenv: 'Session', monkeypatch): +def test_pip_installer_tasks( + qtbot, tmp_virtualenv: 'Session', monkeypatch, caplog +): + caplog.set_level(logging.DEBUG, logger=bqpi.__name__) installer = NapariInstallerQueue() monkeypatch.setattr( NapariPipInstallerTool, "executable", lambda *a: tmp_virtualenv.creator.exe, ) + monkeypatch.setattr( + NapariPipInstallerTool, + "origins", + ("https://pypi.org/simple"), + ) with qtbot.waitSignal(installer.allFinished, timeout=20000): installer.install( tool=InstallerTools.PIP, @@ -142,6 +155,28 @@ def test_pip_installer_tasks(qtbot, tmp_virtualenv: 'Session', monkeypatch): ) +def test_pip_installer_invalid_action(tmp_virtualenv: 'Session', monkeypatch): + installer = NapariInstallerQueue() + monkeypatch.setattr( + NapariPipInstallerTool, + "executable", + lambda *a: tmp_virtualenv.creator.exe, + ) + invalid_action = 'Invalid Action' + with pytest.raises( + ValueError, match=f"Action '{invalid_action}' not supported!" + ): + item = installer._build_queue_item( + tool=InstallerTools.PIP, + action=invalid_action, + pkgs=['pip-install-test'], + prefix=None, + origins=(), + process=installer._create_process(), + ) + installer._queue_item(item) + + def test_installer_failures(qtbot, tmp_virtualenv: 'Session', monkeypatch): installer = NapariInstallerQueue() monkeypatch.setattr( From fb5f527af6eb2a2b270911efe09ca64fd5c7987d Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Fri, 13 Dec 2024 14:47:20 -0500 Subject: [PATCH 4/6] Try to improve code coverage for conda installer tool --- .github/workflows/test_and_deploy.yml | 41 ++++++++++++++++--- .../_tests/test_installer_process.py | 9 +++- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index 6f53b01..922445e 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -25,7 +25,7 @@ concurrency: jobs: test: - name: ${{ matrix.platform }}, py${{ matrix.python-version }}, napari ${{ matrix.napari }} + name: ${{ matrix.platform }}, py${{ matrix.python-version }}, napari ${{ matrix.napari }}, ${{ matrix.tool }} runs-on: ${{ matrix.platform }} strategy: fail-fast: false @@ -33,6 +33,7 @@ jobs: platform: [ubuntu-latest, windows-latest, macos-13] python-version: ["3.9", "3.10", "3.11"] napari: ["latest", "repo"] + tool: ["pip", "conda"] exclude: # TODO: Remove when we have a napari release with the plugin manager changes - napari: "latest" @@ -43,11 +44,19 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} - pip + if: matrix.tool == 'pip' uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Set up Python ${{ matrix.python-version }} - conda + if: matrix.tool == 'conda' + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + python-version: ${{ matrix.python-version }} + - uses: tlambert03/setup-qt-libs@v1 # strategy borrowed from vispy for installing opengl libs on windows @@ -58,14 +67,36 @@ jobs: powershell gl-ci-helpers/appveyor/install_opengl.ps1 if (Test-Path -Path "C:\Windows\system32\opengl32.dll" -PathType Leaf) {Exit 0} else {Exit 1} - - name: Install dependencies + - name: Install dependencies without tox-conda + if: matrix.tool == 'pip' run: | python -m pip install --upgrade pip - pip install setuptools tox tox-gh-actions + python -m pip install setuptools tox tox-gh-actions + + - name: Install dependencies including tox-conda + if: matrix.tool == 'conda' + shell: bash -el {0} + run: | + python -m pip install --upgrade pip + python -m pip install setuptools 'tox<4' tox-gh-actions tox-conda + + - name: Test with tox - pip + if: matrix.tool == 'pip' + uses: aganders3/headless-gui@v2 + with: + run: python -m tox -vv + env: + PYVISTA_OFF_SCREEN: True # required for opengl on windows + NAPARI: ${{ matrix.napari }} + FORCE_COLOR: 1 + # PySide6 only functional with Python 3.10+ + TOX_SKIP_ENV: ".*py39-PySide6.*" - - name: Test with tox + - name: Test with tox - conda + if: matrix.tool == 'conda' uses: aganders3/headless-gui@v2 with: + shell: bash -el {0} run: python -m tox -vv env: PYVISTA_OFF_SCREEN: True # required for opengl on windows diff --git a/napari_plugin_manager/_tests/test_installer_process.py b/napari_plugin_manager/_tests/test_installer_process.py index d0837e3..1195796 100644 --- a/napari_plugin_manager/_tests/test_installer_process.py +++ b/napari_plugin_manager/_tests/test_installer_process.py @@ -1,5 +1,6 @@ import logging import re +import sys import time from pathlib import Path from types import MethodType @@ -90,7 +91,7 @@ def test_pip_installer_tasks( monkeypatch.setattr( NapariPipInstallerTool, "origins", - ("https://pypi.org/simple"), + ("https://pypi.org/simple",), ) with qtbot.waitSignal(installer.allFinished, timeout=20000): installer.install( @@ -235,7 +236,11 @@ def test_cancel_incorrect_job_id(qtbot, tmp_virtualenv: 'Session'): @pytest.mark.skipif( not NapariCondaInstallerTool.available(), reason="Conda is not available." ) -def test_conda_installer(qtbot, tmp_conda_env: Path): +def test_conda_installer(qtbot, caplog, monkeypatch, tmp_conda_env: Path): + if sys.platform == "darwin": + # check handled for `PYTHONEXECUTABLE` env definition on macOS + monkeypatch.setenv("PYTHONEXECUTABLE", sys.executable) + caplog.set_level(logging.DEBUG, logger=bqpi.__name__) conda_meta = tmp_conda_env / "conda-meta" glob_pat = "typing-extensions-*.json" glob_pat_2 = "pyzenhub-*.json" From a4bac7132654f304e1d5bb61f6276432692414a6 Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:23:22 -0500 Subject: [PATCH 5/6] Move 'test_no_implemented_methods' to its own test file --- .../_tests/test_base_installer_process.py | 23 +++++++++++++++++++ .../_tests/test_installer_process.py | 18 --------------- 2 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 napari_plugin_manager/_tests/test_base_installer_process.py diff --git a/napari_plugin_manager/_tests/test_base_installer_process.py b/napari_plugin_manager/_tests/test_base_installer_process.py new file mode 100644 index 0000000..0d7b8e0 --- /dev/null +++ b/napari_plugin_manager/_tests/test_base_installer_process.py @@ -0,0 +1,23 @@ +import pytest + +from napari_plugin_manager.base_qt_package_installer import ( + AbstractInstallerTool, +) + + +def test_not_implemented_methods(): + tool = AbstractInstallerTool('install', ['requests']) + with pytest.raises(NotImplementedError): + tool.executable() + + with pytest.raises(NotImplementedError): + tool.arguments() + + with pytest.raises(NotImplementedError): + tool.environment() + + with pytest.raises(NotImplementedError): + tool.constraints() + + with pytest.raises(NotImplementedError): + tool.available() diff --git a/napari_plugin_manager/_tests/test_installer_process.py b/napari_plugin_manager/_tests/test_installer_process.py index 1195796..9b89103 100644 --- a/napari_plugin_manager/_tests/test_installer_process.py +++ b/napari_plugin_manager/_tests/test_installer_process.py @@ -60,24 +60,6 @@ def environment(self, env=None): return QProcessEnvironment.systemEnvironment() -def test_not_implemented_methods(): - tool = AbstractInstallerTool('install', ['requests']) - with pytest.raises(NotImplementedError): - tool.executable() - - with pytest.raises(NotImplementedError): - tool.arguments() - - with pytest.raises(NotImplementedError): - tool.environment() - - with pytest.raises(NotImplementedError): - tool.constraints() - - with pytest.raises(NotImplementedError): - tool.available() - - def test_pip_installer_tasks( qtbot, tmp_virtualenv: 'Session', monkeypatch, caplog ): From b7faa65c012a778dd7c52d929475a1db5bfa9bed Mon Sep 17 00:00:00 2001 From: dalthviz <16781833+dalthviz@users.noreply.github.com> Date: Mon, 23 Dec 2024 13:31:24 -0500 Subject: [PATCH 6/6] Testing --- napari_plugin_manager/_tests/test_qt_plugin_dialog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/napari_plugin_manager/_tests/test_qt_plugin_dialog.py b/napari_plugin_manager/_tests/test_qt_plugin_dialog.py index fc65727..cb5623a 100644 --- a/napari_plugin_manager/_tests/test_qt_plugin_dialog.py +++ b/napari_plugin_manager/_tests/test_qt_plugin_dialog.py @@ -509,6 +509,7 @@ def test_install_pypi_constructor( plugin_dialog.installer.processFinished, timeout=60_000 ): widget.action_button.click() + qtbot.wait(5000) else: widget.action_button.click() assert mock.called