diff --git a/pyproject.toml b/pyproject.toml index e68dd92d..7f4453f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,7 +98,7 @@ line-length = 88 target-version = "py38" fix = true src = ["src/npe2", "tests"] -select = [ +lint.select = [ "E", "F", "W", #flake8 @@ -110,15 +110,15 @@ select = [ "RUF", # ruff-specific rules ] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "src/npe2/cli.py" = ["B008", "A00"] "**/test_*.py" = ["RUF018"] -[tool.ruff.pyupgrade] +[tool.ruff.lint.pyupgrade] # Preserve types, even if a file imports `from __future__ import annotations`. keep-runtime-typing = true -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ['npe2'] # https://mypy.readthedocs.io/en/stable/config_file.html diff --git a/src/npe2/_inspection/_fetch.py b/src/npe2/_inspection/_fetch.py index 03bb1173..62e85f87 100644 --- a/src/npe2/_inspection/_fetch.py +++ b/src/npe2/_inspection/_fetch.py @@ -3,10 +3,8 @@ import io import json import os -import re import subprocess import tempfile -from concurrent.futures import ProcessPoolExecutor from contextlib import contextmanager from functools import lru_cache from importlib import metadata @@ -20,11 +18,10 @@ Iterator, List, Optional, - Tuple, Union, ) from unittest.mock import patch -from urllib import error, parse, request +from urllib import error, request from zipfile import ZipFile from npe2.manifest import PackageMetadata @@ -264,7 +261,7 @@ def fetch_manifest( return _manifest_from_extracted_wheel(td) except metadata.PackageNotFoundError: return _manifest_from_pypi_sdist(package_or_url, version) - except error.HTTPError: + except error.HTTPError: # pragma: no cover pass # pragma: no cover raise ValueError( # pragma: no cover f"Could not interpret {package_or_url!r} as a PYPI package name or URL to a " @@ -377,75 +374,8 @@ def _tmp_pypi_sdist_download( return _tmp_targz_download(url) -@lru_cache -def _get_packages_by_classifier(classifier: str) -> Dict[str, str]: - """Search for packages declaring ``classifier`` on PyPI. - - Returns - ------- - packages : List[str] - name of all packages at pypi that declare ``classifier`` - """ - PACKAGE_NAME_PATTERN = re.compile('class="package-snippet__name">(.+)') - PACKAGE_VERSION_PATTERN = re.compile('class="package-snippet__version">(.+)') - - packages = {} - page = 1 - url = f"https://pypi.org/search/?c={parse.quote_plus(classifier)}&page=" - while True: - try: - with request.urlopen(f"{url}{page}") as response: - html = response.read().decode() - names = PACKAGE_NAME_PATTERN.findall(html) - versions = PACKAGE_VERSION_PATTERN.findall(html) - packages.update(dict(zip(names, versions))) - page += 1 - except error.HTTPError: - break - - return dict(sorted(packages.items())) - - -def get_pypi_plugins() -> Dict[str, str]: - """Return {name: latest_version} for all plugins found on pypi.""" - NAPARI_CLASSIFIER = "Framework :: napari" - return _get_packages_by_classifier(NAPARI_CLASSIFIER) - - @lru_cache def get_hub_plugin(plugin_name: str) -> Dict[str, Any]: """Return hub information for a specific plugin.""" with request.urlopen(f"https://api.napari-hub.org/plugins/{plugin_name}") as r: return json.load(r) - - -def _try_fetch_and_write_manifest(args: Tuple[str, str, Path, int]): - name, version, dest, indent = args - FORMAT = "json" - - try: # pragma: no cover - mf = fetch_manifest(name, version=version) - manifest_string = getattr(mf, FORMAT)(exclude=set(), indent=indent) - - (dest / f"{name}.{FORMAT}").write_text(manifest_string) - print(f"✅ {name}") - except Exception as e: - print(f"❌ {name}") - return name, {"version": version, "error": str(e)} - - -def fetch_all_manifests(dest: str = "manifests", indent: int = 2) -> None: - """Fetch all manifests for plugins on PyPI and write to ``dest`` directory.""" - _dest = Path(dest) - _dest.mkdir(exist_ok=True, parents=True) - - args = [ - (name, ver, _dest, indent) for name, ver in sorted(get_pypi_plugins().items()) - ] - - # use processes instead of threads, because many of the subroutines in build - # and setuptools use `os.chdir()`, which is not thread-safe - with ProcessPoolExecutor() as executor: - errors = list(executor.map(_try_fetch_and_write_manifest, args)) - _errors = {tup[0]: tup[1] for tup in errors if tup} - (_dest / "errors.json").write_text(json.dumps(_errors, indent=indent)) diff --git a/src/npe2/_plugin_manager.py b/src/npe2/_plugin_manager.py index 8a3c113a..6cc9e3f3 100644 --- a/src/npe2/_plugin_manager.py +++ b/src/npe2/_plugin_manager.py @@ -2,7 +2,6 @@ import contextlib import os -import urllib import warnings from collections import Counter, defaultdict from fnmatch import fnmatch @@ -26,6 +25,7 @@ Tuple, Union, ) +from urllib import parse from psygnal import Signal, SignalGroup @@ -144,7 +144,7 @@ def iter_compatible_readers(self, paths: List[str]) -> Iterator[ReaderContributi yield from (r for pattern, r in self._readers if pattern == "") else: # ensure not a URI - if not urllib.parse.urlparse(path).scheme: + if not parse.urlparse(path).scheme: # lower case the extension for checking manifest pattern base = os.path.splitext(Path(path).stem)[0] ext = "".join(Path(path).suffixes) diff --git a/src/npe2/cli.py b/src/npe2/cli.py index fc4e2479..b50f8af1 100644 --- a/src/npe2/cli.py +++ b/src/npe2/cli.py @@ -1,5 +1,4 @@ import builtins -import sys import warnings from enum import Enum from pathlib import Path @@ -316,23 +315,6 @@ def list( typer.echo(template.format(**r, ncontrib=ncontrib)) -def _fetch_all_manifests(doit: bool): - """Fetch all manifests and dump to "manifests" folder.""" - if not doit: - return - - from npe2._inspection import _fetch - - dest = "manifests" - if "-o" in sys.argv: - dest = sys.argv[sys.argv.index("-o") + 1] - elif "--output" in sys.argv: # pragma: no cover - dest = sys.argv[sys.argv.index("--output") + 1] - - _fetch.fetch_all_manifests(dest) - raise typer.Exit(0) - - @app.command() def fetch( name: List[str], @@ -361,13 +343,6 @@ def fetch( help="If provided, will write manifest to filepath (must end with .yaml, " ".json, or .toml). Otherwise, will print to stdout.", ), - all: Optional[bool] = typer.Option( - None, - "--all", - help="Fetch manifests for ALL known plugins (will be SLOW)", - callback=_fetch_all_manifests, - is_eager=True, - ), ): """Fetch manifest from remote package. diff --git a/tests/test_cli.py b/tests/test_cli.py index d83d721c..e9fda0b4 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,4 @@ import sys -from concurrent.futures import ThreadPoolExecutor -from unittest.mock import patch import pytest from typer.testing import CliRunner @@ -94,21 +92,6 @@ def test_cli_fetch(format, tmp_path, to_file, include_meta): assert "package_metadata" in result.stdout -def test_cli_fetch_all(tmp_path, monkeypatch): - dest = tmp_path / "output" - with patch("npe2._inspection._fetch.get_pypi_plugins") as mock_hub: - mock_hub.return_value = {"a": "0.1.0", "b": "0.2.0", "c": "0.3.0"} - with patch("npe2._inspection._fetch.ProcessPoolExecutor", ThreadPoolExecutor): - cmd = ["fetch", "--all", "-o", str(dest)] - monkeypatch.setattr(sys, "argv", cmd) - result = runner.invoke(app, cmd) - - mock_hub.assert_called_once() - assert result.exit_code == 0 - assert dest.exists() - assert (dest / "errors.json").exists() - - @pytest.mark.filterwarnings("default:Failed to convert") def test_cli_convert_repo(npe1_repo, mock_npe1_pm_with_plugin): result = runner.invoke(app, ["convert", str(npe1_repo)]) diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 35e5a485..00a36d4f 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -10,7 +10,6 @@ _manifest_from_pypi_sdist, get_hub_plugin, get_manifest_from_wheel, - get_pypi_plugins, get_pypi_url, ) @@ -85,11 +84,6 @@ def test_get_hub_plugin(): assert info["name"] == "napari-svg" -def test_get_pypi_plugins(): - plugins = get_pypi_plugins() - assert len(plugins) > 0 - - @pytest.mark.skipif(not os.getenv("CI"), reason="slow, only run on CI") @pytest.mark.parametrize( "url",