Remove utilities for fetching all plugins (#371)

This PR removes all utilities and tests for fetching **all** plugins
from PyPI, as the HTML scraping method is no longer valid. It keeps
utilities for fetching specific manifests from PyPI.

---------

Co-authored-by: Draga Doncila <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

pyproject.toml

@@ -98,7 +98,7 @@ line-length = 88
 target-version = "py38"
 fix = true
 src = ["src/npe2", "tests"]
-select = [
+lint.select = [
     "E",
     "F",
     "W",   #flake8
@@ -110,15 +110,15 @@ select = [
     "RUF", # ruff-specific rules
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "src/npe2/cli.py" = ["B008", "A00"]
 "**/test_*.py" = ["RUF018"]
 
-[tool.ruff.pyupgrade]
+[tool.ruff.lint.pyupgrade]
 # Preserve types, even if a file imports `from __future__ import annotations`.
 keep-runtime-typing = true
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ['npe2']
 
 # https://mypy.readthedocs.io/en/stable/config_file.html

src/npe2/_inspection/_fetch.py

@@ -3,10 +3,8 @@
 import io
 import json
 import os
-import re
 import subprocess
 import tempfile
-from concurrent.futures import ProcessPoolExecutor
 from contextlib import contextmanager
 from functools import lru_cache
 from importlib import metadata
@@ -20,11 +18,10 @@
     Iterator,
     List,
     Optional,
-    Tuple,
     Union,
 )
 from unittest.mock import patch
-from urllib import error, parse, request
+from urllib import error, request
 from zipfile import ZipFile
 
 from npe2.manifest import PackageMetadata
@@ -41,7 +38,6 @@
     "fetch_manifest",
     "get_pypi_url",
     "get_hub_plugin",
-    "get_pypi_plugins",
 ]
 
 
@@ -264,7 +260,7 @@ def fetch_manifest(
                 return _manifest_from_extracted_wheel(td)
         except metadata.PackageNotFoundError:
             return _manifest_from_pypi_sdist(package_or_url, version)
-        except error.HTTPError:
+        except error.HTTPError:  # pragma: no cover
             pass  # pragma: no cover
     raise ValueError(  # pragma: no cover
         f"Could not interpret {package_or_url!r} as a PYPI package name or URL to a "
@@ -377,75 +373,8 @@ def _tmp_pypi_sdist_download(
     return _tmp_targz_download(url)
 
 
-@lru_cache
-def _get_packages_by_classifier(classifier: str) -> Dict[str, str]:
-    """Search for packages declaring ``classifier`` on PyPI.
-
-    Returns
-    -------
-    packages : List[str]
-        name of all packages at pypi that declare ``classifier``
-    """
-    PACKAGE_NAME_PATTERN = re.compile('class="package-snippet__name">(.+)</span>')
-    PACKAGE_VERSION_PATTERN = re.compile('class="package-snippet__version">(.+)</span>')
-
-    packages = {}
-    page = 1
-    url = f"https://pypi.org/search/?c={parse.quote_plus(classifier)}&page="
-    while True:
-        try:
-            with request.urlopen(f"{url}{page}") as response:
-                html = response.read().decode()
-            names = PACKAGE_NAME_PATTERN.findall(html)
-            versions = PACKAGE_VERSION_PATTERN.findall(html)
-            packages.update(dict(zip(names, versions)))
-            page += 1
-        except error.HTTPError:
-            break
-
-    return dict(sorted(packages.items()))
-
-
-def get_pypi_plugins() -> Dict[str, str]:
-    """Return {name: latest_version} for all plugins found on pypi."""
-    NAPARI_CLASSIFIER = "Framework :: napari"
-    return _get_packages_by_classifier(NAPARI_CLASSIFIER)
-
-
 @lru_cache
 def get_hub_plugin(plugin_name: str) -> Dict[str, Any]:
     """Return hub information for a specific plugin."""
     with request.urlopen(f"https://api.napari-hub.org/plugins/{plugin_name}") as r:
         return json.load(r)
-
-
-def _try_fetch_and_write_manifest(args: Tuple[str, str, Path, int]):
-    name, version, dest, indent = args
-    FORMAT = "json"
-
-    try:  # pragma: no cover
-        mf = fetch_manifest(name, version=version)
-        manifest_string = getattr(mf, FORMAT)(exclude=set(), indent=indent)
-
-        (dest / f"{name}.{FORMAT}").write_text(manifest_string)
-        print(f"✅ {name}")
-    except Exception as e:
-        print(f"❌ {name}")
-        return name, {"version": version, "error": str(e)}
-
-
-def fetch_all_manifests(dest: str = "manifests", indent: int = 2) -> None:
-    """Fetch all manifests for plugins on PyPI and write to ``dest`` directory."""
-    _dest = Path(dest)
-    _dest.mkdir(exist_ok=True, parents=True)
-
-    args = [
-        (name, ver, _dest, indent) for name, ver in sorted(get_pypi_plugins().items())
-    ]
-
-    # use processes instead of threads, because many of the subroutines in build
-    # and setuptools use `os.chdir()`, which is not thread-safe
-    with ProcessPoolExecutor() as executor:
-        errors = list(executor.map(_try_fetch_and_write_manifest, args))
-    _errors = {tup[0]: tup[1] for tup in errors if tup}
-    (_dest / "errors.json").write_text(json.dumps(_errors, indent=indent))

src/npe2/_plugin_manager.py

@@ -2,7 +2,6 @@
 
 import contextlib
 import os
-import urllib
 import warnings
 from collections import Counter, defaultdict
 from fnmatch import fnmatch
@@ -26,6 +25,7 @@
     Tuple,
     Union,
 )
+from urllib import parse
 
 from psygnal import Signal, SignalGroup
 
@@ -144,7 +144,7 @@ def iter_compatible_readers(self, paths: List[str]) -> Iterator[ReaderContributi
             yield from (r for pattern, r in self._readers if pattern == "")
         else:
             # ensure not a URI
-            if not urllib.parse.urlparse(path).scheme:
+            if not parse.urlparse(path).scheme:
                 # lower case the extension for checking manifest pattern
                 base = os.path.splitext(Path(path).stem)[0]
                 ext = "".join(Path(path).suffixes)

src/npe2/cli.py

@@ -1,5 +1,4 @@
 import builtins
-import sys
 import warnings
 from enum import Enum
 from pathlib import Path
@@ -316,23 +315,6 @@ def list(
             typer.echo(template.format(**r, ncontrib=ncontrib))
 
 
-def _fetch_all_manifests(doit: bool):
-    """Fetch all manifests and dump to "manifests" folder."""
-    if not doit:
-        return
-
-    from npe2._inspection import _fetch
-
-    dest = "manifests"
-    if "-o" in sys.argv:
-        dest = sys.argv[sys.argv.index("-o") + 1]
-    elif "--output" in sys.argv:  # pragma: no cover
-        dest = sys.argv[sys.argv.index("--output") + 1]
-
-    _fetch.fetch_all_manifests(dest)
-    raise typer.Exit(0)
-
-
 @app.command()
 def fetch(
     name: List[str],
@@ -361,13 +343,6 @@ def fetch(
         help="If provided, will write manifest to filepath (must end with .yaml, "
         ".json, or .toml). Otherwise, will print to stdout.",
     ),
-    all: Optional[bool] = typer.Option(
-        None,
-        "--all",
-        help="Fetch manifests for ALL known plugins (will be SLOW)",
-        callback=_fetch_all_manifests,
-        is_eager=True,
-    ),
 ):
     """Fetch manifest from remote package.
 

tests/test_cli.py

@@ -1,6 +1,4 @@
 import sys
-from concurrent.futures import ThreadPoolExecutor
-from unittest.mock import patch
 
 import pytest
 from typer.testing import CliRunner
@@ -94,21 +92,6 @@ def test_cli_fetch(format, tmp_path, to_file, include_meta):
             assert "package_metadata" in result.stdout
 
 
-def test_cli_fetch_all(tmp_path, monkeypatch):
-    dest = tmp_path / "output"
-    with patch("npe2._inspection._fetch.get_pypi_plugins") as mock_hub:
-        mock_hub.return_value = {"a": "0.1.0", "b": "0.2.0", "c": "0.3.0"}
-        with patch("npe2._inspection._fetch.ProcessPoolExecutor", ThreadPoolExecutor):
-            cmd = ["fetch", "--all", "-o", str(dest)]
-            monkeypatch.setattr(sys, "argv", cmd)
-            result = runner.invoke(app, cmd)
-
-    mock_hub.assert_called_once()
-    assert result.exit_code == 0
-    assert dest.exists()
-    assert (dest / "errors.json").exists()
-
-
 @pytest.mark.filterwarnings("default:Failed to convert")
 def test_cli_convert_repo(npe1_repo, mock_npe1_pm_with_plugin):
     result = runner.invoke(app, ["convert", str(npe1_repo)])

tests/test_fetch.py

@@ -10,7 +10,6 @@
     _manifest_from_pypi_sdist,
     get_hub_plugin,
     get_manifest_from_wheel,
-    get_pypi_plugins,
     get_pypi_url,
 )
 
@@ -85,11 +84,6 @@ def test_get_hub_plugin():
     assert info["name"] == "napari-svg"
 
 
-def test_get_pypi_plugins():
-    plugins = get_pypi_plugins()
-    assert len(plugins) > 0
-
-
 @pytest.mark.skipif(not os.getenv("CI"), reason="slow, only run on CI")
 @pytest.mark.parametrize(
     "url",