From 2aac71d3a2860fce6c2ef0e1378a3afc30ffd166 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 27 Jun 2024 18:01:08 -0400 Subject: [PATCH] Use rapids-metadata (#39) * Use rapids-metadata rapids-metadata was created so that we wouldn't have to cut a new pre-commit-hooks release every time the structure of RAPIDS changes. Use rapids-metadata and get rid of the baked-in list of packages. * Add nightly index to CI * Add type hints * Upgrade to rapids-metadata 0.3.0 --- .pre-commit-hooks.yaml | 2 + ci/build-test.sh | 2 +- pyproject.toml | 1 + src/rapids_pre_commit_hooks/alpha_spec.py | 63 +++++----------- .../test_alpha_spec.py | 75 +++++++++++++++---- 5 files changed, 80 insertions(+), 63 deletions(-) diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index daa5d93..57f8ece 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -18,6 +18,8 @@ language: python files: dependencies[.]yaml$ args: [--fix] + additional_dependencies: + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - id: verify-conda-yes name: pass -y/--yes to conda description: make sure that all calls to conda pass -y/--yes diff --git a/ci/build-test.sh b/ci/build-test.sh index a784659..bb0fd26 100755 --- a/ci/build-test.sh +++ b/ci/build-test.sh @@ -10,6 +10,6 @@ python -m build . for PKG in dist/*; do echo "$PKG" pip uninstall -y rapids-pre-commit-hooks - pip install "$PKG[test]" + pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple "$PKG[test]" pytest done diff --git a/pyproject.toml b/pyproject.toml index 24bb9ae..7ee8526 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "bashlex", "gitpython", "packaging", + "rapids-metadata>=0.3.0,<0.4.0.dev0", "rich", "tomlkit", ] diff --git a/src/rapids_pre_commit_hooks/alpha_spec.py b/src/rapids_pre_commit_hooks/alpha_spec.py index ceddf13..0e308dc 100644 --- a/src/rapids_pre_commit_hooks/alpha_spec.py +++ b/src/rapids_pre_commit_hooks/alpha_spec.py @@ -12,52 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import re -from functools import total_ordering +from functools import cache, total_ordering import yaml from packaging.requirements import InvalidRequirement, Requirement +from rapids_metadata.remote import fetch_latest from .lint import LintMain -RAPIDS_ALPHA_SPEC_PACKAGES = { - "cubinlinker", - "cucim", - "cudf", - "cugraph", - "cugraph-dgl", - "cugraph-equivariant", - "cugraph-pyg", - "cuml", - "cuproj", - "cuspatial", - "cuxfilter", - "dask-cuda", - "dask-cudf", - "distributed-ucxx", - "librmm", - "libucx", - "nx-cugraph", - "ptxcompiler", - "pylibcugraph", - "pylibcugraphops", - "pylibraft", - "pylibwholegraph", - "pynvjitlink", - "raft-dask", - "rmm", - "ucx-py", - "ucxx", -} - -RAPIDS_NON_CUDA_SUFFIXED_PACKAGES = { - "dask-cuda", -} - -RAPIDS_CUDA_SUFFIXED_PACKAGES = ( - RAPIDS_ALPHA_SPEC_PACKAGES - RAPIDS_NON_CUDA_SUFFIXED_PACKAGES -) - ALPHA_SPECIFIER = ">=0.0.0a0" ALPHA_SPEC_OUTPUT_TYPES = { @@ -68,15 +32,21 @@ CUDA_SUFFIX_REGEX = re.compile(r"^(?P.*)-cu[0-9]{2}$") +@cache +def all_metadata(): + return fetch_latest() + + def node_has_type(node, tag_type): return node.tag == f"tag:yaml.org,2002:{tag_type}" -def is_rapids_cuda_suffixed_package(name): - return any( - (match := CUDA_SUFFIX_REGEX.search(name)) and match.group("package") == package - for package in RAPIDS_CUDA_SUFFIXED_PACKAGES - ) +def strip_cuda_suffix(name: str) -> str: + if (match := CUDA_SUFFIX_REGEX.search(name)) and match.group( + "package" + ) in all_metadata().get_current_version(os.getcwd()).cuda_suffixed_packages: + return match.group("package") + return name def check_package_spec(linter, args, anchors, used_anchors, node): @@ -108,8 +78,9 @@ def create_specifier_string(specifiers): req = Requirement(node.value) except InvalidRequirement: return - if req.name in RAPIDS_ALPHA_SPEC_PACKAGES or is_rapids_cuda_suffixed_package( - req.name + if ( + strip_cuda_suffix(req.name) + in all_metadata().get_current_version(os.getcwd()).prerelease_packages ): for key, value in anchors.items(): if value == node: diff --git a/test/rapids_pre_commit_hooks/test_alpha_spec.py b/test/rapids_pre_commit_hooks/test_alpha_spec.py index ddcd685..d71fb23 100644 --- a/test/rapids_pre_commit_hooks/test_alpha_spec.py +++ b/test/rapids_pre_commit_hooks/test_alpha_spec.py @@ -12,15 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib +import os.path from itertools import chain from textwrap import dedent from unittest.mock import MagicMock, Mock, call, patch import pytest import yaml +from packaging.version import Version from rapids_pre_commit_hooks import alpha_spec, lint +latest_version, latest_metadata = max( + alpha_spec.all_metadata().versions.items(), key=lambda item: Version(item[0]) +) + + +@contextlib.contextmanager +def set_cwd(cwd): + old_cwd = os.getcwd() + os.chdir(cwd) + try: + yield + finally: + os.chdir(old_cwd) + def test_anchor_preserving_loader(): loader = alpha_spec.AnchorPreservingLoader("- &a A\n- *a") @@ -32,32 +49,40 @@ def test_anchor_preserving_loader(): @pytest.mark.parametrize( - ["name", "is_suffixed"], + ["name", "stripped_name"], [ *chain( *( [ - (f"{p}-cu11", True), - (f"{p}-cu12", True), - (f"{p}-cuda", False), + (p, p), + (f"{p}-cu11", p), + (f"{p}-cu12", p), + (f"{p}-cuda", f"{p}-cuda"), ] - for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + for p in latest_metadata.cuda_suffixed_packages ) ), *chain( *( [ - (f"{p}-cu11", False), - (f"{p}-cu12", False), - (f"{p}-cuda", False), + (p, p), + (f"{p}-cu11", f"{p}-cu11"), + (f"{p}-cu12", f"{p}-cu12"), + (f"{p}-cuda", f"{p}-cuda"), ] - for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES + for p in latest_metadata.all_packages + - latest_metadata.cuda_suffixed_packages ) ), ], ) -def test_is_rapids_cuda_suffixed_package(name, is_suffixed): - assert alpha_spec.is_rapids_cuda_suffixed_package(name) == is_suffixed +@patch.object( + alpha_spec.all_metadata(), + "get_current_version", + Mock(return_value=latest_metadata), +) +def test_strip_cuda_suffix(name, stripped_name): + assert alpha_spec.strip_cuda_suffix(name) == stripped_name @pytest.mark.parametrize( @@ -71,7 +96,7 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): (p, f"{p}>=0.0.0a0", "development", None), (p, f"{p}>=0.0.0a0", "release", p), ] - for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES + for p in latest_metadata.prerelease_packages ) ), *chain( @@ -82,7 +107,8 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "development", None), (f"{p}-cu11", f"{p}-cu11>=0.0.0a0", "release", f"{p}-cu11"), ] - for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES + for p in latest_metadata.prerelease_packages + & latest_metadata.cuda_suffixed_packages ) ), *chain( @@ -91,7 +117,11 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): (f"{p}-cu12", f"{p}-cu12", "development", None), (f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", None), ] - for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES + for p in latest_metadata.prerelease_packages + & ( + latest_metadata.all_packages + - latest_metadata.cuda_suffixed_packages + ) ) ), ("cuml", "cuml>=24.04,<24.06", "development", "cuml>=24.04,<24.06,>=0.0.0a0"), @@ -109,6 +139,11 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed): (None, "gcc_linux-64=11.*", "release", None), ], ) +@patch.object( + alpha_spec.all_metadata(), + "get_current_version", + Mock(return_value=latest_metadata), +) def test_check_package_spec(package, content, mode, replacement): args = Mock(mode=mode) linter = lint.Linter("dependencies.yaml", content) @@ -134,6 +169,11 @@ def test_check_package_spec(package, content, mode, replacement): assert linter.warnings == expected_linter.warnings +@patch.object( + alpha_spec.all_metadata(), + "get_current_version", + Mock(return_value=latest_metadata), +) def test_check_package_spec_anchor(): CONTENT = dedent( """\ @@ -448,7 +488,7 @@ def test_check_alpha_spec(): ) -def test_check_alpha_spec_integration(): +def test_check_alpha_spec_integration(tmp_path): CONTENT = dedent( """\ dependencies: @@ -463,7 +503,10 @@ def test_check_alpha_spec_integration(): args = Mock(mode="development") linter = lint.Linter("dependencies.yaml", CONTENT) - alpha_spec.check_alpha_spec(linter, args) + with open(os.path.join(tmp_path, "VERSION"), "w") as f: + f.write(f"{latest_version}\n") + with set_cwd(tmp_path): + alpha_spec.check_alpha_spec(linter, args) start = CONTENT.find(REPLACED) end = start + len(REPLACED)