Skip to content

Commit

Permalink
Use rapids-metadata (#39)
Browse files Browse the repository at this point in the history
* Use rapids-metadata

rapids-metadata was created so that we wouldn't have to cut a new
pre-commit-hooks release every time the structure of RAPIDS changes.
Use rapids-metadata and get rid of the baked-in list of packages.

* Add nightly index to CI

* Add type hints

* Upgrade to rapids-metadata 0.3.0
KyleFromNVIDIA authored Jun 27, 2024
1 parent de50ecf commit 2aac71d
Showing 5 changed files with 80 additions and 63 deletions.
2 changes: 2 additions & 0 deletions .pre-commit-hooks.yaml
Original file line number Diff line number Diff line change
@@ -18,6 +18,8 @@
language: python
files: dependencies[.]yaml$
args: [--fix]
additional_dependencies:
- --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
- id: verify-conda-yes
name: pass -y/--yes to conda
description: make sure that all calls to conda pass -y/--yes
2 changes: 1 addition & 1 deletion ci/build-test.sh
Original file line number Diff line number Diff line change
@@ -10,6 +10,6 @@ python -m build .
for PKG in dist/*; do
echo "$PKG"
pip uninstall -y rapids-pre-commit-hooks
pip install "$PKG[test]"
pip install --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple "$PKG[test]"
pytest
done
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -36,6 +36,7 @@ dependencies = [
"bashlex",
"gitpython",
"packaging",
"rapids-metadata>=0.3.0,<0.4.0.dev0",
"rich",
"tomlkit",
]
63 changes: 17 additions & 46 deletions src/rapids_pre_commit_hooks/alpha_spec.py
Original file line number Diff line number Diff line change
@@ -12,52 +12,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
from functools import total_ordering
from functools import cache, total_ordering

import yaml
from packaging.requirements import InvalidRequirement, Requirement
from rapids_metadata.remote import fetch_latest

from .lint import LintMain

RAPIDS_ALPHA_SPEC_PACKAGES = {
"cubinlinker",
"cucim",
"cudf",
"cugraph",
"cugraph-dgl",
"cugraph-equivariant",
"cugraph-pyg",
"cuml",
"cuproj",
"cuspatial",
"cuxfilter",
"dask-cuda",
"dask-cudf",
"distributed-ucxx",
"librmm",
"libucx",
"nx-cugraph",
"ptxcompiler",
"pylibcugraph",
"pylibcugraphops",
"pylibraft",
"pylibwholegraph",
"pynvjitlink",
"raft-dask",
"rmm",
"ucx-py",
"ucxx",
}

RAPIDS_NON_CUDA_SUFFIXED_PACKAGES = {
"dask-cuda",
}

RAPIDS_CUDA_SUFFIXED_PACKAGES = (
RAPIDS_ALPHA_SPEC_PACKAGES - RAPIDS_NON_CUDA_SUFFIXED_PACKAGES
)

ALPHA_SPECIFIER = ">=0.0.0a0"

ALPHA_SPEC_OUTPUT_TYPES = {
@@ -68,15 +32,21 @@
CUDA_SUFFIX_REGEX = re.compile(r"^(?P<package>.*)-cu[0-9]{2}$")


@cache
def all_metadata():
return fetch_latest()


def node_has_type(node, tag_type):
return node.tag == f"tag:yaml.org,2002:{tag_type}"


def is_rapids_cuda_suffixed_package(name):
return any(
(match := CUDA_SUFFIX_REGEX.search(name)) and match.group("package") == package
for package in RAPIDS_CUDA_SUFFIXED_PACKAGES
)
def strip_cuda_suffix(name: str) -> str:
if (match := CUDA_SUFFIX_REGEX.search(name)) and match.group(
"package"
) in all_metadata().get_current_version(os.getcwd()).cuda_suffixed_packages:
return match.group("package")
return name


def check_package_spec(linter, args, anchors, used_anchors, node):
@@ -108,8 +78,9 @@ def create_specifier_string(specifiers):
req = Requirement(node.value)
except InvalidRequirement:
return
if req.name in RAPIDS_ALPHA_SPEC_PACKAGES or is_rapids_cuda_suffixed_package(
req.name
if (
strip_cuda_suffix(req.name)
in all_metadata().get_current_version(os.getcwd()).prerelease_packages
):
for key, value in anchors.items():
if value == node:
75 changes: 59 additions & 16 deletions test/rapids_pre_commit_hooks/test_alpha_spec.py
Original file line number Diff line number Diff line change
@@ -12,15 +12,32 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import contextlib
import os.path
from itertools import chain
from textwrap import dedent
from unittest.mock import MagicMock, Mock, call, patch

import pytest
import yaml
from packaging.version import Version

from rapids_pre_commit_hooks import alpha_spec, lint

latest_version, latest_metadata = max(
alpha_spec.all_metadata().versions.items(), key=lambda item: Version(item[0])
)


@contextlib.contextmanager
def set_cwd(cwd):
old_cwd = os.getcwd()
os.chdir(cwd)
try:
yield
finally:
os.chdir(old_cwd)


def test_anchor_preserving_loader():
loader = alpha_spec.AnchorPreservingLoader("- &a A\n- *a")
@@ -32,32 +49,40 @@ def test_anchor_preserving_loader():


@pytest.mark.parametrize(
["name", "is_suffixed"],
["name", "stripped_name"],
[
*chain(
*(
[
(f"{p}-cu11", True),
(f"{p}-cu12", True),
(f"{p}-cuda", False),
(p, p),
(f"{p}-cu11", p),
(f"{p}-cu12", p),
(f"{p}-cuda", f"{p}-cuda"),
]
for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES
for p in latest_metadata.cuda_suffixed_packages
)
),
*chain(
*(
[
(f"{p}-cu11", False),
(f"{p}-cu12", False),
(f"{p}-cuda", False),
(p, p),
(f"{p}-cu11", f"{p}-cu11"),
(f"{p}-cu12", f"{p}-cu12"),
(f"{p}-cuda", f"{p}-cuda"),
]
for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES
for p in latest_metadata.all_packages
- latest_metadata.cuda_suffixed_packages
)
),
],
)
def test_is_rapids_cuda_suffixed_package(name, is_suffixed):
assert alpha_spec.is_rapids_cuda_suffixed_package(name) == is_suffixed
@patch.object(
alpha_spec.all_metadata(),
"get_current_version",
Mock(return_value=latest_metadata),
)
def test_strip_cuda_suffix(name, stripped_name):
assert alpha_spec.strip_cuda_suffix(name) == stripped_name


@pytest.mark.parametrize(
@@ -71,7 +96,7 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed):
(p, f"{p}>=0.0.0a0", "development", None),
(p, f"{p}>=0.0.0a0", "release", p),
]
for p in alpha_spec.RAPIDS_ALPHA_SPEC_PACKAGES
for p in latest_metadata.prerelease_packages
)
),
*chain(
@@ -82,7 +107,8 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed):
(f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "development", None),
(f"{p}-cu11", f"{p}-cu11>=0.0.0a0", "release", f"{p}-cu11"),
]
for p in alpha_spec.RAPIDS_CUDA_SUFFIXED_PACKAGES
for p in latest_metadata.prerelease_packages
& latest_metadata.cuda_suffixed_packages
)
),
*chain(
@@ -91,7 +117,11 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed):
(f"{p}-cu12", f"{p}-cu12", "development", None),
(f"{p}-cu12", f"{p}-cu12>=0.0.0a0", "release", None),
]
for p in alpha_spec.RAPIDS_NON_CUDA_SUFFIXED_PACKAGES
for p in latest_metadata.prerelease_packages
& (
latest_metadata.all_packages
- latest_metadata.cuda_suffixed_packages
)
)
),
("cuml", "cuml>=24.04,<24.06", "development", "cuml>=24.04,<24.06,>=0.0.0a0"),
@@ -109,6 +139,11 @@ def test_is_rapids_cuda_suffixed_package(name, is_suffixed):
(None, "gcc_linux-64=11.*", "release", None),
],
)
@patch.object(
alpha_spec.all_metadata(),
"get_current_version",
Mock(return_value=latest_metadata),
)
def test_check_package_spec(package, content, mode, replacement):
args = Mock(mode=mode)
linter = lint.Linter("dependencies.yaml", content)
@@ -134,6 +169,11 @@ def test_check_package_spec(package, content, mode, replacement):
assert linter.warnings == expected_linter.warnings


@patch.object(
alpha_spec.all_metadata(),
"get_current_version",
Mock(return_value=latest_metadata),
)
def test_check_package_spec_anchor():
CONTENT = dedent(
"""\
@@ -448,7 +488,7 @@ def test_check_alpha_spec():
)


def test_check_alpha_spec_integration():
def test_check_alpha_spec_integration(tmp_path):
CONTENT = dedent(
"""\
dependencies:
@@ -463,7 +503,10 @@ def test_check_alpha_spec_integration():

args = Mock(mode="development")
linter = lint.Linter("dependencies.yaml", CONTENT)
alpha_spec.check_alpha_spec(linter, args)
with open(os.path.join(tmp_path, "VERSION"), "w") as f:
f.write(f"{latest_version}\n")
with set_cwd(tmp_path):
alpha_spec.check_alpha_spec(linter, args)

start = CONTENT.find(REPLACED)
end = start + len(REPLACED)

0 comments on commit 2aac71d

Please sign in to comment.