From dd228f9f1bea23b74b17dc0f939ff1b0b15cee4f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 13 Jan 2025 08:03:35 -0600 Subject: [PATCH] pylibcugraph: declare cupy and numpy hard dependencies (#4854) While testing stuff for #4804, I found that `pylibcugraph` has a hard runtime dependency on `cupy` and `numpy`, but isn't declaring them ```shell docker run \ --rm \ --gpus 0 \ -it rapidsai/ci-wheel:latest \ bash python -m pip install 'pylibcugraph-cu12==25.2.*,>=0.0.0a0' python -c "import pylibcugraph" ``` ```text Traceback (most recent call last): File "", line 1, in File "/pyenv/versions/3.12.7/lib/python3.12/site-packages/pylibcugraph/__init__.py", line 18, in from pylibcugraph.graphs import SGGraph, MGGraph File "graphs.pyx", line 1, in init pylibcugraph.graphs File "utils.pyx", line 20, in init pylibcugraph.utils ModuleNotFoundError: No module named 'cupy' ``` This declares those dependencies. It also promotes `cugraph-service-server`'s `numpy` dependency to a hard runtime dependency. ## Notes for Reviewers ### Evidence that `pylibcugraph` *already* has a hard dependency on these libraries They're used unconditionally here: https://github.com/rapidsai/cugraph/blob/cddd69ea3f62cabdb3aa2b7b6676e0b74ab4eefc/python/pylibcugraph/pylibcugraph/utils.pyx#L19-L20 But have import guards in other places: https://github.com/rapidsai/cugraph/blob/cddd69ea3f62cabdb3aa2b7b6676e0b74ab4eefc/python/pylibcugraph/pylibcugraph/sssp.pyx#L127-L139 So this PR doesn't introduce new hard dependencies... it just makes them explicit, to make it easier to install and run `pylibcugraph`. ### How was this not caught in CI? Import tests aren't run here for conda packages, because conda builds happen on CPU-only nodes. https://github.com/rapidsai/cugraph/blob/cddd69ea3f62cabdb3aa2b7b6676e0b74ab4eefc/ci/build_python.sh#L27-L30 And `numpy` and `cupy` are probably getting pulled in by some of the wheels' test dependencies, like `cudf`, here: https://github.com/rapidsai/cugraph/blob/cddd69ea3f62cabdb3aa2b7b6676e0b74ab4eefc/ci/test_wheel.sh#L17 ### Should we just make the other unconditional cases conditional with try-catching? No. Talked with @rlratzel, @ChuckHastings, and @eriknw offline, and we agreed to declare these as hard runtime dependencies (and remove the try-catching in places that had it). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Rick Ratzel (https://github.com/rlratzel) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4854 --- conda/recipes/pylibcugraph/meta.yaml | 4 +++- dependencies.yaml | 14 +++++++++++++- python/cugraph-service/server/pyproject.toml | 4 ---- python/pylibcugraph/pylibcugraph/bfs.pyx | 9 +++------ python/pylibcugraph/pylibcugraph/node2vec.pyx | 12 +++--------- python/pylibcugraph/pylibcugraph/pagerank.pyx | 19 ++++--------------- .../pylibcugraph/personalized_pagerank.pyx | 19 ++++--------------- python/pylibcugraph/pylibcugraph/sssp.pyx | 19 ++++--------------- python/pylibcugraph/pyproject.toml | 2 ++ 9 files changed, 36 insertions(+), 66 deletions(-) diff --git a/conda/recipes/pylibcugraph/meta.yaml b/conda/recipes/pylibcugraph/meta.yaml index 54d29a68d91..e8a0286d2b8 100644 --- a/conda/recipes/pylibcugraph/meta.yaml +++ b/conda/recipes/pylibcugraph/meta.yaml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# Copyright (c) 2023-2025, NVIDIA CORPORATION. {% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') + environ.get('VERSION_SUFFIX', '') %} {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} @@ -74,7 +74,9 @@ requirements: {% else %} - cuda-cudart {% endif %} + - cupy >=12.0.0 - libcugraph ={{ version }} + - numpy>=1.23,<3.0a0 - pylibraft ={{ minor_version }} - python - rmm ={{ minor_version }} diff --git a/dependencies.yaml b/dependencies.yaml index 56c0f9deba0..906ccb24cb9 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -141,8 +141,10 @@ files: table: project includes: - cuda_wheels + - depends_on_cupy - depends_on_pylibraft - depends_on_rmm + - python_run_pylibcugraph py_test_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -199,7 +201,7 @@ files: key: test includes: - test_python_common - - test_python_cugraph + - test_python_cugraph_service_server channels: - rapidsai - rapidsai-nightly @@ -421,6 +423,11 @@ dependencies: - matrix: # All CUDA 11 versions packages: - cuda-python>=11.8.5,<12.0a0 + python_run_pylibcugraph: + common: + - output_types: [conda, pyproject, requirements] + packages: + - *numpy python_run_cugraph_service_client: common: - output_types: [conda, pyproject] @@ -468,6 +475,11 @@ dependencies: - output_types: [conda] packages: - *thrift + test_python_cugraph_service_server: + common: + - output_types: [conda, pyproject] + packages: + - *numpy test_python_pylibcugraph: common: - output_types: [conda, pyproject] diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index 29ee41854f8..ec75af55cb3 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -46,16 +46,12 @@ cugraph-service-server = "cugraph_service_server.__main__:main" [project.optional-dependencies] test = [ - "certifi", - "networkx>=2.5.1", "numpy>=1.23,<3.0a0", "pandas", "pytest", "pytest-benchmark", "pytest-cov", "pytest-xdist", - "python-louvain", - "scikit-learn>=0.23.1", "scipy", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibcugraph/pylibcugraph/bfs.pyx b/python/pylibcugraph/pylibcugraph/bfs.pyx index b92afcfd7db..e6a22109b9e 100644 --- a/python/pylibcugraph/pylibcugraph/bfs.pyx +++ b/python/pylibcugraph/pylibcugraph/bfs.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,8 @@ # Have cython use python 3 syntax # cython: language_level = 3 +import cupy + from libc.stdint cimport uintptr_t from libc.stdint cimport int32_t from libc.limits cimport INT_MAX @@ -141,11 +143,6 @@ def bfs(ResourceHandle handle, _GPUGraph graph, >>> }) """ - try: - import cupy - except ModuleNotFoundError: - raise RuntimeError("bfs requires the cupy package, which could not " - "be imported") assert_CAI_type(sources, "sources") if depth_limit <= 0: diff --git a/python/pylibcugraph/pylibcugraph/node2vec.pyx b/python/pylibcugraph/pylibcugraph/node2vec.pyx index 0e0fd73e6c8..e81afb58061 100644 --- a/python/pylibcugraph/pylibcugraph/node2vec.pyx +++ b/python/pylibcugraph/pylibcugraph/node2vec.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,8 @@ # Have cython use python 3 syntax # cython: language_level = 3 +import cupy + from libc.stdint cimport uintptr_t from pylibcugraph._cugraph_c.types cimport ( @@ -124,14 +126,6 @@ def node2vec(ResourceHandle resource_handle, """ - # FIXME: import these modules here for now until a better pattern can be - # used for optional imports (perhaps 'import_optional()' from cugraph), or - # these are made hard dependencies. - try: - import cupy - except ModuleNotFoundError: - raise RuntimeError("node2vec requires the cupy package, which could not " - "be imported") assert_CAI_type(seed_array, "seed_array") cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ diff --git a/python/pylibcugraph/pylibcugraph/pagerank.pyx b/python/pylibcugraph/pylibcugraph/pagerank.pyx index bcb8474ddfa..961e191e054 100644 --- a/python/pylibcugraph/pylibcugraph/pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/pagerank.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,9 @@ # Have cython use python 3 syntax # cython: language_level = 3 +import cupy +import numpy + from pylibcugraph._cugraph_c.types cimport ( bool_t, ) @@ -167,20 +170,6 @@ def pagerank(ResourceHandle resource_handle, array([0.11615585, 0.21488841, 0.2988108 , 0.3701449 ], dtype=float32) """ - # FIXME: import these modules here for now until a better pattern can be - # used for optional imports (perhaps 'import_optional()' from cugraph), or - # these are made hard dependencies. - try: - import cupy - except ModuleNotFoundError: - raise RuntimeError("pagerank requires the cupy package, which could " - "not be imported") - try: - import numpy - except ModuleNotFoundError: - raise RuntimeError("pagerank requires the numpy package, which could " - "not be imported") - cdef cugraph_type_erased_device_array_view_t* \ initial_guess_vertices_view_ptr = \ create_cugraph_type_erased_device_array_view_from_py_obj( diff --git a/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx b/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx index 209d4054491..1926f41b13d 100644 --- a/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx +++ b/python/pylibcugraph/pylibcugraph/personalized_pagerank.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,9 @@ # Have cython use python 3 syntax # cython: language_level = 3 +import cupy +import numpy + from pylibcugraph._cugraph_c.types cimport ( bool_t, ) @@ -177,20 +180,6 @@ def personalized_pagerank(ResourceHandle resource_handle, array([0.00446455, 0.00379487, 0.53607565, 0.45566472 ], dtype=float32) """ - # FIXME: import these modules here for now until a better pattern can be - # used for optional imports (perhaps 'import_optional()' from cugraph), or - # these are made hard dependencies. - try: - import cupy - except ModuleNotFoundError: - raise RuntimeError("pagerank requires the cupy package, which could " - "not be imported") - try: - import numpy - except ModuleNotFoundError: - raise RuntimeError("pagerank requires the numpy package, which could " - "not be imported") - cdef cugraph_type_erased_device_array_view_t* \ initial_guess_vertices_view_ptr = \ create_cugraph_type_erased_device_array_view_from_py_obj( diff --git a/python/pylibcugraph/pylibcugraph/sssp.pyx b/python/pylibcugraph/pylibcugraph/sssp.pyx index 7e40a801e94..2eff824d4d3 100644 --- a/python/pylibcugraph/pylibcugraph/sssp.pyx +++ b/python/pylibcugraph/pylibcugraph/sssp.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,6 +14,9 @@ # Have cython use python 3 syntax # cython: language_level = 3 +import cupy +import numpy + from pylibcugraph._cugraph_c.types cimport ( bool_t, ) @@ -124,20 +127,6 @@ def sssp(ResourceHandle resource_handle, array([-1, -1, 1, 2], dtype=int32) """ - # FIXME: import these modules here for now until a better pattern can be - # used for optional imports (perhaps 'import_optional()' from cugraph), or - # these are made hard dependencies. - try: - import cupy - except ModuleNotFoundError: - raise RuntimeError("sssp requires the cupy package, which could not " - "be imported") - try: - import numpy - except ModuleNotFoundError: - raise RuntimeError("sssp requires the numpy package, which could not " - "be imported") - if compute_predecessors is False: raise ValueError("compute_predecessors must be True for the current " "release.") diff --git a/python/pylibcugraph/pyproject.toml b/python/pylibcugraph/pyproject.toml index 72a5e19c702..cd98e37d327 100644 --- a/python/pylibcugraph/pyproject.toml +++ b/python/pylibcugraph/pyproject.toml @@ -23,6 +23,8 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ + "cupy-cuda11x>=12.0.0", + "numpy>=1.23,<3.0a0", "nvidia-cublas", "nvidia-curand", "nvidia-cusolver",