From 2d32c86fefa88de7c7f29ac9b04a204f4ef027c8 Mon Sep 17 00:00:00 2001 From: Ralph Liu <137829296+nv-rliu@users.noreply.github.com> Date: Wed, 11 Dec 2024 23:40:23 +0900 Subject: [PATCH 1/8] Fix SSL Error (#4825) Addresses errors in `wheel-tests` seen [here](https://github.com/rapidsai/cugraph/issues/4818) Installs SSL certificates with `certifi` as outlined in this issue [here](https://github.com/rapidsai/build-infra/issues/56) --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-125_arch-x86_64.yaml | 1 + dependencies.yaml | 2 ++ python/cugraph-service/server/pyproject.toml | 1 + python/cugraph/cugraph/datasets/dataset.py | 15 ++++++++------- python/cugraph/cugraph/testing/resultset.py | 3 --- python/cugraph/cugraph/tests/conftest.py | 13 ++++++++++++- python/cugraph/pyproject.toml | 1 + 8 files changed, 26 insertions(+), 11 deletions(-) mode change 100644 => 100755 dependencies.yaml diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c045db32c65..23d09269a90 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -10,6 +10,7 @@ dependencies: - aiohttp - breathe - c-compiler +- certifi - cmake>=3.26.4,!=3.30.0 - cuda-nvtx - cuda-version=11.8 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 333d9dc8c2f..afd4dec41c7 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -10,6 +10,7 @@ dependencies: - aiohttp - breathe - c-compiler +- certifi - cmake>=3.26.4,!=3.30.0 - cuda-cudart-dev - cuda-nvcc diff --git a/dependencies.yaml b/dependencies.yaml old mode 100644 new mode 100755 index d081655f8a9..142fe0c987e --- a/dependencies.yaml +++ b/dependencies.yaml @@ -470,6 +470,7 @@ dependencies: common: - output_types: [conda, requirements] packages: + - certifi - ipython - notebook>=0.5.0 - output_types: [conda] @@ -489,6 +490,7 @@ dependencies: common: - output_types: [conda, pyproject] packages: + - certifi - networkx>=2.5.1 - *numpy - python-louvain diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index f388fd4c126..caafd1fe507 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -46,6 +46,7 @@ cugraph-service-server = "cugraph_service_server.__main__:main" [project.optional-dependencies] test = [ + "certifi", "networkx>=2.5.1", "numpy>=1.23,<3.0a0", "pandas", diff --git a/python/cugraph/cugraph/datasets/dataset.py b/python/cugraph/cugraph/datasets/dataset.py index 63389cbc16a..763ae8033f5 100644 --- a/python/cugraph/cugraph/datasets/dataset.py +++ b/python/cugraph/cugraph/datasets/dataset.py @@ -11,14 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import cudf -import dask_cudf -import yaml import os import pandas as pd -import cugraph.dask as dcg +import yaml from pathlib import Path -import urllib.request +from urllib.request import urlretrieve + +import cudf +import cugraph.dask as dcg +import dask_cudf from cugraph.structure.graph_classes import Graph @@ -142,7 +143,7 @@ def __download_csv(self, url): filename = self.metadata["name"] + self.metadata["file_type"] if self._dl_path.path.is_dir(): self._path = self._dl_path.path / filename - urllib.request.urlretrieve(url, str(self._path)) + urlretrieve(url, str(self._path)) else: raise RuntimeError( @@ -458,7 +459,7 @@ def download_all(force=False): filename = meta["name"] + meta["file_type"] save_to = default_download_dir.path / filename if not save_to.is_file() or force: - urllib.request.urlretrieve(meta["url"], str(save_to)) + urlretrieve(meta["url"], str(save_to)) def set_download_dir(path): diff --git a/python/cugraph/cugraph/testing/resultset.py b/python/cugraph/cugraph/testing/resultset.py index f557ad13089..f7d2521752c 100644 --- a/python/cugraph/cugraph/testing/resultset.py +++ b/python/cugraph/cugraph/testing/resultset.py @@ -13,7 +13,6 @@ import warnings import tarfile - import urllib.request import cudf @@ -22,8 +21,6 @@ default_download_dir, ) -# results_dir_path = utils.RAPIDS_DATASET_ROOT_DIR_PATH / "tests" / "resultsets" - class Resultset: """ diff --git a/python/cugraph/cugraph/tests/conftest.py b/python/cugraph/cugraph/tests/conftest.py index d31c2968afe..101a4e6a192 100644 --- a/python/cugraph/cugraph/tests/conftest.py +++ b/python/cugraph/cugraph/tests/conftest.py @@ -23,9 +23,20 @@ # Avoid timeout during shutdown from dask_cuda.utils_test import IncreasedCloseTimeoutNanny -# module-wide fixtures +import certifi +from ssl import create_default_context +from urllib.request import build_opener, HTTPSHandler, install_opener + + +# Install SSL certificates +def pytest_sessionstart(session): + ssl_context = create_default_context(cafile=certifi.where()) + https_handler = HTTPSHandler(context=ssl_context) + install_opener(build_opener(https_handler)) +# module-wide fixtures + # Spoof the gpubenchmark fixture if it's not available so that asvdb and # rapids-pytest-benchmark do not need to be installed to run tests. if "gpubenchmark" not in globals(): diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index d3960ab5d32..bb5b5256603 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -46,6 +46,7 @@ classifiers = [ [project.optional-dependencies] test = [ + "certifi", "networkx>=2.5.1", "numpy>=1.23,<3.0a0", "pandas", From 1f9e3eb5afc302b3462f7bcd159240ac5a660e7f Mon Sep 17 00:00:00 2001 From: Ray Douglass Date: Wed, 11 Dec 2024 13:10:58 -0500 Subject: [PATCH 2/8] Update Changelog [skip ci] --- CHANGELOG.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d934273d0a7..c6320c3527f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,79 @@ +# cugraph 24.12.00 (11 Dec 2024) + +## 🚨 Breaking Changes + +- Fix SSL Error ([#4825](https://github.com/rapidsai/cugraph/pull/4825)) [@nv-rliu](https://github.com/nv-rliu) +- [BUG] Set Max PyTorch Version, Skip 11.4 Tests Using WholeGraph ([#4808](https://github.com/rapidsai/cugraph/pull/4808)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- Remove GNN Packages ([#4765](https://github.com/rapidsai/cugraph/pull/4765)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- remove cugraph-dgl and cugraph-pyg building/testing/publishing ([#4752](https://github.com/rapidsai/cugraph/pull/4752)) [@jameslamb](https://github.com/jameslamb) +- Remove `nx-cugraph` build/test/publish from cugraph ([#4748](https://github.com/rapidsai/cugraph/pull/4748)) [@nv-rliu](https://github.com/nv-rliu) +- Remove CMake/C++ references to cugraph-ops ([#4744](https://github.com/rapidsai/cugraph/pull/4744)) [@ChuckHastings](https://github.com/ChuckHastings) +- Delete the deprecated data_type_id_t enum ([#4737](https://github.com/rapidsai/cugraph/pull/4737)) [@ChuckHastings](https://github.com/ChuckHastings) +- Don't compile int32_t/int64_t vertex_t/edge_t combinations ([#4720](https://github.com/rapidsai/cugraph/pull/4720)) [@ChuckHastings](https://github.com/ChuckHastings) +- Remove deprecated C API functions for graph creation/graph free ([#4718](https://github.com/rapidsai/cugraph/pull/4718)) [@ChuckHastings](https://github.com/ChuckHastings) + +## 🐛 Bug Fixes + +- Fix SSL Error ([#4825](https://github.com/rapidsai/cugraph/pull/4825)) [@nv-rliu](https://github.com/nv-rliu) +- [BUG] Set Max PyTorch Version, Skip 11.4 Tests Using WholeGraph ([#4808](https://github.com/rapidsai/cugraph/pull/4808)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- Fix CCCL 2.7.0-rc2 compile issue by removing reference from values. ([#4799](https://github.com/rapidsai/cugraph/pull/4799)) [@bdice](https://github.com/bdice) +- Re-enable balanced edge cut test ([#4781](https://github.com/rapidsai/cugraph/pull/4781)) [@ChuckHastings](https://github.com/ChuckHastings) +- Fix debug build failure ([#4774](https://github.com/rapidsai/cugraph/pull/4774)) [@seunghwak](https://github.com/seunghwak) +- Add sphinx-lint pre-commit and some docs fixes ([#4771](https://github.com/rapidsai/cugraph/pull/4771)) [@eriknw](https://github.com/eriknw) +- Fix improper initialization of degree_type ([#4755](https://github.com/rapidsai/cugraph/pull/4755)) [@ChuckHastings](https://github.com/ChuckHastings) +- Put a ceiling on cuda-python ([#4747](https://github.com/rapidsai/cugraph/pull/4747)) [@bdice](https://github.com/bdice) +- Fix MG similarity issues ([#4741](https://github.com/rapidsai/cugraph/pull/4741)) [@ChuckHastings](https://github.com/ChuckHastings) +- Address Leiden clustering generating too many clusters ([#4730](https://github.com/rapidsai/cugraph/pull/4730)) [@ChuckHastings](https://github.com/ChuckHastings) +- Add support for storing results for all k-values ([#4728](https://github.com/rapidsai/cugraph/pull/4728)) [@nv-rliu](https://github.com/nv-rliu) +- Remove `--collect-only` Option Left by Accident ([#4727](https://github.com/rapidsai/cugraph/pull/4727)) [@nv-rliu](https://github.com/nv-rliu) +- Update nx-cugraph to NetworkX 3.4 ([#4717](https://github.com/rapidsai/cugraph/pull/4717)) [@eriknw](https://github.com/eriknw) +- Symmetrize edgelist when creating a CSR graph ([#4716](https://github.com/rapidsai/cugraph/pull/4716)) [@jnke2016](https://github.com/jnke2016) + +## 📖 Documentation + +- nx-cugraph: add `bipartite_betweenness_centrality` to supported algorithms ([#4778](https://github.com/rapidsai/cugraph/pull/4778)) [@eriknw](https://github.com/eriknw) +- Notebook to run Centrality against PatentsView data for Medium Blog ([#4769](https://github.com/rapidsai/cugraph/pull/4769)) [@acostadon](https://github.com/acostadon) +- Drop support for NetworkX 3.0 and 3.1 for nx-cugraph ([#4766](https://github.com/rapidsai/cugraph/pull/4766)) [@eriknw](https://github.com/eriknw) +- Update nx-cugraph Docs URLs ([#4761](https://github.com/rapidsai/cugraph/pull/4761)) [@nv-rliu](https://github.com/nv-rliu) +- Update `nx-cugraph` README with New Env Variable ([#4705](https://github.com/rapidsai/cugraph/pull/4705)) [@nv-rliu](https://github.com/nv-rliu) + +## 🚀 New Features + +- [FEA] Support Edge ID Lookup in PyLibcuGraph ([#4687](https://github.com/rapidsai/cugraph/pull/4687)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) + +## 🛠️ Improvements + +- Increase max_iterations in MG HITS TEST ([#4783](https://github.com/rapidsai/cugraph/pull/4783)) [@seunghwak](https://github.com/seunghwak) +- Updates READMEs, updates `core_number` to properly ignore `degree_type`, minor cleanup ([#4776](https://github.com/rapidsai/cugraph/pull/4776)) [@rlratzel](https://github.com/rlratzel) +- Remove edge renumber map from the homogeneous sampling API ([#4775](https://github.com/rapidsai/cugraph/pull/4775)) [@jnke2016](https://github.com/jnke2016) +- Remove GNN Packages ([#4765](https://github.com/rapidsai/cugraph/pull/4765)) [@alexbarghi-nv](https://github.com/alexbarghi-nv) +- Remove cugraph-equivariance ([#4762](https://github.com/rapidsai/cugraph/pull/4762)) [@BradReesWork](https://github.com/BradReesWork) +- deprecate NetworkX Graph as input ([#4759](https://github.com/rapidsai/cugraph/pull/4759)) [@BradReesWork](https://github.com/BradReesWork) +- Remove `nx-cugraph` from this repo ([#4756](https://github.com/rapidsai/cugraph/pull/4756)) [@eriknw](https://github.com/eriknw) +- enforce wheel size limits, README formatting in CI ([#4754](https://github.com/rapidsai/cugraph/pull/4754)) [@jameslamb](https://github.com/jameslamb) +- remove cugraph-dgl and cugraph-pyg building/testing/publishing ([#4752](https://github.com/rapidsai/cugraph/pull/4752)) [@jameslamb](https://github.com/jameslamb) +- Primitives & BFS performance improvements ([#4751](https://github.com/rapidsai/cugraph/pull/4751)) [@seunghwak](https://github.com/seunghwak) +- extract the edgelist from the graph ([#4750](https://github.com/rapidsai/cugraph/pull/4750)) [@jnke2016](https://github.com/jnke2016) +- Remove `nx-cugraph` build/test/publish from cugraph ([#4748](https://github.com/rapidsai/cugraph/pull/4748)) [@nv-rliu](https://github.com/nv-rliu) +- Remove CMake/C++ references to cugraph-ops ([#4744](https://github.com/rapidsai/cugraph/pull/4744)) [@ChuckHastings](https://github.com/ChuckHastings) +- add telemetry ([#4740](https://github.com/rapidsai/cugraph/pull/4740)) [@msarahan](https://github.com/msarahan) +- Delete the deprecated data_type_id_t enum ([#4737](https://github.com/rapidsai/cugraph/pull/4737)) [@ChuckHastings](https://github.com/ChuckHastings) +- Updates README with new dataset, removes mention of script no longer used ([#4736](https://github.com/rapidsai/cugraph/pull/4736)) [@rlratzel](https://github.com/rlratzel) +- devcontainer: replace `VAULT_HOST` with `AWS_ROLE_ARN` ([#4732](https://github.com/rapidsai/cugraph/pull/4732)) [@jjacobelli](https://github.com/jjacobelli) +- Adds new dataset for benchmarking in the 100k node 1M edge range, adds additional k-values for BC benchmarks ([#4726](https://github.com/rapidsai/cugraph/pull/4726)) [@rlratzel](https://github.com/rlratzel) +- re-run all CI when files in .github/workflows change ([#4723](https://github.com/rapidsai/cugraph/pull/4723)) [@jameslamb](https://github.com/jameslamb) +- Don't compile int32_t/int64_t vertex_t/edge_t combinations ([#4720](https://github.com/rapidsai/cugraph/pull/4720)) [@ChuckHastings](https://github.com/ChuckHastings) +- print sccache stats in builds ([#4719](https://github.com/rapidsai/cugraph/pull/4719)) [@jameslamb](https://github.com/jameslamb) +- Remove deprecated C API functions for graph creation/graph free ([#4718](https://github.com/rapidsai/cugraph/pull/4718)) [@ChuckHastings](https://github.com/ChuckHastings) +- Clean up the use of cuco hash functions ([#4707](https://github.com/rapidsai/cugraph/pull/4707)) [@PointKernel](https://github.com/PointKernel) +- Relax PyTorch upper bound (allowing 2.4) ([#4703](https://github.com/rapidsai/cugraph/pull/4703)) [@jakirkham](https://github.com/jakirkham) +- combine pip install calls in wheel-testing scripts ([#4701](https://github.com/rapidsai/cugraph/pull/4701)) [@jameslamb](https://github.com/jameslamb) +- Merge branch-24.10 into branch-24.12 ([#4697](https://github.com/rapidsai/cugraph/pull/4697)) [@jameslamb](https://github.com/jameslamb) +- Merge branch-24.10 into branch-24.12 ([#4682](https://github.com/rapidsai/cugraph/pull/4682)) [@jameslamb](https://github.com/jameslamb) +- Update all rmm imports to use pylibrmm/librmm ([#4671](https://github.com/rapidsai/cugraph/pull/4671)) [@Matt711](https://github.com/Matt711) +- Prune workflows based on changed files ([#4634](https://github.com/rapidsai/cugraph/pull/4634)) [@KyleFromNVIDIA](https://github.com/KyleFromNVIDIA) +- support heterogenous fanout type ([#4608](https://github.com/rapidsai/cugraph/pull/4608)) [@jnke2016](https://github.com/jnke2016) + # cugraph 24.10.00 (9 Oct 2024) ## 🚨 Breaking Changes From d2f53365d982b28e4eb023824cd7a91bf9c1e281 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Thu, 12 Dec 2024 10:42:12 -0600 Subject: [PATCH 3/8] gate telemetry dispatch calls on TELEMETRY_ENABLED env var (#4816) Because of the switch away from certificates/mTLS, we are having to rework a few things. In the meantime, telemetry jobs are failing. This PR adds a switch to turn all of the telemetry stuff off - to skip it instead. It is meant to be controlled by an org-wide environment variable, which can be applied to individual repos by ops. At the time of submitting this PR, the environment variable is 'false' and no telemetry is being reported. Authors: - Mike Sarahan (https://github.com/msarahan) - Ralph Liu (https://github.com/nv-rliu) Approvers: - Jake Awe (https://github.com/AyodeAwe) URL: https://github.com/rapidsai/cugraph/pull/4816 --- .github/workflows/pr.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c9e41475a1e..a018544c96e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -40,6 +40,7 @@ jobs: OTEL_SERVICE_NAME: "pr-cugraph" steps: - name: Telemetry setup + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main changed-files: secrets: inherit @@ -181,7 +182,7 @@ jobs: telemetry-summarize: runs-on: ubuntu-latest needs: pr-builder - if: always() + if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} continue-on-error: true steps: - name: Load stashed telemetry env vars From 3ad4949f2e119f881361500172a04d2d4b243f31 Mon Sep 17 00:00:00 2001 From: Don Acosta <97529984+acostadon@users.noreply.github.com> Date: Fri, 13 Dec 2024 09:07:24 -0500 Subject: [PATCH 4/8] Changed ci build_docs to just upload the xml so the cugraph-docs can create the docs (#4662) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … create the docs --------- Co-authored-by: Jake Awe Co-authored-by: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> --- ci/build_docs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index ad77c9a0e60..46a6005b8dc 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -63,6 +63,8 @@ rapids-logger "Build CPP docs" pushd cpp/doxygen doxygen Doxyfile export XML_DIR_LIBCUGRAPH="$(pwd)/xml" +mkdir -p "${RAPIDS_DOCS_DIR}/libcugraph/xml_tar" +tar -czf "${RAPIDS_DOCS_DIR}/libcugraph/xml_tar"/xml.tar.gz -C xml . popd rapids-logger "Build Python docs" From 9b28c88bd49696e9507523805b7ba2d39153cbae Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Wed, 18 Dec 2024 00:27:34 -0600 Subject: [PATCH 5/8] update telemetry actions to fluent-bit friendly style (#4834) Simplifies telemetry a bit. More details at https://github.com/rapidsai/shared-actions/pull/28. Telemetry will still not be collected until @ajschmidt8 enables the `TELEMETRY_ENABLED` environment variable for this repo. Authors: - Mike Sarahan (https://github.com/msarahan) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4834 --- .github/workflows/pr.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a018544c96e..aea81c152e0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -180,16 +180,11 @@ jobs: build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; sccache -s; telemetry-summarize: - runs-on: ubuntu-latest + # This job must use a self-hosted runner to record telemetry traces. + runs-on: linux-amd64-cpu4 needs: pr-builder if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} continue-on-error: true steps: - - name: Load stashed telemetry env vars - uses: rapidsai/shared-actions/telemetry-dispatch-load-base-env-vars@main - with: - load_service_name: true - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-write-summary@main - with: - cert_concat: "${{ secrets.OTEL_EXPORTER_OTLP_CA_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE }};${{ secrets.OTEL_EXPORTER_OTLP_CLIENT_KEY }}" + uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main From 6783aec7a56f05d5088073c822140bc0cd7d9665 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 18 Dec 2024 09:54:48 -0600 Subject: [PATCH 6/8] Add cuda-python dependency (#4829) cuGraph depends on `cuda-python` but does not currently list that dependency. This PR adds it. Authors: - Bradley Dice (https://github.com/bdice) - Ralph Liu (https://github.com/nv-rliu) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cugraph/pull/4829 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-125_arch-x86_64.yaml | 1 + dependencies.yaml | 9 +++++++++ python/cugraph/pyproject.toml | 1 + 4 files changed, 12 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index d98db2232e2..c5f683afd07 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -13,6 +13,7 @@ dependencies: - certifi - cmake>=3.26.4,!=3.30.0 - cuda-nvtx +- cuda-python>=11.8.5,<12.0a0 - cuda-version=11.8 - cudatoolkit - cudf==25.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 34bef310fed..21eda1c0988 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -16,6 +16,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api +- cuda-python>=12.6.2,<13.0a0 - cuda-version=12.5 - cudf==25.2.*,>=0.0.0a0 - cupy>=12.0.0 diff --git a/dependencies.yaml b/dependencies.yaml index b271abc6276..9ee90314409 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -401,6 +401,15 @@ dependencies: # dataset APIs require [http] extras for use with cudf. - fsspec[http]>=0.6.0 specific: + - output_types: [conda, requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - cuda-python>=12.6.2,<13.0a0 + - matrix: # All CUDA 11 versions + packages: + - cuda-python>=11.8.5,<12.0a0 - output_types: pyproject matrices: - matrix: diff --git a/python/cugraph/pyproject.toml b/python/cugraph/pyproject.toml index 481f151ac22..8fa3a0938e8 100644 --- a/python/cugraph/pyproject.toml +++ b/python/cugraph/pyproject.toml @@ -23,6 +23,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ + "cuda-python>=11.8.5,<12.0a0", "cudf==25.2.*,>=0.0.0a0", "cupy-cuda11x>=12.0.0", "dask-cuda==25.2.*,>=0.0.0a0", From 3fdaba6f711e16fa59264906bc42df1f1bfd072c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 20 Dec 2024 17:29:23 -0500 Subject: [PATCH 7/8] remove 'wget' conda dependency, re-organize dependencies.yaml (#4805) In #4804, I've started working on adding `libcugraph` wheels. This includes a few fixes for things I noticed while doing that: * removes `wget` from `test_notebook` environment - *our CI images already have this system-installed, and removing it helps to remove it from the environment solve for the unified RAPIDS devcontainers* * `dependencies.yaml` re-organization: - breaks `librmm` dependency out into a `depends_on_librmm` to reduce duplication, and for consistency with other RAPIDS dependencies - uses `depends_on_pylibwholegraph` group everywhere instead of repeating `pylibwholegraph` in multiple places - removes unused YAML anchors - alphabetizes lists ## Notes for Reviewers I'd originally wanted to also add a `librmm` wheel dependency for wheel builds here, but looks like doing that resulted in a lot more `sccache` misses, I guess because of building with build isolation. That change will happen in #4804 . Authors: - James Lamb (https://github.com/jameslamb) - Ralph Liu (https://github.com/nv-rliu) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4805 --- ci/release/update-version.sh | 1 - .../all_cuda-118_arch-x86_64.yaml | 4 +- .../all_cuda-125_arch-x86_64.yaml | 4 +- conda/recipes/libcugraph/meta.yaml | 2 - dependencies.yaml | 197 +++++++++++------- python/cugraph-service/client/pyproject.toml | 2 +- python/cugraph-service/server/pyproject.toml | 2 +- 7 files changed, 128 insertions(+), 84 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a73745f2c0e..2890011bc7f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -58,7 +58,6 @@ DEPENDENCIES=( dask-cudf libcudf libraft - libraft-headers librmm pylibcugraph pylibwholegraph diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index c5f683afd07..7b2fddb7422 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -28,7 +28,6 @@ dependencies: - graphviz - ipython - libcudf==25.2.*,>=0.0.0a0 -- libraft-headers==25.2.*,>=0.0.0a0 - libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx @@ -68,11 +67,10 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2!=0.5.0,!=0.5.1 +- thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata - torchmetrics - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 -- wget - wheel name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 21eda1c0988..87a2417bbc4 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -34,7 +34,6 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libraft-headers==25.2.*,>=0.0.0a0 - libraft==25.2.*,>=0.0.0a0 - librmm==25.2.*,>=0.0.0a0 - nbsphinx @@ -73,11 +72,10 @@ dependencies: - sphinx-markdown-tables - sphinx<6 - sphinxcontrib-websupport -- thriftpy2!=0.5.0,!=0.5.1 +- thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata - torchmetrics - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 -- wget - wheel name: all_cuda-125_arch-x86_64 diff --git a/conda/recipes/libcugraph/meta.yaml b/conda/recipes/libcugraph/meta.yaml index 922d5e577b2..eb948e08d14 100644 --- a/conda/recipes/libcugraph/meta.yaml +++ b/conda/recipes/libcugraph/meta.yaml @@ -70,7 +70,6 @@ requirements: - doxygen {{ doxygen_version }} - libcudf ={{ minor_version }} - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu @@ -114,7 +113,6 @@ outputs: - libcusparse {% endif %} - libraft ={{ minor_version }} - - libraft-headers ={{ minor_version }} - librmm ={{ minor_version }} - nccl {{ nccl_version }} - ucx-proc=*=gpu diff --git a/dependencies.yaml b/dependencies.yaml index 9ee90314409..eb4bb68dc9d 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -11,18 +11,23 @@ files: - cpp_build - cuda - cuda_version - - docs - - python_build_rapids - - python_build_wheel - - python_build_cythonize - - depends_on_rmm - depends_on_cudf + - depends_on_cupy + - depends_on_dask_cuda - depends_on_dask_cudf + - depends_on_libcudf + - depends_on_libraft + - depends_on_librmm - depends_on_pylibraft - - depends_on_raft_dask - depends_on_pylibwholegraph - - depends_on_cupy - depends_on_pytorch + - depends_on_raft_dask + - depends_on_rmm + - depends_on_ucx_py + - docs + - python_build_cythonize + - python_build_rapids + - python_build_wheel - python_run_cugraph - test_notebook - test_python_common @@ -48,6 +53,7 @@ files: output: none includes: - cuda_version + - depends_on_pylibwholegraph - py_version - test_notebook - test_python_common @@ -69,8 +75,8 @@ files: extras: table: build-system includes: - - python_build_rapids - python_build_cythonize + - python_build_rapids py_rapids_build_cugraph: output: pyproject pyproject_dir: python/cugraph @@ -79,21 +85,23 @@ files: key: requires includes: - common_build - - depends_on_rmm - - depends_on_pylibraft - depends_on_pylibcugraph + - depends_on_pylibraft + - depends_on_rmm py_run_cugraph: output: pyproject pyproject_dir: python/cugraph extras: table: project includes: - - depends_on_rmm - depends_on_cudf + - depends_on_cupy + - depends_on_dask_cuda - depends_on_dask_cudf - - depends_on_raft_dask - depends_on_pylibcugraph - - depends_on_cupy + - depends_on_raft_dask + - depends_on_rmm + - depends_on_ucx_py - python_run_cugraph py_test_cugraph: output: pyproject @@ -102,17 +110,17 @@ files: table: project.optional-dependencies key: test includes: + - depends_on_pylibwholegraph - test_python_common - test_python_cugraph - - depends_on_pylibwholegraph py_build_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph extras: table: build-system includes: - - python_build_rapids - python_build_cythonize + - python_build_rapids py_rapids_build_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -121,8 +129,8 @@ files: key: requires includes: - common_build - - depends_on_rmm - depends_on_pylibraft + - depends_on_rmm py_run_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -130,8 +138,8 @@ files: table: project includes: - cuda_wheels - - depends_on_rmm - depends_on_pylibraft + - depends_on_rmm py_test_pylibcugraph: output: pyproject pyproject_dir: python/pylibcugraph @@ -171,10 +179,14 @@ files: extras: table: project includes: - - depends_on_rmm - depends_on_cudf - - depends_on_dask_cudf + - depends_on_cugraph + - depends_on_cugraph_service_client - depends_on_cupy + - depends_on_dask_cuda + - depends_on_dask_cudf + - depends_on_rmm + - depends_on_ucx_py - python_run_cugraph_service_server py_test_cugraph_service_server: output: pyproject @@ -290,10 +302,6 @@ dependencies: packages: - c-compiler - cxx-compiler - - libcudf==25.2.*,>=0.0.0a0 - - libraft-headers==25.2.*,>=0.0.0a0 - - libraft==25.2.*,>=0.0.0a0 - - librmm==25.2.*,>=0.0.0a0 - openmpi # Required for building cpp-mgtests (multi-GPU tests) specific: - output_types: [conda] @@ -384,7 +392,6 @@ dependencies: - output_types: [conda, pyproject] packages: - &dask rapids-dask-dependency==25.2.*,>=0.0.0a0 - - &dask_cuda dask-cuda==25.2.*,>=0.0.0a0 - &numba numba>=0.57 - &numpy numpy>=1.23,<3.0a0 - output_types: conda @@ -394,7 +401,6 @@ dependencies: - requests - nccl>=2.19 - ucx-proc=*=gpu - - &ucx_py_unsuffixed ucx-py==0.42.*,>=0.0.0a0 - output_types: pyproject packages: # cudf uses fsspec but is protocol independent. cugraph @@ -410,60 +416,19 @@ dependencies: - matrix: # All CUDA 11 versions packages: - cuda-python>=11.8.5,<12.0a0 - - output_types: pyproject - matrices: - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &ucx_py_cu11 ucx-py-cu11==0.42.*,>=0.0.0a0 - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &ucx_py_cu12 ucx-py-cu12==0.42.*,>=0.0.0a0 - - matrix: - packages: - - *ucx_py_unsuffixed python_run_cugraph_service_client: common: - output_types: [conda, pyproject] packages: - - &thrift thriftpy2!=0.5.0,!=0.5.1 + - &thrift thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 python_run_cugraph_service_server: common: - output_types: [conda, pyproject] packages: - *dask - - *dask_cuda - *numba - *numpy - *thrift - - output_types: conda - packages: - - *ucx_py_unsuffixed - specific: - - output_types: pyproject - matrices: - - matrix: - cuda: "11.*" - cuda_suffixed: "true" - packages: - - &cugraph_cu11 cugraph-cu11==25.2.*,>=0.0.0a0 - - cugraph-service-client-cu11==25.2.*,>=0.0.0a0 - - *ucx_py_cu11 - - matrix: - cuda: "12.*" - cuda_suffixed: "true" - packages: - - &cugraph_cu12 cugraph-cu12==25.2.*,>=0.0.0a0 - - cugraph-service-client-cu12==25.2.*,>=0.0.0a0 - - *ucx_py_cu12 - - matrix: - packages: - - &cugraph_unsuffixed cugraph==25.2.*,>=0.0.0a0 - - cugraph-service-client==25.2.*,>=0.0.0a0 - - *ucx_py_unsuffixed test_cpp: common: - output_types: conda @@ -476,9 +441,6 @@ dependencies: - certifi - ipython - notebook>=0.5.0 - - output_types: [conda] - packages: - - wget test_python_common: common: - output_types: [conda, pyproject] @@ -500,7 +462,6 @@ dependencies: - scikit-learn>=0.23.1 - output_types: [conda] packages: - - &pylibwholegraph_unsuffixed pylibwholegraph==25.2.*,>=0.0.0a0 - *thrift test_python_pylibcugraph: common: @@ -508,11 +469,47 @@ dependencies: packages: - *numpy + depends_on_cugraph: + common: + - output_types: conda + packages: + - &cugraph_unsuffixed cugraph==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - cugraph-cu12==25.2.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - cugraph-cu11==25.2.*,>=0.0.0a0 + - {matrix: null, packages: [*cugraph_unsuffixed]} + + depends_on_cugraph_service_client: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cugraph-service-client==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + depends_on_pytorch: common: - output_types: [conda] packages: - - &pytorch_conda pytorch>=2.3,<2.5a0 + - pytorch>=2.3,<2.5a0 - torchdata - pydantic - ogb @@ -544,7 +541,7 @@ dependencies: common: - output_types: conda packages: - - *pylibwholegraph_unsuffixed + - &pylibwholegraph_unsuffixed pylibwholegraph==25.2.*,>=0.0.0a0 - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file @@ -565,6 +562,24 @@ dependencies: - pylibwholegraph-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_libcudf: + common: + - output_types: conda + packages: + - libcudf==25.2.*,>=0.0.0a0 + + depends_on_libraft: + common: + - output_types: conda + packages: + - libraft==25.2.*,>=0.0.0a0 + + depends_on_librmm: + common: + - output_types: conda + packages: + - librmm==25.2.*,>=0.0.0a0 + depends_on_rmm: common: - output_types: conda @@ -615,6 +630,17 @@ dependencies: - cudf-cu11==25.2.*,>=0.0.0a0 - {matrix: null, packages: [*cudf_unsuffixed]} + depends_on_dask_cuda: + common: + - output_types: [conda, pyproject, requirements] + packages: + - dask-cuda==25.2.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + depends_on_dask_cudf: common: - output_types: conda @@ -733,3 +759,28 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + + depends_on_ucx_py: + common: + - output_types: conda + packages: + - &ucx_py_unsuffixed ucx-py==0.42.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - ucx-py-cu12==0.42.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - ucx-py-cu11==0.42.*,>=0.0.0a0 + - {matrix: null, packages: [*ucx_py_unsuffixed]} diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index ac5e6bad0d5..093a578db2b 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -20,7 +20,7 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.10" dependencies = [ - "thriftpy2!=0.5.0,!=0.5.1", + "thriftpy2>=0.4.15,!=0.5.0,!=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index adb18f57216..29ee41854f8 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "numpy>=1.23,<3.0a0", "rapids-dask-dependency==25.2.*,>=0.0.0a0", "rmm==25.2.*,>=0.0.0a0", - "thriftpy2!=0.5.0,!=0.5.1", + "thriftpy2>=0.4.15,!=0.5.0,!=0.5.1", "ucx-py==0.42.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ From 53afaba88e6138d9e309e351849ba5fe0d5e21ea Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 20 Dec 2024 22:59:02 -0800 Subject: [PATCH 8/8] Remove sphinx pinning (#4839) This pinning was needed for [historical reasons that are no longer relevant](https://github.com/rapidsai/cucim/pull/481/commits/5cbc80b3a7fa49f25959674d3a97e6d8cf503fcb). Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cugraph/pull/4839 --- conda/environments/all_cuda-118_arch-x86_64.yaml | 5 +++-- conda/environments/all_cuda-125_arch-x86_64.yaml | 5 +++-- dependencies.yaml | 6 ++++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7b2fddb7422..27b3779bac7 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -8,7 +8,6 @@ channels: - nvidia dependencies: - aiohttp -- breathe - c-compiler - certifi - cmake>=3.26.4,!=3.30.0 @@ -63,9 +62,9 @@ dependencies: - scikit-learn>=0.23.1 - scipy - setuptools>=61.0.0 +- sphinx - sphinx-copybutton - sphinx-markdown-tables -- sphinx<6 - sphinxcontrib-websupport - thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata @@ -73,4 +72,6 @@ dependencies: - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 - wheel +- pip: + - breathe>=4.35.0 name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index 87a2417bbc4..10ed2595398 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -8,7 +8,6 @@ channels: - nvidia dependencies: - aiohttp -- breathe - c-compiler - certifi - cmake>=3.26.4,!=3.30.0 @@ -68,9 +67,9 @@ dependencies: - scikit-learn>=0.23.1 - scipy - setuptools>=61.0.0 +- sphinx - sphinx-copybutton - sphinx-markdown-tables -- sphinx<6 - sphinxcontrib-websupport - thriftpy2>=0.4.15,!=0.5.0,!=0.5.1 - torchdata @@ -78,4 +77,6 @@ dependencies: - ucx-proc=*=gpu - ucx-py==0.42.*,>=0.0.0a0 - wheel +- pip: + - breathe>=4.35.0 name: all_cuda-125_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index eb4bb68dc9d..cbb1085d8cf 100755 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -334,7 +334,6 @@ dependencies: common: - output_types: [conda] packages: - - breathe - doxygen - graphviz - ipython @@ -344,8 +343,11 @@ dependencies: - recommonmark - sphinx-copybutton - sphinx-markdown-tables - - sphinx<6 + - sphinx - sphinxcontrib-websupport + - pip: + # Need new enough breathe + - breathe>=4.35.0 py_version: specific: - output_types: [conda]