From c3c6780de7111b1a9bc2ddbae02360cb418af647 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 17:45:26 -0500 Subject: [PATCH 1/9] Remving Python from build --- README.md | 40 +++++++++++++++++++++++++++++++++++++++- ci/build_python.sh | 21 --------------------- ci/build_wheel_cuvs.sh | 4 ++-- ci/test_python.sh | 26 +++++++++++++------------- ci/test_wheel_cuvs.sh | 20 ++++++++++---------- docs/source/index.rst | 10 +++------- 6 files changed, 67 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index e86eafda2..428ab04a9 100755 --- a/README.md +++ b/README.md @@ -1,2 +1,40 @@ -#
 cuVS: Vector Search on the GPU
+#
 cuVS: Vector Search and Clustering on the GPU
+ +### NOTE: cuVS is currently being + +## Contents +
+ +1. [Useful Resources](#useful-resources) +2. [What is cuVS?](#what-is-cuvs) +3. [Getting Started](#getting-started) +4. [Installing cuVS](#installing) +5. [Contributing](#contributing) +6. [References](#references) + +
+ +## Useful Resources + +- [cuVS Reference Documentation](https://docs.rapids.ai/api/cuvs/stable/): API Documentation. +- [cuVS Getting Started](./docs/source/quick_start.md): Getting started with RAFT. +- [Build and Install cuVS](./docs/source/build.md): Instructions for installing and building cuVS. +- [Example Notebooks](./notebooks): Example jupyer notebooks +- [RAPIDS Community](https://rapids.ai/community.html): Get help, contribute, and collaborate. +- [GitHub repository](https://github.com/rapidsai/cuvs): Download the cuVS source code. +- [Issue tracker](https://github.com/rapidsai/cuvs/issues): Report issues or request features. + +## What is cuVS? + +cuVS contains many algorithms for running approximate nearest neighbors and clustering on the GPU. + +## Getting Started + + + +## Installing cuVS + +## Contributing + +## References diff --git a/ci/build_python.sh b/ci/build_python.sh index a54c3dcaa..525d21c51 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -34,24 +34,3 @@ rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ conda/recipes/cuvs - - -# Build ann-bench for each cuda and python version -rapids-conda-retry mambabuild \ ---no-test \ ---channel "${CPP_CHANNEL}" \ ---channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ -conda/recipes/cuda-ann-bench - -# Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python -# version -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" -if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cuda-ann-bench-cpu -fi - -rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index 9d2f96996..b4765be38 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CONFIGURE_OPTIONS="-DRAFT_BUILD_WHEELS=ON -DDETECT_CONDA_ENV=OFF -DFIND_RAFT_CPP=OFF" -ci/build_wheel.sh cuvs python/cuvs +#ci/build_wheel.sh cuvs python/cuvs diff --git a/ci/test_python.sh b/ci/test_python.sh index a65469928..9f0c9d6ee 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. set -euo pipefail @@ -31,7 +31,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ --channel "${PYTHON_CHANNEL}" \ - libcuvs cuvs + libcuvs #cuvs rapids-logger "Check GPU usage" nvidia-smi @@ -40,17 +40,17 @@ EXITCODE=0 trap "EXITCODE=1" ERR set +e -rapids-logger "pytest cuvs" -pushd python/cuvs/cuvs -pytest \ - --cache-clear \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cuvs.xml" \ - --cov-config=../.coveragerc \ - --cov=cuvs \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuvs-coverage.xml" \ - --cov-report=term \ - test -popd +#rapids-logger "pytest cuvs" +#pushd python/cuvs/cuvs +#pytest \ +# --cache-clear \ +# --junitxml="${RAPIDS_TESTS_DIR}/junit-cuvs.xml" \ +# --cov-config=../.coveragerc \ +# --cov=cuvs \ +# --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuvs-coverage.xml" \ +# --cov-report=term \ +# test +#popd rapids-logger "Test script exiting with value: $EXITCODE" exit ${EXITCODE} diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index 6b213d399..de7501915 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail @@ -7,12 +7,12 @@ mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/cuvs*.whl)[test] - -# Run smoke tests for aarch64 pull requests -if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then - python ./ci/wheel_smoke_test_cuvs.py -else - python -m pytest ./python/cuvs/cuvs/test -fi +## echo to expand wildcard before adding `[extra]` requires for pip +#python -m pip install $(echo ./dist/cuvs*.whl)[test] +# +## Run smoke tests for aarch64 pull requests +#if [[ "$(arch)" == "aarch64" && "${RAPIDS_BUILD_TYPE}" == "pull-request" ]]; then +# python ./ci/wheel_smoke_test_cuvs.py +#else +# python -m pytest ./python/cuvs/cuvs/test +#fi diff --git a/docs/source/index.rst b/docs/source/index.rst index a161efb42..bf9790610 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,9 +1,5 @@ -cuVS: GPU-accelerated Vector Search -=================================== - -.. image:: ../../img/raft-tech-stack-vss.png - :width: 800 - :alt: cuVS Tech Stack +cuVS: Vector Search and Clustering on the GPU +============================================= Useful Resources ################ @@ -19,7 +15,7 @@ Useful Resources What is cuVS? ############# -cuVS is a library for vector search on the GPU. +cuVS is a library for vector search and clustering on the GPU. .. toctree:: :maxdepth: 1 From 1d337e1402a5b9cd050d181a9d496e7588b97eb1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 18:03:06 -0500 Subject: [PATCH 2/9] More updates --- ci/test_wheel_cuvs.sh | 4 +- cpp/include/cuvs/neighbors/cagra_c.h | 68 ++++++++++++++-------------- cpp/src/neighbors/cagra_c.cpp | 50 ++++++++++---------- cpp/test/neighbors/ann_cagra_c.cu | 10 ++-- cpp/test/neighbors/c_api.c | 6 +-- 5 files changed, 69 insertions(+), 69 deletions(-) diff --git a/ci/test_wheel_cuvs.sh b/ci/test_wheel_cuvs.sh index de7501915..52cfa7ae1 100755 --- a/ci/test_wheel_cuvs.sh +++ b/ci/test_wheel_cuvs.sh @@ -4,8 +4,8 @@ set -euo pipefail mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +#RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +#RAPIDS_PY_WHEEL_NAME="cuvs_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist ## echo to expand wildcard before adding `[extra]` requires for pip #python -m pip install $(echo ./dist/cuvs*.whl)[test] diff --git a/cpp/include/cuvs/neighbors/cagra_c.h b/cpp/include/cuvs/neighbors/cagra_c.h index 59861b502..70d053ecf 100644 --- a/cpp/include/cuvs/neighbors/cagra_c.h +++ b/cpp/include/cuvs/neighbors/cagra_c.h @@ -33,7 +33,7 @@ extern "C" { * @brief Enum to denote which ANN algorithm is used to build CAGRA graph * */ -enum cagraGraphBuildAlgo { +enum cuvsCagraGraphBuildAlgo { /* Use IVF-PQ to build all-neighbors knn graph */ IVF_PQ, /* Experimental, use NN-Descent to build all-neighbors knn graph */ @@ -44,18 +44,18 @@ enum cagraGraphBuildAlgo { * @brief Supplemental parameters to build CAGRA Index * */ -struct cagraIndexParams { +struct cuvsCagraIndexParams { /** Degree of input graph for pruning. */ size_t intermediate_graph_degree; /** Degree of output graph. */ size_t graph_degree; /** ANN algorithm to build knn graph. */ - enum cagraGraphBuildAlgo build_algo; + enum cuvsCagraGraphBuildAlgo build_algo; /** Number of Iterations to run if building with NN_DESCENT */ size_t nn_descent_niter; }; -typedef struct cagraIndexParams* cuvsCagraIndexParams_t; +typedef struct cuvsCagraIndexParams* cuvsCagraIndexParams_t; /** * @brief Allocate CAGRA Index params, and populate with default values @@ -77,7 +77,7 @@ cuvsError_t cuvsCagraIndexParamsDestroy(cuvsCagraIndexParams_t index); * @brief Enum to denote algorithm used to search CAGRA Index * */ -enum cagraSearchAlgo { +enum cuvsCagraSearchAlgo { /** For large batch sizes. */ SINGLE_CTA, /** For small batch sizes. */ @@ -90,13 +90,13 @@ enum cagraSearchAlgo { * @brief Enum to denote Hash Mode used while searching CAGRA index * */ -enum cagraHashMode { HASH, SMALL, AUTO_HASH }; +enum cuvsCagraHashMode { HASH, SMALL, AUTO_HASH }; /** * @brief Supplemental parameters to search CAGRA index * */ -struct cagraSearchParams { +struct cuvsCagraSearchParams { /** Maximum number of queries to search at the same time (batch size). Auto select when 0.*/ size_t max_queries; @@ -114,7 +114,7 @@ struct cagraSearchParams { // Reasonable default values are automatically chosen. /** Which search implementation to use. */ - enum cagraSearchAlgo algo; + enum cuvsCagraSearchAlgo algo; /** Number of threads used to calculate a single distance. 4, 8, 16, or 32. */ size_t team_size; @@ -128,7 +128,7 @@ struct cagraSearchParams { /** Thread block size. 0, 64, 128, 256, 512, 1024. Auto selection when 0. */ size_t thread_block_size; /** Hashmap type. Auto selection when AUTO. */ - enum cagraHashMode hashmap_mode; + enum cuvsCagraHashMode hashmap_mode; /** Lower limit of hashmap bit length. More than 8. */ size_t hashmap_min_bitlen; /** Upper limit of hashmap fill rate. More than 0.1, less than 0.9.*/ @@ -140,7 +140,7 @@ struct cagraSearchParams { uint64_t rand_xor_mask; }; -typedef struct cagraSearchParams* cuvsCagraSearchParams_t; +typedef struct cuvsCagraSearchParams* cuvsCagraSearchParams_t; /** * @brief Allocate CAGRA search params, and populate with default values @@ -166,24 +166,24 @@ typedef struct { uintptr_t addr; DLDataType dtype; -} cagraIndex; +} cuvsCagraIndex; -typedef cagraIndex* cagraIndex_t; +typedef cagraIndex* cuvsCagraIndex_t; /** * @brief Allocate CAGRA index * - * @param[in] index cagraIndex_t to allocate + * @param[in] index cuvsCagraIndex_t to allocate * @return cagraError_t */ -cuvsError_t cagraIndexCreate(cagraIndex_t* index); +cuvsError_t cuvsCagraIndexCreate(cuvsCagraIndex_t* index); /** * @brief De-allocate CAGRA index * - * @param[in] index cagraIndex_t to de-allocate + * @param[in] index cuvsCagraIndex_t to de-allocate */ -cuvsError_t cagraIndexDestroy(cagraIndex_t index); +cuvsError_t cuvsCagraIndexDestroy(cuvsCagraIndex_t index); /** * @brief Build a CAGRA index with a `DLManagedTensor` which has underlying @@ -209,28 +209,28 @@ cuvsError_t cagraIndexDestroy(cagraIndex_t index); * cuvsError_t params_create_status = cuvsCagraIndexParamsCreate(¶ms); * * // Create CAGRA index - * cagraIndex_t index; - * cuvsError_t index_create_status = cagraIndexCreate(&index); + * cuvsCagraIndex_t index; + * cuvsError_t index_create_status = cuvsCagraIndexCreate(&index); * * // Build the CAGRA Index - * cuvsError_t build_status = cagraBuild(res, params, &dataset, index); + * cuvsError_t build_status = cuvsCagraBuild(res, params, &dataset, index); * * // de-allocate `params`, `index` and `res` * cuvsError_t params_destroy_status = cuvsCagraIndexParamsDestroy(params); - * cuvsError_t index_destroy_status = cagraIndexDestroy(index); + * cuvsError_t index_destroy_status = cuvsCagraIndexDestroy(index); * cuvsError_t res_destroy_status = cuvsResourcesDestroy(res); * @endcode * * @param[in] res cuvsResources_t opaque C handle * @param[in] params cuvsCagraIndexParams_t used to build CAGRA index * @param[in] dataset DLManagedTensor* training dataset - * @param[out] index cagraIndex_t Newly built CAGRA index + * @param[out] index cuvsCagraIndex_t Newly built CAGRA index * @return cuvsError_t */ -cuvsError_t cagraBuild(cuvsResources_t res, - cuvsCagraIndexParams_t params, - DLManagedTensor* dataset, - cagraIndex_t index); +cuvsError_t cuvsCagraBuild(cuvsResources_t res, + cuvsCagraIndexParams_t params, + DLManagedTensor* dataset, + cuvsCagraIndex_t index); /** * @brief Search a CAGRA index with a `DLManagedTensor` which has underlying @@ -259,8 +259,8 @@ cuvsError_t cagraBuild(cuvsResources_t res, * cuvsCagraSearchParams_t params; * cuvsError_t params_create_status = cuvsCagraSearchParamsCreate(¶ms); * - * // Search the `index` built using `cagraBuild` - * cuvsError_t search_status = cagraSearch(res, params, index, queries, neighbors, distances); + * // Search the `index` built using `cuvsCagraBuild` + * cuvsError_t search_status = cuvsCagraSearch(res, params, index, queries, neighbors, distances); * * // de-allocate `params` and `res` * cuvsError_t params_destroy_status = cuvsCagraSearchParamsDestroy(params); @@ -269,17 +269,17 @@ cuvsError_t cagraBuild(cuvsResources_t res, * * @param[in] res cuvsResources_t opaque C handle * @param[in] params cuvsCagraSearchParams_t used to search CAGRA index - * @param[in] index cagraIndex which has been returned by `cagraBuild` + * @param[in] index cuvsCagraIndex which has been returned by `cuvsCagraBuild` * @param[in] queries DLManagedTensor* queries dataset to search * @param[out] neighbors DLManagedTensor* output `k` neighbors for queries * @param[out] distances DLManagedTensor* output `k` distances for queries */ -cuvsError_t cagraSearch(cuvsResources_t res, - cuvsCagraSearchParams_t params, - cagraIndex_t index, - DLManagedTensor* queries, - DLManagedTensor* neighbors, - DLManagedTensor* distances); +cuvsError_t cuvsCagraSearch(cuvsResources_t res, + cuvsCagraSearchParams_t params, + cuvsCagraIndex_t index, + DLManagedTensor* queries, + DLManagedTensor* neighbors, + DLManagedTensor* distances); #ifdef __cplusplus } diff --git a/cpp/src/neighbors/cagra_c.cpp b/cpp/src/neighbors/cagra_c.cpp index 638c9a23d..2a9de37f1 100644 --- a/cpp/src/neighbors/cagra_c.cpp +++ b/cpp/src/neighbors/cagra_c.cpp @@ -30,7 +30,7 @@ namespace { template -void* _build(cuvsResources_t res, cagraIndexParams params, DLManagedTensor* dataset_tensor) +void* _build(cuvsResources_t res, cuvsCagraIndexParams params, DLManagedTensor* dataset_tensor) { auto dataset = dataset_tensor->dl_tensor; @@ -59,8 +59,8 @@ void* _build(cuvsResources_t res, cagraIndexParams params, DLManagedTensor* data template void _search(cuvsResources_t res, - cagraSearchParams params, - cagraIndex index, + cuvsCagraSearchParams params, + cuvsCagraIndex index, DLManagedTensor* queries_tensor, DLManagedTensor* neighbors_tensor, DLManagedTensor* distances_tensor) @@ -95,17 +95,17 @@ void _search(cuvsResources_t res, } // namespace -extern "C" cuvsError_t cagraIndexCreate(cagraIndex_t* index) +extern "C" cuvsError_t cuvsCagraIndexCreate(cuvsCagraIndex_t* index) { try { - *index = new cagraIndex{}; + *index = new cuvsCagraIndex{}; return CUVS_SUCCESS; } catch (...) { return CUVS_ERROR; } } -extern "C" cuvsError_t cagraIndexDestroy(cagraIndex_t index_c_ptr) +extern "C" cuvsError_t cuvsCagraIndexDestroy(cuvsCagraIndex_t index_c_ptr) { try { auto index = *index_c_ptr; @@ -130,10 +130,10 @@ extern "C" cuvsError_t cagraIndexDestroy(cagraIndex_t index_c_ptr) } } -extern "C" cuvsError_t cagraBuild(cuvsResources_t res, - cuvsCagraIndexParams_t params, - DLManagedTensor* dataset_tensor, - cagraIndex_t index) +extern "C" cuvsError_t cuvsCagraBuild(cuvsResources_t res, + cuvsCagraIndexParams_t params, + DLManagedTensor* dataset_tensor, + cuvsCagraIndex_t index) { try { auto dataset = dataset_tensor->dl_tensor; @@ -158,12 +158,12 @@ extern "C" cuvsError_t cagraBuild(cuvsResources_t res, } } -extern "C" cuvsError_t cagraSearch(cuvsResources_t res, - cuvsCagraSearchParams_t params, - cagraIndex_t index_c_ptr, - DLManagedTensor* queries_tensor, - DLManagedTensor* neighbors_tensor, - DLManagedTensor* distances_tensor) +extern "C" cuvsError_t cuvsCagraSearch(cuvsResources_t res, + cuvsCagraSearchParams_t params, + cuvsCagraIndex_t index_c_ptr, + DLManagedTensor* queries_tensor, + DLManagedTensor* neighbors_tensor, + DLManagedTensor* distances_tensor) { try { auto queries = queries_tensor->dl_tensor; @@ -205,10 +205,10 @@ extern "C" cuvsError_t cagraSearch(cuvsResources_t res, extern "C" cuvsError_t cuvsCagraIndexParamsCreate(cuvsCagraIndexParams_t* params) { try { - *params = new cagraIndexParams{.intermediate_graph_degree = 128, - .graph_degree = 64, - .build_algo = IVF_PQ, - .nn_descent_niter = 20}; + *params = new cuvsCagraIndexParams{.intermediate_graph_degree = 128, + .graph_degree = 64, + .build_algo = IVF_PQ, + .nn_descent_niter = 20}; return CUVS_SUCCESS; } catch (...) { return CUVS_ERROR; @@ -228,11 +228,11 @@ extern "C" cuvsError_t cuvsCagraIndexParamsDestroy(cuvsCagraIndexParams_t params extern "C" cuvsError_t cuvsCagraSearchParamsCreate(cuvsCagraSearchParams_t* params) { try { - *params = new cagraSearchParams{.itopk_size = 64, - .search_width = 1, - .hashmap_max_fill_rate = 0.5, - .num_random_samplings = 1, - .rand_xor_mask = 0x128394}; + *params = new cuvsCagraSearchParams{.itopk_size = 64, + .search_width = 1, + .hashmap_max_fill_rate = 0.5, + .num_random_samplings = 1, + .rand_xor_mask = 0x128394}; return CUVS_SUCCESS; } catch (...) { return CUVS_ERROR; diff --git a/cpp/test/neighbors/ann_cagra_c.cu b/cpp/test/neighbors/ann_cagra_c.cu index 4870ac3b8..a1c496eaa 100644 --- a/cpp/test/neighbors/ann_cagra_c.cu +++ b/cpp/test/neighbors/ann_cagra_c.cu @@ -56,13 +56,13 @@ TEST(CagraC, BuildSearch) dataset_tensor.dl_tensor.strides = nullptr; // create index - cagraIndex_t index; - cagraIndexCreate(&index); + cuvsCagraIndex_t index; + cuvsCagraIndexCreate(&index); // build index cuvsCagraIndexParams_t build_params; cuvsCagraIndexParamsCreate(&build_params); - cagraBuild(res, build_params, &dataset_tensor, index); + cuvsCagraBuild(res, build_params, &dataset_tensor, index); // create queries DLTensor float* queries_d; @@ -113,7 +113,7 @@ TEST(CagraC, BuildSearch) // search index cuvsCagraSearchParams_t search_params; cuvsCagraSearchParamsCreate(&search_params); - cagraSearch(res, search_params, index, &queries_tensor, &neighbors_tensor, &distances_tensor); + cuvsCagraSearch(res, search_params, index, &queries_tensor, &neighbors_tensor, &distances_tensor); // verify output ASSERT_TRUE(cuvs::devArrMatchHost(neighbors_exp, neighbors_d, 4, cuvs::Compare())); @@ -128,6 +128,6 @@ TEST(CagraC, BuildSearch) // de-allocate index and res cuvsCagraSearchParamsDestroy(search_params); cuvsCagraIndexParamsDestroy(build_params); - cagraIndexDestroy(index); + cuvsCagraIndexDestroy(index); cuvsResourcesDestroy(res); } diff --git a/cpp/test/neighbors/c_api.c b/cpp/test/neighbors/c_api.c index d4f5ad08e..fa1727c51 100644 --- a/cpp/test/neighbors/c_api.c +++ b/cpp/test/neighbors/c_api.c @@ -24,8 +24,8 @@ int main() // simple smoke test to make sure that we can compile the cagra_c.h API // using a c compiler. This isn't aiming to be a full test, just checking // that the exposed C-API is valid C code and doesn't contain C++ features - cagraIndex_t index; - cagraIndexCreate(&index); - cagraIndexDestroy(index); + cuvsCagraIndex_t index; + cuvsCagraIndexCreate(&index); + cuvsCagraIndexDestroy(index); return 0; } From a91844ee6ad4fd8da19a5258dc0ed898aff203fb Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 18:05:13 -0500 Subject: [PATCH 3/9] Fixing cagraIndex --- cpp/include/cuvs/neighbors/cagra_c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/include/cuvs/neighbors/cagra_c.h b/cpp/include/cuvs/neighbors/cagra_c.h index 70d053ecf..c50d7475b 100644 --- a/cpp/include/cuvs/neighbors/cagra_c.h +++ b/cpp/include/cuvs/neighbors/cagra_c.h @@ -168,7 +168,7 @@ typedef struct { } cuvsCagraIndex; -typedef cagraIndex* cuvsCagraIndex_t; +typedef cuvsCagraIndex* cuvsCagraIndex_t; /** * @brief Allocate CAGRA index From 9024fb7847a0b0d9e4f3745c4f4cb292da651828 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 18:36:32 -0500 Subject: [PATCH 4/9] MOre changes to get CI running. Renaming template->examples --- build.sh | 10 +++---- ci/build_python.sh | 28 +++++++++---------- ..._template.sh => build_libcuvs_examples.sh} | 4 +-- conda/recipes/libcuvs/meta.yaml | 6 ++-- cpp/{template => examples}/CMakeLists.txt | 0 cpp/{template => examples}/README.md | 8 +++--- cpp/{template => examples}/build.sh | 2 +- .../cmake/thirdparty/fetch_rapids.cmake | 2 +- .../cmake/thirdparty/get_cuvs.cmake | 0 .../src/cagra_example.cu | 0 cpp/{template => examples}/src/common.cuh | 2 +- 11 files changed, 31 insertions(+), 31 deletions(-) rename conda/recipes/libcuvs/{build_libcuvs_template.sh => build_libcuvs_examples.sh} (61%) rename cpp/{template => examples}/CMakeLists.txt (100%) rename cpp/{template => examples}/README.md (82%) rename cpp/{template => examples}/build.sh (93%) rename cpp/{template => examples}/cmake/thirdparty/fetch_rapids.cmake (95%) rename cpp/{template => examples}/cmake/thirdparty/get_cuvs.cmake (100%) rename cpp/{template => examples}/src/cagra_example.cu (100%) rename cpp/{template => examples}/src/common.cuh (98%) diff --git a/build.sh b/build.sh index 6dd250c51..0d035171e 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # scripts, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libcuvs python docs tests template clean --uninstall -v -g -n --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" +VALIDARGS="clean libcuvs python docs tests examples clean --uninstall -v -g -n --compile-static-lib --allgpuarch --no-nvtx --show_depr_warn --incl-cache-stats --time -h" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--build-metrics=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -27,7 +27,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= is: -v - verbose build mode @@ -433,10 +433,10 @@ if hasArg docs; then fi ################################################################################ -# Initiate build for example CUVS application template (if needed) +# Initiate build for c++ examples (if needed) -if hasArg template; then - pushd ${REPODIR}/cpp/template +if hasArg examples; then + pushd ${REPODIR}/cpp/examples ./build.sh popd fi diff --git a/ci/build_python.sh b/ci/build_python.sh index 525d21c51..2f661c853 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -15,22 +15,22 @@ rapids-print-env rapids-logger "Begin py build" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +#CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) -export RAPIDS_PACKAGE_VERSION=${version} -echo "${version}" > VERSION +#version=$(rapids-generate-version) +#git_commit=$(git rev-parse HEAD) +#export RAPIDS_PACKAGE_VERSION=${version} +#echo "${version}" > VERSION -package_dir="python" -for package_name in cuvs raft-dask; do - underscore_package_name=$(echo "${package_name}" | tr "-" "_") - sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" -done +#package_dir="python" +#for package_name in cuvs; do +# underscore_package_name=$(echo "${package_name}" | tr "-" "_") +# sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py" +#done # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly -rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - conda/recipes/cuvs +#rapids-conda-retry mambabuild \ +# --no-test \ +# --channel "${CPP_CHANNEL}" \ +# conda/recipes/cuvs diff --git a/conda/recipes/libcuvs/build_libcuvs_template.sh b/conda/recipes/libcuvs/build_libcuvs_examples.sh similarity index 61% rename from conda/recipes/libcuvs/build_libcuvs_template.sh rename to conda/recipes/libcuvs/build_libcuvs_examples.sh index bd7719af7..6286a530e 100644 --- a/conda/recipes/libcuvs/build_libcuvs_template.sh +++ b/conda/recipes/libcuvs/build_libcuvs_examples.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # Just building template so we verify it uses libraft.so and fail if it doesn't build -./build.sh template +./build.sh examples diff --git a/conda/recipes/libcuvs/meta.yaml b/conda/recipes/libcuvs/meta.yaml index 4e437f3e6..4b6ff87e9 100644 --- a/conda/recipes/libcuvs/meta.yaml +++ b/conda/recipes/libcuvs/meta.yaml @@ -195,9 +195,9 @@ outputs: home: https://rapids.ai/ license: Apache-2.0 summary: libcuvs tests - - name: libcuvs-template + - name: libcuvs-examples version: {{ version }} - script: build_libcuvs_template.sh + script: build_libcuvs_examples.sh build: script_env: *script_env number: {{ GIT_DESCRIBE_NUMBER }} @@ -241,4 +241,4 @@ outputs: about: home: https://rapids.ai/ license: Apache-2.0 - summary: libcuvs template + summary: libcuvs examples diff --git a/cpp/template/CMakeLists.txt b/cpp/examples/CMakeLists.txt similarity index 100% rename from cpp/template/CMakeLists.txt rename to cpp/examples/CMakeLists.txt diff --git a/cpp/template/README.md b/cpp/examples/README.md similarity index 82% rename from cpp/template/README.md rename to cpp/examples/README.md index 5393c0229..125c6dba2 100644 --- a/cpp/template/README.md +++ b/cpp/examples/README.md @@ -1,14 +1,14 @@ -# Example CUVS Project Template +# cuVS C++ Examples This template project provides a drop-in sample to either start building a new application with, or using CUVS in an existing CMake project. -First, please refer to our [installation docs](https://docs.rapids.ai/api/cuvs/stable/build.html#cuda-gpu-requirements) for the minimum requirements to use CUVS. +First, please refer to our [installation docs](https://docs.rapids.ai/api/cuvs/stable/build.html#cuda-gpu-requirements) for the minimum requirements to use cuVS. Once the minimum requirements are satisfied, this example template application can be built with the provided `build.sh` script. This is a bash script that calls the appropriate CMake commands, so you can look into it to see the typical CMake based build workflow. -This directory (`CUVS_SOURCE/cpp/template`) can be copied directly in order to build a new application with CUVS. +This directory (`CUVS_SOURCE/cpp/examples`) can be copied directly in order to build a new application with CUVS. -CUVS can be integrated into an existing CMake project by copying the contents in the `configure rapids-cmake` and `configure cuvs` sections of the provided `CMakeLists.txt` into your project, along with `cmake/thirdparty/get_cuvs.cmake`. +cuVS can be integrated into an existing CMake project by copying the contents in the `configure rapids-cmake` and `configure cuvs` sections of the provided `CMakeLists.txt` into your project, along with `cmake/thirdparty/get_cuvs.cmake`. Make sure to link against the appropriate Cmake targets. Use `cuvs::cuvs` to utilize the shared library. diff --git a/cpp/template/build.sh b/cpp/examples/build.sh similarity index 93% rename from cpp/template/build.sh rename to cpp/examples/build.sh index 25ccb3461..7a948d9a8 100755 --- a/cpp/template/build.sh +++ b/cpp/examples/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # cuvs empty project template build script diff --git a/cpp/template/cmake/thirdparty/fetch_rapids.cmake b/cpp/examples/cmake/thirdparty/fetch_rapids.cmake similarity index 95% rename from cpp/template/cmake/thirdparty/fetch_rapids.cmake rename to cpp/examples/cmake/thirdparty/fetch_rapids.cmake index 15b6c43a6..4da917e26 100644 --- a/cpp/template/cmake/thirdparty/fetch_rapids.cmake +++ b/cpp/examples/cmake/thirdparty/fetch_rapids.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/cpp/template/cmake/thirdparty/get_cuvs.cmake b/cpp/examples/cmake/thirdparty/get_cuvs.cmake similarity index 100% rename from cpp/template/cmake/thirdparty/get_cuvs.cmake rename to cpp/examples/cmake/thirdparty/get_cuvs.cmake diff --git a/cpp/template/src/cagra_example.cu b/cpp/examples/src/cagra_example.cu similarity index 100% rename from cpp/template/src/cagra_example.cu rename to cpp/examples/src/cagra_example.cu diff --git a/cpp/template/src/common.cuh b/cpp/examples/src/common.cuh similarity index 98% rename from cpp/template/src/common.cuh rename to cpp/examples/src/common.cuh index 0b72d3bf3..757123cea 100644 --- a/cpp/template/src/common.cuh +++ b/cpp/examples/src/common.cuh @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, NVIDIA CORPORATION. + * Copyright (c) 2023-2024, NVIDIA CORPORATION. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From fd72b23d065c4fcba82390ca88a287c5f9334b0d Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 19:05:12 -0500 Subject: [PATCH 5/9] MOre updates --- ci/build_docs.sh | 24 +++++++++++------------- ci/test_python.sh | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 1193285da..f5da8ca32 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -22,11 +22,9 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - libcuvs \ - libcuvs-headers \ - cuvs \ - raft-dask +# --channel "${PYTHON_CHANNEL}" \ + libcuvs #\ +# cuvs \ export RAPIDS_VERSION_NUMBER="24.02" export RAPIDS_DOCS_DIR="$(mktemp -d)" @@ -36,13 +34,13 @@ pushd cpp/doxygen doxygen Doxyfile popd -rapids-logger "Build Python docs" -pushd docs -sphinx-build -b dirhtml source _html -sphinx-build -b text source _text -mkdir -p "${RAPIDS_DOCS_DIR}/cuvs/"{html,txt} -mv _html/* "${RAPIDS_DOCS_DIR}/cuvs/html" -mv _text/* "${RAPIDS_DOCS_DIR}/cuvs/txt" -popd +#rapids-logger "Build Python docs" +#pushd docs +#sphinx-build -b dirhtml source _html +#sphinx-build -b text source _text +#mkdir -p "${RAPIDS_DOCS_DIR}/cuvs/"{html,txt} +#mv _html/* "${RAPIDS_DOCS_DIR}/cuvs/html" +#mv _text/* "${RAPIDS_DOCS_DIR}/cuvs/txt" +#popd rapids-upload-docs diff --git a/ci/test_python.sh b/ci/test_python.sh index 9f0c9d6ee..d8c7c2020 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -20,7 +20,7 @@ set -u rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +#PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} @@ -30,7 +30,7 @@ rapids-print-env rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ +# --channel "${PYTHON_CHANNEL}" \ libcuvs #cuvs rapids-logger "Check GPU usage" From d0183fed5b879d8ba90a16f1a8f8a318d1935e29 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 19:40:54 -0500 Subject: [PATCH 6/9] Fix docs and python tests --- ci/build_docs.sh | 2 +- ci/test_python.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index f5da8ca32..e00b90618 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -18,7 +18,7 @@ rapids-print-env rapids-logger "Downloading artifacts from previous jobs" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) +#PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/test_python.sh b/ci/test_python.sh index d8c7c2020..e70c4555d 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -28,10 +28,10 @@ mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ -# --channel "${PYTHON_CHANNEL}" \ - libcuvs #cuvs +#rapids-mamba-retry install \ +# --channel "${CPP_CHANNEL}" \ +## --channel "${PYTHON_CHANNEL}" \ +# libcuvs #cuvs rapids-logger "Check GPU usage" nvidia-smi From f8443a742825450f364599331bafb30097eb8f02 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 20:13:04 -0500 Subject: [PATCH 7/9] Proper install --- ci/build_docs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index e00b90618..86836f7cd 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -22,9 +22,7 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ -# --channel "${PYTHON_CHANNEL}" \ - libcuvs #\ -# cuvs \ + libcuvs export RAPIDS_VERSION_NUMBER="24.02" export RAPIDS_DOCS_DIR="$(mktemp -d)" From 7839fa5cf79cb902c1f0dfe0fc4e8fc95782cc32 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 21:05:20 -0500 Subject: [PATCH 8/9] Actually building docs to upload --- ci/build_docs.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 86836f7cd..0706b1fca 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -32,13 +32,13 @@ pushd cpp/doxygen doxygen Doxyfile popd -#rapids-logger "Build Python docs" -#pushd docs -#sphinx-build -b dirhtml source _html -#sphinx-build -b text source _text -#mkdir -p "${RAPIDS_DOCS_DIR}/cuvs/"{html,txt} -#mv _html/* "${RAPIDS_DOCS_DIR}/cuvs/html" -#mv _text/* "${RAPIDS_DOCS_DIR}/cuvs/txt" -#popd +rapids-logger "Build Python docs" +pushd docs +sphinx-build -b dirhtml source _html +sphinx-build -b text source _text +mkdir -p "${RAPIDS_DOCS_DIR}/cuvs/"{html,txt} +mv _html/* "${RAPIDS_DOCS_DIR}/cuvs/html" +mv _text/* "${RAPIDS_DOCS_DIR}/cuvs/txt" +popd rapids-upload-docs From 1b18f5c30846c2607bebea6428d3f11fa808e59c Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 5 Feb 2024 22:38:50 -0500 Subject: [PATCH 9/9] Trying proper directory for tests --- cpp/test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 609bc2d4e..f33c14179 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -57,7 +57,7 @@ function(ConfigureTest) ) set_target_properties( ${TEST_NAME} - PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "$" INSTALL_RPATH "\$ORIGIN/../../../lib" CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON