From 07644ec0035abe45f9bc7cc54a90b18535c4edf2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 15:48:21 -0500 Subject: [PATCH 01/81] add option to build faiss shared libs --- cpp/CMakeLists.txt | 2 ++ cpp/cmake/thirdparty/get_faiss.cmake | 15 +++++++++++---- cpp/test/CMakeLists.txt | 1 - 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7ab8a9f46e..cab85e8b9c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,6 +49,8 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) +option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON) + message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") message(VERBOSE "RAFT: Disable depreaction warnings " ${DISABLE_DEPRECATION_WARNINGS}) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index a65401579c..6f4bac6a40 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -15,7 +15,7 @@ #============================================================================= function(find_and_configure_faiss) - set(oneValueArgs VERSION PINNED_TAG) + set(oneValueArgs VERSION PINNED_TAG BUILD_STATIC_LIBS) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) @@ -24,14 +24,20 @@ function(find_and_configure_faiss) LIBRARY_NAMES faiss ) + set(BUILD_SHARED_LIBS OFF) + if (NOT BUILD_STATIC_LIBS) + set(BUILD_SHARED_LIBS ON) + endif() + rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + GLOBAL_TARGETS faiss + BUILD_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} OPTIONS "FAISS_ENABLE_PYTHON OFF" - "BUILD_SHARED_LIBS OFF" + "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" "FAISS_ENABLE_GPU ON" "BUILD_TESTING OFF" @@ -39,7 +45,7 @@ function(find_and_configure_faiss) ) if(FAISS_ADDED) - set(FAISS_GPU_HEADERS ${FAISS_SOURCE_DIR} PARENT_SCOPE) + target_include_directories(faiss INTERFACE $) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) @@ -50,4 +56,5 @@ endfunction() find_and_configure_faiss(VERSION 1.7.0 PINNED_TAG bde7c0027191f29c9dadafe4f6e68ca0ee31fb30 + BUILD_STATIC_LIBS ${RAFT_USE_FAISS_STATIC} ) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 43e1c65695..6040aca26c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -117,7 +117,6 @@ target_compile_options(test_raft target_include_directories(test_raft PUBLIC "$" "$" - "${FAISS_GPU_HEADERS}" ) From f3e1f0927e795d12440fa5188f8484a7e5dba27d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 16:25:41 -0500 Subject: [PATCH 02/81] add FAISS::FAISS to the list of global targets --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 6f4bac6a40..1d0c562cfd 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -30,7 +30,7 @@ function(find_and_configure_faiss) endif() rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + GLOBAL_TARGETS faiss FAISS::FAISS BUILD_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git From e0386442802093398d790dd2818aeb9331d339bb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 17:26:37 -0500 Subject: [PATCH 03/81] update cuco --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 2ee7a1c384..33e28ff622 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG a3c85ee2ea25ddfdd4596c6b9d546f7c7590743f + GIT_TAG 729857a5698a0e8d8f812e0464f65f37854ae17b OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From 3fa9c94860d1dc4a3693666c0affb5879ae9f0a4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 17:58:53 -0500 Subject: [PATCH 04/81] create a faissTargets export-set --- cpp/cmake/thirdparty/get_faiss.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..4e04592b7e 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,6 +52,8 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() + rapids_export(BUILD faiss EXPORT_SET faissTargets) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From 85572d097dfb0a4294e6d497c4e432c42c0161cf Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:10:45 -0500 Subject: [PATCH 05/81] add faiss to raft-exports --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 4e04592b7e..f9d286668a 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,7 +52,7 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() - rapids_export(BUILD faiss EXPORT_SET faissTargets) + rapids_export(BUILD faiss EXPORT_SET raft-exports) endfunction() From b19a4b6c5c059ac7f3c95c99897e40f00736d7f8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:14:38 -0500 Subject: [PATCH 06/81] revert --- cpp/cmake/thirdparty/get_faiss.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 4e04592b7e..1d0c562cfd 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -52,8 +52,6 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() - rapids_export(BUILD faiss EXPORT_SET faissTargets) - endfunction() find_and_configure_faiss(VERSION 1.7.0 From 14785ff90fae3aca8b80d7ab85f9752fc19d95ce Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:28:44 -0500 Subject: [PATCH 07/81] create a faiss-exports export-set --- cpp/cmake/thirdparty/get_faiss.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..ef552c121a 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -46,6 +46,11 @@ function(find_and_configure_faiss) if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + rapids_export(BUILD faiss + EXPORT_SET faiss-exports + GLOBAL_TARGETS faiss + NAMESPACE raft:: + ) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) From 25b62e6986b07e65687aaec95120cb1824e5a0db Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:33:01 -0500 Subject: [PATCH 08/81] mark faiss to be found as part of resolving raft-exports dependencies --- cpp/cmake/thirdparty/get_faiss.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index ef552c121a..2c70756b15 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -57,6 +57,8 @@ function(find_and_configure_faiss) add_library(FAISS::FAISS ALIAS faiss) endif() + rapids_export_package(BUILD faiss cugraph-exports) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From 76edf367c80d4cca7febce5c4a19fea0b7da5aa8 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:45:03 -0500 Subject: [PATCH 09/81] create faiss-exports export set --- cpp/cmake/thirdparty/get_faiss.cmake | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 2c70756b15..8a8ed54859 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -44,21 +44,20 @@ function(find_and_configure_faiss) "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) + if(TARGET faiss AND NOT TARGET FAISS::FAISS) + add_library(FAISS::FAISS ALIAS faiss) + endif() + if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + install(TARGETS faiss EXPORT faiss-exports) rapids_export(BUILD faiss EXPORT_SET faiss-exports GLOBAL_TARGETS faiss NAMESPACE raft:: - ) + LANGUAGES CUDA) endif() - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) - endif() - - rapids_export_package(BUILD faiss cugraph-exports) - endfunction() find_and_configure_faiss(VERSION 1.7.0 From 92851b3acbdd96170653da005e5477605abaac90 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 30 Sep 2021 18:45:03 -0500 Subject: [PATCH 10/81] create faiss-exports export set --- cpp/cmake/thirdparty/get_faiss.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1d0c562cfd..43555cc11b 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -46,6 +46,12 @@ function(find_and_configure_faiss) if(FAISS_ADDED) target_include_directories(faiss INTERFACE $) + install(TARGETS faiss EXPORT faiss-exports) + rapids_export(BUILD faiss + EXPORT_SET faiss-exports + GLOBAL_TARGETS faiss + NAMESPACE raft:: + LANGUAGES CUDA) endif() if(TARGET faiss AND NOT TARGET FAISS::FAISS) From 8e266ef3577a317b53a9b1d84c2e9ba54b12636d Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 5 Oct 2021 17:05:48 -0500 Subject: [PATCH 11/81] export faiss::faiss target to build export set --- cpp/CMakeLists.txt | 12 +++++++++++- cpp/cmake/thirdparty/get_faiss.cmake | 28 +++++++++++++++------------- cpp/test/CMakeLists.txt | 2 +- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index cab85e8b9c..ce51813000 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -105,9 +105,9 @@ rapids_cpm_init() include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_cuco.cmake) +include(cmake/thirdparty/get_faiss.cmake) if(BUILD_TESTS) - include(cmake/thirdparty/get_faiss.cmake) include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) @@ -163,11 +163,20 @@ and common functions for the analytics components of RAPIDS. ]=]) +set(raft_code_string +[=[ +if (TARGET faiss::faiss AND (NOT TARGET faiss)) + add_library(faiss ALIAS faiss::faiss) +endif() +]=] +) + rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string + FINAL_CODE_BLOCK raft_code_string ) ############################################################################## @@ -179,6 +188,7 @@ rapids_export(BUILD raft LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: + FINAL_CODE_BLOCK raft_code_string ) ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 8a8ed54859..b82095bc00 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -29,35 +29,37 @@ function(find_and_configure_faiss) set(BUILD_SHARED_LIBS ON) endif() - rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss FAISS::FAISS - BUILD_EXPORT_SET raft-exports + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss + INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} OPTIONS "FAISS_ENABLE_PYTHON OFF" - "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" "FAISS_ENABLE_GPU ON" "BUILD_TESTING OFF" "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) endif() - if(FAISS_ADDED) - target_include_directories(faiss INTERFACE $) - install(TARGETS faiss EXPORT faiss-exports) + if(faiss_ADDED) rapids_export(BUILD faiss - EXPORT_SET faiss-exports - GLOBAL_TARGETS faiss - NAMESPACE raft:: - LANGUAGES CUDA) + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss::faiss + NAMESPACE faiss::) endif() + # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD faiss raft-exports) + + # Tell cmake where it can find the generated faiss-config.cmake we wrote. + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 6040aca26c..5761f21735 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -129,7 +129,7 @@ PRIVATE CUDA::cusparse rmm::rmm cuco::cuco - FAISS::FAISS + faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From 83e3aa682935ac145bb86458b833c310f73239ac Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 10:10:46 -0500 Subject: [PATCH 12/81] fix faiss GLOBAL_TARGETS --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index b82095bc00..ff16809824 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -50,7 +50,7 @@ function(find_and_configure_faiss) if(faiss_ADDED) rapids_export(BUILD faiss EXPORT_SET faiss-targets - GLOBAL_TARGETS faiss::faiss + GLOBAL_TARGETS faiss NAMESPACE faiss::) endif() From 2773edc97601c64b6f65633547492834c15acc66 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 13:25:55 -0500 Subject: [PATCH 13/81] update rapids-cmake version --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2e4c1b6ad7..2715f3a71f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -15,7 +15,7 @@ #============================================================================= cmake_minimum_required(VERSION 3.20.1 FATAL_ERROR) -file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.10/RAPIDS.cmake +file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-21.12/RAPIDS.cmake ${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(${CMAKE_BINARY_DIR}/RAPIDS.cmake) include(rapids-cmake) From 3ff1c4d0366a3115871189139f1cd29d42d641c2 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 18:58:14 -0500 Subject: [PATCH 14/81] fix naming for generated Findfaiss.cmake module so we find conda-installed faiss --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index ff16809824..6e0ed338d4 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,7 +19,7 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(FAISS + rapids_find_generate_module(faiss HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) From cbb6f2d2626a4d11782f14791845aee091b65aec Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 21:53:09 -0500 Subject: [PATCH 15/81] ensure faiss::faiss target is available in build and install export sets --- cpp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2715f3a71f..13de640c08 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -165,7 +165,9 @@ and common functions for the analytics components of RAPIDS. set(raft_code_string [=[ -if (TARGET faiss::faiss AND (NOT TARGET faiss)) +if(TARGET faiss AND (NOT TARGET faiss::faiss)) + add_library(faiss::faiss ALIAS faiss) +elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() ]=] From 713a872f8d2b598ed8e80700f2c7490f95848d29 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 6 Oct 2021 23:45:09 -0500 Subject: [PATCH 16/81] add gtest to raft-exports, remove dead code --- cpp/CMakeLists.txt | 5 +++-- cpp/cmake/thirdparty/get_gtest.cmake | 14 +++++++++++++- cpp/cmake/thirdparty/get_thrust.cmake | 27 --------------------------- cpp/test/CMakeLists.txt | 9 ++------- 4 files changed, 18 insertions(+), 37 deletions(-) delete mode 100644 cpp/cmake/thirdparty/get_thrust.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 13de640c08..f512df4053 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -102,13 +102,14 @@ endif() # add third party dependencies using CPM rapids_cpm_init() -include(cmake/thirdparty/get_thrust.cmake) include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_cuco.cmake) +# CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) +# CPMFindPackage(GTest) so it's available for raft consumers to use +include(cmake/thirdparty/get_gtest.cmake) if(BUILD_TESTS) - include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) endif() diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 7c234283d5..2354d3beaa 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -17,7 +17,19 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) - rapids_cpm_gtest() + rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports) + + if(GTest_ADDED) + rapids_export(BUILD GTest + VERSION ${GTest_VERSION} + EXPORT_SET GTestTargets + GLOBAL_TARGETS gtest gmock gtest_main gmock_main + NAMESPACE GTest::) + + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) + endif() endfunction() diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake deleted file mode 100644 index c28ff6e66d..0000000000 --- a/cpp/cmake/thirdparty/get_thrust.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# ============================================================================= -# Copyright (c) 2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -# Use CPM to find or clone thrust -function(find_and_configure_thrust) - include(${rapids-cmake-dir}/cpm/thrust.cmake) - - rapids_cpm_thrust( - NAMESPACE raft - BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports - ) - -endfunction() - -find_and_configure_thrust() diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5761f21735..999644361a 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -122,13 +122,8 @@ target_include_directories(test_raft target_link_libraries(test_raft PRIVATE - CUDA::cublas - CUDA::curand - CUDA::cusolver - CUDA::cudart - CUDA::cusparse - rmm::rmm - cuco::cuco + raft::raft + NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main From ceb4d8b9eb6b06a1d7832bbb77cc7bd8799f0777 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 12 Oct 2021 08:47:06 -0500 Subject: [PATCH 17/81] update CMake faiss variable name --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 7dff773b67..ecdf216aef 100755 --- a/build.sh +++ b/build.sh @@ -147,7 +147,7 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft; then -DNVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ -DBUILD_GTEST=${BUILD_GTEST} \ - -DBUILD_STATIC_FAISS=${BUILD_STATIC_FAISS} + -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} # Run all c++ targets at once From 1d3ffab3443be5116c69da59e65910899b3357cc Mon Sep 17 00:00:00 2001 From: ptaylor Date: Tue, 12 Oct 2021 09:05:56 -0500 Subject: [PATCH 18/81] make -v set cmake --log-level=VERBOSE --- build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index ecdf216aef..5dc73ce417 100755 --- a/build.sh +++ b/build.sh @@ -42,6 +42,7 @@ PYTHON_DEPS_CLONE=${REPODIR}/python/external_repositories BUILD_DIRS="${CPP_RAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PYTHON_DEPS_CLONE}" # Set defaults for vars modified by flags to this script +CMAKE_LOG_LEVEL="" VERBOSE_FLAG="" BUILD_ALL_GPU_ARCH=0 BUILD_GTEST=OFF @@ -83,6 +84,7 @@ fi # Process flags if hasArg -v; then VERBOSE_FLAG=-v + CMAKE_LOG_LEVEL="--log-level=VERBOSE" set -x fi if hasArg -g; then @@ -141,7 +143,7 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft; then echo "Building for *ALL* supported GPU architectures..." fi - cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \ + cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} ${CMAKE_LOG_LEVEL} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DNVTX=${NVTX} \ From b48e39384a0b199631c665c4565e0518f3c14c8e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 13 Oct 2021 16:17:07 -0500 Subject: [PATCH 19/81] update cuco hash to the commit that uses rapids-cmake v21.12 --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index 33e28ff622..623ce06466 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG 729857a5698a0e8d8f812e0464f65f37854ae17b + GIT_TAG 23c9a1fdbc06c29687c54016f6bb0b503535032c OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From aedc6d75ee6e5cb796e4061b5a0a7f6e44f4865f Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 19 Nov 2021 15:00:49 -0600 Subject: [PATCH 20/81] update cuco version --- cpp/cmake/thirdparty/get_cuco.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index a0c0faf0a9..fdac7cda11 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -30,4 +30,5 @@ function(find_and_configure_cuco VERSION) endfunction() -find_and_configure_cuco(0.0.1) +# cuCollections doesn't have a version yet +find_and_configure_cuco(0.0) From bb4b7c78a88a589ca6faa6892dd92ec6fe94f810 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 1 Dec 2021 17:42:40 -0600 Subject: [PATCH 21/81] move faiss into separate raft-faiss-exports export set, only include faiss if it's in the raft_FIND_COMPONENTS list --- cpp/CMakeLists.txt | 40 +++++++++++++++++++++++++--- cpp/cmake/thirdparty/get_faiss.cmake | 6 ++--- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 920be0110a..4f61b7b49e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -154,6 +154,12 @@ install(FILES include/raft.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) +include("${rapids-cmake-dir}/export/write_dependencies.cmake") +rapids_export_write_dependencies( + INSTALL raft-faiss-exports + "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-faiss-dependencies.cmake" +) + ############################################################################## # - install export ----------------------------------------------------------- set(doc_string @@ -165,7 +171,7 @@ and common functions for the analytics components of RAPIDS. ]=]) -set(raft_code_string +set(common_code_string [=[ if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) @@ -175,26 +181,52 @@ endif() ]=] ) - rapids_export(INSTALL raft +set(install_code_string +[=[ +if(faiss IN_LIST raft_FIND_COMPONENTS) + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + endif() +endif() +]=] +) + +string(APPEND install_code_string ${common_code_string}) + +rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string - FINAL_CODE_BLOCK raft_code_string + FINAL_CODE_BLOCK install_code_string ) ############################################################################## # - build export ------------------------------------------------------------- +set(build_code_string +[=[ +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") +endif() +]=] +) + +string(APPEND build_code_string ${common_code_string}) + rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: - FINAL_CODE_BLOCK raft_code_string + FINAL_CODE_BLOCK build_code_string ) +rapids_export_write_dependencies( + BUILD raft-faiss-exports "${PROJECT_BINARY_DIR}/raft-faiss-dependencies.cmake" +) + ############################################################################## # - build test executable ---------------------------------------------------- diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 6e0ed338d4..1c1052a5c3 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -31,7 +31,7 @@ function(find_and_configure_faiss) rapids_cpm_find(faiss ${PKG_VERSION} GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-faiss-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} @@ -55,11 +55,11 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` - rapids_export_package(BUILD faiss raft-exports) + rapids_export_package(BUILD faiss raft-faiss-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 From e798ab1403711a8b9e619386ed1a2468008bd254 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 1 Dec 2021 18:07:46 -0600 Subject: [PATCH 22/81] check for faiss in raft_FIND_COMPONENTS for build side export set too --- cpp/CMakeLists.txt | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4f61b7b49e..2e6b20053c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -171,56 +171,39 @@ and common functions for the analytics components of RAPIDS. ]=]) -set(common_code_string -[=[ -if(TARGET faiss AND (NOT TARGET faiss::faiss)) - add_library(faiss::faiss ALIAS faiss) -elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) - add_library(faiss ALIAS faiss::faiss) -endif() -]=] -) - -set(install_code_string +set(code_string [=[ if(faiss IN_LIST raft_FIND_COMPONENTS) if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") endif() + if(TARGET faiss AND (NOT TARGET faiss::faiss)) + add_library(faiss::faiss ALIAS faiss) + elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) + add_library(faiss ALIAS faiss::faiss) + endif() endif() ]=] ) -string(APPEND install_code_string ${common_code_string}) - rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string - FINAL_CODE_BLOCK install_code_string + FINAL_CODE_BLOCK code_string ) ############################################################################## # - build export ------------------------------------------------------------- -set(build_code_string -[=[ -if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") -endif() -]=] -) - -string(APPEND build_code_string ${common_code_string}) - rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: - FINAL_CODE_BLOCK build_code_string + FINAL_CODE_BLOCK code_string ) rapids_export_write_dependencies( From 2d5898d9983afa3fc513b31520da6161e842c90b Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:27:02 -0600 Subject: [PATCH 23/81] pass EXCLUDE_FROM_ALL in get_faiss.cmake --- cpp/cmake/thirdparty/get_faiss.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 1c1052a5c3..506f393c32 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -35,6 +35,7 @@ function(find_and_configure_faiss) CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL OPTIONS "FAISS_ENABLE_PYTHON OFF" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" From 2cb3c3a51f594051f8c0d5651e8cd5aa74e3fca4 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:33:06 -0600 Subject: [PATCH 24/81] enable CUDA language in code_string --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 2e6b20053c..27ded711e7 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -174,6 +174,7 @@ and common functions for the analytics components of RAPIDS. set(code_string [=[ if(faiss IN_LIST raft_FIND_COMPONENTS) + enable_language(CUDA) if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") endif() From 23912f5d4a8d437e579fe32bc3f413c5edd07b75 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:38:01 -0600 Subject: [PATCH 25/81] pass TRUE --- cpp/cmake/thirdparty/get_faiss.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 506f393c32..0427523f27 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -33,9 +33,9 @@ function(find_and_configure_faiss) GLOBAL_TARGETS faiss::faiss INSTALL_EXPORT_SET raft-faiss-exports CPM_ARGS - GIT_REPOSITORY https://github.com/facebookresearch/faiss.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL + GIT_REPOSITORY https://github.com/facebookresearch/faiss.git + GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE OPTIONS "FAISS_ENABLE_PYTHON OFF" "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" From fc306f6dfa14da87b7edc5a7dab7a53d6494a9a7 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 14:47:45 -0600 Subject: [PATCH 26/81] drop CUDA from the list of global languages --- cpp/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 27ded711e7..fd19706061 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -201,7 +201,6 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports GLOBAL_TARGETS raft # since we can't hook into EXPORT SETS - LANGUAGES CUDA DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From 61c263af1c3d96567ee30f39bf17fadf37b48955 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 2 Dec 2021 15:40:36 -0600 Subject: [PATCH 27/81] link raft to faiss if faiss component is requested --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fd19706061..147fab2913 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -183,6 +183,7 @@ if(faiss IN_LIST raft_FIND_COMPONENTS) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() + target_link_libraries(raft INTERFACE faiss::faiss) endif() ]=] ) From 41bd51b944764d35dc13f013323a77ea557ca4f5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 8 Dec 2021 15:35:45 -0600 Subject: [PATCH 28/81] link raft to faiss if faiss component requested --- cpp/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 147fab2913..01d0fdc830 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -183,7 +183,9 @@ if(faiss IN_LIST raft_FIND_COMPONENTS) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() - target_link_libraries(raft INTERFACE faiss::faiss) + if(TARGET raft::raft) + target_link_libraries(raft::raft INTERFACE faiss::faiss) + endif() endif() ]=] ) From ce8a00592387859005d8e0bc6c736256a08106ed Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 15 Dec 2021 18:14:52 -0600 Subject: [PATCH 29/81] fix FAISS::FAISS -> faiss::faiss --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0bf79f8d0f..1b6af1113b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -153,7 +153,7 @@ set(RAFT_LINK_LIBRARIES target_link_libraries(raft INTERFACE ${RAFT_LINK_LIBRARIES}) target_link_libraries(raft_distance PUBLIC ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} FAISS::FAISS) +target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} faiss::faiss) target_compile_options(raft_distance PRIVATE "$<$:${RAFT_CXX_FLAGS}>" From c39685a7aab08eb80c172ebf1612a7d898f9af55 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 09:33:33 -0500 Subject: [PATCH 30/81] Simplify raft component CMake logic, and allow compilation without FAISS --- cpp/CMakeLists.txt | 95 +++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index efebfff429..1cd3ab55d5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -114,31 +114,12 @@ if(BUILD_TESTS) endif() ############################################################################## -# - install targets----------------------------------------------------------- -rapids_cmake_install_lib_dir( lib_dir ) - -include(CPack) - -file(GLOB_RECURSE RAFT_DISTANCE_SOURCES "src/distance/specializations/*.cu") -file(GLOB_RECURSE RAFT_NN_SOURCES "src/nn/specializations/*.cu" ) - -add_library(raft_distance SHARED ${RAFT_DISTANCE_SOURCES}) -add_library(raft::raft_distance ALIAS raft_distance) - -add_library(raft_nn SHARED ${RAFT_NN_SOURCES}) -add_library(raft::raft_nn ALIAS raft_nn) +# - raft --------------------------------------------------------------------- add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) target_include_directories(raft INTERFACE "$" "$") - -target_include_directories(raft_distance PUBLIC "$" - "$") - -target_include_directories(raft_nn PUBLIC "$" - "$") - set(RAFT_LINK_LIBRARIES CUDA::cublas CUDA::curand @@ -149,47 +130,67 @@ set(RAFT_LINK_LIBRARIES rmm::rmm cuco::cuco ) - target_link_libraries(raft INTERFACE ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_distance PUBLIC ${RAFT_LINK_LIBRARIES}) -target_link_libraries(raft_nn PUBLIC ${RAFT_LINK_LIBRARIES} FAISS::FAISS) set(RAFT_COMPILE_DEFINITIONS $<$:NVTX_ENABLED> ) - target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) -target_compile_definitions(raft_distance PRIVATE ${RAFT_COMPILE_DEFINITIONS}) -target_compile_definitions(raft_nn PRIVATE ${RAFT_COMPILE_DEFINITIONS}) - -target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - +target_compile_features(raft INTERFACE cxx_std_17 $) -target_compile_options(raft_nn - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) +############################################################################## +# - raft_distance ------------------------------------------------------------ +add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu +) -target_compile_features(raft_distance PUBLIC cxx_std_17 $) -target_compile_features(raft_nn PUBLIC cxx_std_17 $) -target_compile_features(raft INTERFACE cxx_std_17 $) +add_library(raft::raft_distance ALIAS raft_distance) +target_link_libraries(raft_distance PUBLIC raft::raft) -install(TARGETS raft_distance - DESTINATION ${lib_dir} - EXPORT raft-exports) +############################################################################## +# - raft_nn ------------------------------------------------------------------ + +if(TARGET FAISS::FAISS) + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) + add_library(raft::raft_nn ALIAS raft_nn) + + target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) +endif() -install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) +############################################################################## +# - install targets----------------------------------------------------------- +rapids_cmake_install_lib_dir( lib_dir ) +include(GNUInstallDirs) +include(CPack) -install(TARGETS raft +install(TARGETS raft raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) +if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) +endif() -include(GNUInstallDirs) install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) From af46a094c9211278004eb86d5ed4ec2e4816dc9e Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 13:20:02 -0500 Subject: [PATCH 31/81] Add RAFT_ENABLE_NN_COMPONENT CMake build option --- cpp/CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1cd3ab55d5..924ae60869 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,7 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) +option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -56,6 +57,7 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") +message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -106,8 +108,11 @@ include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) -if(BUILD_TESTS) +if(BUILD_TESTS OR RAFT_ENABLE_NN_COMPONENT) include(cmake/thirdparty/get_faiss.cmake) +endif() + +if(BUILD_TESTS) include(cmake/thirdparty/get_gtest.cmake) include(cmake/thirdparty/get_nccl.cmake) include(cmake/thirdparty/get_ucx.cmake) @@ -164,7 +169,7 @@ target_link_libraries(raft_distance PUBLIC raft::raft) ############################################################################## # - raft_nn ------------------------------------------------------------------ -if(TARGET FAISS::FAISS) +if(RAFT_ENABLE_NN_COMPONENT) add_library(raft_nn SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu From 1e9b9bd496477d812aa1f1b1d34ef8925d77e781 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 14:28:42 -0500 Subject: [PATCH 32/81] Adding option to turn off buildign all shared libs --- cpp/CMakeLists.txt | 86 +++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 924ae60869..8a8235e07c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -48,6 +48,8 @@ option(DETECT_CONDA_ENV "Enable detection of conda environment for dependencies" option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) + +option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") @@ -143,42 +145,53 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) +if(RAFT_BUILD_SHARED_LIBS) ############################################################################## # - raft_distance ------------------------------------------------------------ -add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu -) - -add_library(raft::raft_distance ALIAS raft_distance) -target_link_libraries(raft_distance PUBLIC raft::raft) + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) + + add_library(raft::raft_distance ALIAS raft_distance) + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) ############################################################################## # - raft_nn ------------------------------------------------------------------ -if(RAFT_ENABLE_NN_COMPONENT) - add_library(raft_nn SHARED - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim.cu - src/nn/specializations/fused_l2_knn.cu - src/nn/specializations/knn.cu - ) - add_library(raft::raft_nn ALIAS raft_nn) + if(RAFT_ENABLE_NN_COMPONENT) + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) + add_library(raft::raft_nn ALIAS raft_nn) + + target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) + target_compile_options(raft_nn + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + endif() - target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) endif() ############################################################################## @@ -187,15 +200,24 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft raft_distance +install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(TARGET raft_nn) - install(TARGETS raft_nn + +if(RAFT_BUILD_SHARED_LIBS) + install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) + + if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) + endif() endif() + + install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) From 7fc712cc4efcaa1e58463e25d7491017a691dcae Mon Sep 17 00:00:00 2001 From: Paul Taylor Date: Mon, 20 Dec 2021 14:44:15 -0500 Subject: [PATCH 33/81] Refactor get_faiss to export faiss::faiss build target --- cpp/CMakeLists.txt | 3 +-- cpp/cmake/thirdparty/get_faiss.cmake | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8a8235e07c..4385a2158e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -185,13 +185,12 @@ if(RAFT_BUILD_SHARED_LIBS) ) add_library(raft::raft_nn ALIAS raft_nn) - target_link_libraries(raft_nn PUBLIC raft::raft FAISS::FAISS) + target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) endif() - endif() ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index a65401579c..712c3900d1 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -23,12 +23,12 @@ function(find_and_configure_faiss) HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) - - rapids_cpm_find(FAISS ${PKG_VERSION} - GLOBAL_TARGETS faiss + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE OPTIONS "FAISS_ENABLE_PYTHON OFF" "BUILD_SHARED_LIBS OFF" @@ -38,14 +38,24 @@ function(find_and_configure_faiss) "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" ) - if(FAISS_ADDED) - set(FAISS_GPU_HEADERS ${FAISS_SOURCE_DIR} PARENT_SCOPE) + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) endif() - if(TARGET faiss AND NOT TARGET FAISS::FAISS) - add_library(FAISS::FAISS ALIAS faiss) + if(faiss_ADDED) + rapids_export(BUILD faiss + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss + NAMESPACE faiss::) endif() + # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD faiss raft-faiss-exports) + + # Tell cmake where it can find the generated faiss-config.cmake we wrote. + include("${rapids-cmake-dir}/export/find_package_root.cmake") + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) + endfunction() find_and_configure_faiss(VERSION 1.7.0 From a85b085cbbefba1d71f9ae30b8efd885220b10b6 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Mon, 20 Dec 2021 15:02:59 -0500 Subject: [PATCH 34/81] RAFT_ENABLE_NN_COMPONENT is a dependent option of RAFT_BUILD_SHARED_LIBS --- cpp/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4385a2158e..aa0a07cf38 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,7 +50,8 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) -option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON) +include(CMakeDependentOption) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_BUILD_SHARED_LIBS OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") From 260a727b792ac26af15251e1ad8298603ee23336 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 17:35:07 -0500 Subject: [PATCH 35/81] Using lowercase faiss target --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 712c3900d1..b74ede9381 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,7 +19,7 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(FAISS + rapids_find_generate_module(faiss HEADER_NAMES faiss/IndexFlat.h LIBRARY_NAMES faiss ) From bb4a7f32fed41ef310910dfe299c25fca3f40f69 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 20 Dec 2021 17:52:08 -0500 Subject: [PATCH 36/81] Changing RAFT_BUILD_SHARED_LIBS -> RAFT_COMPILE_LIBRARIES --- cpp/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index aa0a07cf38..0048a2398b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,9 +49,9 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_BUILD_SHARED_LIBS "Enable building raft shared library instantiations" ON) +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_BUILD_SHARED_LIBS OFF) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -146,7 +146,7 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) -if(RAFT_BUILD_SHARED_LIBS) +if(RAFT_COMPILE_LIBRARIES) ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance SHARED @@ -204,7 +204,7 @@ install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(RAFT_BUILD_SHARED_LIBS) +if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) From 8c02d2c395f908300481db0fae9d8d9cde866742 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 21 Dec 2021 12:14:04 -0500 Subject: [PATCH 37/81] Passing deps transitively --- cpp/CMakeLists.txt | 98 +++++++++++++++++++++++++---------------- cpp/test/CMakeLists.txt | 1 - 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0048a2398b..1c2aa8cda1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,9 +49,11 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) + +cmake_dependent_option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON BUILD_TESTS OFF) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON BUILD_TESTS OFF) +cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON BUILD_TESTS OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -126,8 +128,11 @@ endif() add_library(raft INTERFACE) add_library(raft::raft ALIAS raft) -target_include_directories(raft INTERFACE "$" - "$") + +target_include_directories(raft INTERFACE + "$" + "$") + set(RAFT_LINK_LIBRARIES CUDA::cublas CUDA::curand @@ -146,52 +151,71 @@ set(RAFT_COMPILE_DEFINITIONS target_compile_definitions(raft INTERFACE ${RAFT_COMPILE_DEFINITIONS}) target_compile_features(raft INTERFACE cxx_std_17 $) -if(RAFT_COMPILE_LIBRARIES) ############################################################################## # - raft_distance ------------------------------------------------------------ - add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu - ) +if(RAFT_ENABLE_DISTANCE_COMPONENT) + + if(NOT RAFT_COMPILE_LIBRARIES) + add_library(raft_distance INTERFACE) + + else() + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) + + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) + endif() add_library(raft::raft_distance ALIAS raft_distance) - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) +endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ - if(RAFT_ENABLE_NN_COMPONENT) +if(RAFT_ENABLE_NN_COMPONENT) + + set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) + + if(NOT RAFT_COMPILE_LIBRARIES) + add_library(raft_nn INTERFACE) + target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) + + else() add_library(raft_nn SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu src/nn/specializations/fused_l2_knn.cu src/nn/specializations/knn.cu ) - add_library(raft::raft_nn ALIAS raft_nn) target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_include_directories(raft_nn PUBLIC ${RAFT_NN_INCLUDE_DIRECTORIES}) endif() + + add_library(raft::raft_nn ALIAS raft_nn) + endif() ############################################################################## @@ -204,19 +228,17 @@ install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) -if(RAFT_COMPILE_LIBRARIES) +if(TARGET raft_distance) install(TARGETS raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) - - if(TARGET raft_nn) - install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) - endif() endif() - +if(TARGET raft_nn) + install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-exports) +endif() install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft @@ -237,7 +259,7 @@ mathematical computational primitives, and utilities that accelerate building analytics and data science algorithms in the RAPIDS ecosystem. ]=]) - rapids_export(INSTALL raft +rapids_export(INSTALL raft EXPORT_SET raft-exports GLOBAL_TARGETS raft raft_distance# since we can't hook into EXPORT SETS NAMESPACE raft:: diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index b37c671525..ca4999c1df 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -119,7 +119,6 @@ target_compile_options(test_raft target_include_directories(test_raft PUBLIC "$" - "${FAISS_GPU_HEADERS}" ) From ebb7c1d7c3bfa1352fea6662b0f819a8e97f1aed Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 21 Dec 2021 14:47:06 -0500 Subject: [PATCH 38/81] Moving faiss back to test target. We need to figure out how to pass dependencies through transitively --- cpp/CMakeLists.txt | 19 ++++++++++++++----- cpp/test/CMakeLists.txt | 1 + 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 1c2aa8cda1..b68061dcff 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,11 +49,15 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -include(CMakeDependentOption) +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" OFF) +option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" OFF) +option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" OFF) -cmake_dependent_option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON BUILD_TESTS OFF) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON BUILD_TESTS OFF) -cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON BUILD_TESTS OFF) +if(BUILD_TESTS) + set(RAFT_COMPILE_LIBRARIES ON) + set(RAFT_ENABLE_NN_COMPONENT ON) + set(RAFT_ENABLE_DISTANCE_COMPONENT ON) +endif() message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -155,6 +159,8 @@ target_compile_features(raft INTERFACE cxx_std_17 $ # - raft_distance ------------------------------------------------------------ if(RAFT_ENABLE_DISTANCE_COMPONENT) + message("RAFT-distance component enabled.") + if(NOT RAFT_COMPILE_LIBRARIES) add_library(raft_distance INTERFACE) @@ -192,10 +198,13 @@ endif() if(RAFT_ENABLE_NN_COMPONENT) + message("RAFT-nn component enabled.") + set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) if(NOT RAFT_COMPILE_LIBRARIES) add_library(raft_nn INTERFACE) + target_link_libraries(raft_nn INTERFACE raft::raft) target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) else() @@ -206,7 +215,7 @@ if(RAFT_ENABLE_NN_COMPONENT) src/nn/specializations/knn.cu ) - target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) + target_link_libraries(raft_nn PUBLIC raft::raft) target_compile_options(raft_nn PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index ca4999c1df..3666bd8f2c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -127,6 +127,7 @@ PRIVATE raft # transitively links all CUDA libs, etc raft_distance raft_nn + faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From c1e9c2d296fadca04d3a50a2fc7209740aac5b4e Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 22 Dec 2021 09:17:46 -0500 Subject: [PATCH 39/81] Another round of cmake cleanups --- cpp/CMakeLists.txt | 106 ++++++++++----------------- cpp/cmake/thirdparty/get_faiss.cmake | 1 + 2 files changed, 40 insertions(+), 67 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b68061dcff..958666d3bb 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -49,15 +49,10 @@ option(DISABLE_DEPRECATION_WARNINGS "Disable depreaction warnings " ON) option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) -option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" OFF) -option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" OFF) -option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" OFF) - -if(BUILD_TESTS) - set(RAFT_COMPILE_LIBRARIES ON) - set(RAFT_ENABLE_NN_COMPONENT ON) - set(RAFT_ENABLE_DISTANCE_COMPONENT ON) -endif() +option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ${BUILD_TESTS}) +include(CMakeDependentOption) +cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) +cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -67,6 +62,7 @@ message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELIN message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") +message(VERBOSE "RAFT: Enable raft::distance target: ${RAFT_ENABLE_DISTANCE_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -158,73 +154,49 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ if(RAFT_ENABLE_DISTANCE_COMPONENT) - - message("RAFT-distance component enabled.") - - if(NOT RAFT_COMPILE_LIBRARIES) - add_library(raft_distance INTERFACE) - - else() - add_library(raft_distance SHARED - src/distance/specializations/detail - src/distance/specializations/detail/canberra.cu - src/distance/specializations/detail/chebyshev.cu - src/distance/specializations/detail/correlation.cu - src/distance/specializations/detail/cosine.cu - src/distance/specializations/detail/hamming_unexpanded.cu - src/distance/specializations/detail/hellinger_expanded.cu - src/distance/specializations/detail/jensen_shannon.cu - src/distance/specializations/detail/kl_divergence.cu - src/distance/specializations/detail/l1.cu - src/distance/specializations/detail/l2_expanded.cu - src/distance/specializations/detail/l2_sqrt_expanded.cu - src/distance/specializations/detail/l2_sqrt_unexpanded.cu - src/distance/specializations/detail/l2_unexpanded.cu - src/distance/specializations/detail/lp_unexpanded.cu - ) - - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - endif() - + add_library(raft_distance SHARED + src/distance/specializations/detail + src/distance/specializations/detail/canberra.cu + src/distance/specializations/detail/chebyshev.cu + src/distance/specializations/detail/correlation.cu + src/distance/specializations/detail/cosine.cu + src/distance/specializations/detail/hamming_unexpanded.cu + src/distance/specializations/detail/hellinger_expanded.cu + src/distance/specializations/detail/jensen_shannon.cu + src/distance/specializations/detail/kl_divergence.cu + src/distance/specializations/detail/l1.cu + src/distance/specializations/detail/l2_expanded.cu + src/distance/specializations/detail/l2_sqrt_expanded.cu + src/distance/specializations/detail/l2_sqrt_unexpanded.cu + src/distance/specializations/detail/l2_unexpanded.cu + src/distance/specializations/detail/lp_unexpanded.cu + ) add_library(raft::raft_distance ALIAS raft_distance) + + target_link_libraries(raft_distance PUBLIC raft::raft) + target_compile_options(raft_distance + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ if(RAFT_ENABLE_NN_COMPONENT) - - message("RAFT-nn component enabled.") - - set(RAFT_NN_INCLUDE_DIRECTORIES ${FAISS_GPU_INCLUDES}) - - if(NOT RAFT_COMPILE_LIBRARIES) - add_library(raft_nn INTERFACE) - target_link_libraries(raft_nn INTERFACE raft::raft) - target_include_directories(raft_nn INTERFACE ${RAFT_NN_INCLUDE_DIRECTORIES}) - - else() - add_library(raft_nn SHARED - src/nn/specializations/ball_cover.cu - src/nn/specializations/detail/ball_cover_lowdim.cu - src/nn/specializations/fused_l2_knn.cu - src/nn/specializations/knn.cu - ) - - target_link_libraries(raft_nn PUBLIC raft::raft) - target_compile_options(raft_nn - PRIVATE "$<$:${RAFT_CXX_FLAGS}>" - "$<$:${RAFT_CUDA_FLAGS}>" - ) - target_include_directories(raft_nn PUBLIC ${RAFT_NN_INCLUDE_DIRECTORIES}) - endif() - + add_library(raft_nn SHARED + src/nn/specializations/ball_cover.cu + src/nn/specializations/detail/ball_cover_lowdim.cu + src/nn/specializations/fused_l2_knn.cu + src/nn/specializations/knn.cu + ) add_library(raft::raft_nn ALIAS raft_nn) + target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) + target_compile_options(raft_nn + PRIVATE "$<$:${RAFT_CXX_FLAGS}>" + "$<$:${RAFT_CUDA_FLAGS}>" + ) endif() ############################################################################## diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index b74ede9381..09251c0a55 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -51,6 +51,7 @@ function(find_and_configure_faiss) # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` rapids_export_package(BUILD faiss raft-faiss-exports) + rapids_export_package(INSTALL faiss raft-faiss-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") From 52e863559430b473eaad22f7aa9ce0a5e7846dce Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 22 Dec 2021 11:28:19 -0500 Subject: [PATCH 40/81] Refactor again now that I understand the libs are an optimization --- cpp/CMakeLists.txt | 70 +++++++++++++++-------------- cpp/test/CMakeLists.txt | 2 +- cpp/test/distance/distance_base.cuh | 2 + cpp/test/sparse/knn_graph.cu | 2 + cpp/test/spatial/ball_cover.cu | 4 +- cpp/test/spatial/knn.cu | 2 + cpp/test/spatial/selection.cu | 2 + 7 files changed, 48 insertions(+), 36 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 31d50a1b74..f112f5eace 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -51,10 +51,7 @@ option(NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) include(CMakeDependentOption) -cmake_dependent_option(RAFT_ENABLE_NN_COMPONENT "Enable the raft::nn component" ON RAFT_COMPILE_LIBRARIES OFF) -cmake_dependent_option(RAFT_ENABLE_DISTANCE_COMPONENT "Enable the raft::distance component" ON RAFT_COMPILE_LIBRARIES OFF) - -cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_ENABLE_NN_COMPONENT OFF) +cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) message(VERBOSE "RAFT: Build RAFT unit-tests: ${BUILD_TESTS}") message(VERBOSE "RAFT: Enable detection of conda environment for dependencies: ${DETECT_CONDA_ENV}") @@ -112,16 +109,14 @@ rapids_cpm_init() include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) +# CPMFindPackage(faiss) so it's available for raft consumers to use +include(cmake/thirdparty/get_faiss.cmake) # CPMFindPackage(GTest) so it's available for raft consumers to use include(cmake/thirdparty/get_gtest.cmake) -if(BUILD_TESTS OR RAFT_ENABLE_NN_COMPONENT) - include(cmake/thirdparty/get_faiss.cmake) -endif() - if(BUILD_TESTS) - include(cmake/thirdparty/get_nccl.cmake) - include(cmake/thirdparty/get_ucx.cmake) + # include(cmake/thirdparty/get_nccl.cmake) + # include(cmake/thirdparty/get_ucx.cmake) endif() ############################################################################## @@ -149,8 +144,12 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ -if(RAFT_ENABLE_DISTANCE_COMPONENT) - add_library(raft_distance SHARED +add_library(raft_distance INTERFACE) +add_library(raft::raft_distance ALIAS raft_distance) +target_link_libraries(raft_distance INTERFACE raft::raft $) + +if(RAFT_COMPILE_LIBRARIES) + add_library(raft_distance_lib SHARED src/distance/specializations/detail src/distance/specializations/detail/canberra.cu src/distance/specializations/detail/chebyshev.cu @@ -167,32 +166,47 @@ if(RAFT_ENABLE_DISTANCE_COMPONENT) src/distance/specializations/detail/l2_unexpanded.cu src/distance/specializations/detail/lp_unexpanded.cu ) - add_library(raft::raft_distance ALIAS raft_distance) + set_target_properties(raft_distance_lib PROPERTIES OUTPUT_NAME raft_distance) - target_link_libraries(raft_distance PUBLIC raft::raft) - target_compile_options(raft_distance + target_link_libraries(raft_distance_lib PRIVATE raft::raft) + target_compile_options(raft_distance_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_compile_definitions(raft_distance_lib + INTERFACE "RAFT_DISTANCE_COMPILED") + + install(TARGETS raft_distance_lib + DESTINATION ${lib_dir} + EXPORT raft-exports) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ +add_library(raft_nn INTERFACE) +add_library(raft::raft_nn ALIAS raft_nn) +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) -if(RAFT_ENABLE_NN_COMPONENT) - add_library(raft_nn SHARED +if(RAFT_COMPILE_LIBRARIES) + add_library(raft_nn_lib SHARED src/nn/specializations/ball_cover.cu src/nn/specializations/detail/ball_cover_lowdim.cu src/nn/specializations/fused_l2_knn.cu src/nn/specializations/knn.cu ) - add_library(raft::raft_nn ALIAS raft_nn) + set_target_properties(raft_nn_lib PROPERTIES OUTPUT_NAME raft_nn) - target_link_libraries(raft_nn PUBLIC raft::raft faiss::faiss) - target_compile_options(raft_nn + target_link_libraries(raft_nn_lib PRIVATE raft::raft faiss::faiss) + target_compile_options(raft_nn_lib PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) + target_compile_definitions(raft_distance_lib + INTERFACE "RAFT_NN_COMPILED") + + install(TARGETS raft_nn_lib + DESTINATION ${lib_dir} + EXPORT raft-exports) endif() ############################################################################## @@ -201,22 +215,10 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft +install(TARGETS raft raft_nn raft_distance DESTINATION ${lib_dir} EXPORT raft-exports) -if(TARGET raft_distance) - install(TARGETS raft_distance - DESTINATION ${lib_dir} - EXPORT raft-exports) -endif() - -if(TARGET raft_nn) - install(TARGETS raft_nn - DESTINATION ${lib_dir} - EXPORT raft-exports) -endif() - install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft ) @@ -263,7 +265,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 5db9c55703..77c394527d 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -127,7 +127,7 @@ PRIVATE raft::raft raft::raft_distance raft::raft_nn - NCCL::NCCL + # NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main diff --git a/cpp/test/distance/distance_base.cuh b/cpp/test/distance/distance_base.cuh index 475202137b..8f0de29eed 100644 --- a/cpp/test/distance/distance_base.cuh +++ b/cpp/test/distance/distance_base.cuh @@ -20,7 +20,9 @@ #include #include #include +#if defined RAFT_DISTANCE_COMPILED #include +#endif #include namespace raft { diff --git a/cpp/test/sparse/knn_graph.cu b/cpp/test/sparse/knn_graph.cu index df9bb4e3e4..88a3f24df6 100644 --- a/cpp/test/sparse/knn_graph.cu +++ b/cpp/test/sparse/knn_graph.cu @@ -23,7 +23,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif #include diff --git a/cpp/test/spatial/ball_cover.cu b/cpp/test/spatial/ball_cover.cu index 7b44c477aa..257950e4d7 100644 --- a/cpp/test/spatial/ball_cover.cu +++ b/cpp/test/spatial/ball_cover.cu @@ -20,9 +20,11 @@ #include #include #include +#if defined RAFT_NN_COMPILED #include -#include +#endif +#include #include #include diff --git a/cpp/test/spatial/knn.cu b/cpp/test/spatial/knn.cu index 5681f66e25..8af1505bcd 100644 --- a/cpp/test/spatial/knn.cu +++ b/cpp/test/spatial/knn.cu @@ -19,7 +19,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif #include diff --git a/cpp/test/spatial/selection.cu b/cpp/test/spatial/selection.cu index 4409f893a8..8ccf3b6b73 100644 --- a/cpp/test/spatial/selection.cu +++ b/cpp/test/spatial/selection.cu @@ -21,7 +21,9 @@ #include #include +#if defined RAFT_NN_COMPILED #include +#endif namespace raft { namespace spatial { From 293302f3a19c4ff551c6d912482438175ea74435 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Tue, 4 Jan 2022 14:42:35 -0500 Subject: [PATCH 41/81] Move `nn` and `distance` to CMake components --- cpp/CMakeLists.txt | 100 +++++++++++++++++++-------- cpp/cmake/thirdparty/get_faiss.cmake | 7 +- cpp/cmake/thirdparty/get_gtest.cmake | 1 - cpp/test/CMakeLists.txt | 6 +- 4 files changed, 79 insertions(+), 35 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f112f5eace..69f918e88b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -111,12 +111,11 @@ include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) # CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) -# CPMFindPackage(GTest) so it's available for raft consumers to use -include(cmake/thirdparty/get_gtest.cmake) if(BUILD_TESTS) - # include(cmake/thirdparty/get_nccl.cmake) - # include(cmake/thirdparty/get_ucx.cmake) + include(cmake/thirdparty/get_gtest.cmake) + include(cmake/thirdparty/get_nccl.cmake) + include(cmake/thirdparty/get_ucx.cmake) endif() ############################################################################## @@ -145,7 +144,9 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance INTERFACE) -add_library(raft::raft_distance ALIAS raft_distance) +add_library(raft::distance ALIAS raft_distance) +set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) + target_link_libraries(raft_distance INTERFACE raft::raft $) if(RAFT_COMPILE_LIBRARIES) @@ -178,13 +179,15 @@ if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_distance_lib DESTINATION ${lib_dir} - EXPORT raft-exports) + EXPORT raft-distance-exports) endif() ############################################################################## # - raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) -add_library(raft::raft_nn ALIAS raft_nn) +add_library(raft::nn ALIAS raft_nn) +set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) + target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) if(RAFT_COMPILE_LIBRARIES) @@ -206,7 +209,7 @@ if(RAFT_COMPILE_LIBRARIES) install(TARGETS raft_nn_lib DESTINATION ${lib_dir} - EXPORT raft-exports) + EXPORT raft-nn-exports) endif() ############################################################################## @@ -215,9 +218,16 @@ rapids_cmake_install_lib_dir( lib_dir ) include(GNUInstallDirs) include(CPack) -install(TARGETS raft raft_nn raft_distance +install(TARGETS raft DESTINATION ${lib_dir} EXPORT raft-exports) +install(TARGETS raft_distance + DESTINATION ${lib_dir} + EXPORT raft-distance-exports) +install(TARGETS raft_nn + DESTINATION ${lib_dir} + EXPORT raft-nn-exports) + install(DIRECTORY include/raft/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft @@ -227,12 +237,6 @@ install(DIRECTORY include/raft/ install(FILES include/raft.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/raft) -include("${rapids-cmake-dir}/export/write_dependencies.cmake") -rapids_export_write_dependencies( - INSTALL raft-faiss-exports - "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-faiss-dependencies.cmake" -) - ############################################################################## # - install export ----------------------------------------------------------- set(doc_string @@ -242,30 +246,49 @@ Provide targets for the RAFT: RAPIDS Analytics Framework Toolkit. RAPIDS Analytics Framework Toolkit contains shared representations, mathematical computational primitives, and utilities that accelerate building analytics and data science algorithms in the RAPIDS ecosystem. + +Optional Components: + - nn + - distance + +Imported Targets: + - raft::raft + - raft::nn brought in by the `nn` optional component + - raft::distance brought in by the `distance` optional component + ]=]) set(code_string [=[ -if(faiss IN_LIST raft_FIND_COMPONENTS) - enable_language(CUDA) - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-faiss-dependencies.cmake") +set(raft_allowed_components distance nn) +foreach(comp IN LISTS raft_allowed_components) + if(${comp} IN_LIST raft_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/raft-distance-targets.cmake") + + if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") + endif() endif() + + enable_language(CUDA) + +endforeach() + +if(nn IN_LIST raft_FIND_COMPONENTS) if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) add_library(faiss ALIAS faiss::faiss) endif() - if(TARGET raft::raft) - target_link_libraries(raft::raft INTERFACE faiss::faiss) - endif() endif() ]=] ) +# enable_cuda + rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -276,15 +299,38 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn# since we can't hook into EXPORT SETS + GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string ) -rapids_export_write_dependencies( - BUILD raft-faiss-exports "${PROJECT_BINARY_DIR}/raft-faiss-dependencies.cmake" -) +############################################################################## +# - export/install optional components -------------------------------------- + +include("${rapids-cmake-dir}/export/write_dependencies.cmake") + +set(raft_components distance nn) +foreach(comp IN LISTS raft_components) + install( + EXPORT raft-${comp}-exports + FILE raft-${comp}-targets.cmake + NAMESPACE raft:: + DESTINATION "${lib_dir}/cmake/raft" + ) + export( + EXPORT raft-${comp}-exports + FILE ${RAFT_BINARY_DIR}/raft-${comp}-targets.cmake + NAMESPACE raft:: + ) + rapids_export_write_dependencies( + BUILD raft-${comp}-exports "${PROJECT_BINARY_DIR}/raft-${comp}-dependencies.cmake" + ) + rapids_export_write_dependencies( + INSTALL raft-${comp}-exports "${PROJECT_BINARY_DIR}/rapids-cmake/raft/export/raft-${comp}-dependencies.cmake" + ) + +endforeach() ############################################################################## # - build test executable ---------------------------------------------------- diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 666619b2db..bb61a9f4e7 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -31,7 +31,7 @@ function(find_and_configure_faiss) rapids_cpm_find(faiss ${PKG_VERSION} GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-faiss-exports + INSTALL_EXPORT_SET raft-nn-exports CPM_ARGS GIT_REPOSITORY https://github.com/facebookresearch/faiss.git GIT_TAG ${PKG_PINNED_TAG} @@ -56,12 +56,11 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` - rapids_export_package(BUILD faiss raft-faiss-exports) - rapids_export_package(INSTALL faiss raft-faiss-exports) + rapids_export_package(BUILD faiss raft-nn-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") - rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-faiss-exports) + rapids_export_find_package_root(BUILD faiss [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-nn-exports) endfunction() find_and_configure_faiss(VERSION 1.7.0 diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 885c0f6bed..72fb0e18c6 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,7 +18,6 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 77c394527d..59d34e7a2c 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -125,9 +125,9 @@ target_include_directories(test_raft target_link_libraries(test_raft PRIVATE raft::raft - raft::raft_distance - raft::raft_nn - # NCCL::NCCL + raft::distance + raft::nn + NCCL::NCCL faiss::faiss GTest::gtest GTest::gtest_main From 61e5b3cece0aa4c9625eeac131fe326c13114a97 Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Thu, 6 Jan 2022 10:34:54 -0500 Subject: [PATCH 42/81] Make searching for faiss a controllable option --- cpp/CMakeLists.txt | 4 +- cpp/cmake/thirdparty/get_faiss.cmake | 63 +++++++++++++++------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 69f918e88b..c050872e18 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -50,6 +50,7 @@ option(DISABLE_OPENMP "Disable OpenMP" OFF) option(NVTX "Enable nvtx markers" OFF) option(RAFT_COMPILE_LIBRARIES "Enable building raft shared library instantiations" ON) +option(RAFT_ENABLE_NN_DEPENDENCIES "Search for raft::nn dependencies like faiss" ${RAFT_COMPILE_LIBRARIES}) include(CMakeDependentOption) cmake_dependent_option(RAFT_USE_FAISS_STATIC "Build and statically link the FAISS library for nearest neighbors search on GPU" ON RAFT_COMPILE_LIBRARIES OFF) @@ -60,8 +61,6 @@ message(VERBOSE "RAFT: Disable OpenMP: ${DISABLE_OPENMP}") message(VERBOSE "RAFT: Enable kernel resource usage info: ${CUDA_ENABLE_KERNELINFO}") message(VERBOSE "RAFT: Enable lineinfo in nvcc: ${CUDA_ENABLE_LINEINFO}") message(VERBOSE "RAFT: Enable nvtx markers: ${NVTX}") -message(VERBOSE "RAFT: Enable raft::nn target: ${RAFT_ENABLE_NN_COMPONENT}") -message(VERBOSE "RAFT: Enable raft::distance target: ${RAFT_ENABLE_DISTANCE_COMPONENT}") message(VERBOSE "RAFT: Statically link the CUDA runtime: ${CUDA_STATIC_RUNTIME}") # Set RMM logging level @@ -109,7 +108,6 @@ rapids_cpm_init() include(cmake/thirdparty/get_rmm.cmake) include(cmake/thirdparty/get_libcudacxx.cmake) include(cmake/thirdparty/get_cuco.cmake) -# CPMFindPackage(faiss) so it's available for raft consumers to use include(cmake/thirdparty/get_faiss.cmake) if(BUILD_TESTS) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index bb61a9f4e7..59c1d4ba2b 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -19,44 +19,47 @@ function(find_and_configure_faiss) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - rapids_find_generate_module(faiss - HEADER_NAMES faiss/IndexFlat.h - LIBRARY_NAMES faiss - ) + if(RAFT_ENABLE_NN_DEPENDENCIES OR RAFT_COMPILE_LIBRARIES) + rapids_find_generate_module(faiss + HEADER_NAMES faiss/IndexFlat.h + LIBRARY_NAMES faiss + ) - set(BUILD_SHARED_LIBS OFF) - if (NOT BUILD_STATIC_LIBS) - set(BUILD_SHARED_LIBS ON) - endif() + set(BUILD_SHARED_LIBS OFF) + if (NOT BUILD_STATIC_LIBS) + set(BUILD_SHARED_LIBS ON) + endif() - rapids_cpm_find(faiss ${PKG_VERSION} - GLOBAL_TARGETS faiss::faiss - INSTALL_EXPORT_SET raft-nn-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/facebookresearch/faiss.git - GIT_TAG ${PKG_PINNED_TAG} - EXCLUDE_FROM_ALL TRUE - OPTIONS - "FAISS_ENABLE_PYTHON OFF" - "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" - "FAISS_ENABLE_GPU ON" - "BUILD_TESTING OFF" - "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" - ) + rapids_cpm_find(faiss ${PKG_VERSION} + GLOBAL_TARGETS faiss::faiss + INSTALL_EXPORT_SET raft-nn-exports + CPM_ARGS + GIT_REPOSITORY https://github.com/facebookresearch/faiss.git + GIT_TAG ${PKG_PINNED_TAG} + EXCLUDE_FROM_ALL TRUE + OPTIONS + "FAISS_ENABLE_PYTHON OFF" + "CUDAToolkit_ROOT ${CUDAToolkit_LIBRARY_DIR}" + "FAISS_ENABLE_GPU ON" + "BUILD_TESTING OFF" + "CMAKE_MESSAGE_LOG_LEVEL VERBOSE" + ) - if(TARGET faiss AND NOT TARGET faiss::faiss) - add_library(faiss::faiss ALIAS faiss) - endif() + if(TARGET faiss AND NOT TARGET faiss::faiss) + add_library(faiss::faiss ALIAS faiss) + endif() - if(faiss_ADDED) - rapids_export(BUILD faiss - EXPORT_SET faiss-targets - GLOBAL_TARGETS faiss - NAMESPACE faiss::) + if(faiss_ADDED) + rapids_export(BUILD faiss + EXPORT_SET faiss-targets + GLOBAL_TARGETS faiss + NAMESPACE faiss::) + endif() endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` rapids_export_package(BUILD faiss raft-nn-exports) + rapids_export_package(INSTALL faiss raft-nn-exports) # Tell cmake where it can find the generated faiss-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") From f286cf101b8d01afbe249b46b0669f4bce56d05a Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 11 Jan 2022 12:10:58 -0500 Subject: [PATCH 43/81] Adding conda packaging for libraft and pyraft --- build.sh | 17 +++++---- ci/checks/black_lists.sh | 4 +-- ci/checks/changelog.sh | 4 +-- ci/checks/copyright.py | 2 +- ci/gpu/build.sh | 6 ++-- ci/local/README.md | 6 ++-- ci/local/build.sh | 2 +- ci/local/old-gpubuild.sh | 2 +- ci/release/update-version.sh | 2 +- conda/recipes/libraft/build.sh | 7 ++++ conda/recipes/libraft/meta.yaml | 61 +++++++++++++++++++++++++++++++++ conda/recipes/pyraft/build.sh | 4 +++ conda/recipes/pyraft/meta.yaml | 60 ++++++++++++++++++++++++++++++++ 13 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 conda/recipes/libraft/build.sh create mode 100644 conda/recipes/libraft/meta.yaml create mode 100644 conda/recipes/pyraft/build.sh create mode 100644 conda/recipes/pyraft/meta.yaml diff --git a/build.sh b/build.sh index a609670419..1e64ae7c48 100755 --- a/build.sh +++ b/build.sh @@ -18,14 +18,14 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean cppraft pyraft cppdocs -v -g --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" +VALIDARGS="clean libraft pyraft cppdocs -v -g --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" HELP="$0 [ ...] [ ...] where is: clean - remove all existing build artifacts and configuration (start over) - cppraft - build the cuml C++ code only. Also builds the C-wrapper library + libraft - build the raft C++ code only. Also builds the C-wrapper library around the C++ code. - pyraft - build the cuml Python package - cppdocs - build the C++ doxygen documentation + pyraft - build the raft Python package + cppdocs - build the C++ doxygen documentation and is: -v - verbose build mode -g - build for debug @@ -35,7 +35,7 @@ HELP="$0 [ ...] [ ...] --show_depr_warn - show cmake deprecation warnings -h - print this text - default action (no args) is to build both cppraft and pyraft targets + default action (no args) is to build both libraft and pyraft targets " CPP_RAFT_BUILD_DIR=${REPODIR}/cpp/build PY_RAFT_BUILD_DIR=${REPODIR}/python/build @@ -138,7 +138,7 @@ fi ################################################################################ # Configure for building all C++ targets -if (( ${NUMARGS} == 0 )) || hasArg cppraft; then +if (( ${NUMARGS} == 0 )) || hasArg libraft; then if (( ${BUILD_ALL_GPU_ARCH} == 0 )); then RAFT_CMAKE_CUDA_ARCHITECTURES="NATIVE" echo "Building for the architecture of the GPU in the system..." @@ -160,14 +160,13 @@ if (( ${NUMARGS} == 0 )) || hasArg cppraft; then cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${MAKE_TARGETS} ${VERBOSE_FLAG} fi - -# Build and (optionally) install the cuml Python package +# Build and (optionally) install the raft Python package if (( ${NUMARGS} == 0 )) || hasArg pyraft; then cd ${REPODIR}/python if [[ ${INSTALL_TARGET} != "" ]]; then python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace ${SINGLEGPU} else - python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace --library-dir=${LIBCUML_BUILD_DIR} ${SINGLEGPU} + python setup.py build_ext -j${PARALLEL_LEVEL:-1} --inplace --library-dir=${LIBRAFT_BUILD_DIR} ${SINGLEGPU} fi fi diff --git a/ci/checks/black_lists.sh b/ci/checks/black_lists.sh index d96240650d..849b354d08 100755 --- a/ci/checks/black_lists.sh +++ b/ci/checks/black_lists.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. ########################################## -# cuML black listed function call Tester # +# RAFT black listed function call Tester # ########################################## # PR_TARGET_BRANCH is set by the CI enviroment diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh index 5560b34f73..eff64d3b7d 100755 --- a/ci/checks/changelog.sh +++ b/ci/checks/changelog.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. ######################### -# cuML CHANGELOG Tester # +# RAFT CHANGELOG Tester # ######################### # Checkout main for comparison diff --git a/ci/checks/copyright.py b/ci/checks/copyright.py index 738eeb32dd..79a0d70005 100644 --- a/ci/checks/copyright.py +++ b/ci/checks/copyright.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 83cc6fdaef..4f9ca612dc 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2020-2021, NVIDIA CORPORATION. ######################################### -# cuML GPU build and test script for CI # +# RAFT GPU build and test script for CI # ######################################### set -e @@ -115,7 +115,7 @@ gpuci_logger "GoogleTest for raft" cd "$WORKSPACE/cpp/build" GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test_raft -gpuci_logger "Python pytest for cuml" +gpuci_logger "Python pytest for raft" cd "$WORKSPACE/python" -python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-cuml.xml" -v -s +python -m pytest --cache-clear --junitxml="$WORKSPACE/junit-raft.xml" -v -s diff --git a/ci/local/README.md b/ci/local/README.md index 4e1dea789a..3b47ef3b53 100644 --- a/ci/local/README.md +++ b/ci/local/README.md @@ -23,13 +23,13 @@ where: ``` Example Usage: -`bash build.sh -r ~/rapids/cuml -i gpuci/rapidsai-base:cuda9.2-ubuntu16.04-gcc5-py3.6` +`bash build.sh -r ~/rapids/raft -i gpuci/rapidsai-base:cuda9.2-ubuntu16.04-gcc5-py3.6` For a full list of available gpuCI docker images, visit our [DockerHub](https://hub.docker.com/r/gpuci/rapidsai-base/tags) page. Style Check: ```bash -$ bash ci/local/build.sh -r ~/rapids/cuml -s +$ bash ci/local/build.sh -r ~/rapids/raft -s $ . /opt/conda/etc/profile.d/conda.sh $ conda activate rapids #Activate gpuCI conda environment $ cd rapids @@ -43,7 +43,7 @@ There are some caveats to be aware of when using this script, especially if you ### Docker Image Build Repository -The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/cuml/build_rapidsai-base_cuda9.2-ubuntu16.04-gcc5-py3.6/`. Feel free to remove this directory after the script is finished. +The docker image will generate build artifacts in a folder on your machine located in the `root` directory of the repository you passed to the script. For the above example, the directory is named `~/rapids/raft/build_rapidsai-base_cuda9.2-ubuntu16.04-gcc5-py3.6/`. Feel free to remove this directory after the script is finished. *Note*: The script *will not* override your local build repository. Your local environment stays in tact. diff --git a/ci/local/build.sh b/ci/local/build.sh index 9effad4244..cdafd967c7 100644 --- a/ci/local/build.sh +++ b/ci/local/build.sh @@ -6,7 +6,7 @@ MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` DOCKER_IMAGE="gpuci/rapidsai:${MINOR_VERSION}-cuda10.1-devel-ubuntu16.04-py3.7" REPO_PATH=${PWD} RAPIDS_DIR_IN_CONTAINER="/rapids" -CPP_BUILD_DIR="cuML/build" +CPP_BUILD_DIR="raft/build" PYTHON_BUILD_DIR="python/build" CONTAINER_SHELL_ONLY=0 diff --git a/ci/local/old-gpubuild.sh b/ci/local/old-gpubuild.sh index ce6db28291..cdebf0c122 100644 --- a/ci/local/old-gpubuild.sh +++ b/ci/local/old-gpubuild.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2020, NVIDIA CORPORATION. ######################################### -# cuML GPU build and test script for CI # +# RAFT GPU build and test script for CI # ######################################### set -e diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 58eb3bbd67..a832f67aaf 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,7 +1,7 @@ #!/bin/bash # Copyright (c) 2020, NVIDIA CORPORATION. ######################## -# cuML Version Updater # +# RAFT Version Updater # ######################## ## Usage diff --git a/conda/recipes/libraft/build.sh b/conda/recipes/libraft/build.sh new file mode 100644 index 0000000000..35a23bf12d --- /dev/null +++ b/conda/recipes/libraft/build.sh @@ -0,0 +1,7 @@ +if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then + # C++11 requires 10.9 + # but cudatoolkit 8 is build for 10.11 + export MACOSX_DEPLOYMENT_TARGET=10.11 +fi + +./build.sh clean libraft -v --allgpuarch \ No newline at end of file diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft/meta.yaml new file mode 100644 index 0000000000..242ed75533 --- /dev/null +++ b/conda/recipes/libraft/meta.yaml @@ -0,0 +1,61 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +# Usage: + # conda build . -c conda-forge -c nvidia -c rapidsai + {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} + {% set cuda_major=cuda_version.split('.')[0] %} + {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +package: + name: libraft + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - VERSION_SUFFIX + - PROJECT_FLASH + - CCACHE_DIR + - CCACHE_NOHASHDIR + - CCACHE_COMPILERCHECK + - CMAKE_GENERATOR + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + +requirements: + build: + - cmake>=3.20.1 + host: + - nccl>=2.9.9 + - cudatoolkit {{ cuda_version }}.* + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + - lapack + - faiss-proc=*=cuda + - gtest=1.10.0 + - gmock + - libfaiss 1.7.0 *_cuda + run: + - nccl>=2.9.9 + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - faiss-proc=*=cuda + - libfaiss 1.7.0 *_cuda + - libcusolver>=11.2.1 + +about: + home: http://rapids.ai/ + license: Apache-2.0 + # license_file: LICENSE + summary: libraft library \ No newline at end of file diff --git a/conda/recipes/pyraft/build.sh b/conda/recipes/pyraft/build.sh new file mode 100644 index 0000000000..cdad5be8e7 --- /dev/null +++ b/conda/recipes/pyraft/build.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +# This assumes the script is executed from the root of the repo directory +./build.sh pyraft \ No newline at end of file diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml new file mode 100644 index 0000000000..0bda087d25 --- /dev/null +++ b/conda/recipes/pyraft/meta.yaml @@ -0,0 +1,60 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +# Usage: + # conda build . -c conda-forge -c numba -c rapidsai -c pytorch + {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + {% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} + {% set cuda_major=cuda_version.split('.')[0] %} + {% set py_version=environ.get('CONDA_PY', 36) %} + {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} + +package: + name: pyraft + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_py{{ py_version }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CC + - CXX + - VERSION_SUFFIX + +requirements: + build: + - python x.x + - setuptools + - cython>=0.29,<0.30 + - cmake>=3.20.1 + - libraft={{ version }} + - cudatoolkit {{ cuda_version }}.* + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + run: + - python x.x + - dask-cuda {{ minor_version }} + - libraft={{ version }} + - cupy>=7.8.0,<10.0.0a0 + - nccl>=2.9.9 + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + - dask>=2021.11.1,<=2021.11.2 + - distributed>=2021.11.1,<=2021.11.2 + - joblib >=0.11 + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + +tests: # [linux64] + requirements: # [linux64] + - cudatoolkit {{ cuda_version }}.* # [linux64] + imports: # [linux64] + - raft # [linux64] + +about: + home: http://rapids.ai/ + license: Apache-2.0 + # license_file: LICENSE + summary: pyraft library \ No newline at end of file From 5381ccdcfc673f78c06b35f92d24bc219df44d4b Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 11 Jan 2022 12:49:59 -0500 Subject: [PATCH 44/81] Separating out raft dependencies. --- build.sh | 8 +- .../{libraft => libraft_core}/build.sh | 0 .../{libraft => libraft_core}/meta.yaml | 4 - conda/recipes/libraft_distance/build.sh | 7 ++ conda/recipes/libraft_distance/meta.yaml | 113 ++++++++++++++++++ conda/recipes/libraft_nn/build.sh | 7 ++ conda/recipes/libraft_nn/meta.yaml | 56 +++++++++ conda/recipes/pyraft/meta.yaml | 2 +- 8 files changed, 188 insertions(+), 9 deletions(-) rename conda/recipes/{libraft => libraft_core}/build.sh (100%) rename conda/recipes/{libraft => libraft_core}/meta.yaml (93%) create mode 100644 conda/recipes/libraft_distance/build.sh create mode 100644 conda/recipes/libraft_distance/meta.yaml create mode 100644 conda/recipes/libraft_nn/build.sh create mode 100644 conda/recipes/libraft_nn/meta.yaml diff --git a/build.sh b/build.sh index 1e64ae7c48..032ab4d38b 100755 --- a/build.sh +++ b/build.sh @@ -150,10 +150,10 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft; then cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ - -DNVTX=${NVTX} \ - -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ - -DBUILD_GTEST=${BUILD_GTEST} \ - -DBUILD_STATIC_FAISS=${BUILD_STATIC_FAISS} + -DNVTX=${NVTX} +# -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ +# -DBUILD_GTEST=${BUILD_GTEST} \ +# -DBUILD_STATIC_FAISS=${BUILD_STATIC_FAISS} # Run all c++ targets at once diff --git a/conda/recipes/libraft/build.sh b/conda/recipes/libraft_core/build.sh similarity index 100% rename from conda/recipes/libraft/build.sh rename to conda/recipes/libraft_core/build.sh diff --git a/conda/recipes/libraft/meta.yaml b/conda/recipes/libraft_core/meta.yaml similarity index 93% rename from conda/recipes/libraft/meta.yaml rename to conda/recipes/libraft_core/meta.yaml index 242ed75533..84ad70006d 100644 --- a/conda/recipes/libraft/meta.yaml +++ b/conda/recipes/libraft_core/meta.yaml @@ -41,17 +41,13 @@ requirements: - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - lapack - - faiss-proc=*=cuda - gtest=1.10.0 - gmock - - libfaiss 1.7.0 *_cuda run: - nccl>=2.9.9 - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - faiss-proc=*=cuda - - libfaiss 1.7.0 *_cuda - libcusolver>=11.2.1 about: diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft_distance/build.sh new file mode 100644 index 0000000000..41e862d824 --- /dev/null +++ b/conda/recipes/libraft_distance/build.sh @@ -0,0 +1,7 @@ +if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then + # C++11 requires 10.9 + # but cudatoolkit 8 is build for 10.11 + export MACOSX_DEPLOYMENT_TARGET=10.11 +fi + +./build.sh clean libraft -v --allgpuarch # TODO: Compile only distance package instantiations \ No newline at end of file diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml new file mode 100644 index 0000000000..c15a33dc38 --- /dev/null +++ b/conda/recipes/libraft_distance/meta.yaml @@ -0,0 +1,113 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +# Usage: + # conda build . -c conda-forge -c nvidia -c rapidsai + {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} + {% set cuda_major=cuda_version.split('.')[0] %} + {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +package: + name: libraft + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - VERSION_SUFFIX + - PROJECT_FLASH + - CCACHE_DIR + - CCACHE_NOHASHDIR + - CCACHE_COMPILERCHECK + - CMAKE_GENERATOR + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + +requirements: + build: + - cmake>=3.20.1 + host: + - libraft-core {{ + - nccl>=2.9.9 + - cudatoolkit {{ cuda_version }}.* + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + - lapack + - faiss-proc=*=cuda + - gtest=1.10.0 + - gmock + - libfaiss 1.7.0 *_cuda + run: + - nccl>=2.9.9 + - ucx-py {{ ucx_py_version }} + - ucx-proc=*=gpu + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - faiss-proc=*=cuda + - libfaiss 1.7.0 *_cuda + - libcusolver>=11.2.1 + +about: + home: http://rapids.ai/ + license: Apache-2.0 + # license_file: LICENSE + summary: libraft library# Copyright (c) 2022, NVIDIA CORPORATION. + + # Usage: + # conda build . -c conda-forge -c nvidia -c rapidsai + {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} + {% set cuda_major=cuda_version.split('.')[0] %} + {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +package: + name: libraft + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - VERSION_SUFFIX + - PROJECT_FLASH + - CCACHE_DIR + - CCACHE_NOHASHDIR + - CCACHE_COMPILERCHECK + - CMAKE_GENERATOR + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + +requirements: + build: + - cmake>=3.20.1 + host: + - libraft-core {{ version }} + - cudatoolkit {{ cuda_version }}.* + - lapack + - gtest=1.10.0 + - gmock + run: + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - libcusolver>=11.2.1 + +about: + home: http://rapids.ai/ + license: Apache-2.0 + # license_file: LICENSE + summary: libraft library \ No newline at end of file diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft_nn/build.sh new file mode 100644 index 0000000000..7a6e4a8596 --- /dev/null +++ b/conda/recipes/libraft_nn/build.sh @@ -0,0 +1,7 @@ +if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then + # C++11 requires 10.9 + # but cudatoolkit 8 is build for 10.11 + export MACOSX_DEPLOYMENT_TARGET=10.11 +fi + +./build.sh clean libraft -v --allgpuarch # TODO: Compile only nn package instantiations \ No newline at end of file diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml new file mode 100644 index 0000000000..6480f573cf --- /dev/null +++ b/conda/recipes/libraft_nn/meta.yaml @@ -0,0 +1,56 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. + +# Usage: + # conda build . -c conda-forge -c nvidia -c rapidsai + {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} + {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} + {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} + {% set cuda_major=cuda_version.split('.')[0] %} + {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +package: + name: libraft + version: {{ version }} + +source: + git_url: ../../.. + +build: + number: {{ GIT_DESCRIBE_NUMBER }} + string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} + script_env: + - CC + - CXX + - CUDAHOSTCXX + - PARALLEL_LEVEL + - VERSION_SUFFIX + - PROJECT_FLASH + - CCACHE_DIR + - CCACHE_NOHASHDIR + - CCACHE_COMPILERCHECK + - CMAKE_GENERATOR + - CMAKE_C_COMPILER_LAUNCHER + - CMAKE_CXX_COMPILER_LAUNCHER + - CMAKE_CUDA_COMPILER_LAUNCHER + +requirements: + build: + - cmake>=3.20.1 + host: + - libraft-core {{ version }} + - cudatoolkit {{ cuda_version }}.* + - lapack + - faiss-proc=*=cuda + - gtest=1.10.0 + - gmock + - libfaiss 1.7.0 *_cuda + run: + - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} + - faiss-proc=*=cuda + - libfaiss 1.7.0 *_cuda + - libcusolver>=11.2.1 + +about: + home: http://rapids.ai/ + license: Apache-2.0 + # license_file: LICENSE + summary: libraft library \ No newline at end of file diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 0bda087d25..8b7f4c25cd 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -30,7 +30,7 @@ requirements: - setuptools - cython>=0.29,<0.30 - cmake>=3.20.1 - - libraft={{ version }} + - libraft-core={{ version }} - cudatoolkit {{ cuda_version }}.* - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu From 95881a91f748426f5a7a82bd0232a649ccdb44f1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 11 Jan 2022 13:02:23 -0500 Subject: [PATCH 45/81] correct {{ version }} --- conda/recipes/libraft_core/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libraft_core/build.sh b/conda/recipes/libraft_core/build.sh index 35a23bf12d..bfd2d9d723 100644 --- a/conda/recipes/libraft_core/build.sh +++ b/conda/recipes/libraft_core/build.sh @@ -4,4 +4,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch \ No newline at end of file +./build.sh clean libraft -v --allgpuarch # TODO: Compile only the core instantiations \ No newline at end of file From 60c81a72982bb14e9d877f090e90472045b2d254 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 10:47:28 -0800 Subject: [PATCH 46/81] fix typo --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c050872e18..f9f021b697 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -261,7 +261,7 @@ set(code_string set(raft_allowed_components distance nn) foreach(comp IN LISTS raft_allowed_components) if(${comp} IN_LIST raft_FIND_COMPONENTS) - include("${CMAKE_CURRENT_LIST_DIR}/raft-distance-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-targets.cmake") if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") From d20e5efaf29d554eae972b0e0bd8aac8456436c5 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 16:41:27 -0800 Subject: [PATCH 47/81] add GTest targets to install side of raft-exports export set --- cpp/cmake/thirdparty/get_gtest.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index 72fb0e18c6..d81e5e8606 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -27,10 +27,16 @@ function(find_and_configure_gtest ) GLOBAL_TARGETS gtest gmock gtest_main gmock_main NAMESPACE GTest::) + # We generate the GTest-config files when we built GTest locally, so always do `find_dependency` + rapids_export_package(BUILD GTest raft-exports) + + # Tell cmake where it can find the generated GTest-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endif() + rapids_export_package(INSTALL GTest raft-exports) + endfunction() find_and_configure_gtest() From 1594f7f91d62c597020b2eea264042999274a48e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Thu, 13 Jan 2022 17:12:15 -0800 Subject: [PATCH 48/81] install gtest if not already installed --- cpp/cmake/thirdparty/get_gtest.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index d81e5e8606..c28fd5d3fa 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,6 +18,7 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) From 1c74204f9c4ad008e65c17a5989fd08d455b2634 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Fri, 14 Jan 2022 12:48:28 -0800 Subject: [PATCH 49/81] revert get_gtest.cmake changes --- cpp/cmake/thirdparty/get_gtest.cmake | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cpp/cmake/thirdparty/get_gtest.cmake b/cpp/cmake/thirdparty/get_gtest.cmake index c28fd5d3fa..72fb0e18c6 100644 --- a/cpp/cmake/thirdparty/get_gtest.cmake +++ b/cpp/cmake/thirdparty/get_gtest.cmake @@ -18,7 +18,6 @@ function(find_and_configure_gtest ) include(${rapids-cmake-dir}/cpm/gtest.cmake) rapids_cpm_gtest(BUILD_EXPORT_SET raft-exports - INSTALL_EXPORT_SET raft-exports EXCLUDE_FROM_ALL TRUE) if(GTest_ADDED) @@ -28,16 +27,10 @@ function(find_and_configure_gtest ) GLOBAL_TARGETS gtest gmock gtest_main gmock_main NAMESPACE GTest::) - # We generate the GTest-config files when we built GTest locally, so always do `find_dependency` - rapids_export_package(BUILD GTest raft-exports) - - # Tell cmake where it can find the generated GTest-config.cmake we wrote. include("${rapids-cmake-dir}/export/find_package_root.cmake") rapids_export_find_package_root(BUILD GTest [=[${CMAKE_CURRENT_LIST_DIR}]=] raft-exports) endif() - rapids_export_package(INSTALL GTest raft-exports) - endfunction() find_and_configure_gtest() From 0bc750b14a220c4a9faea9f80d6f6cc86f3ca1ad Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 14:08:20 -0800 Subject: [PATCH 50/81] remove raft_ prefix from global targets list --- cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f9f021b697..7c47de2ec3 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -286,7 +286,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + GLOBAL_TARGETS raft distance nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -297,7 +297,7 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + GLOBAL_TARGETS raft distance nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From a3c0370b65b5ff4f88161335cdffe543808a8feb Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 15:45:30 -0800 Subject: [PATCH 51/81] revert previous change, move target_link_libraries below add_library calls that reference the target, fix typo raft_distance_lib -> raft_nn_lib --- cpp/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7c47de2ec3..8b0e0d7193 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -145,8 +145,6 @@ add_library(raft_distance INTERFACE) add_library(raft::distance ALIAS raft_distance) set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) -target_link_libraries(raft_distance INTERFACE raft::raft $) - if(RAFT_COMPILE_LIBRARIES) add_library(raft_distance_lib SHARED src/distance/specializations/detail @@ -180,14 +178,14 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-distance-exports) endif() +target_link_libraries(raft_distance INTERFACE raft::raft $) + ############################################################################## # - raft_nn ------------------------------------------------------------------ add_library(raft_nn INTERFACE) add_library(raft::nn ALIAS raft_nn) set_target_properties(raft_nn PROPERTIES EXPORT_NAME nn) -target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) - if(RAFT_COMPILE_LIBRARIES) add_library(raft_nn_lib SHARED src/nn/specializations/ball_cover.cu @@ -202,7 +200,7 @@ if(RAFT_COMPILE_LIBRARIES) PRIVATE "$<$:${RAFT_CXX_FLAGS}>" "$<$:${RAFT_CUDA_FLAGS}>" ) - target_compile_definitions(raft_distance_lib + target_compile_definitions(raft_nn_lib INTERFACE "RAFT_NN_COMPILED") install(TARGETS raft_nn_lib @@ -210,6 +208,8 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-nn-exports) endif() +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) + ############################################################################## # - install targets----------------------------------------------------------- rapids_cmake_install_lib_dir( lib_dir ) @@ -286,7 +286,7 @@ endif() rapids_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft distance nn + GLOBAL_TARGETS raft raft_distance raft_nn NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -297,7 +297,7 @@ rapids_export(INSTALL raft rapids_export(BUILD raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft distance nn + GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: FINAL_CODE_BLOCK code_string From ea209c14f57ddb80ead1a5b9ab0495856f2dc59e Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 17:29:18 -0800 Subject: [PATCH 52/81] fix _lib name to add prefix --- cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8b0e0d7193..8870b1b1e4 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -178,7 +178,7 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-distance-exports) endif() -target_link_libraries(raft_distance INTERFACE raft::raft $) +target_link_libraries(raft_distance INTERFACE raft::raft $) ############################################################################## # - raft_nn ------------------------------------------------------------------ @@ -208,7 +208,7 @@ if(RAFT_COMPILE_LIBRARIES) EXPORT raft-nn-exports) endif() -target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) +target_link_libraries(raft_nn INTERFACE raft::raft faiss::faiss $) ############################################################################## # - install targets----------------------------------------------------------- From 7093ad50f6ac37a0f3b9d3bb7ab0c2163b282540 Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 20:50:37 -0800 Subject: [PATCH 53/81] add missing PKG_ prefix --- cpp/cmake/thirdparty/get_faiss.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 59c1d4ba2b..46ce522f8c 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -26,7 +26,7 @@ function(find_and_configure_faiss) ) set(BUILD_SHARED_LIBS OFF) - if (NOT BUILD_STATIC_LIBS) + if (NOT PKG_BUILD_STATIC_LIBS) set(BUILD_SHARED_LIBS ON) endif() From f613e39cb8c99e8a0b0da71800dbbfa2a4b5ca8c Mon Sep 17 00:00:00 2001 From: ptaylor Date: Mon, 17 Jan 2022 20:53:29 -0800 Subject: [PATCH 54/81] guard creating alias targets when added as a submodule via CPM --- cpp/CMakeLists.txt | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8870b1b1e4..257adf76b5 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,7 +142,11 @@ target_compile_features(raft INTERFACE cxx_std_17 $ ############################################################################## # - raft_distance ------------------------------------------------------------ add_library(raft_distance INTERFACE) -add_library(raft::distance ALIAS raft_distance) + +if(TARGET raft_distance AND (NOT TARGET raft::distance)) + add_library(raft::distance ALIAS raft_distance) +endif() + set_target_properties(raft_distance PROPERTIES EXPORT_NAME distance) if(RAFT_COMPILE_LIBRARIES) @@ -183,7 +187,11 @@ target_link_libraries(raft_distance INTERFACE raft::raft $ Date: Wed, 19 Jan 2022 11:39:31 -0500 Subject: [PATCH 55/81] Adding `compilelibs` option to build.sh --- build.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build.sh b/build.sh index 032ab4d38b..04281fa838 100755 --- a/build.sh +++ b/build.sh @@ -29,6 +29,7 @@ HELP="$0 [ ...] [ ...] and is: -v - verbose build mode -g - build for debug + --compilelibs - compile shared libraries --allgpuarch - build for all supported GPU architectures --buildfaiss - build faiss statically into raft --nvtx - Enable nvtx for profiling support @@ -47,6 +48,7 @@ VERBOSE_FLAG="" BUILD_ALL_GPU_ARCH=0 BUILD_GTEST=OFF BUILD_STATIC_FAISS=OFF +COMPILE_LIBRARIES=OFF SINGLEGPU="" NVTX=OFF CLEAN=0 @@ -90,6 +92,10 @@ if hasArg -g; then BUILD_TYPE=Debug fi +if hasArg --compilelibs; then + COMPILE_LIBRARIES=ON +fi + if hasArg --allgpuarch; then BUILD_ALL_GPU_ARCH=1 fi @@ -150,6 +156,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft; then cmake -S ${REPODIR}/cpp -B ${CPP_RAFT_BUILD_DIR} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ + -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ -DNVTX=${NVTX} # -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ # -DBUILD_GTEST=${BUILD_GTEST} \ From a5a286c54421b19cdcadd5464b10784141b7f3d7 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 11:42:18 -0500 Subject: [PATCH 56/81] conda packages are using --compilelibs --- conda/recipes/libraft_distance/build.sh | 4 +++- conda/recipes/{libraft_core => libraft_headers}/build.sh | 0 conda/recipes/{libraft_core => libraft_headers}/meta.yaml | 0 conda/recipes/libraft_nn/build.sh | 4 +++- 4 files changed, 6 insertions(+), 2 deletions(-) rename conda/recipes/{libraft_core => libraft_headers}/build.sh (100%) rename conda/recipes/{libraft_core => libraft_headers}/meta.yaml (100%) diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft_distance/build.sh index 41e862d824..5212d67552 100644 --- a/conda/recipes/libraft_distance/build.sh +++ b/conda/recipes/libraft_distance/build.sh @@ -4,4 +4,6 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch # TODO: Compile only distance package instantiations \ No newline at end of file +./build.sh clean libraft -v --allgpuarch --compilelibs + +# TODO: Somehow just extract the libraft_distance.so \ No newline at end of file diff --git a/conda/recipes/libraft_core/build.sh b/conda/recipes/libraft_headers/build.sh similarity index 100% rename from conda/recipes/libraft_core/build.sh rename to conda/recipes/libraft_headers/build.sh diff --git a/conda/recipes/libraft_core/meta.yaml b/conda/recipes/libraft_headers/meta.yaml similarity index 100% rename from conda/recipes/libraft_core/meta.yaml rename to conda/recipes/libraft_headers/meta.yaml diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft_nn/build.sh index 7a6e4a8596..fe557c6c11 100644 --- a/conda/recipes/libraft_nn/build.sh +++ b/conda/recipes/libraft_nn/build.sh @@ -4,4 +4,6 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch # TODO: Compile only nn package instantiations \ No newline at end of file +./build.sh clean libraft -v --allgpuarch --compilelibs + +# TODO: Somehow extract just the libraft_nn.so artiface \ No newline at end of file From 95c84cc5c4561b4fd7a91acd513b8189441a1198 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 12:38:25 -0500 Subject: [PATCH 57/81] Updating final invokation of cppraft --- ci/gpu/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index e87dafe5ef..190e45d775 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -88,7 +88,7 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -"$WORKSPACE/build.sh" cppraft pyraft -v +"$WORKSPACE/build.sh" libraft pyraft -v gpuci_logger "Building doxygen C++ docs" "$WORKSPACE/build.sh" cppdocs -v From b2e0b014d9bcdbb081c9f65a10e589c6f90744da Mon Sep 17 00:00:00 2001 From: ptaylor Date: Wed, 19 Jan 2022 11:00:20 -0800 Subject: [PATCH 58/81] update cuco hash --- cpp/cmake/thirdparty/get_cuco.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/thirdparty/get_cuco.cmake b/cpp/cmake/thirdparty/get_cuco.cmake index fdac7cda11..92d20a27fa 100644 --- a/cpp/cmake/thirdparty/get_cuco.cmake +++ b/cpp/cmake/thirdparty/get_cuco.cmake @@ -22,7 +22,7 @@ function(find_and_configure_cuco VERSION) INSTALL_EXPORT_SET raft-exports CPM_ARGS GIT_REPOSITORY https://github.com/NVIDIA/cuCollections.git - GIT_TAG f0eecb203590f1f4ac4a9f1700229f4434ac64dc + GIT_TAG 0ca860b824f5dc22cf8a41f09912e62e11f07d82 OPTIONS "BUILD_TESTS OFF" "BUILD_BENCHMARKS OFF" "BUILD_EXAMPLES OFF" From 902657ceb030f4f41f06048c5dc9b220dc1b83e8 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:34:34 -0500 Subject: [PATCH 59/81] Replace the use of RMM's CUDA Python bindings with those from CUDA-Python --- python/raft/common/cuda.pxd | 24 +++++----------------- python/raft/common/cuda.pyx | 38 ++++++++++++++++++++--------------- python/raft/common/handle.pxd | 3 +-- python/raft/common/handle.pyx | 9 +++++---- 4 files changed, 33 insertions(+), 41 deletions(-) diff --git a/python/raft/common/cuda.pxd b/python/raft/common/cuda.pxd index e407213f44..0459cb96af 100644 --- a/python/raft/common/cuda.pxd +++ b/python/raft/common/cuda.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2019, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,23 +14,9 @@ # limitations under the License. # -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 +from cuda.ccudart cimport cudaStream_t +cdef class Stream: + cdef cudaStream_t s -# Populate this with more typedef's (eg: events) as and when needed -cdef extern from * nogil: - ctypedef void* _Stream "cudaStream_t" - ctypedef int _Error "cudaError_t" - - -# Populate this with more runtime api method declarations as and when needed -cdef extern from "cuda_runtime_api.h" nogil: - _Error cudaStreamCreate(_Stream* s) - _Error cudaStreamDestroy(_Stream s) - _Error cudaStreamSynchronize(_Stream s) - _Error cudaGetLastError() - const char* cudaGetErrorString(_Error e) - const char* cudaGetErrorName(_Error e) + cdef cudaStream_t getStream(self) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index 0b97eeba67..d00fd88af0 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,10 +19,21 @@ # cython: embedsignature = True # cython: language_level = 3 +from cuda.ccudart cimport ( + cudaStream_t, + cudaError_t, + cudaSuccess, + cudaStreamCreate, + cudaStreamDestroy, + cudaStreamSynchronize, + cudaGetLastError, + cudaGetErrorString, + cudaGetErrorName +) class CudaRuntimeError(RuntimeError): def __init__(self, extraMsg=None): - cdef _Error e = cudaGetLastError() + cdef cudaError_t e = cudaGetLastError() cdef bytes errMsg = cudaGetErrorString(e) cdef bytes errName = cudaGetErrorName(e) msg = "Error! %s reason='%s'" % (errName.decode(), errMsg.decode()) @@ -47,26 +58,22 @@ cdef class Stream: """ # NOTE: - # If we store _Stream directly, this always leads to the following error: - # "Cannot convert Python object to '_Stream'" + # If we store cudaStream_t directly, this always leads to the following error: + # "Cannot convert Python object to 'cudaStream_t'" # I was unable to find a good solution to this in reasonable time. Also, # since cudaStream_t is a pointer anyways, storing it as an integer should # be just fine (although, that certainly is ugly and hacky!). - cdef size_t s def __cinit__(self): - if self.s != 0: - return - cdef _Stream stream - cdef _Error e = cudaStreamCreate(&stream) - if e != 0: + cdef cudaStream_t stream + cdef cudaError_t e = cudaStreamCreate(&stream) + if e != cudaSuccess: raise CudaRuntimeError("Stream create") - self.s = stream + self.s = stream def __dealloc__(self): self.sync() - cdef _Stream stream = <_Stream>self.s - cdef _Error e = cudaStreamDestroy(stream) + cdef cudaError_t e = cudaStreamDestroy(self.s) if e != 0: raise CudaRuntimeError("Stream destroy") @@ -76,10 +83,9 @@ cdef class Stream: could raise exception due to issues with previous asynchronous launches """ - cdef _Stream stream = <_Stream>self.s - cdef _Error e = cudaStreamSynchronize(stream) + cdef cudaError_t e = cudaStreamSynchronize(self.s) if e != 0: raise CudaRuntimeError("Stream sync") - def getStream(self): + cdef cudaStream_t getStream(self): return self.s diff --git a/python/raft/common/handle.pxd b/python/raft/common/handle.pxd index d2ae0a401d..8415b7e3d7 100644 --- a/python/raft/common/handle.pxd +++ b/python/raft/common/handle.pxd @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ from libcpp.memory cimport shared_ptr -from .cuda cimport _Stream from rmm._lib.cuda_stream_view cimport cuda_stream_view from rmm._lib.cuda_stream_pool cimport cuda_stream_pool from libcpp.memory cimport shared_ptr diff --git a/python/raft/common/handle.pyx b/python/raft/common/handle.pyx index 1accf9e679..c4faabb610 100644 --- a/python/raft/common/handle.pyx +++ b/python/raft/common/handle.pyx @@ -1,5 +1,5 @@ # -# Copyright (c) 2020, NVIDIA CORPORATION. +# Copyright (c) 2020-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,9 +24,10 @@ from libcpp.memory cimport shared_ptr from rmm._lib.cuda_stream_view cimport cuda_stream_per_thread from rmm._lib.cuda_stream_view cimport cuda_stream_view -from .cuda cimport _Stream, _Error, cudaStreamSynchronize +from .cuda cimport Stream from .cuda import CudaRuntimeError + cdef class Handle: """ Handle is a lightweight python wrapper around the corresponding C++ class @@ -51,7 +52,7 @@ cdef class Handle: del handle # optional! """ - def __cinit__(self, stream=None, n_streams=0): + def __cinit__(self, stream: Stream=None, n_streams=0): self.n_streams = n_streams if n_streams > 0: self.stream_pool.reset(new cuda_stream_pool(n_streams)) @@ -64,7 +65,7 @@ cdef class Handle: self.stream_pool)) else: # this constructor constructs a handle on user stream - c_stream = cuda_stream_view(<_Stream> stream.getStream()) + c_stream = cuda_stream_view(stream.getStream()) self.c_obj.reset(new handle_t(c_stream, self.stream_pool)) From d7f00745fcc34bd15819998683e7d703450abd9d Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:44:07 -0500 Subject: [PATCH 60/81] Add cuda-python to dev env --- conda/environments/raft_dev_cuda11.5.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/environments/raft_dev_cuda11.5.yml b/conda/environments/raft_dev_cuda11.5.yml index 152f3a8db5..c6d9f3fbf5 100644 --- a/conda/environments/raft_dev_cuda11.5.yml +++ b/conda/environments/raft_dev_cuda11.5.yml @@ -6,6 +6,7 @@ channels: - conda-forge dependencies: - cudatoolkit=11.5 +- cuda-python >=11.5,<12.0 - clang=11.1.0 - clang-tools=11.1.0 - rapids-build-env=22.02.* From c7666ad76d8968a7d4c7213554b79dbfdc561ed8 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 15:51:16 -0500 Subject: [PATCH 61/81] Remove older comment about cudaStream_t attribute --- python/raft/common/cuda.pyx | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index d00fd88af0..8d2f6d8c69 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -56,14 +56,6 @@ cdef class Stream: stream.sync() del stream # optional! """ - - # NOTE: - # If we store cudaStream_t directly, this always leads to the following error: - # "Cannot convert Python object to 'cudaStream_t'" - # I was unable to find a good solution to this in reasonable time. Also, - # since cudaStream_t is a pointer anyways, storing it as an integer should - # be just fine (although, that certainly is ugly and hacky!). - def __cinit__(self): cdef cudaStream_t stream cdef cudaError_t e = cudaStreamCreate(&stream) From 1b896e2a4ea7bd8754d7115dd8c7c962725d4593 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 16:04:03 -0500 Subject: [PATCH 62/81] Removing faiss --- cpp/test/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 07f04ad2ab..614c112052 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -130,7 +130,6 @@ PRIVATE raft::distance raft::nn NCCL::NCCL - faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From 8d7279e21d3bb5de34765aff05bbad0289d954ff Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 16:11:20 -0500 Subject: [PATCH 63/81] Using new compilelibs option --- ci/gpu/build.sh | 2 +- cpp/test/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 190e45d775..525edc5964 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -88,7 +88,7 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -"$WORKSPACE/build.sh" libraft pyraft -v +"$WORKSPACE/build.sh" libraft pyraft -v --compilelibs gpuci_logger "Building doxygen C++ docs" "$WORKSPACE/build.sh" cppdocs -v diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 614c112052..07f04ad2ab 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -130,6 +130,7 @@ PRIVATE raft::distance raft::nn NCCL::NCCL + faiss::faiss GTest::gtest GTest::gtest_main Threads::Threads From 92a89f3c2ed7960bac68b39e38f45e0bf3feec3e Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 16:16:13 -0500 Subject: [PATCH 64/81] Adding compile libs to expected options --- build.sh | 2 +- ci/gpu/build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index e24bb168dc..268ab4ead4 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pyraft cppdocs -v -g --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" +VALIDARGS="clean libraft pyraft cppdocs -v -g --compilelibs --allgpuarch --nvtx --show_depr_warn -h --buildgtest --buildfaiss" HELP="$0 [ ...] [ ...] where is: clean - remove all existing build artifacts and configuration (start over) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index 525edc5964..9a70f96957 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -88,7 +88,7 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -"$WORKSPACE/build.sh" libraft pyraft -v --compilelibs +"$WORKSPACE/build.sh" libraft pyraft -v --compilelibs gpuci_logger "Building doxygen C++ docs" "$WORKSPACE/build.sh" cppdocs -v From bef24e728e7003dc13c9e19baf2ca319143f6d64 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 19 Jan 2022 16:17:51 -0500 Subject: [PATCH 65/81] removing artifact from bad merge --- build.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sh b/build.sh index 268ab4ead4..e2eb8e8080 100755 --- a/build.sh +++ b/build.sh @@ -159,7 +159,6 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft; then -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ - -DNVTX=${NVTX} -DNVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ -DBUILD_GTEST=${BUILD_GTEST} \ From 8446e2f022b22235f15d524c53e73011d4486b06 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 16:31:58 -0500 Subject: [PATCH 66/81] Style --- python/raft/common/cuda.pyx | 3 ++- python/raft/common/handle.pyx | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index 8d2f6d8c69..b5be977bcc 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -19,7 +19,7 @@ # cython: embedsignature = True # cython: language_level = 3 -from cuda.ccudart cimport ( +from cuda.ccudart cimport( cudaStream_t, cudaError_t, cudaSuccess, @@ -31,6 +31,7 @@ from cuda.ccudart cimport ( cudaGetErrorName ) + class CudaRuntimeError(RuntimeError): def __init__(self, extraMsg=None): cdef cudaError_t e = cudaGetLastError() diff --git a/python/raft/common/handle.pyx b/python/raft/common/handle.pyx index c4faabb610..661c5b5f23 100644 --- a/python/raft/common/handle.pyx +++ b/python/raft/common/handle.pyx @@ -52,7 +52,7 @@ cdef class Handle: del handle # optional! """ - def __cinit__(self, stream: Stream=None, n_streams=0): + def __cinit__(self, stream: Stream = None, n_streams=0): self.n_streams = n_streams if n_streams > 0: self.stream_pool.reset(new cuda_stream_pool(n_streams)) From 4de51ed3b0e37c6ca868d68399d3e0301447d838 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 19 Jan 2022 16:33:01 -0500 Subject: [PATCH 67/81] 0 -> cudaSuccess --- python/raft/common/cuda.pyx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/raft/common/cuda.pyx b/python/raft/common/cuda.pyx index b5be977bcc..c3c90936aa 100644 --- a/python/raft/common/cuda.pyx +++ b/python/raft/common/cuda.pyx @@ -67,7 +67,7 @@ cdef class Stream: def __dealloc__(self): self.sync() cdef cudaError_t e = cudaStreamDestroy(self.s) - if e != 0: + if e != cudaSuccess: raise CudaRuntimeError("Stream destroy") def sync(self): @@ -77,7 +77,7 @@ cdef class Stream: launches """ cdef cudaError_t e = cudaStreamSynchronize(self.s) - if e != 0: + if e != cudaSuccess: raise CudaRuntimeError("Stream sync") cdef cudaStream_t getStream(self): From 9d1bdc54ea98f040c1e50954a8d1e95f7ab61c1c Mon Sep 17 00:00:00 2001 From: Robert Maynard Date: Wed, 19 Jan 2022 17:27:40 -0500 Subject: [PATCH 68/81] Add raft_export while rapids-cmake adds a export(COMPONENT) feature --- cpp/CMakeLists.txt | 28 ++-- cpp/cmake/modules/config.cmake.in | 122 ++++++++++++++ cpp/cmake/modules/raft_export.cmake | 238 +++++++++++++++++++++++++++ cpp/cmake/thirdparty/get_faiss.cmake | 1 + 4 files changed, 372 insertions(+), 17 deletions(-) create mode 100644 cpp/cmake/modules/config.cmake.in create mode 100644 cpp/cmake/modules/raft_export.cmake diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 257adf76b5..7e01f19190 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -266,21 +266,13 @@ Imported Targets: set(code_string [=[ -set(raft_allowed_components distance nn) -foreach(comp IN LISTS raft_allowed_components) - if(${comp} IN_LIST raft_FIND_COMPONENTS) - include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-targets.cmake") - - if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") - include("${CMAKE_CURRENT_LIST_DIR}/raft-${comp}-dependencies.cmake") - endif() - endif() - +if(distance IN_LIST raft_FIND_COMPONENTS) enable_language(CUDA) - -endforeach() +endif() if(nn IN_LIST raft_FIND_COMPONENTS) + enable_language(CUDA) + if(TARGET faiss AND (NOT TARGET faiss::faiss)) add_library(faiss::faiss ALIAS faiss) elseif(TARGET faiss::faiss AND (NOT TARGET faiss)) @@ -290,11 +282,12 @@ endif() ]=] ) -# enable_cuda - -rapids_export(INSTALL raft +# Use `rapids_export` for 22.04 as it will have COMPONENT support +include(cmake/modules/raft_export.cmake) +raft_export(INSTALL raft EXPORT_SET raft-exports - GLOBAL_TARGETS raft raft_distance raft_nn + COMPONENTS nn distance + GLOBAL_TARGETS raft nn distance NAMESPACE raft:: DOCUMENTATION doc_string FINAL_CODE_BLOCK code_string @@ -303,8 +296,9 @@ rapids_export(INSTALL raft ############################################################################## # - build export ------------------------------------------------------------- -rapids_export(BUILD raft +raft_export(BUILD raft EXPORT_SET raft-exports + COMPONENTS nn distance GLOBAL_TARGETS raft raft_distance raft_nn DOCUMENTATION doc_string NAMESPACE raft:: diff --git a/cpp/cmake/modules/config.cmake.in b/cpp/cmake/modules/config.cmake.in new file mode 100644 index 0000000000..cde26e1da8 --- /dev/null +++ b/cpp/cmake/modules/config.cmake.in @@ -0,0 +1,122 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +#[=======================================================================[ + +@RAPIDS_PROJECT_DOCUMENTATION@ + +Result Variables +^^^^^^^^^^^^^^^^ + +This module will set the following variables:: + + @project_name_uppercase@_FOUND + @project_name_uppercase@_VERSION + @project_name_uppercase@_VERSION_MAJOR + @project_name_uppercase@_VERSION_MINOR + +#]=======================================================================] + +@PACKAGE_INIT@ + +cmake_minimum_required(VERSION @CMAKE_MINIMUM_REQUIRED_VERSION@) + +set(rapids_global_languages @RAPIDS_LANGUAGES@) +foreach(lang IN LISTS rapids_global_languages) + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${lang}-language.cmake") +endforeach() +unset(rapids_global_languages) + +set(rapids_allowed_components @RAPIDS_COMPONENTS@) + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-dependencies.cmake") +endif() +foreach(comp IN LISTS rapids_allowed_components) + # find dependencies before creating targets that use them + # this way if a dependency can't be found we fail + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND + EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-dependencies.cmake") + endif() +endforeach() + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-targets.cmake") +endif() +foreach(comp IN LISTS rapids_allowed_components) + if(${comp} IN_LIST @project_name@_FIND_COMPONENTS AND + EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-${comp}-targets.cmake") + endif() +endforeach() + +unset(rapids_allowed_components) + +if(EXISTS "${CMAKE_CURRENT_LIST_DIR}/@project_name@-config-version.cmake") + include("${CMAKE_CURRENT_LIST_DIR}/@project_name@-config-version.cmake") +endif() + +# Set our version variables +set(@project_name_uppercase@_VERSION_MAJOR @rapids_orig_major_version@) +set(@project_name_uppercase@_VERSION_MINOR @rapids_orig_minor_version@) +set(@project_name_uppercase@_VERSION_PATCH @rapids_orig_patch_version@) +set(@project_name_uppercase@_VERSION @rapids_orig_version@) + + +set(rapids_global_targets @RAPIDS_GLOBAL_TARGETS@) +set(rapids_namespaced_global_targets @RAPIDS_GLOBAL_TARGETS@) +if(rapids_namespaced_global_targets) + list(TRANSFORM rapids_namespaced_global_targets PREPEND @RAPIDS_NAMESPACE@ ) +endif() + +foreach(target IN LISTS rapids_namespaced_global_targets) + if(TARGET ${target}) + get_target_property(_is_imported ${target} IMPORTED) + get_target_property(_already_global ${target} IMPORTED_GLOBAL) + if(_is_imported AND NOT _already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + endif() +endforeach() + +# For backwards compat +if("rapids_config_@type@" STREQUAL "rapids_config_build") + foreach(target IN LISTS rapids_global_targets) + if(TARGET ${target}) + get_target_property(_is_imported ${target} IMPORTED) + get_target_property(_already_global ${target} IMPORTED_GLOBAL) + if(_is_imported AND NOT _already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + if(NOT TARGET @RAPIDS_NAMESPACE@${target}) + add_library(@RAPIDS_NAMESPACE@${target} ALIAS ${target}) + endif() + endif() + endforeach() +endif() + +unset(rapids_global_targets) +unset(rapids_namespaced_global_targets) + +check_required_components(@project_name@) + +set(${CMAKE_FIND_PACKAGE_NAME}_CONFIG "${CMAKE_CURRENT_LIST_FILE}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(${CMAKE_FIND_PACKAGE_NAME} CONFIG_MODE) + +@RAPIDS_PROJECT_FINAL_CODE_BLOCK@ diff --git a/cpp/cmake/modules/raft_export.cmake b/cpp/cmake/modules/raft_export.cmake new file mode 100644 index 0000000000..4411433336 --- /dev/null +++ b/cpp/cmake/modules/raft_export.cmake @@ -0,0 +1,238 @@ +#============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= +include_guard(GLOBAL) + +#[=======================================================================[.rst: +raft_export +--------------------- + +Generate a projects -Config.cmake module and all related information + +.. code-block:: cmake + + raft_export( (BUILD|INSTALL) + EXPORT_SET + [ COMPONENTS ] + [ GLOBAL_TARGETS ] + [ VERSION ] + [ NAMESPACE ] + [ DOCUMENTATION ] + [ FINAL_CODE_BLOCK ] + [ LANGUAGES ] + ) + +The :cmake:command:`raft_export` function allow projects to easily generate a fully +correct build and install tree `Project-Config.cmake` module including any necessary +calls to :cmake:command:`find_dependency`, or :cmake:command:`CPMFindPackage`. + +.. note:: + :cmake:command:`raft_export` always installs to `lib` and doesn't use GNUInstallDirs + + The files generated by :cmake:command:`raft_export` are completly standalone + and don't require the consuming package to use `rapids-cmake` + +``project_name`` + Name of the project, to be used by consumers when using `find_package` + +``GLOBAL_TARGETS`` + Explicitly list what targets should be made globally visibile to + the consuming project. + +``VERSION`` + Explicitly list the version of the package being exported. By + default :cmake:command:`raft_export` uses the version specified by the + root level :cmake:command:`project` call. If no version has been specified + either way or `OFF` is provided as the `VERSION` value, no version compatibility + checks will be generated. + + Depending on the version string different compatibility modes will be used. + + +------------------+---------------------+ + | Version String | Compatiblity Type | + +==================+=====================+ + | None | No checks perfomed | + +------------------+---------------------+ + | X | SameMajorVersion | + +------------------+---------------------+ + | X.Y | SameMinorVersion | + +------------------+---------------------+ + | X.Y.Z | SameMinorVersion | + +------------------+---------------------+ + +.. note:: + It can be useful to explicitly specify a version string when generating + export rules for a sub-component of alarger project, or an external + project that doesn't have export rules. + +``NAMESPACE`` + Optional value to specify what namespace all targets from the + EXPORT_SET will be placed into. When provided must match the pattern + of `::`. + If not provided all targets will be placed in the `::` + namespace + + Note: When exporting with `BUILD` type, only `GLOBAL_TARGETS` will + be placed in the namespace. + +``DOCUMENTATION`` + Optional value of the variable that holds the documentation + for this config file. + + Note: This requires the documentation variable instead of the contents + so we can handle having CMake code inside the documentation + +``FINAL_CODE_BLOCK`` + Optional value of the variable that holds a string of code that will + be executed at the last step of this config file. + + Note: This requires the code block variable instead of the contents + so that we can properly insert CMake code + +``LANGUAGES`` + Non default languages, such as CUDA that are required by consumers + of your package. This makes sure all consumers properly setup these + languages correctly. + + This is required as CMake's :cmake:command:`enable_language` only supports + enabling languages for the current directory scope, and doesn't support + being called from within functions. Marking languages here overcomes + these limitations and makes it possible for packages included via + `CPM` to enable languages. + + +#]=======================================================================] +# cmake-lint: disable=R0912,R0915,W0105 +function(raft_export type project_name) + include(CMakePackageConfigHelpers) + + list(APPEND CMAKE_MESSAGE_CONTEXT "raft.export") + string(TOLOWER ${type} type) + + set(options "") + set(one_value EXPORT_SET VERSION NAMESPACE DOCUMENTATION FINAL_CODE_BLOCK) + set(multi_value COMPONENTS GLOBAL_TARGETS LANGUAGES) + cmake_parse_arguments(RAPIDS "${options}" "${one_value}" "${multi_value}" ${ARGN}) + + set(rapids_version_set ON) + if(DEFINED RAPIDS_VERSION AND NOT RAPIDS_VERSION) + # We need to capture `VERSION OFF` so we need to make sure it has an off value, and not just + # undefined + set(rapids_version_set OFF) + unset(RAPIDS_VERSION) # unset this so we don't export a version value of `OFF` + elseif(NOT DEFINED RAPIDS_VERSION AND NOT DEFINED PROJECT_VERSION) + set(rapids_version_set OFF) + elseif(DEFINED PROJECT_VERSION AND NOT DEFINED RAPIDS_VERSION) + # Choose the project version when an explicit version isn't provided + set(RAPIDS_VERSION "${PROJECT_VERSION}") + endif() + + if(rapids_version_set) + include("${rapids-cmake-dir}/export/detail/parse_version.cmake") + rapids_export_parse_version(${RAPIDS_VERSION} rapids_orig rapids_project_version) + endif() + + set(RAPIDS_PROJECT_VERSION "${project_name}::") + if(DEFINED RAPIDS_NAMESPACE) + set(RAPIDS_PROJECT_VERSION ${RAPIDS_NAMESPACE}) + endif() + + set(RAPIDS_PROJECT_DOCUMENTATION "Generated ${project_name}-config module") + if(DEFINED RAPIDS_DOCUMENTATION) + if(NOT DEFINED ${RAPIDS_DOCUMENTATION}) + message(FATAL_ERROR "DOCUMENTATION variable `${RAPIDS_DOCUMENTATION}` doesn't exist") + endif() + set(RAPIDS_PROJECT_DOCUMENTATION "${${RAPIDS_DOCUMENTATION}}") + endif() + + if(DEFINED RAPIDS_FINAL_CODE_BLOCK) + if(NOT DEFINED ${RAPIDS_FINAL_CODE_BLOCK}) + message(FATAL_ERROR "FINAL_CODE_BLOCK variable `${RAPIDS_FINAL_CODE_BLOCK}` doesn't exist") + endif() + set(RAPIDS_PROJECT_FINAL_CODE_BLOCK "${${RAPIDS_FINAL_CODE_BLOCK}}") + endif() + + # Write configuration and version files + string(TOLOWER ${project_name} project_name) + string(TOUPPER ${project_name} project_name_uppercase) + if(type STREQUAL "install") + include("${rapids-cmake-dir}/cmake/install_lib_dir.cmake") + rapids_cmake_install_lib_dir(install_location) + set(install_location "${install_location}/cmake/${project_name}") + + set(scratch_dir "${PROJECT_BINARY_DIR}/rapids-cmake/${project_name}/export") + + configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${scratch_dir}/${project_name}-config.cmake" + INSTALL_DESTINATION "${install_location}") + + if(rapids_version_set) + write_basic_package_version_file( + "${scratch_dir}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat}) + endif() + + install(EXPORT ${RAPIDS_EXPORT_SET} FILE ${project_name}-targets.cmake + NAMESPACE ${RAPIDS_PROJECT_VERSION} DESTINATION "${install_location}") + + if(TARGET rapids_export_install_${RAPIDS_EXPORT_SET}) + include("${rapids-cmake-dir}/export/write_dependencies.cmake") + set(destination "${scratch_dir}/${project_name}-dependencies.cmake") + rapids_export_write_dependencies(INSTALL ${RAPIDS_EXPORT_SET} "${destination}") + endif() + + if(DEFINED RAPIDS_LANGUAGES) + include("${rapids-cmake-dir}/export/write_language.cmake") + foreach(lang IN LISTS RAPIDS_LANGUAGES) + set(destination "${scratch_dir}/${project_name}-${lang}-language.cmake") + rapids_export_write_language(INSTALL ${lang} "${destination}") + endforeach() + endif() + + # Install everything we have generated + install(DIRECTORY "${scratch_dir}/" DESTINATION "${install_location}") + + else() + set(install_location "${PROJECT_BINARY_DIR}") + configure_package_config_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/config.cmake.in" + "${install_location}/${project_name}-config.cmake" + INSTALL_DESTINATION "${install_location}") + + if(rapids_version_set) + write_basic_package_version_file( + "${install_location}/${project_name}-config-version.cmake" VERSION ${rapids_project_version} + COMPATIBILITY ${rapids_project_version_compat}) + endif() + + export(EXPORT ${RAPIDS_EXPORT_SET} NAMESPACE ${RAPIDS_PROJECT_VERSION} + FILE "${install_location}/${project_name}-targets.cmake") + + if(TARGET rapids_export_build_${RAPIDS_EXPORT_SET}) + include("${rapids-cmake-dir}/export/write_dependencies.cmake") + rapids_export_write_dependencies(BUILD ${RAPIDS_EXPORT_SET} + "${install_location}/${project_name}-dependencies.cmake") + endif() + + if(DEFINED RAPIDS_LANGUAGES) + include("${rapids-cmake-dir}/export/write_language.cmake") + foreach(lang IN LISTS RAPIDS_LANGUAGES) + rapids_export_write_language(BUILD ${lang} + "${install_location}/${project_name}-${lang}-language.cmake") + endforeach() + endif() + + endif() + +endfunction() diff --git a/cpp/cmake/thirdparty/get_faiss.cmake b/cpp/cmake/thirdparty/get_faiss.cmake index 46ce522f8c..1079db3294 100644 --- a/cpp/cmake/thirdparty/get_faiss.cmake +++ b/cpp/cmake/thirdparty/get_faiss.cmake @@ -58,6 +58,7 @@ function(find_and_configure_faiss) endif() # We generate the faiss-config files when we built faiss locally, so always do `find_dependency` + rapids_export_package(BUILD OpenMP raft-nn-exports) # faiss uses openMP but doesn't export a need for it rapids_export_package(BUILD faiss raft-nn-exports) rapids_export_package(INSTALL faiss raft-nn-exports) From d4346eefc78554497951b33902ee79c16ce4755f Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Mon, 24 Jan 2022 12:54:04 -0500 Subject: [PATCH 69/81] Updates to build.sh --- README.md | 3 --- build.sh | 10 +++++---- conda/recipes/libraft_distance/meta.yaml | 3 --- conda/recipes/pyraft/meta.yaml | 1 + cpp/CMakeLists.txt | 1 + cpp/cmake/thirdparty/get_thrust.cmake | 27 ++++++++++++++++++++++++ 6 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 cpp/cmake/thirdparty/get_thrust.cmake diff --git a/README.md b/README.md index c0eeab75e5..cad1f9f5ef 100755 --- a/README.md +++ b/README.md @@ -54,9 +54,6 @@ raft::distance::pairwise_distance(handle, input.data(), input.data(), workspace.data(), metric); ``` - - - ## Folder Structure and Contents The folder structure mirrors other RAPIDS repos (cuDF, cuML, cuGraph...), with the following folders: diff --git a/build.sh b/build.sh index b58e227a35..895e737ccd 100755 --- a/build.sh +++ b/build.sh @@ -47,11 +47,12 @@ BUILD_DIRS="${CPP_RAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PYTHON_DEPS_CLONE}" # Set defaults for vars modified by flags to this script CMAKE_LOG_LEVEL="" -VERBOSE_FLAG="" +VERBOSE_FLAG="-v" BUILD_ALL_GPU_ARCH=0 -BUILD_GTEST=OFF +BUILD_TESTS=ON BUILD_STATIC_FAISS=OFF COMPILE_LIBRARIES=OFF +ENABLE_NN_DEPENDENCIES=ON SINGLEGPU="" NVTX=OFF CLEAN=0 @@ -157,12 +158,13 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \ -DCMAKE_CUDA_ARCHITECTURES=${RAFT_CMAKE_CUDA_ARCHITECTURES} \ -DRAFT_COMPILE_LIBRARIES=${COMPILE_LIBRARIES} \ + -DRAFT_ENABLE_NN_DEPENDENCIES=${ENABLE_NN_DEPENDENCIES} \ -DNVTX=${NVTX} \ -DDISABLE_DEPRECATION_WARNING=${BUILD_DISABLE_DEPRECATION_WARNING} \ - -DBUILD_GTEST=${BUILD_GTEST} \ + -DBUILD_TESTS=${BUILD_TESTS} \ -DRAFT_USE_FAISS_STATIC=${BUILD_STATIC_FAISS} - if hasArg cppraft; then + if (( ${NUMARGS} == 0 )) || hasArg libraft; then # Run all c++ targets at once cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${MAKE_TARGETS} ${VERBOSE_FLAG} fi diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index c15a33dc38..480b289e6b 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -45,14 +45,11 @@ requirements: - faiss-proc=*=cuda - gtest=1.10.0 - gmock - - libfaiss 1.7.0 *_cuda run: - nccl>=2.9.9 - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - faiss-proc=*=cuda - - libfaiss 1.7.0 *_cuda - libcusolver>=11.2.1 about: diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 8b7f4c25cd..13ba6266fb 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -44,6 +44,7 @@ requirements: - ucx-proc=*=gpu - dask>=2021.11.1,<=2021.11.2 - distributed>=2021.11.1,<=2021.11.2 + - cuda-python >=11.5,<12.0 - joblib >=0.11 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 64c1dbb809..49df39c996 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -188,6 +188,7 @@ endif() target_link_libraries(raft_distance INTERFACE raft::raft $ $ +) ############################################################################## # - raft_nn ------------------------------------------------------------------ diff --git a/cpp/cmake/thirdparty/get_thrust.cmake b/cpp/cmake/thirdparty/get_thrust.cmake new file mode 100644 index 0000000000..ed6a2e48ec --- /dev/null +++ b/cpp/cmake/thirdparty/get_thrust.cmake @@ -0,0 +1,27 @@ +# ============================================================================= +# Copyright (c) 2021, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +# Use CPM to find or clone thrust +function(find_and_configure_thrust) + include(${rapids-cmake-dir}/cpm/thrust.cmake) + + rapids_cpm_thrust( + NAMESPACE raft + BUILD_EXPORT_SET raft-exports + INSTALL_EXPORT_SET raft-exports + ) + +endfunction() + +find_and_configure_thrust() \ No newline at end of file From d8a16d888f53ad22744cd7e7f74d12d0fb52691c Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 25 Jan 2022 15:56:13 -0500 Subject: [PATCH 70/81] Conda packages *should* be ready --- build.sh | 29 ++++++++++++++++++++----- ci/gpu/build.sh | 7 ++++-- conda/recipes/libraft_distance/build.sh | 4 +--- conda/recipes/libraft_headers/build.sh | 2 +- conda/recipes/libraft_nn/build.sh | 4 +--- conda/recipes/pyraft/build.sh | 2 +- 6 files changed, 33 insertions(+), 15 deletions(-) diff --git a/build.sh b/build.sh index 3a140abfcd..ed15017333 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pyraft docs -v -g --compilelibs --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss" +VALIDARGS="clean libraft pyraft docs -v -g --compilelibs --compile-nn --compile-dist --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss" HELP="$0 [ ...] [ ...] where is: clean - remove all existing build artifacts and configuration (start over) @@ -30,7 +30,9 @@ HELP="$0 [ ...] [ ...] and is: -v - verbose build mode -g - build for debug - --compilelibs - compile shared libraries + --compilelibs - compile shared libraries for all components + --compile-nn - compile shared library for nn component + --compile-dist - compile shared library for distance component --allgpuarch - build for all supported GPU architectures --buildfaiss - build faiss statically into raft --nogtest - do not build google tests for libraft @@ -107,10 +109,17 @@ if hasArg --nogtest; then COMPILE_LIBRARIES=OFF ENABLE_NN_DEPENDENCIES=OFF fi -if hasArg --compilelibs; then - COMPILE_LIBRARIES=ON + +if hasArg --compile-nn || hasArg --compilelibs; then ENABLE_NN_DEPENDENCIES=ON + COMPILE_LIBRARIES=ON + CMAKE_TARGET="raft_nn_lib;${CMAKE_TARGET}" +fi +if hasArg --compile-dist || hasArg --compilelibs; then + COMPILE_LIBRARIES=ON + CMAKE_TARGET="raft_distance_lib;${CMAKE_TARGET}" fi + if hasArg --buildfaiss; then BUILD_STATIC_FAISS=ON fi @@ -171,8 +180,18 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then if (( ${NUMARGS} == 0 )) || hasArg libraft; then # Run all c++ targets at once - cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} ${CMAKE_TARGET} + if hasArg --compile-nn || hasArg --compile-dist || hasArg --compilelibs; then + if ! hasArg --nogtest; then + CMAKE_TARGET="test_raft;${CMAKE_TARGET}" + fi + + echo "-- Compiling targets: ${CMAKE_TARGET}" + cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} --target ${CMAKE_TARGET} + else + cmake --build ${CPP_RAFT_BUILD_DIR} -j${PARALLEL_LEVEL} ${VERBOSE_FLAG} + fi fi + fi # Build and (optionally) install the cuml Python package diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index b5366a394b..1ffa0bb8c2 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -89,7 +89,11 @@ export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" -"$WORKSPACE/build.sh" libraft pyraft -v --compilelibs +if hasArg --skip-tests; then + "$WORKSPACE/build.sh" libraft pyraft libraft -v --compilelibs --nogtest +else + "$WORKSPACE/build.sh" libraft pyraft libraft -v --compilelibs +fi gpuci_logger "Building docs" "$WORKSPACE/build.sh" docs -v @@ -99,7 +103,6 @@ gpuci_logger "Resetting LD_LIBRARY_PATH" export LD_LIBRARY_PATH=$LD_LIBRARY_PATH_CACHED export LD_LIBRARY_PATH_CACHED="" - ################################################################################ # TEST - Run GoogleTest and py.tests for RAFT ################################################################################ diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft_distance/build.sh index 5212d67552..713d3fa66e 100644 --- a/conda/recipes/libraft_distance/build.sh +++ b/conda/recipes/libraft_distance/build.sh @@ -4,6 +4,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch --compilelibs - -# TODO: Somehow just extract the libraft_distance.so \ No newline at end of file +./build.sh clean libraft -v --allgpuarch --compile-dist --nogtest diff --git a/conda/recipes/libraft_headers/build.sh b/conda/recipes/libraft_headers/build.sh index bfd2d9d723..e95b20549b 100644 --- a/conda/recipes/libraft_headers/build.sh +++ b/conda/recipes/libraft_headers/build.sh @@ -4,4 +4,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch # TODO: Compile only the core instantiations \ No newline at end of file +./build.sh clean libraft -v --allgpuarch --nogtest diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft_nn/build.sh index fe557c6c11..2fbc0d4194 100644 --- a/conda/recipes/libraft_nn/build.sh +++ b/conda/recipes/libraft_nn/build.sh @@ -4,6 +4,4 @@ if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then export MACOSX_DEPLOYMENT_TARGET=10.11 fi -./build.sh clean libraft -v --allgpuarch --compilelibs - -# TODO: Somehow extract just the libraft_nn.so artiface \ No newline at end of file +./build.sh clean libraft -v --allgpuarch --compile-nn --nogtest diff --git a/conda/recipes/pyraft/build.sh b/conda/recipes/pyraft/build.sh index cdad5be8e7..044a34f906 100644 --- a/conda/recipes/pyraft/build.sh +++ b/conda/recipes/pyraft/build.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # This assumes the script is executed from the root of the repo directory -./build.sh pyraft \ No newline at end of file +./build.sh pyraft From fbe77c405c6c046eff684812eecc861a6a3be401 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 25 Jan 2022 18:35:29 -0500 Subject: [PATCH 71/81] Review feedback --- build.sh | 12 ++--- conda/recipes/libraft_distance/build.sh | 6 +-- conda/recipes/libraft_distance/meta.yaml | 59 ++---------------------- conda/recipes/libraft_headers/build.sh | 6 +-- conda/recipes/libraft_headers/meta.yaml | 5 +- conda/recipes/libraft_nn/build.sh | 6 +-- conda/recipes/libraft_nn/meta.yaml | 6 +-- conda/recipes/pyraft/meta.yaml | 3 +- 8 files changed, 18 insertions(+), 85 deletions(-) diff --git a/build.sh b/build.sh index ed15017333..a636d100b4 100755 --- a/build.sh +++ b/build.sh @@ -18,7 +18,7 @@ ARGS=$* # script, and that this script resides in the repo dir! REPODIR=$(cd $(dirname $0); pwd) -VALIDARGS="clean libraft pyraft docs -v -g --compilelibs --compile-nn --compile-dist --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss" +VALIDARGS="clean libraft pyraft docs -v -g --compile-libs --compile-nn --compile-dist --allgpuarch --nvtx --show_depr_warn -h --nogtest --buildfaiss" HELP="$0 [ ...] [ ...] where is: clean - remove all existing build artifacts and configuration (start over) @@ -30,7 +30,7 @@ HELP="$0 [ ...] [ ...] and is: -v - verbose build mode -g - build for debug - --compilelibs - compile shared libraries for all components + --compile-libs - compile shared libraries for all components --compile-nn - compile shared library for nn component --compile-dist - compile shared library for distance component --allgpuarch - build for all supported GPU architectures @@ -50,7 +50,7 @@ BUILD_DIRS="${CPP_RAFT_BUILD_DIR} ${PY_RAFT_BUILD_DIR} ${PYTHON_DEPS_CLONE}" # Set defaults for vars modified by flags to this script CMAKE_LOG_LEVEL="" -VERBOSE_FLAG="-v" +VERBOSE_FLAG="" BUILD_ALL_GPU_ARCH=0 BUILD_TESTS=YES BUILD_STATIC_FAISS=OFF @@ -110,12 +110,12 @@ if hasArg --nogtest; then ENABLE_NN_DEPENDENCIES=OFF fi -if hasArg --compile-nn || hasArg --compilelibs; then +if hasArg --compile-nn || hasArg --compile-libs; then ENABLE_NN_DEPENDENCIES=ON COMPILE_LIBRARIES=ON CMAKE_TARGET="raft_nn_lib;${CMAKE_TARGET}" fi -if hasArg --compile-dist || hasArg --compilelibs; then +if hasArg --compile-dist || hasArg --compile-libs; then COMPILE_LIBRARIES=ON CMAKE_TARGET="raft_distance_lib;${CMAKE_TARGET}" fi @@ -180,7 +180,7 @@ if (( ${NUMARGS} == 0 )) || hasArg libraft || hasArg docs; then if (( ${NUMARGS} == 0 )) || hasArg libraft; then # Run all c++ targets at once - if hasArg --compile-nn || hasArg --compile-dist || hasArg --compilelibs; then + if hasArg --compile-nn || hasArg --compile-dist || hasArg --compile-libs; then if ! hasArg --nogtest; then CMAKE_TARGET="test_raft;${CMAKE_TARGET}" fi diff --git a/conda/recipes/libraft_distance/build.sh b/conda/recipes/libraft_distance/build.sh index 713d3fa66e..72b43b7cf0 100644 --- a/conda/recipes/libraft_distance/build.sh +++ b/conda/recipes/libraft_distance/build.sh @@ -1,7 +1,3 @@ -if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then - # C++11 requires 10.9 - # but cudatoolkit 8 is build for 10.11 - export MACOSX_DEPLOYMENT_TARGET=10.11 -fi +#!/usr/bin/env bash ./build.sh clean libraft -v --allgpuarch --compile-dist --nogtest diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index 480b289e6b..3ae1e3bada 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -8,7 +8,7 @@ {% set cuda_major=cuda_version.split('.')[0] %} {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: - name: libraft + name: libraft-distance version: {{ version }} source: @@ -36,13 +36,11 @@ requirements: build: - cmake>=3.20.1 host: - - libraft-core {{ + - libraft-headers {{ - nccl>=2.9.9 - cudatoolkit {{ cuda_version }}.* - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - - lapack - - faiss-proc=*=cuda - gtest=1.10.0 - gmock run: @@ -56,55 +54,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft library# Copyright (c) 2022, NVIDIA CORPORATION. - - # Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} - {% set cuda_major=cuda_version.split('.')[0] %} - {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} -package: - name: libraft - version: {{ version }} - -source: - git_url: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - CC - - CXX - - CUDAHOSTCXX - - PARALLEL_LEVEL - - VERSION_SUFFIX - - PROJECT_FLASH - - CCACHE_DIR - - CCACHE_NOHASHDIR - - CCACHE_COMPILERCHECK - - CMAKE_GENERATOR - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - -requirements: - build: - - cmake>=3.20.1 - host: - - libraft-core {{ version }} - - cudatoolkit {{ cuda_version }}.* - - lapack - - gtest=1.10.0 - - gmock - run: - - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - - libcusolver>=11.2.1 - -about: - home: http://rapids.ai/ - license: Apache-2.0 - # license_file: LICENSE - summary: libraft library \ No newline at end of file + summary: libraft-distance library \ No newline at end of file diff --git a/conda/recipes/libraft_headers/build.sh b/conda/recipes/libraft_headers/build.sh index e95b20549b..eec262bb1e 100644 --- a/conda/recipes/libraft_headers/build.sh +++ b/conda/recipes/libraft_headers/build.sh @@ -1,7 +1,3 @@ -if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then - # C++11 requires 10.9 - # but cudatoolkit 8 is build for 10.11 - export MACOSX_DEPLOYMENT_TARGET=10.11 -fi +#!/usr/bin/env bash ./build.sh clean libraft -v --allgpuarch --nogtest diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index 84ad70006d..3eaf449c5a 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -8,7 +8,7 @@ {% set cuda_major=cuda_version.split('.')[0] %} {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: - name: libraft + name: libraft-headers version: {{ version }} source: @@ -40,7 +40,6 @@ requirements: - cudatoolkit {{ cuda_version }}.* - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - - lapack - gtest=1.10.0 - gmock run: @@ -54,4 +53,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft library \ No newline at end of file + summary: libraft-headers library \ No newline at end of file diff --git a/conda/recipes/libraft_nn/build.sh b/conda/recipes/libraft_nn/build.sh index 2fbc0d4194..30dfdbe04f 100644 --- a/conda/recipes/libraft_nn/build.sh +++ b/conda/recipes/libraft_nn/build.sh @@ -1,7 +1,3 @@ -if [ -n "$MACOSX_DEPLOYMENT_TARGET" ]; then - # C++11 requires 10.9 - # but cudatoolkit 8 is build for 10.11 - export MACOSX_DEPLOYMENT_TARGET=10.11 -fi +#!/usr/bin/env bash ./build.sh clean libraft -v --allgpuarch --compile-nn --nogtest diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 6480f573cf..52fa69f38a 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -8,7 +8,7 @@ {% set cuda_major=cuda_version.split('.')[0] %} {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: - name: libraft + name: libraft-nn version: {{ version }} source: @@ -40,9 +40,9 @@ requirements: - cudatoolkit {{ cuda_version }}.* - lapack - faiss-proc=*=cuda + - libfaiss 1.7.0 *_cuda - gtest=1.10.0 - gmock - - libfaiss 1.7.0 *_cuda run: - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} - faiss-proc=*=cuda @@ -53,4 +53,4 @@ about: home: http://rapids.ai/ license: Apache-2.0 # license_file: LICENSE - summary: libraft library \ No newline at end of file + summary: libraft-nn library \ No newline at end of file diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 13ba6266fb..01dd24a05f 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -29,7 +29,6 @@ requirements: - python x.x - setuptools - cython>=0.29,<0.30 - - cmake>=3.20.1 - libraft-core={{ version }} - cudatoolkit {{ cuda_version }}.* - ucx-py {{ ucx_py_version }} @@ -43,7 +42,7 @@ requirements: - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu - dask>=2021.11.1,<=2021.11.2 - - distributed>=2021.11.1,<=2021.11.2 + - distributed>=2021.11.1,<=2022.01.0 - cuda-python >=11.5,<12.0 - joblib >=0.11 - {{ pin_compatible('cudatoolkit', max_pin='x', min_pin='x') }} From 11cdb10ea9de3f81bd05a6ba40e0166e97aabae5 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 25 Jan 2022 18:36:29 -0500 Subject: [PATCH 72/81] Missed one --- conda/recipes/pyraft/meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 01dd24a05f..2ff6c417c1 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -29,14 +29,14 @@ requirements: - python x.x - setuptools - cython>=0.29,<0.30 - - libraft-core={{ version }} + - libraft-headers={{ version }} - cudatoolkit {{ cuda_version }}.* - ucx-py {{ ucx_py_version }} - ucx-proc=*=gpu run: - python x.x - dask-cuda {{ minor_version }} - - libraft={{ version }} + - libraft-headers={{ version }} - cupy>=7.8.0,<10.0.0a0 - nccl>=2.9.9 - ucx-py {{ ucx_py_version }} From 4346f8c5ae1ea2b7b07d419ba674e70df96aa25d Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 25 Jan 2022 18:38:28 -0500 Subject: [PATCH 73/81] And missed another --- ci/gpu/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index c7729ece8b..aa1f51dbcd 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -90,9 +90,9 @@ export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH gpuci_logger "Build C++ and Python targets" if hasArg --skip-tests; then - "$WORKSPACE/build.sh" libraft pyraft libraft -v --compilelibs --nogtest + "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs --nogtest else - "$WORKSPACE/build.sh" libraft pyraft libraft -v --compilelibs + "$WORKSPACE/build.sh" libraft pyraft libraft -v --compile-libs fi gpuci_logger "Building docs" From ae22a56abab1a66684a80992adbfa9c28f99c830 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 10:52:59 -0500 Subject: [PATCH 74/81] Updates based on review feedback --- ci/checks/changelog.sh | 39 ----- ci/local/old-gpubuild.sh | 208 ----------------------- ci/prtest.config | 6 - conda/recipes/libraft_distance/meta.yaml | 12 +- conda/recipes/libraft_headers/meta.yaml | 10 +- conda/recipes/libraft_nn/meta.yaml | 12 +- conda/recipes/pyraft/meta.yaml | 14 +- 7 files changed, 24 insertions(+), 277 deletions(-) delete mode 100755 ci/checks/changelog.sh delete mode 100644 ci/local/old-gpubuild.sh delete mode 100644 ci/prtest.config diff --git a/ci/checks/changelog.sh b/ci/checks/changelog.sh deleted file mode 100755 index eff64d3b7d..0000000000 --- a/ci/checks/changelog.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -######################### -# RAFT CHANGELOG Tester # -######################### - -# Checkout main for comparison -git checkout --force --quiet main - -# Switch back to tip of PR branch -git checkout --force --quiet current-pr-branch - -# Ignore errors during searching -set +e - -# Get list of modified files between matster and PR branch -CHANGELOG=`git diff --name-only main...current-pr-branch | grep CHANGELOG.md` -# Check if CHANGELOG has PR ID -PRNUM=`cat CHANGELOG.md | grep "$PR_ID"` -RETVAL=0 - -# Return status of check result -if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then - echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n" -else - echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n" - RETVAL=1 -fi - -cat << EOF - It should be placed under the section for the appropriate release. - It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate. - It should be formatted as '- PR # ' - Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs' - - -EOF - -exit $RETVAL diff --git a/ci/local/old-gpubuild.sh b/ci/local/old-gpubuild.sh deleted file mode 100644 index 8047b06f1d..0000000000 --- a/ci/local/old-gpubuild.sh +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. -######################################### -# RAFT GPU build and test script for CI # -######################################### - -set -e -NUMARGS=$# -ARGS=$* - -# Arg parsing function -function hasArg { - (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") -} - -# Set path and build parallel level -export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH -export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4} -export CUDA_REL=${CUDA_VERSION%.*} - -# Set home to the job's workspace -export HOME="$WORKSPACE" - -# Parse git describei -cd "$WORKSPACE" -export GIT_DESCRIBE_TAG=`git describe --tags` -export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` - -# Read options for cloning/running downstream repo tests -source "$WORKSPACE/ci/prtest.config" - -# ucx-py version -export UCX_PY_VERSION='0.25.*' - -################################################################################ -# SETUP - Check environment -################################################################################ - -gpuci_logger "Check environment" -env - -gpuci_logger "Check GPU usage" -nvidia-smi - -# temporary usage of gpuci_conda_retry install with packages listed here, looking into -# using the repos yaml files for this -gpuci_logger "Activate conda env" -. /opt/conda/etc/profile.d/conda.sh -conda activate rapids -gpuci_logger "Installing packages needed for RAFT" -gpuci_conda_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ - "cupy>=7,<8.0.0a0" \ - "cudatoolkit=${CUDA_REL}" \ - "cudf=${MINOR_VERSION}" \ - "rmm=${MINOR_VERSION}" \ - "cmake==3.14.3" \ - "nccl>=2.5" \ - "dask>=2.12.0" \ - "distributed>=2.12.0" \ - "dask-cudf=${MINOR_VERSION}" \ - "dask-cuda=${MINOR_VERSION}" \ - "ucx-py=${UCX_PY_VERSION}" - -if [ "$RUN_CUML_LIBCUML_TESTS" = "ON" ] || [ "$RUN_CUML_PRIMS_TESTS" = "ON" ] || [ "$RUN_CUML_PYTHON_TESTS" = "ON" ]; then - gpuci_conda_retry install -c conda-forge -c rapidsai -c rapidsai-nightly -c nvidia \ - "nvstrings=${MINOR_VERSION}" \ - "libcumlprims=${MINOR_VERSION}" \ - "lapack" \ - "umap-learn" \ - "nccl>=2.5" \ - "statsmodels" \ - "xgboost====1.0.2dev.rapidsai0.13" \ - "lightgbm" -fi - -if [ "$RUN_CUGRAPH_LIBCUGRAPH_TESTS" = "ON" ] || [ "$RUN_CUGRAPH_PYTHON_TESTS" = "ON" ]; then - gpuci_conda_retry install -c nvidia -c rapidsai -c rapidsai-nightly -c conda-forge \ - "networkx>=2.3" \ - "python-louvain" \ - "libcypher-parser" \ - "ipython=7.3*" \ - "jupyterlab" -fi - -# Install the master version of dask, distributed, and dask-ml -set -x -pip install "git+https://github.com/dask/distributed.git@2022.01.0" --upgrade --no-deps -pip install "git+https://github.com/dask/dask.git@2022.01.0" --upgrade --no-deps -set +x - - -gpuci_logger "Check versions" -python --version -$CC --version -$CXX --version -conda info -conda config --show-sources -conda list --show-channel-urls - - -################################################################################ -# BUILD - Build RAFT tests -################################################################################ - -gpuci_logger "Adding ${CONDA_PREFIX}/lib to LD_LIBRARY_PATH" - -export LD_LIBRARY_PATH_CACHED=$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH - -gpuci_logger "Build libcuml, cuml, prims and bench targets" -"$WORKSPACE/build.sh" cppraft pyraft -v - -gpuci_logger "Resetting LD_LIBRARY_PATH" - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH_CACHED -export LD_LIBRARY_PATH_CACHED="" - -gpuci_logger "Build treelite for GPU testing" - -cd "$WORKSPACE" - - -################################################################################ -# TEST - Run GoogleTest and py.tests for RAFT -################################################################################ - -if hasArg --skip-tests; then - gpuci_logger "Skipping Tests" - exit 0 -fi - -gpuci_logger "Check GPU usage" -nvidia-smi - -gpuci_logger "GoogleTest for raft" -cd "$WORKSPACE/cpp/build" -GTEST_OUTPUT="xml:$WORKSPACE/test-results/raft_cpp/" ./test/ml - -gpuci_logger "Python pytest for cuml" -cd "$WORKSPACE/python" - -pytest --cache-clear --junitxml="$WORKSPACE/junit-cuml.xml" -v -s - - -################################################################################ -# cuML CI -################################################################################ - -if [ "$RUN_CUML_LIBCUML_TESTS" = "ON" ] || [ "$RUN_CUML_PRIMS_TESTS" = "ON" ] || [ "$RUN_CUML_PYTHON_TESTS" = "ON" ] || [ "$RUN_CUGRAPH_LIBCUGRAPH_TESTS" = "ON" ] || [ "$RUN_CUGRAPH_PYTHON_TESTS" = "ON" ]; then - cd "$WORKSPACE" - mkdir "$WORKSPACE/test_downstream_repos" - cd "$WORKSPACE/test_downstream_repos" - export RAFT_PATH="$WORKSPACE" -fi - -if [ "$RUN_CUML_LIBCUML_TESTS" = "ON" ] || [ "$RUN_CUML_PRIMS_TESTS" = "ON" ] || [ "$RUN_CUML_PYTHON_TESTS" = "ON" ]; then - cd "$WORKSPACE/test_downstream_repos" - - ## Change fork and branch to be tested here: - git clone https://github.com/rapidsai/cuml.git -b branch-0.14 - - - ## Build cuML and run tests, uncomment the tests you want to run - "$WORKSPACE/test_downstream_repos/cuml/build.sh" - - if [ "$RUN_CUML_LIBCUML_TESTS" = "ON" ]; then - gpuci_logger "GoogleTest for libcuml" - cd "$WORKSPACE/cpp/build" - GTEST_OUTPUT="xml:$WORKSPACE/test-results/libcuml_cpp/" ./test/ml - fi - - if [ "$RUN_CUML_PYTHON_TESTS" = "ON" ]; then - gpuci_logger "Python pytest for cuml" - cd "$WORKSPACE/python" - pytest --cache-clear --junitxml="$WORKSPACE/junit-cuml.xml" -v -s -m "not memleak" - fi - - if [ "$RUN_CUML_PRIMS_TESTS" = "ON" ]; then - gpuci_logger "Run ml-prims test" - cd "$WORKSPACE/cpp/build" - GTEST_OUTPUT="xml:$WORKSPACE/test-results/prims/ ./test/prims - fi -fi - - -################################################################################ -# cuGraph CI -################################################################################ - -if [ "$RUN_CUGRAPH_LIBCUGRAPH_TESTS" = "ON" ] || [ "$RUN_CUGRAPH_PYTHON_TESTS" = "ON" ]; then - cd "$WORKSPACE/test_downstream_repos" - - ## Change fork and branch to be tested here: - git clone https://github.com/rapidsai/cugraph.git -b branch-0.14 - - "$WORKSPACE/test_downstream_repos/cugraph/build.sh" clean libcugraph cugraph - - if [ "$RUN_CUGRAPH_LIBCUGRAPH_TESTS" = "ON" ]; then - gpuci_logger "GoogleTest for libcugraph" - cd "$WORKSPACE/cpp/build" - "$WORKSPACE/ci/test.sh" ${TEST_MODE_FLAG} | tee testoutput.txt - fi - - if [ "$RUN_CUGRAPH_PYTHON_TESTS" = "ON" ]; then - gpuci_logger "Python pytest for cugraph" - cd "$WORKSPACE/python" - fi -fi diff --git a/ci/prtest.config b/ci/prtest.config deleted file mode 100644 index 08bdcaa3ab..0000000000 --- a/ci/prtest.config +++ /dev/null @@ -1,6 +0,0 @@ -RUN_CUGRAPH_LIBCUGRAPH_TESTS=OFF -RUN_CUGRAPH_PYTHON_TESTS=OFF - -RUN_CUML_LIBCUML_TESTS=OFF -RUN_CUML_PRIMS_TESTS=OFF -RUN_CUML_PYTHON_TESTS=OFF diff --git a/conda/recipes/libraft_distance/meta.yaml b/conda/recipes/libraft_distance/meta.yaml index 3ae1e3bada..8c3a381a1f 100644 --- a/conda/recipes/libraft_distance/meta.yaml +++ b/conda/recipes/libraft_distance/meta.yaml @@ -1,12 +1,12 @@ # Copyright (c) 2022, NVIDIA CORPORATION. # Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} - {% set cuda_major=cuda_version.split('.')[0] %} - {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +# conda build . -c conda-forge -c nvidia -c rapidsai +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_major=cuda_version.split('.')[0] %} +{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: name: libraft-distance version: {{ version }} diff --git a/conda/recipes/libraft_headers/meta.yaml b/conda/recipes/libraft_headers/meta.yaml index 3eaf449c5a..c7aa793ae5 100644 --- a/conda/recipes/libraft_headers/meta.yaml +++ b/conda/recipes/libraft_headers/meta.yaml @@ -2,11 +2,11 @@ # Usage: # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} - {% set cuda_major=cuda_version.split('.')[0] %} - {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_major=cuda_version.split('.')[0] %} +{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: name: libraft-headers version: {{ version }} diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 52fa69f38a..879f960d71 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -1,12 +1,12 @@ # Copyright (c) 2022, NVIDIA CORPORATION. # Usage: - # conda build . -c conda-forge -c nvidia -c rapidsai - {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} - {% set cuda_major=cuda_version.split('.')[0] %} - {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +# conda build . -c conda-forge -c nvidia -c rapidsai +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version='.'.join(environ.get('CUDA', '9.2').split('.')[:2]) %} +{% set cuda_major=cuda_version.split('.')[0] %} +{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: name: libraft-nn version: {{ version }} diff --git a/conda/recipes/pyraft/meta.yaml b/conda/recipes/pyraft/meta.yaml index 2ff6c417c1..b1588d57c0 100644 --- a/conda/recipes/pyraft/meta.yaml +++ b/conda/recipes/pyraft/meta.yaml @@ -1,13 +1,13 @@ # Copyright (c) 2022, NVIDIA CORPORATION. # Usage: - # conda build . -c conda-forge -c numba -c rapidsai -c pytorch - {% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} - {% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} - {% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} - {% set cuda_major=cuda_version.split('.')[0] %} - {% set py_version=environ.get('CONDA_PY', 36) %} - {% set ucx_py_version=environ.get('UCX_PY_VERSION') %} +# conda build . -c conda-forge -c numba -c rapidsai -c pytorch +{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} +{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set cuda_version='.'.join(environ.get('CUDA', 'unknown').split('.')[:2]) %} +{% set cuda_major=cuda_version.split('.')[0] %} +{% set py_version=environ.get('CONDA_PY', 36) %} +{% set ucx_py_version=environ.get('UCX_PY_VERSION') %} package: name: pyraft From 13d29d7e04bb9be5ba5e9745bec0064b8f75c6c1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 12:22:12 -0500 Subject: [PATCH 75/81] iUpdating readme --- README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 39d6c76264..63b29ff98b 100755 --- a/README.md +++ b/README.md @@ -5,13 +5,15 @@ RAFT is a library containing building-blocks for rapid composition of RAPIDS Ana ##### | Category | Description / Examples | | --- | --- | -| **Data Formats** | tensor representations and conversions for both sparse and dense formats | +| **Data Formats** | sparse & dense, conversions, and data generations | | **Data Generation** | graph, spatial, and machine learning dataset generation | -| **Dense Operations** | linear algebra, statistics | -| **Spatial** | pairwise distances, nearest neighbors, neighborhood / proximity graph construction | -| **Sparse/Graph Operations** | linear algebra, statistics, slicing, msf, spectral embedding/clustering, slhc, vertex degree | -| **Solvers** | eigenvalue decomposition, least squares, lanczos | -| **Tools** | multi-node multi-gpu communicator, utilities | +| **Dense Linear Algebra** | matrix arithmetic, norms, and factorization | +| **Spatial** | pairwise distances, nearest neighbors, and neighborhood graph construction | +| **Sparse Operations** | linear algebra, slicing, norms, degree, and spectral embedding | +| **Basic Clustering** | spectral clustering, single-linkage clustering, and k-means | +| **Optimizers** | eigenvalue decomposition, least squares, and lanczos | +| **Statistics** | sampling, kl-divergence, mean, centering, and variance | +| **Distributed Tools** | multi-node multi-gpu communicator, infrastructure | By taking a primitives-based approach to algorithm development, RAFT accelerates algorithm construction time and reduces the maintenance burden by maximizing reuse across projects. RAFT relies on the [RAPIDS memory manager (RMM)](https://github.com/rapidsai/rmm) which, From c0f5dabd406fb935b88259b509450600a5d765a2 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 12:24:25 -0500 Subject: [PATCH 76/81] iFixing build --- ci/gpu/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/gpu/build.sh b/ci/gpu/build.sh index aa1f51dbcd..e79c21f0c5 100644 --- a/ci/gpu/build.sh +++ b/ci/gpu/build.sh @@ -26,9 +26,6 @@ cd "$WORKSPACE" export GIT_DESCRIBE_TAG=`git describe --tags` export MINOR_VERSION=`echo $GIT_DESCRIBE_TAG | grep -o -E '([0-9]+\.[0-9]+)'` -# Read options for cloning/running downstream repo tests -source "$WORKSPACE/ci/prtest.config" - # ucx-py version export UCX_PY_VERSION='0.25.*' From 0ecba53892266e45f05fba4f8fa1fb7afbffc29e Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 14:01:22 -0500 Subject: [PATCH 77/81] iMore readme updates --- README.md | 49 +++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 63b29ff98b..9baae99872 100755 --- a/README.md +++ b/README.md @@ -1,33 +1,36 @@ #
 RAFT: RAPIDS Analytics Framework Toolkit
-RAFT is a library containing building-blocks for rapid composition of RAPIDS Analytics. These building-blocks include shared representations, mathematical computational primitives, and utilities that accelerate building analytics and data science algorithms in the RAPIDS ecosystem. Both the C++ and Python components can be included in consuming libraries, providing operations for both dense and sparse matrix formats in the following general categories: +RAFT is a [Scipy-like](https://scipy.org/) library for scientific computing, containing building-blocks for rapidly composing analytics in the [RAPIDS](https://rapids.ai) ecosystem. These building-blocks include infrastructure as well as mathematical computational primitives, which accelerate the development of algorithms for data science applications. + +By taking a primitives-based approach to algorithm development, RAFT +1. accelerates algorithm construction time +2. reduces the maintenance burden by maximizing reuse across projects, and +3. centralizes the core computations, allowing future optimizations to benefit all algorithms that use them. + +RAFT provides a header-only C++ API (with optional shared libraries to accelerate build time) that cover the following general categories: ##### | Category | Description / Examples | | --- | --- | | **Data Formats** | sparse & dense, conversions, and data generations | -| **Data Generation** | graph, spatial, and machine learning dataset generation | -| **Dense Linear Algebra** | matrix arithmetic, norms, and factorization | -| **Spatial** | pairwise distances, nearest neighbors, and neighborhood graph construction | -| **Sparse Operations** | linear algebra, slicing, norms, degree, and spectral embedding | -| **Basic Clustering** | spectral clustering, single-linkage clustering, and k-means | +| **Data Generation** | sparse, spatial, machine learning datasets | +| **Dense Linear Algebra** | matrix arithmetic, norms, factorization | +| **Spatial** | pairwise distances, nearest neighbors, neighborhood graph construction | +| **Sparse Operations** | linear algebra, slicing, norms, degree, spectral embedding | +| **Basic Clustering** | spectral clustering, hierarchical clustering, k-means | | **Optimizers** | eigenvalue decomposition, least squares, and lanczos | -| **Statistics** | sampling, kl-divergence, mean, centering, and variance | -| **Distributed Tools** | multi-node multi-gpu communicator, infrastructure | - -By taking a primitives-based approach to algorithm development, RAFT accelerates algorithm construction time and reduces -the maintenance burden by maximizing reuse across projects. RAFT relies on the [RAPIDS memory manager (RMM)](https://github.com/rapidsai/rmm) which, -like other projects in the RAPIDS ecosystem, eases the burden of configuring different allocation strategies globally -across the libraries that use it. RMM also provides RAII wrappers around device arrays that handle the allocation and cleanup. - -RAFT's primary goals are to be fast, simple, reusable, composable, and comprehensive. - -## Build/Install RAFT +| **Statistics** | sampling, moments, metrics | +| **Distributed Tools** | multi-node multi-gpu infrastructure | -Refer to the [Build](BUILD.md) instructions for details on building and including the RAFT library in downstream projects. +RAFT also provides a Python API that enables the building of multi-node multi-GPU algorithms in the [Dask](https://dask.org/) ecosystem. We are continuing to improve the coverage of the Python API to expose the building-blocks from the categories above. ## Getting started +### Rapids Memory Manager (RMM) +RAFT relies heavily on [RMM](https://github.com/rapidsai/rmm) which, +like other projects in the RAPIDS ecosystem, eases the burden of configuring different allocation strategies globally +across the libraries that use it. RMM also provides [RAII](https://en.wikipedia.org/wiki/Resource_acquisition_is_initialization)) wrappers around device arrays that handle the allocation and cleanup. + ### C++ Example Most of the primitives in RAFT accept a `raft::handle_t` object for the management of resources which are expensive to create, such CUDA streams, stream pools, and handles to other CUDA libraries like `cublas` and `cusolver`. @@ -57,13 +60,19 @@ raft::distance::pairwise_distance(handle, input.data(), input.data(), workspace.data(), metric); ``` +## Build/Install RAFT + +Refer to the [Build](BUILD.md) instructions for details on building and including the RAFT library in downstream projects. + ## Folder Structure and Contents The folder structure mirrors other RAPIDS repos (cuDF, cuML, cuGraph...), with the following folders: - `ci`: Scripts for running CI in PRs -- `conda`: conda recipes and development conda environments -- `cpp`: Source code for all C++ code. Headers are in the `include` folder and compiled template specializations for the shared libraries are in `src`. +- `conda`: Conda recipes and development conda environments +- `cpp`: Source code for all C++ code. + - `include`: The C++ API is fully-contained here + - `src`: Compiled template specializations for the shared libraries - `docs`: Source code and scripts for building library documentation - `python`: Source code for all Python source code. From 8217afcd0018e2a0e22733b64f931db038286858 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 15:22:43 -0500 Subject: [PATCH 78/81] libraft-core -> libraft-headers --- conda/recipes/libraft_nn/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libraft_nn/meta.yaml b/conda/recipes/libraft_nn/meta.yaml index 879f960d71..710e99902b 100644 --- a/conda/recipes/libraft_nn/meta.yaml +++ b/conda/recipes/libraft_nn/meta.yaml @@ -36,7 +36,7 @@ requirements: build: - cmake>=3.20.1 host: - - libraft-core {{ version }} + - libraft-headers {{ version }} - cudatoolkit {{ cuda_version }}.* - lapack - faiss-proc=*=cuda From 6909ff0578d1a0f93bae2a11dd440eaed32def4c Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 15:38:43 -0500 Subject: [PATCH 79/81] Adding msf to sparse ops --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9baae99872..96f39a3b30 100755 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ RAFT provides a header-only C++ API (with optional shared libraries to accelerat | **Data Generation** | sparse, spatial, machine learning datasets | | **Dense Linear Algebra** | matrix arithmetic, norms, factorization | | **Spatial** | pairwise distances, nearest neighbors, neighborhood graph construction | -| **Sparse Operations** | linear algebra, slicing, norms, degree, spectral embedding | +| **Sparse Operations** | linear algebra, slicing, norms, degree, spectral embedding, msf | | **Basic Clustering** | spectral clustering, hierarchical clustering, k-means | | **Optimizers** | eigenvalue decomposition, least squares, and lanczos | | **Statistics** | sampling, moments, metrics | From 34f2aab90afc075c17a2236a644b7aa004c93d55 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 15:39:53 -0500 Subject: [PATCH 80/81] Updates --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96f39a3b30..3c4246e198 100755 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ RAFT provides a header-only C++ API (with optional shared libraries to accelerat | **Data Generation** | sparse, spatial, machine learning datasets | | **Dense Linear Algebra** | matrix arithmetic, norms, factorization | | **Spatial** | pairwise distances, nearest neighbors, neighborhood graph construction | -| **Sparse Operations** | linear algebra, slicing, norms, degree, spectral embedding, msf | +| **Sparse Operations** | linear algebra, slicing, symmetrization, norms, spectral embedding, msf | | **Basic Clustering** | spectral clustering, hierarchical clustering, k-means | | **Optimizers** | eigenvalue decomposition, least squares, and lanczos | | **Statistics** | sampling, moments, metrics | From 8fd8f566c93220746645df2933a272b472492436 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 26 Jan 2022 15:46:14 -0500 Subject: [PATCH 81/81] adding cuda-accelerated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c4246e198..88612f13f3 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ #
 RAFT: RAPIDS Analytics Framework Toolkit
-RAFT is a [Scipy-like](https://scipy.org/) library for scientific computing, containing building-blocks for rapidly composing analytics in the [RAPIDS](https://rapids.ai) ecosystem. These building-blocks include infrastructure as well as mathematical computational primitives, which accelerate the development of algorithms for data science applications. +RAFT is a [Scipy-like](https://scipy.org/) library for scientific computing, containing CUDA-accelerated building-blocks for rapidly composing analytics in the [RAPIDS](https://rapids.ai) ecosystem. These building-blocks include infrastructure as well as mathematical computational primitives, which accelerate the development of algorithms for data science applications. By taking a primitives-based approach to algorithm development, RAFT 1. accelerates algorithm construction time