From f2b23720565d560d70d271f19c4af8548a314736 Mon Sep 17 00:00:00 2001 From: corey-derochie-amd <161367113+corey-derochie-amd@users.noreply.github.com> Date: Mon, 22 Jul 2024 00:41:31 -0600 Subject: [PATCH] Only initialize MSCCL++ when runtime-enabled. (#1266) [ROCm/rccl commit: b31b4082ddb85a81f67389d4e7a020628687146d] --- projects/rccl/CMakeLists.txt | 29 +++++++++++++++-------- projects/rccl/cmake/DownloadProject.cmake | 4 ++-- projects/rccl/cmake/MSCCLPP.cmake | 12 +++++----- projects/rccl/src/init.cc | 10 ++++---- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index 73fa176fa3..483bcc3678 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -26,7 +26,8 @@ option(BUILD_SHARED_LIBS "Build as shared library" option(BUILD_TESTS "Build unit test programs" OFF) option(COLLTRACE "Collective Trace Option" ON) option(ENABLE_MSCCL_KERNEL "Enable MSCCL while compiling" ON) -option(ENABLE_MSCCLPP "Enable MSCCL++" OFF) +option(ENABLE_MSCCLPP "Enable MSCCL++" ON) +option(BUILD_MSCCLPP "Build MSCCL++" ON) option(ENABLE_IFC "Enable indirect function call" OFF) option(INSTALL_DEPENDENCIES "Force install dependencies" OFF) option(ROCTX "Enable ROCTX" OFF) @@ -245,12 +246,20 @@ if (HAVE_KERNARG_PRELOAD) message(STATUS "Kernarg preloading to SGPR enabled") endif() -# Check for IBVerbs; disable MSCCL++ if not present -if (ENABLE_MSCCLPP) - find_package(IBVerbs) - if (NOT IBVerbs_FOUND) - set(ENABLE_MSCCLPP OFF) - message(WARNING "IBVerbs not found; disabling MSCCL++") +## Disable building MSCCL++ if the build environment is invalid +if (BUILD_MSCCLPP) + if (NOT ENABLE_MSCCLPP) + set(BUILD_MSCCLPP OFF) + message(WARNING "ENABLE_MSCCLPP not set; disabling MSCCL++ build") + else() + find_package(IBVerbs) + if (NOT IBVerbs_FOUND) + set(BUILD_MSCCLPP OFF) + message(WARNING "IBVerbs not found; disabling MSCCL++ build") + elseif(NOT "gfx942" IN_LIST GPU_TARGETS) + set(BUILD_MSCCLPP OFF) + message(WARNING "Can only build MSCCL++ for gfx942; disabling MSCCL++ build") + endif() endif() endif() @@ -707,9 +716,9 @@ if (HAVE_KERNARG_PRELOAD) target_link_options(rccl PRIVATE -Xoffload-linker -mllvm=-amdgpu-kernarg-preload-count=16) endif() -if(ENABLE_MSCCLPP) +if(BUILD_MSCCLPP) include(cmake/MSCCLPP.cmake) - message(STATUS "Building RCCL with MSCCL++ support") + message(STATUS "Building MSCCL++ with NCCL API support") endif() ## Track linking time @@ -729,7 +738,7 @@ file(COPY tools/msccl-unit-test-algorithms DESTINATION ${PROJECT_BINARY_DIR}) ## Install Algorithm files under share folder rocm_install(DIRECTORY ${PROJECT_BINARY_DIR}/msccl-algorithms DESTINATION ${CMAKE_INSTALL_DATADIR}/rccl) rocm_install(DIRECTORY ${PROJECT_BINARY_DIR}/msccl-unit-test-algorithms DESTINATION ${CMAKE_INSTALL_DATADIR}/rccl) -if(ENABLE_MSCCLPP) +if(BUILD_MSCCLPP) rocm_install(FILES ${MSCCLPP_OUT_LIBS} DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT "runtime") endif() diff --git a/projects/rccl/cmake/DownloadProject.cmake b/projects/rccl/cmake/DownloadProject.cmake index 110bbd5c4f..765b1f6d21 100644 --- a/projects/rccl/cmake/DownloadProject.cmake +++ b/projects/rccl/cmake/DownloadProject.cmake @@ -158,7 +158,7 @@ function(download_project) if(result) message(FATAL_ERROR "CMake step for ${DL_ARGS_PROJ} failed: ${result}") endif() - execute_process(COMMAND ${CMAKE_COMMAND} --build . + execute_process(COMMAND ${CMAKE_COMMAND} --build . -j16 RESULT_VARIABLE result ${OUTPUT_QUIET} WORKING_DIRECTORY "${DL_ARGS_DOWNLOAD_DIR}" @@ -167,4 +167,4 @@ function(download_project) message(FATAL_ERROR "Build step for ${DL_ARGS_PROJ} failed: ${result}") endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/projects/rccl/cmake/MSCCLPP.cmake b/projects/rccl/cmake/MSCCLPP.cmake index 8d3792850a..0000f99bbb 100644 --- a/projects/rccl/cmake/MSCCLPP.cmake +++ b/projects/rccl/cmake/MSCCLPP.cmake @@ -41,18 +41,18 @@ if(ENABLE_MSCCLPP) find_package(mscclpp_nccl) if(NOT mscclpp_nccl_FOUND) - message(STATUS "MSCCL++ not found. Downloading and building MSCCL++.") + message(STATUS "MSCCL++ not found. Downloading and building MSCCL++ only for gfx942.") # Download, build and install mscclpp download_project(PROJ mscclpp_nccl GIT_REPOSITORY https://github.com/microsoft/mscclpp.git GIT_TAG b1b9d0626cfa40319c18c05f8c16650568395c29 INSTALL_DIR ${MSCCLPP_ROOT} - CMAKE_ARGS -DGPU_TARGETS=${GPU_TARGETS} -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX= - LOG_DOWNLOAD TRUE - LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE + CMAKE_ARGS -DGPU_TARGETS=gfx942 -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX= + LOG_DOWNLOAD FALSE + LOG_CONFIGURE FALSE + LOG_BUILD FALSE + LOG_INSTALL FALSE UPDATE_DISCONNECTED TRUE ) diff --git a/projects/rccl/src/init.cc b/projects/rccl/src/init.cc index 8c1f4250b5..78b9da6426 100644 --- a/projects/rccl/src/init.cc +++ b/projects/rccl/src/init.cc @@ -100,6 +100,9 @@ size_t std::hash::operator ()(const ncclUniqueId& uniqueId) const bool operator ==(const ncclUniqueId& a, const ncclUniqueId& b) { return memcmp(a.internal, b.internal, NCCL_UNIQUE_ID_BYTES) == 0; } + +RCCL_PARAM(EnableMscclpp, "ENABLE_MSCCLPP", 0); +RCCL_PARAM(MscclppThreshold, "MSCCLPP_THRESHOLD", (size_t)(1024*1024)); #endif // GDRCOPY support: Off by default @@ -164,7 +167,7 @@ static ncclResult_t ncclInit() { initNvtxRegisteredEnums(); #endif #ifdef ENABLE_MSCCLPP - if (!mscclpp_init()) { + if (rcclParamEnableMscclpp() && !mscclpp_init()) { return ncclSystemError; } #endif @@ -181,11 +184,6 @@ ncclResult_t ncclGetVersion(int* version) { return ncclSuccess; } -#ifdef ENABLE_MSCCLPP -RCCL_PARAM(EnableMscclpp, "ENABLE_MSCCLPP", 0); -RCCL_PARAM(MscclppThreshold, "MSCCLPP_THRESHOLD", (size_t)(1024*1024)); -#endif - NCCL_API(ncclResult_t, ncclGetUniqueId, ncclUniqueId* out); ncclResult_t ncclGetUniqueId(ncclUniqueId* out) { NCCLCHECK(ncclInit());