Only initialize MSCCL++ when runtime-enabled. (#1266)
[ROCm/rccl commit: b31b4082dd]
This commit is contained in:
corey-derochie-amd
کامیت شده توسط
GitHub
GitHub
والد
863b213fd2
کامیت
f2b2372056
@@ -26,7 +26,8 @@ option(BUILD_SHARED_LIBS "Build as shared library"
|
||||
option(BUILD_TESTS "Build unit test programs" OFF)
|
||||
option(COLLTRACE "Collective Trace Option" ON)
|
||||
option(ENABLE_MSCCL_KERNEL "Enable MSCCL while compiling" ON)
|
||||
option(ENABLE_MSCCLPP "Enable MSCCL++" OFF)
|
||||
option(ENABLE_MSCCLPP "Enable MSCCL++" ON)
|
||||
option(BUILD_MSCCLPP "Build MSCCL++" ON)
|
||||
option(ENABLE_IFC "Enable indirect function call" OFF)
|
||||
option(INSTALL_DEPENDENCIES "Force install dependencies" OFF)
|
||||
option(ROCTX "Enable ROCTX" OFF)
|
||||
@@ -245,12 +246,20 @@ if (HAVE_KERNARG_PRELOAD)
|
||||
message(STATUS "Kernarg preloading to SGPR enabled")
|
||||
endif()
|
||||
|
||||
# Check for IBVerbs; disable MSCCL++ if not present
|
||||
if (ENABLE_MSCCLPP)
|
||||
find_package(IBVerbs)
|
||||
if (NOT IBVerbs_FOUND)
|
||||
set(ENABLE_MSCCLPP OFF)
|
||||
message(WARNING "IBVerbs not found; disabling MSCCL++")
|
||||
## Disable building MSCCL++ if the build environment is invalid
|
||||
if (BUILD_MSCCLPP)
|
||||
if (NOT ENABLE_MSCCLPP)
|
||||
set(BUILD_MSCCLPP OFF)
|
||||
message(WARNING "ENABLE_MSCCLPP not set; disabling MSCCL++ build")
|
||||
else()
|
||||
find_package(IBVerbs)
|
||||
if (NOT IBVerbs_FOUND)
|
||||
set(BUILD_MSCCLPP OFF)
|
||||
message(WARNING "IBVerbs not found; disabling MSCCL++ build")
|
||||
elseif(NOT "gfx942" IN_LIST GPU_TARGETS)
|
||||
set(BUILD_MSCCLPP OFF)
|
||||
message(WARNING "Can only build MSCCL++ for gfx942; disabling MSCCL++ build")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -707,9 +716,9 @@ if (HAVE_KERNARG_PRELOAD)
|
||||
target_link_options(rccl PRIVATE -Xoffload-linker -mllvm=-amdgpu-kernarg-preload-count=16)
|
||||
endif()
|
||||
|
||||
if(ENABLE_MSCCLPP)
|
||||
if(BUILD_MSCCLPP)
|
||||
include(cmake/MSCCLPP.cmake)
|
||||
message(STATUS "Building RCCL with MSCCL++ support")
|
||||
message(STATUS "Building MSCCL++ with NCCL API support")
|
||||
endif()
|
||||
|
||||
## Track linking time
|
||||
@@ -729,7 +738,7 @@ file(COPY tools/msccl-unit-test-algorithms DESTINATION ${PROJECT_BINARY_DIR})
|
||||
## Install Algorithm files under share folder
|
||||
rocm_install(DIRECTORY ${PROJECT_BINARY_DIR}/msccl-algorithms DESTINATION ${CMAKE_INSTALL_DATADIR}/rccl)
|
||||
rocm_install(DIRECTORY ${PROJECT_BINARY_DIR}/msccl-unit-test-algorithms DESTINATION ${CMAKE_INSTALL_DATADIR}/rccl)
|
||||
if(ENABLE_MSCCLPP)
|
||||
if(BUILD_MSCCLPP)
|
||||
rocm_install(FILES ${MSCCLPP_OUT_LIBS} DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT "runtime")
|
||||
endif()
|
||||
|
||||
|
||||
@@ -158,7 +158,7 @@ function(download_project)
|
||||
if(result)
|
||||
message(FATAL_ERROR "CMake step for ${DL_ARGS_PROJ} failed: ${result}")
|
||||
endif()
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} --build .
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} --build . -j16
|
||||
RESULT_VARIABLE result
|
||||
${OUTPUT_QUIET}
|
||||
WORKING_DIRECTORY "${DL_ARGS_DOWNLOAD_DIR}"
|
||||
@@ -167,4 +167,4 @@ function(download_project)
|
||||
message(FATAL_ERROR "Build step for ${DL_ARGS_PROJ} failed: ${result}")
|
||||
endif()
|
||||
|
||||
endfunction()
|
||||
endfunction()
|
||||
|
||||
@@ -41,18 +41,18 @@ if(ENABLE_MSCCLPP)
|
||||
find_package(mscclpp_nccl)
|
||||
|
||||
if(NOT mscclpp_nccl_FOUND)
|
||||
message(STATUS "MSCCL++ not found. Downloading and building MSCCL++.")
|
||||
message(STATUS "MSCCL++ not found. Downloading and building MSCCL++ only for gfx942.")
|
||||
# Download, build and install mscclpp
|
||||
|
||||
download_project(PROJ mscclpp_nccl
|
||||
GIT_REPOSITORY https://github.com/microsoft/mscclpp.git
|
||||
GIT_TAG b1b9d0626cfa40319c18c05f8c16650568395c29
|
||||
INSTALL_DIR ${MSCCLPP_ROOT}
|
||||
CMAKE_ARGS -DGPU_TARGETS=${GPU_TARGETS} -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
|
||||
LOG_DOWNLOAD TRUE
|
||||
LOG_CONFIGURE TRUE
|
||||
LOG_BUILD TRUE
|
||||
LOG_INSTALL TRUE
|
||||
CMAKE_ARGS -DGPU_TARGETS=gfx942 -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
|
||||
LOG_DOWNLOAD FALSE
|
||||
LOG_CONFIGURE FALSE
|
||||
LOG_BUILD FALSE
|
||||
LOG_INSTALL FALSE
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
)
|
||||
|
||||
|
||||
@@ -100,6 +100,9 @@ size_t std::hash<ncclUniqueId>::operator ()(const ncclUniqueId& uniqueId) const
|
||||
bool operator ==(const ncclUniqueId& a, const ncclUniqueId& b) {
|
||||
return memcmp(a.internal, b.internal, NCCL_UNIQUE_ID_BYTES) == 0;
|
||||
}
|
||||
|
||||
RCCL_PARAM(EnableMscclpp, "ENABLE_MSCCLPP", 0);
|
||||
RCCL_PARAM(MscclppThreshold, "MSCCLPP_THRESHOLD", (size_t)(1024*1024));
|
||||
#endif
|
||||
|
||||
// GDRCOPY support: Off by default
|
||||
@@ -164,7 +167,7 @@ static ncclResult_t ncclInit() {
|
||||
initNvtxRegisteredEnums();
|
||||
#endif
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
if (!mscclpp_init()) {
|
||||
if (rcclParamEnableMscclpp() && !mscclpp_init()) {
|
||||
return ncclSystemError;
|
||||
}
|
||||
#endif
|
||||
@@ -181,11 +184,6 @@ ncclResult_t ncclGetVersion(int* version) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
#ifdef ENABLE_MSCCLPP
|
||||
RCCL_PARAM(EnableMscclpp, "ENABLE_MSCCLPP", 0);
|
||||
RCCL_PARAM(MscclppThreshold, "MSCCLPP_THRESHOLD", (size_t)(1024*1024));
|
||||
#endif
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGetUniqueId, ncclUniqueId* out);
|
||||
ncclResult_t ncclGetUniqueId(ncclUniqueId* out) {
|
||||
NCCLCHECK(ncclInit());
|
||||
|
||||
مرجع در شماره جدید
Block a user