Re-enabled MSCCL++ (#1325)
* Added restrictions around calling MSCCL++ collectives (#1281)
* Added restriction to non-zero 32-byte multiple message sizes to MSCCL++ AllGather.
* Renamed and refactored some mscclpp types.
* Only transmit the MSCCL++ unique id for non-split comm init. For splitting comm, it has already been transmitted. Instead, save the MSCCL++ communicator in child communicators when calling `ncclCommSplit`. Only destroy MSCCL++ communicators when no RCCL communicators remain that use it. Also improved trace logging.
* Disable MSCCL++ when using managed memory buffers as it isn't supported.
* Added datatype and op constraints for MSCCL++ AllReduce.
* Added documentation on MSCCL++ restrictions to the README.
* [BUILD] Support custom CMake flags in MSCCLPP (#1275)
* [BUILD] Support custom CMAKE_PREFIX_PATH in MSCCLPP
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* [BUILD] CMake flags to support build-id in MSCCLPP
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* [BUILD] Fix CMake warnings in MSCCLPP build
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
* Wrapped all cmake arguments passed to mscclpp to remove empty arguments and properly format them.
---------
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
Co-authored-by: Corey Derochie <corey.derochie@amd.com>
* Link to libmscclpp_nccl statically (#1282)
* Switched mscclpp_nccl to static linking. Added a build step to rename the NCCL API functions.
* Undid separation of building libmscclpp_nccl from building librccl with MSCCL++ integration. With a static build, it's either fully enabled or fully disabled.
* `nm` isn't always available in docker containers due to being stripped down. Removed use of `nm` in `cmake` and hard-coded the output into mscclpp_nccl_syms.txt.
* Removed IBVerbs dependency for integrating with MSCCL++ (#1313)
* Renamed `RCCL_ENABLE_MSCCLPP` to `RCCL_MSCCLPP_ENABLE` to conform to MSCCL. Set `RCCL_MSCCLPP_ENABLE` to 1 by default if `ENABLE_MSCCLPP` is defined, or 0 otherwise. Added a log warning if `RCCL_MSCCLPP_ENABLE` is set to 1 but `ENABLE_MSCCLPP` is not defined. (#1294)
* Include mscclpp as a git submodule (#1314)
* Added the desired mscclpp commit as a git submodule.
* Added step to automatically checkout the mscclpp submodule if it isn't already present, in case the user forgot to clone recursively.
* Added instruction to README to clone using --recurse-submodules to get the mscclpp submodule.
* Enabled MSCCL++ feature build.
---------
Signed-off-by: nileshnegi <Nilesh.Negi@amd.com>
Co-authored-by: Nilesh M Negi <Nilesh.Negi@amd.com>
[ROCm/rccl commit: 736a705875]
This commit is contained in:
corey-derochie-amd
committed by
GitHub
GitHub
parent
e3b39ab309
commit
9ffd893c5a
@@ -32,7 +32,20 @@
|
||||
# For downloading, building, and installing required dependencies
|
||||
include(cmake/DownloadProject.cmake)
|
||||
|
||||
function(mscclpp_cmake_arg NAME)
|
||||
string (REPLACE ";" "$<SEMICOLON>" ARG_VALUE "${${NAME}}") # Replace ; with non-escapable SEMICOLON symbol to avoid CMake errors
|
||||
string(STRIP "${ARG_VALUE}" ARG_VALUE) # Eliminate whitespace, reducing to empty string if necessary
|
||||
|
||||
# Only add a cmake argument if it has a value
|
||||
set(${NAME}_ARG "-D${NAME}=\"${ARG_VALUE}\"" PARENT_SCOPE)
|
||||
if("${ARG_VALUE}" STREQUAL "")
|
||||
set(${NAME}_ARG "" PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
|
||||
if(ENABLE_MSCCLPP)
|
||||
# Try to find the mscclpp install
|
||||
set(MSCCLPP_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/ext/mscclpp CACHE PATH "")
|
||||
execute_process(
|
||||
COMMAND mkdir -p ${MSCCLPP_ROOT}
|
||||
@@ -41,30 +54,45 @@ if(ENABLE_MSCCLPP)
|
||||
find_package(mscclpp_nccl)
|
||||
|
||||
if(NOT mscclpp_nccl_FOUND)
|
||||
message(STATUS "MSCCL++ not found. Downloading and building MSCCL++ only for gfx942.")
|
||||
# Download, build and install mscclpp
|
||||
# Ensure the source code is checked out
|
||||
set(MSCCLPP_SOURCE ${CMAKE_CURRENT_SOURCE_DIR}/ext-src/mscclpp CACHE PATH "")
|
||||
if(NOT EXISTS ${MSCCLPP_SOURCE}/CMakeLists.txt)
|
||||
message(STATUS "Checking out microsoft/mscclpp")
|
||||
execute_process(
|
||||
COMMAND git submodule update --init --recursive
|
||||
WORKING_DIRECTORY ${MSCCLPP_SOURCE}
|
||||
)
|
||||
endif()
|
||||
|
||||
message(STATUS "Building mscclpp only for gfx942.")
|
||||
|
||||
mscclpp_cmake_arg(CMAKE_PREFIX_PATH)
|
||||
mscclpp_cmake_arg(CMAKE_SHARED_LINKER_FLAGS_INIT)
|
||||
mscclpp_cmake_arg(CMAKE_EXE_LINKER_FLAGS_INIT)
|
||||
mscclpp_cmake_arg(CMAKE_INSTALL_RPATH_USE_LINK_PATH)
|
||||
mscclpp_cmake_arg(HIP_COMPILER)
|
||||
|
||||
download_project(PROJ mscclpp_nccl
|
||||
GIT_REPOSITORY https://github.com/microsoft/mscclpp.git
|
||||
GIT_TAG b1b9d0626cfa40319c18c05f8c16650568395c29
|
||||
# GIT_REPOSITORY https://github.com/microsoft/mscclpp.git
|
||||
# GIT_TAG 1e82dd444fc1ed8b7add354eebaab8a94e67d5fc
|
||||
INSTALL_DIR ${MSCCLPP_ROOT}
|
||||
CMAKE_ARGS -DGPU_TARGETS=gfx942 -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=Release -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR>
|
||||
CMAKE_ARGS -DGPU_TARGETS=gfx942 -DBYPASS_GPU_CHECK=ON -DUSE_ROCM=ON -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_APPS_NCCL=ON -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> "${CMAKE_PREFIX_PATH_ARG}" "${CMAKE_SHARED_LINKER_FLAGS_INIT_ARG}" "${CMAKE_EXE_LINKER_FLAGS_INIT_ARG}" -DCMAKE_VERBOSE_MAKEFILE=1 "${CMAKE_INSTALL_RPATH_USE_LINK_PATH_ARG}" "${HIP_COMPILER_ARG}"
|
||||
LOG_DOWNLOAD FALSE
|
||||
LOG_CONFIGURE FALSE
|
||||
LOG_BUILD FALSE
|
||||
LOG_INSTALL FALSE
|
||||
UPDATE_DISCONNECTED TRUE
|
||||
SOURCE_DIR ${MSCCLPP_SOURCE}
|
||||
)
|
||||
|
||||
find_package(mscclpp_nccl REQUIRED)
|
||||
endif()
|
||||
|
||||
# Copy the outputs to the PROJECT_BINARY_DIR, list them in MSCCLPP_OUT_LIBS
|
||||
file(GLOB MSCCLPP_LIB_FILES "${MSCCLPP_ROOT}/lib/*")
|
||||
file(GLOB MSCCLPP_LIB_NAMES RELATIVE ${MSCCLPP_ROOT}/lib "${MSCCLPP_ROOT}/lib/*")
|
||||
set(MSCCLPP_OUT_LIBS "")
|
||||
foreach(LIB_NAME ${MSCCLPP_LIB_NAMES})
|
||||
list(APPEND MSCCLPP_OUT_LIBS ${PROJECT_BINARY_DIR}/${LIB_NAME})
|
||||
endforeach()
|
||||
file(COPY ${MSCCLPP_LIB_FILES} DESTINATION ${PROJECT_BINARY_DIR})
|
||||
execute_process(COMMAND objcopy
|
||||
--redefine-syms=${CMAKE_CURRENT_SOURCE_DIR}/src/misc/mscclpp/mscclpp_nccl_syms.txt
|
||||
"${MSCCLPP_ROOT}/lib/libmscclpp_nccl_static.a"
|
||||
"${PROJECT_BINARY_DIR}/libmscclpp_nccl.a"
|
||||
)
|
||||
add_library(mscclpp_nccl STATIC IMPORTED)
|
||||
set_target_properties(mscclpp_nccl PROPERTIES IMPORTED_LOCATION ${PROJECT_BINARY_DIR}/libmscclpp_nccl.a)
|
||||
endif()
|
||||
|
||||
Reference in New Issue
Block a user