From c96ff57ac79522e813b0661bb5831d71012f5366 Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Tue, 14 Feb 2023 22:31:54 +0000 Subject: [PATCH 1/3] auto-detect and enable MPI --- CMakeLists.txt | 50 ++++++++++++++++++++++++++++++++++++++++++-------- README.md | 16 ++++++++++++++++ 2 files changed, 58 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d950565e2f..f3138fa2fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,33 @@ # ######################################################################## #Adding pthread flag for linking set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") +macro(check_mpi mpi_compiler mpi_lib_a mpi_lib_so) + find_program(MPI_MPICXX ${mpi_compiler}) + if (MPI_MPICXX) + message ("-- ${mpi_compiler} found @ ${MPI_MPICXX}") + if (${CMAKE_VERSION} VERSION_LESS "3.20.0") + get_filename_component(mpi.tmpdir ${MPI_MPICXX} DIRECTORY) + get_filename_component(mpi_base_dir ${mpi.tmpdir} DIRECTORY) + else() + cmake_path(GET MPI_MPICXX PARENT_PATH mpi.tmpdir) + cmake_path(GET mpi.tmpdir PARENT_PATH mpi_base_dir) + endif() + find_file(MPI_H mpi.h PATHS ${mpi_base_dir} PATH_SUFFIXES include include/x86_64-linux-gnu ${ARGN} {REQUIRED) + if (${CMAKE_VERSION} VERSION_LESS "3.20.0") + get_filename_component(mpi_inc_dir ${MPI_H} DIRECTORY) + else() + cmake_path(GET MPI_H PARENT_PATH mpi_inc_dir) + endif() + message ("-- mpi.h is in ${mpi_inc_dir}") + find_file(MPI_LIB NAMES ${mpi_lib_so} ${mpi_lib_a} PATHS ${mpi_base_dir} PATH_SUFFIXES lib lib64 lib/x86_64-linux-gnu REQIRED) + message ("-- libmpi is ${MPI_LIB}") + add_definitions(-DMPI_SUPPORT) + include_directories(${mpi_inc_dir}) + link_libraries(${MPI_LIB}) + else() + message ("-- ${mpi_compiler} not found") + endif() +endmacro() cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR) @@ -32,8 +59,7 @@ include(ROCMCheckTargetIds) include(ROCMClients) # Build variables -option(USE_MPI "Build RCCL-tests with MPI support. Requires the MPI path to be set.") -set(MPI_PATH "" CACHE PATH "Path to MPI installation") +option(NO_MPI "Build RCCL-tests without MPI support.") ## Get default GPU targets using rocm_check_target_ids rocm_check_target_ids( DEFAULT_AMDGPU_TARGETS @@ -41,13 +67,21 @@ rocm_check_target_ids( ) set(AMDGPU_TARGETS "${DEFAULT_AMDGPU_TARGETS}" CACHE STRING "List of specific machine types for these tests to target.") -# Find the MPI package if we're using MPI -if (USE_MPI) - if(NOT MPI_PATH STREQUAL "") - set(MPI_HOME "${MPI_PATH}") +if (NOT NO_MPI) + # Check for MPICH first + check_mpi(mpicxx.mpich libmpich.a libmpich.so include/x86_64-linux-gnu/mpich) + + # Check for MPI in general. If we find mpicxx, we don't know whether its + # MPICH or another MPI implementation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpi.a libmpi.so) endif() - find_package(MPI REQUIRED MODULE) - add_definitions(-DOMPI_SKIP_MPICXX -DMPI_SUPPORT) + + if (NOT MPI_MPICXX) + message ("-- no MPI library found") + endif() +else() + message ("-- MPI support explicitely disabled") endif() set(ROCM_USE_DEV_COMPONENT OFF) # This repo doesn't have a dev component diff --git a/README.md b/README.md index c2847232e6..0a88c5d384 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,22 @@ RCCL tests rely on MPI to work on multiple processes, hence multiple nodes. If y $ make MPI=1 MPI_HOME=/path/to/mpi HIP_HOME=/path/to/hip RCCL_HOME=/path/to/rccl ``` +RCCL tests can also be built using cmake. A typical sequence will be: + +```shell +$ mkdir build +$ cd build +$ CXX=/opt/rocm/bin/hipcc cmake -DCMAKE_PREFIX_PATH=/path/to/rccl .. +$ make +``` + +When using the cmake build procedure, please make sure that RCCL has also been built using cmake (i.e. not using the install.sh script), since cmake will check +for cmake target and config files that are created during the RCCL build. + +Using the cmake method also has the advantage that the build is automatically checking for MPI installations, i.e. it is not necessary to explicitley request +MPI builds. A user can explicitely disable MPI builds by adding the -DNO_MPI=1 flag to the cmake command line. + + ## Usage RCCL tests can run on multiple processes, multiple threads, and multiple HIP devices per thread. The number of process is managed by MPI and is therefore not passed to the tests as argument. The total number of ranks (=HIP devices) will be equal to (number of processes)\*(number of threads)\*(number of GPUs per thread). From 8fc00ec32ef9185f277c26d03a4824c75f2072cb Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Thu, 2 Mar 2023 18:22:03 +0000 Subject: [PATCH 2/3] revamp cmake MPI detection we honor user requested MPI installations using MPI_PATH first, and check afterwards for MPICH and Open MPI in the default Ubuntu and RHEL installation directories. --- CMakeLists.txt | 69 +++++++++++++++++++++++++++++++------------------- README.md | 5 ++-- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f3138fa2fa..3cdbad6f75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,29 +3,23 @@ # ######################################################################## #Adding pthread flag for linking set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") -macro(check_mpi mpi_compiler mpi_lib_a mpi_lib_so) - find_program(MPI_MPICXX ${mpi_compiler}) +macro(check_mpi mpi_compiler mpi_lib_a mpi_lib_so mpi_bin_dir mpi_base_lib_dir mpi_inc_dir) + find_program(MPI_MPICXX ${mpi_compiler} PATHS ${mpi_bin_dir} NO_DEFAULT_PATH) if (MPI_MPICXX) message ("-- ${mpi_compiler} found @ ${MPI_MPICXX}") - if (${CMAKE_VERSION} VERSION_LESS "3.20.0") - get_filename_component(mpi.tmpdir ${MPI_MPICXX} DIRECTORY) - get_filename_component(mpi_base_dir ${mpi.tmpdir} DIRECTORY) - else() - cmake_path(GET MPI_MPICXX PARENT_PATH mpi.tmpdir) - cmake_path(GET mpi.tmpdir PARENT_PATH mpi_base_dir) - endif() - find_file(MPI_H mpi.h PATHS ${mpi_base_dir} PATH_SUFFIXES include include/x86_64-linux-gnu ${ARGN} {REQUIRED) - if (${CMAKE_VERSION} VERSION_LESS "3.20.0") - get_filename_component(mpi_inc_dir ${MPI_H} DIRECTORY) - else() - cmake_path(GET MPI_H PARENT_PATH mpi_inc_dir) - endif() - message ("-- mpi.h is in ${mpi_inc_dir}") - find_file(MPI_LIB NAMES ${mpi_lib_so} ${mpi_lib_a} PATHS ${mpi_base_dir} PATH_SUFFIXES lib lib64 lib/x86_64-linux-gnu REQIRED) + find_file(MPI_H mpi.h PATHS ${mpi_inc_dir} NO_DEFAULT_PATH) + message ("-- mpi.h is in ${MPI_H}") + find_file(MPI_LIB NAMES ${mpi_lib_so} ${mpi_lib_a} PATHS ${mpi_base_lib_dir} PATH_SUFFIXES lib lib64 lib/x86_64-linux-gnu NO_DEFAULT_PATH) message ("-- libmpi is ${MPI_LIB}") - add_definitions(-DMPI_SUPPORT) - include_directories(${mpi_inc_dir}) - link_libraries(${MPI_LIB}) + if (NOT MPI_H OR NOT MPI_LIB) + set (MPI_MPICXX "MPI_MPICXX-NOTFOUND") + set (MPI_H "MPI_H-NOTFOUND") + set (MPI_LIB "MPI_LIB-NOTFOUND") + else() + add_definitions(-DMPI_SUPPORT) + include_directories(${mpi_inc_dir}) + link_libraries(${MPI_LIB}) + endif() else() message ("-- ${mpi_compiler} not found") endif() @@ -60,6 +54,7 @@ include(ROCMClients) # Build variables option(NO_MPI "Build RCCL-tests without MPI support.") +option(MPI_PATH "Use MPI in the specified directory.") ## Get default GPU targets using rocm_check_target_ids rocm_check_target_ids( DEFAULT_AMDGPU_TARGETS @@ -68,13 +63,35 @@ rocm_check_target_ids( set(AMDGPU_TARGETS "${DEFAULT_AMDGPU_TARGETS}" CACHE STRING "List of specific machine types for these tests to target.") if (NOT NO_MPI) - # Check for MPICH first - check_mpi(mpicxx.mpich libmpich.a libmpich.so include/x86_64-linux-gnu/mpich) + # CHECK for MPI Path first. User requested this directory explicitely + if (MPI_PATH) + set(mpi_spec_bin_dir "${MPI_PATH}/bin") + set(mpi_spec_inc_dir "${MPI_PATH}/include") + check_mpi(mpicxx libmpi.a libmpi.so ${mpi_spec_bin_dir} ${MPI_PATH} ${mpi_spec_inc_dir}) + if (NOT MPI_MPICXX) + # Since the user explicitely requested this directory, abort if something went wrong. + MESSAGE(FATAL_ERROR "Could not find MPI in ${MPI_PATH}") + endif() + endif() - # Check for MPI in general. If we find mpicxx, we don't know whether its - # MPICH or another MPI implementation + # Check for MPICH Ubuntu installation if (NOT MPI_MPICXX) - check_mpi(mpicxx libmpi.a libmpi.so) + check_mpi(mpicxx.mpich libmpich.a libmpich.so /usr/bin /usr /usr/include/x86_64-linux-gnu/mpich) + endif() + + # Check for Open MPI Ubuntu installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx.openmpi libmpi.a libmpi.so /usr/bin /usr/lib/x86_64-linux-gnu/openmpi /usr/lib/x86_64-linux-gnu/openmpi/include) + endif() + + # Check for MPICH RHEL installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpich.a libmpich.so /usr/lib64/mpich/bin /usr/lib64/mpich /usr/include/mpich-x86_64) + endif() + + # Check for Open MPI RHEL installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/openmpi/bin /usr/lib64/openmpi /usr/include/openmpi-x64_64) endif() if (NOT MPI_MPICXX) @@ -91,7 +108,7 @@ add_subdirectory(src) # Create ROCm standard packages rocm_create_package( - NAME rccl-separate-tests + NAME rccl-tests DESCRIPTION "Tests for the ROCm Communication Collectives Library" MAINTAINER "RCCL Maintainer " ) diff --git a/README.md b/README.md index 0a88c5d384..74f15515b4 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,9 @@ $ make When using the cmake build procedure, please make sure that RCCL has also been built using cmake (i.e. not using the install.sh script), since cmake will check for cmake target and config files that are created during the RCCL build. -Using the cmake method also has the advantage that the build is automatically checking for MPI installations, i.e. it is not necessary to explicitley request -MPI builds. A user can explicitely disable MPI builds by adding the -DNO_MPI=1 flag to the cmake command line. +Using the cmake method also has the advantage that the build is automatically checking for MPI installations, i.e. it is not necessary to explicitly request +MPI builds. A user can request to use a particular MPI library by using the MPI_PATH variable. MPI support can be explicitely disabled by adding the -DNO_MPI=1 +flag to the cmake command line. ## Usage From efdd4ad40bcc59cd2fecae4cd34eed3644db917c Mon Sep 17 00:00:00 2001 From: Edgar Gabriel Date: Mon, 24 Jul 2023 12:02:44 -0700 Subject: [PATCH 3/3] search SLES install paths for MPI --- CMakeLists.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cdbad6f75..296c01c28b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,26 @@ if (NOT NO_MPI) check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/openmpi/bin /usr/lib64/openmpi /usr/include/openmpi-x64_64) endif() + # Check for MPICH SLES installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpich.a libmpich.so /usr/lib64/mpi/gcc/mpich/bin /usr/lib64/mpi/gcc/mpich /usr/lib64/mpi/gcc/mpich/include) + endif() + + # Check for Open MPI v4 SLES installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/mpi/gcc/openmpi4/bin /usr/lib64/mpi/gcc/openmpi4 /usr/lib64/mpi/gcc/openmpi4/include) + endif() + + # Check for Open MPI v3 SLES installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/mpi/gcc/openmpi3/bin /usr/lib64/mpi/gcc/openmpi3 /usr/lib64/mpi/gcc/openmpi3/include) + endif() + + # Check for Open MPI v2 SLES installation + if (NOT MPI_MPICXX) + check_mpi(mpicxx libmpi.a libmpi.so /usr/lib64/mpi/gcc/openmpi2/bin /usr/lib64/mpi/gcc/openmpi2 /usr/lib64/mpi/gcc/openmpi2/include) + endif() + if (NOT MPI_MPICXX) message ("-- no MPI library found") endif()