From 4a4c053a6ace3e758136b4b77e1a1cec3c782f2c Mon Sep 17 00:00:00 2001 From: Nirmal Unnikrishnan Date: Tue, 8 Mar 2022 17:32:02 +0000 Subject: [PATCH 1/2] File reorganization as per the new defined standard The header files will in /opt/rocm-xxx/include/rccl Libraries and cmake will be in /opt/rocm-xxx/lib folder. Added wrappers for header files using rocm-cmake functions for backward compatibility. [ROCm/rccl commit: 676a4737c116d17280ed4d7fcdf7e9b8c2885eb8] --- projects/rccl/CMakeLists.txt | 21 ++++++++++++++------ projects/rccl/cmake/Dependencies.cmake | 1 + projects/rccl/install.sh | 14 ++++++++++++- projects/rccl/test/common/CollectiveArgs.hpp | 2 +- projects/rccl/test/common/EnvVars.hpp | 2 +- projects/rccl/test/common/PtrUnion.hpp | 2 +- projects/rccl/test/common/TestBed.cpp | 2 +- projects/rccl/test/common/TestBedChild.hpp | 2 +- 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index daae126281..2d49acd11b 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -48,6 +48,8 @@ option(BUILD_TESTS "Build test programs" OFF) option(INSTALL_DEPENDENCIES "Force install dependencies" OFF) option(BUILD_ADDRESS_SANITIZER "Build with address sanitizer enabled" OFF) option(BUILD_ALLREDUCE_ONLY "Build AllReduce + sum + float kernel only" OFF) +#Set the header wrapper ON by default. +option(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY "Build with file/folder reorg with backward compatibility enabled" ON) # parse version from Makefile NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH must exist # NCCL_SUFFIX is optional NCCL_VERSION formatting is ((X) * 1000 + (Y) * 100 + @@ -271,19 +273,27 @@ rocm_set_soversion(rccl "1.0") rocm_install_targets(TARGETS rccl - PREFIX - rccl) + ) rocm_install(FILES ${PROJECT_BINARY_DIR}/rccl.h - DESTINATION rccl/${CMAKE_INSTALL_INCLUDEDIR}) + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rccl) rocm_export_targets(NAMESPACE roc:: - PREFIX - rccl TARGETS rccl DEPENDS hip) +if(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY) + #Create wrapper files + rocm_wrap_header_dir( "${PROJECT_BINARY_DIR}" + PATTERNS "*.h" + rccl.h + GUARDS SYMLINK WRAPPER + WRAPPER_LOCATIONS rccl ) + #install the wrapper header file to package + rocm_install( FILES "${PROJECT_BINARY_DIR}/rccl/include/rccl.h" + DESTINATION "./rccl/include/" ) +endif() set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip-rocclr (>= 3.5.0)") set(CPACK_DEBIAN_PACKAGE_DEPENDS "rocm-smi-lib (>= 4.0.0)") @@ -331,7 +341,6 @@ rocm_create_package( "RCCL Maintainer " LDCONFIG) -rocm_install_symlink_subdir(rccl) if(BUILD_TESTS) add_subdirectory(test) diff --git a/projects/rccl/cmake/Dependencies.cmake b/projects/rccl/cmake/Dependencies.cmake index ba1a5593b8..ae6ee02eeb 100644 --- a/projects/rccl/cmake/Dependencies.cmake +++ b/projects/rccl/cmake/Dependencies.cmake @@ -111,4 +111,5 @@ include(ROCMCreatePackage) include(ROCMInstallTargets) include(ROCMPackageConfigHelpers) include(ROCMInstallSymlinks) +include( ROCMHeaderWrapper ) include(ROCMCheckTargetIds OPTIONAL) diff --git a/projects/rccl/install.sh b/projects/rccl/install.sh index 6de243c401..ef011ee2b6 100755 --- a/projects/rccl/install.sh +++ b/projects/rccl/install.sh @@ -20,6 +20,7 @@ function display_help() echo " [--prefix] Specify custom directory to install RCCL to (default: /opt/rocm)." echo " [--address-sanitizer] Build with address sanitizer enabled" echo " [--build_allreduce_only] Build only AllReduce + sum + float kernel" + echo " [--rm-legacy-include-dir] Remove legacy include dir Packaging added for file/folder reorg backward compatibility" } # ################################################# @@ -38,6 +39,7 @@ clean_build=true install_dependencies=false build_static=false build_allreduce_only=false +build_freorg_bkwdcomp=true # ################################################# # Parameter parsing @@ -46,7 +48,7 @@ build_allreduce_only=false # check if we have a modern version of getopt that can handle whitespace and long parameters getopt -T if [[ $? -eq 4 ]]; then - GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,dependencies,package_build,tests_build,run_tests_quick,static,run_tests_all,hcc,hip-clang,no_clean,prefix:,address-sanitizer,build_allreduce_only --options hidptrs -- "$@") + GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,dependencies,package_build,tests_build,run_tests_quick,static,run_tests_all,hcc,hip-clang,no_clean,prefix:,address-sanitizer,build_allreduce_only,rm-legacy-include-dir --options hidptrs -- "$@") else echo "Need a new version of getopt" exit 1 @@ -102,6 +104,9 @@ while true; do --build_allreduce_only) build_allreduce_only=true shift ;; + --rm-legacy-include-dir) + build_freorg_bkwdcomp=false + shift ;; --prefix) install_prefix=${2} shift 2 ;; @@ -183,6 +188,13 @@ if [[ "${build_address_sanitizer}" == true ]]; then cmake_common_options="${cmake_common_options} -DBUILD_ADDRESS_SANITIZER=ON" fi +#Enable backward compatibility wrappers +if [[ "${build_freorg_bkwdcomp}" == true ]]; then + cmake_common_options="${cmake_common_options} -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=ON" +else + cmake_common_options="${cmake_common_options} -DBUILD_FILE_REORG_BACKWARD_COMPATIBILITY=OFF" +fi + compiler=hipcc if [[ "${build_hip_clang}" == false ]]; then compiler=hcc diff --git a/projects/rccl/test/common/CollectiveArgs.hpp b/projects/rccl/test/common/CollectiveArgs.hpp index 39fef1096c..e7d294bc81 100644 --- a/projects/rccl/test/common/CollectiveArgs.hpp +++ b/projects/rccl/test/common/CollectiveArgs.hpp @@ -7,7 +7,7 @@ b #pragma once #include "PtrUnion.hpp" #include "PrepDataFuncs.hpp" -#include "rccl.h" +#include "rccl/rccl.h" namespace RcclUnitTesting { diff --git a/projects/rccl/test/common/EnvVars.hpp b/projects/rccl/test/common/EnvVars.hpp index 87b9f07f07..5574df2321 100644 --- a/projects/rccl/test/common/EnvVars.hpp +++ b/projects/rccl/test/common/EnvVars.hpp @@ -7,7 +7,7 @@ #pragma once #include #include -#include "rccl.h" +#include "rccl/rccl.h" namespace RcclUnitTesting { diff --git a/projects/rccl/test/common/PtrUnion.hpp b/projects/rccl/test/common/PtrUnion.hpp index 36e5d6ec69..67323e93d5 100644 --- a/projects/rccl/test/common/PtrUnion.hpp +++ b/projects/rccl/test/common/PtrUnion.hpp @@ -1,6 +1,6 @@ #pragma once #include "ErrCode.hpp" -#include "rccl.h" +#include "rccl/rccl.h" #include "rccl_bfloat16.h" namespace RcclUnitTesting diff --git a/projects/rccl/test/common/TestBed.cpp b/projects/rccl/test/common/TestBed.cpp index 6f0c2aba8f..edbab4faec 100644 --- a/projects/rccl/test/common/TestBed.cpp +++ b/projects/rccl/test/common/TestBed.cpp @@ -5,7 +5,7 @@ ************************************************************************/ #include #include "TestBed.hpp" -#include +#include #define PIPE_WRITE(childId, val) \ ASSERT_EQ(write(childList[childId]->parentWriteFd, &val, sizeof(val)), sizeof(val)) diff --git a/projects/rccl/test/common/TestBedChild.hpp b/projects/rccl/test/common/TestBedChild.hpp index e6ccd42444..43b511317e 100644 --- a/projects/rccl/test/common/TestBedChild.hpp +++ b/projects/rccl/test/common/TestBedChild.hpp @@ -9,7 +9,7 @@ #include #include #include "CollectiveArgs.hpp" -#include "rccl.h" +#include "rccl/rccl.h" #define MAX_RANKS 32 namespace RcclUnitTesting From e740088560fb8ec590066c373e9a006566bfe196 Mon Sep 17 00:00:00 2001 From: Nirmal Unnikrishnan Date: Thu, 10 Mar 2022 01:28:41 +0000 Subject: [PATCH 2/2] File reorganization with backward compatibility Updated the header file location and export path [ROCm/rccl commit: 115461cc04310503268fe5a39b44d9475a188717] --- projects/rccl/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index 2d49acd11b..842caef63c 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -112,10 +112,11 @@ if(BUILD_ADDRESS_SANITIZER) add_link_options(-fuse-ld=lld) endif() -configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/rccl.h) -configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/nccl.h) +configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/include/rccl/rccl.h) +configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/include/rccl/nccl.h) -include_directories(${PROJECT_BINARY_DIR}) # for generated rccl.h header +include_directories(${PROJECT_BINARY_DIR}/include) # for generated rccl.h header +include_directories(${PROJECT_BINARY_DIR}/include/rccl) # for generated rccl.h header include_directories(src) include_directories(src/include) include_directories(src/collectives) @@ -274,7 +275,7 @@ rocm_set_soversion(rccl "1.0") rocm_install_targets(TARGETS rccl ) -rocm_install(FILES ${PROJECT_BINARY_DIR}/rccl.h +rocm_install(FILES ${PROJECT_BINARY_DIR}/include/rccl/rccl.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rccl) rocm_export_targets(NAMESPACE @@ -285,7 +286,7 @@ rocm_export_targets(NAMESPACE hip) if(BUILD_FILE_REORG_BACKWARD_COMPATIBILITY) #Create wrapper files - rocm_wrap_header_dir( "${PROJECT_BINARY_DIR}" + rocm_wrap_header_dir( "${PROJECT_BINARY_DIR}/include/rccl" PATTERNS "*.h" rccl.h GUARDS SYMLINK WRAPPER