diff --git a/projects/rccl-tests/.jenkins/common.groovy b/projects/rccl-tests/.jenkins/common.groovy index 70dbbd7a3c..7426d35d75 100644 --- a/projects/rccl-tests/.jenkins/common.groovy +++ b/projects/rccl-tests/.jenkins/common.groovy @@ -13,7 +13,10 @@ def runCompileCommand(platform, project, jobName) ${getRCCL} ${auxiliary.exitIfNotSuccess()} cd ${project.paths.project_build_prefix} - ${project.paths.build_command} + cmake \ + -DCMAKE_CXX_COMPILER=/opt/rocm/hip/bin/hipcc \ + -S . -B build + make -C build -j\$(nproc) ${auxiliary.exitIfNotSuccess()} """ diff --git a/projects/rccl-tests/CMakeLists.txt b/projects/rccl-tests/CMakeLists.txt new file mode 100644 index 0000000000..539a1eae2b --- /dev/null +++ b/projects/rccl-tests/CMakeLists.txt @@ -0,0 +1,61 @@ +# ######################################################################## +# Copyright 2022 Advanced Micro Devices, Inc. +# ######################################################################## + +cmake_minimum_required(VERSION 3.16.3 FATAL_ERROR) + +project(RCCL-tests VERSION 2.12.10 LANGUAGES CXX) + +# Get ROCm path from environment if available +if (DEFINED ENV{ROCM_PATH}) + set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to ROCm installation") +else() + set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to ROCm installation") +endif() + +# Set CMake/CPack variables +list( APPEND CMAKE_PREFIX_PATH ${ROCM_PATH} ${ROCM_PATH}/llvm) +set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}/install" CACHE PATH "Prefix install path") +set(CPACK_PACKAGING_INSTALL_PREFIX "${ROCM_PATH}" CACHE PATH "Path to install to when packaged.") +set(CMAKE_CXX_STANDARD 14) + +# Get additional packages required +find_package(ROCM 0.7.3 CONFIG REQUIRED PATHS "${ROCM_PATH}") +find_package(RCCL HINTS CONFIG REQUIRED PATHS "${ROCM_PATH}") + +include(ROCMSetupVersion) +include(ROCMCreatePackage) +include(ROCMInstallTargets) +include(ROCMCheckTargetIds) +include(ROCMClients) + +# Build variables +option(USE_MPI "Build RCCL-tests with MPI support. Requires the MPI path to be set.") +set(MPI_PATH "" CACHE PATH "Path to MPI installation") +## Get default GPU targets using rocm_check_target_ids +rocm_check_target_ids( + DEFAULT_AMDGPU_TARGETS + TARGETS "gfx803;gfx900:xnack-;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack-;gfx90a:xnack+;gfx1030" +) +set(AMDGPU_TARGETS "${DEFAULT_AMDGPU_TARGETS}" CACHE STRING "List of specific machine types for these tests to target.") + +# Find the MPI package if we're using MPI +if (USE_MPI) + if(NOT MPI_PATH STREQUAL "") + set(MPI_HOME "${MPI_PATH}") + endif() + find_package(MPI REQUIRED MODULE) + add_definitions(-DOMPI_SKIP_MPICXX -DMPI_SUPPORT) +endif() + +set(ROCM_USE_DEV_COMPONENT OFF) # This repo doesn't have a dev component + +# Add all of the tests +add_subdirectory(src) + +# Create ROCm standard packages +rocm_create_package( + NAME rccl-separate-tests + DESCRIPTION "Tests for the ROCm Communication Collectives Library" + MAINTAINER "RCCL Maintainer " +) diff --git a/projects/rccl-tests/src/CMakeLists.txt b/projects/rccl-tests/src/CMakeLists.txt new file mode 100644 index 0000000000..b5a40aefc1 --- /dev/null +++ b/projects/rccl-tests/src/CMakeLists.txt @@ -0,0 +1,71 @@ +# ######################################################################## +# Copyright 2022 Advanced Micro Devices, Inc. +# ######################################################################## + +# Compile common object library +set_property(SOURCE common.cu PROPERTY LANGUAGE CXX) +add_library(rccl_common OBJECT common.cu) +if(USE_MPI) + target_link_libraries(rccl_common roc::rccl MPI::MPI_CXX) +else() + target_link_libraries(rccl_common roc::rccl) +endif() + +function(add_relative_test test_name test_target) + get_target_property(EXE_PATH ${test_target} RUNTIME_OUTPUT_DIRECTORY) + if(EXE_PATH STREQUAL "EXE_PATH-NOTFOUND") + set(EXE_PATH ".") + endif() + get_filename_component(EXE_PATH "${EXE_PATH}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}") + get_target_property(EXE_NAME ${test_target} RUNTIME_OUTPUT_NAME) + if(EXE_NAME STREQUAL "EXE_NAME-NOTFOUND") + get_target_property(EXE_NAME ${test_target} OUTPUT_NAME) + if(EXE_NAME STREQUAL "EXE_NAME-NOTFOUND") + set(EXE_NAME "${test_target}") + endif() + endif() + file(RELATIVE_PATH rel_path "${CMAKE_CURRENT_BINARY_DIR}" "${EXE_PATH}/${EXE_NAME}") + add_test(NAME "${test_name}" COMMAND "./${rel_path}") +endfunction() + +function(add_rccl_test TEST) + set(TEST_SOURCE "${TEST}.cu") + set_property(SOURCE ${TEST_SOURCE} PROPERTY LANGUAGE CXX) + set(TEST_TARGET "${TEST}_perf") + add_executable(${TEST_TARGET} ${TEST_SOURCE}) + target_link_libraries( + ${TEST_TARGET} + PRIVATE + rccl_common + ) + if (NOT WIN32) + foreach(amdgpu_target ${AMDGPU_TARGETS}) + target_link_libraries(${TEST_TARGET} PRIVATE --amdgpu-target=${amdgpu_target}) + endforeach() + endif() + set_target_properties( + ${TEST_TARGET} + PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + # LINKER_LANGUAGE CXX + ) + add_relative_test(${TEST} ${TEST_TARGET}) + rocm_install(TARGETS ${TEST_TARGET}) + # TODO: copy/install DLLs on Windows + set_target_properties( + ${TEST_TARGET} PROPERTIES + INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${ROCM_PATH}/lib" + ) +endfunction() + +add_rccl_test(all_gather) +add_rccl_test(all_reduce) +add_rccl_test(alltoall) +add_rccl_test(alltoallv) +add_rccl_test(broadcast) +add_rccl_test(gather) +add_rccl_test(hypercube) +add_rccl_test(reduce_scatter) +add_rccl_test(reduce) +add_rccl_test(scatter) +add_rccl_test(sendrecv)