From 55a4b22ad7e84430c60a933c5fe916d1f1e3e10a Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Thu, 16 May 2019 16:16:18 +0000 Subject: [PATCH 01/47] Updating RCCL based on NCCL 2.3.7 - Contains modifications to support AMD hardware - Adds unit tests --- CMakeLists.txt | 180 +++++++++++ LICENSE.txt | 1 + NOTICES.txt | 66 +++++ README.md | 93 +++--- hipify.sh | 112 +++++++ install.sh | 69 +++++ jenkinsfile | 89 ++++++ src/collectives/all_gather.cu | 9 +- src/collectives/all_reduce.cu | 9 +- src/collectives/broadcast.cu | 13 +- src/collectives/collectives.h | 3 +- src/collectives/device/all_gather.h | 23 +- src/collectives/device/all_gather_0.cpp | 8 + src/collectives/device/all_reduce.h | 23 +- src/collectives/device/all_reduce_0.cpp | 8 + src/collectives/device/all_reduce_1.cpp | 8 + src/collectives/device/all_reduce_2.cpp | 8 + src/collectives/device/all_reduce_3.cpp | 8 + src/collectives/device/broadcast.h | 23 +- src/collectives/device/broadcast_0.cpp | 8 + src/collectives/device/common.h | 42 ++- src/collectives/device/common_kernel.h | 32 +- src/collectives/device/functions.cu | 11 +- src/collectives/device/ll_kernel.h | 54 +++- src/collectives/device/primitives.h | 89 +++--- src/collectives/device/reduce.h | 13 +- src/collectives/device/reduce_0.cpp | 8 + src/collectives/device/reduce_1.cpp | 8 + src/collectives/device/reduce_2.cpp | 8 + src/collectives/device/reduce_3.cpp | 8 + src/collectives/device/reduce_kernel.h | 121 ++++++++ src/collectives/device/reduce_scatter.cu | 1 + src/collectives/device/reduce_scatter.h | 13 +- src/collectives/device/reduce_scatter_0.cpp | 8 + src/collectives/device/reduce_scatter_1.cpp | 8 + src/collectives/device/reduce_scatter_2.cpp | 8 + src/collectives/device/reduce_scatter_3.cpp | 8 + src/collectives/reduce.cu | 8 +- src/collectives/reduce_scatter.cu | 8 +- src/include/common_coll.h | 21 +- src/include/core.h | 65 ++-- src/include/debug.h | 1 + src/include/enqueue.h | 5 +- src/include/group.h | 3 +- src/include/nvlink.h | 1 + src/include/nvlink_stub.h | 20 ++ src/include/rings.h | 5 + src/include/shm.h | 9 +- src/include/topo.h | 32 +- src/init.cu | 114 +++++-- src/misc/enqueue.cu | 86 +++--- src/misc/group.cu | 19 +- src/misc/nvmlwrap.cu | 1 + src/misc/nvmlwrap_stub.cu | 53 ++++ src/misc/rings.cu | 5 + src/misc/utils.cu | 2 +- src/nccl.h.in | 39 +-- src/ring.cu | 13 +- src/transport.cu | 35 +-- src/transport/net.cu | 58 ++-- src/transport/net_ib.cu | 15 +- src/transport/net_socket.cu | 8 +- src/transport/p2p.cu | 96 ++++-- src/transport/shm.cu | 5 +- test/CMakeLists.txt | 63 ++++ test/CMakeLists.txt.in | 15 + test/CorrectnessTest.hpp | 311 ++++++++++++++++++++ test/test_AllGather.cpp | 63 ++++ test/test_AllGather.hpp | 32 ++ test/test_AllReduce.cpp | 59 ++++ test/test_AllReduce.hpp | 76 +++++ test/test_Broadcast.cpp | 67 +++++ test/test_Broadcast.hpp | 26 ++ test/test_GroupCalls.cpp | 125 ++++++++ test/test_GroupCalls.hpp | 17 ++ test/test_Reduce.cpp | 67 +++++ test/test_Reduce.hpp | 80 +++++ test/test_ReduceScatter.cpp | 65 ++++ test/test_ReduceScatter.hpp | 83 ++++++ 79 files changed, 2678 insertions(+), 399 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 NOTICES.txt create mode 100755 hipify.sh create mode 100755 install.sh create mode 100644 jenkinsfile create mode 100644 src/collectives/device/all_gather_0.cpp create mode 100644 src/collectives/device/all_reduce_0.cpp create mode 100644 src/collectives/device/all_reduce_1.cpp create mode 100644 src/collectives/device/all_reduce_2.cpp create mode 100644 src/collectives/device/all_reduce_3.cpp create mode 100644 src/collectives/device/broadcast_0.cpp create mode 100644 src/collectives/device/reduce_0.cpp create mode 100644 src/collectives/device/reduce_1.cpp create mode 100644 src/collectives/device/reduce_2.cpp create mode 100644 src/collectives/device/reduce_3.cpp create mode 100644 src/collectives/device/reduce_scatter_0.cpp create mode 100644 src/collectives/device/reduce_scatter_1.cpp create mode 100644 src/collectives/device/reduce_scatter_2.cpp create mode 100644 src/collectives/device/reduce_scatter_3.cpp create mode 100644 src/include/nvlink_stub.h create mode 100644 src/misc/nvmlwrap_stub.cu create mode 100644 test/CMakeLists.txt create mode 100644 test/CMakeLists.txt.in create mode 100644 test/CorrectnessTest.hpp create mode 100644 test/test_AllGather.cpp create mode 100644 test/test_AllGather.hpp create mode 100644 test/test_AllReduce.cpp create mode 100644 test/test_AllReduce.hpp create mode 100644 test/test_Broadcast.cpp create mode 100644 test/test_Broadcast.hpp create mode 100644 test/test_GroupCalls.cpp create mode 100644 test/test_GroupCalls.hpp create mode 100644 test/test_Reduce.cpp create mode 100644 test/test_Reduce.hpp create mode 100644 test/test_ReduceScatter.cpp create mode 100644 test/test_ReduceScatter.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..bc0b59fb5d --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,180 @@ +# Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + +cmake_minimum_required(VERSION 2.8.12) + +set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "") + +project(rccl CXX) + +find_package(ROCM + REQUIRED + PATHS + /opt/rocm) + +include(ROCMInstallTargets) +include(ROCMPackageConfigHelpers) +include(ROCMSetupVersion) +include(ROCMInstallSymlinks) +include(ROCMCreatePackage) + +option(BUILD_TESTS "Build test programs" ON) + +# In order to support function calls within the kernel, we must use hcc as the +# compiler +set(CMAKE_CXX_COMPILER "/opt/rocm/bin/hcc") + +# parse version from Makefile NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH must exist +# NCCL_SUFFIX is optional NCCL_VERSION formatting is ((X) * 1000 + (Y) * 100 + +# (Z)) so we must first detect one or two digits first +file(READ makefiles/version.mk version_mk_text) +if("${version_mk_text}" MATCHES "NCCL_MAJOR *:= *([0-9]*)") + set(NCCL_MAJOR ${CMAKE_MATCH_1}) +else() + message(FATAL_ERROR "Failed to parse NCCL_MAJOR") +endif() +if("${version_mk_text}" MATCHES "NCCL_MINOR *:= *([0-9]*)") + set(NCCL_MINOR ${CMAKE_MATCH_1}) +else() + message(FATAL_ERROR "Failed to parse NCCL_MINOR") +endif() +if("${version_mk_text}" MATCHES "NCCL_PATCH *:= *([0-9]*)") + set(NCCL_PATCH ${CMAKE_MATCH_1}) +else() + message(FATAL_ERROR "Failed to parse NCCL_PATCH") +endif() +if("${version_mk_text}" MATCHES "NCCL_SUFFIX *:= *([0-9]*)") + set(NCCL_SUFFIX ${CMAKE_MATCH_1}) +else() + set(NCCL_SUFFIX) +endif() +if("${version_mk_text}" MATCHES "PKG_REVISION *:= *([0-9]*)") + set(PKG_REVISION ${CMAKE_MATCH_1}) +else() + message(FATAL_ERROR "Failed to parse PKG_REVISION") +endif() +if("${NCCL_PATCH}" MATCHES "[0-9][0-9]") + set(NCCL_VERSION "${NCCL_MAJOR}${NCCL_MINOR}${NCCL_PATCH}") +else() + set(NCCL_VERSION "${NCCL_MAJOR}${NCCL_MINOR}0${NCCL_PATCH}") +endif() + +rocm_setup_version(VERSION + "${NCCL_MAJOR}.${NCCL_MINOR}.${NCCL_PATCH}-${PKG_REVISION}") + +list(APPEND CMAKE_PREFIX_PATH + /opt/rocm + /opt/rocm/hip + /opt/rocm/hcc) + +find_package(hip REQUIRED) + +link_libraries(-amdgpu-target=gfx803 + -amdgpu-target=gfx900 + -amdgpu-target=gfx906 + -hc-function-calls) + +option(BUILD_SHARED_LIBS "Build as a shared library" ON) + +configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/rccl.h) +configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/nccl.h) + +include_directories(${PROJECT_BINARY_DIR}) # for generated rccl.h header +include_directories(src) +include_directories(src/include) +include_directories(src/collectives) +include_directories(src/collectives/device) + +set(CU_SOURCES + src/bootstrap.cu + src/collectives/all_gather.cu + src/collectives/all_reduce.cu + src/collectives/broadcast.cu + src/collectives/reduce.cu + src/collectives/reduce_scatter.cu + src/collectives/device/functions.cu + src/init.cu + src/misc/enqueue.cu + src/misc/group.cu + src/misc/ibvwrap.cu + src/misc/nvmlwrap_stub.cu + src/misc/rings.cu + src/misc/utils.cu + src/ring.cu + src/transport.cu + src/transport/net.cu + src/transport/net_ib.cu + src/transport/net_socket.cu + src/transport/p2p.cu + src/transport/shm.cu) + +set(CPP_SOURCES) +foreach(filename ${CU_SOURCES}) + string(REPLACE ".cu" + ".cpp" + cpp_filename + ${filename}) + configure_file(${filename} ${cpp_filename} COPYONLY) + list(APPEND CPP_SOURCES ${cpp_filename}) +endforeach(filename) + +list(APPEND CPP_SOURCES src/collectives/device/all_gather_0.cpp) +list(APPEND CPP_SOURCES src/collectives/device/all_reduce_0.cpp) +list(APPEND CPP_SOURCES src/collectives/device/all_reduce_1.cpp) +list(APPEND CPP_SOURCES src/collectives/device/all_reduce_2.cpp) +list(APPEND CPP_SOURCES src/collectives/device/all_reduce_3.cpp) +list(APPEND CPP_SOURCES src/collectives/device/broadcast_0.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_0.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_1.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_2.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_3.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_scatter_0.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_scatter_1.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_scatter_2.cpp) +list(APPEND CPP_SOURCES src/collectives/device/reduce_scatter_3.cpp) + +add_library(rccl ${CPP_SOURCES}) + +if(TRACE) + add_definitions(-DENABLE_TRACE) +endif() + +if(TARGET hip::device) + target_link_libraries(rccl PRIVATE hip::device) + target_link_libraries(rccl INTERFACE hip::host) +else() + target_link_libraries(rccl PUBLIC hip::hip_hcc ${hcc_LIBRARIES} numa) +endif() + +rocm_install_targets(TARGETS + rccl + PREFIX + rccl) +install(FILES ${PROJECT_BINARY_DIR}/rccl.h + DESTINATION rccl/${CMAKE_INSTALL_INCLUDEDIR}) + +rocm_export_targets(NAMESPACE + roc:: + PREFIX + rccl + TARGETS + rccl + DEPENDS + hip) + +set(CPACK_DEBIAN_PACKAGE_DEPENDS "hip_hcc") +set(CPACK_RPM_PACKAGE_REQUIRES "hip_hcc") + +rocm_create_package( + NAME + rccl + DESCRIPTION + "Optimized primitives for collective multi-GPU communication" + MAINTAINER + "Jeff Daily " + LDCONFIG) + +rocm_install_symlink_subdir(rccl) + +if(BUILD_TESTS) + add_subdirectory(test) +endif() diff --git a/LICENSE.txt b/LICENSE.txt index 3593a7aa69..6b9c6a3138 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,5 +1,6 @@ Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions diff --git a/NOTICES.txt b/NOTICES.txt new file mode 100644 index 0000000000..1b9bcc8eec --- /dev/null +++ b/NOTICES.txt @@ -0,0 +1,66 @@ +Notices and Licenses file +_______________________________________________________________ + +Dependencies on nvidia-nccl v2.3.7-1 (BSD3) +Copyright (c) 2015-2018, NVIDIA CORPORATION. +Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of NVIDIA CORPORATION, Lawrence Berkeley National + Laboratory, the U.S. Department of Energy, nor the names of their + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The U.S. Department of Energy funded the development of this software +under subcontract 7078610 with Lawrence Berkeley National Laboratory. + + +nvidia-nccl v2.3.7-1 (BSD2) +Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of NVIDIA CORPORATION, Lawrence Berkeley National + Laboratory, the U.S. Department of Energy, nor the names of their + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The U.S. Department of Energy funded the development of this software +under subcontract 7078610 with Lawrence Berkeley National Laboratory. \ No newline at end of file diff --git a/README.md b/README.md index fa5145323b..fc3b738227 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ -# NCCL +# RCCL -Optimized primitives for collective multi-GPU communication. +ROCm Communication Collectives Library ## Introduction -NCCL (pronounced "Nickel") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on platforms using PCIe, NVLink, NVswitch, as well as networking using InfiniBand Verbs or TCP/IP sockets. NCCL supports an arbitrary number of GPUs installed in a single node or across multiple nodes, and can be used in either single- or multi-process (e.g., MPI) applications. - -For more information on NCCL usage, please refer to the [NCCL documentation](https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/index.html). +RCCL (pronounced "Rickle") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on platforms using PCIe, xGMI as well as networking using InfiniBand Verbs or TCP/IP sockets. RCCL supports an arbitrary number of GPUs installed in a single node or across multiple nodes, and can be used in either single- or multi-process (e.g., MPI) applications. ## What's inside @@ -22,71 +20,62 @@ These operations are implemented using ring algorithms and have been optimized f ## Requirements -NCCL requires at least CUDA 7.0 and Kepler or newer GPUs. For PCIe based platforms, best performance is achieved when all GPUs are located on a common PCIe root complex, but multi-socket configurations are also supported. +1. ROCm supported GPUs +2. ROCm stack installed on the system (HIP runtime & HCC) -## Build +## Quickstart RCCL Build -Note: the official and tested builds of NCCL can be downloaded from: https://developer.nvidia.com/nccl. You can skip the following build steps if you choose to use the official builds. +RCCL directly depends on HIP runtime & HCC C++ compiler which are part of the ROCm software stack. +In addition, HC Direct Function call support needs to be present on your machine. There are binaries for hcc and HIP that need to be installed to get HC Direct Function call support. These binaries are currently packaged with roc-master, and will be included in ROCm 2.4. -To build the library : +The root of this repository has a helper script 'install.sh' to build and install RCCL on Ubuntu with a single command. It does not take a lot of options and hard-codes configuration that can be specified through invoking cmake directly, but it's a great way to get started quickly and can serve as an example of how to build/install. + +* `./install.sh` -- builds library including unit tests +* `./install.sh -h` -- shows help +* `./install.sh -t` -- builds library including unit tests, and also runs unit tests + +## Manual build +#### To build the library : ```shell -$ cd nccl -$ make -j src.build +$ git clone https://github.com/ROCmSoftwarePlatform/rccl.git +$ cd rccl +$ mkdir build +$ cd build +$ CXX=/opt/rocm/bin/hcc cmake -DCMAKE_INSTALL_PREFIX=$PWD/rccl-install .. +$ make -j 8 install ``` +You may substitute a path of your own choosing for CMAKE_INSTALL_PREFIX. -If CUDA is not installed in the default /usr/local/cuda path, you can define the CUDA path with : +#### To build the RCCL package and install package : + +Assuming you have already cloned this repository and built the library as shown in the previous section: ```shell -$ make src.build CUDA_HOME= +$ cd rccl/build +$ make package +$ sudo dpkg -i *.deb ``` -NCCL will be compiled and installed in `build/` unless `BUILDDIR` is set. - -By default, NCCL is compiled for all supported architectures. To accelerate the compilation and reduce the binary size, consider redefining `NVCC_GENCODE` (defined in `makefiles/common.mk`) to only include the architecture of the target platform : -```shell -$ make -j src.build NVCC_GENCODE="-gencode=arch=compute_70,code=sm_70" -``` - -## Install - -To install NCCL on the system, create a package then install it as root. - -Debian/Ubuntu : -```shell -$ # Install tools to create debian packages -$ sudo apt install build-essential devscripts debhelper -$ # Build NCCL deb package -$ make pkg.debian.build -$ ls build/pkg/deb/ -``` - -RedHat/CentOS : -```shell -$ # Install tools to create rpm packages -$ sudo yum install rpm-build rpmdevtools -$ # Build NCCL rpm package -$ make pkg.redhat.build -$ ls build/pkg/rpm/ -``` - -OS-agnostic tarball : -```shell -$ make pkg.txz.build -$ ls build/pkg/txz/ -``` +RCCL package install requires sudo/root access because it creates a directory called "rccl" under /opt/rocm/. This is an optional step and RCCL can be used directly by including the path containing librccl.so. ## Tests -Tests for NCCL are maintained separately at https://github.com/nvidia/nccl-tests. +There are unit tests implemented with the Googletest framework in RCCL, which are currently a work-in-progress. To invoke the unit tests, go to the rccl-install folder, then the test/ subfolder, and execute the appropriate unit test executable(s). Several notes for running the unit tests: +1. The LD_LIBRARY_PATH environment variable will need to be set to include /path/to/rccl-install/lib/ in order to run the unit tests. +2. The HSA_FORCE_FINE_GRAIN_PCIE environment variable will need to be set to 1 in order to run the unit tests. + +An example call to the unit tests: ```shell -$ git clone https://github.com/NVIDIA/nccl-tests.git -$ cd nccl-tests -$ make -$ ./build/all_reduce_perf -b 8 -e 256M -f 2 -g +$ LD_LIBRARY_PATH=rccl-install/lib/ HSA_FORCE_FINE_GRAIN_PCIE=1 rccl-install/test/UnitTests ``` +There are also other performance and error-checking tests for RCCL. These are maintained separately at https://github.com/ROCmSoftwarePlatform/rccl-tests. +See the rccl-tests README for more information on how to build and run those tests. + ## Copyright All source code and accompanying documentation is copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + +All modifications are copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. diff --git a/hipify.sh b/hipify.sh new file mode 100755 index 0000000000..e389fb519e --- /dev/null +++ b/hipify.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + +FILES=" +./src/nccl.h.in +./src/bootstrap.cu +./src/collectives/all_gather.cu +./src/collectives/all_reduce.cu +./src/collectives/broadcast.cu +./src/collectives/collectives.h +./src/collectives/device/all_gather.cu +./src/collectives/device/all_gather.h +./src/collectives/device/all_reduce.cu +./src/collectives/device/all_reduce.h +./src/collectives/device/broadcast.cu +./src/collectives/device/broadcast.h +./src/collectives/device/common.h +./src/collectives/device/common_kernel.h +./src/collectives/device/functions.cu +./src/collectives/device/ll_kernel.h +./src/collectives/device/primitives.h +./src/collectives/device/reduce.cu +./src/collectives/device/reduce.h +./src/collectives/device/reduce_kernel.h +./src/collectives/device/reduce_scatter.cu +./src/collectives/device/reduce_scatter.h +./src/collectives/reduce.cu +./src/collectives/reduce_scatter.cu +./src/include/bootstrap.h +./src/include/common_coll.h +./src/include/core.h +./src/include/debug.h +./src/include/enqueue.h +./src/include/group.h +./src/include/ibvwrap.h +./src/include/nccl_net.h +./src/include/net.h +./src/include/nvlink.h +./src/include/nvmlwrap.h +./src/include/param.h +./src/include/ring.h +./src/include/rings.h +./src/include/shm.h +./src/include/socket.h +./src/include/topo.h +./src/include/transport.h +./src/include/utils.h +./src/init.cu +./src/misc/enqueue.cu +./src/misc/group.cu +./src/misc/ibvwrap.cu +./src/misc/nvmlwrap.cu +./src/misc/rings.cu +./src/misc/utils.cu +./src/ring.cu +./src/transport.cu +./src/transport/net.cu +./src/transport/net_ib.cu +./src/transport/net_socket.cu +./src/transport/p2p.cu +./src/transport/shm.cu +" + +for f in $FILES +do + sed -i \ + -e 's@cuda_runtime.h@hip/hip_runtime_api.h@g' \ + -e 's@cuda_fp16.h@hip/hip_fp16.h@g' \ + -e 's/cudaDeviceCanAccessPeer/hipDeviceCanAccessPeer/g' \ + -e 's/cudaDeviceEnablePeerAccess/hipDeviceEnablePeerAccess/g' \ + -e 's/cudaDeviceGetPCIBusId/hipDeviceGetPCIBusId/g' \ + -e 's/cudaErrorPeerAccessAlreadyEnabled/hipErrorPeerAccessAlreadyEnabled/g' \ + -e 's/cudaError_t/hipError_t/g' \ + -e 's/cudaEventCreateWithFlags/hipEventCreateWithFlags/g' \ + -e 's/cudaEventDestroy/hipEventDestroy/g' \ + -e 's/cudaEventDisableTiming/hipEventDisableTiming/g' \ + -e 's/cudaEventRecord/hipEventRecord/g' \ + -e 's/cudaEvent_t/hipEvent_t/g' \ + -e 's/cudaFree/hipFree/g' \ + -e 's/cudaFreeHost/hipHostFree/g' \ + -e 's/cudaGetDevice/hipGetDevice/g' \ + -e 's/cudaGetErrorString/hipGetErrorString/g' \ + -e 's/cudaGetLastError/hipGetLastError/g' \ + -e 's/cudaHostAlloc/hipHostMalloc/g' \ + -e 's/cudaHostAllocMapped/hipHostMallocMapped/g' \ + -e 's/cudaHostGetDevicePointer/hipHostGetDevicePointer/g' \ + -e 's/cudaHostRegister/hipHostRegister/g' \ + -e 's/cudaHostRegisterMapped/hipHostRegisterMapped/g' \ + -e 's/cudaHostUnregister/hipHostUnregister/g' \ + -e 's/cudaIpcCloseMemHandle/hipIpcCloseMemHandle/g' \ + -e 's/cudaIpcGetMemHandle/hipIpcGetMemHandle/g' \ + -e 's/cudaIpcMemHandle_t/hipIpcMemHandle_t/g' \ + -e 's/cudaIpcMemLazyEnablePeerAccess/hipIpcMemLazyEnablePeerAccess/g' \ + -e 's/cudaIpcOpenMemHandle/hipIpcOpenMemHandle/g' \ + -e 's/cudaMalloc/hipMalloc/g' \ + -e 's/cudaMemcpy/hipMemcpy/g' \ + -e 's/cudaMemcpyAsync/hipMemcpyAsync/g' \ + -e 's/cudaMemcpyDefault/hipMemcpyDefault/g' \ + -e 's/cudaMemcpyDeviceToDevice/hipMemcpyDeviceToDevice/g' \ + -e 's/cudaMemoryTypeDevice/hipMemoryTypeDevice/g' \ + -e 's/cudaMemset/hipMemset/g' \ + -e 's/cudaPointerAttributes/hipPointerAttribute_t/g' \ + -e 's/cudaPointerGetAttributes/hipPointerGetAttributes/g' \ + -e 's/cudaSetDevice/hipSetDevice/g' \ + -e 's/cudaStreamCreateWithFlags/hipStreamCreateWithFlags/g' \ + -e 's/cudaStreamDestroy/hipStreamDestroy/g' \ + -e 's/cudaStreamNonBlocking/hipStreamNonBlocking/g' \ + -e 's/cudaStreamWaitEvent/hipStreamWaitEvent/g' \ + -e 's/cudaStream_t/hipStream_t/g' \ + -e 's/cudaSuccess/hipSuccess/g' \ + $f +done diff --git a/install.sh b/install.sh new file mode 100755 index 0000000000..0069c5bce5 --- /dev/null +++ b/install.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + +export RCCL_DIR=$PWD/rccl-internal +export RCCL_INSTALL=$PWD/rccl-install +export ROCM_PATH=/opt/rocm/bin + +# ################################################# +# helper functions +# ################################################# +function display_help() +{ + echo "RCCL build & installation helper script" + echo "./install [-h|--help] " + echo " [-h|--help] prints this help message" + echo " [-t|--test] run RCCL unit tests too" +} + +# ################################################# +# Parameter parsing +# ################################################# + +# check if we have a modern version of getopt that can handle whitespace and long parameters +getopt -T +if [[ $? -eq 4 ]]; then + GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,clients,debug,test --options hicdt -- "$@") +else + echo "Need a new version of getopt" + exit 1 +fi + +if [[ $? -ne 0 ]]; then + echo "getopt invocation failed; could not parse the command line"; + exit 1 +fi + +eval set -- "${GETOPT_PARSE}" + +run_tests=false + +while true; do + case "${1}" in + -h|--help) + display_help + exit 0 + ;; + -t|--test) + run_tests=true + shift ;; + --) shift ; break ;; + *) echo "Unexpected command line parameter received; aborting"; + exit 1 + ;; + esac + done + +# Install the pre-commit hook +#bash ./githooks/install + +rm -rf build +mkdir build +cd build +CXX=$ROCM_PATH/hcc cmake -DCMAKE_INSTALL_PREFIX=$RCCL_INSTALL .. +make -j 8 install + +if ($run_tests); then +# Optionally, run tests if they're enabled. +HSA_FORCE_FINE_GRAIN_PCIE=1 $RCCL_INSTALL/test/UnitTests +fi diff --git a/jenkinsfile b/jenkinsfile new file mode 100644 index 0000000000..655f9dc7e0 --- /dev/null +++ b/jenkinsfile @@ -0,0 +1,89 @@ +#!/usr/bin/env groovy +// Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. +// This shared library is available at https://github.com/ROCmSoftwarePlatform/rccl +@Library('rocJenkins') _ + +// This is file for internal AMD use. +// If you are interested in running your own Jenkins, please raise a github issue for assistance. + +import com.amd.project.* +import com.amd.docker.* + +//////////////////////////////////////////////////////////////////////// +// Mostly generated from snippet generator 'properties; set job properties' +// Time-based triggers added to execute nightly tests, eg '30 2 * * *' means 2:30 AM +properties([ + pipelineTriggers([cron('0 1 * * *'), [$class: 'PeriodicFolderTrigger', interval: '5m']]), + buildDiscarder(logRotator( + artifactDaysToKeepStr: '', + artifactNumToKeepStr: '', + daysToKeepStr: '', + numToKeepStr: '10')), + disableConcurrentBuilds(), + [$class: 'CopyArtifactPermissionProperty', projectNames: '*'] + ]) + + +//////////////////////////////////////////////////////////////////////// +import java.nio.file.Path; + +rcclCI: +{ + + def rccl = new rocProject('rccl') + // customize for project + rccl.paths.build_command = './install.sh' + + // Define test architectures, optional rocm version argument is available + def nodes = new dockerNodes(['gfx906'], rccl) + + boolean formatCheck = false + + def compileCommand = + { + platform, project-> + + project.paths.construct_build_prefix() + def command = """#!/usr/bin/env bash + set -x + cd ${project.paths.project_build_prefix} + LD_LIBRARY_PATH=/opt/rocm/hcc/lib CXX=${project.compiler.compiler_path} ${project.paths.build_command} + """ + + platform.runCommand(this, command) + } + + def testCommand = + { + platform, project-> + + def command = """#!/usr/bin/env bash + set -x + cd ${project.paths.project_build_prefix}/rccl-install/test + ./UnitTest --gtest_output=xml --gtest_color=yes + """ + + platform.runCommand(this, command) + junit "${project.paths.project_build_prefix}/rccl-install/*.xml" + } + + def packageCommand = + { + platform, project-> + + def command = """ + set -x + cd ${project.paths.project_build_prefix}/build + make package + rm -rf package && mkdir -p package + mv *.deb package/ + sudo dpkg -i package/*.deb + """ + + platform.runCommand(this, command) + platform.archiveArtifacts(this, """${project.paths.project_build_prefix}/build/package/*.deb""") + } + + buildProject(rccl, formatCheck, nodes.dockerArray, compileCommand, testCommand, packageCommand) + +} \ No newline at end of file diff --git a/src/collectives/all_gather.cu b/src/collectives/all_gather.cu index 8dec28e63b..7ad36c777b 100644 --- a/src/collectives/all_gather.cu +++ b/src/collectives/all_gather.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -10,12 +11,12 @@ #include "collectives.h" ncclResult_t ncclAllGatherFunc(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { size_t nbytes = count*ncclTypeSize(datatype); INFO(NCCL_COLL,"AllGather: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p", comm->opCount, sendbuff, recvbuff, count, datatype, op, root, comm, comm->nRanks, stream); if (comm->nRanks == 1) { if (sendbuff != recvbuff) - CUDACHECK(cudaMemcpyAsync(recvbuff, sendbuff, nbytes, cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(hipMemcpyAsync(recvbuff, sendbuff, nbytes, hipMemcpyDeviceToDevice, stream)); } else { NCCLCHECK(transportSaveProxies(ALLGATHER_SUBSTEPS, ALLGATHER_BUFCHUNKS, comm->nRanks-1, comm->nRanks, nbytes*comm->nRanks, proxyPatternRing, comm)); NCCLCHECK(saveKernel(ncclCollAllGather, sendbuff, recvbuff, nbytes, ncclInt8, op, root, comm, stream, nbytes*comm->nRanks, 1)); @@ -24,9 +25,9 @@ ncclResult_t ncclAllGatherFunc(const void* sendbuff, void* recvbuff, size_t coun } NCCL_API(ncclResult_t, ncclAllGather, const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclComm_t comm, hipStream_t stream); ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclComm_t comm, hipStream_t stream) { return ncclEnqueueCheck(ncclAllGatherFunc, "AllGather", sendbuff, recvbuff, sendcount, datatype, ncclSum, 0, comm, stream); } diff --git a/src/collectives/all_reduce.cu b/src/collectives/all_reduce.cu index cc14083ab7..234af2c898 100644 --- a/src/collectives/all_reduce.cu +++ b/src/collectives/all_reduce.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -10,12 +11,12 @@ #include "collectives.h" ncclResult_t ncclAllReduceFunc(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { size_t nbytes = count*ncclTypeSize(datatype); INFO(NCCL_COLL,"AllReduce: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p", comm->opCount, sendbuff, recvbuff, count, datatype, op, root, comm, comm->nRanks, stream); if (comm->nRanks == 1) { if (sendbuff != recvbuff) - CUDACHECK(cudaMemcpyAsync(recvbuff, sendbuff, nbytes, cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(hipMemcpyAsync(recvbuff, sendbuff, nbytes, hipMemcpyDeviceToDevice, stream)); } else { NCCLCHECK(transportSaveProxies(ALLREDUCE_SUBSTEPS, ALLREDUCE_BUFCHUNKS, (comm->nRanks)*2-2, comm->nRanks, nbytes, proxyPatternRing, comm)); NCCLCHECK(saveKernel(ncclCollAllReduce, sendbuff, recvbuff, count, datatype, op, root, comm, stream, nbytes, comm->nRanks)); @@ -24,9 +25,9 @@ ncclResult_t ncclAllReduceFunc(const void* sendbuff, void* recvbuff, size_t coun } NCCL_API(ncclResult_t, ncclAllReduce, const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, hipStream_t stream); ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, hipStream_t stream) { return ncclEnqueueCheck(ncclAllReduceFunc, "AllReduce", sendbuff, recvbuff, count, datatype, op, 0, comm, stream); } diff --git a/src/collectives/broadcast.cu b/src/collectives/broadcast.cu index 91ce905440..a2b65995f8 100644 --- a/src/collectives/broadcast.cu +++ b/src/collectives/broadcast.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -10,12 +11,12 @@ #include "collectives.h" ncclResult_t ncclBroadcastFunc(const void* sendbuff, void* recvbuff, const size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { size_t nbytes = count*ncclTypeSize(datatype); INFO(NCCL_COLL,"Broadcast: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p", comm->opCount, sendbuff, recvbuff, count, datatype, op, root, comm, comm->nRanks, stream); if (comm->nRanks == 1) { if (sendbuff != recvbuff) - CUDACHECK(cudaMemcpyAsync(recvbuff, sendbuff, nbytes, cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(hipMemcpyAsync(recvbuff, sendbuff, nbytes, hipMemcpyDeviceToDevice, stream)); } else { NCCLCHECK(transportSaveProxies(BROADCAST_SUBSTEPS, BROADCAST_BUFCHUNKS, 1, 1, nbytes, proxyPatternFrom(root), comm)); NCCLCHECK(saveKernel(ncclCollBroadcast, sendbuff, recvbuff, nbytes, ncclInt8, op, root, comm, stream, nbytes, 1)); @@ -26,17 +27,17 @@ ncclResult_t ncclBroadcastFunc(const void* sendbuff, void* recvbuff, const size_ /* Deprecated original "in place" function, similar to MPI */ NCCL_API(ncclResult_t, ncclBcast, void* buff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream) { + ncclComm_t comm, hipStream_t stream) { return ncclEnqueueCheck(ncclBroadcastFunc, "Bcast", buff, buff, count, datatype, ncclSum, root, comm, stream); } NCCL_API(ncclResult_t, ncclBroadcast, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream) { + ncclComm_t comm, hipStream_t stream) { return ncclEnqueueCheck(ncclBroadcastFunc, "Broadcast", sendbuff, recvbuff, count, datatype, ncclSum, root, comm, stream); } diff --git a/src/collectives/collectives.h b/src/collectives/collectives.h index 4a5cb7a98d..5b2f0f13f4 100644 --- a/src/collectives/collectives.h +++ b/src/collectives/collectives.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -19,7 +20,7 @@ typedef enum { ncclCollBroadcast, ncclCollReduce, ncclCollAllGather, ncclCollRed /* Declare all collective operations */ #define DECL_COLL4(coll, op, dtype) \ - extern __device__ void NCCL_COLL_NAME(coll, op, dtype)(struct CollectiveArgs* args); \ + extern __device__ __attribute__((noinline)) void NCCL_COLL_NAME(coll, op, dtype)(struct CollectiveArgs* args); \ extern __global__ void NCCL_KERN_NAME(coll, op, dtype)(struct ncclColl coll); \ #define DECL_COLL3(coll, op, dtype) \ diff --git a/src/collectives/device/all_gather.h b/src/collectives/device/all_gather.h index a30e575570..677435071e 100644 --- a/src/collectives/device/all_gather.h +++ b/src/collectives/device/all_gather.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -16,15 +17,16 @@ if (noffset == buffSize) noffset = 0; template +__attribute__((noinline)) __device__ void ncclAllGatherKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; - const int nthreads = blockDim.x - 1; + const int nthreads = blockDim.x; const int bid = args->bid; __shared__ T* sharedNextOutput; struct ncclComm* comm = args->comm; struct ncclRing* ring = comm->rings+blockIdx.x; - int prevdirect = ring->recv.conn.direct; - int nextdirect = ring->send.conn.direct; + int prevdirect = 0; + int nextdirect = 0; WaitFlag waitDoneFromNext(ring->send.conn.head, ALLGATHER_BUFCHUNKS*ALLGATHER_SUBSTEPS); WaitFlag waitReadyFromPrev(ring->recv.conn.tail, ALLGATHER_SUBSTEPS); @@ -41,7 +43,7 @@ __device__ void ncclAllGatherKernel(struct CollectiveArgs* args) { if (tid == 0) { // Update in case we skipped some collectives - *ring->recv.conn.opCount = args->opCount; + STORE(ring->recv.conn.opCount, args->opCount); // Wait for next to be ready WaitFlag waitOpCountNext(ring->send.conn.opCount, 0); waitOpCountNext.wait(args->opCount); @@ -50,9 +52,9 @@ __device__ void ncclAllGatherKernel(struct CollectiveArgs* args) { } if (nextdirect) { void* volatile* ptr = &(ring->devMemSend->ptrExchange); - while (*ptr == nullptr); - sharedNextOutput = (T*)*ptr; - *ptr = nullptr; + while (LOAD(ptr) == nullptr); + sharedNextOutput = (T*)LOAD(ptr); + STORE(ptr, nullptr); } } __syncthreads(); @@ -158,10 +160,10 @@ __device__ void ncclAllGatherKernel(struct CollectiveArgs* args) { if (tid == 0) { waitDoneFromNext.wait(ALLGATHER_SUBSTEPS*(step + ALLGATHER_BUFCHUNKS)); - *ring->send.conn.head = 0ULL; - *ring->recv.conn.tail = 0ULL; + STORE(ring->send.conn.head, 0ULL); + STORE(ring->recv.conn.tail, 0ULL); __threadfence_system(); - *ring->recv.conn.opCount = args->opCount+1; + STORE(ring->recv.conn.opCount, args->opCount+1); } } @@ -176,6 +178,7 @@ __device__ void ncclAllGatherKernel(struct CollectiveArgs* args) { step++; template +__attribute__((noinline)) __device__ void ncclAllGatherLLKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; const int bid = args->bid; diff --git a/src/collectives/device/all_gather_0.cpp b/src/collectives/device/all_gather_0.cpp new file mode 100644 index 0000000000..75f90ca5e6 --- /dev/null +++ b/src/collectives/device/all_gather_0.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 0 +#include "device/all_gather.cu" diff --git a/src/collectives/device/all_reduce.h b/src/collectives/device/all_reduce.h index d7abc6445b..a323d4a70b 100644 --- a/src/collectives/device/all_reduce.h +++ b/src/collectives/device/all_reduce.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -16,15 +17,16 @@ if (noffset == buffSize) noffset = 0; template +__attribute__((noinline)) __device__ void ncclAllReduceKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; - const int nthreads = blockDim.x - 1; + const int nthreads = blockDim.x; const int bid = args->bid; __shared__ T* sharedNextOutput; struct ncclComm* comm = args->comm; struct ncclRing* ring = comm->rings+blockIdx.x; - int prevdirect = ring->recv.conn.direct; - int nextdirect = ring->send.conn.direct; + int prevdirect = 0; + int nextdirect = 0; WaitFlag waitDoneFromNext(ring->send.conn.head, ALLREDUCE_BUFCHUNKS*ALLREDUCE_SUBSTEPS); WaitFlag waitReadyFromPrev(ring->recv.conn.tail, ALLREDUCE_SUBSTEPS); @@ -42,7 +44,7 @@ __device__ void ncclAllReduceKernel(struct CollectiveArgs* args) { if (tid == 0) { // Update in case we skipped some collectives - *ring->recv.conn.opCount = args->opCount; + STORE(ring->recv.conn.opCount, args->opCount); // Wait for next to be ready WaitFlag waitOpCountNext(ring->send.conn.opCount, 0); waitOpCountNext.wait(args->opCount); @@ -51,9 +53,9 @@ __device__ void ncclAllReduceKernel(struct CollectiveArgs* args) { } if (nextdirect) { void* volatile* ptr = &(ring->devMemSend->ptrExchange); - while (*ptr == nullptr); - sharedNextOutput = (T*)*ptr; - *ptr = nullptr; + while (LOAD(ptr) == nullptr); + sharedNextOutput = (T*)LOAD(ptr); + STORE(ptr, nullptr); } } __syncthreads(); @@ -189,10 +191,10 @@ __device__ void ncclAllReduceKernel(struct CollectiveArgs* args) { if (tid == 0) { // Wait for next to have consumed all data before we reset the flag waitDoneFromNext.wait(ALLREDUCE_SUBSTEPS*(step + ALLREDUCE_BUFCHUNKS)); - *ring->send.conn.head = 0ULL; - *ring->recv.conn.tail = 0ULL; + STORE(ring->send.conn.head, 0ULL); + STORE(ring->recv.conn.tail, 0ULL); __threadfence_system(); - *ring->recv.conn.opCount = args->opCount+1; + STORE(ring->recv.conn.opCount, args->opCount+1); } } @@ -207,6 +209,7 @@ __device__ void ncclAllReduceKernel(struct CollectiveArgs* args) { step++; template +__attribute__((noinline)) __device__ void ncclAllReduceLLKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; const int bid = args->bid; diff --git a/src/collectives/device/all_reduce_0.cpp b/src/collectives/device/all_reduce_0.cpp new file mode 100644 index 0000000000..235005af1a --- /dev/null +++ b/src/collectives/device/all_reduce_0.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 0 +#include "device/all_reduce.cu" diff --git a/src/collectives/device/all_reduce_1.cpp b/src/collectives/device/all_reduce_1.cpp new file mode 100644 index 0000000000..dda4b5d517 --- /dev/null +++ b/src/collectives/device/all_reduce_1.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 1 +#include "device/all_reduce.cu" diff --git a/src/collectives/device/all_reduce_2.cpp b/src/collectives/device/all_reduce_2.cpp new file mode 100644 index 0000000000..745435b60f --- /dev/null +++ b/src/collectives/device/all_reduce_2.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 2 +#include "device/all_reduce.cu" diff --git a/src/collectives/device/all_reduce_3.cpp b/src/collectives/device/all_reduce_3.cpp new file mode 100644 index 0000000000..d7f45f03dd --- /dev/null +++ b/src/collectives/device/all_reduce_3.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 3 +#include "device/all_reduce.cu" diff --git a/src/collectives/device/broadcast.h b/src/collectives/device/broadcast.h index c2f6d001e1..0baef3f476 100644 --- a/src/collectives/device/broadcast.h +++ b/src/collectives/device/broadcast.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -15,15 +16,16 @@ if (boffset == buffSize) boffset = 0; template +__attribute__((noinline)) __device__ void ncclBroadcastKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; - const int nthreads = blockDim.x - 1; + const int nthreads = blockDim.x; const int bid = args->bid; __shared__ T* sharedNextOutput; struct ncclComm* comm = args->comm; struct ncclRing* ring = comm->rings+blockIdx.x; - int prevdirect = ring->recv.conn.direct; - int nextdirect = ring->send.conn.direct; + int prevdirect = 0; + int nextdirect = 0; WaitFlag waitDoneFromNext(ring->send.conn.head, (BROADCAST_BUFCHUNKS-1)*BROADCAST_SUBSTEPS); WaitFlag waitReadyFromPrev(ring->recv.conn.tail, 0); @@ -42,7 +44,7 @@ __device__ void ncclBroadcastKernel(struct CollectiveArgs* args) { if (tid == 0) { // Update in case we skipped some collectives - *ring->recv.conn.opCount = args->opCount; + STORE(ring->recv.conn.opCount, args->opCount); if (nextRank != root) { // Wait for next to be ready WaitFlag waitOpCountNext(ring->send.conn.opCount, 0); @@ -53,9 +55,9 @@ __device__ void ncclBroadcastKernel(struct CollectiveArgs* args) { } if (nextRank != root && nextdirect) { void* volatile* ptr = &(ring->devMemSend->ptrExchange); - while (*ptr == nullptr); - sharedNextOutput = (T*)*ptr; - *ptr = nullptr; + while (LOAD(ptr) == nullptr); + sharedNextOutput = (T*)LOAD(ptr); + STORE(ptr, nullptr); } } __syncthreads(); @@ -130,11 +132,11 @@ __device__ void ncclBroadcastKernel(struct CollectiveArgs* args) { if (nextRank != root) { // Wait for next to have consumed data before resetting the flag waitDoneFromNext.wait(BROADCAST_SUBSTEPS*(step + BROADCAST_BUFCHUNKS - 1)); - *ring->send.conn.head = 0ULL; + STORE(ring->send.conn.head, 0ULL); } - *ring->recv.conn.tail = 0ULL; + STORE(ring->recv.conn.tail, 0ULL); __threadfence_system(); - *ring->recv.conn.opCount = args->opCount+1; + STORE(ring->recv.conn.opCount, args->opCount+1); } } @@ -147,6 +149,7 @@ __device__ void ncclBroadcastKernel(struct CollectiveArgs* args) { step++; template +__attribute__((noinline)) __device__ void ncclBroadcastLLKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; const int bid = args->bid; diff --git a/src/collectives/device/broadcast_0.cpp b/src/collectives/device/broadcast_0.cpp new file mode 100644 index 0000000000..75b75ad9cf --- /dev/null +++ b/src/collectives/device/broadcast_0.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 0 +#include "device/broadcast.cu" diff --git a/src/collectives/device/common.h b/src/collectives/device/common.h index c9889133eb..2f230c6b4b 100644 --- a/src/collectives/device/common.h +++ b/src/collectives/device/common.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -7,12 +8,49 @@ #ifndef NCCL_DEVICE_COMMON_H_ #define NCCL_DEVICE_COMMON_H_ +#include + #include "../collectives.h" #include "core.h" #include "nccl.h" +#include + typedef void(*ncclKern_t)(struct CollectiveArgs* args); -extern __device__ ncclKern_t ncclFuncs[]; +extern __device__ const ncclKern_t ncclFuncs[]; + +template +struct Caller { + static + void call(ncclColl* const c) noexcept + { + constexpr unsigned short m = f + (l - f) / 2; + + return (c->funcIndex < m) ? Caller::call(c) : Caller::call(c); + } +}; + +template +struct Caller{ + static + void call(struct ncclColl* const c) noexcept { ncclFuncs[f](&c->args); } +}; + +inline +__device__ +void NCCL_CALL_FUNCTIONS(struct ncclColl* const c) noexcept +{ + if (c->funcIndex < 72) { + if (c->funcIndex % 2) ncclBroadcastLL_copy_i8(&c->args); + else ncclBroadcast_copy_i8(&c->args); + } + else if (c->funcIndex < 144) Caller<72, 144>::call(c); + else if (c->funcIndex < 216) { + if (c->funcIndex % 2) ncclAllGatherLL_copy_i8(&c->args); + else ncclAllGather_copy_i8(&c->args); + } + else Caller<216, 360>::call(c); +} static __device__ void load_parallel(void* dst, void* src, size_t size, int tid) { int* d = (int*)dst; @@ -54,7 +92,7 @@ __global__ void NCCL_KERN_NAME(coll, op, dtype)(struct ncclColl firstColl) { \ if (c->funcIndex == fIndex) { \ coll##Kernel, ctype>(&c->args); \ } else { \ - ncclFuncs[c->funcIndex](&c->args); \ + NCCL_CALL_FUNCTIONS(c); \ } \ } \ int nextIndex = c->nextIndex; \ diff --git a/src/collectives/device/common_kernel.h b/src/collectives/device/common_kernel.h index 0eaa0610d0..e8194bf4e3 100644 --- a/src/collectives/device/common_kernel.h +++ b/src/collectives/device/common_kernel.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -11,13 +12,25 @@ #include #include -#include +#include // Define min for ssize_t static __device__ int min(int a, ssize_t b) { return (a < b) ? a : b; } typedef uint64_t PackType; +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + +template +struct MULTI { + __device__ PackType operator()(const PackType x, const PackType y) const + { + return FUNC()(x, y); + } +}; + +#else + // unpack x and y to elements of type T and apply FUNC to each element template struct MULTI { @@ -192,6 +205,8 @@ struct MULTI { } }; +#endif //defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + #define ALIGNUP(x, a) ((((x)-1) & ~((a)-1)) + (a)) template @@ -210,7 +225,7 @@ void vStore(volatile T* ptr, const T val) { *ptr = val; } -#if CUDART_VERSION < 9000 +#if CUDART_VERSION < 9000 && !(defined(__HIP_PLATFORM_HCC__) || defined(__HCC__)) template<> inline __device__ half vFetch(const volatile half* ptr) { half r; @@ -237,6 +252,7 @@ void vStore(volatile half* ptr, const half val) { #endif template +__attribute__((noinline)) __device__ inline void ReduceCopy( const int tid, const int nthreads, const volatile T * __restrict__ const src0, @@ -266,14 +282,25 @@ struct MULTI128 { }; inline __device__ void Fetch128(Pack128& v, Pack128* p) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + v.x = p->x; + v.y = p->y; +#else asm volatile("ld.volatile.global.v2.u64 {%0,%1}, [%2];" : "=l"(v.x), "=l"(v.y) : "l"(p) : "memory"); +#endif } inline __device__ void Store128(Pack128* p, Pack128& v) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + p->x = v.x; + p->y = v.y; +#else asm volatile("st.volatile.global.v2.u64 [%0], {%1,%2};" :: "l"(p), "l"(v.x), "l"(v.y) : "memory"); +#endif } #define WARP_SIZE 32 template +__attribute__((noinline)) __device__ inline void ReduceCopy128b( const int w, const int nw, const int t, Pack128 * src0, Pack128 * src1, Pack128 * dest0, Pack128 * dest1, const int N) { @@ -303,6 +330,7 @@ __device__ inline void ReduceCopy128b( const int w, const int nw, const int t, } template +__attribute__((noinline)) __device__ inline void ReduceOrCopy(const int tid, const int nthreads, volatile T * __restrict__ dest0, volatile T * __restrict__ dest1, const volatile T * __restrict__ src0, const volatile T * __restrict__ src1, diff --git a/src/collectives/device/functions.cu b/src/collectives/device/functions.cu index 1fb8108166..9226c576d4 100644 --- a/src/collectives/device/functions.cu +++ b/src/collectives/device/functions.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -55,7 +56,15 @@ NCCL_FUNCS2A(ncclAllReduce) } // Must be consistent with the ncclFuncSet enum -__device__ ncclKern_t ncclFuncs[ncclCollCount*ncclNumOps*ncclNumTypes*2] = { +using ncclKern_t = void (*)(struct CollectiveArgs*); +__device__ constexpr ncclKern_t ncclFuncs[]{ +#if defined(__HIP_DEVICE_COMPILE__) + NCCL_FUNCS2B(ncclBroadcast), + NCCL_FUNCS2A(ncclReduce), + NCCL_FUNCS2B(ncclAllGather), + NCCL_FUNCS2A(ncclReduceScatter), + NCCL_FUNCS2A(ncclAllReduce) +#endif // Don't try to initialize the host shadow copy of this device-side global // variable. There is no host pointer to a device-side function, which // confuses clang. This will be fixed in the next clang release. diff --git a/src/collectives/device/ll_kernel.h b/src/collectives/device/ll_kernel.h index 5ec3c9a871..ca7e4d63e5 100644 --- a/src/collectives/device/ll_kernel.h +++ b/src/collectives/device/ll_kernel.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -7,17 +8,41 @@ #ifndef NCCL_LL_KERNEL_H_ #define NCCL_LL_KERNEL_H_ -static __device__ uint64_t readLL(union ncclLLFifoLine* src, uint32_t flag) { +static __device__ __attribute__((noinline)) uint64_t readLL(union ncclLLFifoLine* src, uint32_t flag) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + using Vec = uint32_t __attribute__((ext_vector_type(4))); + Vec i4; + do { + asm volatile ("flat_load_dwordx4 %0, %1, glc\n" + "s_waitcnt vmcnt(0)\n" + "buffer_wbinvl1_vol\n" : "=v"(i4) : "v"(src)); + } while (i4[1] != flag || i4[3] != flag); + uint64_t val64 = (uint64_t)(i4[0]) + (((uint64_t)i4[2]) << 32); + return val64; +#else uint32_t data1, flag1, data2, flag2; do { asm volatile("ld.volatile.global.v4.u32 {%0,%1,%2,%3}, [%4];" : "=r"(data1), "=r"(flag1), "=r"(data2), "=r"(flag2) : "l"(&src->i4)); } while ((flag1 != flag) || (flag2 != flag)); uint64_t val64 = data1 + (((uint64_t)data2) << 32); return val64; +#endif } -static __device__ void storeLL(union ncclLLFifoLine* dst, uint64_t val, uint32_t flag) { +static __device__ __attribute__((noinline)) void storeLL(union ncclLLFifoLine* dst, uint64_t val, uint32_t flag) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + using Vec = uint32_t __attribute__((ext_vector_type(4))); + Vec i4; + i4[0] = val & 0xffffffff; + i4[1] = flag; + i4[2] = (val >> 32); + i4[3] = flag; + asm volatile ("flat_store_dwordx4 %0, %1, glc\n" + "s_waitcnt vmcnt(0)\n" + "buffer_wbinvl1_vol\n" : : "v"(dst), "v"(i4)); +#else asm volatile("st.volatile.global.v4.u32 [%0], {%1,%2,%3,%4};" :: "l"(&dst->i4), "r"((uint32_t)val), "r"(flag), "r"((uint32_t)(val >> 32)), "r"(flag)); +#endif } // Using memcpy handles misaligned pointers. @@ -34,6 +59,7 @@ template class LLPrimitives { private: template + __attribute__((noinline)) static __device__ void ReduceCopyGeneric(const T* src1, union ncclLLFifoLine* src2, T* dst1, union ncclLLFifoLine* dst2, int size, uint32_t iflag, uint32_t oflag, int nthreads) { if (size <= 0) return; size_t size64 = size * sizeof(T) / sizeof(uint64_t); @@ -117,23 +143,29 @@ class LLPrimitives { #define STEP_TO_SLOT(step) \ (step % NCCL_LL_CHUNKS) +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#define SYNC __syncthreads() +#else +#define SYNC asm volatile ("bar.sync 1, %0;" :: "r"(llNthreads)) +#endif + #define WAIT_NEXT \ if (tid == 0) { \ while (sendHead + NCCL_LL_CHUNKS <= step) { \ - sendHead = sendHeadPtr[0]; \ + sendHead = LOAD(sendHeadPtr); \ } \ } \ - asm volatile ("bar.sync 1, %0;" :: "r"(llNthreads)); + SYNC; #define POST_SIZE \ - if (tid == 0 && sizesFifo) sizesFifo[step % NCCL_LL_CHUNKS] = (maxOffset <= 0) ? -1 : (maxOffset*2*(int)sizeof(T)); + if (tid == 0 && sizesFifo) { STORE(sizesFifo + step % NCCL_LL_CHUNKS, (maxOffset <= 0) ? -1 : (maxOffset*2*(int)sizeof(T))); } #define ACK_PREV \ - asm volatile ("bar.sync 1, %0;" :: "r"(llNthreads)); \ - if (tid == 0) recvHeadPtr[0] = step; + SYNC; \ + if (tid == 0) STORE(recvHeadPtr,step); #define FIFO_CLEANING_AND_SAVE_STEP(flag) do { \ - if (step > ring->send.conn.llLastCleaning + NCCL_LL_CLEAN_FREQ) { \ + if (step > LOAD(&ring->send.conn.llLastCleaning) + NCCL_LL_CLEAN_FREQ) { \ /* Reset all flags */ \ static_assert((NCCL_LL_BUFF_SIZE % NCCL_LL_MAX_NTHREADS) == 0, "NCCL_LL_BUFF_SIZE must be a multiple of THREADS"); \ static_assert(NCCL_LL_BUFF_SIZE/(sizeof(union ncclLLFifoLine)*NCCL_LL_MAX_NTHREADS) > 0, "NCCL_LL_BUFF_SIZE is less than 16 bytes*THREADS"); \ @@ -145,10 +177,10 @@ class LLPrimitives { /* Restart from the same slot, only make sure sender waits for data to be reset */ \ step += NCCL_LL_CHUNKS; \ ACK_PREV; \ - while (sendHeadPtr[0] < step); \ - if (tid == 0) ring->send.conn.llLastCleaning = step; \ + while (LOAD(sendHeadPtr) < step); \ + { if (tid == 0) STORE(&ring->send.conn.llLastCleaning, step); }\ } \ - ring->send.conn.llStep = step; \ + STORE(&ring->send.conn.llStep, step); \ } while (0); #endif diff --git a/src/collectives/device/primitives.h b/src/collectives/device/primitives.h index e2baa4b301..1ef5ac802e 100644 --- a/src/collectives/device/primitives.h +++ b/src/collectives/device/primitives.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -31,10 +32,10 @@ class WaitFlag { volatile uint64_t * const flag; const int shift; public: - __device__ __forceinline__ + __device__ WaitFlag(volatile uint64_t * const flag, const int shift) : flag(flag), shift(shift) { } - __device__ __forceinline__ - void wait(uint64_t val) { while ((*flag + shift) < val) /*SPIN*/; } + __device__ + void wait(uint64_t val) { while ((LOAD(flag) + shift) < val) /*SPIN*/; } }; @@ -44,83 +45,83 @@ class PostFlag { volatile int * const fifo; const int fifo_size; public: - __device__ __forceinline__ + __device__ PostFlag(volatile uint64_t* const flag, const int shift, volatile int* const fifo, const int fifo_size) : flag(flag), shift(shift), fifo(fifo), fifo_size(fifo_size) { } - __device__ __forceinline__ - void post(uint64_t val) { *flag = (val - shift); } - __device__ __forceinline__ - void postSize(uint64_t step, int size) { if (fifo != NULL) fifo[step%fifo_size] = size; }; + __device__ + void post(uint64_t val) { STORE(flag, (val - shift)); } + __device__ + void postSize(uint64_t step, int size) { if (fifo != NULL) STORE(fifo + step%fifo_size, size); }; }; // Helper to check if any argument is of type T. // e.g. AnyAre(Flag1, Flag2, ...) -template __device__ __forceinline__ +template __device__ bool AnyAre() { return false; } template -__device__ __forceinline__ +__device__ bool AnyAre(FIRST_T first, TAIL_Ts... tail) { return std::is_same::value || AnyAre(tail...); } // Wait on all WaitFlags, ignore PostFlags -__device__ __forceinline__ -void WaitOnFlags(uint64_t val) { } +__device__ +static void WaitOnFlags(uint64_t val) { } -template __device__ __forceinline__ -void WaitOnFlags(uint64_t val, WaitFlag flag, TAIL_Ts... tail) { +template __device__ +static void WaitOnFlags(uint64_t val, WaitFlag flag, TAIL_Ts... tail) { flag.wait(val); WaitOnFlags(val, tail...); } -template __device__ __forceinline__ -void WaitOnFlags(uint64_t val, PostFlag, TAIL_Ts... tail) { +template __device__ +static void WaitOnFlags(uint64_t val, PostFlag, TAIL_Ts... tail) { WaitOnFlags(val, tail...); } // Post all PostFlags, ignore WaitFlags -__device__ __forceinline__ -void PostToFlags(uint64_t val) { } +__device__ +static void PostToFlags(uint64_t val) { } -template __device__ __forceinline__ -void PostToFlags(uint64_t val, WaitFlag flag, TAIL_Ts... tail) { +template __device__ +static void PostToFlags(uint64_t val, WaitFlag flag, TAIL_Ts... tail) { PostToFlags(val, tail...); } -template __device__ __forceinline__ -void PostToFlags(uint64_t val, PostFlag flag, TAIL_Ts... tail) { +template __device__ +static void PostToFlags(uint64_t val, PostFlag flag, TAIL_Ts... tail) { flag.post(val); PostToFlags(val, tail...); } // Post sizes for PostFlags, ignore WaitFlags -__device__ __forceinline__ -void PostSizeToFlags(uint64_t step, int size) { } +__device__ +static void PostSizeToFlags(uint64_t step, int size) { } -template __device__ __forceinline__ -void PostSizeToFlags(uint64_t step, int size, WaitFlag flag, TAIL_Ts... tail) { +template __device__ +static void PostSizeToFlags(uint64_t step, int size, WaitFlag flag, TAIL_Ts... tail) { PostSizeToFlags(step, size, tail...); } -template __device__ __forceinline__ -void PostSizeToFlags(uint64_t step, int size, PostFlag flag, TAIL_Ts... tail) { +template __device__ +static void PostSizeToFlags(uint64_t step, int size, PostFlag flag, TAIL_Ts... tail) { flag.postSize(step, size); PostSizeToFlags(step, size, tail...); } // Create pointer arithmetic syntax that doesn't break for std::nullptr_t -template __device__ __forceinline__ -Tptr ptradd(Tptr ptr, int i) { +template __device__ +static Tptr ptradd(Tptr ptr, int i) { return ptr + i; } -__device__ __forceinline__ -std::nullptr_t ptradd(std::nullptr_t ptr, int i) { +__device__ +static std::nullptr_t ptradd(std::nullptr_t ptr, int i) { return nullptr; } @@ -132,7 +133,7 @@ class Primitives { template // either WaitFunc or PostFunc - static __device__ __forceinline__ void + static __device__ __attribute__((noinline)) void GenericOp(const int tid, const int nthreads, const T* src1, const SRC2_T src2, @@ -160,7 +161,11 @@ class Primitives { if (tid == 0) { WaitOnFlags(SUBSTEPS*step + sub + 1, flags...); } +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + __syncthreads(); +#else asm volatile ("bar.sync 1, %0;" :: "r"(nthreads)); +#endif } ReduceOrCopy < @@ -180,13 +185,11 @@ class Primitives { ); if (AnyAre(flags...)) { __syncthreads(); - } - } else { - if (AnyAre(flags...)) { - __syncthreads(); - PostSizeToFlags(SUBSTEPS*step+sub, realSize*sizeof(T), flags...); + if(tid == 0) + PostSizeToFlags(SUBSTEPS*step+sub, realSize*sizeof(T), flags...); __threadfence_system(); - PostToFlags(SUBSTEPS*step + sub + 1, flags...); + if(tid == 0) + PostToFlags(SUBSTEPS*step + sub + 1, flags...); } } sliceOffset += sliceSize; @@ -195,28 +198,28 @@ class Primitives { public: template - static __device__ __forceinline__ void + static __device__ void Copy(const int tid, const int nthreads, const T* src, T* dst, int len, int maxOffset, uint64_t step, SYNC_Ts... flags) { GenericOp(tid, nthreads, src, nullptr, dst, nullptr, len, maxOffset, step, flags...); } template - static __device__ __forceinline__ void + static __device__ void DoubleCopy(const int tid, const int nthreads, const T* src, T* dst1, T* dst2, int len, int maxOffset, uint64_t step, SYNC_Ts... flags) { GenericOp(tid, nthreads, src, nullptr, dst1, dst2, len, maxOffset, step, flags...); } template - static __device__ __forceinline__ void + static __device__ void Reduce(const int tid, const int nthreads, const T* src1, const T* src2, T* dst, int len, int maxOffset, uint64_t step, SYNC_Ts... flags) { GenericOp(tid, nthreads, src1, src2, dst, nullptr, len, maxOffset, step, flags...); } template - static __device__ __forceinline__ void + static __device__ void ReduceCopy(const int tid, const int nthreads, const T* src1, const T* src2, T* dst1, T* dst2, int len, int maxOffset, uint64_t step, SYNC_Ts... flags) { GenericOp(tid, nthreads, src1, src2, dst1, dst2, len, maxOffset, step, flags...); diff --git a/src/collectives/device/reduce.h b/src/collectives/device/reduce.h index f5694b1456..91eb83f60d 100644 --- a/src/collectives/device/reduce.h +++ b/src/collectives/device/reduce.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -15,9 +16,10 @@ if (boffset == buffSize) boffset = 0; template +__attribute__((noinline)) __device__ void ncclReduceKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; - const int nthreads = blockDim.x - 1; + const int nthreads = blockDim.x; const int bid = args->bid; struct ncclComm* comm = args->comm; struct ncclRing* ring = comm->rings+blockIdx.x; @@ -40,7 +42,7 @@ __device__ void ncclReduceKernel(struct CollectiveArgs* args) { if (tid == 0) { // Update in case we skipped some collectives - *ring->recv.conn.opCount = args->opCount; + STORE(ring->recv.conn.opCount, args->opCount); if (rank != root) { // Wait for next to be ready @@ -98,11 +100,11 @@ __device__ void ncclReduceKernel(struct CollectiveArgs* args) { if (rank != root) { // Wait for next to have consumed data before resetting the flag waitDoneFromNext.wait(REDUCE_SUBSTEPS*(step + REDUCE_BUFCHUNKS - 1)); - *ring->send.conn.head = 0ULL; + STORE(ring->send.conn.head, 0ULL); } - *ring->recv.conn.tail = 0ULL; + STORE(ring->recv.conn.tail, 0ULL); __threadfence_system(); - *ring->recv.conn.opCount = args->opCount+1; + STORE(ring->recv.conn.opCount, args->opCount+1); } } @@ -115,6 +117,7 @@ __device__ void ncclReduceKernel(struct CollectiveArgs* args) { step++; template +__attribute__((noinline)) __device__ void ncclReduceLLKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; const int bid = args->bid; diff --git a/src/collectives/device/reduce_0.cpp b/src/collectives/device/reduce_0.cpp new file mode 100644 index 0000000000..f1b83bc655 --- /dev/null +++ b/src/collectives/device/reduce_0.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 0 +#include "device/reduce.cu" diff --git a/src/collectives/device/reduce_1.cpp b/src/collectives/device/reduce_1.cpp new file mode 100644 index 0000000000..63b157075e --- /dev/null +++ b/src/collectives/device/reduce_1.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 1 +#include "device/reduce.cu" diff --git a/src/collectives/device/reduce_2.cpp b/src/collectives/device/reduce_2.cpp new file mode 100644 index 0000000000..7c84b0ada3 --- /dev/null +++ b/src/collectives/device/reduce_2.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 2 +#include "device/reduce.cu" diff --git a/src/collectives/device/reduce_3.cpp b/src/collectives/device/reduce_3.cpp new file mode 100644 index 0000000000..c590bdd3c6 --- /dev/null +++ b/src/collectives/device/reduce_3.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 3 +#include "device/reduce.cu" diff --git a/src/collectives/device/reduce_kernel.h b/src/collectives/device/reduce_kernel.h index 0cb8f139f7..86e0f56a12 100644 --- a/src/collectives/device/reduce_kernel.h +++ b/src/collectives/device/reduce_kernel.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -18,6 +19,123 @@ struct FuncNull { } }; +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + +//we really don't need any specializations and we don't need +//to break things into uint32_t +template +__device__ inline T ncclMinFunc(T x, T y) { return y < x ? y : x; } + +template +__device__ inline T ncclMaxFunc(T x, T y) { return y < x ? x : y; } + +template +class FuncBase { +protected: + static constexpr auto n = sizeof(PackType) / sizeof(T); + + union Cvt { + using Vec = T __attribute__((ext_vector_type(n))); + + PackType data; + Vec vec; + + static_assert(sizeof(Vec) == sizeof(data), "Vec must be the same size of data."); + }; +}; + +template<> +class FuncBase { +protected: + static constexpr auto n = sizeof(PackType) / sizeof(_Float16); + union Cvt { + using Vec = _Float16 __attribute__((ext_vector_type(n))); + + PackType data; + Vec vec; + + static_assert(sizeof(Vec) == sizeof(data), "Vec must be the same size of data."); + }; +}; + +template +struct FuncSum : private FuncBase { + __device__ PackType operator()(PackType x, PackType y) const + { + using Cvt = typename FuncBase::Cvt; + + Cvt tmp_x{x}; + tmp_x.vec += Cvt{y}.vec; + + return tmp_x.data; + } + template{}>* = nullptr> + __device__ T operator()(const T x, const T y) const { + return x + y; + } +}; + +template +struct FuncProd : private FuncBase { + __device__ PackType operator()(PackType x, PackType y) const + { + using Cvt = typename FuncBase::Cvt; + + Cvt tmp_x{x}; + tmp_x.vec *= Cvt{y}.vec; + + return tmp_x.data; + } + template{}>* = nullptr> + __device__ T operator()(const T x, const T y) const { + return x * y; + } +}; + +template +struct FuncMax : private FuncBase { + __device__ PackType operator()(PackType x, PackType y) const + { + using Cvt = typename FuncBase::Cvt; + + Cvt tmp_x{x}; + Cvt tmp_y{y}; + + for (auto i = 0u; i != FuncBase::n; ++i) { + tmp_x.vec[i] = ncclMaxFunc(tmp_x.vec[i], tmp_y.vec[i]); + } + + return tmp_x.data; + } + template{}>* = nullptr> + __device__ T operator()(const T x, const T y) const { + return (x < y) ? y : x; + } +}; + +template +struct FuncMin : private FuncBase { + __device__ PackType operator()(PackType x, PackType y) const + { + using Cvt = typename FuncBase::Cvt; + + Cvt tmp_x{x}; + Cvt tmp_y{y}; + + for (auto i = 0u; i != FuncBase::n; ++i) { + tmp_x.vec[i] = ncclMinFunc(tmp_x.vec[i], tmp_y.vec[i]); + } + + return tmp_x.data; + } + template{}>* = nullptr> + __device__ T operator()(const T x, const T y) const { + return (x < y) ? x : y; + } +}; + +#else + template struct FuncSum { __device__ T operator()(const T x, const T y) const { @@ -361,4 +479,7 @@ struct FuncMin { return __float2half(fm); } }; + +#endif // defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + #endif // REDUCE_KERNEL_H_ diff --git a/src/collectives/device/reduce_scatter.cu b/src/collectives/device/reduce_scatter.cu index b16053c41b..efff65deba 100644 --- a/src/collectives/device/reduce_scatter.cu +++ b/src/collectives/device/reduce_scatter.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ diff --git a/src/collectives/device/reduce_scatter.h b/src/collectives/device/reduce_scatter.h index cad011b22e..aad151211d 100644 --- a/src/collectives/device/reduce_scatter.h +++ b/src/collectives/device/reduce_scatter.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -16,9 +17,10 @@ if (noffset == buffSize) noffset = 0; template +__attribute__((noinline)) __device__ void ncclReduceScatterKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; - const int nthreads = blockDim.x - 1; + const int nthreads = blockDim.x; const int bid = args->bid; struct ncclComm* comm = args->comm; struct ncclRing* ring = comm->rings+blockIdx.x; @@ -38,7 +40,7 @@ __device__ void ncclReduceScatterKernel(struct CollectiveArgs* args) { if (tid == 0) { // Update in case we skipped some collectives - *ring->recv.conn.opCount = args->opCount; + STORE(ring->recv.conn.opCount, args->opCount); // Wait for next to be ready WaitFlag waitOpCountNext(ring->send.conn.opCount, 0); waitOpCountNext.wait(args->opCount); @@ -112,10 +114,10 @@ __device__ void ncclReduceScatterKernel(struct CollectiveArgs* args) { if (tid == 0) { waitDoneFromNext.wait(REDUCESCATTER_SUBSTEPS*(step + REDUCESCATTER_BUFCHUNKS)); - *ring->send.conn.head = 0ULL; - *ring->recv.conn.tail = 0ULL; + STORE(ring->send.conn.head, 0ULL); + STORE(ring->recv.conn.tail, 0ULL); __threadfence_system(); - *ring->recv.conn.opCount = args->opCount+1; + STORE(ring->recv.conn.opCount, args->opCount+1); } } @@ -130,6 +132,7 @@ __device__ void ncclReduceScatterKernel(struct CollectiveArgs* args) { step++; template +__attribute__((noinline)) __device__ void ncclReduceScatterLLKernel(struct CollectiveArgs* args) { const int tid = threadIdx.x; const int bid = args->bid; diff --git a/src/collectives/device/reduce_scatter_0.cpp b/src/collectives/device/reduce_scatter_0.cpp new file mode 100644 index 0000000000..936f164605 --- /dev/null +++ b/src/collectives/device/reduce_scatter_0.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 0 +#include "device/reduce_scatter.cu" diff --git a/src/collectives/device/reduce_scatter_1.cpp b/src/collectives/device/reduce_scatter_1.cpp new file mode 100644 index 0000000000..3dbd2466d7 --- /dev/null +++ b/src/collectives/device/reduce_scatter_1.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 1 +#include "device/reduce_scatter.cu" diff --git a/src/collectives/device/reduce_scatter_2.cpp b/src/collectives/device/reduce_scatter_2.cpp new file mode 100644 index 0000000000..7302f55739 --- /dev/null +++ b/src/collectives/device/reduce_scatter_2.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 2 +#include "device/reduce_scatter.cu" diff --git a/src/collectives/device/reduce_scatter_3.cpp b/src/collectives/device/reduce_scatter_3.cpp new file mode 100644 index 0000000000..95a2fc93b7 --- /dev/null +++ b/src/collectives/device/reduce_scatter_3.cpp @@ -0,0 +1,8 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#define NCCL_OP 3 +#include "device/reduce_scatter.cu" diff --git a/src/collectives/reduce.cu b/src/collectives/reduce.cu index d8fde80baa..89dc804b7f 100644 --- a/src/collectives/reduce.cu +++ b/src/collectives/reduce.cu @@ -10,12 +10,12 @@ #include "collectives.h" ncclResult_t ncclReduceFunc(const void* sendbuff, void* recvbuff, const size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { size_t nbytes = count*ncclTypeSize(datatype); INFO(NCCL_COLL,"Reduce: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p", comm->opCount, sendbuff, recvbuff, count, datatype, op, root, comm, comm->nRanks, stream); if (comm->nRanks == 1) { if (sendbuff != recvbuff) - CUDACHECK(cudaMemcpyAsync(recvbuff, sendbuff, nbytes, cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(hipMemcpyAsync(recvbuff, sendbuff, nbytes, hipMemcpyDeviceToDevice, stream)); } else { NCCLCHECK(transportSaveProxies(REDUCE_SUBSTEPS, REDUCE_BUFCHUNKS, 1, 1, nbytes, proxyPatternTo(root), comm)); NCCLCHECK(saveKernel(ncclCollReduce, sendbuff, recvbuff, count, datatype, op, root, comm, stream, nbytes, 1)); @@ -25,9 +25,9 @@ ncclResult_t ncclReduceFunc(const void* sendbuff, void* recvbuff, const size_t c } NCCL_API(ncclResult_t, ncclReduce, const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream); ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { return ncclEnqueueCheck(ncclReduceFunc, "Reduce", sendbuff, recvbuff, count, datatype, op, root, comm, stream); } diff --git a/src/collectives/reduce_scatter.cu b/src/collectives/reduce_scatter.cu index 1447d4a91b..f73d50948d 100644 --- a/src/collectives/reduce_scatter.cu +++ b/src/collectives/reduce_scatter.cu @@ -10,12 +10,12 @@ #include "collectives.h" ncclResult_t ncclReduceScatterFunc(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream) { size_t nbytes = count*ncclTypeSize(datatype); INFO(NCCL_COLL,"ReduceScatter: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p", comm->opCount, sendbuff, recvbuff, count, datatype, op, root, comm, comm->nRanks, stream); if (comm->nRanks == 1) { if (sendbuff != recvbuff) - CUDACHECK(cudaMemcpyAsync(recvbuff, sendbuff, nbytes, cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(hipMemcpyAsync(recvbuff, sendbuff, nbytes, hipMemcpyDeviceToDevice, stream)); } else { NCCLCHECK(transportSaveProxies(REDUCESCATTER_SUBSTEPS, REDUCESCATTER_BUFCHUNKS, comm->nRanks-1, comm->nRanks, nbytes*comm->nRanks, proxyPatternRing, comm)); NCCLCHECK(saveKernel(ncclCollReduceScatter, sendbuff, recvbuff, count, datatype, op, root, comm, stream, nbytes*comm->nRanks, 1)); @@ -24,9 +24,9 @@ ncclResult_t ncclReduceScatterFunc(const void* sendbuff, void* recvbuff, size_t } NCCL_API(ncclResult_t, ncclReduceScatter, const void* sendbuff, void* recvbuff, size_t recvcount, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream); + ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, hipStream_t stream); ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, cudaStream_t stream) { + ncclDataType_t datatype, ncclRedOp_t op, ncclComm* comm, hipStream_t stream) { return ncclEnqueueCheck(ncclReduceScatterFunc, "ReduceScatter", sendbuff, recvbuff, recvcount, datatype, op, 0, comm, stream); } diff --git a/src/include/common_coll.h b/src/include/common_coll.h index 3ec7354f5d..be9aa0023f 100644 --- a/src/include/common_coll.h +++ b/src/include/common_coll.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -12,16 +13,16 @@ #include "collectives/collectives.h" static ncclResult_t PointerCheck(const void* pointer, struct ncclComm* comm, const char* ptrname, const char* opname) { - cudaPointerAttributes attr; - cudaError_t err = cudaPointerGetAttributes(&attr, pointer); - if (err != cudaSuccess || attr.devicePointer == NULL) { + hipPointerAttribute_t attr; + hipError_t err = hipPointerGetAttributes(&attr, pointer); + if (err != hipSuccess || attr.devicePointer == NULL) { WARN("%s : %s is not a valid pointer", opname, ptrname); return ncclInvalidArgument; } #if CUDART_VERSION >= 10000 - if (attr.type == cudaMemoryTypeDevice && attr.device != comm->cudaDev) { + if (attr.type == hipMemoryTypeDevice && attr.device != comm->cudaDev) { #else - if (attr.memoryType == cudaMemoryTypeDevice && attr.device != comm->cudaDev) { + if (attr.memoryType == hipMemoryTypeDevice && attr.device != comm->cudaDev) { #endif WARN("%s : %s allocated on device %d mismatchs with NCCL device %d", opname, ptrname, attr.device, comm->cudaDev); return ncclInvalidArgument; @@ -99,7 +100,7 @@ static inline void ncclGetCollResource(ncclComm_t comm, size_t nbytes, int* nrin int llEnforced = 0; /* see if the size falls in the NCCL_LL_THRESHOLD range set by user */ if (comm->llThreshold >= 0) { /* user sets total LL threshold */ if (nbytes > comm->llThreshold) { /* non-LL */ - *nthreads = comm->nThreads+1; + *nthreads = comm->nThreads; *nrings = comm->nRings; return; } else { @@ -129,12 +130,12 @@ static inline void ncclGetCollResource(ncclComm_t comm, size_t nbytes, int* nrin nr = DIVUP(nbytes, (NCCL_LL_RING_THRESHOLD*ll_max_nthreads*comm->nRanks)); /* else we try the max number of LL threads */ nr = nr == 0 ? 1 : nr > comm->nRings ? comm->nRings : nr; *ll = nbytes > comm->nRanks*nr*ll_max_nthreads*comm->threadThreshold ? llEnforced : 1; - *nthreads = *ll ? ll_max_nthreads : comm->nThreads+1; + *nthreads = *ll ? ll_max_nthreads : comm->nThreads; *nrings = *ll ? (int)nr : comm->nRings; } static ncclResult_t saveKernel(int coll, const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t dtype, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream, size_t nbytes, int loopFactor) { + ncclDataType_t dtype, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream, size_t nbytes, int loopFactor) { int llMode, nBlocks, nThreads; ncclGetCollResource(comm, nbytes, &nBlocks, &nThreads, &llMode); comm->myParams->blockDim.x = std::max((int)comm->myParams->blockDim.x, nThreads); @@ -164,7 +165,7 @@ static ncclResult_t saveKernel(int coll, const void* sendbuff, void* recvbuff, s int opIndex = ring->collFifoTail; struct ncclColl* c = ring->collectives+opIndex; volatile uint8_t* activePtr = (volatile uint8_t*)&c->active; - while (activePtr[0] != 0) sched_yield(); + while (LOAD(activePtr) != 0) sched_yield(); struct CollectiveArgs* args = &c->args; args->root = root; @@ -180,7 +181,7 @@ static ncclResult_t saveKernel(int coll, const void* sendbuff, void* recvbuff, s c->nThreads = nThreads; c->funcIndex = FUNC_INDEX(coll, op, dtype, llMode); - c->active = 1; + STORE(&c->active, 1); opIndex = (opIndex+1)%NCCL_MAX_OPS; c->nextIndex = opIndex; ring->collFifoTail = opIndex; diff --git a/src/include/core.h b/src/include/core.h index 8285df5d45..1e5950553b 100644 --- a/src/include/core.h +++ b/src/include/core.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -16,16 +17,17 @@ #include // std::min/std::max #include #include -#include +#include +#include #if CUDART_VERSION < 9000 struct cudaLaunchParams { - void *func; + void (*func)(struct ncclColl); dim3 gridDim; dim3 blockDim; - void **args; + struct ncclColl **args; size_t sharedMem; - cudaStream_t stream; + hipStream_t stream; }; #endif @@ -35,10 +37,10 @@ struct cudaLaunchParams { // Rings / LL tuning #define NCCL_LL_RING_THRESHOLD 8 // Per thread size before we start increasing nrings -#define NCCL_THREAD_THRESHOLD 64 // Per thread size before we switch to non-LL for Volta and above +#define NCCL_THREAD_THRESHOLD 256 // Per thread size before we switch to non-LL for Volta and above #define NCCL_THREAD_THRESHOLD_PREVOLTA 32 // Per thread size before we switch to non-LL for pre-Volta archs #define NCCL_LL_MAX_NTHREADS 256 -#define NCCL_LL_MIN_NTHREADS 64 +#define NCCL_LL_MIN_NTHREADS 256 #define DIVUP(x, y) \ (((x)+(y)-1)/(y)) @@ -90,9 +92,9 @@ struct ncclConnector { struct ncclConnInfo conn; }; -#define CACHE_LINE_SIZE 128 +#define CACHE_LINE_SIZE 64 #define MEM_ALIGN 4096 -#define SIZES_FIFO_SIZE 32 +#define SIZES_FIFO_SIZE 16 #define CUDA_IPC_MIN 2097152UL /* 2MiB - not currently used */ #define NCCL_LL_CHUNKS 8 @@ -164,6 +166,8 @@ struct ncclRing { }; static_assert(sizeof(struct ncclRing) == 0x80*sizeof(int), "ncclRing must have a pow2 size"); +#pragma pack(push) /* push current alignment to stack */ +#pragma pack(4) /* set alignment to 4 bytes boundary */ /* CollectiveArgs + ncclColl are to be a power of two, currently 64 bytes, */ /* to make sure reads to host from the CUDA kernel are aligned. */ /* Make sure to adjust padding at the end of ncclColl. */ @@ -197,6 +201,7 @@ struct ncclColl { }; }; static_assert(sizeof(struct ncclColl) == (0x10*sizeof(int)), "ncclColl must have a pow2 size"); +#pragma pack(pop) /* restore original alignment from stack */ struct ncclComm { struct ncclRing rings[MAXRINGS]; @@ -206,9 +211,9 @@ struct ncclComm { int cudaDev; // my cuda device index enum { GROUP, PARALLEL } launchMode; - cudaStream_t userStream; + hipStream_t userStream; bool userStreamSet; - cudaEvent_t doneEvent; + hipEvent_t doneEvent; bool checkPointers; // Counter to make sure collectives match (needed for bcast/reduce @@ -225,7 +230,7 @@ struct ncclComm { // An internal CUDA stream for NCCL kernel CGMD launches int groupCudaStream; - cudaStream_t groupStream; + hipStream_t groupStream; // Device copy of the communicator struct ncclComm *devComm; @@ -243,22 +248,31 @@ struct ncclComm { int* intraCGMode; // Whether we can use CUDA9 CGMD or not int* intraCC; // Only to check all have the same ComputeCap and disable CGMode if not struct ncclColl args; - void* argsptr; + struct ncclColl* argsptr; }; +// Convert volatile access to atomic +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#define LOAD(VAR) __atomic_load_n((VAR), __ATOMIC_SEQ_CST) +#define STORE(DST, SRC) __atomic_store_n((DST), (SRC), __ATOMIC_SEQ_CST) +#else +#define LOAD(VAR) *(VAR) +#define STORE(DST, SRC) *(DST) = (SRC) +#endif + // Check CUDA calls #define CUDACHECK(cmd) do { \ - cudaError_t e = cmd; \ - if( e != cudaSuccess ) { \ - WARN("Cuda failure '%s'", cudaGetErrorString(e)); \ + hipError_t e = cmd; \ + if( e != hipSuccess ) { \ + WARN("Cuda failure '%s'", hipGetErrorString(e)); \ return ncclUnhandledCudaError; \ } \ } while(false) #define CUDACHECKGOTO(cmd, res, label) do { \ - cudaError_t e = cmd; \ - if( e != cudaSuccess ) { \ - WARN("Cuda failure '%s'", cudaGetErrorString(e)); \ + hipError_t e = cmd; \ + if( e != hipSuccess ) { \ + WARN("Cuda failure '%s'", hipGetErrorString(e)); \ res = ncclUnhandledCudaError; \ goto label; \ } \ @@ -327,14 +341,14 @@ int ncclCudaCompCap(); #include static inline ncclResult_t ncclCudaHostAlloc(void** ptr, void** devPtr, size_t size) { - CUDACHECK(cudaHostAlloc(ptr, size, cudaHostAllocMapped)); + CUDACHECK(hipHostMalloc(ptr, size, hipHostMallocMapped)); memset(*ptr, 0, size); *devPtr = *ptr; return ncclSuccess; } static inline ncclResult_t ncclCudaHostFree(void* ptr) { - CUDACHECK(cudaFreeHost(ptr)); + CUDACHECK(hipHostFree(ptr)); return ncclSuccess; } @@ -351,15 +365,18 @@ static ncclResult_t ncclCalloc(T** ptr, size_t nelem) { } template -static ncclResult_t ncclCudaCalloc(T** ptr, size_t nelem) { - CUDACHECK(cudaMalloc(ptr, nelem*sizeof(T))); - CUDACHECK(cudaMemset(*ptr, 0, nelem*sizeof(T))); +static ncclResult_t ncclCudaCalloc(T** ptr, size_t nelem, bool isFineGrain = false) { + if (isFineGrain) + CUDACHECK(hipExtMallocWithFlags((void**)ptr, nelem*sizeof(T), hipDeviceMallocFinegrained)); + else + CUDACHECK(hipMalloc(ptr, nelem*sizeof(T))); + CUDACHECK(hipMemset(*ptr, 0, nelem*sizeof(T))); return ncclSuccess; } template static ncclResult_t ncclCudaMemcpy(T* dst, T* src, size_t nelem) { - CUDACHECK(cudaMemcpy(dst, src, nelem*sizeof(T), cudaMemcpyDefault)); + CUDACHECK(hipMemcpy(dst, src, nelem*sizeof(T), hipMemcpyDefault)); return ncclSuccess; } diff --git a/src/include/debug.h b/src/include/debug.h index 55dee1838c..1ef87d9f6a 100644 --- a/src/include/debug.h +++ b/src/include/debug.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ diff --git a/src/include/enqueue.h b/src/include/enqueue.h index 69d0463d99..f17639826e 100644 --- a/src/include/enqueue.h +++ b/src/include/enqueue.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -11,11 +12,11 @@ #include "group.h" typedef ncclResult_t(*ncclFunc_t)(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t type, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t type, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream); ncclResult_t ncclEnqueueCheck(ncclFunc_t func, const char* primName, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t type, ncclRedOp_t op, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); ncclResult_t ncclCpuBarrierIn(ncclComm_t comm, int* isLast); ncclResult_t ncclCpuBarrierLast(ncclComm_t comm); ncclResult_t ncclCpuBarrierOut(ncclComm_t comm); diff --git a/src/include/group.h b/src/include/group.h index 76da30f8c0..183e823516 100644 --- a/src/include/group.h +++ b/src/include/group.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2017, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -18,7 +19,7 @@ typedef ncclResult_t(*ncclInitFunc_t)(ncclComm_t* newcomm, int ndev, ncclUniqueI ncclResult_t ncclAsyncInit(ncclInitFunc_t func, int cudaDev, ncclComm_t* newcomm, int ndev, ncclUniqueId commId, int myrank); typedef ncclResult_t(*ncclCollFunc_t)(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t type, ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t type, ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream); ncclResult_t ncclAsyncColl(ncclComm_t comm); #endif diff --git a/src/include/nvlink.h b/src/include/nvlink.h index 7eb74c9f91..28976386bb 100644 --- a/src/include/nvlink.h +++ b/src/include/nvlink.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ diff --git a/src/include/nvlink_stub.h b/src/include/nvlink_stub.h new file mode 100644 index 0000000000..9ee176edf0 --- /dev/null +++ b/src/include/nvlink_stub.h @@ -0,0 +1,20 @@ +/************************************************************************* + * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef NCCL_NVLINK_H_ +#define NCCL_NVLINK_H_ + +#include "topo.h" + +#define CONNECT_NVLINK 0x10 +#define CONNECT_NVSWITCH 0x100 + +static int getNumNvlinks(const char* busId) { + return 0; +} + +#endif diff --git a/src/include/rings.h b/src/include/rings.h index 751846c63a..3b4c311102 100644 --- a/src/include/rings.h +++ b/src/include/rings.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -9,7 +10,11 @@ static int getDefaultThreads() { // On Kepler, rings are doubled later. +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + return 256; +#else return ncclCudaCompCap() == 3 ? 128 : 256; +#endif } ncclResult_t ncclGetRings(int* nrings, int* nthreads, int rank, int nranks, int* transports, ncclTvalue_t* values, int* prev, int* next); diff --git a/src/include/shm.h b/src/include/shm.h index 4fb49cbb8f..850ecae5ce 100644 --- a/src/include/shm.h +++ b/src/include/shm.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -39,14 +40,14 @@ static ncclResult_t shmOpen(const char* shmname, const int shmsize, void** shmPt ncclResult_t res = ncclSuccess; NCCLCHECKGOTO(shmSetup(shmname, shmsize, &fd, &ptr, create), res, sysError); - CUDACHECKGOTO(cudaHostRegister(ptr, shmsize, cudaHostRegisterMapped), res, cudaError); - CUDACHECKGOTO(cudaHostGetDevicePointer(devShmPtr, ptr, 0), res, cudaError); + CUDACHECKGOTO(hipHostRegister(ptr, shmsize, hipHostRegisterMapped), res, hipError_t); + CUDACHECKGOTO(hipHostGetDevicePointer(devShmPtr, ptr, 0), res, hipError_t); *shmPtr = ptr; return ncclSuccess; sysError: WARN("Error while %s shared memory segment %s (size %d)\n", create ? "creating" : "attaching to", shmname, shmsize); -cudaError: +hipError_t: if (fd != -1) close(fd); if (create) shm_unlink(shmname); if (ptr != MAP_FAILED) munmap(ptr, shmsize); @@ -60,7 +61,7 @@ static ncclResult_t shmUnlink(const char* shmname) { } static ncclResult_t shmClose(void* shmPtr, void* devShmPtr, const int shmsize) { - CUDACHECK(cudaHostUnregister(shmPtr)); + CUDACHECK(hipHostUnregister(shmPtr)); if (munmap(shmPtr, shmsize) != 0) { WARN("munmap of shared memory failed"); return ncclSystemError; diff --git a/src/include/topo.h b/src/include/topo.h index e824a81023..d14e38690e 100644 --- a/src/include/topo.h +++ b/src/include/topo.h @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -11,13 +12,41 @@ #include #include #include +#include +#include +#include #define BUSID_SIZE (sizeof("0000:00:00.0")) #define BUSID_REDUCED_SIZE (sizeof("0000:00")) +static bool isEPYC() { + std::ifstream cpuinfo("/proc/cpuinfo"); + std::string line; + int needed = 2; + static bool vendor_id = true, cpu_family = false, initialized = false; + if (initialized) return (vendor_id && cpu_family); + while (std::getline(cpuinfo, line)) { + if (line.compare(0, 9, "vendor_id") == 0) { + if(line.find("AuthenticAMD") == std::string::npos) + vendor_id = false; + needed --; + } + if (line.compare(0, 10, "cpu family") == 0) { + std::string family_str = line.substr(line.find(": ") + 2); + if (std::stoi(family_str) >= 23) + cpu_family = true; + needed --; + } + if (!needed) + break; + } + initialized = true; + return (vendor_id && cpu_family); +} + static ncclResult_t getCudaPath(int cudaDev, char** path) { char busId[BUSID_SIZE]; - CUDACHECK(cudaDeviceGetPCIBusId(busId, BUSID_SIZE, cudaDev)); + CUDACHECK(hipDeviceGetPCIBusId(busId, BUSID_SIZE, cudaDev)); for (int i=0; i #include #include #include @@ -24,7 +27,7 @@ #include #include #include -#include +#include #include #include #include @@ -58,17 +61,17 @@ ncclNet_t* ncclNet = NULL; #pragma weak ncclCudaCompCap int ncclCudaCompCap() { int cudaDev; - if (cudaGetDevice(&cudaDev) != cudaSuccess) return 0; + if (hipGetDevice(&cudaDev) != hipSuccess) return 0; int ccMajor; - if (cudaDeviceGetAttribute(&ccMajor, cudaDevAttrComputeCapabilityMajor, cudaDev) != cudaSuccess) return 0; + if (hipDeviceGetAttribute(&ccMajor, hipDeviceAttributeComputeCapabilityMajor, cudaDev) != hipSuccess) return 0; return ccMajor; } int ncclCudaFullCompCap() { int cudaDev; - if (cudaGetDevice(&cudaDev) != cudaSuccess) return 0; + if (hipGetDevice(&cudaDev) != hipSuccess) return 0; int ccMajor, ccMinor; - if (cudaDeviceGetAttribute(&ccMajor, cudaDevAttrComputeCapabilityMajor, cudaDev) != cudaSuccess) return 0; - if (cudaDeviceGetAttribute(&ccMinor, cudaDevAttrComputeCapabilityMinor, cudaDev) != cudaSuccess) return 0; + if (hipDeviceGetAttribute(&ccMajor, hipDeviceAttributeComputeCapabilityMajor, cudaDev) != hipSuccess) return 0; + if (hipDeviceGetAttribute(&ccMinor, hipDeviceAttributeComputeCapabilityMinor, cudaDev) != hipSuccess) return 0; return ccMajor*10+ccMinor; } @@ -144,6 +147,22 @@ int ncclThreadThreshold(int minCompCap, int multiNode) { return threshold; } +bool useFineGrainVramPcie = false; + +void parseHsaForceFineGrainVramPcie() { + char* str = getenv("HSA_FORCE_FINE_GRAIN_PCIE"); + if (str && strlen(str) > 0) { + errno = 0; + int64_t v = strtoll(str, NULL, 0); + if (errno || (v != 0 && v != 1)) { + INFO(NCCL_ALL,"Invalid value %s for %s, using default %u.", str, "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); \ + } else { + useFineGrainVramPcie = v; + INFO(NCCL_ALL,"%s set by environment to %u.", "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); \ + } + } +} + pthread_mutex_t initLock = PTHREAD_MUTEX_INITIALIZER; static bool initialized = false; static ncclResult_t ncclInit() { @@ -153,6 +172,8 @@ static ncclResult_t ncclInit() { initEnv(); initDebug(); initNet(); + // Check if HSA_FORCE_FINE_GRAIN_PCIE is set in env + parseHsaForceFineGrainVramPcie(); initialized = true; } pthread_mutex_unlock(&initLock); @@ -177,16 +198,16 @@ static ncclResult_t commFree(ncclComm_t comm) { if (comm == NULL) return ncclSuccess; - CUDACHECK(cudaFree(comm->devComm)); + CUDACHECK(hipFree(comm->devComm)); for (int ring=0; ringnRings; ring++) NCCLCHECK(freeRing(comm->rings+ring)); if (comm->doneEvent != NULL) - CUDACHECK(cudaEventDestroy(comm->doneEvent)); + CUDACHECK(hipEventDestroy(comm->doneEvent)); if (comm->launchMode == ncclComm::GROUP) { - CUDACHECK(cudaStreamDestroy(comm->groupStream)); + CUDACHECK(hipStreamDestroy(comm->groupStream)); } // Last rank frees shared resources between threads @@ -216,8 +237,8 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) { // Try to create a CUDA object right away. If there is something wrong with // the device we're on (failure cause #1) , better know it early. - cudaEvent_t doneEvent; - CUDACHECK(cudaEventCreateWithFlags(&doneEvent, cudaEventDisableTiming)); + hipEvent_t doneEvent; + CUDACHECK(hipEventCreateWithFlags(&doneEvent, hipEventDisableTiming)); struct ncclComm* comm; NCCLCHECK(ncclCalloc(&comm, 1)); @@ -225,7 +246,7 @@ static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) { INFO(NCCL_INIT,"comm %p rank %d nranks %d", comm, rank, ndev); comm->rank = rank; comm->nRanks = ndev; - cudaGetDevice(&comm->cudaDev); + hipGetDevice(&comm->cudaDev); comm->doneEvent = doneEvent; comm->llThreshold = ncclParamLlThreshold(); comm->checkPointers = ncclParamCheckPointers() == 1 ? true : false; @@ -255,7 +276,11 @@ static ncclResult_t devCommSetup(ncclComm_t comm) { } // Pre-process the string so that running "strings" on the lib can quickly reveal the version. +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#define VERSION_STRING "NCCL version " STR(NCCL_MAJOR) "." STR(NCCL_MINOR) "." STR(NCCL_PATCH) NCCL_SUFFIX "+hip" +#else #define VERSION_STRING "NCCL version " STR(NCCL_MAJOR) "." STR(NCCL_MINOR) "." STR(NCCL_PATCH) NCCL_SUFFIX "+cuda" STR(CUDA_MAJOR) "." STR(CUDA_MINOR) +#endif static void showVersion() { static int shown = 0; if (shown == 0 && ncclDebugLevel >= NCCL_LOG_VERSION) { @@ -416,8 +441,8 @@ static ncclResult_t buildRings(int nrings, int* rings, int rank, int nranks, int void* waitForNonNullPtr(void* p) { volatile void** ptr = (volatile void**) p; - while (*ptr == NULL) sched_yield(); - return (void*)*ptr; + while (LOAD(ptr) == NULL) sched_yield(); + return (void*)LOAD(ptr); } ncclResult_t initParams(struct ncclComm* comm) { @@ -472,11 +497,11 @@ ncclResult_t ncclCommSetIntra(struct ncclComm* comm, int rank, int ranks, struct comm->launchMode = ncclComm::PARALLEL; } if (comm->launchMode == ncclComm::GROUP) { - CUDACHECK(cudaStreamCreateWithFlags(&comm->groupStream, cudaStreamNonBlocking)); + CUDACHECK(hipStreamCreateWithFlags(&comm->groupStream, hipStreamNonBlocking)); #if CUDART_VERSION >= 9000 if (*comm->intraCC && (ncclCudaFullCompCap() == *comm->intraCC)) { // Check whether the GPU supports Cooperative Group Multi Device Launch - (void) cudaDeviceGetAttribute(&cgMdLaunch, cudaDevAttrCooperativeMultiDeviceLaunch, comm->cudaDev); + (void) hipDeviceGetAttribute(&cgMdLaunch, cudaDevAttrCooperativeMultiDeviceLaunch, comm->cudaDev); } #endif } @@ -614,14 +639,41 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm } bool SetCpuAffinity(int cudaDev, nvmlDevice_t* nvmlDevice) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + if (numa_available() < 0) { + WARN("System does not support NUMA API!"); + return false; + } + char* cudaPath; + NCCLCHECK(getCudaPath(cudaDev, &cudaPath)); + strcat(cudaPath, "/numa_node"); + int fd; + SYSCHECKVAL(open(cudaPath, O_RDONLY), "open", fd); + char numa_node[5]; + int len; + SYSCHECKVAL(read(fd, numa_node, 4), "read", len); + SYSCHECK(close(fd), "close"); + errno = 0; + long node = strtol(numa_node, NULL, 10); + if (errno == ERANGE || errno == EINVAL) { + INFO(NCCL_ALL,"%s: Call to strtol returned %s", __func__, strerror(errno)); + free(cudaPath); + return false; + } + numa_run_on_node(node); + numa_set_preferred(node); + free(cudaPath); + return true; +#else char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; - if (cudaDeviceGetPCIBusId(busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, cudaDev) != cudaSuccess) return false; + if (hipDeviceGetPCIBusId(busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, cudaDev) != hipSuccess) return false; if (wrapNvmlDeviceGetHandleByPciBusId(busId, nvmlDevice) != ncclSuccess) return false; if (wrapNvmlDeviceSetCpuAffinity(*nvmlDevice) != ncclSuccess) { WARN("Failed to set CPU affinity"); return false; } return true; +#endif } ncclResult_t ncclCommInitRankSync(ncclComm_t* newcomm, int nranks, ncclUniqueId commId, int myrank) { @@ -634,7 +686,7 @@ ncclResult_t ncclCommInitRankSync(ncclComm_t* newcomm, int nranks, ncclUniqueId // Make sure all host memory allocation are close to the GPU int cudaDev; nvmlDevice_t nvmlDevice; - CUDACHECK(cudaGetDevice(&cudaDev)); + CUDACHECK(hipGetDevice(&cudaDev)); SetCpuAffinity(cudaDev, &nvmlDevice); ncclResult_t res; @@ -667,7 +719,7 @@ ncclResult_t ncclCommInitRank(ncclComm_t* newcomm, int nranks, ncclUniqueId comm INFO(NCCL_INIT,"rank %d nranks %d", myrank, nranks); // Make sure the CUDA runtime is initialized. - CUDACHECK(cudaFree(NULL)); + CUDACHECK(hipFree(NULL)); NCCLCHECK(PtrCheck(newcomm, "CommInitRank", "newcomm")); if (nranks < 1 || myrank < 0 || myrank >= nranks) { @@ -677,7 +729,7 @@ ncclResult_t ncclCommInitRank(ncclComm_t* newcomm, int nranks, ncclUniqueId comm if (ncclAsyncMode()) { int cudaDev; - CUDACHECK(cudaGetDevice(&cudaDev)); + CUDACHECK(hipGetDevice(&cudaDev)); return ncclAsyncInit(ncclCommInitRankSync, cudaDev, newcomm, nranks, commId, myrank); } else { return ncclCommInitRankSync(newcomm, nranks, commId, myrank); @@ -688,7 +740,7 @@ static ncclResult_t initTransportsAll(struct ncclComm** comms, const int* devs, struct ncclInfo* allInfo; NCCLCHECK(ncclCalloc(&allInfo, nranks)); for (int rank=0; rankrings+r; NCCLCHECK(ring->send.transport->send.connect(connect+2*rank+1, &ring->send)); NCCLCHECK(ring->recv.transport->recv.connect(connect+2*rank+0, &ring->recv)); @@ -800,7 +852,7 @@ ncclResult_t ncclCommInitAll(ncclComm_t* comms, int ndev, const int* devlist) { ncclDevList[i] = devlist ? devlist[i] : i; } - cudaGetDevice(&savedDevice); + hipGetDevice(&savedDevice); for(rank=0; rankcudaDev; if (savedDevice != commDevice) { - CUDACHECK(cudaSetDevice(commDevice)); + CUDACHECK(hipSetDevice(commDevice)); } NCCLCHECK(commFree(comm)); if (savedDevice != commDevice) - CUDACHECK(cudaSetDevice(savedDevice)); + CUDACHECK(hipSetDevice(savedDevice)); return ncclSuccess; } diff --git a/src/misc/enqueue.cu b/src/misc/enqueue.cu index 80846dd656..be5daf3926 100644 --- a/src/misc/enqueue.cu +++ b/src/misc/enqueue.cu @@ -1,9 +1,12 @@ /************************************************************************* * Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ +#include + #include "enqueue.h" #include "common_coll.h" #include "param.h" @@ -11,30 +14,30 @@ #include "collectives/collectives.h" #define NCCL_FUNC4(coll, op, dtype) \ - (void*)NCCL_KERN_NAME(coll, op, dtype), \ - (void*)NCCL_KERN_NAME(coll##LL, op, dtype) + NCCL_KERN_NAME(coll, op, dtype), \ + NCCL_KERN_NAME(coll##LL, op, dtype) // Must be consistent with ncclDataType_t #define NCCL_FUNCS3A(coll, op) \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, u8), \ - (void*)NCCL_FUNC4(coll, op, i32), \ - (void*)NCCL_FUNC4(coll, op, u32), \ - (void*)NCCL_FUNC4(coll, op, i64), \ - (void*)NCCL_FUNC4(coll, op, u64), \ - (void*)NCCL_FUNC4(coll, op, f16), \ - (void*)NCCL_FUNC4(coll, op, f32), \ - (void*)NCCL_FUNC4(coll, op, f64) + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, u8), \ + NCCL_FUNC4(coll, op, i32), \ + NCCL_FUNC4(coll, op, u32), \ + NCCL_FUNC4(coll, op, i64), \ + NCCL_FUNC4(coll, op, u64), \ + NCCL_FUNC4(coll, op, f16), \ + NCCL_FUNC4(coll, op, f32), \ + NCCL_FUNC4(coll, op, f64) #define NCCL_FUNCS3B(coll, op) \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8), \ - (void*)NCCL_FUNC4(coll, op, i8) + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8), \ + NCCL_FUNC4(coll, op, i8) // Must be consistent with ncclRedOp_t #define NCCL_FUNCS2A(coll) \ @@ -48,8 +51,9 @@ NCCL_FUNCS3B(coll, copy), \ NCCL_FUNCS3B(coll, copy) +typedef void(*ncclKern_t)(struct ncclColl); // Must be consistent with the ncclFuncSet enum -static void* const ncclKerns[ncclCollCount*ncclNumOps*ncclNumTypes*2] = { +static ncclKern_t const ncclKerns[ncclCollCount*ncclNumOps*ncclNumTypes*2] = { NCCL_FUNCS2B(ncclBroadcast), NCCL_FUNCS2A(ncclReduce), NCCL_FUNCS2B(ncclAllGather), @@ -67,13 +71,13 @@ ncclResult_t ncclLaunchCooperativeKernelMultiDevice(struct cudaLaunchParams *par } #endif int savedDev; - CUDACHECK(cudaGetDevice(&savedDev)); + CUDACHECK(hipGetDevice(&savedDev)); for (int i = 0; i < numDevices; i++) { struct cudaLaunchParams* params = paramsList+i; - CUDACHECK(cudaSetDevice(cudaDevs[i])); - CUDACHECK(cudaLaunchKernel(params->func, params->gridDim, params->blockDim, params->args, params->sharedMem, params->stream)); + CUDACHECK(hipSetDevice(cudaDevs[i])); + hipLaunchKernelGGL(params->func, params->gridDim, params->blockDim, params->sharedMem, params->stream, **params->args); } - CUDACHECK(cudaSetDevice(savedDev)); + CUDACHECK(hipSetDevice(savedDev)); return ncclSuccess; } @@ -83,7 +87,7 @@ ncclResult_t setupLaunch(struct ncclComm* comm, struct cudaLaunchParams* params) // Set active = 2 for the last operation for (int r=0; rgridDim.x; r++) { struct ncclRing* ring = comm->rings+r; - ring->collectives[(ring->collStart+ring->collCount-1)%NCCL_MAX_OPS].active = 2; + STORE(&ring->collectives[(ring->collStart+ring->collCount-1)%NCCL_MAX_OPS].active, 2); } // Find the first operation, choose the kernel accordingly and pass it @@ -91,7 +95,7 @@ ncclResult_t setupLaunch(struct ncclComm* comm, struct cudaLaunchParams* params) struct ncclColl* coll = comm->rings[0].collectives+comm->rings[0].collStart; memcpy(&comm->args, coll, sizeof(struct ncclColl)); // As we pass that coll directly, we can free it immediately. - coll->active = 0; + STORE(&coll->active, 0); params->func = ncclKerns[coll->funcIndex]; return ncclSuccess; @@ -99,7 +103,7 @@ ncclResult_t setupLaunch(struct ncclComm* comm, struct cudaLaunchParams* params) ncclResult_t ncclCpuBarrierIn(struct ncclComm* comm, int* isLast) { volatile int* ptr = (volatile int*)(comm->intraBarrier+comm->intraPhase); - int val = *ptr; + int val = LOAD(ptr); bool done = false; while (done == false) { if (val >= comm->intraRanks) { @@ -121,7 +125,7 @@ ncclResult_t ncclCpuBarrierIn(struct ncclComm* comm, int* isLast) { ncclResult_t ncclCpuBarrierLast(struct ncclComm* comm) { volatile int* ptr = (volatile int*)(comm->intraBarrier+comm->intraPhase); - int val = *ptr; + int val = LOAD(ptr); if (__sync_bool_compare_and_swap(ptr, val, val+1) != true) { WARN("Trying to launch too many collectives"); return ncclInternalError; @@ -131,7 +135,7 @@ ncclResult_t ncclCpuBarrierLast(struct ncclComm* comm) { ncclResult_t ncclCpuBarrierOut(struct ncclComm* comm) { volatile int* ptr = (volatile int*)(comm->intraBarrier+comm->intraPhase); - while (*ptr < comm->intraRanks) pthread_yield(); + while (LOAD(ptr) < comm->intraRanks) pthread_yield(); comm->intraPhase ^= 1; return ncclSuccess; } @@ -145,14 +149,14 @@ ncclResult_t ncclBarrierEnqueue(struct ncclComm* comm) { // Use internal NCCL stream for CGMD/GROUP launch if required or if the user stream is NULL if (comm->launchMode == ncclComm::GROUP && (comm->groupCudaStream || comm->userStream == NULL)) { // Enqueue event in user stream - CUDACHECK(cudaEventRecord(comm->doneEvent, comm->userStream)); + CUDACHECK(hipEventRecord(comm->doneEvent, comm->userStream)); // Create dependency between user stream and internal NCCL stream - CUDACHECK(cudaStreamWaitEvent(comm->groupStream, comm->doneEvent, 0)); + CUDACHECK(hipStreamWaitEvent(comm->groupStream, comm->doneEvent, 0)); params->stream = comm->groupStream; } else { if (comm->userStream != params->stream) { // Stream changed from last call, create dependency against last NCCL kernel launch - CUDACHECK(cudaStreamWaitEvent(comm->userStream, comm->doneEvent, 0)); + CUDACHECK(hipStreamWaitEvent(comm->userStream, comm->doneEvent, 0)); } params->stream = comm->userStream; } @@ -185,10 +189,10 @@ ncclResult_t ncclBarrierEnqueueWait(ncclComm_t comm) { struct cudaLaunchParams *params = comm->myParams; if (comm->launchMode == ncclComm::PARALLEL) { - CUDACHECK(cudaLaunchKernel(params->func, params->gridDim, params->blockDim, params->args, params->sharedMem, params->stream)); + hipLaunchKernelGGL(params->func, params->gridDim, params->blockDim, params->sharedMem, params->stream, **params->args); } // Start the network proxies as soon as the kernel has been launched. We can't - // perform any CUDA call between the two or having a cudaFree between the CUDA + // perform any CUDA call between the two or having a hipFree between the CUDA // launch and the transportStartProxies call could cause a deadlock. // Also, starting the proxies after the CUDA launch seems to be better for // performance (latency). @@ -205,11 +209,11 @@ ncclResult_t ncclBarrierEnqueueWait(ncclComm_t comm) { ncclResult_t ncclEnqueueEvents(ncclComm_t comm) { struct cudaLaunchParams *params = comm->myParams; // Enqueue event after NCCL kernel - CUDACHECK(cudaEventRecord(comm->doneEvent, params->stream)); + CUDACHECK(hipEventRecord(comm->doneEvent, params->stream)); // Use internal NCCL stream for CGMD/GROUP launch if required or if the user stream is NULL if (comm->launchMode == ncclComm::GROUP && (comm->groupCudaStream || comm->userStream == NULL)) { // Create dependency between NCCL internal stream and user stream - CUDACHECK(cudaStreamWaitEvent(comm->userStream, comm->doneEvent, 0)); + CUDACHECK(hipStreamWaitEvent(comm->userStream, comm->doneEvent, 0)); } comm->userStreamSet = false; return ncclSuccess; @@ -217,15 +221,15 @@ ncclResult_t ncclEnqueueEvents(ncclComm_t comm) { ncclResult_t ncclEnqueueCheck(ncclFunc_t func, const char* primName, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t type, ncclRedOp_t op, int root, - ncclComm_t comm, cudaStream_t stream) { + ncclComm_t comm, hipStream_t stream) { if (comm == NULL) return ncclInvalidArgument; // Launch asynchronously if needed if (ncclAsyncMode()) { ncclResult_t ret = ncclSuccess; int savedDev = -1; if (comm->checkPointers) { - CUDACHECKGOTO(cudaGetDevice(&savedDev), ret, end); - CUDACHECKGOTO(cudaSetDevice(comm->cudaDev), ret, end); + CUDACHECKGOTO(hipGetDevice(&savedDev), ret, end); + CUDACHECKGOTO(hipSetDevice(comm->cudaDev), ret, end); } // Check arguments NCCLCHECKGOTO(ArgsCheck(sendbuff, recvbuff, count, type, op, root, comm, primName), ret, end); @@ -234,7 +238,7 @@ ncclResult_t ncclEnqueueCheck(ncclFunc_t func, const char* primName, const void* NCCLCHECK(ncclAsyncColl(comm)); NCCLCHECKGOTO(func(sendbuff, recvbuff, count, type, op, root, comm, stream), ret, end); end: - if (savedDev != -1) CUDACHECK(cudaSetDevice(savedDev)); + if (savedDev != -1) CUDACHECK(hipSetDevice(savedDev)); ncclAsyncErrCheck(ret); return ret; } else { diff --git a/src/misc/group.cu b/src/misc/group.cu index 1716a75643..0144bee78d 100644 --- a/src/misc/group.cu +++ b/src/misc/group.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -52,7 +53,7 @@ struct ncclAsyncArgs { thread_local struct ncclAsyncArgs ncclGroupArgs[MAX_ASYNC_OPS]; ncclResult_t ncclSetDevice(int cudaDev) { - CUDACHECK(cudaSetDevice(cudaDev)); + CUDACHECK(hipSetDevice(cudaDev)); return ncclSuccess; } @@ -116,7 +117,7 @@ ncclResult_t ncclGroupEnd() { ncclGroupMode--; if (ncclGroupMode > 0) return ncclSuccess; int savedDev; - CUDACHECK(cudaGetDevice(&savedDev)); + CUDACHECK(hipGetDevice(&savedDev)); int done = ncclGroupIndex; int doneArray[ncclGroupIndex]; for (int i=0; ifuncType == ASYNC_FUNC_COLL) { if (args->coll.comm->userStream == NULL) - CUDACHECKGOTO(cudaSetDevice(args->coll.comm->cudaDev), ret, end); + CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end); NCCLCHECKGOTO(ncclBarrierEnqueue(args->coll.comm), ret, end); } } for (int i=0; ifuncType == ASYNC_FUNC_COLL) { - CUDACHECKGOTO(cudaSetDevice(args->coll.comm->cudaDev), ret, end); + CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end); NCCLCHECKGOTO(ncclBarrierEnqueueWait(args->coll.comm), ret, end); } } @@ -152,7 +153,7 @@ ncclResult_t ncclGroupEnd() { struct ncclAsyncArgs* args = ncclGroupArgs+i; if (args->funcType == ASYNC_FUNC_COLL) { if (args->coll.comm->userStream == NULL) - CUDACHECKGOTO(cudaSetDevice(args->coll.comm->cudaDev), ret, end); + CUDACHECKGOTO(hipSetDevice(args->coll.comm->cudaDev), ret, end); NCCLCHECKGOTO(ncclEnqueueEvents(args->coll.comm), ret, end); doneArray[i] = 1; done--; @@ -182,7 +183,7 @@ group_cleanup: for (int r=0; rnRings; r++) { struct ncclRing* ring = comm->rings+r; for (int i=0; icollCount; i++) { - ring->collectives[(ring->collStart + i)%NCCL_MAX_OPS].active = 0; + STORE(&ring->collectives[(ring->collStart + i)%NCCL_MAX_OPS].active, 0); } ring->collFifoTail = ring->collStart; ring->collCount = 0; @@ -193,6 +194,6 @@ group_cleanup: end: ncclGroupError = ncclSuccess; ncclGroupIndex = 0; - CUDACHECK(cudaSetDevice(savedDev)); // do other clean-ups first before calling cudaSetDevice, because this call can fail too + CUDACHECK(hipSetDevice(savedDev)); // do other clean-ups first before calling hipSetDevice, because this call can fail too return ret; } diff --git a/src/misc/nvmlwrap.cu b/src/misc/nvmlwrap.cu index d9407f4686..f3ee2ac9ae 100644 --- a/src/misc/nvmlwrap.cu +++ b/src/misc/nvmlwrap.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ diff --git a/src/misc/nvmlwrap_stub.cu b/src/misc/nvmlwrap_stub.cu new file mode 100644 index 0000000000..85a389a1a9 --- /dev/null +++ b/src/misc/nvmlwrap_stub.cu @@ -0,0 +1,53 @@ +/************************************************************************* + * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "nvmlwrap.h" + +ncclResult_t wrapNvmlSymbols(void) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlInit(void) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlShutdown(void) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetHandleByPciBusId(const char* pciBusId, nvmlDevice_t* device) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetIndex(nvmlDevice_t device, unsigned* index) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceSetCpuAffinity(nvmlDevice_t device) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceClearCpuAffinity(nvmlDevice_t device) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetPciInfo(nvmlDevice_t device, nvmlPciInfo_t* pci) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetNvLinkState(nvmlDevice_t device, unsigned int link, nvmlEnableState_t *isActive) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetNvLinkRemotePciInfo(nvmlDevice_t device, unsigned int link, nvmlPciInfo_t *pci) { + return ncclSuccess; +} + +ncclResult_t wrapNvmlDeviceGetNvLinkCapability(nvmlDevice_t device, unsigned int link, + nvmlNvLinkCapability_t capability, unsigned int *capResult) { + return ncclSuccess; +} \ No newline at end of file diff --git a/src/misc/rings.cu b/src/misc/rings.cu index a5d4616019..359e26b359 100644 --- a/src/misc/rings.cu +++ b/src/misc/rings.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -337,7 +338,11 @@ ncclResult_t ncclGetRings(int* nrings, int* nthreads, int rank, int nranks, int* if (rank == 0) INFO(NCCL_INIT,"Limiting to %d rings per user request.", maxNrings); *nrings = maxNrings; } else { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + int defaultMinNrings = 1; +#else int defaultMinNrings = ncclCudaCompCap() == 3 ? 2 : 1; +#endif if (minNrings < defaultMinNrings) minNrings = defaultMinNrings; if (minNrings > 0 && minNrings > *nrings) { if (rank == 0 && minNrings > defaultMinNrings) INFO(NCCL_INIT,"Duplicating rings to %d per user request.", minNrings); diff --git a/src/misc/utils.cu b/src/misc/utils.cu index d8e3aec5f5..f5b61c5498 100644 --- a/src/misc/utils.cu +++ b/src/misc/utils.cu @@ -32,7 +32,7 @@ void ncclDebugLog(ncclDebugLogLevel level, unsigned long flags, const char *file char hostname[1024]; getHostName(hostname, 1024); int cudaDev; - cudaGetDevice(&cudaDev); + hipGetDevice(&cudaDev); char buffer[1024]; size_t len = 0; diff --git a/src/nccl.h.in b/src/nccl.h.in index 72276254cd..be5f095541 100644 --- a/src/nccl.h.in +++ b/src/nccl.h.in @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -7,15 +8,15 @@ #ifndef NCCL_H_ #define NCCL_H_ -#include -#include +#include +#include -#define NCCL_MAJOR ${nccl:Major} -#define NCCL_MINOR ${nccl:Minor} -#define NCCL_PATCH ${nccl:Patch} -#define NCCL_SUFFIX "${nccl:Suffix}" +#define NCCL_MAJOR ${NCCL_MAJOR} +#define NCCL_MINOR ${NCCL_MINOR} +#define NCCL_PATCH ${NCCL_PATCH} +#define NCCL_SUFFIX "${NCCL_SUFFIX}" -#define NCCL_VERSION_CODE ${nccl:Version} +#define NCCL_VERSION_CODE ${NCCL_VERSION} #define NCCL_VERSION(X,Y,Z) ((X) * 1000 + (Y) * 100 + (Z)) #ifdef __cplusplus @@ -132,9 +133,9 @@ typedef enum { ncclInt8 = 0, ncclChar = 0, * In-place operation will happen if sendbuff == recvbuff. */ ncclResult_t ncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, - ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream); ncclResult_t pncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, - ncclRedOp_t op, int root, ncclComm_t comm, cudaStream_t stream); + ncclRedOp_t op, int root, ncclComm_t comm, hipStream_t stream); /* * (deprecated) Broadcast (in-place) @@ -146,9 +147,9 @@ ncclResult_t pncclReduce(const void* sendbuff, void* recvbuff, size_t count, ncc * This operation is implicitely in place. */ ncclResult_t ncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); ncclResult_t pncclBcast(void* buff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); /* * Broadcast @@ -160,9 +161,9 @@ ncclResult_t pncclBcast(void* buff, size_t count, ncclDataType_t datatype, int r * In-place operation will happen if sendbuff == recvbuff. */ ncclResult_t ncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); ncclResult_t pncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, int root, - ncclComm_t comm, cudaStream_t stream); + ncclComm_t comm, hipStream_t stream); /* * All-Reduce @@ -173,9 +174,9 @@ ncclResult_t pncclBroadcast(const void* sendbuff, void* recvbuff, size_t count, * In-place operation will happen if sendbuff == recvbuff. */ ncclResult_t ncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, hipStream_t stream); ncclResult_t pncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, - ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, hipStream_t stream); /* * Reduce-Scatter @@ -190,10 +191,10 @@ ncclResult_t pncclAllReduce(const void* sendbuff, void* recvbuff, size_t count, */ ncclResult_t ncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, - cudaStream_t stream); + hipStream_t stream); ncclResult_t pncclReduceScatter(const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, ncclComm_t comm, - cudaStream_t stream); + hipStream_t stream); /* * All-Gather @@ -206,9 +207,9 @@ ncclResult_t pncclReduceScatter(const void* sendbuff, void* recvbuff, * In-place operations will happen if sendbuff == recvbuff + rank * sendcount. */ ncclResult_t ncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclComm_t comm, hipStream_t stream); ncclResult_t pncclAllGather(const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, ncclComm_t comm, cudaStream_t stream); + ncclDataType_t datatype, ncclComm_t comm, hipStream_t stream); /* * Group semantics diff --git a/src/ring.cu b/src/ring.cu index fede79387f..23e27571f0 100644 --- a/src/ring.cu +++ b/src/ring.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2015-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -7,6 +8,8 @@ #include "ring.h" #include "param.h" +extern bool useFineGrainVramPcie; + NCCL_PARAM(Buffsize, "BUFFSIZE", DEFAULT_BUFFER_SIZE_BYTES); ncclResult_t initRing(struct ncclComm* comm, int ringid) { @@ -18,12 +21,12 @@ ncclResult_t initRing(struct ncclComm* comm, int ringid) { const int sendSize = ring->devMemSendSize = sizeof(struct ncclSendMem); struct ncclSendMem* sendMem; - NCCLCHECK(ncclCudaCalloc((char**)&sendMem, sendSize)); + NCCLCHECK(ncclCudaCalloc((char**)&sendMem, sendSize, useFineGrainVramPcie)); ring->devMemSend = sendMem; const int recvSize = ring->devMemRecvSize = offsetof(struct ncclRecvMem, buff)+ring->buffSize; struct ncclRecvMem* recvMem; - NCCLCHECK(ncclCudaCalloc((char**)&recvMem, recvSize)); + NCCLCHECK(ncclCudaCalloc((char**)&recvMem, recvSize, useFineGrainVramPcie)); ring->devMemRecv = recvMem; TRACE(NCCL_INIT,"sendMem %p size %d recvMem %p size %d", sendMem, sendSize, recvMem, recvSize); @@ -51,12 +54,12 @@ ncclResult_t initRing(struct ncclComm* comm, int ringid) { ncclResult_t freeRing(struct ncclRing* ring) { // Intermediate buffering - CUDACHECK(cudaFree(ring->devMemSend)); - CUDACHECK(cudaFree(ring->devMemRecv)); + CUDACHECK(hipFree(ring->devMemSend)); + CUDACHECK(hipFree(ring->devMemRecv)); // Index to rank table free(ring->userRanks); - CUDACHECK(cudaFree(ring->devUserRanks)); + CUDACHECK(hipFree(ring->devUserRanks)); // Operation list NCCLCHECK(ncclCudaHostFree(ring->collectives)); diff --git a/src/transport.cu b/src/transport.cu index 7c13d5c351..c78adcc1de 100644 --- a/src/transport.cu +++ b/src/transport.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -18,35 +19,35 @@ struct ncclTransport ncclTransports[NTRANSPORTS] = { }; static void FifoPullArgs(struct transportProxyInfo* info, struct ncclProxyArgs *args) { - struct ncclProxyArgs *fifoArgs = info->argsFifo + (info->argsFifoHead % TRANSPORT_PROXY_FIFO_SIZE); + struct ncclProxyArgs *fifoArgs = info->argsFifo + (LOAD(&info->argsFifoHead) % TRANSPORT_PROXY_FIFO_SIZE); pthread_mutex_lock(&info->mutex); - while (fifoArgs->active == 0) + while (LOAD(&fifoArgs->active) == 0) pthread_cond_wait(&info->cond, &info->mutex); __sync_synchronize(); memcpy(args, fifoArgs, sizeof(struct ncclProxyArgs)); __sync_synchronize(); - fifoArgs->active = 0; + STORE(&fifoArgs->active, 0); pthread_cond_signal(&info->cond); pthread_mutex_unlock(&info->mutex); - info->argsFifoHead++; + __atomic_fetch_add(&info->argsFifoHead, 1, __ATOMIC_SEQ_CST); } static struct ncclProxyArgs* FifoGetNextArgs(struct transportProxyInfo* info) { if (info == NULL) return NULL; - struct ncclProxyArgs* fifoArgs = info->argsFifo + (info->argsFifoTail % TRANSPORT_PROXY_FIFO_SIZE); + struct ncclProxyArgs* fifoArgs = info->argsFifo + (LOAD(&info->argsFifoTail) % TRANSPORT_PROXY_FIFO_SIZE); pthread_mutex_lock(&info->mutex); - while (fifoArgs->active == 1) + while (LOAD(&fifoArgs->active) == 1) pthread_cond_wait(&info->cond, &info->mutex); pthread_mutex_unlock(&info->mutex); - info->argsFifoTail++; + __atomic_fetch_add(&info->argsFifoTail, 1, __ATOMIC_SEQ_CST); return fifoArgs; } static void FifoPushArgs(struct transportProxyInfo* info) { if (info == NULL) return; - struct ncclProxyArgs* fifoArgs = info->argsFifo + ((info->argsFifoTail-1) % TRANSPORT_PROXY_FIFO_SIZE); - if (fifoArgs->active == 0) return; + struct ncclProxyArgs* fifoArgs = info->argsFifo + ((LOAD(&info->argsFifoTail)-1) % TRANSPORT_PROXY_FIFO_SIZE); + if (LOAD(&fifoArgs->active) == 0) return; pthread_mutex_lock(&info->mutex); pthread_cond_signal(&info->cond); @@ -55,21 +56,21 @@ static void FifoPushArgs(struct transportProxyInfo* info) { static void WaitProxyReady(struct transportProxyInfo* info) { pthread_mutex_lock(&info->mutex); - while (info->proxyReady == 0) + while (LOAD(&info->proxyReady) == 0) pthread_cond_wait(&info->cond, &info->mutex); pthread_mutex_unlock(&info->mutex); } static void SetProxyReady(struct transportProxyInfo* info) { pthread_mutex_lock(&info->mutex); - info->proxyReady = 1; + STORE(&info->proxyReady, 1); pthread_cond_signal(&info->cond); pthread_mutex_unlock(&info->mutex); } static void StopProxy(struct transportProxyInfo* info) { struct ncclProxyArgs* fifoArgs = FifoGetNextArgs(info); - fifoArgs->active = -1; + STORE(&fifoArgs->active, -1); FifoPushArgs(info); } @@ -100,7 +101,7 @@ static void SaveProxy(struct ncclConnector* connector, struct ncclProxyArgs* arg __sync_synchronize(); memcpy(fifoArgs, args, sizeof(struct ncclProxyArgs)); __sync_synchronize(); - fifoArgs->active = 1; + STORE(&fifoArgs->active, 1); } ncclResult_t transportSaveProxies(int substeps, int subchunks, int nstepsPerRound, int nblocksPerRound, size_t nbytes, int pattern, struct ncclComm* comm) { @@ -136,9 +137,9 @@ ncclResult_t transportStartProxies(ncclComm* comm) { void* persistentThread(void *opaqueInfo) { struct transportProxyInfo* info = (struct transportProxyInfo*)opaqueInfo; // We need to initialize the context before launching any NCCL cuda kernel, - // otherwise we would create it during the first cudaMemcpyAsync inside the + // otherwise we would create it during the first hipMemcpyAsync inside the // proxy function and that would cause a deadlock - cudaSetDevice(info->comm->cudaDev); + hipSetDevice(info->comm->cudaDev); // Signal the main thread the context is created and it can proceed. SetProxyReady(info); while (1) { @@ -167,8 +168,8 @@ ncclResult_t transportCreateProxy(int type, struct ncclRing* ring, struct ncclCo info->cond = PTHREAD_COND_INITIALIZER; info->mutex = PTHREAD_MUTEX_INITIALIZER; info->func = proxyfunc; - info->argsFifoHead = info->argsFifoTail = 0; - info->proxyReady = 0; + STORE(&info->argsFifoHead, 0); STORE(&info->argsFifoTail, 0); + STORE(&info->proxyReady, 0); pthread_create(&connector->proxyInfo->thread, NULL, persistentThread, info); // Wait for thread to initialize its CUDA context. WaitProxyReady(info); diff --git a/src/transport/net.cu b/src/transport/net.cu index 9c366b32f5..fe3a7f7c56 100644 --- a/src/transport/net.cu +++ b/src/transport/net.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -9,9 +10,13 @@ #include "nvmlwrap.h" #include "net.h" #include "param.h" -#include "nvlink.h" -#include +#include #include +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#include "nvlink_stub.h" +#else +#include "nvlink.h" +#endif #define NET_MAX_IFS 16 @@ -83,7 +88,7 @@ ncclResult_t netFillInfo(ncclTinfo_t* opaqueInfo, int rank) { // Find distance with current GPU int cudaDev; - cudaGetDevice(&cudaDev); + hipGetDevice(&cudaDev); char* cudaPath; NCCLCHECK(getCudaPath(cudaDev, &cudaPath)); @@ -214,18 +219,25 @@ int getDev(int ringId, int nDev, short* distances) { NCCL_PARAM(NetGdrRead, "NET_GDR_READ", -2); NCCL_PARAM(NetGdrLevel, "NET_GDR_LEVEL", PATH_PHB); +extern bool useFineGrainVramPcie; + static ncclResult_t netGetGdrSupport(int dev, int distance, int read, int* useGdr) { *useGdr = 0; int cudaDev; - CUDACHECK(cudaGetDevice(&cudaDev)); + CUDACHECK(hipGetDevice(&cudaDev)); + + if (!useFineGrainVramPcie) { + INFO(NCCL_INIT|NCCL_NET,"NET/%s : GPU Direct RDMA Disabled for GPU %d / Need Fine Grain VRAM over PCIe", ncclNetName(), cudaDev); + return ncclSuccess; + } if (read) { // For reads (sends) only enable under certain conditions int gdrReadParam = ncclParamNetGdrRead(); if (gdrReadParam == 0) return ncclSuccess; else if (gdrReadParam < 0) { // default : enable only on DGX2 char busId[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE]; - CUDACHECK(cudaDeviceGetPCIBusId(busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, cudaDev)); + CUDACHECK(hipDeviceGetPCIBusId(busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, cudaDev)); int nvlinks = getNumNvlinks(busId); if (nvlinks < CONNECT_NVSWITCH || ncclCudaCompCap() < 7) return ncclSuccess; } @@ -260,7 +272,7 @@ ncclResult_t netSendSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo int size = offsetof(struct ncclRecvMem, buff)+ring->buffSize; if (resources->useGdr) { - NCCLCHECK(ncclCudaCalloc((char**)(&resources->devNetMem), size)); + NCCLCHECK(ncclCudaCalloc((char**)(&resources->devNetMem), size, true)); } NCCLCHECK(ncclCudaHostAlloc((void**)&resources->hostRecvMem, (void**)&resources->devHostRecvMem, size)); @@ -351,7 +363,7 @@ ncclResult_t netSendFree(void* transportResources) { NCCLCHECK(ncclCudaHostFree(resources->hostSendMem)); NCCLCHECK(ncclCudaHostFree(resources->hostRecvMem)); if (resources->useGdr) - CUDACHECK(cudaFree(resources->devNetMem)); + CUDACHECK(hipFree(resources->devNetMem)); NCCLCHECK(ncclNetCloseSend(resources->netSendComm)); free(resources); return ncclSuccess; @@ -403,7 +415,7 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) { if (llMode) { if (tail < end && tail < head + args->substeps) { int slot = tail%args->substeps; - int size = sizesFifo[slot]; + int size = LOAD(&sizesFifo[slot]); if (size != 0) { if (size == -1) size = 0; uint32_t flag = tail + 1; @@ -413,20 +425,21 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) { for (int i=0; inetSendComm, lines, size, ptrType, requests+slot)); if (requests[slot] != NULL) { - sizesFifo[slot] = size; + STORE(&sizesFifo[slot], size); tail++; idle = 0; } } } - } else while (tail < *prevTail) { + } else while (tail < LOAD(prevTail)) { // Send through network int slot = tail%args->substeps; - NCCLCHECK(ncclNetIsend(resources->netSendComm, localBuff+slot*sliceSize, sizesFifo[slot], ptrType, requests+slot)); + //TRACE(NCCL_NET,"head %d tail %d prevTail %d slot %d size %d ptrType %d", head, tail, LOAD(prevTail), slot, LOAD(&sizesFifo[slot]), ptrType); + NCCLCHECK(ncclNetIsend(resources->netSendComm, localBuff+slot*sliceSize, LOAD(&sizesFifo[slot]), ptrType, requests+slot)); if (requests[slot] != NULL) { tail++; idle = 0; @@ -438,12 +451,12 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) { NCCLCHECK(ncclNetTest(requests[slot], &done, NULL)); if (done) { if (llMode) { - sizesFifo[slot] = 0; + STORE(&sizesFifo[slot], 0); // Make sure size is reset to zero before we update the head. __sync_synchronize(); } head++; - *prevHead = head; + STORE(prevHead, head); idle = 0; } } @@ -451,17 +464,17 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) { } // Reset - if (llMode == 0) *prevTail = 0; + if (llMode == 0) STORE(prevTail, 0); nextColl: if (llMode) { resources->llStep += args->nsteps; // Don't forget to ack otherwise the GPU won't be able to push data. - *prevHead = resources->llStep; + STORE(prevHead, resources->llStep); if (resources->llStep > resources->llLastCleaning + NCCL_LL_CLEAN_FREQ) { memset(localBuff, 0, NCCL_LL_BUFF_SIZE); resources->llStep += NCCL_LL_CHUNKS; - *prevHead = resources->llStep; + STORE(prevHead, resources->llStep); resources->llLastCleaning = resources->llStep; } } @@ -503,7 +516,7 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) { while (head < end) { idle++; - if ((tail < head + args->substeps) && (tail < *nextHead + args->substeps) && (tail < end)) { + if ((tail < head + args->substeps) && (tail < LOAD(nextHead) + args->substeps) && (tail < end)) { int slot = tail%args->substeps; NCCLCHECK(ncclNetIrecv(resources->netRecvComm, localBuff+slot*sliceSize, sliceSize, ptrType, requests+slot)); if (requests[slot] != NULL) { @@ -521,7 +534,8 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) { head++; if (llMode == 0) { if (ptrType == NCCL_PTR_CUDA) ncclNetFlush(resources->netRecvComm, localBuff+slot*sliceSize, size); - *nextTail = head; + //TRACE(NCCL_NET,"head %d tail %d slot %d size %d ptrType %d", head, tail, slot, size, ptrType); + STORE(nextTail, head); } idle = 0; } @@ -531,8 +545,8 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) { // Wait for last ack and reset if (llMode == 0) { - transportProxyWait([=] { return *nextHead == head; }); - *nextHead = 0; + transportProxyWait([=] { return LOAD(nextHead) == head; }); + STORE(nextHead, 0); } nextColl: @@ -540,7 +554,7 @@ nextColl: resources->llStep += args->nsteps; if (resources->llStep > resources->llLastCleaning + NCCL_LL_CLEAN_FREQ) { resources->llStep += NCCL_LL_CHUNKS; - while (*nextHead < resources->llStep); + while (LOAD(nextHead) < resources->llStep); resources->llLastCleaning = resources->llStep; } } diff --git a/src/transport/net_ib.cu b/src/transport/net_ib.cu index 18e158df7c..cbe2f9c45f 100644 --- a/src/transport/net_ib.cu +++ b/src/transport/net_ib.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -182,12 +183,16 @@ ncclResult_t ncclIbPciPath(int dev, char** path) { ncclResult_t ncclIbGdrSupport(int ibDev) { static int moduleLoaded = -1; if (moduleLoaded == -1) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + moduleLoaded = (access("/sys/kernel/mm/memory_peers/amdkfd/version", F_OK) == -1) ? 0 : 1; +#else moduleLoaded = (access("/sys/kernel/mm/memory_peers/nv_mem/version", F_OK) == -1) ? 0 : 1; +#endif } if (moduleLoaded == 0) return ncclSystemError; ncclResult_t ret = ncclSystemError; void* ptr; - if (cudaMalloc(&ptr, sizeof(int)) == cudaSuccess) { + if (hipMalloc(&ptr, sizeof(int)) == hipSuccess) { struct ibv_mr* mr; struct ibv_pd* pd; if (wrap_ibv_alloc_pd(&pd, ncclIbDevs[ibDev].context) == ncclSuccess) { @@ -197,7 +202,7 @@ ncclResult_t ncclIbGdrSupport(int ibDev) { } wrap_ibv_dealloc_pd(pd); } - cudaFree(ptr); + hipFree(ptr); } return ret; } @@ -206,7 +211,7 @@ ncclResult_t ncclIbPtrSupport(int dev, int* supportedTypes) { *supportedTypes = NCCL_PTR_HOST; int cudaDev; - CUDACHECK(cudaGetDevice(&cudaDev)); + CUDACHECK(hipGetDevice(&cudaDev)); if (ncclIbGdrSupport(dev) != ncclSuccess) { INFO(NCCL_INIT|NCCL_NET,"NET/IB : GPU Direct RDMA Disabled for GPU %d / HCA %s (no module or not supported by GPU)", cudaDev, ncclIbDevs[dev].devName); @@ -637,7 +642,7 @@ ncclResult_t ncclIbIsend(void* sendComm, void* data, int size, int type, void** // Wait for the receiver to have posted the corresponding receive volatile struct ncclIbSendFifo* slot = comm->fifo + (comm->fifoHead%MAX_REQUESTS); volatile uint32_t * readyPtr = &slot->ready; - if (*readyPtr == 0) { *request = NULL; return ncclSuccess; } + if (LOAD(readyPtr) == 0) { *request = NULL; return ncclSuccess; } struct ncclIbRequest* req; NCCLCHECK(ncclIbGetRequest(comm->reqs, &req)); @@ -679,7 +684,7 @@ ncclResult_t ncclIbIsend(void* sendComm, void* data, int size, int type, void** #endif // We must clear slot->ready, but reset other fields to aid // debugging and sanity checks - slot->ready = 0; + STORE(&slot->ready, 0); slot->addr = 0ULL; slot->rkey = slot->size = slot->seq = 0; comm->fifoHead++; diff --git a/src/transport/net_socket.cu b/src/transport/net_socket.cu index 1efee15dda..b09e2e7234 100644 --- a/src/transport/net_socket.cu +++ b/src/transport/net_socket.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -50,7 +51,12 @@ ncclResult_t ncclSocketDevices(int* ndev) { ncclResult_t ncclSocketPciPath(int dev, char** path) { char devicepath[PATH_MAX]; - snprintf(devicepath, PATH_MAX, "/sys/class/net/%s/device", ncclNetIfNames+dev*MAX_IF_NAME_SIZE); + snprintf(devicepath, PATH_MAX, "/sys/class/net/%s", ncclNetIfNames+dev*MAX_IF_NAME_SIZE); + *path = realpath(devicepath, NULL); + const char* string_virual_network_device_path="/sys/devices/virtual/net/"; + if (*path && !strncmp(*path, string_virual_network_device_path, strlen(string_virual_network_device_path))) + return ncclSuccess; + free(*path); *path = realpath(devicepath, NULL); if (*path == NULL) { INFO(NCCL_NET|NCCL_INIT, "Could not find real path of %s", devicepath); diff --git a/src/transport/p2p.cu b/src/transport/p2p.cu index 6c4626a77c..301ad3e90d 100644 --- a/src/transport/p2p.cu +++ b/src/transport/p2p.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -10,10 +11,16 @@ #include "transport.h" #include "param.h" #include -#include +#include #include "nvmlwrap.h" #include +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#include "nvlink_stub.h" +#else #include "nvlink.h" +#endif + +extern bool useFineGrainVramPcie; struct p2pInfo { int rank; @@ -27,7 +34,7 @@ struct p2pConnectInfo { int direct; union { void* directPtr; - cudaIpcMemHandle_t devIpc; + hipIpcMemHandle_t devIpc; }; }; @@ -39,7 +46,7 @@ ncclResult_t p2pFillInfo(ncclTinfo_t* opaqueInfo, int rank) { struct p2pInfo* info = (struct p2pInfo*)opaqueInfo; static_assert(sizeof(struct p2pInfo) <= sizeof(ncclTinfo_t), "p2p Info too large"); info->rank = rank; - CUDACHECK(cudaGetDevice(&info->cudaDev)); + CUDACHECK(hipGetDevice(&info->cudaDev)); info->hostHash=getHostHash(); info->pidHash=getPidHash(); @@ -47,12 +54,15 @@ ncclResult_t p2pFillInfo(ncclTinfo_t* opaqueInfo, int rank) { // cudaDev is a CUDA runtime dev number which could be different from the // NVML device number. Then we get the busID from NVML to be sure it is // consistent with NVML remote PCI bus Ids. - CUDACHECK(cudaDeviceGetPCIBusId(info->busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, info->cudaDev)); + CUDACHECK(hipDeviceGetPCIBusId(info->busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE, info->cudaDev)); +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) +#else nvmlDevice_t nvmlDevice; NCCLCHECK(wrapNvmlDeviceGetHandleByPciBusId(info->busId, &nvmlDevice)); nvmlPciInfo_t pciInfo; NCCLCHECK(wrapNvmlDeviceGetPciInfo(nvmlDevice, &pciInfo)); strncpy(info->busId, pciInfo.busId, NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE); +#endif return ncclSuccess; } @@ -84,15 +94,41 @@ ncclResult_t p2pCanConnect(ncclTvalue_t* ret, ncclTinfo_t* myOpaqueInfo, ncclTin // See if CUDA can do P2P int p2p; - if (cudaDeviceCanAccessPeer(&p2p, myInfo->cudaDev, peerInfo->cudaDev) != cudaSuccess) { + if (hipDeviceCanAccessPeer(&p2p, myInfo->cudaDev, peerInfo->cudaDev) != hipSuccess) { INFO(NCCL_INIT|NCCL_P2P,"peer query failed between dev %d and dev %d", myInfo->cudaDev, peerInfo->cudaDev); return ncclSuccess; } + + if (!useFineGrainVramPcie) p2p = 0; + if (p2p == 0) return ncclSuccess; - // Check for NVLink/NVswitch +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + uint32_t link_type, hops; + if (hipExtGetLinkTypeAndHopCount(myInfo->cudaDev, peerInfo->cudaDev, &link_type, &hops) != hipSuccess) { + p2p = 0; + return ncclSuccess; + } + static const char* link_type_name[] = {"HT", "QPI", "PCIE", "IB", "XGMI"}; + static unsigned long long link_status_print_once_mask = 0; + if (!(link_status_print_once_mask & (1 << (myInfo->cudaDev*8 + peerInfo->cudaDev)))) { + INFO(NCCL_INIT, "%d -> %d: link type %s hops %d", myInfo->cudaDev, peerInfo->cudaDev, + link_type_name[link_type], hops); + link_status_print_once_mask |= (1 << (myInfo->cudaDev*8 + peerInfo->cudaDev)); + } + if (link_type != HSA_AMD_LINK_INFO_TYPE_XGMI) { + // enable below lines on release only: disable PCIe P2P until HDP flush is implemented. + // p2p = 0; + // return ncclSuccess; + } + int nvlinkp2p = 0; + if (link_type == HSA_AMD_LINK_INFO_TYPE_XGMI && hops == 1) + nvlinkp2p = CONNECT_NVLINK; +#else + // Check for NVLink/NVswitch int nvlinkp2p = getNvlinkGpu(myInfo->busId, peerInfo->busId); +#endif if (nvlinkp2p > 0) { *ret = nvlinkp2p; return ncclSuccess; @@ -457,12 +493,12 @@ ncclResult_t p2pSendSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo INFO(NCCL_INIT|NCCL_P2P,"Ring %02d : %d -> %d via P2P/common device", ring->id, myInfo->rank, peerInfo->rank); } else { // Enable P2P access - cudaError_t err = cudaDeviceEnablePeerAccess(peerInfo->cudaDev, 0); - if (err == cudaErrorPeerAccessAlreadyEnabled) { - cudaGetLastError(); - } else if (err != cudaSuccess) { + hipError_t err = hipDeviceEnablePeerAccess(peerInfo->cudaDev, 0); + if (err == hipErrorPeerAccessAlreadyEnabled) { + hipGetLastError(); + } else if (err != hipSuccess) { WARN("failed to peer with device %d: %d %s", - peerInfo->cudaDev, err, cudaGetErrorString(err)); + peerInfo->cudaDev, err, hipGetErrorString(err)); return ncclInternalError; } INFO(NCCL_INIT|NCCL_P2P,"Ring %02d : %d[%d] -> %d[%d] via P2P/direct pointer", @@ -471,10 +507,10 @@ ncclResult_t p2pSendSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo } else { info.direct = 0; // Map IPC and enable P2P access - cudaError_t err = cudaIpcGetMemHandle(&info.devIpc, (void*)ring->devMemSend); - if (err != cudaSuccess) { + hipError_t err = hipIpcGetMemHandle(&info.devIpc, (void*)ring->devMemSend); + if (err != hipSuccess) { WARN("rank %d failed to get CUDA IPC handle to device %d : %d %s", - myInfo->rank, peerInfo->cudaDev, err, cudaGetErrorString(err)); + myInfo->rank, peerInfo->cudaDev, err, hipGetErrorString(err)); return ncclInternalError; } INFO(NCCL_INIT|NCCL_P2P,"Ring %02d : %d[%d] -> %d[%d] via P2P/IPC", @@ -498,12 +534,12 @@ ncclResult_t p2pRecvSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo TRACE(NCCL_INIT|NCCL_P2P,"%d <- %d via P2P/common device", myInfo->rank, peerInfo->rank); } else { // Enable P2P access - cudaError_t err = cudaDeviceEnablePeerAccess(peerInfo->cudaDev, 0); - if (err == cudaErrorPeerAccessAlreadyEnabled) { - cudaGetLastError(); - } else if (err != cudaSuccess) { + hipError_t err = hipDeviceEnablePeerAccess(peerInfo->cudaDev, 0); + if (err == hipErrorPeerAccessAlreadyEnabled) { + hipGetLastError(); + } else if (err != hipSuccess) { WARN("failed to peer with device %d: %d %s", - peerInfo->cudaDev, err, cudaGetErrorString(err)); + peerInfo->cudaDev, err, hipGetErrorString(err)); return ncclInternalError; } TRACE(NCCL_INIT|NCCL_P2P,"Ring %02d : %d[%d] <- %d[%d] via P2P/direct pointer", ring->id, myInfo->rank, myInfo->cudaDev, peerInfo->rank, peerInfo->cudaDev); @@ -511,10 +547,10 @@ ncclResult_t p2pRecvSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo } else { info.direct = 0; // Map IPC and enable P2P access - cudaError_t err = cudaIpcGetMemHandle(&info.devIpc, (void*)ring->devMemRecv); - if (err != cudaSuccess) { - WARN("rank %d failed to get CUDA IPC handle to device %d : %d %s", - myInfo->rank, peerInfo->cudaDev, err, cudaGetErrorString(err)); + hipError_t err = hipIpcGetMemHandle(&info.devIpc, (void*)ring->devMemRecv); + if (err != hipSuccess) { + WARN("rank %d failed to get HIP IPC handle to device %d : %d %s", + myInfo->rank, peerInfo->cudaDev, err, hipGetErrorString(err)); return ncclInternalError; } TRACE(NCCL_INIT|NCCL_P2P,"Ring %02d : %d[%d] <- %d[%d] via P2P/IPC", ring->id, myInfo->rank, myInfo->cudaDev, peerInfo->rank, peerInfo->cudaDev); @@ -537,15 +573,15 @@ static ncclResult_t p2pSendConnect(struct ncclConnect* connectInfo, struct ncclC } else { void* remPtr = NULL; //TRACE_DUMP_IPC(&info->devIpc); - cudaError_t err = cudaIpcOpenMemHandle(&remPtr, info->devIpc, cudaIpcMemLazyEnablePeerAccess); + hipError_t err = hipIpcOpenMemHandle(&remPtr, info->devIpc, hipIpcMemLazyEnablePeerAccess); void** ipcPtrSave; NCCLCHECK(ncclCalloc(&ipcPtrSave, 1)); *resources = ipcPtrSave; *ipcPtrSave = remPtr; remDevMem = (struct ncclRecvMem*)remPtr; - if (err != cudaSuccess) { + if (err != hipSuccess) { WARN("failed to open CUDA IPC handle : %d %s", - err, cudaGetErrorString(err)); + err, hipGetErrorString(err)); return ncclUnhandledCudaError; } } @@ -571,15 +607,15 @@ ncclResult_t p2pRecvConnect(struct ncclConnect* connectInfo, struct ncclConnecto } else { void* remPtr = NULL; //TRACE_DUMP_IPC(&info->devIpc); - cudaError_t err = cudaIpcOpenMemHandle(&remPtr, info->devIpc, cudaIpcMemLazyEnablePeerAccess); + hipError_t err = hipIpcOpenMemHandle(&remPtr, info->devIpc, hipIpcMemLazyEnablePeerAccess); void** ipcPtrSave; NCCLCHECK(ncclCalloc(&ipcPtrSave, 1)); *resources = ipcPtrSave; *ipcPtrSave = remPtr; remDevMem = (struct ncclSendMem*)remPtr; - if (err != cudaSuccess) { + if (err != hipSuccess) { WARN("failed to open CUDA IPC handle : %d %s", - err, cudaGetErrorString(err)); + err, hipGetErrorString(err)); return ncclUnhandledCudaError; } } @@ -595,7 +631,7 @@ ncclResult_t p2pRecvConnect(struct ncclConnect* connectInfo, struct ncclConnecto ncclResult_t p2pFree(void* resources) { if (resources != NULL) { void** ipcPtrSave = (void**) resources; - CUDACHECK(cudaIpcCloseMemHandle(*ipcPtrSave)); + CUDACHECK(hipIpcCloseMemHandle(*ipcPtrSave)); free(resources); } return ncclSuccess; diff --git a/src/transport/shm.cu b/src/transport/shm.cu index 317f652dac..557a32c86b 100644 --- a/src/transport/shm.cu +++ b/src/transport/shm.cu @@ -1,5 +1,6 @@ /************************************************************************* * Copyright (c) 2016-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ @@ -10,7 +11,7 @@ #include "param.h" #include "shm.h" #include -#include +#include struct shmInfo { int rank; @@ -57,7 +58,7 @@ ncclResult_t shmFillInfo(ncclTinfo_t* opaqueInfo, int rank) { struct shmInfo* info = (struct shmInfo*)opaqueInfo; static_assert(sizeof(struct shmInfo) <= sizeof(ncclTinfo_t), "shm Info too large"); info->rank = rank; - CUDACHECK(cudaGetDevice(&info->cudaDev)); + CUDACHECK(hipGetDevice(&info->cudaDev)); info->hostHash=getHostHash(); info->pidHash=getPidHash(); return ncclSuccess; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000000..1af7886276 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,63 @@ +cmake_minimum_required(VERSION 2.8.12) + +if(BUILD_TESTS) + + message("Going to build unit tests (Installed in /test/UnitTests)") + + # OpenMP is used to drive GPUs (one per thread) + find_package(OpenMP REQUIRED) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") + + # Download and unpack googletest at configure time + configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) + execute_process( + COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download) + if(result) + message(FATAL_ERROR "CMake step for googletest failed: ${result}") + endif() + execute_process( + COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download) + if(result) + message(FATAL_ERROR "Build step for googletest failed: ${result}") + endif() + + # Add googletest directly to our build. This adds the following targets: + # gtest, gtest_main, gmock and gmock_main + add_subdirectory("${CMAKE_BINARY_DIR}/googletest-src" + "${CMAKE_BINARY_DIR}/googletest-build") + + # Add googletest directly to our build. This defines the gtest and gtest_main + # targets. add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src + # ${CMAKE_CURRENT_BINARY_DIR}/googletest-build EXCLUDE_FROM_ALL) + + # Collect source files for tests + set(TEST_SOURCES + test_AllGather.cpp + test_AllReduce.cpp + test_Broadcast.cpp + test_Reduce.cpp + test_ReduceScatter.cpp + test_GroupCalls.cpp + ) + + add_executable(UnitTests ${TEST_SOURCES}) + target_include_directories(UnitTests PRIVATE /opt/rocm) + target_link_libraries(UnitTests PRIVATE gtest_main PRIVATE rccl) + install(TARGETS UnitTests RUNTIME DESTINATION test) + + # HCC adds /opt/rocm/lib as RPATH, even though the install process is supposed to + # remove RPATH. As a work-around, set the correct RPATH for the unit test executable + # as a post-install step + install( + CODE + "execute_process(COMMAND chrpath -r ${CMAKE_INSTALL_PREFIX}/lib:/opt/rocm/lib ${CMAKE_INSTALL_PREFIX}/test/UnitTests)" + ) +else() + message("Not building unit tests") +endif() diff --git a/test/CMakeLists.txt.in b/test/CMakeLists.txt.in new file mode 100644 index 0000000000..128d29e7c2 --- /dev/null +++ b/test/CMakeLists.txt.in @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 2.8.2) + +project(googletest-download NONE) + +include(ExternalProject) +ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src" + BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/test/CorrectnessTest.hpp b/test/CorrectnessTest.hpp new file mode 100644 index 0000000000..638c028788 --- /dev/null +++ b/test/CorrectnessTest.hpp @@ -0,0 +1,311 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef CORRECTNESSTEST_HPP +#define CORRECTNESSTEST_HPP + +#include +#include +#include +#include +#include "rccl.h" + +#define HIP_CALL(x) ASSERT_EQ(x, hipSuccess) +#define NCCL_CALL(x) ASSERT_EQ(x, ncclSuccess) + +namespace CorrectnessTests +{ + // Performs the various basic reduction operations + template + T ReduceOp(ncclRedOp_t const op, T const A, T const B) + { + switch (op) + { + case ncclSum: return A + B; + case ncclProd: return A * B; + case ncclMax: return std::max(A, B); + case ncclMin: return std::min(A, B); + default: + fprintf(stderr, "[ERROR] Unsupported reduction operator (%d)\n", op); + exit(0); + } + } + + // Returns the number of bytes per element for each supported datatype + static int DataTypeToBytes(ncclDataType_t const dataType) + { + switch (dataType) + { + case ncclInt8: return 1; + case ncclUint8: return 1; + case ncclInt32: return 4; + case ncclUint32: return 4; + case ncclInt64: return 8; + case ncclUint64: return 8; + case ncclFloat16: return 2; + case ncclFloat32: return 4; + case ncclFloat64: return 8; + default: + fprintf(stderr, "[ERROR] Unsupported datatype (%d)\n", dataType); + exit(0); + } + } + + // Encapsulates all the memory used per devices for collectives, as well as reference results + struct Dataset + { + int numDevices; // Number of devices participating + size_t numElements; // Number of elements per array + ncclDataType_t dataType; // Data type of each input/output pointer + bool inPlace; // Whether or not output pointers are same as input pointers + std::vector inputs; // Input pointers (1 per device) + std::vector outputs; // Output pointers (1 per device) + // May be identical to input pointers for in-place tests + std::vector expected; // Expected output (1 per device) + + size_t NumBytes() const + { + return numElements * DataTypeToBytes(dataType); + } + + void Initialize(int const numDevices_, + size_t const numElements_, + ncclDataType_t const dataType_, + bool const inPlace_) + { + numDevices = numDevices_; + numElements = numElements_; + dataType = dataType_; + inPlace = inPlace_; + + inputs.resize(numDevices); + outputs.resize(numDevices); + expected.resize(numDevices); + + // Allocate per-device memory + size_t const numBytes = NumBytes(); + + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + HIP_CALL(hipMalloc((void **)&inputs[i], numBytes)); + if (inPlace) + outputs[i] = inputs[i]; + else + HIP_CALL(hipMalloc((void **)&outputs[i], numBytes)); + + expected[i] = malloc(numBytes); + } + } + + ~Dataset() + { + for (int i = 0; i < outputs.size(); i++) + { + if (!inPlace) hipFree(outputs[i]); + hipFree(inputs[i]); + free(expected[i]); + } + } + }; + + typedef std::tuple TestTuple; + + // Base class for each collective test + // - Each test is instantiated with a different TestTuple + class CorrectnessTest : public testing::TestWithParam + { + protected: + + // This code is called per test-tuple + void SetUp() override + { + // Check for fine-grained env variable (otherwise will hang) + if (!getenv("HSA_FORCE_FINE_GRAIN_PCIE")) + { + printf("Must set HSA_FORCE_FINE_GRAIN_PCIE=1 prior to execution\n"); + exit(0); + } + + // Make the test tuple parameters accessible + std::tie(op, dataType, numElements, numDevices, inPlace) = GetParam(); + + // Collect the number of available GPUs + HIP_CALL(hipGetDeviceCount(&numDevicesAvailable)); + + // Only proceed with testing if there are enough GPUs + if (numDevices > numDevicesAvailable) + { + fprintf(stdout, "Skipping test requring %d devices (only %d available)\n", + numDevices, numDevicesAvailable); + return; + } + + // Initialize communicators + comms.resize(numDevices); + NCCL_CALL(ncclCommInitAll(comms.data(), numDevices, NULL)); + + // Create streams + streams.resize(numDevices); + for (int i = 0; i < numDevices; i++) + HIP_CALL(hipStreamCreate(&streams[i])); + } + + // Clean up per TestTuple + void TearDown() override + { + // Release communicators and streams + for (int i = 0; i < numDevices; i++) + { + NCCL_CALL(ncclCommDestroy(comms[i])); + HIP_CALL(hipStreamDestroy(streams[i])); + } + } + + void FillDatasetWithPattern(Dataset& dataset) + { + int8_t* arrayI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + + // NOTE: Currently half-precision float tests are unsupported due to half being supported + // on GPU only and not host + + // Fills input data[i][j] with (i + j) % 6 + // - Keeping range small to reduce likelihood of overflow + // - Sticking with floating points values that are perfectly representable + for (int i = 0; i < dataset.numDevices; i++) + { + for (int j = 0; j < dataset.numElements; j++) + { + int valueI = (i + j) % 6; + float valueF = (float)valueI; + + switch (dataset.dataType) + { + case ncclInt8: arrayI1[j] = valueI; break; + case ncclUint8: arrayU1[j] = valueI; break; + case ncclInt32: arrayI4[j] = valueI; break; + case ncclUint32: arrayU4[j] = valueI; break; + case ncclInt64: arrayI8[j] = valueI; break; + case ncclUint64: arrayU8[j] = valueI; break; + case ncclFloat32: arrayF4[j] = valueF; break; + case ncclFloat64: arrayF8[j] = valueF; break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + + HIP_CALL(hipSetDevice(i)); + HIP_CALL(hipMemcpy(dataset.inputs[i], arrayI1, dataset.NumBytes(), hipMemcpyHostToDevice)); + + // Fills output data[i][j] with 0 (if not inplace) + if (!dataset.inPlace) + HIP_CALL(hipMemset(dataset.outputs[i], 0, dataset.NumBytes())); + } + + free(arrayI1); + } + + void ValidateResults(Dataset const& dataset) const + { + int8_t* outputI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* outputU1 = (uint8_t *)outputI1; + int32_t* outputI4 = (int32_t *)outputI1; + uint32_t* outputU4 = (uint32_t *)outputI1; + int64_t* outputI8 = (int64_t *)outputI1; + uint64_t* outputU8 = (uint64_t *)outputI1; + float* outputF4 = (float *)outputI1; + double* outputF8 = (double *)outputI1; + + bool isMatch = true; + + // Loop over each device's output and compare it to the expected output + // (Each collective operation computes its own expected results) + for (int i = 0; i < dataset.numDevices && isMatch; i++) + { + HIP_CALL(hipMemcpy(outputI1, dataset.outputs[i], dataset.NumBytes(), hipMemcpyDeviceToHost)); + + int8_t* expectedI1 = (int8_t *)dataset.expected[i]; + uint8_t* expectedU1 = (uint8_t *)expectedI1; + int32_t* expectedI4 = (int32_t *)expectedI1; + uint32_t* expectedU4 = (uint32_t *)expectedI1; + int64_t* expectedI8 = (int64_t *)expectedI1; + uint64_t* expectedU8 = (uint64_t *)expectedI1; + float* expectedF4 = (float *)expectedI1; + double* expectedF8 = (double *)expectedI1; + + for (int j = 0; j < dataset.numElements && isMatch; j++) + { + switch (dataset.dataType) + { + case ncclInt8: isMatch &= (outputI1[j] == expectedI1[j]); break; + case ncclUint8: isMatch &= (outputU1[j] == expectedU1[j]); break; + case ncclInt32: isMatch &= (outputI4[j] == expectedI4[j]); break; + case ncclUint32: isMatch &= (outputU4[j] == expectedU4[j]); break; + case ncclInt64: isMatch &= (outputI8[j] == expectedI8[j]); break; + case ncclUint64: isMatch &= (outputU8[j] == expectedU8[j]); break; + case ncclFloat32: isMatch &= (outputF4[j] == expectedF4[j]); break; + case ncclFloat64: isMatch &= (outputF8[j] == expectedF8[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + + if (!isMatch) + { + switch (dataset.dataType) + { + case ncclInt8: + printf("Expected %d. Output %d on device %d[%d]\n", outputI1[j], expectedI1[j], i, j); break; + case ncclUint8: + printf("Expected %u. Output %u on device %d[%d]\n", outputU1[j], expectedU1[j], i, j); break; + case ncclInt32: + printf("Expected %d. Output %d on device %d[%d]\n", outputI4[j], expectedI4[j], i, j); break; + case ncclUint32: + printf("Expected %u. Output %u on device %d[%d]\n", outputU4[j], expectedU4[j], i, j); break; + case ncclInt64: + printf("Expected %ld. Output %ld on device %d[%d]\n", outputI8[j], expectedI8[j], i, j); break; + case ncclUint64: + printf("Expected %lu. Output %lu on device %d[%d]\n", outputU8[j], expectedU8[j], i, j); break; + case ncclFloat32: + printf("Expected %f. Output %f on device %d[%d]\n", outputF4[j], expectedF4[j], i, j); break; + case ncclFloat64: + printf("Expected %lf. Output %lf on device %d[%d]\n", outputF8[j], expectedF8[j], i, j); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + ASSERT_EQ(isMatch, true); + } + } + + // Passed in parameters from TestTuple + ncclRedOp_t op; + ncclDataType_t dataType; + size_t numElements; + int numDevices; + bool inPlace; + + int numDevicesAvailable; + std::vector comms; + std::vector streams; + }; + +} + +#endif diff --git a/test/test_AllGather.cpp b/test/test_AllGather.cpp new file mode 100644 index 0000000000..d65a45a28e --- /dev/null +++ b/test/test_AllGather.cpp @@ -0,0 +1,63 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_AllGather.hpp" +#include + +namespace CorrectnessTests +{ + TEST_P(AllGatherCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + if (numElements % numDevices != 0) return; + + // Prepare input / output / expected results + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + FillDatasetWithPattern(dataset); + ComputeExpectedResults(dataset); + + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + size_t const sendCount = dataset.numElements / dataset.numDevices; + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclAllGather((int8_t *)dataset.inputs[i] + (i * byteCount), + dataset.outputs[i], sendCount, + dataType, comms[i], streams[i]); + } + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results + ValidateResults(dataset); + } + + INSTANTIATE_TEST_SUITE_P(AllGatherCorrectnessSweep, + AllGatherCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_AllGather.hpp b/test/test_AllGather.hpp new file mode 100644 index 0000000000..3146fbe287 --- /dev/null +++ b/test/test_AllGather.hpp @@ -0,0 +1,32 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_ALLGATHER_HPP +#define TEST_ALLGATHER_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class AllGatherCorrectnessTest : public CorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset) + { + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + + int8_t* result = (int8_t *)malloc(dataset.NumBytes()); + + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(result + i * byteCount, (int8_t *)dataset.inputs[i] + (i * byteCount), + byteCount, hipMemcpyDeviceToHost)); + + for (int i = 0; i < dataset.numDevices; i++) + memcpy(dataset.expected[i], result, dataset.NumBytes()); + } + }; +} + +#endif diff --git a/test/test_AllReduce.cpp b/test/test_AllReduce.cpp new file mode 100644 index 0000000000..d4b35b6890 --- /dev/null +++ b/test/test_AllReduce.cpp @@ -0,0 +1,59 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_AllReduce.hpp" +#include + +namespace CorrectnessTests +{ + TEST_P(AllReduceCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + + // Prepare input / output / expected results + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + FillDatasetWithPattern(dataset); + ComputeExpectedResults(dataset, op); + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclAllReduce(dataset.inputs[i], dataset.outputs[i], + numElements, dataType, op, comms[i], streams[i]); + } + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results + ValidateResults(dataset); + } + + INSTANTIATE_TEST_SUITE_P(AllReduceCorrectnessSweep, + AllReduceCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_AllReduce.hpp b/test/test_AllReduce.hpp new file mode 100644 index 0000000000..d8867cb649 --- /dev/null +++ b/test/test_AllReduce.hpp @@ -0,0 +1,76 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_ALLREDUCE_HPP +#define TEST_ALLREDUCE_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class AllReduceCorrectnessTest : public CorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, ncclRedOp_t const op) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.inputs[i], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + // Allocate temporary host array to accumulate results + int8_t* resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + + // Perform reduction on the other device arrays + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + + // Copy results into expected arrays + for (int i = 0; i < dataset.numDevices; i++) + memcpy(dataset.expected[i], resultI1, dataset.NumBytes()); + + free(resultI1); + } + }; +} + +#endif diff --git a/test/test_Broadcast.cpp b/test/test_Broadcast.cpp new file mode 100644 index 0000000000..c2f47b30ad --- /dev/null +++ b/test/test_Broadcast.cpp @@ -0,0 +1,67 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_Broadcast.hpp" +#include + +namespace CorrectnessTests +{ + TEST_P(BroadcastCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + + // Allocate data + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset); + ComputeExpectedResults(dataset, root); + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclBroadcast(dataset.inputs[i], + dataset.outputs[i], + numElements, dataType, + root, comms[i], streams[i]); + } + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results + ValidateResults(dataset); + } + } + + INSTANTIATE_TEST_SUITE_P(BroadcastCorrectnessSweep, + BroadcastCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_Broadcast.hpp b/test/test_Broadcast.hpp new file mode 100644 index 0000000000..5ccf351592 --- /dev/null +++ b/test/test_Broadcast.hpp @@ -0,0 +1,26 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_BROADCAST_HPP +#define TEST_BROADCAST_HPP + +#include "CorrectnessTest.hpp" +#include + +namespace CorrectnessTests +{ + class BroadcastCorrectnessTest : public CorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, int const root) + { + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.inputs[root], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + }; +} + +#endif diff --git a/test/test_GroupCalls.cpp b/test/test_GroupCalls.cpp new file mode 100644 index 0000000000..9bf0dd5497 --- /dev/null +++ b/test/test_GroupCalls.cpp @@ -0,0 +1,125 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_GroupCalls.hpp" + +#include "test_AllGather.hpp" +#include "test_AllReduce.hpp" +#include "test_Broadcast.hpp" +#include "test_Reduce.hpp" +#include "test_ReduceScatter.hpp" + +#include + +namespace CorrectnessTests +{ + TEST_P(GroupCallsCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + + // Create multiple datasets for group operation + std::vector datasets(5); + for (int i = 0; i < datasets.size(); i++) + { + datasets[i].Initialize(numDevices, numElements, dataType, inPlace); + FillDatasetWithPattern(datasets[i]); + } + + // Compute expected results for each dataset in group + int const root = 0; + AllGatherCorrectnessTest::ComputeExpectedResults(datasets[0]); + AllReduceCorrectnessTest::ComputeExpectedResults(datasets[1], op); + BroadcastCorrectnessTest::ComputeExpectedResults(datasets[2], root); + ReduceCorrectnessTest::ComputeExpectedResults(datasets[3], op, root); + ReduceScatterCorrectnessTest::ComputeExpectedResults(datasets[4], op); + + // Start a group call + ncclGroupStart(); + + // AllGather + size_t const byteCount = datasets[0].NumBytes() / numDevices; + size_t const elemCount = numElements / numDevices; + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), + datasets[0].outputs[i], elemCount, + dataType, comms[i], streams[i]); + } + + // AllReduce + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + ncclAllReduce(datasets[1].inputs[i], datasets[1].outputs[i], + numElements, dataType, op, comms[i], streams[i]); + } + + // Broadcast + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + ncclBroadcast(datasets[2].inputs[i], + datasets[2].outputs[i], + numElements, dataType, + root, comms[i], streams[i]); + } + + // Reduce + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + ncclReduce(datasets[3].inputs[i], + datasets[3].outputs[i], + numElements, dataType, op, + root, comms[i], streams[i]); + } + + // ReduceScatter + for (int i = 0; i < numDevices; i++) + { + ncclReduceScatter(datasets[4].inputs[i], + (int8_t *)datasets[4].outputs[i] + (i * byteCount), + elemCount, dataType, op, + comms[i], streams[i]); + HIP_CALL(hipSetDevice(i)); + } + + // Signal end of group call + ncclGroupEnd(); + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results for each collective in the group + for (int i = 0; i < 5; i++) + { + ValidateResults(datasets[i]); + } + } + + INSTANTIATE_TEST_SUITE_P(GroupCallsCorrectnessSweep, + GroupCallsCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_GroupCalls.hpp b/test/test_GroupCalls.hpp new file mode 100644 index 0000000000..00f84ce6a9 --- /dev/null +++ b/test/test_GroupCalls.hpp @@ -0,0 +1,17 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef TEST_GROUPCALLS_HPP +#define TEST_GROUPCALLS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class GroupCallsCorrectnessTest : public CorrectnessTest {}; +} + +#endif diff --git a/test/test_Reduce.cpp b/test/test_Reduce.cpp new file mode 100644 index 0000000000..089cc97593 --- /dev/null +++ b/test/test_Reduce.cpp @@ -0,0 +1,67 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_Reduce.hpp" +#include + +namespace CorrectnessTests +{ + TEST_P(ReduceCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + + // Allocate data + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset); + ComputeExpectedResults(dataset, op, root); + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclReduce(dataset.inputs[i], + dataset.outputs[i], + numElements, dataType, op, + root, comms[i], streams[i]); + } + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results + ValidateResults(dataset); + } + } + + INSTANTIATE_TEST_SUITE_P(ReduceCorrectnessSweep, + ReduceCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_Reduce.hpp b/test/test_Reduce.hpp new file mode 100644 index 0000000000..3ab9d66b44 --- /dev/null +++ b/test/test_Reduce.hpp @@ -0,0 +1,80 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_REDUCE_HPP +#define TEST_REDUCE_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class ReduceCorrectnessTest : public CorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, ncclRedOp_t const op, int const root) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.inputs[i], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + // Allocate temporary host array to accumulate results + int8_t* resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + + // Perform reduction on the other device arrays + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + + // Copy results into expected arrays + for (int i = 0; i < dataset.numDevices; i++) + { + if (i == root) + memcpy(dataset.expected[root], resultI1, dataset.NumBytes()); + else + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.outputs[i], dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + free(resultI1); + } + }; +} + +#endif diff --git a/test/test_ReduceScatter.cpp b/test/test_ReduceScatter.cpp new file mode 100644 index 0000000000..10ae2affc8 --- /dev/null +++ b/test/test_ReduceScatter.cpp @@ -0,0 +1,65 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_ReduceScatter.hpp" +#include + +namespace CorrectnessTests +{ + TEST_P(ReduceScatterCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + if (numElements % numDevices != 0) return; + + // Prepare input / output / expected results + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + FillDatasetWithPattern(dataset); + ComputeExpectedResults(dataset, op); + + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + size_t const recvCount = dataset.numElements / dataset.numDevices; + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclReduceScatter(dataset.inputs[i], + (int8_t *)dataset.outputs[i] + (i * byteCount), + recvCount, dataType, op, + comms[i], streams[i]); + } + + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + hipStreamSynchronize(streams[i]); + + // Check results + ValidateResults(dataset); + } + + INSTANTIATE_TEST_SUITE_P(ReduceScatterCorrectnessSweep, + ReduceScatterCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_ReduceScatter.hpp b/test/test_ReduceScatter.hpp new file mode 100644 index 0000000000..a1731f13b2 --- /dev/null +++ b/test/test_ReduceScatter.hpp @@ -0,0 +1,83 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_REDUCE_SCATTER_HPP +#define TEST_REDUCE_SCATTER_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class ReduceScatterCorrectnessTest : public CorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, ncclRedOp_t const op) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.inputs[i], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + // Allocate temporary host array to accumulate results + int8_t* resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + + // Perform reduction on the other device arrays + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + + // Copy results into expected arrays + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], dataset.outputs[i], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + for (int i = 0; i < dataset.numDevices; i++) + memcpy((int8_t *)dataset.expected[i] + (i * byteCount), + resultI1 + (i * byteCount), byteCount); + + free(resultI1); + } + }; +} + +#endif From 11f78df04d750bcfe618f4f0d77cf00d7d62ffb9 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Thu, 16 May 2019 21:39:39 +0000 Subject: [PATCH 02/47] Reducing scope of linker flags to rccl target to avoid warnings --- CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bc0b59fb5d..fec673021b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,11 +68,6 @@ list(APPEND CMAKE_PREFIX_PATH find_package(hip REQUIRED) -link_libraries(-amdgpu-target=gfx803 - -amdgpu-target=gfx900 - -amdgpu-target=gfx906 - -hc-function-calls) - option(BUILD_SHARED_LIBS "Build as a shared library" ON) configure_file(src/nccl.h.in ${PROJECT_BINARY_DIR}/rccl.h) @@ -138,6 +133,12 @@ if(TRACE) add_definitions(-DENABLE_TRACE) endif() +target_link_libraries(rccl + PRIVATE -amdgpu-target=gfx803 + PRIVATE -amdgpu-target=gfx900 + PRIVATE -amdgpu-target=gfx906 + PRIVATE -hc-function-calls) + if(TARGET hip::device) target_link_libraries(rccl PRIVATE hip::device) target_link_libraries(rccl INTERFACE hip::host) From 08fcce5ec9048d102601ad53d589caba4d767698 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Thu, 16 May 2019 23:13:49 +0000 Subject: [PATCH 03/47] Fixing GoogleTest to 1.8.1 and making changes to tests to support older API --- test/CMakeLists.txt.in | 2 +- test/CorrectnessTest.hpp | 13 ++++++++++ test/test_AllGather.cpp | 45 ++++++++++++++++----------------- test/test_AllReduce.cpp | 45 ++++++++++++++++----------------- test/test_Broadcast.cpp | 46 +++++++++++++++++----------------- test/test_GroupCalls.cpp | 50 ++++++++++++++++--------------------- test/test_Reduce.cpp | 45 ++++++++++++++++----------------- test/test_ReduceScatter.cpp | 46 +++++++++++++++++----------------- 8 files changed, 148 insertions(+), 144 deletions(-) diff --git a/test/CMakeLists.txt.in b/test/CMakeLists.txt.in index 128d29e7c2..af8783aad6 100644 --- a/test/CMakeLists.txt.in +++ b/test/CMakeLists.txt.in @@ -5,7 +5,7 @@ project(googletest-download NONE) include(ExternalProject) ExternalProject_Add(googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG master + GIT_TAG release-1.8.1 SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src" BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build" CONFIGURE_COMMAND "" diff --git a/test/CorrectnessTest.hpp b/test/CorrectnessTest.hpp index 638c028788..a4dbd567ac 100644 --- a/test/CorrectnessTest.hpp +++ b/test/CorrectnessTest.hpp @@ -154,7 +154,10 @@ namespace CorrectnessTests // Create streams streams.resize(numDevices); for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); HIP_CALL(hipStreamCreate(&streams[i])); + } } // Clean up per TestTuple @@ -219,6 +222,16 @@ namespace CorrectnessTests free(arrayI1); } + void Synchronize() const + { + // Wait for reduction to complete + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + HIP_CALL(hipStreamSynchronize(streams[i])); + } + } + void ValidateResults(Dataset const& dataset) const { int8_t* outputI1 = (int8_t *)malloc(dataset.NumBytes()); diff --git a/test/test_AllGather.cpp b/test/test_AllGather.cpp index d65a45a28e..c6e79fe9c8 100644 --- a/test/test_AllGather.cpp +++ b/test/test_AllGather.cpp @@ -32,32 +32,31 @@ namespace CorrectnessTests } // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results ValidateResults(dataset); } - INSTANTIATE_TEST_SUITE_P(AllGatherCorrectnessSweep, - AllGatherCorrectnessTest, - testing::Combine( - // Reduction operator (not used) - testing::Values(ncclSum), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(3072, 3145728), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(AllGatherCorrectnessSweep, + AllGatherCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace diff --git a/test/test_AllReduce.cpp b/test/test_AllReduce.cpp index d4b35b6890..f77651c84e 100644 --- a/test/test_AllReduce.cpp +++ b/test/test_AllReduce.cpp @@ -28,32 +28,31 @@ namespace CorrectnessTests } // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results ValidateResults(dataset); } - INSTANTIATE_TEST_SUITE_P(AllReduceCorrectnessSweep, - AllReduceCorrectnessTest, - testing::Combine( - // Reduction operator - testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(1024, 1048576), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(AllReduceCorrectnessSweep, + AllReduceCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace diff --git a/test/test_Broadcast.cpp b/test/test_Broadcast.cpp index c2f47b30ad..2f2a091a6d 100644 --- a/test/test_Broadcast.cpp +++ b/test/test_Broadcast.cpp @@ -34,34 +34,34 @@ namespace CorrectnessTests root, comms[i], streams[i]); } + // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results ValidateResults(dataset); } } - INSTANTIATE_TEST_SUITE_P(BroadcastCorrectnessSweep, - BroadcastCorrectnessTest, - testing::Combine( - // Reduction operator is not used - testing::Values(ncclSum), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(1024, 1048576), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(BroadcastCorrectnessSweep, + BroadcastCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace diff --git a/test/test_GroupCalls.cpp b/test/test_GroupCalls.cpp index 9bf0dd5497..d713588c13 100644 --- a/test/test_GroupCalls.cpp +++ b/test/test_GroupCalls.cpp @@ -43,7 +43,6 @@ namespace CorrectnessTests size_t const elemCount = numElements / numDevices; for (int i = 0; i < numDevices; i++) { - HIP_CALL(hipSetDevice(i)); ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), datasets[0].outputs[i], elemCount, dataType, comms[i], streams[i]); @@ -52,7 +51,6 @@ namespace CorrectnessTests // AllReduce for (int i = 0; i < numDevices; i++) { - HIP_CALL(hipSetDevice(i)); ncclAllReduce(datasets[1].inputs[i], datasets[1].outputs[i], numElements, dataType, op, comms[i], streams[i]); } @@ -60,7 +58,6 @@ namespace CorrectnessTests // Broadcast for (int i = 0; i < numDevices; i++) { - HIP_CALL(hipSetDevice(i)); ncclBroadcast(datasets[2].inputs[i], datasets[2].outputs[i], numElements, dataType, @@ -70,7 +67,6 @@ namespace CorrectnessTests // Reduce for (int i = 0; i < numDevices; i++) { - HIP_CALL(hipSetDevice(i)); ncclReduce(datasets[3].inputs[i], datasets[3].outputs[i], numElements, dataType, op, @@ -84,15 +80,13 @@ namespace CorrectnessTests (int8_t *)datasets[4].outputs[i] + (i * byteCount), elemCount, dataType, op, comms[i], streams[i]); - HIP_CALL(hipSetDevice(i)); } // Signal end of group call ncclGroupEnd(); // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results for each collective in the group for (int i = 0; i < 5; i++) @@ -101,25 +95,25 @@ namespace CorrectnessTests } } - INSTANTIATE_TEST_SUITE_P(GroupCallsCorrectnessSweep, - GroupCallsCorrectnessTest, - testing::Combine( - // Reduction operator (not used) - testing::Values(ncclSum), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(3072, 3145728), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(GroupCallsCorrectnessSweep, + GroupCallsCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace diff --git a/test/test_Reduce.cpp b/test/test_Reduce.cpp index 089cc97593..bf6c2164e5 100644 --- a/test/test_Reduce.cpp +++ b/test/test_Reduce.cpp @@ -35,33 +35,32 @@ namespace CorrectnessTests } // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results ValidateResults(dataset); } } - INSTANTIATE_TEST_SUITE_P(ReduceCorrectnessSweep, - ReduceCorrectnessTest, - testing::Combine( - // Reduction operator - testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(1024, 1048576), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(ReduceCorrectnessSweep, + ReduceCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace diff --git a/test/test_ReduceScatter.cpp b/test/test_ReduceScatter.cpp index 10ae2affc8..567ce1fef1 100644 --- a/test/test_ReduceScatter.cpp +++ b/test/test_ReduceScatter.cpp @@ -33,33 +33,33 @@ namespace CorrectnessTests comms[i], streams[i]); } + // Wait for reduction to complete - for (int i = 0; i < numDevices; i++) - hipStreamSynchronize(streams[i]); + Synchronize(); // Check results ValidateResults(dataset); } - INSTANTIATE_TEST_SUITE_P(ReduceScatterCorrectnessSweep, - ReduceScatterCorrectnessTest, - testing::Combine( - // Reduction operator - testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), - // Data types - testing::Values(ncclInt8, - ncclUint8, - ncclInt32, - ncclUint32, - ncclInt64, - ncclUint64, - //ncclFloat16, - ncclFloat32, - ncclFloat64), - // Number of elements - testing::Values(3072, 3145728), - // Number of devices - testing::Values(2,3,4), - // In-place or not - testing::Values(false, true))); + INSTANTIATE_TEST_CASE_P(ReduceScatterCorrectnessSweep, + ReduceScatterCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); } // namespace From 248e1d61cc5e6f39c439a3beb481899ab374e3a6 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Fri, 17 May 2019 17:13:40 +0000 Subject: [PATCH 04/47] Adding check for chrpath for unit tests --- test/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1af7886276..452cb4fd06 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,6 +4,12 @@ if(BUILD_TESTS) message("Going to build unit tests (Installed in /test/UnitTests)") + # chrpath is required to properly set rpath for the UnitTests executable + find_program(CHRPATH chrpath) + if(NOT CHRPATH) + message(FATAL_ERROR "chrpath is required for UnitTests. Please install (e.g. sudo apt-get install chrpath)") + endif() + # OpenMP is used to drive GPUs (one per thread) find_package(OpenMP REQUIRED) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") From 0d6a5a3d25895db5266ee2fd0dbbc5af81048844 Mon Sep 17 00:00:00 2001 From: Stanley Tsang Date: Fri, 17 May 2019 11:42:14 -0600 Subject: [PATCH 05/47] Update README.md Adding mention of requirement for chrpath for unit tests. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fc3b738227..5e6a92ff86 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ These operations are implemented using ring algorithms and have been optimized f 1. ROCm supported GPUs 2. ROCm stack installed on the system (HIP runtime & HCC) +3. For building and running the unit tests, chrpath will need to be installed on your machine first. (sudo apt-get install chrpath) ## Quickstart RCCL Build From a50c85285135e9b0ba610f1d68623ff2993ce370 Mon Sep 17 00:00:00 2001 From: Gilbert Lee Date: Fri, 17 May 2019 23:57:11 +0000 Subject: [PATCH 06/47] Adding support for alignment tests via sub-datasets Added sample alignment test for AllGather Datasets no longer free memory on destruction so Release() must be used --- test/CorrectnessTest.hpp | 33 ++++++++++++++++++++++++- test/test_AllGather.cpp | 49 +++++++++++++++++++++++++++++++++++++ test/test_AllReduce.cpp | 2 ++ test/test_Broadcast.cpp | 2 ++ test/test_GroupCalls.cpp | 1 + test/test_Reduce.cpp | 2 ++ test/test_ReduceScatter.cpp | 2 ++ 7 files changed, 90 insertions(+), 1 deletion(-) diff --git a/test/CorrectnessTest.hpp b/test/CorrectnessTest.hpp index a4dbd567ac..d0a4680ef3 100644 --- a/test/CorrectnessTest.hpp +++ b/test/CorrectnessTest.hpp @@ -60,6 +60,7 @@ namespace CorrectnessTests size_t numElements; // Number of elements per array ncclDataType_t dataType; // Data type of each input/output pointer bool inPlace; // Whether or not output pointers are same as input pointers + std::vector inputs; // Input pointers (1 per device) std::vector outputs; // Output pointers (1 per device) // May be identical to input pointers for in-place tests @@ -100,7 +101,8 @@ namespace CorrectnessTests } } - ~Dataset() + // Explicit memory release to avoid double-free from subDatasets + void Release() { for (int i = 0; i < outputs.size(); i++) { @@ -108,6 +110,35 @@ namespace CorrectnessTests hipFree(inputs[i]); free(expected[i]); } + + outputs.clear(); + } + + // Creates a dataset by pointing to an existing dataset + // Primarily to allow for testing with different starting byte-alignments + void ExtractSubDataset(size_t const startElement, + size_t const lastElement, + Dataset& subDataset) + { + ASSERT_LE(startElement, lastElement); + ASSERT_LT(lastElement, numElements); + + subDataset.numDevices = numDevices; + subDataset.numElements = lastElement - startElement + 1; + subDataset.dataType = dataType; + subDataset.inPlace = inPlace; + + subDataset.inputs.resize(numDevices); + subDataset.outputs.resize(numDevices); + subDataset.expected.resize(numDevices); + + size_t const byteOffset = (startElement * DataTypeToBytes(dataType)); + for (int i = 0; i < numDevices; i++) + { + subDataset.inputs[i] = (int8_t *)inputs[i] + byteOffset; + subDataset.outputs[i] = (int8_t *)outputs[i] + byteOffset; + subDataset.expected[i] = (int8_t *)expected[i] + byteOffset; + } } }; diff --git a/test/test_AllGather.cpp b/test/test_AllGather.cpp index c6e79fe9c8..2727514186 100644 --- a/test/test_AllGather.cpp +++ b/test/test_AllGather.cpp @@ -36,8 +36,57 @@ namespace CorrectnessTests // Check results ValidateResults(dataset); + dataset.Release(); } + TEST_P(AllGatherCorrectnessTest, Alignment) + { + if (numDevices > numDevicesAvailable) return; + if (numElements % numDevices != 0) return; + + // Allocate dataset + Dataset dataset; + dataset.Initialize(numDevices, numElements, dataType, inPlace); + + // Loop over several offsets (so that device pointers are not aligned) + for (int firstElement = 1; firstElement <= 11; firstElement += 2) + { + if (firstElement < numElements) + { + // Select last element so that total number of elements is multiple of numDevices + int const lastElement = firstElement + ((numElements - firstElement) / numDevices) * numDevices - 1; + if (lastElement >= numElements) break; + + Dataset subDataset; + dataset.ExtractSubDataset(firstElement, lastElement, subDataset); + + // Compute reference results for sub-dataset + FillDatasetWithPattern(subDataset); + ComputeExpectedResults(subDataset); + + size_t const byteCount = subDataset.NumBytes() / subDataset.numDevices; + size_t const sendCount = subDataset.numElements / subDataset.numDevices; + + // Launch the reduction (1 thread per GPU) + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclAllGather((int8_t *)subDataset.inputs[i] + (i * byteCount), + subDataset.outputs[i], sendCount, + dataType, comms[i], streams[i]); + } + + // Wait for reduction to complete + Synchronize(); + + // Check results + ValidateResults(subDataset); + } + } + dataset.Release(); + } + + INSTANTIATE_TEST_CASE_P(AllGatherCorrectnessSweep, AllGatherCorrectnessTest, testing::Combine( diff --git a/test/test_AllReduce.cpp b/test/test_AllReduce.cpp index f77651c84e..0fb7474d0e 100644 --- a/test/test_AllReduce.cpp +++ b/test/test_AllReduce.cpp @@ -32,6 +32,8 @@ namespace CorrectnessTests // Check results ValidateResults(dataset); + + dataset.Release(); } INSTANTIATE_TEST_CASE_P(AllReduceCorrectnessSweep, diff --git a/test/test_Broadcast.cpp b/test/test_Broadcast.cpp index 2f2a091a6d..3ed6964785 100644 --- a/test/test_Broadcast.cpp +++ b/test/test_Broadcast.cpp @@ -41,6 +41,8 @@ namespace CorrectnessTests // Check results ValidateResults(dataset); } + + dataset.Release(); } INSTANTIATE_TEST_CASE_P(BroadcastCorrectnessSweep, diff --git a/test/test_GroupCalls.cpp b/test/test_GroupCalls.cpp index d713588c13..de1ad0bd76 100644 --- a/test/test_GroupCalls.cpp +++ b/test/test_GroupCalls.cpp @@ -92,6 +92,7 @@ namespace CorrectnessTests for (int i = 0; i < 5; i++) { ValidateResults(datasets[i]); + datasets[i].Release(); } } diff --git a/test/test_Reduce.cpp b/test/test_Reduce.cpp index bf6c2164e5..dfca79ccf4 100644 --- a/test/test_Reduce.cpp +++ b/test/test_Reduce.cpp @@ -40,6 +40,8 @@ namespace CorrectnessTests // Check results ValidateResults(dataset); } + + dataset.Release(); } INSTANTIATE_TEST_CASE_P(ReduceCorrectnessSweep, diff --git a/test/test_ReduceScatter.cpp b/test/test_ReduceScatter.cpp index 567ce1fef1..d55b514689 100644 --- a/test/test_ReduceScatter.cpp +++ b/test/test_ReduceScatter.cpp @@ -39,6 +39,8 @@ namespace CorrectnessTests // Check results ValidateResults(dataset); + + dataset.Release(); } INSTANTIATE_TEST_CASE_P(ReduceScatterCorrectnessSweep, From e517dbed5c03c94100198521feee935f59a1e98d Mon Sep 17 00:00:00 2001 From: Wenkai Du Date: Mon, 20 May 2019 16:51:14 +0000 Subject: [PATCH 07/47] By default will not build test program --- CMakeLists.txt | 2 +- install.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fec673021b..6fbe58af90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ include(ROCMSetupVersion) include(ROCMInstallSymlinks) include(ROCMCreatePackage) -option(BUILD_TESTS "Build test programs" ON) +option(BUILD_TESTS "Build test programs" OFF) # In order to support function calls within the kernel, we must use hcc as the # compiler diff --git a/install.sh b/install.sh index 0069c5bce5..a96261b4bc 100755 --- a/install.sh +++ b/install.sh @@ -60,7 +60,11 @@ while true; do rm -rf build mkdir build cd build +if ($run_tests); then +CXX=$ROCM_PATH/hcc cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$RCCL_INSTALL .. +else CXX=$ROCM_PATH/hcc cmake -DCMAKE_INSTALL_PREFIX=$RCCL_INSTALL .. +fi make -j 8 install if ($run_tests); then From 4bfa506a6b5bc7db5f8d160e162e23a02fad10c7 Mon Sep 17 00:00:00 2001 From: Wenkai Du Date: Tue, 21 May 2019 00:08:42 +0000 Subject: [PATCH 08/47] Remove extra compiler path setting --- CMakeLists.txt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fbe58af90..258ebe47de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,10 +19,6 @@ include(ROCMCreatePackage) option(BUILD_TESTS "Build test programs" OFF) -# In order to support function calls within the kernel, we must use hcc as the -# compiler -set(CMAKE_CXX_COMPILER "/opt/rocm/bin/hcc") - # parse version from Makefile NCCL_MAJOR, NCCL_MINOR, NCCL_PATCH must exist # NCCL_SUFFIX is optional NCCL_VERSION formatting is ((X) * 1000 + (Y) * 100 + # (Z)) so we must first detect one or two digits first From afa945d6e679799cab04434ef4b14b5eae92db10 Mon Sep 17 00:00:00 2001 From: Stanley Tsang Date: Tue, 21 May 2019 15:54:41 +0000 Subject: [PATCH 09/47] Renaming jenkinsfile --- jenkinsfile => Jenkinsfile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename jenkinsfile => Jenkinsfile (100%) diff --git a/jenkinsfile b/Jenkinsfile similarity index 100% rename from jenkinsfile rename to Jenkinsfile From a115f577ddc089a7afb65d546852c57ba86b95c7 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Tue, 21 May 2019 14:34:20 -0600 Subject: [PATCH 10/47] Adding fix for unsufficient devices / better logging for skipped tests (#63) --- test/CorrectnessTest.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/CorrectnessTest.hpp b/test/CorrectnessTest.hpp index d0a4680ef3..e7d3c75382 100644 --- a/test/CorrectnessTest.hpp +++ b/test/CorrectnessTest.hpp @@ -173,8 +173,13 @@ namespace CorrectnessTests // Only proceed with testing if there are enough GPUs if (numDevices > numDevicesAvailable) { - fprintf(stdout, "Skipping test requring %d devices (only %d available)\n", + fprintf(stdout, "[ SKIPPED ] Test requires %d devices (only %d available)\n", numDevices, numDevicesAvailable); + + // Modify the number of devices so that tear-down doesn't occur + // This is temporary until GTEST_SKIP() becomes available + numDevices = 0; + numDevicesAvailable = -1; return; } From 6e8f40eb222ec994316f55b7a90af139a5bab0d2 Mon Sep 17 00:00:00 2001 From: Aaron Enye Shi Date: Wed, 22 May 2019 17:29:59 -0400 Subject: [PATCH 11/47] Update README to note install rocm-cmake (#68) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e6a92ff86..8f798687ba 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ $ cd build $ CXX=/opt/rocm/bin/hcc cmake -DCMAKE_INSTALL_PREFIX=$PWD/rccl-install .. $ make -j 8 install ``` -You may substitute a path of your own choosing for CMAKE_INSTALL_PREFIX. +You may substitute a path of your own choosing for CMAKE_INSTALL_PREFIX. Note: ensure rocm-cmake is installed, `apt install rocm-cmake`. #### To build the RCCL package and install package : From ffe2054ed2ee00c29b00a889465edf1274adb38d Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Wed, 22 May 2019 15:30:37 -0600 Subject: [PATCH 12/47] Test combined calls (#64) * Adding test for queueing multiple different collectives, 1 device per thread --- test/CMakeLists.txt | 1 + test/test_CombinedCalls.cpp | 99 +++++++++++++++++++++++++++++++++++++ test/test_CombinedCalls.hpp | 17 +++++++ 3 files changed, 117 insertions(+) create mode 100644 test/test_CombinedCalls.cpp create mode 100644 test/test_CombinedCalls.hpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 452cb4fd06..86709ee761 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -50,6 +50,7 @@ if(BUILD_TESTS) test_Reduce.cpp test_ReduceScatter.cpp test_GroupCalls.cpp + test_CombinedCalls.cpp ) add_executable(UnitTests ${TEST_SOURCES}) diff --git a/test/test_CombinedCalls.cpp b/test/test_CombinedCalls.cpp new file mode 100644 index 0000000000..bdbf55bc20 --- /dev/null +++ b/test/test_CombinedCalls.cpp @@ -0,0 +1,99 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_CombinedCalls.hpp" + +#include "test_AllGather.hpp" +#include "test_AllReduce.hpp" +#include "test_Broadcast.hpp" +#include "test_Reduce.hpp" +#include "test_ReduceScatter.hpp" + +#include + +namespace CorrectnessTests +{ + TEST_P(CombinedCallsCorrectnessTest, Correctness) + { + if (numDevices > numDevicesAvailable) return; + + // Create multiple datasets for combined operation + std::vector datasets(5); + for (int i = 0; i < datasets.size(); i++) + { + datasets[i].Initialize(numDevices, numElements, dataType, inPlace); + FillDatasetWithPattern(datasets[i]); + } + + // Compute expected results for each dataset in combined + int const root = 0; + AllGatherCorrectnessTest::ComputeExpectedResults(datasets[0]); + AllReduceCorrectnessTest::ComputeExpectedResults(datasets[1], op); + BroadcastCorrectnessTest::ComputeExpectedResults(datasets[2], root); + ReduceCorrectnessTest::ComputeExpectedResults(datasets[3], op, root); + ReduceScatterCorrectnessTest::ComputeExpectedResults(datasets[4], op); + + size_t const byteCount = datasets[0].NumBytes() / numDevices; + size_t const elemCount = numElements / numDevices; + + #pragma omp parallel for num_threads(numDevices) + for (int i = 0; i < numDevices; i++) + { + ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), + datasets[0].outputs[i], elemCount, + dataType, comms[i], streams[i]); + + ncclAllReduce(datasets[1].inputs[i], datasets[1].outputs[i], + numElements, dataType, op, comms[i], streams[i]); + + ncclBroadcast(datasets[2].inputs[i], + datasets[2].outputs[i], + numElements, dataType, + root, comms[i], streams[i]); + + ncclReduce(datasets[3].inputs[i], + datasets[3].outputs[i], + numElements, dataType, op, + root, comms[i], streams[i]); + + ncclReduceScatter(datasets[4].inputs[i], + (int8_t *)datasets[4].outputs[i] + (i * byteCount), + elemCount, dataType, op, + comms[i], streams[i]); + } + + // Wait for reduction to complete + Synchronize(); + + // Check results for each collective in the combined + for (int i = 0; i < 5; i++) + { + ValidateResults(datasets[i]); + datasets[i].Release(); + } + } + + INSTANTIATE_TEST_CASE_P(CombinedCallsCorrectnessSweep, + CombinedCallsCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true))); +} // namespace diff --git a/test/test_CombinedCalls.hpp b/test/test_CombinedCalls.hpp new file mode 100644 index 0000000000..f09e925757 --- /dev/null +++ b/test/test_CombinedCalls.hpp @@ -0,0 +1,17 @@ +/************************************************************************* + * Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef TEST_COMBINEDCALLS_HPP +#define TEST_COMBINEDCALLS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class CombinedCallsCorrectnessTest : public CorrectnessTest {}; +} + +#endif From 42c3e4b93d80441f01a6b9460aeef13cdaaf30d4 Mon Sep 17 00:00:00 2001 From: saadrahim <44449863+saadrahim@users.noreply.github.com> Date: Wed, 22 May 2019 15:31:12 -0600 Subject: [PATCH 13/47] Updating readme for 2.5 release (#67) --- README.md | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8f798687ba..d0d2356e6d 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,9 @@ ROCm Communication Collectives Library ## Introduction -RCCL (pronounced "Rickle") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on platforms using PCIe, xGMI as well as networking using InfiniBand Verbs or TCP/IP sockets. RCCL supports an arbitrary number of GPUs installed in a single node or across multiple nodes, and can be used in either single- or multi-process (e.g., MPI) applications. +RCCL (pronounced "Rickle") is a stand-alone library of standard collective communication routines for GPUs, implementing all-reduce, all-gather, reduce, broadcast, and reduce-scatter. It has been optimized to achieve high bandwidth on platforms using PCIe, xGMI as well as networking using InfiniBand Verbs or TCP/IP sockets. RCCL supports an arbitrary number of GPUs installed in a single node, and can be used in either single- or multi-process (e.g., MPI) applications. Multi node support is planned for a future release. -## What's inside - -At present, the library implements the following collectives operations: - -- all-reduce -- all-gather -- reduce-scatter -- reduce -- broadcast - -These operations are implemented using ring algorithms and have been optimized for throughput and latency. For best performance, small operations can be either batched into larger operations or aggregated through the API. +The collective operations are implemented using ring algorithms and have been optimized for throughput and latency. For best performance, small operations can be either batched into larger operations or aggregated through the API. ## Requirements @@ -32,8 +22,9 @@ In addition, HC Direct Function call support needs to be present on your machine The root of this repository has a helper script 'install.sh' to build and install RCCL on Ubuntu with a single command. It does not take a lot of options and hard-codes configuration that can be specified through invoking cmake directly, but it's a great way to get started quickly and can serve as an example of how to build/install. * `./install.sh` -- builds library including unit tests +* `./install.sh -i` -- builds and installs the library to /opt/rocm/rccl * `./install.sh -h` -- shows help -* `./install.sh -t` -- builds library including unit tests, and also runs unit tests +* `./install.sh -t` -- builds library including unit tests ## Manual build #### To build the library : @@ -44,7 +35,7 @@ $ cd rccl $ mkdir build $ cd build $ CXX=/opt/rocm/bin/hcc cmake -DCMAKE_INSTALL_PREFIX=$PWD/rccl-install .. -$ make -j 8 install +$ make -j 8 ``` You may substitute a path of your own choosing for CMAKE_INSTALL_PREFIX. Note: ensure rocm-cmake is installed, `apt install rocm-cmake`. From 4c4351673bf1857a1775a3671d8ef3143ecebad8 Mon Sep 17 00:00:00 2001 From: saadrahim <44449863+saadrahim@users.noreply.github.com> Date: Wed, 22 May 2019 15:32:32 -0600 Subject: [PATCH 14/47] Jenkinsfile (#65) * Changing Jenkinsfile to support runs without docker * Updating install file for build options * Fixing command execution * Fixing Jenkinsfile * fixing test execution * Removing junit search --- Jenkinsfile | 22 +++++------ install.sh | 109 +++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 97 insertions(+), 34 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 655f9dc7e0..c53a81da98 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,7 +1,7 @@ #!/usr/bin/env groovy // Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. // This shared library is available at https://github.com/ROCmSoftwarePlatform/rccl -@Library('rocJenkins') _ +@Library('rocJenkins@noDocker') _ // This is file for internal AMD use. // If you are interested in running your own Jenkins, please raise a github issue for assistance. @@ -32,10 +32,10 @@ rcclCI: def rccl = new rocProject('rccl') // customize for project - rccl.paths.build_command = './install.sh' + rccl.paths.build_command = './install.sh -t' // Define test architectures, optional rocm version argument is available - def nodes = new dockerNodes(['gfx906'], rccl) + def nodes = new dockerNodes(['RCCL'], rccl) boolean formatCheck = false @@ -50,7 +50,7 @@ rcclCI: LD_LIBRARY_PATH=/opt/rocm/hcc/lib CXX=${project.compiler.compiler_path} ${project.paths.build_command} """ - platform.runCommand(this, command) + sh command } def testCommand = @@ -59,12 +59,12 @@ rcclCI: def command = """#!/usr/bin/env bash set -x - cd ${project.paths.project_build_prefix}/rccl-install/test - ./UnitTest --gtest_output=xml --gtest_color=yes + cd ${project.paths.project_build_prefix}/build/release/test + HSA_FORCE_FINE_GRAIN_PCIE=1 ./UnitTests --gtest_output=xml --gtest_color=yes """ - platform.runCommand(this, command) - junit "${project.paths.project_build_prefix}/rccl-install/*.xml" + sh command + //junit "${project.paths.project_build_prefix}/build/release/*.xml" } def packageCommand = @@ -80,10 +80,10 @@ rcclCI: sudo dpkg -i package/*.deb """ - platform.runCommand(this, command) - platform.archiveArtifacts(this, """${project.paths.project_build_prefix}/build/package/*.deb""") + + //platform.archiveArtifacts(this, """${project.paths.project_build_prefix}/build/package/*.deb""") } - buildProject(rccl, formatCheck, nodes.dockerArray, compileCommand, testCommand, packageCommand) + buildProjectNoDocker(rccl, formatCheck, nodes.dockerArray, compileCommand, testCommand, packageCommand) } \ No newline at end of file diff --git a/install.sh b/install.sh index a96261b4bc..1554736762 100755 --- a/install.sh +++ b/install.sh @@ -1,10 +1,6 @@ #!/bin/bash # Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. -export RCCL_DIR=$PWD/rccl-internal -export RCCL_INSTALL=$PWD/rccl-install -export ROCM_PATH=/opt/rocm/bin - # ################################################# # helper functions # ################################################# @@ -12,10 +8,23 @@ function display_help() { echo "RCCL build & installation helper script" echo "./install [-h|--help] " - echo " [-h|--help] prints this help message" - echo " [-t|--test] run RCCL unit tests too" + echo " [-h|--help] prints this help message." + echo " [-p|--package_build] Build RCCL package." + echo " [-t|--tests_build] Build unit tests, but do not run." + echo " [-r|--run_tests] Run unit tests (must be built already.)" + echo " [--prefix] Specify custom directory to install RCCL to (default: /opt/rocm/rccl)." } +# ################################################# +# global variables +# ################################################# +build_package=false +install_prefix=/opt/rocm/rccl +build_tests=false +run_tests=false +run_tests_only=false +build_release=true +install_library=false # ################################################# # Parameter parsing # ################################################# @@ -23,7 +32,7 @@ function display_help() # check if we have a modern version of getopt that can handle whitespace and long parameters getopt -T if [[ $? -eq 4 ]]; then - GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,install,clients,debug,test --options hicdt -- "$@") + GETOPT_PARSE=$(getopt --name "${0}" --longoptions help,package_build_only,tests_build,run_tests,prefix: --options hptr -- "$@") else echo "Need a new version of getopt" exit 1 @@ -36,17 +45,27 @@ fi eval set -- "${GETOPT_PARSE}" -run_tests=false - while true; do case "${1}" in -h|--help) - display_help - exit 0 - ;; - -t|--test) + display_help + exit 0 + ;; + -i|--install) + install_library=true + shift ;; + -p|--package_build) + build_package=true + shift ;; + -t|--tests_build) + build_tests=true + shift ;; + -r|--run_tests) run_tests=true shift ;; + --prefix) + install_prefix=${2} + shift 2 ;; --) shift ; break ;; *) echo "Unexpected command line parameter received; aborting"; exit 1 @@ -57,17 +76,61 @@ while true; do # Install the pre-commit hook #bash ./githooks/install -rm -rf build -mkdir build -cd build -if ($run_tests); then -CXX=$ROCM_PATH/hcc cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$RCCL_INSTALL .. +rocm_path=/opt/rocm/bin +#build_dir=./build +# ################################################# +# prep +# ################################################# +# ensure a clean build environment +if [[ "${build_release}" == true ]]; then + rm -rf ${build_dir}/release else -CXX=$ROCM_PATH/hcc cmake -DCMAKE_INSTALL_PREFIX=$RCCL_INSTALL .. + rm -rf ${build_dir}/debug fi -make -j 8 install -if ($run_tests); then -# Optionally, run tests if they're enabled. -HSA_FORCE_FINE_GRAIN_PCIE=1 $RCCL_INSTALL/test/UnitTests + +# Create and go to the build directory. +mkdir -p build; cd build + +if ($build_release); then + mkdir -p release; cd release +else + mkdir -p debug; cd debug +fi + + +# build type +if [[ "${build_release}" == true ]]; then + #mkdir -p ${build_dir}/release/clients && cd ${build_dir}/release + cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Release" +else + #mkdir -p ${build_dir}/debug/clients && cd ${build_dir}/debug + cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Debug" +fi + + +#if !($run_tests_only); then +# cd build + +if ($build_tests); then + CXX=$rocm_path/hcc cmake -DBUILD_TESTS=ON -DCMAKE_INSTALL_PREFIX=$install_prefix ../../. +else + CXX=$rocm_path/hcc cmake -DBUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=$install_prefix ../../. +fi + +make -j$(nproc) + + if ($build_package); then + make package + fi +#fi + +# Optionally, run tests if they're enabled. +if ($run_tests); then + if (test -f "./test/UnitTests"); then + HSA_FORCE_FINE_GRAIN_PCIE=1 ./test/UnitTests + else + echo "Unit tests have not been built yet; please re-run script with -t to build unit tests." + exit 1 + fi fi From 1bb6d2104c07a1c7470f11c0700b145164ccbfe5 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Thu, 23 May 2019 15:52:17 -0700 Subject: [PATCH 15/47] Add RCCL primitive testing (#70) --- tools/rccl-prim-test/Makefile | 16 ++ tools/rccl-prim-test/copy_kernel.h | 255 +++++++++++++++++ tools/rccl-prim-test/rccl_prim_test.cpp | 362 ++++++++++++++++++++++++ 3 files changed, 633 insertions(+) create mode 100644 tools/rccl-prim-test/Makefile create mode 100644 tools/rccl-prim-test/copy_kernel.h create mode 100644 tools/rccl-prim-test/rccl_prim_test.cpp diff --git a/tools/rccl-prim-test/Makefile b/tools/rccl-prim-test/Makefile new file mode 100644 index 0000000000..8a65d7db8c --- /dev/null +++ b/tools/rccl-prim-test/Makefile @@ -0,0 +1,16 @@ +HIP_PATH?= $(wildcard /opt/rocm/hip) +ifeq (,$(HIP_PATH)) + HIP_PATH=../../.. +endif +HIPCC=$(HIP_PATH)/bin/hipcc + +EXE=rccl_prim_test +CXXFLAGS = -O3 -g -I/opt/rocm/rocrand/include + +all: $(EXE) + +$(EXE): rccl_prim_test.cpp + $(HIPCC) $(CXXFLAGS) $^ -o $@ + +clean: + rm -f *.o $(EXE) \ No newline at end of file diff --git a/tools/rccl-prim-test/copy_kernel.h b/tools/rccl-prim-test/copy_kernel.h new file mode 100644 index 0000000000..f4de543efd --- /dev/null +++ b/tools/rccl-prim-test/copy_kernel.h @@ -0,0 +1,255 @@ +/************************************************************************* + * Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + + +#ifndef COPY_KERNEL_H_ +#define COPY_KERNEL_H_ +#include +#include + +// Define min for ssize_t +static __device__ int min(int a, ssize_t b) { return (a < b) ? a : b; } + +typedef uint64_t PackType; + +template +struct MULTI { + __device__ PackType operator()(const PackType x, const PackType y) const + { + return FUNC()(x, y); + } +}; + +#define ALIGNUP(x, a) ((((x)-1) & ~((a)-1)) + (a)) + +template +__device__ inline volatile T* AlignUp(volatile T * ptr, size_t align) { + size_t ptrval = reinterpret_cast(ptr); + return reinterpret_cast(ALIGNUP(ptrval, align)); +} + +template inline __device__ +T vFetch(const volatile T* ptr) { + return *ptr; +} + +template inline __device__ +void vStore(volatile T* ptr, const T val) { + *ptr = val; +} + +template +__attribute__((noinline)) +__device__ inline void ReduceCopy( + const int tid, const int nthreads, + const volatile T * __restrict__ const src0, + const volatile T * __restrict__ const src1, + volatile T * __restrict__ const dest0, + volatile T * __restrict__ const dest1, const int N) { + for (int idx = tid; idx < N; idx += nthreads) { + T val = vFetch(src0+idx); + if (TWO_INPUTS) { + val = FUNC()(val, vFetch(src1+idx)); + } + vStore(dest0+idx, val); + if (TWO_OUTPUTS) { + vStore(dest1+idx, val); + } + } +} + +typedef ulong2 Pack128; + +template +struct MULTI128 { + __device__ void operator()(Pack128& x, Pack128& y) { + x.x = MULTI()(x.x, y.x); + x.y = MULTI()(x.y, y.y); + } +}; + +inline __device__ void Fetch128(Pack128& v, Pack128* p) { + v.x = p->x; + v.y = p->y; +} + +inline __device__ void Store128(Pack128* p, Pack128& v) { + p->x = v.x; + p->y = v.y; +} + +#define WARP_SIZE 32 +template +__attribute__((noinline)) +__device__ inline void ReduceCopy128b( const int w, const int nw, const int t, + Pack128 * src0, Pack128 * src1, Pack128 * dest0, Pack128 * dest1, + const int N) { + Pack128 t0[UNROLL]; + Pack128 t1[UNROLL]; + const Pack128* src0_end = src0 + N; + const int inc = nw * UNROLL * WARP_SIZE; + const int offset = w * UNROLL * WARP_SIZE + t; + src0 += offset; if (TWO_INPUTS) src1 += offset; + dest0 += offset; if (TWO_OUTPUTS) dest1 += offset; + + while (src0 < src0_end) { +#pragma unroll + for (int u = 0; u < UNROLL; ++u) { + Fetch128(t0[u], src0+u*WARP_SIZE); + if (TWO_INPUTS) Fetch128(t1[u], src1+u*WARP_SIZE); + } +#pragma unroll + for (int u = 0; u < UNROLL; ++u) { + if (TWO_INPUTS) MULTI128()(t0[u], t1[u]); + Store128(dest0+u*WARP_SIZE, t0[u]); + if (TWO_OUTPUTS) Store128(dest1+u*WARP_SIZE, t0[u]); + } + src0 += inc; if (TWO_INPUTS) src1 += inc; + dest0 += inc; if (TWO_OUTPUTS) dest1 += inc; + } +} + +template +__attribute__((noinline)) +__device__ inline void ReduceOrCopy(const int tid, const int nthreads, + volatile T * __restrict__ dest0, volatile T * __restrict__ dest1, + const volatile T * __restrict__ src0, const volatile T * __restrict__ src1, + int N) { + int Nrem = N; + if (Nrem <= 0) return; + + int Npreamble = (Nrem(tid, nthreads, src0, src1, dest0, dest1, Npreamble); + + Nrem -= Npreamble; + if (Nrem == 0) return; + + dest0 += Npreamble; if (HAS_DEST1) { dest1 += Npreamble; } + src0 += Npreamble; if (HAS_SRC1) { src1 += Npreamble; } + + // stage 2: fast path: use 128b loads/stores to do the bulk of the work, + // assuming the pointers we have are all 128-bit alignable. + int w = tid / WARP_SIZE; // Warp number + int nw = nthreads / WARP_SIZE; // Number of warps + int t = tid % WARP_SIZE; // Thread (inside the warp) + + const int PackFactor = sizeof(Pack128) / sizeof(T); + + // stage 2a: main loop + int Nalign2a = (Nrem / (PackFactor * UNROLL * nthreads)) + * (UNROLL * nthreads); // round down + + ReduceCopy128b(w, nw, t, (Pack128*)src0, (Pack128*)src1, (Pack128*)dest0, (Pack128*)dest1, Nalign2a); + + int Ndone2a = Nalign2a * PackFactor; + Nrem -= Ndone2a; + if (Nrem == 0) return; + dest0 += Ndone2a; if (HAS_DEST1) { dest1 += Ndone2a; } + src0 += Ndone2a; if (HAS_SRC1) { src1 += Ndone2a; } + + // stage 2b: slightly less optimized for section when we don't have full + // UNROLLs + + int Nalign2b = Nrem / PackFactor; + + ReduceCopy128b(w, nw, t, (Pack128*)src0, (Pack128*)src1, (Pack128*)dest0, (Pack128*)dest1, Nalign2b); + + int Ndone2b = Nalign2b * PackFactor; + Nrem -= Ndone2b; + if (Nrem == 0) return; + dest0 += Ndone2b; if (HAS_DEST1) { dest1 += Ndone2b; } + src0 += Ndone2b; if (HAS_SRC1) { src1 += Ndone2b; } + + // stage 2c: tail + ReduceCopy(tid, nthreads, src0, src1, dest0, dest1, Nrem); +} + +template +struct FuncPassA { + __device__ T operator()(const T x, const T y) const { + return x; + } +}; + +template +struct FuncSum { + __device__ T operator()(const T x, const T y) const { + return x + y; + } +}; + +template +struct MULTI { + static_assert(sizeof(PackType) == 2 * sizeof(float), + "PackType must be twice the size of float."); + union converter { + PackType storage; + struct { + float a, b; + }; + }; + + __device__ PackType operator()(const PackType x, const PackType y) const { + converter cx, cy, cr; + cx.storage = x; + cy.storage = y; + + cr.a = FUNC()(cx.a, cy.a); + cr.b = FUNC()(cx.b, cy.b); + + return cr.storage; + } +}; + +// Assumptions: +// - there is exactly 1 block +// - THREADS is the number of producer threads +// - this function is called by all producer threads +template +__device__ void Copy(volatile T * __restrict__ const dest, + const volatile T * __restrict__ const src, const int N) { + ReduceOrCopy, T, false, false>(threadIdx.x, THREADS, + dest, nullptr, src, nullptr, N); +} + +template +__device__ void DoubleCopy(volatile T * __restrict__ const dest0, + volatile T * __restrict__ const dest1, + const volatile T * __restrict__ const src, const int N) { + ReduceOrCopy, T, true, false>(threadIdx.x, THREADS, + dest0, dest1, src, nullptr, N); +} + +template +__device__ void Reduce(volatile T * __restrict__ const dest, + const volatile T * __restrict__ const src0, + const volatile T * __restrict__ const src1, const int N) { + ReduceOrCopy, T, false, true>(threadIdx.x, THREADS, + dest, nullptr, src0, src1, N); +} + +template +__device__ void ReduceCopy(volatile T * __restrict__ const dest0, + volatile T * __restrict__ const dest1, + const volatile T * __restrict__ const src0, + const volatile T * __restrict__ const src1, const int N) { + ReduceOrCopy, T, true, true>(threadIdx.x, THREADS, + dest0, dest1, src0, src1, N); +} +#endif // COPY_KERNEL_H_ diff --git a/tools/rccl-prim-test/rccl_prim_test.cpp b/tools/rccl-prim-test/rccl_prim_test.cpp new file mode 100644 index 0000000000..07aeea567b --- /dev/null +++ b/tools/rccl-prim-test/rccl_prim_test.cpp @@ -0,0 +1,362 @@ +/* +Copyright (c) 2019 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +/** + * @file rccl_prim_test.cpp + * + * test performance if individual rccl primitives + */ +#include //fprintf +#include //cerr +#include //usleep +#include +#include +#include +#include "copy_kernel.h" + +#define MAX_WORKGROUPS 8 +#define THREADS 256 +#define UNROLL 8 + +#define NUM_ITERS 10 + +struct transfer_data_t { + float *dest0; //remote fine grain + float *src0; //local fine grain + float *dest1; //local coarse grain + float *src1; //local coarse grain + int N; + int gpu; +}; + +struct profiling_data_t { + uint64_t write_cycles; + uint64_t bytes_transferred; +}; + + +#define LOAD(VAR) __atomic_load_n((VAR), __ATOMIC_SEQ_CST) +#define STORE(DST, SRC) __atomic_store_n((DST), (SRC), __ATOMIC_SEQ_CST) + +enum Ops { + OP_COPY, + OP_LOCALCOPY, + OP_DOUBLECOPY, + OP_REDUCE, + OP_REDUCECOPY, + NUM_OPS, +}; + +template +__global__ void flag_sync_kernel(struct transfer_data_t* transfer_data, struct profiling_data_t* profiling_data) { + size_t idx = threadIdx.x; + uint64_t curr_time, next_time; + + if (idx == 0) { + curr_time = clock(); + } + __syncthreads(); + + int offset = transfer_data->N * blockIdx.x / gridDim.x; + int n = transfer_data->N / gridDim.x; + if (op == OP_COPY) Copy(transfer_data->dest0 + offset, transfer_data->src0 + offset, n); + if (op == OP_LOCALCOPY) Copy(transfer_data->dest1 + offset, transfer_data->src0 + offset, n); + if (op == OP_DOUBLECOPY) DoubleCopy(transfer_data->dest0 + offset, transfer_data->dest1 + offset, transfer_data->src0 + offset, n); + if (op == OP_REDUCE) Reduce(transfer_data->dest0 + offset, transfer_data->src0 + offset, transfer_data->src1 + offset, n); + if (op == OP_REDUCECOPY) ReduceCopy(transfer_data->dest0 + offset, transfer_data->dest1 + offset, transfer_data->src0 + offset, transfer_data->src1 + offset, n); + + if (idx == 0) { + next_time = clock(); + __atomic_fetch_add(&(profiling_data->write_cycles), next_time - curr_time, __ATOMIC_SEQ_CST); + curr_time = next_time; + __atomic_fetch_add(&(profiling_data->bytes_transferred), n * sizeof(float), __ATOMIC_SEQ_CST); + } +} + +typedef void(*flag_sync_kernel_t)(struct transfer_data_t* transfer_data, struct profiling_data_t* profiling_data); + +static flag_sync_kernel_t const flagSyncKerns[NUM_OPS] = { + flag_sync_kernel, + flag_sync_kernel, + flag_sync_kernel, + flag_sync_kernel, + flag_sync_kernel, +}; + +__global__ void initTestDataKernel(float* data, const size_t N, const int gpu) { + int tid = threadIdx.x + blockIdx.x * blockDim.x; + while (tid < N) { + data[tid] = 1.0/(float)(gpu*17 + tid%77); + tid += blockDim.x * gridDim.x; + } +} + +#define HIPCHECK(cmd) \ +do { \ + hipError_t error = (cmd); \ + if (error != hipSuccess) \ + { \ + std::cerr << "Encountered HIP error (" << error << ") at line " \ + << __LINE__ << " in file " << __FILE__ << "\n"; \ + exit(-1); \ + } \ +} while (0) + +static void setupPeers() { + int deviceCnt, dev; + + HIPCHECK(hipGetDeviceCount(&deviceCnt)); + HIPCHECK(hipGetDevice(&dev)); + //! If gpus are not peer enabled, enable them + for (int i = 0; i < deviceCnt; i++) { + HIPCHECK(hipSetDevice(i)); + for (int j = 0; j < deviceCnt; j++) { + if (i != j) { + HIPCHECK(hipDeviceEnablePeerAccess(j, 0)); + } + } + } + HIPCHECK(hipSetDevice(dev)); +} + +char* getCmdOption(char ** begin, char ** end, const std::string & option) { + char ** itr = std::find(begin, end, option); + if (itr != end && ++itr != end) + { + return *itr; + } + return 0; +} + +bool cmdOptionExists(char** begin, char** end, const std::string& option) { + return std::find(begin, end, option) != end; +} + +int main(int argc,char* argv[]) +{ + if (cmdOptionExists(argv, argv + argc, "-h")) { + printf("./rccl_prim_test -w num_workgroups -p copy|localcopy|doublecopy|reduce|reducecopy|all\n"); + exit(0); + } + + int workgroups = 1; + char *wg = getCmdOption(argv, argv + argc, "-w"); + if (wg) + workgroups = atol(wg); + printf("Benchmarking using %d workgroups\n", workgroups); + + const char *ops[] = {"copy", "localcopy", "doublecopy", "reduce", "reducecopy", "all"}; + char *prim = getCmdOption(argv, argv + argc, "-p"); + int op = 5, begin_op, end_op; + if (prim) { + for (op = 0; op < sizeof(ops); op++) + if (!strcmp((const char *)prim, ops[op])) + break; + } + if (op < NUM_OPS ) { + begin_op = op; + end_op = op + 1; + } else { + begin_op = 0; + end_op = NUM_OPS; + printf("Benchmarking all ops\n"); + } + + // Enable peer access + setupPeers(); + + // data buffers + float *buff_0, *buff_1, *buff_coarse_0, *buff_coarse_1; + struct transfer_data_t h_transfer_data_0, h_transfer_data_1, *transfer_data_0, *transfer_data_1; + struct profiling_data_t *profiling_data_0, *profiling_data_1, *d_profiling_data_0, *d_profiling_data_1; + uint64_t N = 2097152*4*MAX_WORKGROUPS; + + HIPCHECK(hipSetDevice(0)); + HIPCHECK(hipExtMallocWithFlags((void**) &transfer_data_0, sizeof(struct transfer_data_t), hipDeviceMallocFinegrained)); + //printf("GPU 0: allocated fine grain VRAM at %llx\n", (unsigned long long)transfer_data_0); + HIPCHECK(hipExtMallocWithFlags((void**) &buff_0, 2*N*sizeof(float), hipDeviceMallocFinegrained)); + //printf("GPU 0: allocated fine grain VRAM at %llx\n", (unsigned long long)buff_0); + HIPCHECK(hipMalloc((void**) &buff_coarse_0, 2*N*sizeof(float))); + //printf("GPU 0: allocated coarse grain VRAM at %llx\n", (unsigned long long)buff_coarse_0); + profiling_data_0 = (struct profiling_data_t *)malloc(sizeof(struct profiling_data_t)); + HIPCHECK(hipMalloc((void**) &d_profiling_data_0, sizeof(struct profiling_data_t))); + //create stream + hipStream_t stream_0; + HIPCHECK(hipStreamCreate(&stream_0)); + //randomize test data + hipLaunchKernelGGL(initTestDataKernel, + /*grid dim x,y,z*/ dim3(32, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_0, + /*kernel args*/ buff_0, 2*N, 0); + hipLaunchKernelGGL(initTestDataKernel, + /*grid dim x,y,z*/ dim3(32, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_0, + /*kernel args*/ buff_coarse_0, 2*N, 0); + + HIPCHECK(hipSetDevice(1)); + HIPCHECK(hipExtMallocWithFlags((void**) &transfer_data_1, sizeof(struct transfer_data_t), hipDeviceMallocFinegrained)); + //printf("GPU 1: allocated fine grain VRAM at %llx\n", (unsigned long long)transfer_data_1); + HIPCHECK(hipExtMallocWithFlags((void**) &buff_1, 2*N*sizeof(float), hipDeviceMallocFinegrained)); + //printf("GPU 1: allocated fine grain VRAM at %llx\n", (unsigned long long)buff_1); + HIPCHECK(hipMalloc((void**) &buff_coarse_1, 2*N*sizeof(float))); + //printf("GPU 1: allocated coarse grain VRAM at %llx\n", (unsigned long long)buff_coarse_1); + profiling_data_1 = (struct profiling_data_t *)malloc(sizeof(struct profiling_data_t)); + HIPCHECK(hipMalloc((void**) &d_profiling_data_1, sizeof(struct profiling_data_t))); + //create stream + hipStream_t stream_1; + HIPCHECK(hipStreamCreate(&stream_1)); + //randomize test data + hipLaunchKernelGGL(initTestDataKernel, + /*grid dim x,y,z*/ dim3(32, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_1, + /*kernel args*/ buff_1, 2*N, 1); + hipLaunchKernelGGL(initTestDataKernel, + /*grid dim x,y,z*/ dim3(32, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_1, + /*kernel args*/ buff_coarse_1, 2*N, 1); + + h_transfer_data_0.dest0 = buff_1; + h_transfer_data_0.dest1 = buff_coarse_0 + N; + h_transfer_data_0.src0 = buff_0; + h_transfer_data_0.src1 = buff_coarse_0; + h_transfer_data_0.N = N; + h_transfer_data_0.gpu = 0; + + h_transfer_data_1.dest0 = buff_0 + N; + h_transfer_data_1.dest1 = buff_coarse_1; + h_transfer_data_1.src0 = buff_1 + N; + h_transfer_data_1.src1 = buff_coarse_1 + N; + h_transfer_data_1.N = N; + h_transfer_data_1.gpu = 1; + + HIPCHECK(hipSetDevice(0)); + HIPCHECK(hipMemcpyAsync(transfer_data_0, &h_transfer_data_0, + sizeof(struct transfer_data_t), hipMemcpyHostToDevice, + stream_0)); + HIPCHECK(hipStreamSynchronize(stream_0)); + + HIPCHECK(hipSetDevice(1)); + HIPCHECK(hipMemcpyAsync(transfer_data_1, &h_transfer_data_1, + sizeof(struct transfer_data_t), hipMemcpyHostToDevice, + stream_1)); + HIPCHECK(hipStreamSynchronize(stream_1)); + + for (int op = begin_op; op < end_op; op ++) { + const char *OpsName[] = {"Copy", "Local Copy", "Double Copy", "Reduce", "ReduceCopy"}; + printf("Testing %s: \n", OpsName[op]); + // 2 warm up cycles + for (int i = 0; i < 2; i ++) { + HIPCHECK(hipSetDevice(0)); + //launch the kernel + hipLaunchKernelGGL(flagSyncKerns[op], + /*grid dim x,y,z*/ dim3(workgroups, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_0, + /*kernel args*/ transfer_data_0, d_profiling_data_0); + + HIPCHECK(hipSetDevice(1)); + //launch the kernel + hipLaunchKernelGGL(flagSyncKerns[op], + /*grid dim x,y,z*/ dim3(workgroups, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_1, + /*kernel args*/ transfer_data_1, d_profiling_data_1); + } + + HIPCHECK(hipSetDevice(0)); + HIPCHECK(hipStreamSynchronize(stream_0)); + HIPCHECK(hipMemset(d_profiling_data_0, 0, sizeof(struct profiling_data_t))); + HIPCHECK(hipSetDevice(1)); + HIPCHECK(hipStreamSynchronize(stream_1)); + HIPCHECK(hipMemset(d_profiling_data_1, 0, sizeof(struct profiling_data_t))); + + auto start = std::chrono::high_resolution_clock::now(); + for (int i = 0; i < NUM_ITERS; i ++) { + HIPCHECK(hipSetDevice(0)); + //launch the kernel + hipLaunchKernelGGL(flagSyncKerns[op], + /*grid dim x,y,z*/ dim3(workgroups, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_0, + /*kernel args*/ transfer_data_0, d_profiling_data_0); + + HIPCHECK(hipSetDevice(1)); + //launch the kernel + hipLaunchKernelGGL(flagSyncKerns[op], + /*grid dim x,y,z*/ dim3(workgroups, 1, 1), + /*block dim x,y,z*/ dim3(THREADS, 1, 1), + /*dynamic shared mem*/ 0, + /*stream*/ stream_1, + /*kernel args*/ transfer_data_1, d_profiling_data_1); + } + + HIPCHECK(hipSetDevice(0)); + HIPCHECK(hipStreamSynchronize(stream_0)); + HIPCHECK(hipSetDevice(1)); + HIPCHECK(hipStreamSynchronize(stream_1)); + auto delta = std::chrono::high_resolution_clock::now() - start; + double deltaSec = std::chrono::duration_cast>(delta).count(); + + HIPCHECK(hipMemcpyAsync(profiling_data_0, d_profiling_data_0, + sizeof(struct profiling_data_t), hipMemcpyDeviceToHost, + stream_0)); + HIPCHECK(hipStreamSynchronize(stream_0)); + + HIPCHECK(hipMemcpyAsync(profiling_data_1, d_profiling_data_1, + sizeof(struct profiling_data_t), hipMemcpyDeviceToHost, + stream_1)); + HIPCHECK(hipStreamSynchronize(stream_1)); + + double speed = (double)(profiling_data_0->bytes_transferred) / (deltaSec*1.0E9); + printf("Transfered %lu bytes in %f s. Throughput %f GB/s\n", profiling_data_0->bytes_transferred, deltaSec, speed); + + fprintf(stderr, "GPU 0: write_cycles %ld bytes_transferred %ld\n", + profiling_data_0->write_cycles, profiling_data_0->bytes_transferred); + + fprintf(stderr, "GPU 1: write_cycles %ld bytes_transferred %ld\n", + profiling_data_1->write_cycles, profiling_data_1->bytes_transferred); + } + + HIPCHECK(hipStreamDestroy(stream_0)); + HIPCHECK(hipStreamDestroy(stream_1)); + HIPCHECK(hipFree((void*) transfer_data_0)); + HIPCHECK(hipFree((void*) buff_0)); + HIPCHECK(hipFree((void*) buff_coarse_0)); + HIPCHECK(hipFree((void*) d_profiling_data_0)); + free(profiling_data_0); + HIPCHECK(hipFree((void*) transfer_data_1)); + HIPCHECK(hipFree((void*) buff_1)); + HIPCHECK(hipFree((void*) buff_coarse_1)); + HIPCHECK(hipFree((void*) d_profiling_data_1)); + free(profiling_data_1); +} From b921279a21bed17997da05943eb72ab943c7e8da Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Thu, 23 May 2019 18:53:25 -0400 Subject: [PATCH 16/47] Fix build failure for hip-clang (#69) --- CMakeLists.txt | 22 ++++++++++++++++++---- src/collectives/device/common.h | 4 ++-- src/transport/p2p.cu | 1 + 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 258ebe47de..f21f596e19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,8 @@ list(APPEND CMAKE_PREFIX_PATH /opt/rocm/hcc) find_package(hip REQUIRED) +message(STATUS "HIP compiler: ${HIP_COMPILER}") +message(STATUS "HIP runtime: ${HIP_RUNTIME}") option(BUILD_SHARED_LIBS "Build as a shared library" ON) @@ -130,10 +132,22 @@ if(TRACE) endif() target_link_libraries(rccl - PRIVATE -amdgpu-target=gfx803 - PRIVATE -amdgpu-target=gfx900 - PRIVATE -amdgpu-target=gfx906 - PRIVATE -hc-function-calls) + PRIVATE --amdgpu-target=gfx803 + PRIVATE --amdgpu-target=gfx900 + PRIVATE --amdgpu-target=gfx906) + +if("${HIP_COMPILER}" MATCHES "clang") + target_compile_options(rccl + PRIVATE --amdgpu-target=gfx803 + PRIVATE --amdgpu-target=gfx900 + PRIVATE --amdgpu-target=gfx906 + PRIVATE -fgpu-rdc) + target_link_libraries(rccl PRIVATE -fgpu-rdc) +endif() + +if("${HIP_COMPILER}" MATCHES "hcc") + target_link_libraries(rccl PRIVATE -hc-function-calls) +endif() if(TARGET hip::device) target_link_libraries(rccl PRIVATE hip::device) diff --git a/src/collectives/device/common.h b/src/collectives/device/common.h index 2f230c6b4b..9048011aa3 100644 --- a/src/collectives/device/common.h +++ b/src/collectives/device/common.h @@ -22,7 +22,7 @@ extern __device__ const ncclKern_t ncclFuncs[]; template struct Caller { static - void call(ncclColl* const c) noexcept + __device__ void call(ncclColl* const c) noexcept { constexpr unsigned short m = f + (l - f) / 2; @@ -33,7 +33,7 @@ struct Caller { template struct Caller{ static - void call(struct ncclColl* const c) noexcept { ncclFuncs[f](&c->args); } + __device__ void call(struct ncclColl* const c) noexcept { ncclFuncs[f](&c->args); } }; inline diff --git a/src/transport/p2p.cu b/src/transport/p2p.cu index 301ad3e90d..a616ad5f6d 100644 --- a/src/transport/p2p.cu +++ b/src/transport/p2p.cu @@ -12,6 +12,7 @@ #include "param.h" #include #include +#include #include "nvmlwrap.h" #include #if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) From f45566a8bd16352e4189b2a874dac2f522ae761f Mon Sep 17 00:00:00 2001 From: Wenkai Du Date: Fri, 24 May 2019 20:58:51 +0000 Subject: [PATCH 17/47] Increase number of rings with XGMI connection Improve throughput for about 20%. Also remove P2P over PCIe which was left enabled at initial release. Signed-off-by: Wenkai Du --- src/transport/p2p.cu | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/transport/p2p.cu b/src/transport/p2p.cu index a616ad5f6d..065e2a7bc7 100644 --- a/src/transport/p2p.cu +++ b/src/transport/p2p.cu @@ -119,9 +119,9 @@ ncclResult_t p2pCanConnect(ncclTvalue_t* ret, ncclTinfo_t* myOpaqueInfo, ncclTin link_status_print_once_mask |= (1 << (myInfo->cudaDev*8 + peerInfo->cudaDev)); } if (link_type != HSA_AMD_LINK_INFO_TYPE_XGMI) { - // enable below lines on release only: disable PCIe P2P until HDP flush is implemented. - // p2p = 0; - // return ncclSuccess; + // disable PCIe P2P until HDP flush is implemented. + p2p = 0; + return ncclSuccess; } int nvlinkp2p = 0; if (link_type == HSA_AMD_LINK_INFO_TYPE_XGMI && hops == 1) @@ -290,7 +290,11 @@ int p2pComputeRingsNvLink(ncclTvalue_t* values, int nranks, int* rings, int nrin } // Duplicate the rings for direct NVLink +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) + compNrings = copyRings(nranks, rings, compNrings, compNrings*3); +#else compNrings = copyRings(nranks, rings, compNrings, compNrings*2); +#endif if (ncclCudaCompCap() == 6) *nthreads /= 2; return compNrings; From bb7542c1d9c59bba0a10c1c6d19f3c9ba944ccf9 Mon Sep 17 00:00:00 2001 From: saadrahim <44449863+saadrahim@users.noreply.github.com> Date: Fri, 24 May 2019 15:03:56 -0600 Subject: [PATCH 18/47] Readthedocs documentation support (#71) --- docs/Doxyfile | 2456 ++++++++++++++++++++++++++++++++++ docs/rocm.jpg | Bin 0 -> 6761 bytes docs/run_doc.sh | 16 + docs/run_doxygen.sh | 12 + docs/source/Makefile | 20 + docs/source/allapi.rst | 11 + docs/source/api.rst | 103 ++ docs/source/conf.py | 185 +++ docs/source/index.rst | 21 + docs/source/library.rst | 13 + docs/source/requirements.txt | 3 + src/nccl.h.in | 119 +- 12 files changed, 2908 insertions(+), 51 deletions(-) create mode 100644 docs/Doxyfile create mode 100644 docs/rocm.jpg create mode 100755 docs/run_doc.sh create mode 100755 docs/run_doxygen.sh create mode 100644 docs/source/Makefile create mode 100644 docs/source/allapi.rst create mode 100644 docs/source/api.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/library.rst create mode 100644 docs/source/requirements.txt diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 0000000000..3d28cf5388 --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,2456 @@ +# Doxyfile 1.8.10 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a double hash (##) is considered a comment and is placed in +# front of the TAG it is preceding. +# +# All text after a single hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists, items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (\" \"). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all text +# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv +# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv +# for the list of possible encodings. +# The default value is: UTF-8. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "RCCL" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = v3.0.1.0 + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "prototype interfaces compatible with ROCm platform and HiP" + +# With the PROJECT_LOGO tag one can specify a logo or an icon that is included +# in the documentation. The maximum height of the logo should not exceed 55 +# pixels and the maximum width should not exceed 200 pixels. Doxygen will copy +# the logo to the output directory. + +PROJECT_LOGO = ./rocm.jpg + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) path +# into which the generated documentation will be written. If a relative path is +# entered, it will be relative to the location where doxygen was started. If +# left blank the current directory will be used. + +OUTPUT_DIRECTORY = docBin + +# If the CREATE_SUBDIRS tag is set to YES then doxygen will create 4096 sub- +# directories (in 2 levels) under the output directory of each output format and +# will distribute the generated files over these directories. Enabling this +# option can be useful when feeding doxygen a huge amount of source files, where +# putting all generated files in the same directory would otherwise causes +# performance problems for the file system. +# The default value is: NO. + +CREATE_SUBDIRS = NO + +# If the ALLOW_UNICODE_NAMES tag is set to YES, doxygen will allow non-ASCII +# characters to appear in the names of generated files. If set to NO, non-ASCII +# characters will be escaped, for example _xE3_x81_x84 will be used for Unicode +# U+3044. +# The default value is: NO. + +ALLOW_UNICODE_NAMES = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# Possible values are: Afrikaans, Arabic, Armenian, Brazilian, Catalan, Chinese, +# Chinese-Traditional, Croatian, Czech, Danish, Dutch, English (United States), +# Esperanto, Farsi (Persian), Finnish, French, German, Greek, Hungarian, +# Indonesian, Italian, Japanese, Japanese-en (Japanese with English messages), +# Korean, Korean-en (Korean with English messages), Latvian, Lithuanian, +# Macedonian, Norwegian, Persian (Farsi), Polish, Portuguese, Romanian, Russian, +# Serbian, Serbian-Cyrillic, Slovak, Slovene, Spanish, Swedish, Turkish, +# Ukrainian and Vietnamese. +# The default value is: English. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member +# descriptions after the members that are listed in the file and class +# documentation (similar to Javadoc). Set to NO to disable this. +# The default value is: YES. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES, doxygen will prepend the brief +# description of a member or function before the detailed description +# +# Note: If both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. +# The default value is: YES. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator that is +# used to form the text in various listings. Each string in this list, if found +# as the leading text of the brief description, will be stripped from the text +# and the result, after processing the whole list, is used as the annotated +# text. Otherwise, the brief description is used as-is. If left blank, the +# following values are used ($name is automatically replaced with the name of +# the entity):The $name class, The $name widget, The $name file, is, provides, +# specifies, contains, represents, a, an and the. + +ABBREVIATE_BRIEF = "The $name class" \ + "The $name widget" \ + "The $name file" \ + is \ + provides \ + specifies \ + contains \ + represents \ + a \ + an \ + the + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# doxygen will generate a detailed section even if there is only a brief +# description. +# The default value is: NO. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. +# The default value is: NO. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES, doxygen will prepend the full path +# before files name in the file list and in the header files. If set to NO the +# shortest path that makes the file name unique will be used +# The default value is: YES. + +FULL_PATH_NAMES = YES + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of the +# path mentioned in the documentation of a class, which tells the reader which +# header file to include in order to use a class. If left blank only the name of +# the header file containing the class definition is used. Otherwise one should +# specify the list of include paths that are normally passed to the compiler +# using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but +# less readable) file names. This can be useful is your file systems doesn't +# support long names like on DOS, Mac, or CD-ROM. +# The default value is: NO. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then doxygen will interpret the +# first line (until the first dot) of a Javadoc-style comment as the brief +# description. If set to NO, the Javadoc-style will behave just like regular Qt- +# style comments (thus requiring an explicit @brief command for a brief +# description.) +# The default value is: NO. + +JAVADOC_AUTOBRIEF = NO + +# If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first +# line (until the first dot) of a Qt-style comment as the brief description. If +# set to NO, the Qt-style will behave just like regular Qt-style comments (thus +# requiring an explicit \brief command for a brief description.) +# The default value is: NO. + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make doxygen treat a +# multi-line C++ special comment block (i.e. a block of //! or /// comments) as +# a brief description. This used to be the default behavior. The new default is +# to treat a multi-line C++ comment block as a detailed description. Set this +# tag to YES if you prefer the old behavior instead. +# +# Note that setting this tag to YES also means that rational rose comments are +# not recognized any more. +# The default value is: NO. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES then doxygen will produce a new +# page for each member. If set to NO, the documentation of a member will be part +# of the file/class/namespace that contains it. +# The default value is: NO. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. Doxygen +# uses this value to replace tabs by spaces in code fragments. +# Minimum value: 1, maximum value: 16, default value: 4. + +TAB_SIZE = 4 + +# This tag can be used to specify a number of aliases that act as commands in +# the documentation. An alias has the form: +# name=value +# For example adding +# "sideeffect=@par Side Effects:\n" +# will allow you to put the command \sideeffect (or @sideeffect) in the +# documentation, which will result in a user-defined paragraph with heading +# "Side Effects:". You can put \n's in the value part of an alias to insert +# newlines. + +ALIASES = + +# This tag can be used to specify a number of word-keyword mappings (TCL only). +# A mapping has the form "name=value". For example adding "class=itcl::class" +# will allow you to use the command class in the itcl::class meaning. + +TCL_SUBST = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources +# only. Doxygen will then generate output that is more tailored for C. For +# instance, some of the names that are used will be different. The list of all +# members will be omitted, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java or +# Python sources only. Doxygen will then generate output that is more tailored +# for that language. For instance, namespaces will be presented as packages, +# qualified scopes will look different, etc. +# The default value is: NO. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources. Doxygen will then generate output that is tailored for Fortran. +# The default value is: NO. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for VHDL. +# The default value is: NO. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given +# extension. Doxygen has a built-in mapping, but you can override or extend it +# using this tag. The format is ext=language, where ext is a file extension, and +# language is one of the parsers supported by doxygen: IDL, Java, Javascript, +# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran: +# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran: +# Fortran. In the later case the parser tries to guess whether the code is fixed +# or free formatted code, this is the default for Fortran type files), VHDL. For +# instance to make doxygen treat .inc files as Fortran files (default is PHP), +# and .f files as C (default is Fortran), use: inc=Fortran f=C. +# +# Note: For files without extension you can use no_extension as a placeholder. +# +# Note that for custom extensions you also need to set FILE_PATTERNS otherwise +# the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should set this +# tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); +# versus func(std::string) {}). This also make the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. +# The default value is: NO. + +BUILTIN_STL_SUPPORT = NO + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. +# The default value is: NO. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip (see: +# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen +# will parse them like normal C++ but will assume all classes use public instead +# of private inheritance when no explicit protection keyword is present. +# The default value is: NO. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate +# getter and setter methods for a property. Setting this option to YES will make +# doxygen to replace the get and set methods by a property in the documentation. +# This will only work if the methods are indeed getting or setting a simple +# type. If this is not the case, or you want to show the methods anyway, you +# should set this option to NO. +# The default value is: YES. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. +# The default value is: NO. + +DISTRIBUTE_GROUP_DOC = YES + +# If one adds a struct or class to a group and this option is enabled, then also +# any nested class or struct is added to the same group. By default this option +# is disabled and one has to add nested compounds explicitly via \ingroup. +# The default value is: NO. + +GROUP_NESTED_COMPOUNDS = NO + +# Set the SUBGROUPING tag to YES to allow class member groups of the same type +# (for instance a group of public functions) to be put as a subgroup of that +# type (e.g. under the Public Functions section). Set it to NO to prevent +# subgrouping. Alternatively, this can be done per class using the +# \nosubgrouping command. +# The default value is: YES. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and unions +# are shown inside the group in which they are included (e.g. using \ingroup) +# instead of on a separate page (for HTML and Man pages) or section (for LaTeX +# and RTF). +# +# Note that this feature does not work in combination with +# SEPARATE_MEMBER_PAGES. +# The default value is: NO. + +INLINE_GROUPED_CLASSES = NO + +# When the INLINE_SIMPLE_STRUCTS tag is set to YES, structs, classes, and unions +# with only public data fields or simple typedef fields will be shown inline in +# the documentation of the scope in which they are defined (i.e. file, +# namespace, or group documentation), provided this scope is documented. If set +# to NO, structs, classes, and unions are shown on a separate page (for HTML and +# Man pages) or section (for LaTeX and RTF). +# The default value is: NO. + +INLINE_SIMPLE_STRUCTS = NO + +# When TYPEDEF_HIDES_STRUCT tag is enabled, a typedef of a struct, union, or +# enum is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically be +# useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. +# The default value is: NO. + +TYPEDEF_HIDES_STRUCT = YES + +# The size of the symbol lookup cache can be set using LOOKUP_CACHE_SIZE. This +# cache is used to resolve symbols given their name and scope. Since this can be +# an expensive process and often the same symbol appears multiple times in the +# code, doxygen keeps a cache of pre-resolved symbols. If the cache is too small +# doxygen will become slower. If the cache is too large, memory is wasted. The +# cache size is given by this formula: 2^(16+LOOKUP_CACHE_SIZE). The valid range +# is 0..9, the default is 0, corresponding to a cache size of 2^16=65536 +# symbols. At the end of a run doxygen will report the cache usage and suggest +# the optimal cache size from a speed point of view. +# Minimum value: 0, maximum value: 9, default value: 0. + +LOOKUP_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +SHOW_NAMESPACES = NO + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES, all private members of a class will +# be included in the documentation. +# The default value is: NO. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal +# scope will be included in the documentation. +# The default value is: NO. + +EXTRACT_PACKAGE = NO + +# If the EXTRACT_STATIC tag is set to YES, all static members of a file will be +# included in the documentation. +# The default value is: NO. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined +# locally in source files will be included in the documentation. If set to NO, +# only classes defined in header files are included. Does not have any effect +# for Java sources. +# The default value is: YES. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. If set to YES, local methods, +# which are defined in the implementation section but not in the interface are +# included in the documentation. If set to NO, only methods in the interface are +# included. +# The default value is: NO. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base name of +# the file that contains the anonymous namespace. By default anonymous namespace +# are hidden. +# The default value is: NO. + +EXTRACT_ANON_NSPACES = NO + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend +# (class|struct|union) declarations. If set to NO, these declarations will be +# included in the documentation. +# The default value is: NO. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, doxygen will hide any +# documentation blocks found inside the body of a function. If set to NO, these +# blocks will be appended to the function's detailed documentation block. +# The default value is: NO. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation that is typed after a +# \internal command is included. If the tag is set to NO then the documentation +# will be excluded. Set it to YES to include the internal documentation. +# The default value is: NO. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file +# names in lower-case letters. If set to YES, upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. +# The default value is: system dependent. + +CASE_SENSE_NAMES = NO + +# If the HIDE_SCOPE_NAMES tag is set to NO then doxygen will show members with +# their full class and namespace scopes in the documentation. If set to YES, the +# scope will be hidden. +# The default value is: NO. + +HIDE_SCOPE_NAMES = NO + +# If the HIDE_COMPOUND_REFERENCE tag is set to NO (default) then doxygen will +# append additional text to a page's title, such as Class Reference. If set to +# YES the compound reference will be hidden. +# The default value is: NO. + +HIDE_COMPOUND_REFERENCE= NO + +# If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of +# the files that are included by a file in the documentation of that file. +# The default value is: YES. + +SHOW_INCLUDE_FILES = YES + +# If the SHOW_GROUPED_MEMB_INC tag is set to YES then Doxygen will add for each +# grouped member an include statement to the documentation, telling the reader +# which file to include in order to use the member. +# The default value is: NO. + +SHOW_GROUPED_MEMB_INC = NO + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then doxygen will list include +# files with double quotes in the documentation rather than with sharp brackets. +# The default value is: NO. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES then a tag [inline] is inserted in the +# documentation for inline members. +# The default value is: YES. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES then doxygen will sort the +# (detailed) documentation of file and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. +# The default value is: YES. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the brief +# descriptions of file, namespace and class members alphabetically by member +# name. If set to NO, the members will appear in declaration order. Note that +# this will also influence the order of the classes in the class list. +# The default value is: NO. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen will sort the +# (brief and detailed) documentation of class members so that constructors and +# destructors are listed first. If set to NO the constructors will appear in the +# respective orders defined by SORT_BRIEF_DOCS and SORT_MEMBER_DOCS. +# Note: If SORT_BRIEF_DOCS is set to NO this option is ignored for sorting brief +# member documentation. +# Note: If SORT_MEMBER_DOCS is set to NO this option is ignored for sorting +# detailed member documentation. +# The default value is: NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the hierarchy +# of group names into alphabetical order. If set to NO the group names will +# appear in their defined order. +# The default value is: NO. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by +# fully-qualified names, including namespaces. If set to NO, the class list will +# be sorted only by class name, not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the alphabetical +# list. +# The default value is: NO. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to do proper +# type resolution of all parameters of a function it will reject a match between +# the prototype and the implementation of a member function even if there is +# only one candidate or it is obvious which candidate to choose by doing a +# simple string match. By disabling STRICT_PROTO_MATCHING doxygen will still +# accept a match between prototype and implementation in such cases. +# The default value is: NO. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or disable (NO) the todo +# list. This list is created by putting \todo commands in the documentation. +# The default value is: YES. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or disable (NO) the test +# list. This list is created by putting \test commands in the documentation. +# The default value is: YES. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or disable (NO) the bug +# list. This list is created by putting \bug commands in the documentation. +# The default value is: YES. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or disable (NO) +# the deprecated list. This list is created by putting \deprecated commands in +# the documentation. +# The default value is: YES. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional documentation +# sections, marked by \if ... \endif and \cond +# ... \endcond blocks. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines that the +# initial value of a variable or macro / define can have for it to appear in the +# documentation. If the initializer consists of more lines than specified here +# it will be hidden. Use a value of 0 to hide initializers completely. The +# appearance of the value of individual variables and macros / defines can be +# controlled using \showinitializer or \hideinitializer command in the +# documentation regardless of this setting. +# Minimum value: 0, maximum value: 10000, default value: 30. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated at +# the bottom of the documentation of classes and structs. If set to YES, the +# list will mention the files that were used to generate the documentation. +# The default value is: YES. + +SHOW_USED_FILES = YES + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. This +# will remove the Files entry from the Quick Index and from the Folder Tree View +# (if specified). +# The default value is: YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces +# page. This will remove the Namespaces entry from the Quick Index and from the +# Folder Tree View (if specified). +# The default value is: YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command command input-file, where command is the value of the +# FILE_VERSION_FILTER tag, and input-file is the name of an input file provided +# by doxygen. Whatever the program writes to standard output is used as the file +# version. For an example see the documentation. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. To create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. You can +# optionally specify a file name after the option, if omitted DoxygenLayout.xml +# will be used as the name of the layout file. +# +# Note that if you run doxygen from a directory containing a file called +# DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE +# tag is left empty. + +LAYOUT_FILE = + +# The CITE_BIB_FILES tag can be used to specify one or more bib files containing +# the reference definitions. This must be a list of .bib files. The .bib +# extension is automatically appended if omitted. This requires the bibtex tool +# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info. +# For LaTeX the style of the bibliography can be controlled using +# LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the +# search path. See also \cite for info how to create references. + +CITE_BIB_FILES = + +#--------------------------------------------------------------------------- +# Configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated to +# standard output by doxygen. If QUIET is set to YES this implies that the +# messages are off. +# The default value is: NO. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated to standard error (stderr) by doxygen. If WARNINGS is set to YES +# this implies that the warnings are on. +# +# Tip: Turn warnings on while writing the documentation. +# The default value is: YES. + +WARNINGS = YES + +# If the WARN_IF_UNDOCUMENTED tag is set to YES then doxygen will generate +# warnings for undocumented members. If EXTRACT_ALL is set to YES then this flag +# will automatically be disabled. +# The default value is: YES. + +WARN_IF_UNDOCUMENTED = YES + +# If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some parameters +# in a documented function, or documenting parameters that don't exist or using +# markup commands wrongly. +# The default value is: YES. + +WARN_IF_DOC_ERROR = YES + +# This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that +# are documented, but have no documentation for their parameters or return +# value. If set to NO, doxygen will only warn about wrong or incomplete +# parameter documentation, but not about the absence of documentation. +# The default value is: NO. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that doxygen +# can produce. The string should contain the $file, $line, and $text tags, which +# will be replaced by the file and line number from which the warning originated +# and the warning text. Optionally the format may contain $version, which will +# be replaced by the version of the file (if it could be obtained via +# FILE_VERSION_FILTER) +# The default value is: $file:$line: $text. + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning and error +# messages should be written. If left blank the output is written to standard +# error (stderr). + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +INPUT = nccl.h + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: http://www.gnu.org/software/libiconv) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.f90, *.f, *.for, *.tcl, *.vhd, +# *.vhdl, *.ucf, *.qsf, *.as and *.js. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.f90 \ + *.f \ + *.for \ + *.tcl \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf \ + *.as \ + *.js + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or directories +# that contain example code fragments that are included (see the \include +# command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and +# *.h) to filter out the source-files in the directories. If left blank all +# files are included. + +EXAMPLE_PATTERNS = * + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude commands +# irrespective of the value of the RECURSIVE tag. +# The default value is: NO. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or directories +# that contain images that are to be included in the documentation (see the +# \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command: +# +# +# +# where is the value of the INPUT_FILTER tag, and is the +# name of an input file. Doxygen will then use the output that the filter +# program writes to standard output. If FILTER_PATTERNS is specified, this tag +# will be ignored. +# +# Note that the filter must not add or remove lines; it is applied before the +# code is scanned, but not when the output code is generated. If lines are added +# or removed, the anchors will not be placed correctly. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. The filters are a list of the form: pattern=filter +# (like *.cpp=my_cpp_filter). See INPUT_FILTER for further information on how +# filters are used. If the FILTER_PATTERNS tag is empty or if none of the +# patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will also be used to filter the input files that are used for +# producing the source files to browse (i.e. when SOURCE_BROWSER is set to YES). +# The default value is: NO. + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) and +# it is also possible to disable source filtering for a specific pattern using +# *.ext= (so without naming a filter). +# This tag requires that the tag FILTER_SOURCE_FILES is set to YES. + +FILTER_SOURCE_PATTERNS = + +# If the USE_MDFILE_AS_MAINPAGE tag refers to the name of a markdown file that +# is part of the input, its contents will be placed on the main page +# (index.html). This can be useful if you have a project on for instance GitHub +# and want to reuse the introduction page also for the doxygen output. + +USE_MDFILE_AS_MAINPAGE = ../README.md + +#--------------------------------------------------------------------------- +# Configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will be +# generated. Documented entities will be cross-referenced with these sources. +# +# Note: To get rid of all source code in the generated output, make sure that +# also VERBATIM_HEADERS is set to NO. +# The default value is: NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body of functions, +# classes and enums directly into the documentation. +# The default value is: NO. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES will instruct doxygen to hide any +# special comment blocks from generated source code fragments. Normal C, C++ and +# Fortran comments will always remain visible. +# The default value is: YES. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES then for each documented +# function all documented functions referencing it will be listed. +# The default value is: NO. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES then for each documented function +# all documented entities called/used by that function will be listed. +# The default value is: NO. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES and SOURCE_BROWSER tag is set +# to YES then the hyperlinks from functions in REFERENCES_RELATION and +# REFERENCED_BY_RELATION lists will link to the source code. Otherwise they will +# link to the documentation. +# The default value is: YES. + +REFERENCES_LINK_SOURCE = YES + +# If SOURCE_TOOLTIPS is enabled (the default) then hovering a hyperlink in the +# source code will show a tooltip with additional information such as prototype, +# brief description and links to the definition and documentation. Since this +# will make the HTML file larger and loading of large files a bit slower, you +# can opt to disable this feature. +# The default value is: YES. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +SOURCE_TOOLTIPS = YES + +# If the USE_HTAGS tag is set to YES then the references to source code will +# point to the HTML generated by the htags(1) tool instead of doxygen built-in +# source browser. The htags tool is part of GNU's global source tagging system +# (see http://www.gnu.org/software/global/global.html). You will need version +# 4.8.6 or higher. +# +# To use it do the following: +# - Install the latest version of global +# - Enable SOURCE_BROWSER and USE_HTAGS in the config file +# - Make sure the INPUT points to the root of the source tree +# - Run doxygen as normal +# +# Doxygen will invoke htags (and that will in turn invoke gtags), so these +# tools must be available from the command line (i.e. in the search path). +# +# The result: instead of the source browser generated by doxygen, the links to +# source code will now point to the output of htags. +# The default value is: NO. +# This tag requires that the tag SOURCE_BROWSER is set to YES. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set the YES then doxygen will generate a +# verbatim copy of the header file for each class for which an include is +# specified. Set to NO to disable this. +# See also: Section \class. +# The default value is: YES. + +VERBATIM_HEADERS = YES + +# If the CLANG_ASSISTED_PARSING tag is set to YES then doxygen will use the +# clang parser (see: http://clang.llvm.org/) for more accurate parsing at the +# cost of reduced performance. This can be particularly helpful with template +# rich C++ code for which doxygen's built-in parser lacks the necessary type +# information. +# Note: The availability of this option depends on whether or not doxygen was +# compiled with the --with-libclang option. +# The default value is: NO. + +CLANG_ASSISTED_PARSING = NO + +# If clang assisted parsing is enabled you can provide the compiler with command +# line options that you would normally use when invoking the compiler. Note that +# the include paths will already be set by doxygen for the files and directories +# specified with INPUT and INCLUDE_PATH. +# This tag requires that the tag CLANG_ASSISTED_PARSING is set to YES. + +CLANG_OPTIONS = + +#--------------------------------------------------------------------------- +# Configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index of all +# compounds will be generated. Enable this if the project contains a lot of +# classes, structs, unions or interfaces. +# The default value is: YES. + +ALPHABETICAL_INDEX = YES + +# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in +# which the alphabetical index list will be split. +# Minimum value: 1, maximum value: 20, default value: 5. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all classes will +# be put under the same header in the alphabetical index. The IGNORE_PREFIX tag +# can be used to specify a prefix (or a list of prefixes) that should be ignored +# while generating the index headers. +# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. If a +# relative path is entered the value of OUTPUT_DIRECTORY will be put in front of +# it. +# The default directory is: html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for each +# generated HTML page (for example: .htm, .php, .asp). +# The default value is: .html. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a user-defined HTML header file for +# each generated HTML page. If the tag is left blank doxygen will generate a +# standard header. +# +# To get valid HTML the header file that includes any scripts and style sheets +# that doxygen needs, which is dependent on the configuration options used (e.g. +# the setting GENERATE_TREEVIEW). It is highly recommended to start with a +# default header using +# doxygen -w html new_header.html new_footer.html new_stylesheet.css +# YourConfigFile +# and then modify the file new_header.html. See also section "Doxygen usage" +# for information on how to generate the default header that doxygen normally +# uses. +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. For a description +# of the possible markers and block names see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a user-defined HTML footer for each +# generated HTML page. If the tag is left blank doxygen will generate a standard +# footer. See HTML_HEADER for more information on how to generate a default +# footer and what special commands can be used inside the footer. See also +# section "Doxygen usage" for information on how to generate the default footer +# that doxygen normally uses. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading style +# sheet that is used by each HTML page. It can be used to fine-tune the look of +# the HTML output. If left blank doxygen will generate a default style sheet. +# See also section "Doxygen usage" for information on how to generate the style +# sheet that doxygen normally uses. +# Note: It is recommended to use HTML_EXTRA_STYLESHEET instead of this tag, as +# it is more robust and this tag (HTML_STYLESHEET) will in the future become +# obsolete. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_STYLESHEET = + +# The HTML_EXTRA_STYLESHEET tag can be used to specify additional user-defined +# cascading style sheets that are included after the standard style sheets +# created by doxygen. Using this option one can overrule certain style aspects. +# This is preferred over using HTML_STYLESHEET since it does not replace the +# standard style sheet and is therefore more robust against future updates. +# Doxygen will copy the style sheet files to the output directory. +# Note: The order of the extra style sheet files is of importance (e.g. the last +# style sheet in the list overrules the setting of the previous ones in the +# list). For an example see the documentation. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath^ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that the +# files will be copied as-is; there are no commands or markers available. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen +# will adjust the colors in the style sheet and background images according to +# this color. Hue is specified as an angle on a colorwheel, see +# http://en.wikipedia.org/wiki/Hue for more information. For instance the value +# 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 +# purple, and 360 is red again. +# Minimum value: 0, maximum value: 359, default value: 220. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors +# in the HTML output. For a value of 0 the output will use grayscales only. A +# value of 255 will produce the most vivid colors. +# Minimum value: 0, maximum value: 255, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to the +# luminance component of the colors in the HTML output. Values below 100 +# gradually make the output lighter, whereas values above 100 make the output +# darker. The value divided by 100 is the actual gamma applied, so 80 represents +# a gamma of 0.8, The value 220 represents a gamma of 2.2, and 100 does not +# change the gamma. +# Minimum value: 40, maximum value: 240, default value: 80. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting this +# to YES can help to show when doxygen was last run and thus if the +# documentation is up to date. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_TIMESTAMP = NO + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_DYNAMIC_SECTIONS = NO + +# With HTML_INDEX_NUM_ENTRIES one can control the preferred number of entries +# shown in the various tree structured indices initially; the user can expand +# and collapse entries dynamically later on. Doxygen will expand the tree to +# such a level that at most the specified number of entries are visible (unless +# a fully collapsed tree already exceeds this amount). So setting the number of +# entries 1 will produce a full collapsed tree by default. 0 is a special value +# representing an infinite number of entries and will result in a full expanded +# tree by default. +# Minimum value: 0, maximum value: 9999, default value: 100. +# This tag requires that the tag GENERATE_HTML is set to YES. + +HTML_INDEX_NUM_ENTRIES = 100 + +# If the GENERATE_DOCSET tag is set to YES, additional index files will be +# generated that can be used as input for Apple's Xcode 3 integrated development +# environment (see: http://developer.apple.com/tools/xcode/), introduced with +# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a +# Makefile in the HTML output directory. Running make will produce the docset in +# that directory and running make install will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at +# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_DOCSET = NO + +# This tag determines the name of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# The default value is: Doxygen generated docs. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# This tag specifies a string that should uniquely identify the documentation +# set bundle. This should be a reverse domain-name style string, e.g. +# com.mycompany.MyDocSet. Doxygen will append .docset to the name. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# The DOCSET_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. +# The default value is: org.doxygen.Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The DOCSET_PUBLISHER_NAME tag identifies the documentation publisher. +# The default value is: Publisher. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three +# additional HTML index files: index.hhp, index.hhc, and index.hhk. The +# index.hhp is a project file that can be read by Microsoft's HTML Help Workshop +# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on +# Windows. +# +# The HTML Help Workshop contains a compiler that can convert all HTML output +# generated by doxygen into a single compiled HTML file (.chm). Compiled HTML +# files are now used as the Windows 98 help format, and will replace the old +# Windows help format (.hlp) on all Windows platforms in the future. Compressed +# HTML files also contain an index, a table of contents, and you can search for +# words in the documentation. The HTML workshop also contains a viewer for +# compressed HTML files. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_HTMLHELP = NO + +# The CHM_FILE tag can be used to specify the file name of the resulting .chm +# file. You can add a path in front of the file if the result should not be +# written to the html output directory. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_FILE = + +# The HHC_LOCATION tag can be used to specify the location (absolute path +# including file name) of the HTML help compiler (hhc.exe). If non-empty, +# doxygen will try to run the HTML help compiler on the generated index.hhp. +# The file has to be specified with full path. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +HHC_LOCATION = + +# The GENERATE_CHI flag controls if a separate .chi index file is generated +# (YES) or that it should be included in the master .chm file (NO). +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +GENERATE_CHI = NO + +# The CHM_INDEX_ENCODING is used to encode HtmlHelp index (hhk), content (hhc) +# and project file content. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +CHM_INDEX_ENCODING = + +# The BINARY_TOC flag controls whether a binary table of contents is generated +# (YES) or a normal table of contents (NO) in the .chm file. Furthermore it +# enables the Previous and Next buttons. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members to +# the table of contents of the HTML help documentation and to the tree view. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTMLHELP is set to YES. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated that +# can be used as input for Qt's qhelpgenerator to generate a Qt Compressed Help +# (.qch) of the generated HTML documentation. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can be used to specify +# the file name of the resulting .qch file. The path specified is relative to +# the HTML output folder. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help +# Project output. For more information please see Qt Help Project / Namespace +# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace). +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt +# Help Project output. For more information please see Qt Help Project / Virtual +# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual- +# folders). +# The default value is: doc. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_VIRTUAL_FOLDER = doc + +# If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom +# filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see Qt Help Project / Custom +# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom- +# filters). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's filter section matches. Qt Help Project / Filter Attributes (see: +# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes). +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHP_SECT_FILTER_ATTRS = + +# The QHG_LOCATION tag can be used to specify the location of Qt's +# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the +# generated .qhp file. +# This tag requires that the tag GENERATE_QHP is set to YES. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files will be +# generated, together with the HTML files, they form an Eclipse help plugin. To +# install this plugin and make it available under the help contents menu in +# Eclipse, the contents of the directory containing the HTML and XML files needs +# to be copied into the plugins directory of eclipse. The name of the directory +# within the plugins directory should be the same as the ECLIPSE_DOC_ID value. +# After copying Eclipse needs to be restarted before the help appears. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the Eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have this +# name. Each documentation set should have its own identifier. +# The default value is: org.doxygen.Project. +# This tag requires that the tag GENERATE_ECLIPSEHELP is set to YES. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# If you want full control over the layout of the generated HTML pages it might +# be necessary to disable the index and replace it with your own. The +# DISABLE_INDEX tag can be used to turn on/off the condensed index (tabs) at top +# of each HTML page. A value of NO enables the index and the value YES disables +# it. Since the tabs in the index contain the same information as the navigation +# tree, you can set this option to YES if you also set GENERATE_TREEVIEW to YES. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +DISABLE_INDEX = NO + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. If the tag +# value is set to YES, a side panel will be generated containing a tree-like +# index structure (just like the one that is generated for HTML Help). For this +# to work a browser that supports JavaScript, DHTML, CSS and frames is required +# (i.e. any modern browser). Windows users are probably better off using the +# HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can +# further fine-tune the look of the index. As an example, the default style +# sheet generated by doxygen has an example that shows how to put an image at +# the root of the tree instead of the PROJECT_NAME. Since the tree basically has +# the same information as the tab index, you could consider setting +# DISABLE_INDEX to YES when enabling this option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +GENERATE_TREEVIEW = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that +# doxygen will group on one line in the generated HTML documentation. +# +# Note that a value of 0 will completely suppress the enum values from appearing +# in the overview section. +# Minimum value: 0, maximum value: 20, default value: 4. +# This tag requires that the tag GENERATE_HTML is set to YES. + +ENUM_VALUES_PER_LINE = 1 + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be used +# to set the initial width (in pixels) of the frame in which the tree is shown. +# Minimum value: 0, maximum value: 1500, default value: 250. +# This tag requires that the tag GENERATE_HTML is set to YES. + +TREEVIEW_WIDTH = 250 + +# If the EXT_LINKS_IN_WINDOW option is set to YES, doxygen will open links to +# external symbols imported via tag files in a separate window. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of LaTeX formulas included as images in +# the HTML documentation. When you change the font size after a successful +# doxygen run you need to manually remove any form_*.png images from the HTML +# output directory to force them to be regenerated. +# Minimum value: 8, maximum value: 50, default value: 10. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are not +# supported properly for IE 6.0, but are supported on all modern browsers. +# +# Note that when changing this option you need to delete any form_*.png files in +# the HTML output directory before the changes have effect. +# The default value is: YES. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see +# http://www.mathjax.org) which uses client side Javascript for the rendering +# instead of using pre-rendered bitmaps. Use this if you do not have LaTeX +# installed or if you want to formulas look prettier in the HTML output. When +# enabled you may also need to install MathJax separately and configure the path +# to it using the MATHJAX_RELPATH option. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +USE_MATHJAX = YES + +# When MathJax is enabled you can set the default output format to be used for +# the MathJax output. See the MathJax site (see: +# http://docs.mathjax.org/en/latest/output.html) for more details. +# Possible values are: HTML-CSS (which is slower, but has the best +# compatibility), NativeMML (i.e. MathML) and SVG. +# The default value is: HTML-CSS. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_FORMAT = HTML-CSS + +# When MathJax is enabled you need to specify the location relative to the HTML +# output directory using the MATHJAX_RELPATH option. The destination directory +# should contain the MathJax.js script. For instance, if the mathjax directory +# is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax +# Content Delivery Network so you can quickly see the result without installing +# MathJax. However, it is strongly recommended to install a local copy of +# MathJax from http://www.mathjax.org before deployment. +# The default value is: http://cdn.mathjax.org/mathjax/latest. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest + +# The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax +# extension names that should be enabled during MathJax rendering. For example +# MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_EXTENSIONS = + +# The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces +# of code that will be used on startup of the MathJax code. See the MathJax site +# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# example see the documentation. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_CODEFILE = + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box for +# the HTML output. The underlying search engine uses javascript and DHTML and +# should work on any modern browser. Note that when using HTML help +# (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets (GENERATE_DOCSET) +# there is already a search function so this one should typically be disabled. +# For large projects the javascript based search engine can be slow, then +# enabling SERVER_BASED_SEARCH may provide a better solution. It is possible to +# search using the keyboard; to jump to the search box use + S +# (what the is depends on the OS and browser, but it is typically +# , /