From 273974393e8995aaa1701e211cd1d925b0c0ac0b Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Tue, 10 Nov 2020 11:10:48 -0800 Subject: [PATCH 01/17] Use target id of xnack off (#298) [ROCm/rccl commit: 1fdb216f87a8599c953785bc9d4195a4577b59f9] --- projects/rccl/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index 66e3e0edb0..e10c785a5c 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -11,7 +11,7 @@ set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "") project(rccl CXX) -set(AMDGPU_TARGETS gfx803;gfx900;gfx906;gfx908 CACHE STRING "List of specific machine types for library to target") +set(AMDGPU_TARGETS gfx803;gfx900;gfx906:xnack-;gfx908:xnack- CACHE STRING "List of specific machine types for library to target") option(BUILD_TESTS "Build test programs" OFF) option(INSTALL_DEPENDENCIES "Force install dependencies" OFF) From a7ef699687d8e8e85b9c4c1e36b1c366ee122a74 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Tue, 10 Nov 2020 15:44:10 -0700 Subject: [PATCH 02/17] Clique kernel support (#295) * Adding experimental clique-based kernels (opt-in only) Co-authored-by: Stanley Tsang Co-authored-by: Gilbert Lee Co-authored-by: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> [ROCm/rccl commit: 41bcfb8878c99fff1c7916fc27f2845af7ddcc70] --- projects/rccl/CHANGELOG.md | 9 + projects/rccl/CMakeLists.txt | 6 + projects/rccl/src/bootstrap.cc | 36 +- .../rccl/src/clique/AllReduceCliqueKernel.h | 75 +++ projects/rccl/src/clique/CliqueCommon.h | 93 ++++ projects/rccl/src/clique/CliqueManager.cc | 519 ++++++++++++++++++ projects/rccl/src/clique/CliqueManager.h | 128 +++++ projects/rccl/src/clique/CliqueShmNames.h | 37 ++ projects/rccl/src/clique/HandleCache.cc | 31 ++ projects/rccl/src/clique/HandleCache.h | 142 +++++ projects/rccl/src/clique/HandleShm.cc | 67 +++ projects/rccl/src/clique/HandleShm.h | 53 ++ projects/rccl/src/clique/Hash.cc | 34 ++ projects/rccl/src/clique/Hash.h | 28 + projects/rccl/src/clique/MsgQueue.cc | 72 +++ projects/rccl/src/clique/MsgQueue.h | 42 ++ projects/rccl/src/clique/SharedMemHelper.h | 43 ++ projects/rccl/src/clique/ShmObject.cc | 45 ++ projects/rccl/src/clique/ShmObject.h | 210 +++++++ .../rccl/src/collectives/device/all_reduce.h | 75 +-- projects/rccl/src/collectives/device/common.h | 36 +- .../src/collectives/device/common_kernel.h | 5 + projects/rccl/src/enqueue.cc | 49 +- projects/rccl/src/include/bootstrap.h | 2 +- projects/rccl/src/include/comm.h | 11 +- projects/rccl/src/include/devcomm.h | 15 + projects/rccl/src/init.cc | 59 +- projects/rccl/test/CorrectnessTest.hpp | 19 +- projects/rccl/test/test_AllReduce.cpp | 2 +- projects/rccl/test/test_CombinedCalls.cpp | 2 +- projects/rccl/test/test_GroupCalls.cpp | 2 +- projects/rccl/tools/HelloRccl/HelloRccl.cpp | 257 +++++++++ projects/rccl/tools/HelloRccl/HelloRccl.hpp | 49 ++ projects/rccl/tools/HelloRccl/Makefile | 21 + projects/rccl/tools/HelloRccl/runTest.sh | 22 + 35 files changed, 2204 insertions(+), 92 deletions(-) create mode 100644 projects/rccl/src/clique/AllReduceCliqueKernel.h create mode 100644 projects/rccl/src/clique/CliqueCommon.h create mode 100644 projects/rccl/src/clique/CliqueManager.cc create mode 100644 projects/rccl/src/clique/CliqueManager.h create mode 100644 projects/rccl/src/clique/CliqueShmNames.h create mode 100644 projects/rccl/src/clique/HandleCache.cc create mode 100644 projects/rccl/src/clique/HandleCache.h create mode 100644 projects/rccl/src/clique/HandleShm.cc create mode 100644 projects/rccl/src/clique/HandleShm.h create mode 100644 projects/rccl/src/clique/Hash.cc create mode 100644 projects/rccl/src/clique/Hash.h create mode 100644 projects/rccl/src/clique/MsgQueue.cc create mode 100644 projects/rccl/src/clique/MsgQueue.h create mode 100644 projects/rccl/src/clique/SharedMemHelper.h create mode 100644 projects/rccl/src/clique/ShmObject.cc create mode 100644 projects/rccl/src/clique/ShmObject.h create mode 100644 projects/rccl/tools/HelloRccl/HelloRccl.cpp create mode 100644 projects/rccl/tools/HelloRccl/HelloRccl.hpp create mode 100644 projects/rccl/tools/HelloRccl/Makefile create mode 100755 projects/rccl/tools/HelloRccl/runTest.sh diff --git a/projects/rccl/CHANGELOG.md b/projects/rccl/CHANGELOG.md index 60276a7b01..2388e7c640 100644 --- a/projects/rccl/CHANGELOG.md +++ b/projects/rccl/CHANGELOG.md @@ -3,8 +3,17 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https://rccl.readthedocs.io) ## [Unreleased] +### Added +- Experimental support for clique-based kernels (opt in with RCCL_ENABLE_CLIQUE=1) +- Clique-based kernels may offer better performance for smaller input sizes +- Clique-based kernels are currently only enabled for AllReduce under a certain byte limit (controlled via RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT) ### Optimizations - Performance improvements for Rome-based systems +### Known issues +- Clique-based kernels are currently experimental and have not been fully tested on all topologies. By default, clique-based kernels are disabled if the detected topology is not supported (override with RCCL_FORCE_ENABLE_CLIQUE) +- Clique-based kernels may hang if there are differences between environment variables set across ranks. +- Clique-based kernels may fail if the input / output device pointers are not the base device pointers returned by hipMalloc. + ## [RCCL-2.7.8 for ROCm 3.9.0] ### Added diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index e10c785a5c..cc6ddcb929 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -126,6 +126,12 @@ set(CC_SOURCES src/collectives/all_to_all_api.cc src/collectives/all_to_allv_api.cc src/channel.cc + src/clique/CliqueManager.cc # RCCL + src/clique/HandleCache.cc # RCCL + src/clique/HandleShm.cc # RCCL + src/clique/Hash.cc # RCCL + src/clique/MsgQueue.cc # RCCL + src/clique/ShmObject.cc # RCCL src/misc/argcheck.cc src/misc/nvmlwrap_stub.cc src/misc/utils.cc diff --git a/projects/rccl/src/bootstrap.cc b/projects/rccl/src/bootstrap.cc index e90dd66823..71e7070284 100644 --- a/projects/rccl/src/bootstrap.cc +++ b/projects/rccl/src/bootstrap.cc @@ -12,6 +12,11 @@ #include "socket.h" #include #include +// [RCCL] +#include "clique/CliqueManager.h" +#include "clique/CliqueShmNames.h" +#include "clique/Hash.h" +// [/RCCL] struct bootstrapNetComm { int fd; @@ -163,7 +168,14 @@ static ncclResult_t setFilesLimit() { return ncclSuccess; } -static void *bootstrapRoot(void* listenComm) { +static void *bootstrapRoot(void* bootstrapRootStruct) { // [RCCL] Modified to include hash argument) + // [RCCL] Unpack bootstrapRootStruct + struct bootstrapRootStruct* rootStruct = (struct bootstrapRootStruct*) bootstrapRootStruct; + void* listenComm = rootStruct->listenComm; + unsigned long hash = rootStruct->hash; + int pid = getpid(); // sharing PID to other ranks for creating shared memory files for CliqueManager + // [/RCCL] + struct extInfo info; ncclNetHandle_t *rankHandles = NULL; ncclNetHandle_t *rankHandlesRoot = NULL; // for initial rank <-> root information exchange @@ -205,12 +217,19 @@ static void *bootstrapRoot(void* listenComm) { } while (c < nranks); TRACE(NCCL_INIT, "COLLECTED ALL %d HANDLES", nranks); + { // [RCCL] Initialize message queues / shared memory files + NCCLCHECKGOTO(CliqueManager::BootstrapRootInit(pid, hash), res, out); + } // [/RCCL] + // Send the connect handle for the next rank in the AllGather ring for (int r=0; rhash = djb2Hash(id->internal); + rootStruct->listenComm = listenComm; + pthread_create(&thread, NULL, bootstrapRoot, (void *)rootStruct); + // [/RCCL] + return ncclSuccess; } @@ -267,9 +293,10 @@ struct extState { int rank; int nranks; int dev; + int rootPid; // [RCCL] PID of root }; -ncclResult_t bootstrapInit(ncclUniqueId * id, int rank, int nranks, void** commState) { +ncclResult_t bootstrapInit(ncclUniqueId * id, int rank, int nranks, void** commState, int* rootPid) { // [RCCL] Adding rootPid ncclNetHandle_t* netHandle = (ncclNetHandle_t*) id; bool idFromEnv = getenv("NCCL_COMM_ID") != NULL; struct extState* state; @@ -314,6 +341,9 @@ ncclResult_t bootstrapInit(ncclUniqueId * id, int rank, int nranks, void** commS ncclNetHandle_t extHandleNext; NCCLCHECK(bootstrapNetAccept(extBstrapListenCommRoot, &tmpRecvComm)); NCCLCHECK(bootstrapNetRecv(tmpRecvComm, &extHandleNext, sizeof(extHandleNext))); + { // [RCCL] Receive PID from root + NCCLCHECK(bootstrapNetRecv(tmpRecvComm, rootPid, sizeof(int))); + } // [/RCCL] NCCLCHECK(bootstrapNetCloseRecv(tmpRecvComm)); NCCLCHECK(bootstrapNetCloseListen(extBstrapListenCommRoot)); diff --git a/projects/rccl/src/clique/AllReduceCliqueKernel.h b/projects/rccl/src/clique/AllReduceCliqueKernel.h new file mode 100644 index 0000000000..a46ed6a219 --- /dev/null +++ b/projects/rccl/src/clique/AllReduceCliqueKernel.h @@ -0,0 +1,75 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef ALLREDUCECLIQUEKERNEL_H +#define ALLREDUCECLIQUEKERNEL_H + +#include "CliqueCommon.h" +#include "devcomm.h" +#include "reduce_kernel.h" +#include "common_kernel.h" + +template +__device__ void AllReduceCliqueSplitKernel(struct CollectiveArgs* args) +{ + // Clique-specific kernel arguments + cliqueDevicePtrs_t* cliquePtrs = args->clique.ptrs; // Collection of all input/output pointers across ranks in clique + size_t const N = args->clique.count; // Total number of elements to reduce + int const nBlocks = args->clique.nChannels; // Total number of blocks assigned to this kernel (may be different than gridDim.x) + int const blockId = args->clique.bid; // 0-indexed blockIdx for this threadblock (may be different than blockIdx.x) + int const rank = args->comm->rank; // Current rank + + // Each threadblock works independently of others on a subsection of the input + // First split evently across ranks, while maintaining multiples of blocksize + size_t const perRankN = RoundUp((N + NUM_RANKS - 1) / NUM_RANKS, blockDim.x); + size_t const perBlockN = RoundUp((perRankN + nBlocks - 1) / nBlocks, blockDim.x); + size_t const currBlockStart = min((rank * nBlocks + blockId) * perBlockN, N); + size_t const currBlockStop = min(currBlockStart + perBlockN, N); + size_t const blockN = currBlockStop - currBlockStart; + + if (blockN > 0) + { + // Prepare input / output subarrays + T const** inputs = (T const**)cliquePtrs->inputs; + T** outputs = (T **)cliquePtrs->outputs; + T const* srcs[NUM_RANKS]; + T* dsts[NUM_RANKS]; + + #pragma unroll + for (int r = 0; r < NUM_RANKS; r++) + { + srcs[r] = inputs[r] + currBlockStart; + dsts[r] = outputs[r] + currBlockStart; + } + + // Perform the reduction + #define ALL_REDUCE_CLIQUE_UNROLL 2 + ReduceOrCopyMulti( + threadIdx.x, blockDim.x, NUM_RANKS, srcs, NUM_RANKS, dsts, blockN); + } + + // Even if there was nothing for this GPU to do, it must participate in a barrier + // because other GPUs may be modifying this GPUs output buffer still + if (blockId == 0) WaitForBarrier(cliquePtrs->barrier); +} + +#endif diff --git a/projects/rccl/src/clique/CliqueCommon.h b/projects/rccl/src/clique/CliqueCommon.h new file mode 100644 index 0000000000..b7bdee19ae --- /dev/null +++ b/projects/rccl/src/clique/CliqueCommon.h @@ -0,0 +1,93 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef CLIQUE_COMMON_H +#define CLIQUE_COMMON_H + +#include "nccl.h" +#include + +#define MIN_CLIQUE_SIZE 2 +#define MAX_CLIQUE_SIZE 8 + +typedef struct +{ + int* globalCount; // Shared across GPUs + int* globalSense; // Shared across GPUs + int* localSense; // Local to this GPU +} gpuBarrier_t; + +typedef struct +{ + // Input/output pointers from participating ranks + void const* inputs[MAX_CLIQUE_SIZE]; + void* outputs[MAX_CLIQUE_SIZE]; + + // Barrier variable + gpuBarrier_t barrier; +} cliqueDevicePtrs_t; + +// Helper macro to launch an appropriate kernel by converting rank to a template argument +#define LAUNCH_CLIQUE_KERNEL(kernelname, FUNC, T, args) \ + { \ + switch (args->comm->nRanks){ \ + case 2: kernelname(args); break; \ + case 3: kernelname(args); break; \ + case 4: kernelname(args); break; \ + case 5: kernelname(args); break; \ + case 6: kernelname(args); break; \ + case 7: kernelname(args); break; \ + case 8: kernelname(args); break; \ + } \ + } + +// Multi-GPU (on same node) barrier. One thread per grid per GPU updates barrier / waits +template +__forceinline__ __device__ void WaitForBarrier(gpuBarrier_t const& barrier) +{ + if (threadIdx.x == 0) + { + // Sense inversion barrier + *barrier.localSense = 1 - *barrier.localSense; + int localSense = *barrier.localSense; + + int val = __atomic_add_fetch(barrier.globalCount, 1, __ATOMIC_SEQ_CST); + if (val == NUM_RANKS) + { + // Last arrival resets barrier + __atomic_store_n(barrier.globalCount, 0, __ATOMIC_SEQ_CST); + __atomic_store_n(barrier.globalSense, localSense, __ATOMIC_SEQ_CST); + } + else + { + // Wait for all ranks to reach barrier + while (__atomic_load_n(barrier.globalSense, __ATOMIC_SEQ_CST) != localSense); + } + } +} + +__forceinline__ __host__ __device__ size_t RoundUp(size_t X, size_t Y) +{ + return (X+Y-1)/Y * Y; +} + +#endif diff --git a/projects/rccl/src/clique/CliqueManager.cc b/projects/rccl/src/clique/CliqueManager.cc new file mode 100644 index 0000000000..3afd3cfa31 --- /dev/null +++ b/projects/rccl/src/clique/CliqueManager.cc @@ -0,0 +1,519 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "CliqueManager.h" +#include "CliqueShmNames.h" +#include "MsgQueue.h" + +#include "nccl.h" +#include "core.h" + +#include "Hash.h" + +#include "AllReduceCliqueKernel.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +cliqueDevicePtrs_t CliqueManager::m_staticCliquePtrs[NCCL_MAX_OPS] = {}; +int* CliqueManager::m_staticGpuBarrierMem = NULL; + +// Define some environment variables that affect clique-based kernels +RCCL_PARAM(EnableClique, "ENABLE_CLIQUE", 0); // Opt-in environment variable for clique-based kernels +RCCL_PARAM(AllReduceCliqueByteLimit, "CLIQUE_ALLREDUCE_BYTE_LIMIT", 2097152); // Max number of bytes to use clique-based kernels for all reduce +RCCL_PARAM(AllReduceNumChannels, "CLIQUE_ALLREDUCE_NCHANNELS", 4); // Number of channels to use for all-reduce + +CliqueManager::CliqueManager(int const rank, + int const numRanks, + cliqueMode_t const cliqueMode) : + m_rank(rank), + m_numRanks(numRanks), + m_cliqueMode(cliqueMode), + m_init(false), + m_pinnedCliquePtrs(NULL), + m_fineGrainBarrierMem(NULL) +{ +} + +CliqueManager::~CliqueManager() +{ + if (m_init) + { + CleanUp(); + } +} + +void CliqueManager::CleanUp() +{ + if (m_cliqueMode == CLIQUE_DISABLED) return; + + // Free variables that are shared between SINGLE_PROCESS / SINGLE_NODE + if (m_pinnedCliquePtrs) hipHostFree(m_pinnedCliquePtrs); + if (m_gpuBarrierLocalSense) hipFree(m_gpuBarrierLocalSense); + + if (m_cliqueMode == CLIQUE_SINGLE_NODE) + { + // Release caches + if (m_ipcHandleSendCache) delete m_ipcHandleSendCache; + if (m_ipcHandleSendCache) delete m_ipcHandleRecvCache; + + // Close shared memory + m_shmHandles.Close(); + m_sharedCpuMemory.Close(); + m_sharedIpcHandle.Close(); + + if (m_fineGrainBarrierMem) + { + if (m_rank == 0) + hipFree(m_fineGrainBarrierMem); + else + hipIpcCloseMemHandle(m_fineGrainBarrierMem); + } + } + else if (m_cliqueMode == CLIQUE_SINGLE_PROCESS) + { + if (m_rank == 0 && m_staticGpuBarrierMem) + hipFree(m_staticGpuBarrierMem); + } + m_init = false; +} + +ncclResult_t CliqueManager::Init(ncclUniqueId const* commId, int suffix) +{ + ncclResult_t res; + + if (m_init) return ncclSuccess; + m_init = true; + + if (m_cliqueMode == CLIQUE_DISABLED) return ncclSuccess; + + // Check parameters + if (m_rank < 0 || m_rank >= m_numRanks) + { + WARN("Invalid rank specified. Expected 0 <= %d < %d for CliqueManager", m_rank, m_numRanks); + return ncclInvalidUsage; + } + if (commId == NULL) + { + WARN("CommId should not be empty"); + return ncclInvalidUsage; + } + + // For now, opt-into clique based kernels via RCCL_ENABLE_CLIQUE env var + if (!rcclParamEnableClique()) + { + INFO(NCCL_INIT, "Disabling clique-based kernels (did not find env var RCCL_ENABLE_CLIQUE)"); + m_cliqueMode = CLIQUE_DISABLED; + return ncclSuccess; + } + + // Allocate pinned CPU memory for holding clique pointers, which kernels will have access to + if (hipHostMalloc(&m_pinnedCliquePtrs, sizeof(cliqueDevicePtrs_t) * NCCL_MAX_OPS) != hipSuccess) + { + WARN("Unable to allocated pinned host memory for clique pointers. Disabling clique-based kernels"); + m_cliqueMode = CLIQUE_DISABLED; + m_init = true; + return ncclSuccess; + } + + unsigned long hash = djb2Hash(commId->internal); + std::string shmSuffix = std::to_string(hash) + "_" + std::to_string(suffix); + + // Allocate sense barrier variable on local GPU + NCCLCHECKGOTO(ncclCudaCalloc(&m_gpuBarrierLocalSense, NCCL_MAX_OPS * sizeof(int)), res, dropback); + + if (m_cliqueMode == CLIQUE_SINGLE_NODE) + { + // Initialize shared memory file for IPC handles (based on commId hash) + m_shmHandles = NcclIpcHandleShm(m_rank, m_numRanks, hash, NUM_HANDLES_PER_RANK, NCCL_MAX_OPS, shmSuffix); + NCCLCHECKGOTO(m_shmHandles.Open(), res, dropback); + + // Initialize IPC caches + m_ipcHandleSendCache = new NcclIpcHandleSendCache(m_numRanks * NUM_HANDLES_PER_RANK * NCCL_MAX_OPS); + m_ipcHandleRecvCache = new NcclIpcHandleRecvCache(m_numRanks * NUM_HANDLES_PER_RANK * NCCL_MAX_OPS, + 100, + hipIpcMemHandleHash, + hipIpcMemHandleEqual); + + // Initialize shared object for GPU barrier IPC handle + m_sharedIpcHandle = ShmObject(std::max(4096LU, sizeof(hipIpcMemHandle_t)), + CliqueShmNames["Barriers"] + shmSuffix, + m_rank, + m_numRanks, + hash); + NCCLCHECKGOTO(m_sharedIpcHandle.Open(), res, dropback); + + if (m_rank == 0) + { + hipIpcMemHandle_t handle; + // Allocate fine-grained device memory on rank 0 and get IPC handle for it + // Re-usable barrier consists of (globalCount / globalSense) pair of integers + NCCLCHECKGOTO(ncclCudaCalloc(&m_fineGrainBarrierMem, NCCL_MAX_OPS * 2 * sizeof(int), true), res, dropback); + if (hipIpcGetMemHandle(&handle, m_fineGrainBarrierMem) != hipSuccess) + { + WARN("Unable to get IPC handle for barrier memory"); + goto dropback; + } + // Write IPC handle to shared memory for other ranks to receive + *m_sharedIpcHandle.Get() = handle; + + // Set up global count/sense for first rank + m_gpuBarrierGlobalCount = &m_fineGrainBarrierMem[0]; + m_gpuBarrierGlobalSense = &m_fineGrainBarrierMem[NCCL_MAX_OPS]; + } + + // Initialize shared CPU memory to be used for barrier variables + m_sharedCpuMemory = ShmObject(2 * sizeof(int32_t), + CliqueShmNames["SharedCounters"] + shmSuffix, + m_rank, + m_numRanks, + hash); + NCCLCHECKGOTO(m_sharedCpuMemory.Open(), res, dropback); + + // Split up the shared CPU memory for barrier counters / global sense + m_cpuBarrierGlobalCount = &m_sharedCpuMemory.Get()[0]; + m_cpuBarrierGlobalSense = &m_sharedCpuMemory.Get()[1]; + + // Initialize CPU barriers + if (m_rank == 0) + { + *m_cpuBarrierGlobalCount = 0; + *m_cpuBarrierGlobalSense = 0; + } + m_cpuBarrierLocalSense = 0; + } + else if (m_cliqueMode == CLIQUE_SINGLE_PROCESS) + { + // First rank prepares fine-grained memory shared across ranks used for the two barrier variables + if (m_rank == 0) + { + NCCLCHECKGOTO(ncclCudaCalloc(&m_staticGpuBarrierMem, NCCL_MAX_OPS * 2 * sizeof(int), true), res, dropback); + + // Prepare all barriers + for (int opIndex = 0; opIndex < NCCL_MAX_OPS; opIndex++) + { + m_staticCliquePtrs[opIndex].barrier.globalCount = &m_staticGpuBarrierMem[opIndex]; + m_staticCliquePtrs[opIndex].barrier.globalSense = &m_staticGpuBarrierMem[opIndex + NCCL_MAX_OPS];; + } + } + } + + + m_init = true; + INFO(NCCL_INIT, "Clique-based kernels enabled (mode %d)", m_cliqueMode); + return ncclSuccess; + +dropback: + // NOTE: This currently assumes that all ranks will fail the same way + // Additional support is required to handle cases when some processes succeed while others fail + WARN("Unable to initialize shared memory. Disabling clique-based kernels"); + CleanUp(); + m_cliqueMode = CLIQUE_DISABLED; + return ncclSuccess; +} + +bool CliqueManager::IsSupported(ncclFunc_t const coll, + size_t const count, + ncclDataType_t const datatype, + ncclRedOp_t const op) const +{ + if (m_cliqueMode == CLIQUE_DISABLED) return false; + + // Filter based on total input size for each collective type + size_t totalBytes = count * ncclTypeSize(datatype); + if (coll == ncclCollAllReduce && (totalBytes <= rcclParamAllReduceCliqueByteLimit())) return true; + + return false; +} + +ncclResult_t CliqueManager::DeclarePointers(uint64_t opCount, void const* inputPtr, void* outputPtr) +{ + // Do nothing if disabled + if (m_cliqueMode == CLIQUE_DISABLED) return ncclSuccess; + + if (!m_init) + { + WARN("CliqueManager must be initialized before use"); + return ncclInvalidUsage; + } + + int const opIndex = opCount % NCCL_MAX_OPS; + + // Add opIndex to queue of in-progress collectives + m_inProgress.push(opIndex); + + if (m_cliqueMode == CLIQUE_SINGLE_NODE) + { + // Get fine-grained device memory if not already done + if (m_fineGrainBarrierMem == NULL) + { + hipIpcMemHandle_t handle = *m_sharedIpcHandle.Get(); + CUDACHECK(hipIpcOpenMemHandle((void**)&m_fineGrainBarrierMem, handle, hipIpcMemLazyEnablePeerAccess)); + + // Prepare global count/sense barrier variables used the ipc-shared gpu device memory + m_gpuBarrierGlobalCount = &m_fineGrainBarrierMem[0]; + m_gpuBarrierGlobalSense = &m_fineGrainBarrierMem[NCCL_MAX_OPS]; + } + + std::vector> handles(NUM_HANDLES_PER_RANK); + + // Get IPC handles for input/output pointers from cache + NCCLCHECK(CheckCacheForPtr(const_cast(inputPtr), m_ipcHandleSendCache, m_rank, &handles[0])); + NCCLCHECK(CheckCacheForPtr(outputPtr , m_ipcHandleSendCache, m_rank, &handles[1])); + + // Prepare barrier pointers (done after the IpcOpenMemory) + m_pinnedCliquePtrs[opIndex].barrier.globalCount = &m_gpuBarrierGlobalCount[opIndex]; + m_pinnedCliquePtrs[opIndex].barrier.globalSense = &m_gpuBarrierGlobalSense[opIndex]; + m_pinnedCliquePtrs[opIndex].barrier.localSense = &m_gpuBarrierLocalSense[opIndex]; + + // Write IPC handles to shared memory for given rank / opCount + NCCLCHECK(m_shmHandles.WriteHandles(opIndex, handles)); + } + else if (m_cliqueMode == CLIQUE_SINGLE_PROCESS) + { + // Store this rank's input/output pointers into static member + m_staticCliquePtrs[opIndex].inputs[m_rank] = inputPtr; + m_staticCliquePtrs[opIndex].outputs[m_rank] = outputPtr; + } + + return ncclSuccess; +} + +ncclResult_t CliqueManager::GetNumChannelsToUse(ncclFunc_t const coll, + size_t const count, + ncclDataType_t const datatype, + ncclRedOp_t const op, + int const totalNumChannels, + uint8_t* numChannelstoUse) +{ + size_t const totalBytes = count * ncclTypeSize(datatype); + *numChannelstoUse = 1; + + if (coll == ncclCollAllReduce) { + *numChannelstoUse = std::min((int)rcclParamAllReduceNumChannels(), totalNumChannels); + } + + return ncclSuccess; +} + + + +ncclResult_t CliqueManager::SetCliqueCollectiveArgs(CollectiveArgs* args) +{ + // Do nothing if disabled + if (m_cliqueMode == CLIQUE_DISABLED) return ncclSuccess; + if (!m_init) + { + WARN("CliqueManager must be initialized before use"); + return ncclInvalidUsage; + } + + // Prepare clique argments (NOTE: clique pointers are not ready yet) + int opIndex = args->opCount % NCCL_MAX_OPS; + args->clique.ptrs = &m_pinnedCliquePtrs[opIndex]; + + + // Determine number of channels to use for this collective + args->clique.nChannels = rcclParamAllReduceNumChannels(); + + return ncclSuccess; +} + +ncclResult_t CliqueManager::WaitForPointers() +{ + // Do nothing if disabled + if (m_cliqueMode == CLIQUE_DISABLED) return ncclSuccess; + + if (!m_init) + { + WARN("CliqueManager must be initialized before use"); + return ncclInvalidUsage; + } + + // Do nothing if there are no outstanding clique-kernels + if (m_inProgress.empty()) return ncclSuccess; + + // Copy clique device pointers to pinned device memory + if (m_cliqueMode == CLIQUE_SINGLE_NODE) + { + // Wait for all ranks to arrive + WaitForBarrier(); + + int numHandles = m_numRanks * NUM_HANDLES_PER_RANK; + std::vector> handles(numHandles); + + while (!m_inProgress.empty()) + { + int const opIndex = m_inProgress.front(); + m_inProgress.pop(); + + // Collect the ready handles from shared memory and convert them to device pointers + NCCLCHECK(m_shmHandles.ReadHandles(opIndex, handles)); + for (int i = 0; i < m_numRanks; i++) + { + void *input; + NCCLCHECK(CheckCacheForHandle(handles[i * NUM_HANDLES_PER_RANK], + m_ipcHandleRecvCache, &input)); + m_pinnedCliquePtrs[opIndex].inputs[i] = const_cast(input); + + NCCLCHECK(CheckCacheForHandle(handles[(i * NUM_HANDLES_PER_RANK) + 1], + m_ipcHandleRecvCache, &m_pinnedCliquePtrs[opIndex].outputs[i])); + } + } + } + else if (m_cliqueMode == CLIQUE_SINGLE_PROCESS) + { + while (!m_inProgress.empty()) + { + int const opIndex = m_inProgress.front(); + m_inProgress.pop(); + + // Copy from static memory to pinned host memory and set local sense + memcpy(&m_pinnedCliquePtrs[opIndex], &m_staticCliquePtrs[opIndex], sizeof(cliqueDevicePtrs_t)); + m_pinnedCliquePtrs[opIndex].barrier.localSense = &m_gpuBarrierLocalSense[opIndex]; + } + } + return ncclSuccess; +} + +std::string HandleToString(hipIpcMemHandle_t handle) +{ + char mapping[17] = "0123456789ABCDEF"; + std::string result; + for (int i = 0; i < 4; i++) + { + unsigned char val = (unsigned char)handle.reserved[i]; + result += mapping[val / 16]; + result += mapping[val % 16]; + } + return result; +} + + +ncclResult_t CliqueManager::CheckCacheForPtr(void* devPtr, + NcclIpcHandleSendCache* cache, + int rank, + std::pair* handlePair) +{ + // Get the base address for this device allocation + hsa_status_t status; + hsa_amd_pointer_info_t info; + info.size = sizeof(hsa_amd_pointer_info_t); + status = hsa_amd_pointer_info(devPtr, &info, NULL, NULL, NULL); + if (status != HSA_STATUS_SUCCESS) { + WARN("Uanble to get pointer information for %p", devPtr); + return ncclInvalidArgument; + } + + // Compute the offset between the device addres and the base address + uint64_t baseAddr = (uint64_t)info.agentBaseAddress; + uint64_t realAddr = (uint64_t)devPtr; + handlePair->second = realAddr - baseAddr; + + // IPC handles are only supported for base address pointers + NcclIpcHandleSendCache::iterator it = cache->find(baseAddr); + + if (it == cache->end()) + { + CUDACHECK(hipIpcGetMemHandle(&handlePair->first, (void*)baseAddr)); + cache->insert(baseAddr, handlePair->first); + } + else + { + handlePair->first = (it->second).first; + } + return ncclSuccess; +} + +ncclResult_t CliqueManager::CheckCacheForHandle(std::pair const& handlePair, + NcclIpcHandleRecvCache* cache, + void** ptr) +{ + NcclIpcHandleRecvCache::iterator it = cache->find(handlePair.first); + + // Get base address pointer from cache if it exists + void* baseAddr; + if (it == cache->end()) + { + CUDACHECK(hipIpcOpenMemHandle(&baseAddr, handlePair.first, hipIpcMemLazyEnablePeerAccess)); + cache->insert(handlePair.first, baseAddr); + } + else + { + baseAddr = (it->second).first; + } + + // Modify base address pointer with offset + uint64_t realAddr = (uint64_t)baseAddr + handlePair.second; + *ptr = (void*)realAddr; + return ncclSuccess; +} + +void CliqueManager::WaitForBarrier() +{ + // Sense inversion barrier + m_cpuBarrierLocalSense = 1 - m_cpuBarrierLocalSense; + + if (__sync_add_and_fetch(m_cpuBarrierGlobalCount, 1) == m_numRanks) + { + // Reset the barrier + STORE(m_cpuBarrierGlobalCount, 0); + STORE(m_cpuBarrierGlobalSense, m_cpuBarrierLocalSense); + } else { + while (LOAD(m_cpuBarrierGlobalSense) != m_cpuBarrierLocalSense); + } +} + +ncclResult_t CliqueManager::BootstrapRootInit(int pid, unsigned long hash) +{ + for (auto it = CliqueShmNames.begin(); it != CliqueShmNames.end(); it++) + { + int msgid, fd; + std::string msgQueueName = "/tmp/" + it->second + std::to_string(hash) + "_" + std::to_string(pid); + SYSCHECKVAL(open(msgQueueName.c_str(), O_CREAT | O_RDWR, 0606), "open", fd); + NCCLCHECK(MsgQueueGetId(msgQueueName, hash, true, msgid)); + SYSCHECK(close(fd), "close"); + } + + std::string shmDir = "/dev/shm/"; + + for (auto it = CliqueShmNames.begin(); it != CliqueShmNames.end(); it++) + { + struct stat fileStatus; + std::string shmFileName = it->second + std::to_string(hash) + "_" + std::to_string(pid); + std::string shmFullPath = shmDir + shmFileName; + + // Check if shm file already exists; if so, unlink it + if (stat(shmFullPath.c_str(), &fileStatus) == 0) + { + NCCLCHECK(shmUnlink(shmFileName.c_str())); + } + } + return ncclSuccess; +} diff --git a/projects/rccl/src/clique/CliqueManager.h b/projects/rccl/src/clique/CliqueManager.h new file mode 100644 index 0000000000..bf8f028e58 --- /dev/null +++ b/projects/rccl/src/clique/CliqueManager.h @@ -0,0 +1,128 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef RCCL_CLIQUE_MANAGER_HPP_ +#define RCCL_CLIQUE_MANAGER_HPP_ + +#include +#include +#include + +#include "nccl.h" +#include "devcomm.h" +#include "CliqueCommon.h" +#include "HandleCache.h" +#include "HandleShm.h" + +#define NUM_HANDLES_PER_RANK 2 + +class CliqueManager +{ +public: + typedef enum + { + CLIQUE_DISABLED = 0, + CLIQUE_SINGLE_PROCESS = 1, + CLIQUE_SINGLE_NODE = 2 + } cliqueMode_t; + + CliqueManager(int const rank, int const numRanks, cliqueMode_t const cliqueMode); + + ~CliqueManager(); + + void CleanUp(); + + ncclResult_t Init(ncclUniqueId const* commId, int suffix); + + // Returns true if the collective is supported via a clique-based kernel + bool IsSupported(ncclFunc_t const coll, + size_t const count, + ncclDataType_t const datatype, + ncclRedOp_t const op) const; + + // Provide the pointers to be exchanged across the clique for the given rank / opCount + ncclResult_t DeclarePointers(uint64_t opCount, void const* inputPtr, void* outputPtr); + + // Determine the number of channels / CUs to use for this call + ncclResult_t GetNumChannelsToUse(ncclFunc_t const coll, + size_t const count, + ncclDataType_t const datatype, + ncclRedOp_t const op, + int const totalNumChannels, + uint8_t* numChannelstoUse); + + // Set pointers for where clique-related arguments will be found + // This sets pointers to device-accessible memory where the arguments will eventually reside + ncclResult_t SetCliqueCollectiveArgs(CollectiveArgs* args); + + // Blocking call that only returns after all out-standing clique pointers are ready + ncclResult_t WaitForPointers(); + + // Prepares shared memory files upon initialization + static ncclResult_t BootstrapRootInit(int pid, unsigned long hash); + +protected: + ncclResult_t CheckCacheForPtr(void* devPtr, + NcclIpcHandleSendCache* cache, + int rank, + std::pair* handlePair); + + ncclResult_t CheckCacheForHandle(std::pair const& handlePair, + NcclIpcHandleRecvCache* cache, + void** ptr); + + // Race-condition helper functions + void WaitForBarrier(); + + int m_rank; // Associated rank + int m_numRanks; // Total number of ranks + cliqueMode_t m_cliqueMode; // Clique mode (off/single process/single node) + bool m_init; // Whether CliqueManager has been initialized + cliqueDevicePtrs_t* m_pinnedCliquePtrs; // Pinned-host-memory (device accessible) containing device pointers + int* m_gpuBarrierGlobalCount; // Part of GPU barrier (count variable shared across ranks) + int* m_gpuBarrierGlobalSense; // Part of GPU barrier (reset variable shared across ranks) + int* m_gpuBarrierLocalSense; // Part of GPU barrier (reset variable local to this rank) + std::queue m_inProgress; // Queue of clique-based collectives waiting for pointers + + // IPC-related (CLIQUE_SINGLE_NODE) + NcclIpcHandleShm m_shmHandles; // Used to exchange IPC handles between ranks + NcclIpcHandleSendCache* m_ipcHandleSendCache; // Caches pointers to IPC handles (to send to other processes) + NcclIpcHandleRecvCache* m_ipcHandleRecvCache; // Caches IPC handles to pointers (received from other processes) + ShmObject m_sharedCpuMemory; // Used to pass shared memory used for CPU barrier + ShmObject m_sharedIpcHandle; // Used to pass fine-grained device memory buffer IPC handle + int* m_fineGrainBarrierMem; // Fine-grained GPU memory barrier (allocated only on 1st rank, shared on others) + int* m_cpuBarrierGlobalCount; // Part of CPU barrier (count variable shared across ranks) + int* m_cpuBarrierGlobalSense; // Part of CPU barrier (reset variable shared across ranks) + int m_cpuBarrierLocalSense; // Part of CPU barrier (reset variable local to this rank) + + // Single-process (CLIQUE_SINGLE_PROCESS) + static cliqueDevicePtrs_t m_staticCliquePtrs[NCCL_MAX_OPS]; // Use shared static memory to exchange pointer info + static int* m_staticGpuBarrierMem; // Static storage backing for fine-grained gpu barrier +}; + +// For use in bootstrapping code +struct bootstrapRootStruct { + void* listenComm; + unsigned long hash; +}; + +#endif diff --git a/projects/rccl/src/clique/CliqueShmNames.h b/projects/rccl/src/clique/CliqueShmNames.h new file mode 100644 index 0000000000..577af8be8d --- /dev/null +++ b/projects/rccl/src/clique/CliqueShmNames.h @@ -0,0 +1,37 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef NCCL_CLIQUE_SHM_NAMES_H_ +#define NCCL_CLIQUE_SHM_NAMES_H_ + +#include +#include + +static std::map CliqueShmNames = +{ + {"SharedCounters", "RcclCounters" }, + {"Mutexes" , "RcclMutexes" }, + {"IpcHandles" , "RcclIpcHandles"}, + {"Barriers" , "RcclBarriers" } +}; + +#endif diff --git a/projects/rccl/src/clique/HandleCache.cc b/projects/rccl/src/clique/HandleCache.cc new file mode 100644 index 0000000000..8fe4b6ab4b --- /dev/null +++ b/projects/rccl/src/clique/HandleCache.cc @@ -0,0 +1,31 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "HandleCache.h" + +#include "Hash.h" + +// djb2 hash function for hashing char array in hipIpcMemHandle_t +unsigned long hipIpcMemHandleHash(const hipIpcMemHandle_t& handle) +{ + return djb2Hash(handle.reserved); +} diff --git a/projects/rccl/src/clique/HandleCache.h b/projects/rccl/src/clique/HandleCache.h new file mode 100644 index 0000000000..dc479e00e8 --- /dev/null +++ b/projects/rccl/src/clique/HandleCache.h @@ -0,0 +1,142 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef NCCL_HANDLE_CACHE_H_ +#define NCCL_HANDLE_CACHE_H_ + +#include +#include +#include + +#include "core.h" + +//#include "llvm/ADT/DenseMap.h" + +template < + class Key, + class Value, + class Hash, + class KeyEqual, + class Allocator +> +class NcclIpcHandleCache +{ +public: + typedef std::pair::iterator> NcclIpcHandleCacheValueType; + typedef std::unordered_map LRUCache; + using iterator = typename LRUCache::iterator; + NcclIpcHandleCache(size_t size, + size_t bucket_count = 100, + const Hash& hash = Hash(), + const KeyEqual& eql = KeyEqual(), + const Allocator& alloc = Allocator() ) : m_cache(bucket_count, hash, eql, alloc) + { + m_capacity = size; + } + + ~NcclIpcHandleCache() + { + m_lruHistory.clear(); + m_cache.clear(); + } + + iterator begin() + { + return m_cache.begin(); + } + + iterator end() + { + return m_cache.end(); + } + + iterator find(const Key& key) + { + iterator it = m_cache.find(key); + if (it != m_cache.end()) + { + updateHistory(it); + } + + return it; + } + + std::pair insert(const Key& key, const Value& value) + { + if (m_cache.size() == m_capacity) + { + // remove entry + pop(); + } + + typename LRUCache::iterator it = m_cache.find(key); + bool inserted; + if (it == m_cache.end()) + { + typename std::list::iterator it = m_lruHistory.insert(m_lruHistory.end(), key); + m_cache.insert(std::make_pair(key, std::make_pair(value, it))); + inserted = true; + } + else + { + inserted = false; + } + + return std::pair(it, inserted); + } + +private: + void pop() + { + typename LRUCache::iterator it = m_cache.find(m_lruHistory.front()); + m_cache.erase(it); + m_lruHistory.pop_front(); + } + + void updateHistory(const iterator& it) + { + if (m_lruHistory.size() > 0) + { + m_lruHistory.splice(m_lruHistory.end(), m_lruHistory, (it->second).second); + } + } + size_t m_capacity; + std::list m_lruHistory; + LRUCache m_cache; +}; + +// djb2 hash function for hashing char array in hipIpcMemHandle_t +unsigned long hipIpcMemHandleHash(const hipIpcMemHandle_t& handle); + +// equality function required for unordered_map +auto hipIpcMemHandleEqual = [](const hipIpcMemHandle_t& l, const hipIpcMemHandle_t& r) +{ + return memcmp(l.reserved, r.reserved, sizeof(l.reserved)) == 0; +}; + +//typedef llvm::DenseMap SendCache; +//typedef llvm::DenseMap RecvCache; + +typedef NcclIpcHandleCache, std::equal_to, std::allocator< std::pair::iterator>>>> NcclIpcHandleSendCache; +typedef NcclIpcHandleCache::iterator>>>> NcclIpcHandleRecvCache; + +#endif diff --git a/projects/rccl/src/clique/HandleShm.cc b/projects/rccl/src/clique/HandleShm.cc new file mode 100644 index 0000000000..937390cf20 --- /dev/null +++ b/projects/rccl/src/clique/HandleShm.cc @@ -0,0 +1,67 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include + +#include "HandleShm.h" +#include "CliqueShmNames.h" +#include "core.h" +#include "Hash.h" +#include "shm.h" + +NcclIpcHandleShm::NcclIpcHandleShm(int rank, int numRanks, int projid, int numHandlesPerRank, int capacity, std::string suffix) : + ShmObject>(numRanks * numHandlesPerRank * capacity * sizeof(std::pair), + CliqueShmNames["IpcHandles"] + suffix, + rank, + numRanks, + projid), + m_numHandlesPerRank(numHandlesPerRank), + m_numHandlesPerOpCount(numRanks * numHandlesPerRank) +{ +} + +NcclIpcHandleShm::NcclIpcHandleShm() +{ +} + +NcclIpcHandleShm::~NcclIpcHandleShm() +{ +} + +ncclResult_t NcclIpcHandleShm::Open() +{ + return ShmObject::Open(); +} + +ncclResult_t NcclIpcHandleShm::WriteHandles(uint64_t opCount, std::vector> const& sendHandles) +{ + size_t idx = (opCount * m_numHandlesPerOpCount) + (m_rank * m_numHandlesPerRank); + memcpy(m_shmPtr + idx, sendHandles.data(), sizeof(std::pair) * m_numHandlesPerRank); + return ncclSuccess; +} + +ncclResult_t NcclIpcHandleShm::ReadHandles(uint64_t opCount, std::vector>& recvHandles) +{ + size_t idx = opCount * m_numHandlesPerOpCount; + memcpy(recvHandles.data(), m_shmPtr + idx, m_numHandlesPerOpCount * sizeof(std::pair)); + return ncclSuccess; +} diff --git a/projects/rccl/src/clique/HandleShm.h b/projects/rccl/src/clique/HandleShm.h new file mode 100644 index 0000000000..c681de0eb4 --- /dev/null +++ b/projects/rccl/src/clique/HandleShm.h @@ -0,0 +1,53 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef NCCL_IPC_HANDLE_SHM_H_ +#define NCCL_IPC_HANDLE_SHM_H_ + +#include +#include +#include + +#include "nccl.h" +#include "ShmObject.h" + +class NcclIpcHandleShm : public ShmObject> +{ +public: + NcclIpcHandleShm(int rank, int numRanks, int projid, int numHandlesPerRank, int capacity, std::string suffix); + + NcclIpcHandleShm(); + + ~NcclIpcHandleShm(); + + ncclResult_t Open(); + + ncclResult_t WriteHandles(uint64_t opCount, std::vector> const& sendHandles); + + ncclResult_t ReadHandles(uint64_t opCount, std::vector>& recvHandles); + +private: + int m_numHandlesPerRank; + int m_numHandlesPerOpCount; +}; + +#endif diff --git a/projects/rccl/src/clique/Hash.cc b/projects/rccl/src/clique/Hash.cc new file mode 100644 index 0000000000..249c66b329 --- /dev/null +++ b/projects/rccl/src/clique/Hash.cc @@ -0,0 +1,34 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "Hash.h" + +unsigned long djb2Hash(const char* data) +{ + unsigned long hash = 5381; + int c; + + while ((c = *(data)++)) + hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ + + return hash; +} diff --git a/projects/rccl/src/clique/Hash.h b/projects/rccl/src/clique/Hash.h new file mode 100644 index 0000000000..e6cbbaa569 --- /dev/null +++ b/projects/rccl/src/clique/Hash.h @@ -0,0 +1,28 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef NCCL_HASH_H_ +#define NCCL_HASH_H_ + +unsigned long djb2Hash(const char* data); + +#endif diff --git a/projects/rccl/src/clique/MsgQueue.cc b/projects/rccl/src/clique/MsgQueue.cc new file mode 100644 index 0000000000..716c449e8f --- /dev/null +++ b/projects/rccl/src/clique/MsgQueue.cc @@ -0,0 +1,72 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "MsgQueue.h" + +#include +#include + +#define MSG_QUEUE_PERM 0666 + +ncclResult_t MsgQueueGetId(std::string name, int projid, bool exclusive, int& msgid) +{ + key_t key; + SYSCHECKVAL(ftok(name.c_str(), projid), "ftok", key); + int flag = (exclusive == true ? IPC_CREAT | IPC_EXCL : IPC_CREAT); + + msgid = msgget(key, MSG_QUEUE_PERM | flag); + // Check if we're trying to create message queue and it already exists; if so, delete existing queue + if (msgid == -1 && exclusive == true && errno == EEXIST) + { + NCCLCHECK(MsgQueueClose(name, projid)); + SYSCHECKVAL(msgget(key, MSG_QUEUE_PERM | flag), "msgget", msgid); + } + else if (msgid == -1) + { + WARN("Call to MsgQueueGetId failed : %s", strerror(errno)); + return ncclSystemError; + } + return ncclSuccess; +} + +ncclResult_t MsgQueueSend(int msgid, const void* msgp, size_t msgsz, int msgflg) +{ + SYSCHECK(msgsnd(msgid, msgp, msgsz, msgflg), "msgsnd"); + return ncclSuccess; +} + +ncclResult_t MsgQueueRecv(int msgid, void* msgp, size_t msgsz, long msgtyp, bool wait) +{ + int msgflg = (wait == false ? IPC_NOWAIT : 0); + SYSCHECK(msgrcv(msgid, msgp, msgsz, msgtyp, msgflg), "msgrcv"); + return ncclSuccess; +} + +ncclResult_t MsgQueueClose(std::string name, int projid) +{ + key_t key; + int msgid; + key = ftok(name.c_str(), projid); + SYSCHECKVAL(msgget(key, IPC_CREAT), "msgget", msgid); + SYSCHECK(msgctl(msgid, IPC_RMID, NULL), "msgctl"); + return ncclSuccess; +} diff --git a/projects/rccl/src/clique/MsgQueue.h b/projects/rccl/src/clique/MsgQueue.h new file mode 100644 index 0000000000..346208a6e8 --- /dev/null +++ b/projects/rccl/src/clique/MsgQueue.h @@ -0,0 +1,42 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef RCCL_MSG_QUEUE_HPP_ +#define RCCL_MSG_QUEUE_HPP_ + +#include + +#include "nccl.h" +#include "core.h" + +struct MsgBuffer +{ + long msg_type; + char msg_text[1]; +}; + +ncclResult_t MsgQueueGetId(std::string name, int projid, bool exclusive, int& msgid); +ncclResult_t MsgQueueSend(int msgid, const void* msgp, size_t msgsz, int msgflg); +ncclResult_t MsgQueueRecv(int msgid, void* msgp, size_t msgsz, long msgtyp, bool wait); +ncclResult_t MsgQueueClose(std::string name, int projid); + +#endif diff --git a/projects/rccl/src/clique/SharedMemHelper.h b/projects/rccl/src/clique/SharedMemHelper.h new file mode 100644 index 0000000000..6fc269e7fc --- /dev/null +++ b/projects/rccl/src/clique/SharedMemHelper.h @@ -0,0 +1,43 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef SHAREDMEMHELPER_H +#define SHAREDMEMHELPER_H + + +class SharedMemHelper +{ +public: + SharedMemHelper(int rank, int numRanks, int numEntries); + + ncclStatus_t Init(std::string const& baseFilename); + + ncclStatus_t + + +protected: + bool m_initialized; + int m_rank; + int m_numRanks; +}; + +#endif diff --git a/projects/rccl/src/clique/ShmObject.cc b/projects/rccl/src/clique/ShmObject.cc new file mode 100644 index 0000000000..353779e41d --- /dev/null +++ b/projects/rccl/src/clique/ShmObject.cc @@ -0,0 +1,45 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "ShmObject.h" +#include + +// Template specializations for sem_t objects which require additional initialization +template<> +ncclResult_t ShmObject::Close() +{ + size_t numMutexes = m_shmSize / sizeof(sem_t); + + for (size_t i = 0; i < numMutexes; i++) + { + sem_destroy(static_cast(&m_shmPtr[i])); + } + + int retVal = shm_unlink(m_shmName.c_str()); + if (retVal == -1 && errno != ENOENT) + { + WARN("Call to shm_unlink in ShmObject failed : %s", strerror(errno)); + return ncclSystemError; + } + + return ncclSuccess; +} diff --git a/projects/rccl/src/clique/ShmObject.h b/projects/rccl/src/clique/ShmObject.h new file mode 100644 index 0000000000..f458014f5e --- /dev/null +++ b/projects/rccl/src/clique/ShmObject.h @@ -0,0 +1,210 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef NCCL_SHM_OBJECT_H_ +#define NCCL_SHM_OBJECT_H_ + +#include +#include +#include +#include +#include +#include + +#include "MsgQueue.h" +#include "nccl.h" +#include "core.h" +#include "shm.h" + +// ShmObject abstracts away the nitty-gritty when multiple processes need to handle opening a shared +// memory object at the same time. + +static ncclResult_t shmSetupExclusive(const char* shmname, const int shmsize, int* fd, void** ptr, int create) { + *fd = shm_open(shmname, O_CREAT | O_RDWR | O_EXCL, S_IRUSR | S_IWUSR); + if (*fd == -1) return ncclSystemError; + if (create) SYSCHECK(shm_allocate(*fd, shmsize), "posix_fallocate"); + SYSCHECK(shm_map(*fd, shmsize, ptr), "mmap"); + close(*fd); + *fd = -1; + if (create) memset(*ptr, 0, shmsize); + return ncclSuccess; +} + +template +class ShmObject +{ +public: +ShmObject(size_t size, std::string fileName, int rank, int numRanks, int projid) : + m_shmSize(size), + m_shmName(fileName), + m_rank(rank), + m_numRanks(numRanks), + m_projid(projid), + m_alloc(false), + m_shmPtr(nullptr) {} + + ShmObject() {} + + ~ShmObject() {} + + ncclResult_t Open(); + + ncclResult_t Close() + { + if (m_alloc) + { + if (m_rank == 0) + { + std::string tmpFileName = "/tmp/" + m_shmName; + remove(tmpFileName.c_str()); + } + int retVal = shm_unlink(m_shmName.c_str()); + if (retVal == -1 && errno != ENOENT) + { + WARN("Call to shm_unlink in ShmObject failed : %s", strerror(errno)); + return ncclSystemError; + } + } + return ncclSuccess; + } + + T*& Get() + { + return m_shmPtr; + } +protected: + ncclResult_t BroadcastMessage(int msgid, bool pass) + { + MsgBuffer msg; + msg.msg_text[0] = (pass == 0 ? 'F': 'P'); + for (int rank = 0; rank < m_numRanks; rank++) + { + if (rank == m_rank) continue; + msg.msg_type = rank; + NCCLCHECK(MsgQueueSend(msgid, &msg, sizeof(msg), 0)); + } + return ncclSuccess; + } + + // tag for dispatch + template + struct OpenTag{}; + + ncclResult_t InitIfSemaphore(OpenTag tag); + ncclResult_t InitIfSemaphore(OpenTag tag); + ncclResult_t InitIfSemaphore(OpenTag tag); + ncclResult_t InitIfSemaphore(OpenTag tag); + ncclResult_t InitIfSemaphore(OpenTag> tag); + + size_t m_shmSize; + std::string m_shmName; + int m_rank; + int m_numRanks; + int m_projid; + bool m_alloc; + T* m_shmPtr; +}; + +template +ncclResult_t ShmObject::Open() +{ + if (m_alloc == false) + { + int shmFd; + int protection = PROT_READ | PROT_WRITE; + int visibility = MAP_SHARED; + + int msgid; + std::string tmpFileName = "/tmp/" + m_shmName; + NCCLCHECK(MsgQueueGetId(tmpFileName, m_projid, false, msgid)); + if (m_rank == 0) + { + ncclResult_t resultSetup = shmSetupExclusive(m_shmName.c_str(), m_shmSize, &shmFd, (void**)&m_shmPtr, 1); + ncclResult_t resultSemInit = InitIfSemaphore(OpenTag{}); + if ((resultSetup != ncclSuccess && errno != EEXIST) || (resultSemInit != ncclSuccess)) + { + NCCLCHECK(BroadcastMessage(msgid, false)); + WARN("Call to ShmObject::Open in root rank failed : %s", strerror(errno)); + return ncclSystemError; + } + NCCLCHECK(BroadcastMessage(msgid, true)); + } + else + { + MsgBuffer msg; + NCCLCHECK(MsgQueueRecv(msgid, &msg, sizeof(msg), m_rank, true)); + if (msg.msg_text[0] == 'P') + { + NCCLCHECK(shmSetup(m_shmName.c_str(), m_shmSize, &shmFd, (void**)&m_shmPtr, 0)); + } + else + { + WARN("Call to shm_open from non-root rank in ShmObject failed : %s", strerror(errno)); + return ncclSystemError; + } + } + m_alloc = true; + } + else + { + WARN("Cannot allocate ShmObject twice.\n"); + return ncclInvalidUsage; + } + return ncclSuccess; +} + +template +ncclResult_t ShmObject::InitIfSemaphore(OpenTag tag) +{ + return ncclSuccess; +} + +template +ncclResult_t ShmObject::InitIfSemaphore(OpenTag tag) +{ + return ncclSuccess; +} + +template +ncclResult_t ShmObject::InitIfSemaphore(OpenTag tag) +{ + return ncclSuccess; +} + +template +ncclResult_t ShmObject::InitIfSemaphore(OpenTag> tag) +{ + return ncclSuccess; +} + +template +ncclResult_t ShmObject::InitIfSemaphore(OpenTag tag) +{ + size_t numMutexes = m_shmSize / sizeof(sem_t); + + for (size_t i = 0; i < numMutexes; i++) + { + SYSCHECK(sem_init(static_cast(&m_shmPtr[i]), 1, 1), "sem_init"); + } + return ncclSuccess; +} +#endif diff --git a/projects/rccl/src/collectives/device/all_reduce.h b/projects/rccl/src/collectives/device/all_reduce.h index 3af59a4c14..15d0c6768f 100644 --- a/projects/rccl/src/collectives/device/all_reduce.h +++ b/projects/rccl/src/collectives/device/all_reduce.h @@ -8,6 +8,7 @@ #include "devcomm.h" #include "primitives.h" #include "collectives.h" +#include "clique/AllReduceCliqueKernel.h" // [RCCL] AllReduce Clique-based kernel support template __attribute__((noinline)) @@ -310,6 +311,7 @@ __device__ void ncclAllReduceTreeLLKernel(struct CollectiveArgs* args) { const ssize_t loopSize = nChannels*chunkSize; const ssize_t size = args->coll.count; + if (loopSize > size) { chunkSize = DIVUP(size, nChannels*minChunkSize)*minChunkSize; } @@ -417,76 +419,10 @@ __device__ void ncclAllReduceCollNetLLKernel(struct CollectiveArgs* args) { template __attribute__((noinline)) __device__ void ncclAllReduceRingLL128Kernel(struct CollectiveArgs* args) { - const int tid = threadIdx.x; - const int nthreads = args->coll.nThreads; - const int bid = args->coll.bid; - const int nChannels = args->coll.nChannels; - struct ncclDevComm* comm = args->comm; - struct ncclChannel* channel = comm->channels+blockIdx.x; - struct ncclRing* ring = &channel->ring; - const int stepSize = comm->buffSizes[NCCL_PROTO_LL128] / (sizeof(uint64_t)*NCCL_STEPS); - ssize_t chunkSize = stepSize*NCCL_LL128_DATAELEMS*sizeof(uint64_t) / (NCCL_LL128_LINEELEMS*sizeof(T)); - // We should not need the final /2 but it makes performance much, much smoother. Might be a bug somewhere. - const ssize_t minChunkSize = (NCCL_LL128_SHMEM_ELEMS_PER_THREAD*nthreads*NCCL_LL128_DATAELEMS*sizeof(uint64_t))/(NCCL_LL128_LINEELEMS*sizeof(T))/2; - const int nranks = comm->nRanks; - const ssize_t loopSize = nChannels*nranks*chunkSize; - const ssize_t size = args->coll.count; - ncclLL128Primitives LLprims(tid, nthreads, &ring->prev, &ring->next, stepSize, channel, comm); - - // Compute pointers - const T * __restrict__ thisInput = (const T*)args->sendbuff; - T * __restrict__ thisOutput = (T*)args->recvbuff; - - for (ssize_t gridOffset = 0; gridOffset < size; gridOffset += loopSize) { - chunkSize = min(DIVUP(size-gridOffset, nChannels*nranks*minChunkSize)*minChunkSize, chunkSize); - - /////////////// begin AllReduce steps /////////////// - ssize_t offset; - int nelem; - int chunk; - - // step 0: push data to next GPU - chunk = ring->devUserRanks[nranks-1]; - offset = gridOffset + (chunk*nChannels+bid) * chunkSize; - nelem = min(chunkSize, size-offset); - - LLprims.send(thisInput+offset, nelem); - - // k-2 steps: reduce and copy to next GPU - for (int j=2; jdevUserRanks[nranks-j]; - offset = gridOffset + (chunk*nChannels+bid) * chunkSize; - nelem = min(chunkSize, size-offset); - - LLprims.recvReduceSend(thisInput+offset, nelem); - } - - // step k-1: reduce this buffer and data, which will produce the final - // result that we store in this data and push to the next GPU - chunk = ring->devUserRanks[0]; - offset = gridOffset + (chunk*nChannels+bid) * chunkSize; - nelem = min(chunkSize, size-offset); - - LLprims.recvReduceCopySend(thisInput+offset, thisOutput+offset, nelem); - - // k-2 steps: copy to next GPU - for (int j=1; jdevUserRanks[nranks-j]; - offset = gridOffset + (chunk*nChannels+bid) * chunkSize; - nelem = min(chunkSize, size-offset); - - LLprims.recvCopySend(thisOutput+offset, nelem); - } - - // Make final copy from buffer to dest. - chunk = ring->devUserRanks[1]; - offset = gridOffset + (chunk*nChannels+bid) * chunkSize; - nelem = min(chunkSize, size-offset); - - // Here we need to copy from buffer to this output. - LLprims.recv(thisOutput+offset, nelem); - } + // [RCCL] RingLL128 is re-purposed as clique-based kernel + LAUNCH_CLIQUE_KERNEL(AllReduceCliqueSplitKernel, FUNC, T, args); + // [/RCCL] } template @@ -507,6 +443,7 @@ __device__ void ncclAllReduceTreeLL128Kernel(struct CollectiveArgs* args) { int nthreadsSplit = NCCL_LL128_SPLIT(nthreads); const ssize_t size = args->coll.count; + if (loopSize > size) { chunkSize = DIVUP(size, nChannels*minChunkSize)*minChunkSize; } diff --git a/projects/rccl/src/collectives/device/common.h b/projects/rccl/src/collectives/device/common.h index f13957dd7e..efcdf75276 100644 --- a/projects/rccl/src/collectives/device/common.h +++ b/projects/rccl/src/collectives/device/common.h @@ -89,13 +89,45 @@ static inline __device__ void exitIfAbortBarrier(int abort) { NCCL_FUNCS3B(coll, copy), \ NCCL_FUNCS3B(coll, copy) +// [RCCL] Adding clique-based kernels for AllReduce, in-place of unused RingLL28 kernels +#define NCCL_FUNC5B(coll, op, dtype) \ + NCCL_COLL_NAME(coll##LL, op, dtype), \ + NCCL_COLL_NAME(coll##LL128, op, dtype), \ + NCCL_COLL_NAME(coll, op, dtype) + +#define NCCL_FUNC4B(coll, op, dtype) \ + NCCL_FUNC5(coll##Tree, op, dtype), \ + NCCL_FUNC5B(coll##Ring, op, dtype), \ + NCCL_FUNC5(coll##CollNet, op, dtype) + +#define NCCL_FUNCS3C(coll, op) \ + NCCL_FUNC4B(coll, op, i8), \ + NCCL_FUNC4B(coll, op, u8), \ + NCCL_FUNC4B(coll, op, i32), \ + NCCL_FUNC4B(coll, op, u32), \ + NCCL_FUNC4B(coll, op, i64), \ + NCCL_FUNC4B(coll, op, u64), \ + NCCL_FUNC4B(coll, op, f16), \ + NCCL_FUNC4B(coll, op, f32), \ + NCCL_FUNC4B(coll, op, f64), \ + NCCL_FUNC4B(coll, op, b16) + +#define NCCL_FUNCS2C(coll) \ + NCCL_FUNCS3C(coll, sum ), \ + NCCL_FUNCS3C(coll, prod), \ + NCCL_FUNCS3C(coll, max ), \ + NCCL_FUNCS3C(coll, min ) + +// [/RCCL] + + // Must be consistent with ncclFunc_t #define NCCL_FUNCS() { \ NCCL_FUNCS2B(ncclBroadcast), \ NCCL_FUNCS2A(ncclReduce), \ NCCL_FUNCS2B(ncclAllGather), \ NCCL_FUNCS2A(ncclReduceScatter), \ - NCCL_FUNCS2A(ncclAllReduce), \ + NCCL_FUNCS2C(ncclAllReduce), \ NCCL_COLL_NAME(ncclGather, copy, i8), \ NCCL_COLL_NAME(ncclScatter, copy, i8), \ NCCL_COLL_NAME(ncclAllToAll, copy, i8), \ @@ -114,7 +146,7 @@ static const __device__ constexpr ncclKernelFunc_t ncclFuncs[]{ NCCL_FUNCS2A(ncclReduce), NCCL_FUNCS2B(ncclAllGather), NCCL_FUNCS2A(ncclReduceScatter), - NCCL_FUNCS2A(ncclAllReduce), + NCCL_FUNCS2C(ncclAllReduce), NCCL_COLL_NAME(ncclGather, copy, i8), NCCL_COLL_NAME(ncclScatter, copy, i8), NCCL_COLL_NAME(ncclAllToAll, copy, i8), diff --git a/projects/rccl/src/collectives/device/common_kernel.h b/projects/rccl/src/collectives/device/common_kernel.h index c5092cf52a..efa0b7a1aa 100644 --- a/projects/rccl/src/collectives/device/common_kernel.h +++ b/projects/rccl/src/collectives/device/common_kernel.h @@ -350,9 +350,14 @@ __device__ int ptrAlign128(T* ptr) { return (uint64_t)ptr % alignof(int32_t); } __device__ int ptrAlign128(T* ptr) { return (uint64_t)ptr % alignof(Pack128); } #endif +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__) +// Multiply UNROLL by 2 if single source/single destination +#define AUTOUNROLL (UNROLL*((MINSRCS==1 && MINDSTS==1) ? 2 : 1)) +#else // Try to limit consecutive load/stores to 8. // Use UNROLL 8 when we have a single source and a single destination, 4 otherwise #define AUTOUNROLL (UNROLL*(4/(MINDSTS+MINSRCS))) +#endif template __device__ void ReduceOrCopyMulti(const int tid, const int nthreads, diff --git a/projects/rccl/src/enqueue.cc b/projects/rccl/src/enqueue.cc index 7f52e089ce..ff5d982b1e 100644 --- a/projects/rccl/src/enqueue.cc +++ b/projects/rccl/src/enqueue.cc @@ -9,6 +9,8 @@ #include "argcheck.h" #include "coll_net.h" #include "../graph/topo.h" +#include +#include // Only generate inline kernels for LL #define NCCL_FUNC5(coll, op, dtype) \ @@ -116,6 +118,10 @@ ncclResult_t setupLaunch(struct ncclComm* comm, hipLaunchParams* params) { STORE(&channel->collectives[(channel->collStart+channel->collCount-1)%NCCL_MAX_OPS].active, 2); } + { // [RCCL] Wait for any clique-based collectives + NCCLCHECK(comm->cliqueManager->WaitForPointers()); + } // [/RCCL] + // Find the first operation, choose the kernel accordingly and pass it // as the first argument. struct ncclColl* coll = comm->channels[0].collectives+comm->channels[0].collStart; @@ -210,7 +216,8 @@ ncclResult_t ncclBarrierEnqueueWait(ncclComm_t comm) { (comm->launchMode == ncclComm::GROUP && comm->groupCudaStream) ? "/Stream" : ""); } - + hipEvent_t startEvent; + hipEvent_t stopEvent; if (comm->launchMode == ncclComm::PARALLEL) { hipLaunchKernelGGL(((void (*)(struct ncclDevComm*))params->func), params->gridDim, params->blockDim, params->sharedMem, params->stream, **((struct ncclDevComm ***)(params->args))); } else { @@ -257,6 +264,7 @@ static ncclResult_t getAlgoInfo(struct ncclInfo* info) { info->algorithm = -1; info->protocol = -1; int nAlgos = NCCL_NUM_ALGORITHMS; + // Check collNet support int collNetTypeSupport = 0; if (info->comm->collNetSupport) @@ -373,6 +381,7 @@ static ncclResult_t computeColl(struct ncclInfo* info /* input */, struct ncclCo #endif return ncclSuccess; } + // Set nstepsPerLoop and nchunksPerLoop NCCLCHECK(getAlgoInfo(info)); NCCLCHECK(getPatternInfo(info)); @@ -391,6 +400,33 @@ static ncclResult_t computeColl(struct ncclInfo* info /* input */, struct ncclCo coll->funcIndex = FUNC_INDEX(info->coll, info->op, info->datatype, info->algorithm, info->protocol); + { // [RCCL] Check for clique-based kernel support + if (info->comm->cliqueManager->IsSupported(info->coll, + info->count, + info->datatype, + info->op)) + { + // Declare the input / output pointers being used (to exchange via IPC with other ranks) + NCCLCHECK(info->comm->cliqueManager->DeclarePointers(info->comm->opCount, + info->sendbuff, + info->recvbuff)); + + + info->algorithm = NCCL_ALGO_RING; + info->protocol = NCCL_PROTO_CLIQUE; + // Determine the number of channels to use for clique-kernel + NCCLCHECK(info->comm->cliqueManager->GetNumChannelsToUse(info->coll, + info->count, + info->datatype, + info->op, + info->comm->nChannels, + &coll->args.clique.nChannels)); + coll->args.clique.count = info->count; + coll->funcIndex = FUNC_INDEX(info->coll, info->op, info->datatype, info->algorithm, info->protocol); + return ncclSuccess; + } + } // [RCCL] + int stepSize = info->comm->buffSizes[info->protocol]/NCCL_STEPS; int chunkSteps = (info->protocol == NCCL_PROTO_SIMPLE && info->algorithm == NCCL_ALGO_RING) ? info->chunkSteps : 1; int sliceSteps = (info->protocol == NCCL_PROTO_SIMPLE && info->algorithm == NCCL_ALGO_RING) ? info->sliceSteps : 1; @@ -478,6 +514,7 @@ ncclResult_t ncclSaveKernel(struct ncclInfo* info) { info->comm->myParams->blockDim.x = std::max(info->comm->myParams->blockDim.x, info->nThreads); int nChannels = info->coll == ncclCollSendRecv ? 1 : coll.args.coll.nChannels; + int nSubChannels = (info->pattern == ncclPatternCollTreeUp || info->pattern == ncclPatternCollTreeDown) ? 2 : 1; for (int bid=0; bidargs.a2av.extra+info->comm->nRanks*2, info->recvcounts, sizeof(size_t*)*(info->comm->nRanks)); memcpy(c->args.a2av.extra+info->comm->nRanks*3, info->rdispls, sizeof(size_t*)*(info->comm->nRanks)); c->args.a2av.bid = bid % coll.args.coll.nChannels; - } else if (info->coll != ncclCollSendRecv) + } else if (info->coll != ncclCollSendRecv) { c->args.coll.bid = bid % coll.args.coll.nChannels; + } + + // [RCCL] Setup pointers to where all the input/output pointers will be + if (info->protocol == NCCL_PROTO_CLIQUE) { + NCCLCHECK(info->comm->cliqueManager->SetCliqueCollectiveArgs(&c->args)); + } + // [/RCCL] STORE(&c->active, 1); opIndex = (opIndex+1)%NCCL_MAX_OPS; @@ -599,6 +643,7 @@ ncclResult_t ncclEnqueueCheck(struct ncclInfo* info) { } else { NCCLCHECKGOTO(ncclSaveKernel(info), ret, end); } + end: if (savedDev != -1) CUDACHECK(hipSetDevice(savedDev)); ncclAsyncErrCheck(ret); diff --git a/projects/rccl/src/include/bootstrap.h b/projects/rccl/src/include/bootstrap.h index a7d6be965e..aa671b7066 100644 --- a/projects/rccl/src/include/bootstrap.h +++ b/projects/rccl/src/include/bootstrap.h @@ -12,7 +12,7 @@ ncclResult_t bootstrapNetInit(); ncclResult_t bootstrapCreateRoot(ncclUniqueId* commId, bool idFromEnv); ncclResult_t bootstrapGetUniqueId(ncclUniqueId* out); -ncclResult_t bootstrapInit(ncclUniqueId* id, int rank, int nranks, void** commState); +ncclResult_t bootstrapInit(ncclUniqueId* id, int rank, int nranks, void** commState, int* rootPid); // [RCCL] Adding rootPid ncclResult_t bootstrapAllGather(void* commState, void* allData, int size); ncclResult_t bootstrapSend(void* commState, int peer, void* data, int size); ncclResult_t bootstrapRecv(void* commState, int peer, void* data, int size); diff --git a/projects/rccl/src/include/comm.h b/projects/rccl/src/include/comm.h index 6df4f750d3..d527e83801 100644 --- a/projects/rccl/src/include/comm.h +++ b/projects/rccl/src/include/comm.h @@ -10,6 +10,9 @@ #include "transport.h" #include "p2p.h" +// [RCCL] +#include "clique/CliqueManager.h" +// [/RCCL] #if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__) #else @@ -143,8 +146,12 @@ struct ncclComm { //list of async p2p operation queued in a group semantics struct ncclP2Plist p2plist; - // RCCL AllToAll/Scatter/Gather API - bool alltoallDisable; + // [RCCL] + bool alltoallDisable; // RCCL AllToAll/Scatter/Gather API + CliqueManager* cliqueManager; // CliqueManager handles pointer collection / distribution for clique-based kernels + int rootPid; // Process ID of root + // [/RCCL] + }; #endif diff --git a/projects/rccl/src/include/devcomm.h b/projects/rccl/src/include/devcomm.h index f7b870c5e5..d9c3b02ff2 100644 --- a/projects/rccl/src/include/devcomm.h +++ b/projects/rccl/src/include/devcomm.h @@ -12,6 +12,9 @@ #include "rccl_bfloat16.h" #include "align.h" #include +// [RCCL] Support for clique-based kernels +#include "clique/CliqueCommon.h" +// [/RCCL] // Convert volatile access to atomic #if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__) @@ -22,6 +25,7 @@ #define STORE(DST, SRC) *(DST) = (SRC) #endif + #define NCCL_NUM_FUNCTIONS 5 // SendRecv not included for now typedef enum { ncclCollBroadcast, ncclCollReduce, ncclCollAllGather, ncclCollReduceScatter, ncclCollAllReduce, ncclCollGather, ncclCollScatter, ncclCollAllToAll, ncclCollAllToAllv, ncclCollSendRecv} ncclFunc_t; extern const char* ncclFuncStr[NCCL_NUM_FUNCTIONS+4]; @@ -35,6 +39,7 @@ extern const char* ncclAlgoStr[NCCL_NUM_ALGORITHMS]; #define NCCL_NUM_PROTOCOLS 3 // Simple/LL/LL128 #define NCCL_PROTO_LL 0 #define NCCL_PROTO_LL128 1 +#define NCCL_PROTO_CLIQUE 1 // [RCCL] Clique takes up same protocol as unused LL128 #define NCCL_PROTO_SIMPLE 2 extern const char* ncclProtoStr[NCCL_NUM_PROTOCOLS]; @@ -190,8 +195,18 @@ struct CollectiveArgs { size_t count; size_t* extra; } a2av; + // [RCCL] Clique-based arguments + struct { + uint16_t nThreads; + uint8_t bid; + uint8_t nChannels; + size_t count; + cliqueDevicePtrs_t* ptrs; + } clique; + // [/RCCL] }; }; + struct ncclColl { union { struct { diff --git a/projects/rccl/src/init.cc b/projects/rccl/src/init.cc index 98c41b68e1..3c67b78708 100644 --- a/projects/rccl/src/init.cc +++ b/projects/rccl/src/init.cc @@ -28,6 +28,10 @@ #include #include "graph/topo.h" +// [RCCL] +#include "clique/CliqueManager.h" +// [/RCCL] + #define STR2(v) #v #define STR(v) STR2(v) @@ -300,6 +304,7 @@ static ncclResult_t commFree(ncclComm_t comm) { } RCCL_PARAM(AllToAllDisable, "ALLTOALL_KERNEL_DISABLE", 1); +RCCL_PARAM(ForceEnableClique, "FORCE_ENABLE_CLIQUE", 0); static ncclResult_t commAlloc(ncclComm_t* comret, int ndev, int rank) { if (ndev < 1) { @@ -678,7 +683,10 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm int nranks = comm->nRanks; uint64_t commHash = getHash(commId->internal, NCCL_UNIQUE_ID_BYTES); TRACE(NCCL_INIT, "comm %p, commHash %lx, rank %d nranks %d - BEGIN", comm, commHash, rank, nranks); - NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap)); + // [RCCL] Collect the PID of the root + int rootPid; + NCCLCHECK(bootstrapInit(commId, rank, nranks, &comm->bootstrap, &rootPid)); + // [/RCCL] // AllGather1 - begin struct { @@ -1016,8 +1024,53 @@ static ncclResult_t initTransportsRank(struct ncclComm* comm, ncclUniqueId* comm return ncclInternalError; } NCCLCHECK(ncclCommSetIntra(comm, intraRank, intraRanks, allGather1Data[intraRank0].comm)); + + { // [RCCL] Check if clique-based kernels can be enabled and initialize CliqueManager if so + CliqueManager::cliqueMode_t cliqueMode = CliqueManager::CLIQUE_DISABLED; + if (comm->localRanks == comm->nRanks) + { + // Check that all the GPUs have peer access to one another + bool hasPeerAccess = true; + for (int i = 0; i < nranks && hasPeerAccess; i++) + { + int cudaDev1 = allGather1Data[i].peerInfo.cudaDev; + for (int j = 0; j < nranks; j++) + { + if (i == j) continue; + int cudaDev2 = allGather1Data[j].peerInfo.cudaDev; + int p2p; + if (hipDeviceCanAccessPeer(&p2p, cudaDev1, cudaDev2) != hipSuccess || !p2p) + { + hasPeerAccess = false; + break; + } + } + } + if (hasPeerAccess) + { + if (intraRanks == nranks) + cliqueMode = CliqueManager::CLIQUE_SINGLE_PROCESS; + else + cliqueMode = CliqueManager::CLIQUE_SINGLE_NODE; + } + + // For now, only enable clique-based kernels on CR8_G topologies, unless explicitly asked + if (!rcclParamForceEnableClique()) + { + // Disable clique-kernel support if not on CR8 topology + if (!(comm->topo->nodes[NET].count == 0 && comm->topo->type == RCCL_TOPO_CR8G)) + { + INFO(NCCL_INIT, "Disabling clique-based kernels due to topology (force enable with RCCL_FORCE_ENABLE_CLIQUE)"); + cliqueMode = CliqueManager::CLIQUE_DISABLED; + } + } + } + comm->cliqueManager = new CliqueManager(rank, nranks, cliqueMode); + NCCLCHECK(comm->cliqueManager->Init(commId, rootPid)); + } // [/RCCL] } while(0); + // Done with AllGather1 data free(allGather1Data); @@ -1147,6 +1200,10 @@ ncclResult_t ncclCommDestroy(ncclComm_t comm) { return ncclInvalidArgument; } + // [RCCL] Delete CliqueManager if it exists + if (comm->cliqueManager) delete comm->cliqueManager; + // [/RCCL] + return commDestroy(comm); } diff --git a/projects/rccl/test/CorrectnessTest.hpp b/projects/rccl/test/CorrectnessTest.hpp index 95010ee701..fb11027b64 100644 --- a/projects/rccl/test/CorrectnessTest.hpp +++ b/projects/rccl/test/CorrectnessTest.hpp @@ -418,23 +418,24 @@ namespace CorrectnessTests switch (dataset.dataType) { case ncclInt8: - printf("Expected %d. Output %d on device %d[%d]\n", outputI1[j], expectedI1[j], i, j); break; + printf("Expected %d. Output %d on device %d[%d]\n", expectedI1[j], outputI1[j], i, j); + break; case ncclUint8: - printf("Expected %u. Output %u on device %d[%d]\n", outputU1[j], expectedU1[j], i, j); break; + printf("Expected %u. Output %u on device %d[%d]\n", expectedU1[j], outputU1[j], i, j); break; case ncclInt32: - printf("Expected %d. Output %d on device %d[%d]\n", outputI4[j], expectedI4[j], i, j); break; + printf("Expected %d. Output %d on device %d[%d]\n", expectedI4[j], outputI4[j], i, j); break; case ncclUint32: - printf("Expected %u. Output %u on device %d[%d]\n", outputU4[j], expectedU4[j], i, j); break; + printf("Expected %u. Output %u on device %d[%d]\n", expectedU4[j], outputU4[j], i, j); break; case ncclInt64: - printf("Expected %ld. Output %ld on device %d[%d]\n", outputI8[j], expectedI8[j], i, j); break; + printf("Expected %ld. Output %ld on device %d[%d]\n", expectedI8[j], outputI8[j], i, j); break; case ncclUint64: - printf("Expected %lu. Output %lu on device %d[%d]\n", outputU8[j], expectedU8[j], i, j); break; + printf("Expected %lu. Output %lu on device %d[%d]\n", expectedU8[j], outputU8[j], i, j); break; case ncclFloat32: - printf("Expected %f. Output %f on device %d[%d]\n", outputF4[j], expectedF4[j], i, j); break; + printf("Expected %f. Output %f on device %d[%d]\n", expectedF4[j], outputF4[j], i, j); break; case ncclFloat64: - printf("Expected %lf. Output %lf on device %d[%d]\n", outputF8[j], expectedF8[j], i, j); break; + printf("Expected %lf. Output %lf on device %d[%d]\n", expectedF8[j], outputF8[j], i, j); break; case ncclBfloat16: - printf("Expected %f. Output %f on device %d[%d]\n", (float)outputB2[j], (float)expectedB2[j], i, j); break; + printf("Expected %f. Output %f on device %d[%d]\n", (float)expectedB2[j], (float)outputB2[j], i, j); break; default: fprintf(stderr, "[ERROR] Unsupported datatype\n"); exit(0); diff --git a/projects/rccl/test/test_AllReduce.cpp b/projects/rccl/test/test_AllReduce.cpp index 992b7c32d4..28f76e52d2 100644 --- a/projects/rccl/test/test_AllReduce.cpp +++ b/projects/rccl/test/test_AllReduce.cpp @@ -58,6 +58,6 @@ namespace CorrectnessTests testing::Values(2,3,4,5,6,7,8), // In-place or not testing::Values(false, true), - testing::Values("")), + testing::Values("RCCL_ENABLE_CLIQUE=0", "RCCL_ENABLE_CLIQUE=1")), CorrectnessTest::PrintToStringParamName()); } // namespace diff --git a/projects/rccl/test/test_CombinedCalls.cpp b/projects/rccl/test/test_CombinedCalls.cpp index db4f938fc0..7d1b23c98c 100644 --- a/projects/rccl/test/test_CombinedCalls.cpp +++ b/projects/rccl/test/test_CombinedCalls.cpp @@ -96,6 +96,6 @@ namespace CorrectnessTests testing::Values(2,3,4,5,6,7,8), // In-place or not testing::Values(false, true), - testing::Values("")), + testing::Values("RCCL_ENABLE_CLIQUE=0", "RCCL_ENABLE_CLIQUE=1")), CorrectnessTest::PrintToStringParamName()); } // namespace diff --git a/projects/rccl/test/test_GroupCalls.cpp b/projects/rccl/test/test_GroupCalls.cpp index b3ab2ca09b..f94dd6cf6a 100644 --- a/projects/rccl/test/test_GroupCalls.cpp +++ b/projects/rccl/test/test_GroupCalls.cpp @@ -116,6 +116,6 @@ namespace CorrectnessTests testing::Values(2,3,4,5,6,7,8), // In-place or not testing::Values(false, true), - testing::Values("")), + testing::Values("RCCL_ENABLE_CLIQUE=0", "RCCL_ENABLE_CLIQUE=1")), CorrectnessTest::PrintToStringParamName()); } // namespace diff --git a/projects/rccl/tools/HelloRccl/HelloRccl.cpp b/projects/rccl/tools/HelloRccl/HelloRccl.cpp new file mode 100644 index 0000000000..4eb72b6a52 --- /dev/null +++ b/projects/rccl/tools/HelloRccl/HelloRccl.cpp @@ -0,0 +1,257 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "HelloRccl.hpp" + + +void Usage(char *argv0); +void ExecuteTest(int numIntraRank, int intraRankStartId, int numTotalRanks, ncclComm_t* comm); + +int main(int argc, char **argv) +{ + if (getenv("NCCL_COMM_ID") && argc == 3) // Run in multi-process mode + { + int nranks = atoi(argv[1]); + int rank = atoi(argv[2]); + if (rank == 0) printf("Running in multi-process mode\n"); + + // Create communicator for this rank + ncclUniqueId commId; + NCCL_CALL(ncclGetUniqueId(&commId)); + + // Initialize communicator + ncclComm_t comm; + HIP_CALL(hipSetDevice(rank)); + NCCL_CALL(ncclCommInitRank(&comm, nranks, commId, rank)); + + // Run the test + ExecuteTest(1, rank, nranks, &comm); + } + else if (argc == 2) // Run in single-process mode + { + printf("Running in single-process mode\n"); + + int nranks = atoi(argv[1]); + + // Initialize communicators for each rank + ncclComm_t comm[nranks]; + NCCL_CALL(ncclCommInitAll(comm, nranks, NULL)); + + // Run the test + ExecuteTest(nranks, 0, nranks, comm); + } + else + { + Usage(argv[0]); + return 1; + } + return 0; +} + +void ExecuteTest(int numIntraRank, int intraRankStartId, int numTotalRanks, ncclComm_t* comm) +{ + // Test configuration settings + int minPow = 10; // Starting power of 2 input size + int maxPow = 28; // Ending power of 2 input size + int numWarmups = 3; // Number of untimed warmup iterations + int numIterations = 10; // Number of timed iterations + + // Allocate GPU resources for this process + hipStream_t stream[numIntraRank]; + hipEvent_t startEvent[numIntraRank]; + hipEvent_t stopEvent[numIntraRank]; + for (int i = 0; i < numIntraRank; i++) + { + HIP_CALL(hipSetDevice(intraRankStartId + i)); + HIP_CALL(hipStreamCreate(&stream[i])); + HIP_CALL(hipEventCreate(&startEvent[i])); + HIP_CALL(hipEventCreate(&stopEvent[i])); + } + + if (intraRankStartId == 0) + { + printf("AllReduce Performance (sum of floats):\n"); + printf("%10s %10s %10s\n", "Bytes", "CpuTime(ms)", "GpuTime(ms)"); + } + + // Loop over power-of-two input sizes + for (int power = minPow; power <= maxPow; power++) + { + int N = 1 << power; + + // Allocate GPU memory + float *iputGpu[numIntraRank], *oputGpu[numIntraRank]; + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipSetDevice(intraRankStartId + r)); + HIP_CALL(hipMalloc((void **)&iputGpu[r], N * sizeof(float))); + HIP_CALL(hipMalloc((void **)&oputGpu[r], N * sizeof(float))); + } + + // Allocate CPU memory for input/output + float *iputCpu = (float *)malloc(N * sizeof(float)); + float *oputCpu = (float *)malloc(N * sizeof(float)); + + // Fill CPU memory with a simple pattern + for (int i = 0; i < N; i++) + { + iputCpu[i] = 1.0f; + oputCpu[i] = 0.0f; + } + + // Copy the input from CPU memory to GPU memory + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipSetDevice(intraRankStartId + r)); + HIP_CALL(hipMemcpy(iputGpu[r], iputCpu, N * sizeof(float), hipMemcpyHostToDevice)); + } + + // Perform some untimed initial warmup iterations + for (int iteration = 0; iteration < numWarmups; iteration++) + { + NCCL_CALL(ncclGroupStart()); + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipSetDevice(intraRankStartId + r)); + NCCL_CALL(ncclAllReduce(iputGpu[r], oputGpu[r], N, ncclFloat, ncclSum, comm[r], stream[r])); + } + NCCL_CALL(ncclGroupEnd()); + } + for (int r = 0; r < numIntraRank; r++) + HIP_CALL(hipStreamSynchronize(stream[r])); + + // Perform timed iterations + auto cpuStart = std::chrono::high_resolution_clock::now(); + for (int r = 0; r < numIntraRank; r++) + HIP_CALL(hipEventRecord(startEvent[r], stream[r])); + + for (int iteration = 0; iteration < numIterations; iteration++) + { + NCCL_CALL(ncclGroupStart()); + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipSetDevice(intraRankStartId + r)); + NCCL_CALL(ncclAllReduce(iputGpu[r], oputGpu[r], N, ncclFloat, ncclSum, comm[r], stream[r])); + } + NCCL_CALL(ncclGroupEnd()); + } + + for (int r = 0; r < numIntraRank; r++) + HIP_CALL(hipEventRecord(stopEvent[r], stream[r])); + + for (int r = 0; r < numIntraRank; r++) + HIP_CALL(hipStreamSynchronize(stream[r])); + + auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; + double totalCpuTime = std::chrono::duration_cast>(cpuDelta).count(); + + float totalGpuTime; + HIP_CALL(hipEventElapsedTime(&totalGpuTime, startEvent[0], stopEvent[0])); + + if (intraRankStartId == 0) printf("%10lu %10.3f %10.3f\n", N * sizeof(float), (totalCpuTime / numIterations), (totalGpuTime / numIterations)); + + // Validate results + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipMemcpy(oputCpu, oputGpu[r], N * sizeof(float), hipMemcpyDeviceToHost)); + bool isOK = true; + int expected = numTotalRanks; + for (int i = 0; i < N; i++) + { + isOK &= (oputCpu[i] == expected); + } + if (!isOK) + { + printf("[ERROR] Rank %d Incorrect results for N = %d\n", intraRankStartId + r, N); + NCCL_CALL(ncclCommDestroy(comm[r])); + exit(1); + } + } + + // Release GPU resources + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipFree(oputGpu[r])); + HIP_CALL(hipFree(iputGpu[r])); + } + free(iputCpu); + free(oputCpu); + } + + for (int r = 0; r < numIntraRank; r++) + { + HIP_CALL(hipStreamDestroy(stream[r])); + HIP_CALL(hipEventDestroy(startEvent[r])); + HIP_CALL(hipEventDestroy(stopEvent[r])); + NCCL_CALL(ncclCommDestroy(comm[r])); + } +} + +void Usage(char *argv0) +{ + printf("Single Process Usage: %s numRanks\n", argv0); + printf("\n"); + printf("Multi Process Usage: %s numRanks rank\n", argv0); + printf(" - NCCL_COMM_ID must be set in order to use this\n\n"); + printf(" - To use this process as the root process you may use any of the following:\n"); + + char hostname[256]; + gethostname(hostname, 256); + printf(" export NCCL_COMM_ID=%s:12345\n", hostname); + + // Loop over linked list of interfaces + struct ifaddrs *ifaddr; + getifaddrs(&ifaddr); + for (struct ifaddrs* ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) + { + // Skip anything not based on IPv4 / IPv6 + int family = ifa->ifa_addr->sa_family; + if (family != AF_INET && family != AF_INET6) continue; + + // Skip iPv6 loopback interface + if (family == AF_INET6) + { + struct sockaddr_in6* sa = (struct sockaddr_in6*)(ifa->ifa_addr); + if (IN6_IS_ADDR_LOOPBACK(&sa->sin6_addr)) continue; + } + + socklen_t saLen = (family == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6)); + char host[NI_MAXHOST]; + char service[NI_MAXSERV]; + + getnameinfo(ifa->ifa_addr, saLen, host, NI_MAXHOST, service, NI_MAXSERV, + NI_NUMERICHOST|NI_NUMERICSERV); + + std::string result = std::string(host) + ":12345"; + printf(" export NCCL_COMM_ID=%s\n", result.c_str()); + } + freeifaddrs(ifaddr); +} diff --git a/projects/rccl/tools/HelloRccl/HelloRccl.hpp b/projects/rccl/tools/HelloRccl/HelloRccl.hpp new file mode 100644 index 0000000000..55d24ecd5d --- /dev/null +++ b/projects/rccl/tools/HelloRccl/HelloRccl.hpp @@ -0,0 +1,49 @@ +/* +Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#ifndef HELLORCCL_HPP +#define HELLORCCL_HPP +#include + +#define HIP_CALL(cmd) \ + do { \ + hipError_t error = (cmd); \ + if (error != hipSuccess) \ + { \ + std::cerr << "Encountered HIP error (" << hipGetErrorString(error) << ") at line " \ + << __LINE__ << " in file " << __FILE__ << "\n"; \ + exit(-1); \ + } \ + } while (0) + +#define NCCL_CALL(cmd) \ + do { \ + ncclResult_t error = (cmd); \ + if (error != ncclSuccess) \ + { \ + std::cerr << "Encountered NCCL error (" << ncclGetErrorString(error) << ") at line " \ + << __LINE__ << " in file " << __FILE__ << "\n"; \ + exit(-1); \ + } \ + } while (0) + +#endif diff --git a/projects/rccl/tools/HelloRccl/Makefile b/projects/rccl/tools/HelloRccl/Makefile new file mode 100644 index 0000000000..8729c17feb --- /dev/null +++ b/projects/rccl/tools/HelloRccl/Makefile @@ -0,0 +1,21 @@ +# Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. + +# Set to where RCCL is installed +RCCL_INSTALL=../../build/release + +HIP_PATH?= $(wildcard /opt/rocm/hip) +ifeq (,$(HIP_PATH)) +HIP_PATH=../../.. +endif +HIPCC=$(HIP_PATH)/bin/hipcc + +EXE=HelloRccl +CXXFLAGS = -std=c++11 -O3 -I../../src/include -I$(RCCL_INSTALL) -L$(RCCL_INSTALL) -lrccl + +all: $(EXE) + +$(EXE): $(EXE).cpp $(shell find -regex ".*\.\hpp") + $(HIPCC) $(CXXFLAGS) $< -o $@ + +clean: + rm -f *.o $(EXE) diff --git a/projects/rccl/tools/HelloRccl/runTest.sh b/projects/rccl/tools/HelloRccl/runTest.sh new file mode 100755 index 0000000000..c6e832e3c4 --- /dev/null +++ b/projects/rccl/tools/HelloRccl/runTest.sh @@ -0,0 +1,22 @@ +#!/bin/bash +RCCL_INSTALL=../../build/release +EXE=$PWD/HelloRccl +LDPATH=$LD_LIBRARY_PATH:$RCCL_INSTALL + +echo "Single process - With clique-based kernels:" +RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT=1073741824 RCCL_FORCE_ENABLE_CLIQUE=1 NCCL_DEBUG=INFO RCCL_ENABLE_CLIQUE=1 LD_LIBRARY_PATH=$LDPATH $EXE 4 + +echo "Single process - Without clique-based kernels:" +NCCL_DEBUG=INFO LD_LIBRARY_PATH=$LDPATH $EXE 4 + +echo "With clique-based kernels:" +RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT=1073741824 RCCL_FORCE_ENABLE_CLIQUE=1 NCCL_DEBUG=INFO RCCL_ENABLE_CLIQUE=1 NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 0 & +RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT=1073741824 RCCL_FORCE_ENABLE_CLIQUE=1 NCCL_DEBUG=INFO RCCL_ENABLE_CLIQUE=1 NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 1 & +RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT=1073741824 RCCL_FORCE_ENABLE_CLIQUE=1 NCCL_DEBUG=INFO RCCL_ENABLE_CLIQUE=1 NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 2 & +RCCL_CLIQUE_ALLREDUCE_BYTE_LIMIT=1073741824 RCCL_FORCE_ENABLE_CLIQUE=1 NCCL_DEBUG=INFO RCCL_ENABLE_CLIQUE=1 NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 3 + +echo "Without clique-based kernels:" +NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 0 & +NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 1 & +NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 2 & +NCCL_COMM_ID=$HOSTNAME:12345 LD_LIBRARY_PATH=$LDPATH $EXE 4 3 From 62d21047b8c0c4affb96e23649541759e9454869 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Thu, 12 Nov 2020 15:47:34 -0800 Subject: [PATCH 03/17] Skip unused peer connection in scatter and gather (#301) [ROCm/rccl commit: 4e68229c8b365d2e9108f06ae86ab39ebf1f311e] --- projects/rccl/src/collectives/device/gather.h | 5 +++-- projects/rccl/src/collectives/device/scatter.h | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/projects/rccl/src/collectives/device/gather.h b/projects/rccl/src/collectives/device/gather.h index c704e39421..b6c3865875 100644 --- a/projects/rccl/src/collectives/device/gather.h +++ b/projects/rccl/src/collectives/device/gather.h @@ -52,9 +52,10 @@ __device__ void ncclGatherKernel(struct CollectiveArgs* args) { else { int peerSend = (rank+(blockIdx.x*peersPerChan)+i)%nranks; int peerRecv = (2*nranks+rank-((blockIdx.x*peersPerChan)%nranks)-(i%nranks))%nranks; + int peerNone = -1; if (rank == root) { ncclPrimitives - prims(tid, nthreads, &peerRecv, &peerSend, NULL, stepSize, channel, comm); + prims(tid, nthreads, &peerRecv, &peerNone, NULL, stepSize, channel, comm); ssize_t recv_offset = chunkOffset + peerRecv*size; prims.recv(thisOutput+recv_offset, nelem); @@ -62,7 +63,7 @@ __device__ void ncclGatherKernel(struct CollectiveArgs* args) { else { if (peerSend == root) { ncclPrimitives - prims(tid, nthreads, &peerRecv, &peerSend, NULL, stepSize, channel, comm); + prims(tid, nthreads, &peerNone, &peerSend, NULL, stepSize, channel, comm); ssize_t send_offset = chunkOffset; prims.send(thisInput+send_offset, nelem); diff --git a/projects/rccl/src/collectives/device/scatter.h b/projects/rccl/src/collectives/device/scatter.h index f08544068a..848b47713d 100644 --- a/projects/rccl/src/collectives/device/scatter.h +++ b/projects/rccl/src/collectives/device/scatter.h @@ -52,9 +52,10 @@ __device__ void ncclScatterKernel(struct CollectiveArgs* args) { else { int peerSend = (rank+(blockIdx.x*peersPerChan)+i)%nranks; int peerRecv = (2*nranks+rank-((blockIdx.x*peersPerChan)%nranks)-(i%nranks))%nranks; + int peerNone = -1; if (rank == root) { ncclPrimitives - prims(tid, nthreads, &peerRecv, &peerSend, NULL, stepSize, channel, comm); + prims(tid, nthreads, &peerNone, &peerSend, NULL, stepSize, channel, comm); ssize_t send_offset = chunkOffset + peerSend*size; prims.send(thisInput+send_offset, nelem); @@ -62,7 +63,7 @@ __device__ void ncclScatterKernel(struct CollectiveArgs* args) { else { if (peerRecv == root) { ncclPrimitives - prims(tid, nthreads, &peerRecv, &peerSend, NULL, stepSize, channel, comm); + prims(tid, nthreads, &peerRecv, &peerNone, NULL, stepSize, channel, comm); ssize_t recv_offset = chunkOffset; prims.recv(thisOutput+recv_offset, nelem); From f66d05193aed26650049cb019c5d3a3af8e3e454 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Fri, 13 Nov 2020 09:07:11 -0700 Subject: [PATCH 04/17] Adding RCCL_CLIQUE_DEBUG to help debug experimental clique feature (#300) [ROCm/rccl commit: c8d08a7c2fd74cad2f371be0bac836bf56fe9c2e] --- .../rccl/src/clique/AllReduceCliqueKernel.h | 12 +++++++++- projects/rccl/src/clique/CliqueCommon.h | 16 +++++++++++-- projects/rccl/src/clique/CliqueManager.cc | 24 ++++++++++++++++--- projects/rccl/src/include/devcomm.h | 1 + 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/projects/rccl/src/clique/AllReduceCliqueKernel.h b/projects/rccl/src/clique/AllReduceCliqueKernel.h index a46ed6a219..67a306d3c7 100644 --- a/projects/rccl/src/clique/AllReduceCliqueKernel.h +++ b/projects/rccl/src/clique/AllReduceCliqueKernel.h @@ -36,6 +36,7 @@ __device__ void AllReduceCliqueSplitKernel(struct CollectiveArgs* args) size_t const N = args->clique.count; // Total number of elements to reduce int const nBlocks = args->clique.nChannels; // Total number of blocks assigned to this kernel (may be different than gridDim.x) int const blockId = args->clique.bid; // 0-indexed blockIdx for this threadblock (may be different than blockIdx.x) + int const verbose = args->clique.verbose; // For debug purposes int const rank = args->comm->rank; // Current rank // Each threadblock works independently of others on a subsection of the input @@ -46,6 +47,10 @@ __device__ void AllReduceCliqueSplitKernel(struct CollectiveArgs* args) size_t const currBlockStop = min(currBlockStart + perBlockN, N); size_t const blockN = currBlockStop - currBlockStart; + if (verbose && threadIdx.x == 0) + { + printf("Rank %d block %d of %d %lu -> %lu [%lu]\n", rank, blockId, nBlocks, currBlockStart, currBlockStop, blockN); + } if (blockN > 0) { // Prepare input / output subarrays @@ -69,7 +74,12 @@ __device__ void AllReduceCliqueSplitKernel(struct CollectiveArgs* args) // Even if there was nothing for this GPU to do, it must participate in a barrier // because other GPUs may be modifying this GPUs output buffer still - if (blockId == 0) WaitForBarrier(cliquePtrs->barrier); + if (blockId == 0) + { + if (verbose && threadIdx.x == 0) printf("Rank %d enters GPU barrier\n", rank); + WaitForBarrier(cliquePtrs->barrier, rank, verbose); + if (verbose && threadIdx.x == 0) printf("Rank %d exits GPU barrier\n", rank); + } } #endif diff --git a/projects/rccl/src/clique/CliqueCommon.h b/projects/rccl/src/clique/CliqueCommon.h index b7bdee19ae..0fab8cc8ad 100644 --- a/projects/rccl/src/clique/CliqueCommon.h +++ b/projects/rccl/src/clique/CliqueCommon.h @@ -62,7 +62,7 @@ typedef struct // Multi-GPU (on same node) barrier. One thread per grid per GPU updates barrier / waits template -__forceinline__ __device__ void WaitForBarrier(gpuBarrier_t const& barrier) +__forceinline__ __device__ void WaitForBarrier(gpuBarrier_t const& barrier, int const rank, int const verbose) { if (threadIdx.x == 0) { @@ -71,6 +71,7 @@ __forceinline__ __device__ void WaitForBarrier(gpuBarrier_t const& barrier) int localSense = *barrier.localSense; int val = __atomic_add_fetch(barrier.globalCount, 1, __ATOMIC_SEQ_CST); + if (verbose) printf("Rank %d arrived at GPU barrier %d\n", rank, val); if (val == NUM_RANKS) { // Last arrival resets barrier @@ -80,7 +81,18 @@ __forceinline__ __device__ void WaitForBarrier(gpuBarrier_t const& barrier) else { // Wait for all ranks to reach barrier - while (__atomic_load_n(barrier.globalSense, __ATOMIC_SEQ_CST) != localSense); + int counter = 0; + while (__atomic_load_n(barrier.globalSense, __ATOMIC_SEQ_CST) != localSense) + { + if (verbose) + { + counter++; + if (counter == 100000000) + { + printf("Rank %d waiting on GPU barrier: (%d != %d)", rank, *barrier.globalSense, localSense); + } + } + } } } } diff --git a/projects/rccl/src/clique/CliqueManager.cc b/projects/rccl/src/clique/CliqueManager.cc index 3afd3cfa31..fef2a91599 100644 --- a/projects/rccl/src/clique/CliqueManager.cc +++ b/projects/rccl/src/clique/CliqueManager.cc @@ -48,6 +48,7 @@ int* CliqueManager::m_staticGpuBarrierMem = NULL; RCCL_PARAM(EnableClique, "ENABLE_CLIQUE", 0); // Opt-in environment variable for clique-based kernels RCCL_PARAM(AllReduceCliqueByteLimit, "CLIQUE_ALLREDUCE_BYTE_LIMIT", 2097152); // Max number of bytes to use clique-based kernels for all reduce RCCL_PARAM(AllReduceNumChannels, "CLIQUE_ALLREDUCE_NCHANNELS", 4); // Number of channels to use for all-reduce +RCCL_PARAM(CliqueDebug, "CLIQUE_DEBUG", 0); // Emit debug messages CliqueManager::CliqueManager(int const rank, int const numRanks, @@ -337,7 +338,7 @@ ncclResult_t CliqueManager::SetCliqueCollectiveArgs(CollectiveArgs* args) // Prepare clique argments (NOTE: clique pointers are not ready yet) int opIndex = args->opCount % NCCL_MAX_OPS; args->clique.ptrs = &m_pinnedCliquePtrs[opIndex]; - + args->clique.verbose = rcclParamCliqueDebug(); // Determine number of channels to use for this collective args->clique.nChannels = rcclParamAllReduceNumChannels(); @@ -477,16 +478,33 @@ ncclResult_t CliqueManager::CheckCacheForHandle(std::pair Date: Fri, 13 Nov 2020 10:32:42 -0700 Subject: [PATCH 05/17] Fixing IPC handle leak (#302) [ROCm/rccl commit: 2958f7eacedc2e498ee232ba9c76e54239cf829d] --- projects/rccl/src/clique/CliqueManager.cc | 6 +- projects/rccl/src/clique/HandleCache.h | 72 ++++++++++++++++++++--- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/projects/rccl/src/clique/CliqueManager.cc b/projects/rccl/src/clique/CliqueManager.cc index fef2a91599..dde0cf0085 100644 --- a/projects/rccl/src/clique/CliqueManager.cc +++ b/projects/rccl/src/clique/CliqueManager.cc @@ -82,7 +82,11 @@ void CliqueManager::CleanUp() { // Release caches if (m_ipcHandleSendCache) delete m_ipcHandleSendCache; - if (m_ipcHandleSendCache) delete m_ipcHandleRecvCache; + if (m_ipcHandleRecvCache) + { + m_ipcHandleRecvCache->close(); + delete m_ipcHandleRecvCache; + } // Close shared memory m_shmHandles.Close(); diff --git a/projects/rccl/src/clique/HandleCache.h b/projects/rccl/src/clique/HandleCache.h index dc479e00e8..245b78f80d 100644 --- a/projects/rccl/src/clique/HandleCache.h +++ b/projects/rccl/src/clique/HandleCache.h @@ -103,14 +103,16 @@ public: return std::pair(it, inserted); } - + ncclResult_t close(); private: - void pop() - { - typename LRUCache::iterator it = m_cache.find(m_lruHistory.front()); - m_cache.erase(it); - m_lruHistory.pop_front(); - } + // tag for dispatch + template + struct CloseTag{}; + + hipError_t CloseIfPointer(CloseTag tag, iterator it); + hipError_t CloseIfPointer(CloseTag tag, iterator it); + + void pop(); void updateHistory(const iterator& it) { @@ -133,8 +135,60 @@ auto hipIpcMemHandleEqual = [](const hipIpcMemHandle_t& l, const hipIpcMemHandle return memcmp(l.reserved, r.reserved, sizeof(l.reserved)) == 0; }; -//typedef llvm::DenseMap SendCache; -//typedef llvm::DenseMap RecvCache; +template < + class Key, + class Value, + class Hash, + class KeyEqual, + class Allocator +> +ncclResult_t NcclIpcHandleCache::close() +{ + for (auto it = m_cache.begin(); it != m_cache.end(); it ++) + { + CUDACHECK(CloseIfPointer(CloseTag{}, it)); + } + return ncclSuccess; +} + +template < + class Key, + class Value, + class Hash, + class KeyEqual, + class Allocator +> +void NcclIpcHandleCache::pop() +{ + typename LRUCache::iterator it = m_cache.find(m_lruHistory.front()); + CloseIfPointer(CloseTag{}, it); + m_cache.erase(it); + m_lruHistory.pop_front(); +} + +template < + class Key, + class Value, + class Hash, + class KeyEqual, + class Allocator +> +hipError_t NcclIpcHandleCache::CloseIfPointer(CloseTag tag, iterator it) +{ + return hipIpcCloseMemHandle(it->second.first); +} + +template < + class Key, + class Value, + class Hash, + class KeyEqual, + class Allocator +> +hipError_t NcclIpcHandleCache::CloseIfPointer(CloseTag tag, iterator it) +{ + return hipSuccess; +} typedef NcclIpcHandleCache, std::equal_to, std::allocator< std::pair::iterator>>>> NcclIpcHandleSendCache; typedef NcclIpcHandleCache::iterator>>>> NcclIpcHandleRecvCache; From 36935cfbeec6ab9823288d395f33c1264c6eeb41 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Fri, 13 Nov 2020 17:57:44 -0800 Subject: [PATCH 06/17] gtest: add scatter to combined calls and use loops (#303) * gtest: add scatter to combined calls and use loops * gtest: run validation inside loop * gtest: revert small element count to 2520 * gtest: fix memory leak in validation [ROCm/rccl commit: b0853ccd516cee04879e890a897967bbaa368375] --- projects/rccl/test/CorrectnessTest.hpp | 1 + projects/rccl/test/test_CombinedCalls.cpp | 73 ++++++++++++++--------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/projects/rccl/test/CorrectnessTest.hpp b/projects/rccl/test/CorrectnessTest.hpp index fb11027b64..12c5fbe3f1 100644 --- a/projects/rccl/test/CorrectnessTest.hpp +++ b/projects/rccl/test/CorrectnessTest.hpp @@ -444,6 +444,7 @@ namespace CorrectnessTests } ASSERT_EQ(isMatch, true); } + free(outputI1); } // Passed in parameters from TestTuple diff --git a/projects/rccl/test/test_CombinedCalls.cpp b/projects/rccl/test/test_CombinedCalls.cpp index 7d1b23c98c..3916b4b8cc 100644 --- a/projects/rccl/test/test_CombinedCalls.cpp +++ b/projects/rccl/test/test_CombinedCalls.cpp @@ -10,6 +10,7 @@ #include "test_Broadcast.hpp" #include "test_Reduce.hpp" #include "test_ReduceScatter.hpp" +#include "test_Scatter.hpp" namespace CorrectnessTests { @@ -25,6 +26,10 @@ namespace CorrectnessTests FillDatasetWithPattern(datasets[i]); } + Dataset scatter_dataset; + scatter_dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollScatter); + FillDatasetWithPattern(scatter_dataset); + // Compute expected results for each dataset in combined int const root = 0; AllGatherCorrectnessTest::ComputeExpectedResults(datasets[0]); @@ -32,46 +37,56 @@ namespace CorrectnessTests BroadcastCorrectnessTest::ComputeExpectedResults(datasets[2], root); ReduceCorrectnessTest::ComputeExpectedResults(datasets[3], op, root); ReduceScatterCorrectnessTest::ComputeExpectedResults(datasets[4], op); + ScatterCorrectnessTest::ComputeExpectedResults(scatter_dataset, root); size_t const byteCount = datasets[0].NumBytes() / numDevices; size_t const elemCount = numElements / numDevices; - ncclGroupStart(); - for (int i = 0; i < numDevices; i++) + for (int j = 0; j < 10; j++) { - ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), - datasets[0].outputs[i], elemCount, - dataType, comms[i], streams[i]); + ncclGroupStart(); + for (int i = 0; i < numDevices; i++) + { + ncclScatter(scatter_dataset.inputs[i], + scatter_dataset.outputs[i], + numElements, dataType, + root, comms[i], streams[i]); - ncclAllReduce(datasets[1].inputs[i], datasets[1].outputs[i], - numElements, dataType, op, comms[i], streams[i]); + ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), + datasets[0].outputs[i], elemCount, + dataType, comms[i], streams[i]); - ncclBroadcast(datasets[2].inputs[i], - datasets[2].outputs[i], - numElements, dataType, - root, comms[i], streams[i]); + ncclAllReduce(datasets[1].inputs[i], datasets[1].outputs[i], + numElements, dataType, op, comms[i], streams[i]); - ncclReduce(datasets[3].inputs[i], - datasets[3].outputs[i], - numElements, dataType, op, - root, comms[i], streams[i]); + ncclBroadcast(datasets[2].inputs[i], + datasets[2].outputs[i], + numElements, dataType, + root, comms[i], streams[i]); - ncclReduceScatter(datasets[4].inputs[i], - (int8_t *)datasets[4].outputs[i] + (i * byteCount), - elemCount, dataType, op, - comms[i], streams[i]); + ncclReduce(datasets[3].inputs[i], + datasets[3].outputs[i], + numElements, dataType, op, + root, comms[i], streams[i]); + + ncclReduceScatter(datasets[4].inputs[i], + (int8_t *)datasets[4].outputs[i] + (i * byteCount), + elemCount, dataType, op, + comms[i], streams[i]); + } + ncclGroupEnd(); + // Wait for reduction to complete + Synchronize(); + // Check results for each collective in the combined + for (int i = 0; i < 5; i++) + ValidateResults(datasets[i]); + + ValidateResults(scatter_dataset); } - ncclGroupEnd(); - // Wait for reduction to complete - Synchronize(); - - // Check results for each collective in the combined for (int i = 0; i < 5; i++) - { - ValidateResults(datasets[i]); datasets[i].Release(); - } + scatter_dataset.Release(); } INSTANTIATE_TEST_SUITE_P(CombinedCallsCorrectnessSweep, @@ -95,7 +110,7 @@ namespace CorrectnessTests // Number of devices testing::Values(2,3,4,5,6,7,8), // In-place or not - testing::Values(false, true), - testing::Values("RCCL_ENABLE_CLIQUE=0", "RCCL_ENABLE_CLIQUE=1")), + testing::Values(false), + testing::Values("RCCL_ENABLE_CLIQUE=0", "RCCL_ENABLE_CLIQUE=1", "RCCL_ALLTOALL_KERNEL_DISABLE=0", "RCCL_ALLTOALL_KERNEL_DISABLE=1")), CorrectnessTest::PrintToStringParamName()); } // namespace From f19cbc8e512c49d58cac1c5dac1c2ca83457f42e Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Fri, 13 Nov 2020 17:58:04 -0800 Subject: [PATCH 07/17] Use device's link width and speed if port doesn't report (#304) [ROCm/rccl commit: 554729079d98a1d95eaf69ded5f8b2f286532dd8] --- projects/rccl/src/graph/xml.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/projects/rccl/src/graph/xml.cc b/projects/rccl/src/graph/xml.cc index a1f936babe..3bf4fa7838 100644 --- a/projects/rccl/src/graph/xml.cc +++ b/projects/rccl/src/graph/xml.cc @@ -492,7 +492,10 @@ ncclResult_t ncclTopoGetXmlFromSys(struct ncclXmlNode* pciNode, struct ncclXml* char portSpeedStr[MAX_STR_LEN]; float portSpeed; NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_speed", portSpeedStr)); - sscanf(portSpeedStr, "%f GT/s", &portSpeed); + if (portSpeedStr[0]) + sscanf(portSpeedStr, "%f GT/s", &portSpeed); + else + portSpeed = deviceSpeed; NCCLCHECK(xmlSetAttr(pciNode, "link_speed", portSpeed < deviceSpeed ? portSpeedStr : deviceSpeedStr)); } NCCLCHECK(xmlGetAttrIndex(pciNode, "link_width", &index)); @@ -502,7 +505,11 @@ ncclResult_t ncclTopoGetXmlFromSys(struct ncclXmlNode* pciNode, struct ncclXml* NCCLCHECK(ncclTopoGetStrFromSys(path, "max_link_width", strValue)); int deviceWidth = strtol(strValue, NULL, 0); NCCLCHECK(ncclTopoGetStrFromSys(path, "../max_link_width", strValue)); - int portWidth = strtol(strValue, NULL, 0); + int portWidth; + if (strValue[0]) + portWidth = strtol(strValue, NULL, 0); + else + portWidth = deviceWidth; NCCLCHECK(xmlSetAttrInt(pciNode, "link_width", std::min(deviceWidth,portWidth))); } struct ncclXmlNode* parent = pciNode->parent; From b68ff1ebba13b584b782010e6ce98d248c2d1ecf Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Tue, 17 Nov 2020 14:55:06 -0800 Subject: [PATCH 08/17] Add Rome model and improve search (#305) [ROCm/rccl commit: 975b14dffa2b6ccf093c316b7383c6752fe5ac72] --- projects/rccl/src/graph/rome_models.h | 24 +++++ projects/rccl/src/graph/search.cc | 4 +- projects/rccl/tools/scripts/topo_val.sh | 2 +- .../topo_expl/models/topo_8p_rome_n4_1.xml | 93 +++++++++++++++++++ projects/rccl/tools/topo_expl/topo_expl.cpp | 2 + 5 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml diff --git a/projects/rccl/src/graph/rome_models.h b/projects/rccl/src/graph/rome_models.h index 286d860842..20d08fe5b5 100755 --- a/projects/rccl/src/graph/rome_models.h +++ b/projects/rccl/src/graph/rome_models.h @@ -213,6 +213,28 @@ static struct rcclRomeModel rome_model_44 = { .ringBase = "5 4 7 6 2 1 3 0|5 6 7 4 1 0 2 3", }; +static struct rcclRomeModel rome_model_45 = { + .nGpus = 8, .nCpus = 7, .nNics = 0, .nLinks = 3, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .nicIds = { }, + .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, }, + .nicNuma = { }, + .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, }, + .pattern = "00102010002010", + .ringBase = "0 1 2 3 4 5 6 7|0 2 5 7 4 6 1 3|0 3 1 6 4 7 5 2|0 7 6 5 4 3 2 1", +}; + +static struct rcclRomeModel rome_model_46 = { + .nGpus = 8, .nCpus = 7, .nNics = 1, .nLinks = 3, + .gpuIds = { 0x43000, 0x23000, 0x26000, 0x3000, 0xc3000, 0xc6000, 0xa3000, 0x83000, }, + .nicIds = { 0xe1000, }, + .gpuNuma = { 1, 2, 2, 3, 5, 5, 6, 7, }, + .nicNuma = { 4, }, + .connMatrix = { 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, }, + .pattern = "00102010012010", + .ringBase = "6 5 7 4 1 2 3 0|7 4 6 5 1 0 3 2", +}; + static struct rcclRomeModel romeTopoModels[] = { rome_model_22, rome_model_25, @@ -230,4 +252,6 @@ static struct rcclRomeModel romeTopoModels[] = { rome_model_40, rome_model_42, rome_model_44, + rome_model_45, + rome_model_46, }; diff --git a/projects/rccl/src/graph/search.cc b/projects/rccl/src/graph/search.cc index 0a887b6a1e..a2c0d0b710 100644 --- a/projects/rccl/src/graph/search.cc +++ b/projects/rccl/src/graph/search.cc @@ -973,8 +973,10 @@ static bool permuteGpuIds(int *g, int n, int last, struct rcclRomeModel* ref, st if (i < ref->nGpus) return false; // match XGMI connection for (i = 0; i < ref->nGpus; i++) { - for (j = 0; j < ref->nGpus; j++) + for (j = 0; j < ref->nGpus; j++) { if (ref->connMatrix[i*ref->nGpus+j] != topo->connMatrix[g[i]*ref->nGpus+g[j]]) break; + if ((ref->gpuIds[i]-ref->gpuIds[j])*(topo->gpuIds[g[i]]-topo->gpuIds[g[j]]) < 0) break; + } if (j < ref->nGpus) break; } if (i < ref->nGpus) return false; diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh index fcc07c3876..9afe237aa6 100755 --- a/projects/rccl/tools/scripts/topo_val.sh +++ b/projects/rccl/tools/scripts/topo_val.sh @@ -21,7 +21,7 @@ DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -for i in {0..44} +for i in {0..46} do $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log" $DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log" diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml new file mode 100644 index 0000000000..b4af22a14f --- /dev/null +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_n4_1.xml @@ -0,0 +1,93 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp index ff283cc5a1..48709bb14a 100644 --- a/projects/rccl/tools/topo_expl/topo_expl.cpp +++ b/projects/rccl/tools/topo_expl/topo_expl.cpp @@ -114,6 +114,8 @@ NodeModelDesc model_descs[] = { {4, "topo_4p3l_n4.xml", "4 nodes 8 gfx908 Rome NPS=4"}, {1, "topo_4p3l_n2_1.xml", "single node 8 gfx908 Rome"}, {4, "topo_4p3l_n2_1.xml", "4 nodes 8 gfx908 Rome"}, + {1, "topo_8p_rome_n4_1.xml", "single node 8 gfx908 Rome NPS=4"}, + {4, "topo_8p_rome_n4_1.xml", "4 nodes node 8 gfx908 Rome NPS=4"}, }; int main(int argc,char* argv[]) From 9e83df4ad3c60297518bb77f0f114e53fec38aff Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Fri, 4 Dec 2020 13:55:56 -0800 Subject: [PATCH 09/17] Adding backward compatibility for target-id syntax for AMDGPU_TARGETS (#306) [ROCm/rccl commit: 882d52ad7ec57e8055f71d978519c3352667fd5c] --- projects/rccl/CMakeLists.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/projects/rccl/CMakeLists.txt b/projects/rccl/CMakeLists.txt index cc6ddcb929..5d7bf15bb3 100644 --- a/projects/rccl/CMakeLists.txt +++ b/projects/rccl/CMakeLists.txt @@ -11,7 +11,21 @@ set(CMAKE_INSTALL_PREFIX "/opt/rocm" CACHE PATH "") project(rccl CXX) -set(AMDGPU_TARGETS gfx803;gfx900;gfx906:xnack-;gfx908:xnack- CACHE STRING "List of specific machine types for library to target") +# Detect compiler support for target ID +if( CMAKE_CXX_COMPILER MATCHES ".*/hipcc$" ) + execute_process(COMMAND ${CMAKE_CXX_COMPILER} "--help" + OUTPUT_VARIABLE CXX_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE) + string(REGEX MATCH ".mcode\-object\-version" TARGET_ID_SUPPORT ${CXX_OUTPUT}) +endif() + +# Use target ID syntax if supported for AMDGPU_TARGETS +if(TARGET_ID_SUPPORT) + set(AMDGPU_TARGETS gfx803;gfx900;gfx906:xnack-;gfx908:xnack- CACHE STRING "List of specific machine types for library to target") +else() + set(AMDGPU_TARGETS gfx803;gfx900;gfx906;gfx908 CACHE STRING "List of specific machine types for library to target") +endif() option(BUILD_TESTS "Build test programs" OFF) option(INSTALL_DEPENDENCIES "Force install dependencies" OFF) From 9b48f92d724faec0886a731c9494825b1c8440b9 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Fri, 4 Dec 2020 14:57:13 -0700 Subject: [PATCH 10/17] [TransferBench] Support multiple of 4 byte sizes, changing default GPU timing mechanism (#307) * Changing default timing mechanism, adjusting CPU bandwidth calc, adding flag to use combined timing * Adding support for smaller transfers (byte size must be multiple of 4 instead of 128) [ROCm/rccl commit: b80ae551b1491a60e3833c870740ad3e6c237763] --- .../tools/TransferBench/TransferBench.cpp | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/projects/rccl/tools/TransferBench/TransferBench.cpp b/projects/rccl/tools/TransferBench/TransferBench.cpp index 732a405ee7..531589695d 100644 --- a/projects/rccl/tools/TransferBench/TransferBench.cpp +++ b/projects/rccl/tools/TransferBench/TransferBench.cpp @@ -54,6 +54,7 @@ int main(int argc, char **argv) bool useSingleSync = getenv("USE_SINGLE_SYNC"); // Perform synchronization only once after all iterations instead of per iteration bool useInteractive = getenv("USE_INTERACTIVE"); // Pause for user-input before starting transfer loop bool useSleep = getenv("USE_SLEEP"); // Adds a 100ms sleep after each synchronization + bool combineTiming = getenv("COMBINE_TIMING"); // Combines the timing with kernel launch bool reuseStreams = getenv("REUSE_STREAMS"); // Re-use streams instead of creating / destroying per test bool showAddr = getenv("SHOW_ADDR"); // Print out memory addresses for each Link bool outputToCsv = getenv("OUTPUT_TO_CSV"); // Output in CSV format @@ -67,9 +68,9 @@ int main(int argc, char **argv) std::vector valuesOfN; size_t const numBytesPerLink = argc > 2 ? atoll(argv[2]) : DEFAULT_BYTES_PER_LINK; - if (numBytesPerLink % 128) + if (numBytesPerLink % 4) { - printf("[ERROR] numBytesPerLink (%lu) must be a multiple of 128\n", numBytesPerLink); + printf("[ERROR] numBytesPerLink (%lu) must be a multiple of 4\n", numBytesPerLink); exit(1); } @@ -130,6 +131,9 @@ int main(int argc, char **argv) printf("%-20s %8s: %s\n", "USE_SLEEP", useSleep ? "(set)" : "(unset)", useSleep ? "Add sleep after each sync" : "No sleep per sync"); + printf("%-20s %8s: %s\n", + "COMBINE_TIMING", combineTiming ? "(set)" : "(unset)", + combineTiming ? "Using combined timing+launch" : "Using separate timing / launch"); printf("%-20s %8s: %s\n", "REUSE_STREAMS", reuseStreams ? "(set)" : "(unset)", reuseStreams ? "Re-using streams per topology" : "Creating/destroying streams per topology"); @@ -200,7 +204,6 @@ int main(int argc, char **argv) hipStream_t streams[numLinks]; // hipStream to use per Link hipEvent_t startEvents[numLinks]; // Start event per Link hipEvent_t stopEvents[numLinks]; // Stop event per Link - hipEvent_t dummyEvents[numLinks]; // Dummy event per Link std::vector cpuBlockParams[numLinks]; // CPU copy of block parameters BlockParam* gpuBlockParams[numLinks]; // GPU copy of block parameters @@ -251,7 +254,6 @@ int main(int argc, char **argv) HIP_CALL(hipSetDevice(exeIndex)); HIP_CALL(hipEventCreate(&startEvents[i])); HIP_CALL(hipEventCreate(&stopEvents[i])); - HIP_CALL(hipEventCreate(&dummyEvents[i])); HIP_CALL(hipMalloc((void**)&gpuBlockParams[i], sizeof(BlockParam) * numLinks)); if (reuseStreams) { @@ -278,21 +280,23 @@ int main(int argc, char **argv) // Count # of links / total blocks each GPU will be working on linkCount[exeIndex]++; + // Each block needs to know src/dst pointers and how many elements to transfer - // Figure out the sub-array each block does for this link - // NOTE: Have each sub-array to work on multiple of 32-floats (128-bytes), - // but divide as evenly as possible - // NOTE: N is always a multiple of 32 - int blocksWithExtra = (N / 32) % links[i].numBlocksToUse; - int perBlockBaseN = (N / 32) / links[i].numBlocksToUse * 32; + // Figure out the sub-array each block does for this Link + // - Partition N as evenly as posible, but try to keep blocks as multiples of 32, + // except the very last one, for alignment reasons + size_t assigned = 0; + int maxNumBlocksToUse = std::min((N + 31) / 32, (size_t)links[i].numBlocksToUse); for (int j = 0; j < links[i].numBlocksToUse; j++) { BlockParam param; - param.N = perBlockBaseN + ((j < blocksWithExtra) ? 32 : 0); - param.src = linkSrcMem[i] + ((j * perBlockBaseN) + ((j < blocksWithExtra) ? - j : blocksWithExtra) * 32) + initOffset; - param.dst = linkDstMem[i] + ((j * perBlockBaseN) + ((j < blocksWithExtra) ? - j : blocksWithExtra) * 32) + initOffset; + int blocksLeft = std::max(0, maxNumBlocksToUse - j); + size_t leftover = N - assigned; + size_t roundedN = (leftover + 31) / 32; + param.N = blocksLeft ? std::min(leftover, ((roundedN / blocksLeft) * 32)) : 0; + param.src = linkSrcMem[i] + assigned + initOffset; + param.dst = linkDstMem[i] + assigned + initOffset; + assigned += param.N; cpuBlockParams[i].push_back(param); } @@ -345,13 +349,15 @@ int main(int argc, char **argv) } else { + if (!combineTiming && recordStart) HIP_CALL(hipEventRecord(startEvents[i], streams[i])); hipExtLaunchKernelGGL(useMemset ? MemsetKernel : CopyKernel, dim3(links[i].numBlocksToUse, 1, 1), dim3(BLOCKSIZE, 1, 1), 0, streams[i], - recordStart ? startEvents[i] : NULL, - recordStop ? stopEvents[i] : NULL, + (combineTiming && recordStart) ? startEvents[i] : NULL, + (combineTiming && recordStop) ? stopEvents[i] : NULL, 0, gpuBlockParams[i]); + if (!combineTiming & recordStop) HIP_CALL(hipEventRecord(stopEvents[i], streams[i])); } } @@ -381,7 +387,6 @@ int main(int argc, char **argv) for (int i = 0; i < numLinks; i++) { HIP_CALL(hipSetDevice(links[i].exeIndex)); - HIP_CALL(hipEventSynchronize(startEvents[i])); HIP_CALL(hipEventSynchronize(stopEvents[i])); float gpuDeltaMsec; HIP_CALL(hipEventElapsedTime(&gpuDeltaMsec, startEvents[i], stopEvents[i])); @@ -404,12 +409,11 @@ int main(int argc, char **argv) // Report timings totalCpuTime = totalCpuTime / (1.0 * numIterations) * 1000; - double totalBandwidthGbs = 0.0; + double totalBandwidthGbs = (numLinks * N * sizeof(float) / 1.0E6) / totalCpuTime; for (int i = 0; i < numLinks; i++) { double linkDurationMsec = totalGpuTime[i] / (1.0 * numIterations); double linkBandwidthGbs = (N * sizeof(float) / 1.0E9) / linkDurationMsec * 1000.0f; - totalBandwidthGbs += linkBandwidthGbs; if (!outputToCsv) { printf(" Link %02d: %c%02d -> [GPU %02d:%02d] -> %c%02d | %9.3f GB/s | %8.3f ms | %9s |", @@ -443,7 +447,7 @@ int main(int argc, char **argv) // Display aggregate statistics if (!outputToCsv) { - printf(" Aggregate Bandwidth | %9.3f GB/s | %8.3f ms |\n", totalBandwidthGbs, totalCpuTime); + printf(" Aggregate Bandwidth (CPU timed) | %9.3f GB/s | %8.3f ms |\n", totalBandwidthGbs, totalCpuTime); } else { @@ -488,7 +492,7 @@ void DisplayUsage(char const* cmdName) printf(" configFile: File containing Links to execute (see below for format)\n"); printf(" N : (Optional) Number of bytes to transfer per link.\n"); - printf(" If not specified, defaults to %lu bytes. Must be a multiple of 128 bytes\n", DEFAULT_BYTES_PER_LINK); + printf(" If not specified, defaults to %lu bytes. Must be a multiple of 4 bytes\n", DEFAULT_BYTES_PER_LINK); printf(" If 0 is specified, a range of Ns will be benchmarked\n"); printf(" If a negative number is specified, a configFile gets generated with this number as default number of CUs per link\n"); printf("\n"); @@ -538,6 +542,7 @@ void DisplayUsage(char const* cmdName) printf(" USE_SINGLE_SYNC - Perform synchronization only once after all iterations instead of per iteration\n"); printf(" USE_INTERACTIVE - Pause for user-input before starting transfer loop\n"); printf(" USE_SLEEP - Adds a 100ms sleep after each synchronization\n"); + printf(" COMBINE_TIMING - Combines timing with launch (potentially lower timing overhead)\n"); printf(" REUSE_STREAMS - Re-use streams instead of creating / destroying per test\n"); printf(" SHOW_ADDR - Print out memory addresses for each Link\n"); printf(" OUTPUT_TO_CSV - Outputs to CSV format if set\n"); From 5155abb25020a32e26924f7581284714d33ba148 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Fri, 11 Dec 2020 10:21:14 -0700 Subject: [PATCH 11/17] [TransferBench] Adding ability to perform CPU-executed copies, various upgrades (#309) * Adding CPU based execution, fixing typos, adding Fine-grained mem * Exposing sampling factor when generating range of data sizes * Refactoring how Links are launched, now once per thread * Documentation updates [ROCm/rccl commit: ae0c4092c7c0149391dfc30e6dcbc3ea68726bfa] --- projects/rccl/tools/TransferBench/EnvVars.hpp | 137 +++ projects/rccl/tools/TransferBench/Makefile | 2 +- .../tools/TransferBench/TransferBench.cpp | 988 ++++++++++-------- .../tools/TransferBench/TransferBench.hpp | 104 +- projects/rccl/tools/TransferBench/example.cfg | 68 +- 5 files changed, 802 insertions(+), 497 deletions(-) create mode 100644 projects/rccl/tools/TransferBench/EnvVars.hpp diff --git a/projects/rccl/tools/TransferBench/EnvVars.hpp b/projects/rccl/tools/TransferBench/EnvVars.hpp new file mode 100644 index 0000000000..258f0ee33c --- /dev/null +++ b/projects/rccl/tools/TransferBench/EnvVars.hpp @@ -0,0 +1,137 @@ +#ifndef ENVVARS_HPP +#define ENVVARS_HPP + +// This class manages environment variable that affect TransferBench +class EnvVars +{ +public: + // Default configuration values + int const DEFAULT_NUM_WARMUPS = 3; + int const DEFAULT_NUM_ITERATIONS = 10; + int const DEFAULT_SAMPLING_FACTOR = 1; + int const DEFAULT_NUM_CPU_PER_LINK = 4; + + // Environment variables + int useHipCall; // Use hipMemcpy/hipMemset instead of custom shader kernels + int useMemset; // Perform a memset instead of a copy (ignores source memory) + int useSingleSync; // Perform synchronization only once after all iterations instead of per iteration + int useInteractive; // Pause for user-input before starting transfer loop + int useSleep; // Adds a 100ms sleep after each synchronization + int combineTiming; // Combines the timing with kernel launch + int showAddr; // Print out memory addresses for each Link + int outputToCsv; // Output in CSV format + int byteOffset; // Byte-offset for memory allocations + int numWarmups; // Number of un-timed warmup iterations to perform + int numIterations; // Number of timed iterations to perform + int samplingFactor; // Affects how many different values of N are generated (when N set to 0) + int numCpuPerLink; // Number of CPU child threads to use per CPU link + + // Constructor that collects values + EnvVars() + { + useHipCall = GetEnvVar("USE_HIP_CALL" , 0); + useMemset = GetEnvVar("USE_MEMSET" , 0); + useSingleSync = GetEnvVar("USE_SINGLE_SYNC" , 0); + useInteractive = GetEnvVar("USE_INTERACTIVE" , 0); + combineTiming = GetEnvVar("COMBINE_TIMING" , 0); + showAddr = GetEnvVar("SHOW_ADDR" , 0); + outputToCsv = GetEnvVar("OUTPUT_TO_CSV" , 0); + byteOffset = GetEnvVar("BYTE_OFFSET" , 0); + numWarmups = GetEnvVar("NUM_WARMUPS" , DEFAULT_NUM_WARMUPS); + numIterations = GetEnvVar("NUM_ITERATIONS" , DEFAULT_NUM_ITERATIONS); + samplingFactor = GetEnvVar("SAMPLING_FACTOR" , DEFAULT_SAMPLING_FACTOR); + numCpuPerLink = GetEnvVar("NUM_CPU_PER_LINK" , DEFAULT_NUM_CPU_PER_LINK); + + // Perform some basic validation + if (byteOffset % sizeof(float)) + { + printf("[ERROR] BYTE_OFFSET must be set to multiple of %lu\n", sizeof(float)); + exit(1); + } + if (numWarmups < 0) + { + printf("[ERROR] NUM_WARMUPS must be set to a non-negative number\n"); + exit(1); + } + if (numIterations <= 0) + { + printf("[ERROR] NUM_ITERATIONS must be set to a positive number\n"); + exit(1); + } + if (samplingFactor < 1) + { + printf("[ERROR] SAMPLING_FACTOR must be greater or equal to 1\n"); + exit(1); + } + if (numCpuPerLink < 1) + { + printf("[ERROR] NUM_CPU_PER_LINK must be greater or equal to 1\n"); + exit(1); + } + } + + // Display info on the env vars that can be used + static void DisplayUsage() + { + printf("Environment variables:\n"); + printf("======================\n"); + printf(" USE_HIP_CALL - Use hipMemcpy/hipMemset instead of custom shader kernels for GPU-executed copies\n"); + printf(" USE_MEMSET - Perform a memset instead of a copy (ignores source memory)\n"); + printf(" USE_SINGLE_SYNC - Perform synchronization only once after all iterations instead of per iteration\n"); + printf(" USE_INTERACTIVE - Pause for user-input before starting transfer loop\n"); + printf(" COMBINE_TIMING - Combines timing with launch (potentially lower timing overhead)\n"); + printf(" SHOW_ADDR - Print out memory addresses for each Link\n"); + printf(" OUTPUT_TO_CSV - Outputs to CSV format if set\n"); + printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4. Defaults to 0\n"); + printf(" NUM_WARMUPS=W - Perform W untimed warmup iteration(s) per test\n"); + printf(" NUM_ITERATIONS=I - Perform I timed iteration(s) per test\n"); + printf(" SAMPLING_FACTOR=F - Add F samples (when possible) between powers of 2 when auto-generating data sizes\n"); + printf(" NUM_CPU_PER_LINK=C - Use C threads per Link for CPU-executed copies\n"); + } + + // Display env var settings + void DisplayEnvVars() const + { + if (!outputToCsv) + { + printf("Run configuration\n"); + printf("=====================================================\n"); + printf("%-20s = %12d : Using %s for GPU-executed copies\n", "USE_HIP_CALL", useHipCall, + useHipCall ? "HIP functions" : "custom kernels"); + printf("%-20s = %12d : Performing %s\n", "USE_MEMSET", useMemset, + useMemset ? "memset" : "memcopy"); + if (useHipCall && !useMemset) + { + char* env = getenv("HSA_ENABLE_SDMA"); + printf("%-20s = %12s : %s\n", "HSA_ENABLE_SDMA", env, + (env && !strcmp(env, "0")) ? "Using blit kernels for hipMemcpy" : "Using DMA copy engines"); + } + printf("%-20s = %12d : %s\n", "USE_SINGLE_SYNC", useSingleSync, + useSingleSync ? "Synchronizing only once, after all iterations" : "Synchronizing per iteration"); + printf("%-20s = %12d : Running in %s mode\n", "USE_INTERACTIVE", useInteractive, + useInteractive ? "interactive" : "non-interactive"); + printf("%-20s = %12d : %s\n", "COMBINE_TIMING", combineTiming, + combineTiming ? "Using combined timing+launch" : "Using separate timing / launch"); + printf("%-20s = %12d : %s\n", "SHOW_ADDR", showAddr, + showAddr ? "Displaying src/dst mem addresses" : "Not displaying src/dst mem addresses"); + printf("%-20s = %12d : Output to %s\n", "OUTPUT_TO_CSV", outputToCsv, + outputToCsv ? "CSV" : "console"); + printf("%-20s = %12d : Using byte offset of %d\n", "BYTE_OFFSET", byteOffset, byteOffset); + printf("%-20s = %12d : Running %d warmup iteration(s) per topology\n", "NUM_WARMUPS", numWarmups, numWarmups); + printf("%-20s = %12d : Running %d timed iteration(s) per topology\n", "NUM_ITERATIONS", numIterations, numIterations); + printf("%-20s = %12d : Using %d CPU thread(s) per CPU-based-copy Link\n", "NUM_CPU_PER_LINK", numCpuPerLink, numCpuPerLink); + printf("\n"); + } + }; + +private: + // Helper function that gets parses environment variable or sets to default value + int GetEnvVar(std::string const varname, int defaultValue) + { + if (getenv(varname.c_str())) + return atoi(getenv(varname.c_str())); + return defaultValue; + } +}; + +#endif diff --git a/projects/rccl/tools/TransferBench/Makefile b/projects/rccl/tools/TransferBench/Makefile index af0b4597b7..c51bd47b4c 100644 --- a/projects/rccl/tools/TransferBench/Makefile +++ b/projects/rccl/tools/TransferBench/Makefile @@ -6,7 +6,7 @@ endif HIPCC=$(HIP_PATH)/bin/hipcc EXE=TransferBench -CXXFLAGS = -O3 -I../../src/include -I. +CXXFLAGS = -O3 -I../../src/include -I. -lnuma all: $(EXE) diff --git a/projects/rccl/tools/TransferBench/TransferBench.cpp b/projects/rccl/tools/TransferBench/TransferBench.cpp index 531589695d..f6d90fdb29 100644 --- a/projects/rccl/tools/TransferBench/TransferBench.cpp +++ b/projects/rccl/tools/TransferBench/TransferBench.cpp @@ -24,11 +24,13 @@ THE SOFTWARE. // on the same node #include "TransferBench.hpp" +#include +#include +#include +#include // Simple configuration parameters -size_t const DEFAULT_BYTES_PER_LINK = (1<<26); -int const DEFAULT_NUM_WARMUPS = 3; -int const DEFAULT_NUM_ITERATIONS = 10; +size_t const DEFAULT_BYTES_PER_LINK = (1<<26); // Amount of data transferred per Link int main(int argc, char **argv) { @@ -41,130 +43,13 @@ int main(int argc, char **argv) } // If a negative value is listed for N, generate a comprehensive config file for this node - if (argc > 2 && atoi(argv[2]) < 0) + if (argc > 2 && atoll(argv[2]) < 0) { GenerateConfigFile(argv[1], -1*atoi(argv[2])); exit(0); } - // Collect environment variables / display current run configuration - bool useHipCall = getenv("USE_HIP_CALL"); // Use hipMemcpy/hipMemset instead of custom shader kernels - bool useMemset = getenv("USE_MEMSET"); // Perform a memset instead of a copy (ignores source memory) - bool useFineGrainMem = getenv("USE_FINEGRAIN_MEM"); // Allocate fine-grained GPU memory instead of coarse-grained GPU memory - bool useSingleSync = getenv("USE_SINGLE_SYNC"); // Perform synchronization only once after all iterations instead of per iteration - bool useInteractive = getenv("USE_INTERACTIVE"); // Pause for user-input before starting transfer loop - bool useSleep = getenv("USE_SLEEP"); // Adds a 100ms sleep after each synchronization - bool combineTiming = getenv("COMBINE_TIMING"); // Combines the timing with kernel launch - bool reuseStreams = getenv("REUSE_STREAMS"); // Re-use streams instead of creating / destroying per test - bool showAddr = getenv("SHOW_ADDR"); // Print out memory addresses for each Link - bool outputToCsv = getenv("OUTPUT_TO_CSV"); // Output in CSV format - int byteOffset = getenv("BYTE_OFFSET") ? atoi(getenv("BYTE_OFFSET")) : 0; // Byte-offset for memory allocations - int numWarmups = getenv("NUM_WARMUPS") ? atoi(getenv("NUM_WARMUPS")) : DEFAULT_NUM_WARMUPS; - int numIterations = getenv("NUM_ITERATIONS") ? atoi(getenv("NUM_ITERATIONS")) : DEFAULT_NUM_ITERATIONS; - - // Determine number of bytes to run per link - // If a non-zero number of bytes is specified, use it - // Otherwise generate array of bytes values to execute over - std::vector valuesOfN; - size_t const numBytesPerLink = argc > 2 ? atoll(argv[2]) : DEFAULT_BYTES_PER_LINK; - - if (numBytesPerLink % 4) - { - printf("[ERROR] numBytesPerLink (%lu) must be a multiple of 4\n", numBytesPerLink); - exit(1); - } - - if (numBytesPerLink != 0) - { - size_t N = numBytesPerLink / sizeof(float); - valuesOfN.push_back(N); - } - else - { - for (int N = 256; N <= (1<<27); N *= 2) - { - int decimationFactor = 1; // This can be modified to increase number of samples between powers of two - int delta = std::max(32, N / decimationFactor); - int curr = N; - while (curr < N * 2) - { - valuesOfN.push_back(curr); - curr += delta; - } - } - } - - if (byteOffset % 4) - { - printf("[ERROR] byteOffset must be a multiple of 4\n"); - exit(1); - } - int initOffset = byteOffset / sizeof(float); - - char *env; - if (!outputToCsv) - { - printf("Run configuration\n"); - printf("=====================================================\n"); - printf("%-20s %8s: Using %s\n", - "USE_HIP_CALL", useHipCall ? "(set)" : "(unset)", - useHipCall ? "HIP functions" : "custom kernels"); - printf("%-20s %8s: Performing %s\n", - "USE_MEMSET", useMemset ? "(set)" : "(unset)", - useMemset ? "memset" : "memcopy"); - if (useHipCall && !useMemset) - { - env = getenv("HSA_ENABLE_SDMA"); - printf("%-20s %8s: %s\n", - "HSA_ENABLE_SDMA", env ? env : "(unset)", - (env && !strcmp(env, "0")) ? "Using blit kernels for hipMemcpy" : "Using DMA copy engines"); - } - printf("%-20s %8s: GPU destination memory type: %s-grained\n", - "USE_FINEGRAIN_MEM", useFineGrainMem ? "(set)" : "(unset)", - useFineGrainMem ? "fine" : "coarse"); - printf("%-20s %8s: %s\n", - "USE_SINGLE_SYNC", useSingleSync ? "(set)" : "(unset)", - useSingleSync ? "Synchronizing only once, after all iterations" : "Synchronizing per iteration"); - printf("%-20s %8s: Running in %s mode\n", - "USE_INTERACTIVE", useInteractive ? "(set)" : "(unset)", - useInteractive ? "interactive" : "non-interactive"); - printf("%-20s %8s: %s\n", - "USE_SLEEP", useSleep ? "(set)" : "(unset)", - useSleep ? "Add sleep after each sync" : "No sleep per sync"); - printf("%-20s %8s: %s\n", - "COMBINE_TIMING", combineTiming ? "(set)" : "(unset)", - combineTiming ? "Using combined timing+launch" : "Using separate timing / launch"); - printf("%-20s %8s: %s\n", - "REUSE_STREAMS", reuseStreams ? "(set)" : "(unset)", - reuseStreams ? "Re-using streams per topology" : "Creating/destroying streams per topology"); - printf("%-20s %8s: %s\n", - "SHOW_ADDR", showAddr ? "(set)" : "(unset)", - showAddr ? "Displaying src/dst mem addresses" : "Not displaying src/dst mem addresses"); - env = getenv("OUTPUT_TO_CSV"); - printf("%-20s %8s: Output to csv\n", - "OUTPUT_TO_CSV", env ? env : "(unset)"); - env = getenv("BYTE_OFFSET"); - printf("%-20s %8s: Using byte offset of %d\n", - "BYTE_OFFSET", env ? env : "(unset)", byteOffset); - env = getenv("NUM_WARMUPS"); - printf("%-20s %8s: Running %d warmup iteration(s) per topology\n", - "NUM_WARMUPS", env ? env : "(unset)", numWarmups); - env = getenv("NUM_ITERATIONS"); - printf("%-20s %8s: Running %d timed iteration(s) per topology\n", - "NUM_ITERATIONS", env ? env : "(unset)", numIterations); - printf("\n"); - } - - // Collect the number of available CPUs/GPUs on this machine - int numGpuDevices; - HIP_CALL(hipGetDeviceCount(&numGpuDevices)); - if (numGpuDevices < 1) - { - printf("[ERROR] No GPU devices found\n"); - exit(1); - } - - // Read configuration file + // Check that Link configuration file can be opened FILE* fp = fopen(argv[1], "r"); if (!fp) { @@ -172,148 +57,177 @@ int main(int argc, char **argv) exit(1); } - // Track links that get used - std::map, int> linkMap; - std::vector> streamCache(numGpuDevices); - - // Print CSV header - if (outputToCsv) + // Check for NUMA library support + if (numa_available() == -1) { - printf("Test,NumBytes,ExeGpu,SrcMem,DstMem,BW(GB/s),Time(ms),LinkDesc,SrcAddr,DstAddr,numWarmups,numIters,useHipCall,useMemSet,useFineGrain,useSingleSync,resuseStreams\n"); + printf("[ERROR] NUMA library not supported. Check to see if libnuma has been installed on this system\n"); + exit(1); } - // Loop over each line in the configuration file + // Collect environment variables / display current run configuration + EnvVars ev; + ev.DisplayEnvVars(); + + // Determine number of bytes to run per Link + // If a non-zero number of bytes is specified, use it + // Otherwise generate array of bytes values to execute over + std::vector valuesOfN; + size_t const numBytesPerLink = argc > 2 ? atoll(argv[2]) : DEFAULT_BYTES_PER_LINK; + PopulateTestSizes(numBytesPerLink, ev.samplingFactor, valuesOfN); + + // Find the largest N to be used - memory will only be allocated once per link config + size_t maxN = valuesOfN[0]; + for (auto N : valuesOfN) + maxN = std::max(maxN, N); + + int const initOffset = ev.byteOffset / sizeof(float); + std::stack threads; + + // Collect the number of available CPUs/GPUs on this machine + int numGpuDevices; + HIP_CALL(hipGetDeviceCount(&numGpuDevices)); + int const numCpuDevices = numa_num_configured_nodes(); + + // Track links that get used + std::set> peerAccessTracker; + + // Print CSV header + if (ev.outputToCsv) + { + printf("Test,NumBytes,Executor,SrcMem,DstMem,CUs,BW(GB/s),Time(ms),LinkDesc,SrcAddr,DstAddr,ByteOffset,numWarmups,numIters,useHipCall,useMemSet,useSingleSync,combinedTiming\n"); + } + + // Loop over each line in the Link configuration file int testNum = 0; char line[2048]; while(fgets(line, 2048, fp)) { // Parse links from configuration file std::vector links; - ParseLinks(line, links); + ParseLinks(line, numCpuDevices, numGpuDevices, links); int const numLinks = links.size(); if (numLinks == 0) continue; testNum++; + // Prepare link + for (int i = 0; i < numLinks; i++) + { + // Get some aliases to link variables + MemType const& exeMemType = links[i].exeMemType; + int const& exeIndex = links[i].exeIndex; + MemType const& srcMemType = links[i].srcMemType; + MemType const& dstMemType = links[i].dstMemType; + int const& srcIndex = links[i].srcIndex; + int const& dstIndex = links[i].dstIndex; + int const& blocksToUse = links[i].numBlocksToUse; + + // Enable peer-to-peer access if necessary (can only be called once per unique pair) + if (exeMemType == MEM_GPU) + { + // Ensure executing GPU can access source memory + if ((srcMemType == MEM_GPU || srcMemType == MEM_GPU_FINE) && srcIndex != exeIndex) + { + auto exeSrcPair = std::make_pair(exeIndex, srcIndex); + if (!peerAccessTracker.count(exeSrcPair)) + { + EnablePeerAccess(exeIndex, srcIndex); + peerAccessTracker.insert(exeSrcPair); + } + } + + // Ensure executing GPU can access destination memory + if ((dstMemType == MEM_GPU || dstMemType == MEM_GPU_FINE) && dstIndex != exeIndex) + { + auto exeDstPair = std::make_pair(exeIndex, dstIndex); + if (!peerAccessTracker.count(exeDstPair)) + { + EnablePeerAccess(exeIndex, dstIndex); + peerAccessTracker.insert(exeDstPair); + } + } + } + + // Allocate (maximum) source / destination memory based on type / device index + AllocateMemory(srcMemType, srcIndex, maxN * sizeof(float) + ev.byteOffset, &links[i].srcMem); + AllocateMemory(dstMemType, dstIndex, maxN * sizeof(float) + ev.byteOffset, &links[i].dstMem); + + // Prepare execution agent + if (exeMemType == MEM_GPU) + { + HIP_CALL(hipSetDevice(exeIndex)); + HIP_CALL(hipEventCreate(&links[i].startEvent)); + HIP_CALL(hipEventCreate(&links[i].stopEvent)); + HIP_CALL(hipMalloc((void**)&links[i].blockParam, sizeof(BlockParam) * blocksToUse)); + HIP_CALL(hipStreamCreate(&links[i].stream)); + } + else if (exeMemType == MEM_CPU) + { + links[i].blockParam = (BlockParam*)malloc(ev.numCpuPerLink * sizeof(BlockParam)); + } + } + // Loop over all the different number of bytes to use per Link for (auto N : valuesOfN) { - if (!outputToCsv) printf("Test %d: [%lu bytes]\n", testNum, N * sizeof(float)); - float* linkSrcMem[numLinks]; // Source memory per Link - float* linkDstMem[numLinks]; // Destination memory per Link - hipStream_t streams[numLinks]; // hipStream to use per Link - hipEvent_t startEvents[numLinks]; // Start event per Link - hipEvent_t stopEvents[numLinks]; // Stop event per Link - std::vector cpuBlockParams[numLinks]; // CPU copy of block parameters - BlockParam* gpuBlockParams[numLinks]; // GPU copy of block parameters - - // Clear counters - int linkCount[numGpuDevices]; - for (int i = 0; i < numGpuDevices; i++) - linkCount[i] = 0; + if (!ev.outputToCsv) printf("Test %d: [%lu bytes]\n", testNum, N * sizeof(float)); + // Prepare links based on current N for (int i = 0; i < numLinks; i++) { - MemType srcMemType = links[i].srcMemType; - MemType dstMemType = links[i].dstMemType; - int exeIndex = links[i].exeIndex; - int srcIndex = links[i].srcIndex; - int dstIndex = links[i].dstIndex; - int blocksToUse = links[i].numBlocksToUse; - - // Check for valid src/dst indices - if ((srcIndex < 0 || srcIndex >= numGpuDevices) || - (dstIndex < 0 || dstIndex >= numGpuDevices) || - (exeIndex < 0 || exeIndex >= numGpuDevices)) - { - printf("[ERROR] Invalid link %d:(%c%d->%c%d) GPU index must be between 0 and %d inclusively\n", - exeIndex, MemTypeStr[srcMemType], srcIndex, MemTypeStr[dstMemType], dstIndex, numGpuDevices-1); - exit(1); - } - - // Enable peer-to-peer access if this is the first time seeing this pair - if (srcMemType == MEM_GPU && dstMemType == MEM_GPU) - { - auto linkPair = std::make_pair(srcIndex, dstIndex); - linkMap[linkPair]++; - if (linkMap[linkPair] == 1 && srcIndex != dstIndex) - { - int canAccess; - HIP_CALL(hipDeviceCanAccessPeer(&canAccess, srcIndex, dstIndex)); - if (!canAccess) - { - printf("[ERROR] Unable to enable peer access between GPU devices %d and %d\n", srcIndex, dstIndex); - exit(1); - } - HIP_CALL(hipSetDevice(srcIndex)); - HIP_CALL(hipDeviceEnablePeerAccess(dstIndex, 0)); - } - } - - // Allocate hipEvents / hipStreams on executing GPU - HIP_CALL(hipSetDevice(exeIndex)); - HIP_CALL(hipEventCreate(&startEvents[i])); - HIP_CALL(hipEventCreate(&stopEvents[i])); - HIP_CALL(hipMalloc((void**)&gpuBlockParams[i], sizeof(BlockParam) * numLinks)); - if (reuseStreams) - { - // If re-using streams, create new stream, otherwise point to existing stream - if (streamCache[exeIndex].size() <= linkCount[exeIndex]) - { - streamCache[exeIndex].resize(linkCount[exeIndex] + 1); - HIP_CALL(hipStreamCreate(&streamCache[exeIndex][linkCount[exeIndex]])); - } - streams[i] = streamCache[exeIndex][linkCount[exeIndex]]; - } - else - { - HIP_CALL(hipStreamCreate(&streams[i])); - } - - // Allocate source / destination memory based on type / device index - AllocateMemory(srcMemType, srcIndex, N * sizeof(float) + byteOffset, useFineGrainMem, &linkSrcMem[i]); - AllocateMemory(dstMemType, dstIndex, N * sizeof(float) + byteOffset, useFineGrainMem, &linkDstMem[i]); - // Initialize source memory with patterned data - CheckOrFill(MODE_FILL, N, useMemset, useHipCall, linkSrcMem[i] + initOffset); - - // Count # of links / total blocks each GPU will be working on - linkCount[exeIndex]++; - + CheckOrFill(MODE_FILL, N, ev.useMemset, ev.useHipCall, links[i].srcMem + initOffset); // Each block needs to know src/dst pointers and how many elements to transfer // Figure out the sub-array each block does for this Link // - Partition N as evenly as posible, but try to keep blocks as multiples of 32, // except the very last one, for alignment reasons - size_t assigned = 0; - int maxNumBlocksToUse = std::min((N + 31) / 32, (size_t)links[i].numBlocksToUse); - for (int j = 0; j < links[i].numBlocksToUse; j++) + if (links[i].exeMemType == MEM_GPU) { - BlockParam param; - int blocksLeft = std::max(0, maxNumBlocksToUse - j); - size_t leftover = N - assigned; - size_t roundedN = (leftover + 31) / 32; - param.N = blocksLeft ? std::min(leftover, ((roundedN / blocksLeft) * 32)) : 0; - param.src = linkSrcMem[i] + assigned + initOffset; - param.dst = linkDstMem[i] + assigned + initOffset; - assigned += param.N; - cpuBlockParams[i].push_back(param); + size_t assigned = 0; + int maxNumBlocksToUse = std::min((N + 31) / 32, (size_t)links[i].numBlocksToUse); + for (int j = 0; j < links[i].numBlocksToUse; j++) + { + BlockParam param; + int blocksLeft = std::max(0, maxNumBlocksToUse - j); + size_t leftover = N - assigned; + size_t roundedN = (leftover + 31) / 32; + param.N = blocksLeft ? std::min(leftover, ((roundedN / blocksLeft) * 32)) : 0; + param.src = links[i].srcMem + assigned + initOffset; + param.dst = links[i].dstMem + assigned + initOffset; + assigned += param.N; + + HIP_CALL(hipMemcpy(&links[i].blockParam[j], ¶m, sizeof(BlockParam), hipMemcpyHostToDevice)); + } + } + else if (links[i].exeMemType == MEM_CPU) + { + // For CPU-based copy, divded based on the number of child threads + size_t assigned = 0; + int maxNumBlocksToUse = std::min((N + 31) / 32, (size_t)ev.numCpuPerLink); + for (int j = 0; j < ev.numCpuPerLink; j++) + { + int blocksLeft = std::max(0, maxNumBlocksToUse - j); + size_t leftover = N - assigned; + size_t roundedN = (leftover + 31) / 32; + links[i].blockParam[j].N = blocksLeft ? std::min(leftover, ((roundedN / blocksLeft) * 32)) : 0; + links[i].blockParam[j].src = links[i].srcMem + assigned + initOffset; + links[i].blockParam[j].dst = links[i].dstMem + assigned + initOffset; + assigned += links[i].blockParam[j].N; + } } - HIP_CALL(hipMemcpy(gpuBlockParams[i], cpuBlockParams[i].data(), - sizeof(BlockParam) * links[i].numBlocksToUse, hipMemcpyHostToDevice)); + // Initialize timing + links[i].totalTime = 0.0; } - // Launch kernels (warmup iterations are not counted) double totalCpuTime = 0; - double totalGpuTime[numLinks]; - for (int i = 0; i < numLinks; i++) totalGpuTime[i] = 0.0; - - for (int iteration = -numWarmups; iteration < numIterations; iteration++) + // Launch kernels (warmup iterations are not counted) + for (int iteration = -ev.numWarmups; iteration < ev.numIterations; iteration++) { // Pause before starting first timed iteration in interactive mode - if (useInteractive && iteration == 0) + if (ev.useInteractive && iteration == 0) { printf("Hit to continue: "); scanf("%*c"); @@ -323,80 +237,26 @@ int main(int argc, char **argv) // Start CPU timing for this iteration auto cpuStart = std::chrono::high_resolution_clock::now(); - // Enqueue all links + // Execute all links in parallel + for (int i = 0; i < numLinks; i++) + threads.push(std::thread(RunLink, std::ref(ev), N, iteration, std::ref(links[i]))); + + // Wait for all threads to finish for (int i = 0; i < numLinks; i++) { - HIP_CALL(hipSetDevice(links[i].exeIndex)); - - bool recordStart = (!useSingleSync || iteration == 0); - bool recordStop = (!useSingleSync || iteration == numIterations - 1); - - if (useHipCall) - { - // Record start event - if (recordStart) HIP_CALL(hipEventRecord(startEvents[i], streams[i])); - - // Execute hipMemset / hipMemcpy - if (useMemset) - HIP_CALL(hipMemsetAsync(linkDstMem[i] + initOffset, 42, N * sizeof(float), streams[i])); - else - HIP_CALL(hipMemcpyAsync(linkDstMem[i] + initOffset, - linkSrcMem[i] + initOffset, - N * sizeof(float), hipMemcpyDeviceToDevice, - streams[i])); - // Record stop event - if (recordStop) HIP_CALL(hipEventRecord(stopEvents[i], streams[i])); - } - else - { - if (!combineTiming && recordStart) HIP_CALL(hipEventRecord(startEvents[i], streams[i])); - hipExtLaunchKernelGGL(useMemset ? MemsetKernel : CopyKernel, - dim3(links[i].numBlocksToUse, 1, 1), - dim3(BLOCKSIZE, 1, 1), - 0, streams[i], - (combineTiming && recordStart) ? startEvents[i] : NULL, - (combineTiming && recordStop) ? stopEvents[i] : NULL, - 0, gpuBlockParams[i]); - if (!combineTiming & recordStop) HIP_CALL(hipEventRecord(stopEvents[i], streams[i])); - } - } - - // Synchronize per iteration, unless in single sync mode, in which case - // synchronize during last warmup / last actual iteration - if (!useSingleSync || iteration == -1 || iteration == numIterations - 1) - { - for (int i = 0; i < numLinks; i++) - { - HIP_CALL(hipSetDevice(links[i].exeIndex)); - hipStreamSynchronize(streams[i]); - } + threads.top().join(); + threads.pop(); } // Stop CPU timing for this iteration auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; double deltaSec = std::chrono::duration_cast>(cpuDelta).count(); - if (useSleep) usleep(100000); - if (iteration >= 0) - { - totalCpuTime += deltaSec; - - // Record GPU timing - if (!useSingleSync || iteration == numIterations - 1) - { - for (int i = 0; i < numLinks; i++) - { - HIP_CALL(hipSetDevice(links[i].exeIndex)); - HIP_CALL(hipEventSynchronize(stopEvents[i])); - float gpuDeltaMsec; - HIP_CALL(hipEventElapsedTime(&gpuDeltaMsec, startEvents[i], stopEvents[i])); - totalGpuTime[i] += gpuDeltaMsec; - } - } - } + if (iteration >= 0) totalCpuTime += deltaSec; } - if (useInteractive) + // Pause for interactive mode + if (ev.useInteractive) { printf("Transfers complete. Hit to continue: "); scanf("%*c"); @@ -405,89 +265,100 @@ int main(int argc, char **argv) // Validate that each link has transferred correctly for (int i = 0; i < numLinks; i++) - CheckOrFill(MODE_CHECK, N, useMemset, useHipCall, linkDstMem[i] + initOffset); + CheckOrFill(MODE_CHECK, N, ev.useMemset, ev.useHipCall, links[i].dstMem + initOffset); // Report timings - totalCpuTime = totalCpuTime / (1.0 * numIterations) * 1000; + totalCpuTime = totalCpuTime / (1.0 * ev.numIterations) * 1000; double totalBandwidthGbs = (numLinks * N * sizeof(float) / 1.0E6) / totalCpuTime; for (int i = 0; i < numLinks; i++) { - double linkDurationMsec = totalGpuTime[i] / (1.0 * numIterations); + double linkDurationMsec = links[i].totalTime / (1.0 * ev.numIterations); double linkBandwidthGbs = (N * sizeof(float) / 1.0E9) / linkDurationMsec * 1000.0f; - if (!outputToCsv) + if (!ev.outputToCsv) { - printf(" Link %02d: %c%02d -> [GPU %02d:%02d] -> %c%02d | %9.3f GB/s | %8.3f ms | %9s |", + printf(" Link %02d: %c%02d -> [%cPU %02d:%02d] -> %c%02d | %9.3f GB/s | %8.3f ms | %-16s", i + 1, MemTypeStr[links[i].srcMemType], links[i].srcIndex, - links[i].exeIndex, links[i].numBlocksToUse, + MemTypeStr[links[i].exeMemType], links[i].exeIndex, + links[i].exeMemType == MEM_CPU ? ev.numCpuPerLink : links[i].numBlocksToUse, MemTypeStr[links[i].dstMemType], links[i].dstIndex, linkBandwidthGbs, linkDurationMsec, GetLinkDesc(links[i]).c_str()); - if (showAddr) printf(" %16p | %16p |", linkSrcMem[i] + initOffset, linkDstMem[i] + initOffset); + if (ev.showAddr) printf(" %16p | %16p |", links[i].srcMem + initOffset, links[i].dstMem + initOffset); printf("\n"); } else { - printf("%d,%lu,%02d,%c%02d,%c%02d,%9.3f,%8.3f,%s,%p,%p,%d,%d,%s,%s,%s,%s,%s\n", - testNum, N * sizeof(float), links[i].exeIndex, + printf("%d,%lu,%c%02d,%c%02d,%c%02d,%d,%9.3f,%8.3f,%s,%p,%p,%d,%d,%d,%s,%s,%s,%s\n", + testNum, N * sizeof(float), MemTypeStr[links[i].srcMemType], links[i].srcIndex, + MemTypeStr[links[i].exeMemType], links[i].exeIndex, MemTypeStr[links[i].dstMemType], links[i].dstIndex, + links[i].exeMemType == MEM_CPU ? ev.numCpuPerLink : links[i].numBlocksToUse, linkBandwidthGbs, linkDurationMsec, GetLinkDesc(links[i]).c_str(), - linkSrcMem[i] + initOffset, linkDstMem[i] + initOffset, - numWarmups, numIterations, - useHipCall ? "true" : "false", - useMemset ? "true" : "false", - useFineGrainMem ? "true" : "false", - useSingleSync ? "true" : "false", - reuseStreams ? "true" : "false"); + links[i].srcMem + initOffset, links[i].dstMem + initOffset, + ev.byteOffset, + ev.numWarmups, ev.numIterations, + ev.useHipCall ? "true" : "false", + ev.useMemset ? "true" : "false", + ev.useSingleSync ? "true" : "false", + ev.combineTiming ? "true" : "false"); } } // Display aggregate statistics - if (!outputToCsv) + if (!ev.outputToCsv) { printf(" Aggregate Bandwidth (CPU timed) | %9.3f GB/s | %8.3f ms |\n", totalBandwidthGbs, totalCpuTime); } else { - printf("%d,%lu,ALL,ALL,ALL,%9.3f,%8.3f,ALL,ALL,ALL,%d,%d,%s,%s,%s,%s,%s\n", - testNum, N * sizeof(float), totalBandwidthGbs, totalCpuTime, numWarmups, numIterations, - useHipCall ? "true" : "false", - useMemset ? "true" : "false", - useFineGrainMem ? "true" : "false", - useSingleSync ? "true" : "false", - reuseStreams ? "true" : "false"); + printf("%d,%lu,ALL,ALL,ALL,ALL,%9.3f,%8.3f,ALL,ALL,ALL,%d,%d,%d,%s,%s,%s,%s\n", + testNum, N * sizeof(float), totalBandwidthGbs, totalCpuTime, ev.byteOffset, + ev.numWarmups, ev.numIterations, + ev.useHipCall ? "true" : "false", + ev.useMemset ? "true" : "false", + ev.useSingleSync ? "true" : "false", + ev.combineTiming ? "true" : "false"); } + } - // Release GPU memory - for (int i = 0; i < numLinks; i++) + // Release GPU memory + for (int i = 0; i < numLinks; i++) + { + DeallocateMemory(links[i].srcMemType, links[i].srcIndex, links[i].srcMem); + DeallocateMemory(links[i].dstMemType, links[i].dstIndex, links[i].dstMem); + + if (links[i].exeMemType == MEM_GPU) { - DeallocateMemory(links[i].srcMemType, links[i].srcIndex, linkSrcMem[i]); - DeallocateMemory(links[i].dstMemType, links[i].dstIndex, linkDstMem[i]); - HIP_CALL(hipFree(gpuBlockParams[i])); - if (!reuseStreams) - HIP_CALL(hipStreamDestroy(streams[i])); - HIP_CALL(hipEventDestroy(startEvents[i])); - HIP_CALL(hipEventDestroy(stopEvents[i])); + HIP_CALL(hipEventDestroy(links[i].startEvent)); + HIP_CALL(hipEventDestroy(links[i].stopEvent)); + HIP_CALL(hipStreamDestroy(links[i].stream)); + HIP_CALL(hipFree(links[i].blockParam)); + } + else if (links[i].exeMemType == MEM_CPU) + { + free(links[i].blockParam); } } } fclose(fp); - // Clean up stream cache if re-using streams - if (reuseStreams) - { - for (auto streamVector : streamCache) - for (auto stream : streamVector) - HIP_CALL(hipStreamDestroy(stream)); - } - return 0; } void DisplayUsage(char const* cmdName) { + if (numa_available() == -1) + { + printf("[ERROR] NUMA library not supported. Check to see if libnuma has been installed on this system\n"); + exit(1); + } + int numGpuDevices; + HIP_CALL(hipGetDeviceCount(&numGpuDevices)); + int const numCpuDevices = numa_num_configured_nodes(); + printf("Usage: %s configFile \n", cmdName); printf(" configFile: File containing Links to execute (see below for format)\n"); @@ -498,57 +369,48 @@ void DisplayUsage(char const* cmdName) printf("\n"); printf("Configfile Format:\n"); printf("==================\n"); - printf("A Link is defined as a uni-directional transfer from src memory location to dst memory location\n"); + printf("A Link is defined as a uni-directional transfer from src memory location to dst memory location executed by either CPU or GPU\n"); printf("Each single line in the configuration file defines a set of Links to run in parallel\n"); printf("\n"); printf("There are two ways to specify the configuration file:\n"); printf("\n"); printf("1) Basic\n"); - printf(" The basic specification assumes the same number of threadblocks/CUs used per link\n"); + printf(" The basic specification assumes the same number of threadblocks/CUs used per GPU-executed Link\n"); printf(" A positive number of Links is specified followed by that number of triplets describing each Link\n"); printf("\n"); - printf(" #Links #CUs (GPUIndex1 srcMem1 dstMem1) ... (GPUIndexL srcMemL dstMemL)\n"); + printf(" #Links #CUs (srcMem1->Executor1->dstMem1) ... (srcMemL->ExecutorL->dstMemL)\n"); printf("\n"); printf("2) Advanced\n"); - printf(" The advanced specification allows different number of threadblocks/CUs used per Link\n"); + printf(" The advanced specification allows different number of threadblocks/CUs used per GPU-executed Link\n"); printf(" A negative number of links is specified, followed by quadruples describing each Link\n"); - printf(" -#Links (GPUIndex1 #CUs1 srcMem1 dstMem1) ... (GPUIndexL #CUsL srcMemL dstMemL)\n"); + printf(" -#Links (srcMem1->Executor1->dstMem1 #CUs1) ... (srcMemL->ExecutorL->dstMemL #CUsL)\n"); printf("\n"); printf("Argument Details:\n"); printf(" #Links : Number of Links to be run in parallel\n"); - printf(" #CUs : Number of threadblocks/CUs to use for a Link\n"); - printf(" GpuIndex: 0-indexed GPU id executing the Link\n"); + printf(" #CUs : Number of threadblocks/CUs to use for a GPU-executed Link\n"); printf(" srcMemL : Source memory location (Where the data is to be read from). Ignored in memset mode\n"); + printf(" Executor: Executor are specified by a character indicating executor type, followed by device index (0-indexed)\n"); + printf(" - C: CPU-executed (Indexed from 0 to %d)\n", numCpuDevices-1); + printf(" - G: GPU-executed (Indexed from 0 to %d)\n", numGpuDevices-1); printf(" dstMemL : Destination memory location (Where the data is to be written to)\n"); - printf(" Memory locations are specified by a character indicating memory type, followed by GPU device index (0-indexed)\n"); + printf("\n"); + printf(" Memory locations are specified by a character indicating memory type, followed by device index (0-indexed)\n"); printf(" Supported memory locations are:\n"); - printf(" - P: Pinned host memory (on CPU, on NUMA node closest to provided GPU index)\n"); - printf(" - G: Global device memory (on GPU)\n"); - printf("Round brackets may be included for human clarity, but will be ignored\n"); + printf(" - C: Pinned host memory (on NUMA node, indexed from 0 to %d)\n", numCpuDevices-1); + printf(" - G: Global device memory (on GPU device indexed from 0 to %d)\n", numGpuDevices-1); + printf(" - F: Fine-grain device memory (on GPU device indexed from 0 to %d)\n", numGpuDevices-1); printf("\n"); printf("Examples:\n"); - printf("1 4 (0 G0 G1) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 0 and copies it to memory on GPU 1\n"); - printf("1 4 (0 G1 G0) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 1 and copies it to memory on GPU 0\n"); - printf("1 4 (2 P0 G2) Single Link that uses 4 CUs on GPU 2 that reads memory from CPU 0 and copies it to memory on GPU 2\n"); - printf("2 4 (0 G0 G1) (1 G1 G0) Runs 2 Links in parallel. GPU 0 - > GPU1, and GP1 -> GPU 0, each with 4 CUs\n"); - printf("-2 (0 G0 G1 4) (1 G1 G0 2) Runs 2 Links in parallel. GPU 0 - > GPU 1 using four CUs, and GPU1 -> GPU 0 using two CUs\n"); + printf("1 4 (G0->G0->G1) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 0 and copies it to memory on GPU 1\n"); + printf("1 4 (G1->C0->G0) Single Link that uses 4 CUs on GPU 0 that reads memory from CPU 1 and copies it to memory on GPU 0\n"); + printf("1 4 (C0->G2->G2) Single Link that uses 4 CUs on GPU 2 that reads memory from CPU 0 and copies it to memory on GPU 2\n"); + printf("2 4 G0->G0->G1 G1->G1->G0 Runs 2 Links in parallel. GPU 0 - > GPU1, and GP1 -> GPU 0, each with 4 CUs\n"); + printf("-2 (G0 G0 G1 4) (G1 G1 G0 2) Runs 2 Links in parallel. GPU 0 - > GPU 1 using four CUs, and GPU1 -> GPU 0 using two CUs\n"); printf("\n"); + printf("Round brackets and arrows' ->' may be included for human clarity, but will be ignored and are unnecessary\n"); printf("\n"); - printf("Environment variables:\n"); - printf("======================\n"); - printf(" USE_HIP_CALL - Use hipMemcpy/hipMemset instead of custom shader kernels\n"); - printf(" USE_MEMSET - Perform a memset instead of a copy (ignores source memory)\n"); - printf(" USE_FINEGRAIN_MEM - Allocate fine-grained GPU memory instead of coarse-grained GPU memory\n"); - printf(" USE_SINGLE_SYNC - Perform synchronization only once after all iterations instead of per iteration\n"); - printf(" USE_INTERACTIVE - Pause for user-input before starting transfer loop\n"); - printf(" USE_SLEEP - Adds a 100ms sleep after each synchronization\n"); - printf(" COMBINE_TIMING - Combines timing with launch (potentially lower timing overhead)\n"); - printf(" REUSE_STREAMS - Re-use streams instead of creating / destroying per test\n"); - printf(" SHOW_ADDR - Print out memory addresses for each Link\n"); - printf(" OUTPUT_TO_CSV - Outputs to CSV format if set\n"); - printf(" BYTE_OFFSET - Initial byte-offset for memory allocations. Must be multiple of 4. Defaults to 0\n"); - printf(" NUM_WARMUPS=W - Perform W untimed warmup iteration(s) per test\n"); - printf(" NUM_ITERATIONS=I - Perform I timed iteration(s) per test\n"); + + EnvVars::DisplayUsage(); } void GenerateConfigFile(char const* cfgFile, int numBlocks) @@ -556,13 +418,28 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) // Detect number of available GPUs and skip if less than 2 int numGpuDevices; HIP_CALL(hipGetDeviceCount(&numGpuDevices)); - printf("Generated configFile %s for %d device(s) / %d CUs per link\n", cfgFile, numGpuDevices, numBlocks); + printf("Generating configFile %s for %d device(s) / %d CUs per link\n", cfgFile, numGpuDevices, numBlocks); if (numGpuDevices < 2) { printf("Skipping. (Less than 2 GPUs detected)\n"); exit(0); } + // Check first to see if file exists, and issue warning + FILE* exists = fopen(cfgFile, "r"); + if (exists) + { + fclose(exists); + printf("[WARN] File %s alreadys exists. Enter 'Y' to confirm overwrite\n", cfgFile); + char ch; + scanf(" %c", &ch); + if (ch != 'Y' && ch != 'y') + { + printf("Aborting\n"); + exit(0); + } + } + // Open config file for writing FILE* fp = fopen(cfgFile, "w"); if (!fp) @@ -574,18 +451,18 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) // CU testing fprintf(fp, "# CU scaling tests\n"); for (int i = 1; i < 16; i++) - fprintf(fp, "1 %d (0 G0 G1)\n", i); + fprintf(fp, "1 %d (G0->G0->G1)\n", i); fprintf(fp, "\n"); // Pinned memory testing fprintf(fp, "# Pinned CPU memory read tests\n"); for (int i = 0; i < numGpuDevices; i++) - fprintf(fp, "1 %d (%d C%d G%d)\n", numBlocks, i, i, i); + fprintf(fp, "1 %d (C0->G%d->G%d)\n", numBlocks, i, i); fprintf(fp, "\n"); fprintf(fp, "# Pinned CPU memory write tests\n"); for (int i = 0; i < numGpuDevices; i++) - fprintf(fp, "1 %d (%d G%d C%d)\n", numBlocks, i, i, i); + fprintf(fp, "1 %d (G%d->G%d->C0)\n", numBlocks, i, i); fprintf(fp, "\n"); // Single link testing GPU testing @@ -594,7 +471,7 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) for (int j = 0; j < numGpuDevices; j++) { if (i == j) continue; - fprintf(fp, "1 %d (%d G%d G%d)\n", numBlocks, i, i, j); + fprintf(fp, "1 %d (G%d->G%d->G%d)\n", numBlocks, i, i, j); } fprintf(fp, "\n"); @@ -604,7 +481,7 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) for (int j = 0; j < numGpuDevices; j++) { if (i == j) continue; - fprintf(fp, "2 %d (%d G%d G%d) (%d G%d G%d)\n", numBlocks, i, i, j, j, j, i); + fprintf(fp, "2 %d (G%d->G%d->G%d) (G%d->G%d->G%d)\n", numBlocks, i, i, j, j, j, i); } fprintf(fp, "\n"); @@ -613,7 +490,7 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) fprintf(fp, "%d %d", numGpuDevices, numBlocks); for (int i = 0; i < numGpuDevices; i++) { - fprintf(fp, " (%d G%d G%d)", i, i, (i+1)%numGpuDevices); + fprintf(fp, " (G%d->G%d->G%d)", i, i, (i+1)%numGpuDevices); } fprintf(fp, "\n\n"); @@ -621,23 +498,23 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) fprintf(fp, "# Simple bi-directional ring\n"); fprintf(fp, "%d %d", numGpuDevices * 2, numBlocks); for (int i = 0; i < numGpuDevices; i++) - fprintf(fp, " (%d G%d G%d)", i, i, (i+1)%numGpuDevices); + fprintf(fp, " (G%d->G%d->G%d)", i, i, (i+1)%numGpuDevices); for (int i = 0; i < numGpuDevices; i++) - fprintf(fp, " (%d G%d G%d)", i, i, (i+numGpuDevices-1)%numGpuDevices); + fprintf(fp, " (G%d->G%d->G%d)", i, i, (i+numGpuDevices-1)%numGpuDevices); fprintf(fp, "\n\n"); // Broadcast from GPU 0 fprintf(fp, "# GPU 0 Broadcast\n"); fprintf(fp, "%d %d", numGpuDevices-1, numBlocks); for (int i = 1; i < numGpuDevices; i++) - fprintf(fp, " (%d G%d G%d)", 0, 0, i); + fprintf(fp, " (G%d->G%d->G%d)", 0, 0, i); fprintf(fp, "\n\n"); // Gather to GPU 0 fprintf(fp, "# GPU 0 Gather\n"); fprintf(fp, "%d %d", numGpuDevices-1, numBlocks); for (int i = 1; i < numGpuDevices; i++) - fprintf(fp, " (%d G%d G%d)", 0, i, 0); + fprintf(fp, " (G%d->G%d->G%d)", 0, i, 0); fprintf(fp, "\n\n"); // Full stress test @@ -647,7 +524,7 @@ void GenerateConfigFile(char const* cfgFile, int numBlocks) for (int j = 0; j < numGpuDevices; j++) { if (i == j) continue; - fprintf(fp, " (%d G%d G%d)", i, i, j); + fprintf(fp, " (G%d->G%d->G%d)", i, i, j); } fprintf(fp, "\n\n"); @@ -692,29 +569,87 @@ void DisplayTopology() } } -void ParseMemType(std::string const& token, MemType* memType, int* memIndex) +void PopulateTestSizes(size_t const numBytesPerLink, + int const samplingFactor, + std::vector& valuesOfN) +{ + valuesOfN.clear(); + + // If the number of bytes is specified, use it + if (numBytesPerLink != 0) + { + if (numBytesPerLink % 4) + { + printf("[ERROR] numBytesPerLink (%lu) must be a multiple of 4\n", numBytesPerLink); + exit(1); + } + size_t N = numBytesPerLink / sizeof(float); + valuesOfN.push_back(N); + } + else + { + // Otherwise generate a range of values + // (Powers of 2, with samplingFactor samples between successive powers of 2) + for (int N = 256; N <= (1<<27); N *= 2) + { + int delta = std::max(32, N / samplingFactor); + int curr = N; + while (curr < N * 2) + { + valuesOfN.push_back(curr); + curr += delta; + } + } + } +} + +void ParseMemType(std::string const& token, int const numCpus, int const numGpus, MemType* memType, int* memIndex) { char typeChar; if (sscanf(token.c_str(), " %c %d", &typeChar, memIndex) != 2) { - printf("Error parsing memory type token %s\n", token.c_str()); + printf("[ERROR] Unable to parse memory type token %s - expecting either 'C' or 'G' or 'F' followed by an index\n", token.c_str()); exit(1); } switch (typeChar) { - case 'C': case 'c': *memType = MEM_CPU; break; - case 'G': case 'g': *memType = MEM_GPU; break; - default: printf("Unrecognized memory type %s\n", token.c_str()); exit(1); + case 'C': case 'c': + *memType = MEM_CPU; + if (*memIndex < 0 || *memIndex >= numCpus) + { + printf("[ERROR] CPU index must be between 0 and %d (instead of %d)\n", numCpus-1, *memIndex); + exit(1); + } + break; + case 'G': case 'g': + *memType = MEM_GPU; + if (*memIndex < 0 || *memIndex >= numGpus) + { + printf("[ERROR] GPU index must be between 0 and %d (instead of %d)\n", numGpus-1, *memIndex); + exit(1); + } + break; + case 'F': case 'f': + *memType = MEM_GPU_FINE; + if (*memIndex < 0 || *memIndex >= numGpus) + { + printf("[ERROR] GPU index must be between 0 and %d (instead of %d)\n", numGpus-1, *memIndex); + exit(1); + } + break; + default: + printf("[ERROR] Unrecognized memory type %s. Expecting either 'C' or 'G' or 'F'\n", token.c_str()); + exit(1); } } -// Helper function to parse a link of link definitions -void ParseLinks(char* line, std::vector& links) +// Helper function to parse a list of link definitions +void ParseLinks(char* line, int numCpus, int numGpus, std::vector& links) { - // Replace any round brackets with spaces + // Replace any round brackets or '->' with spaces, for (int i = 0; line[i]; i++) - if (line[i] == '(' || line[i] == ')') line[i] = ' '; + if (line[i] == '(' || line[i] == ')' || line[i] == '-' || line[i] == '>' ) line[i] = ' '; links.clear(); int numLinks = 0; @@ -725,11 +660,12 @@ void ParseLinks(char* line, std::vector& links) iss >> numLinks; if (iss.fail()) return; + std::string exeMem; std::string srcMem; std::string dstMem; if (numLinks > 0) { - // Method 1: Take in triples (exeGpu, srcMem, dstMem) + // Method 1: Take in triples (srcMem, exeMem, dstMem) int numBlocksToUse; iss >> numBlocksToUse; if (numBlocksToUse <= 0) @@ -740,47 +676,99 @@ void ParseLinks(char* line, std::vector& links) links.resize(numLinks); for (int i = 0; i < numLinks; i++) { - iss >> links[i].exeIndex >> srcMem >> dstMem; - ParseMemType(srcMem, &links[i].srcMemType, &links[i].srcIndex); - ParseMemType(dstMem, &links[i].dstMemType, &links[i].dstIndex); + iss >> srcMem >> exeMem >> dstMem; + ParseMemType(srcMem, numCpus, numGpus, &links[i].srcMemType, &links[i].srcIndex); + ParseMemType(exeMem, numCpus, numGpus, &links[i].exeMemType, &links[i].exeIndex); + ParseMemType(dstMem, numCpus, numGpus, &links[i].dstMemType, &links[i].dstIndex); links[i].numBlocksToUse = numBlocksToUse; + if (links[i].exeMemType != MEM_CPU && links[i].exeMemType != MEM_GPU) + { + printf("[ERROR] Executor must either be CPU ('C') or GPU ('G')\n"); + exit(1); + } } } else { - // Method 2: Read in quads (exeGpu, srcMem, dstMem, Read common # blocks to use, then read (src, dst) doubles + // Method 2: Read in quads (srcMem, exeMem, dstMem, Read common # blocks to use, then read (src, dst) doubles numLinks *= -1; links.resize(numLinks); for (int i = 0; i < numLinks; i++) { - iss >> links[i].exeIndex >> srcMem >> dstMem >> links[i].numBlocksToUse; - ParseMemType(srcMem, &links[i].srcMemType, &links[i].srcIndex); - ParseMemType(dstMem, &links[i].dstMemType, &links[i].dstIndex); + iss >> srcMem >> exeMem >> dstMem >> links[i].numBlocksToUse; + ParseMemType(srcMem, numCpus, numGpus, &links[i].srcMemType, &links[i].srcIndex); + ParseMemType(exeMem, numCpus, numGpus, &links[i].exeMemType, &links[i].exeIndex); + ParseMemType(dstMem, numCpus, numGpus, &links[i].dstMemType, &links[i].dstIndex); + if (links[i].exeMemType != MEM_CPU || links[i].exeMemType != MEM_GPU) + { + printf("[ERROR] Executor must either be CPU ('C') or GPU ('G')\n"); + exit(1); + } + } } } -void AllocateMemory(MemType memType, int devIndex, size_t numBytes, bool useFineGrainMem, float** memPtr) +void EnablePeerAccess(int const deviceId, int const peerDeviceId) { - HIP_CALL(hipSetDevice(devIndex)); + int canAccess; + HIP_CALL(hipDeviceCanAccessPeer(&canAccess, deviceId, peerDeviceId)); + if (!canAccess) + { + printf("[ERROR] Unable to enable peer access from GPU devices %d to %d\n", peerDeviceId, deviceId); + exit(1); + } + HIP_CALL(hipSetDevice(deviceId)); + HIP_CALL(hipDeviceEnablePeerAccess(peerDeviceId, 0)); +} + +void AllocateMemory(MemType memType, int devIndex, size_t numBytes, float** memPtr) +{ + if (numBytes == 0) + { + printf("[ERROR] Unable to allocate 0 bytes\n"); + exit(1); + } if (memType == MEM_CPU) { - // // Allocate pinned-memory on NUMA node closest to the selected GPU - HIP_CALL(hipHostMalloc((void **)memPtr, numBytes, hipHostMallocPortable)); + // Set numa policy prior to call to hipHostMalloc + unsigned long nodemask = (1ULL << devIndex); + long retCode = set_mempolicy(MPOL_BIND, &nodemask, sizeof(nodemask)*8); + if (retCode) + { + printf("[ERROR] Unable to set NUMA memory policy to bind to NUMA node %d\n", devIndex); + exit(1); + } + + // Allocate host-pinned memory (should respect NUMA mem policy) + HIP_CALL(hipHostMalloc((void **)memPtr, numBytes, hipHostMallocNumaUser)); + + // Check that the allocated pages are actually on the correct NUMA node + CheckPages((char*)*memPtr, numBytes, devIndex); + + // Reset to default numa mem policy + retCode = set_mempolicy(MPOL_DEFAULT, NULL, 8); + if (retCode) + { + printf("[ERROR] Unable reset to default NUMA memory policy\n"); + exit(1); + } } else if (memType == MEM_GPU) { - // Allocate GPU memory - if (useFineGrainMem) - HIP_CALL(hipExtMallocWithFlags((void**)memPtr, numBytes, hipDeviceMallocFinegrained)); - else - HIP_CALL(hipMalloc((void**)memPtr, numBytes)); + // Allocate GPU memory on appropriate device + HIP_CALL(hipSetDevice(devIndex)); + HIP_CALL(hipMalloc((void**)memPtr, numBytes)); + } + else if (memType == MEM_GPU_FINE) + { + HIP_CALL(hipExtMallocWithFlags((void**)memPtr, numBytes, hipDeviceMallocFinegrained)); } else { - printf("Error: Unsupported memory type %d\n", memType); + printf("[ERROR] Unsupported memory type %d\n", memType); exit(1); } } @@ -791,12 +779,51 @@ void DeallocateMemory(MemType memType, int devIndex, float* memPtr) { HIP_CALL(hipHostFree(memPtr)); } - else if (memType == MEM_GPU) + else if (memType == MEM_GPU || memType == MEM_GPU_FINE) { HIP_CALL(hipFree(memPtr)); } } +void CheckPages(char* array, size_t numBytes, int targetId) +{ + unsigned long const pageSize = getpagesize(); + unsigned long const numPages = (numBytes + pageSize - 1) / pageSize; + + std::vector pages(numPages); + std::vector status(numPages); + + pages[0] = array; + for (int i = 1; i < numPages; i++) + { + pages[i] = (char*)pages[i-1] + pageSize; + } + + long const retCode = move_pages(0, numPages, pages.data(), NULL, status.data(), 0); + if (retCode) + { + printf("[ERROR] Unable to collect page info\n"); + exit(1); + } + + size_t mistakeCount = 0; + for (int i = 0; i < numPages; i++) + { + if (status[i] < 0) + { + printf("[ERROR] Unexpected page status %d for page %d\n", status[i], i); + exit(1); + } + if (status[i] != targetId) mistakeCount++; + } + if (mistakeCount > 0) + { + printf("[ERROR] %lu out of %lu pages for memory allocation were not on NUMA node %d\n", mistakeCount, numPages, targetId); + // NOTE: Some older versions of HIP do not properly respect NUMA policy so avoid failing for now + // exit(1); + } +} + // Helper function to either fill a device pointer with pseudo-random data, or to check to see if it matches void CheckOrFill(ModeType mode, int N, bool isMemset, bool isHipCall, float* ptr) { @@ -858,31 +885,132 @@ std::string GetLinkTypeDesc(uint32_t linkType, uint32_t hopCount) return result; } +std::string GetDesc(MemType srcMemType, int srcIndex, + MemType dstMemType, int dstIndex) +{ + if (srcMemType == MEM_CPU) + { + if (dstMemType == MEM_CPU) + return (srcIndex == dstIndex) ? "LOCAL" : "NUMA"; + else if (dstMemType == MEM_GPU || dstMemType == MEM_GPU_FINE) + return "PCIE"; + else + goto error; + } + else if (srcMemType == MEM_GPU || srcMemType == MEM_GPU_FINE) + { + if (dstMemType == MEM_CPU) + return "PCIE"; + else if (dstMemType == MEM_GPU || dstMemType == MEM_GPU_FINE) + { + if (srcIndex == dstIndex) return "LOCAL"; + else + { + uint32_t linkType, hopCount; + HIP_CALL(hipExtGetLinkTypeAndHopCount(srcIndex, dstIndex, &linkType, &hopCount)); + return GetLinkTypeDesc(linkType, hopCount); + } + } + else + goto error; + } +error: + printf("[ERROR] Unrecognized memory type\n"); + exit(1); +} + std::string GetLinkDesc(Link const& link) { - std::string result = ""; - - // Currently only describe links between src/dst on GPU - if (link.srcMemType == MEM_GPU && link.dstMemType == MEM_GPU) - { - if (link.exeIndex != link.srcIndex) - { - uint32_t linkType, hopCount; - HIP_CALL(hipExtGetLinkTypeAndHopCount(link.srcIndex, link.exeIndex, &linkType, &hopCount)); - result += GetLinkTypeDesc(linkType, hopCount); - } - - if (link.exeIndex != link.dstIndex) - { - uint32_t linkType, hopCount; - HIP_CALL(hipExtGetLinkTypeAndHopCount(link.exeIndex, link.dstIndex, &linkType, &hopCount)); - if (result != "") result += "+"; - result += GetLinkTypeDesc(linkType, hopCount); - } - } - else - { - result = "???"; - } - return result; + return GetDesc(link.srcMemType, link.srcIndex, link.exeMemType, link.exeIndex) + "-" + + GetDesc(link.exeMemType, link.exeIndex, link.dstMemType, link.dstIndex); +} + +void RunLink(EnvVars const& ev, size_t const N, int const iteration, Link& link) +{ + // GPU execution agent + if (link.exeMemType == MEM_GPU) + { + // Switch to executing GPU + HIP_CALL(hipSetDevice(link.exeIndex)); + + bool recordStart = (!ev.useSingleSync || iteration == 0); + bool recordStop = (!ev.useSingleSync || iteration == ev.numIterations - 1); + + int const initOffset = ev.byteOffset / sizeof(float); + + if (ev.useHipCall) + { + // Record start event + if (recordStart) HIP_CALL(hipEventRecord(link.startEvent, link.stream)); + + // Execute hipMemset / hipMemcpy + if (ev.useMemset) + HIP_CALL(hipMemsetAsync(link.dstMem + initOffset, 42, N * sizeof(float), link.stream)); + else + HIP_CALL(hipMemcpyAsync(link.dstMem + initOffset, + link.srcMem + initOffset, + N * sizeof(float), hipMemcpyDefault, + link.stream)); + // Record stop event + if (recordStop) HIP_CALL(hipEventRecord(link.stopEvent, link.stream)); + } + else + { + if (!ev.combineTiming && recordStart) HIP_CALL(hipEventRecord(link.startEvent, link.stream)); + hipExtLaunchKernelGGL(ev.useMemset ? GpuMemsetKernel : GpuCopyKernel, + dim3(link.numBlocksToUse, 1, 1), + dim3(BLOCKSIZE, 1, 1), + 0, link.stream, + (ev.combineTiming && recordStart) ? link.startEvent : NULL, + (ev.combineTiming && recordStop) ? link.stopEvent : NULL, + 0, link.blockParam); + if (!ev.combineTiming & recordStop) HIP_CALL(hipEventRecord(link.stopEvent, link.stream)); + } + + // Synchronize per iteration, unless in single sync mode, in which case + // synchronize during last warmup / last actual iteration + if (!ev.useSingleSync || iteration == -1 || iteration == ev.numIterations - 1) + { + HIP_CALL(hipStreamSynchronize(link.stream)); + } + + if (iteration >= 0) + { + // Record GPU timing + if (!ev.useSingleSync || iteration == ev.numIterations - 1) + { + HIP_CALL(hipEventSynchronize(link.stopEvent)); + float gpuDeltaMsec; + HIP_CALL(hipEventElapsedTime(&gpuDeltaMsec, link.startEvent, link.stopEvent)); + link.totalTime += gpuDeltaMsec; + } + } + } + else if (link.exeMemType == MEM_CPU) // CPU execution agent + { + // Force this thread and all child threads onto correct NUMA node + if (numa_run_on_node(link.exeIndex)) + { + printf("[ERROR] Unable to set CPU to NUMA node %d\n", link.exeIndex); + exit(1); + } + + std::vector childThreads; + + auto cpuStart = std::chrono::high_resolution_clock::now(); + + // Launch child-threads to perform memcopies + for (int i = 0; i < ev.numCpuPerLink; i++) + childThreads.push_back(std::thread(ev.useMemset ? CpuMemsetKernel : CpuCopyKernel, std::ref(link.blockParam[i]))); + + // Wait for child-threads to finish + for (int i = 0; i < ev.numCpuPerLink; i++) + childThreads[i].join(); + + auto cpuDelta = std::chrono::high_resolution_clock::now() - cpuStart; + + // Record time if not a warmup iteration + if (iteration >= 0) + link.totalTime += (std::chrono::duration_cast>(cpuDelta).count() * 1000.0); + } } diff --git a/projects/rccl/tools/TransferBench/TransferBench.hpp b/projects/rccl/tools/TransferBench/TransferBench.hpp index 1291bf68f7..6a08e80f4e 100644 --- a/projects/rccl/tools/TransferBench/TransferBench.hpp +++ b/projects/rccl/tools/TransferBench/TransferBench.hpp @@ -33,7 +33,9 @@ THE SOFTWARE. #include #include #include + #include "copy_kernel.h" +#include "EnvVars.hpp" // Helper macro for catching HIP errors #define HIP_CALL(cmd) \ @@ -50,11 +52,12 @@ THE SOFTWARE. // Different src/dst memory types supported typedef enum { - MEM_CPU = 0, // Pinned CPU memory - MEM_GPU = 1 // Global GPU memory + MEM_CPU = 0, // Pinned CPU memory + MEM_GPU = 1, // Coarse-grained global GPU memory + MEM_GPU_FINE = 2 // Fine-grained global GPU memory } MemType; -char const MemTypeStr[3] = "CG"; +char const MemTypeStr[4] = "CGF"; typedef enum { @@ -62,17 +65,6 @@ typedef enum MODE_CHECK = 1 // Check data against pattern } ModeType; -// Each Link is a uni-direction operation from a src memory to dst memory executed by a specific GPU -struct Link -{ - int exeIndex; // GPU to execute on - MemType srcMemType; // Source memory type - int srcIndex; // Source device index - MemType dstMemType; // Destination memory type - int dstIndex; // Destination device index - int numBlocksToUse; // Number of threadblocks to use for this Link -}; - // Each threadblock copies N floats from src to dst struct BlockParam { @@ -81,46 +73,90 @@ struct BlockParam float* dst; }; +// Each Link is a uni-direction operation from a src memory to dst memory executed by a specific GPU +struct Link +{ + // Link config + MemType exeMemType; // Link executor type (CPU or GPU) + int exeIndex; // Executor index (NUMA node for CPU / device ID for GPU) + MemType srcMemType; // Source memory type + int srcIndex; // Source device index + MemType dstMemType; // Destination memory type + int dstIndex; // Destination device index + int numBlocksToUse; // Number of threadblocks to use for this Link + + // Link implementation + float* srcMem; // Source memory + float* dstMem; // Destination memory + + hipEvent_t startEvent; + hipEvent_t stopEvent; + hipStream_t stream; + BlockParam* blockParam; + + double totalTime; +}; + void DisplayUsage(char const* cmdName); // Display usage instructions void GenerateConfigFile(char const* cfgFile, int numBlocks); // Generate a sample config file void DisplayTopology(); // Display GPU topology -void ParseLinks(char* line, std::vector& links); // Parse Link information -void AllocateMemory(MemType memType, int devIndex, size_t numBytes, bool useFineGrainMem, float** memPtr); +void PopulateTestSizes(size_t const numBytesPerLink, int const samplingFactor, std::vector& valuesofN); +void ParseMemType(std::string const& token, int const numCpus, int const numGpus, MemType* memType, int* memIndex); +void ParseLinks(char* line, int numCpus, int numGpus, std::vector& links); // Parse Link information +void EnablePeerAccess(int const deviceId, int const peerDeviceId); +void AllocateMemory(MemType memType, int devIndex, size_t numBytes, float** memPtr); void DeallocateMemory(MemType memType, int devIndex, float* memPtr); +void CheckPages(char* byteArray, size_t numBytes, int targetId); void CheckOrFill(ModeType mode, int N, bool isMemset, bool isHipCall, float* ptr); +void RunLink(EnvVars const& ev, size_t const N, int const iteration, Link& link); + + std::string GetLinkTypeDesc(uint32_t linkType, uint32_t hopCount); +std::string GetDesc(MemType srcMemType, int srcIndex, + MemType dstMemType, int dstIndex); std::string GetLinkDesc(Link const& link); - -#define MAX_NAME_LEN 64 #define BLOCKSIZE 256 #define COPY_UNROLL 4 #define MEMSET_UNROLL 4 // GPU copy kernel __global__ void __launch_bounds__(BLOCKSIZE) -CopyKernel(BlockParam* blockParams) +GpuCopyKernel(BlockParam* blockParams) { - // Collect the arguments for this block - int N = blockParams[blockIdx.x].N; - const float* __restrict__ src = (float* )blockParams[blockIdx.x].src; - float* __restrict__ dst = (float* )blockParams[blockIdx.x].dst; + // Collect the arguments for this block + int N = blockParams[blockIdx.x].N; + const float* __restrict__ src = (float* )blockParams[blockIdx.x].src; + float* __restrict__ dst = (float* )blockParams[blockIdx.x].dst; - Copy(dst, src, N); + Copy(dst, src, N); } // GPU set kernel __global__ void __launch_bounds__(BLOCKSIZE) -MemsetKernel(BlockParam* blockParams) +GpuMemsetKernel(BlockParam* blockParams) { - // Collect the arguments for this block - int N = blockParams[blockIdx.x].N; - float* __restrict__ dst = (float*)blockParams[blockIdx.x].dst; + // Collect the arguments for this block + int N = blockParams[blockIdx.x].N; + float* __restrict__ dst = (float*)blockParams[blockIdx.x].dst; - // Use non-zero value - #pragma unroll MEMSET_UNROLL - for (int tid = threadIdx.x; tid < N; tid += BLOCKSIZE) - { - dst[tid] = 1234.0; - } + // Use non-zero value + #pragma unroll MEMSET_UNROLL + for (int tid = threadIdx.x; tid < N; tid += BLOCKSIZE) + { + dst[tid] = 1234.0; + } +} + +// CPU copy kernel +void CpuCopyKernel(BlockParam const& blockParams) +{ + memcpy(blockParams.dst, blockParams.src, blockParams.N * sizeof(float)); +} + +// CPU memset kernel +void CpuMemsetKernel(BlockParam const& blockParams) +{ + for (int i = 0; i < blockParams.N; i++) + blockParams.dst[i] = 1234.0; } diff --git a/projects/rccl/tools/TransferBench/example.cfg b/projects/rccl/tools/TransferBench/example.cfg index b14762f9e3..2d546f433b 100644 --- a/projects/rccl/tools/TransferBench/example.cfg +++ b/projects/rccl/tools/TransferBench/example.cfg @@ -1,39 +1,43 @@ -#Configfile Format: -#================== -#A Link is defined as a uni-directional transfer from src memory location to dst memory location -#Each single line in the configuration file defines a set of Links to run in parallel +# Configfile Format: +# ================== +# A Link is defined as a uni-directional transfer from src memory location to dst memory location executed by either CPU or GPU +# Each single line in the configuration file defines a set of Links to run in parallel -#There are two ways to specify the configuration file: +# There are two ways to specify the configuration file: -#1) Basic -# The basic specification assumes the same number of threadblocks/CUs used per link -# A positive number of Links is specified followed by that number of triplets describing each Link +# 1) Basic +# The basic specification assumes the same number of threadblocks/CUs used per GPU-executed Link +# A positive number of Links is specified followed by that number of triplets describing each Link - #Links #CUs (GPUIndex1 srcMem1 dstMem1) ... (GPUIndexL srcMemL dstMemL) +# #Links #CUs (srcMem1->Executor1->dstMem1) ... (srcMemL->ExecutorL->dstMemL) -#2) Advanced -# The advanced specification allows different number of threadblocks/CUs used per Link -# A negative number of links is specified, followed by quadruples describing each Link -# -#Links (GPUIndex1 #CUs1 srcMem1 dstMem1) ... (GPUIndexL #CUsL srcMemL dstMemL) +# 2) Advanced +# The advanced specification allows different number of threadblocks/CUs used per GPU-executed Link +# A negative number of links is specified, followed by quadruples describing each Link +# -#Links (srcMem1->Executor1->dstMem1 #CUs1) ... (srcMemL->ExecutorL->dstMemL #CUsL) -#Argument Details: -# #Links : Number of Links to be run in parallel -# #CUs : Number of threadblocks/CUs to use for a Link -# GpuIndex: 0-indexed GPU id executing the Link -# srcMemL : Source memory location (Where the data is to be read from). Ignored in memset mode -# dstMemL : Destination memory location (Where the data is to be written to) -# Memory locations are specified by a character indicating memory type, followed by GPU device index (0-indexed) -# Supported memory locations are: -# - P: Pinned host memory (on CPU, on NUMA node closest to provided GPU index) -# - G: Global device memory (on GPU) -#Round brackets may be included for human clarity, but will be ignored +# Argument Details: +# #Links : Number of Links to be run in parallel +# #CUs : Number of threadblocks/CUs to use for a GPU-executed Link +# srcMemL : Source memory location (Where the data is to be read from). Ignored in memset mode +# Executor: Executor are specified by a character indicating executor type, followed by device index (0-indexed) +# - C: CPU-executed (Indexed from 0 to 1) +# - G: GPU-executed (Indexed from 0 to 3) +# dstMemL : Destination memory location (Where the data is to be written to) -#Examples: -#1 4 (0 G0 G1) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 0 and copies it to memory on GPU 1 -#1 4 (0 G1 G0) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 1 and copies it to memory on GPU 0 -#1 4 (2 P0 G2) Single Link that uses 4 CUs on GPU 2 that reads memory from CPU 0 and copies it to memory on GPU 2 -#2 4 (0 G0 G1) (1 G1 G0) Runs 2 Links in parallel. GPU 0 - > GPU1, and GP1 -> GPU 0, each with 4 CUs -#-2 (0 G0 G1 4) (1 G1 G0 2) Runs 2 Links in parallel. GPU 0 - > GPU 1 using four CUs, and GPU1 -> GPU 0 using two CUs +# Memory locations are specified by a character indicating memory type, followed by device index (0-indexed) +# Supported memory locations are: +# - C: Pinned host memory (on NUMA node, indexed from 0 to 1) +# - G: Global device memory (on GPU device indexed from 0 to 3) +# - F: Fine-grain device memory (on GPU device indexed from 0 to 3) -# Single link between GPUs 0 and 1 -1 1 (0 G0 G1) +# Examples: +# 1 4 (G0->G0->G1) Single Link that uses 4 CUs on GPU 0 that reads memory from GPU 0 and copies it to memory on GPU 1 +# 1 4 (G1->C0->G0) Single Link that uses 4 CUs on GPU 0 that reads memory from CPU 1 and copies it to memory on GPU 0 +# 1 4 (C0->G2->G2) Single Link that uses 4 CUs on GPU 2 that reads memory from CPU 0 and copies it to memory on GPU 2 +# 2 4 G0->G0->G1 G1->G1->G0 Runs 2 Links in parallel. GPU 0 - > GPU1, and GP1 -> GPU 0, each with 4 CUs +# -2 (G0 G0 G1 4) (G1 G1 G0 2) Runs 2 Links in parallel. GPU 0 - > GPU 1 using four CUs, and GPU1 -> GPU 0 using two CUs +# Round brackets and arrows' ->' may be included for human clarity, but will be ignored and are unnecessary + +# Single GPU-executed link between GPUs 0 and 1 using 4 CUs +1 4 (G0->G0->G1) From c570f09681b2e30646457e8c7adcd846b75d5b47 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Tue, 15 Dec 2020 17:37:31 -0700 Subject: [PATCH 12/17] [TransferBench] Fixing bug with fine-grained memory allocation (#311) * Fixing bug with fine-grained memory [ROCm/rccl commit: 41c35dad485d20c519a31e50c38ddb57445353df] --- projects/rccl/tools/TransferBench/TransferBench.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/rccl/tools/TransferBench/TransferBench.cpp b/projects/rccl/tools/TransferBench/TransferBench.cpp index f6d90fdb29..faed8dae44 100644 --- a/projects/rccl/tools/TransferBench/TransferBench.cpp +++ b/projects/rccl/tools/TransferBench/TransferBench.cpp @@ -764,6 +764,7 @@ void AllocateMemory(MemType memType, int devIndex, size_t numBytes, float** memP } else if (memType == MEM_GPU_FINE) { + HIP_CALL(hipSetDevice(devIndex)); HIP_CALL(hipExtMallocWithFlags((void**)memPtr, numBytes, hipDeviceMallocFinegrained)); } else From 4ea285c527aad3008a74b053593053bbcaa5c5e5 Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Tue, 15 Dec 2020 17:16:17 -0800 Subject: [PATCH 13/17] Fix Rome PCIe 2 node topology generation (#310) [ROCm/rccl commit: 373a10851678a42da2ed47f55bc523d4e0210327] --- projects/rccl/src/graph/search.cc | 2 +- projects/rccl/tools/scripts/topo_val.sh | 2 +- projects/rccl/tools/topo_expl/Makefile | 2 +- .../topo_expl/models/topo_8p_rome_pcie.xml | 52 +++++++++++++++++++ projects/rccl/tools/topo_expl/topo_expl.cpp | 1 + 5 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 projects/rccl/tools/topo_expl/models/topo_8p_rome_pcie.xml diff --git a/projects/rccl/src/graph/search.cc b/projects/rccl/src/graph/search.cc index a2c0d0b710..4701a7f45f 100644 --- a/projects/rccl/src/graph/search.cc +++ b/projects/rccl/src/graph/search.cc @@ -426,7 +426,7 @@ ncclResult_t ncclTopoSearchRecNet(struct ncclTopoSystem* system, struct ncclTopo for (int i = 0; inodes[GPU].count; i++) if (paths[i].count < paths[f].count) f = i; int t = 1 << 10; - NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, FORCED_ORDER_PCI, &t, NET, n, f)); + NCCLCHECK(ncclTopoSearchTryGpu(system, graph, saveGraph, 0, backToNet, backToFirstRank, (f == 0) ? FORCED_ORDER_PCI : 0, &t, NET, n, f)); // [RCCL] Event if forced order PCI is found, continue the search instead of ending early // if (t == -1) *time = -1; // [/RCCL] diff --git a/projects/rccl/tools/scripts/topo_val.sh b/projects/rccl/tools/scripts/topo_val.sh index 9afe237aa6..fe11d8b26e 100755 --- a/projects/rccl/tools/scripts/topo_val.sh +++ b/projects/rccl/tools/scripts/topo_val.sh @@ -21,7 +21,7 @@ DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -for i in {0..46} +for i in {0..47} do $DIR/../topo_expl/topo_expl -m $i > "topo_m$i.log" $DIR/../TopoVisual/topo_visual.sh -i "topo_m$i.log" diff --git a/projects/rccl/tools/topo_expl/Makefile b/projects/rccl/tools/topo_expl/Makefile index fcf838d232..75bab55396 100644 --- a/projects/rccl/tools/topo_expl/Makefile +++ b/projects/rccl/tools/topo_expl/Makefile @@ -6,7 +6,7 @@ endif HIPCC = $(HIP_PATH)/bin/hipcc EXE = topo_expl -CXXFLAGS = -g -O3 -Iinclude -I../../src/include -I../../src/graph/ -DTOPO_EXPL -DENABLE_TRACE -lnuma +CXXFLAGS = -g -O3 -Iinclude -I../../src/include -I../../src/graph -I../../src -DTOPO_EXPL -DENABLE_TRACE -lnuma files = $(EXE).cpp model.cpp utils.cpp ../../src/graph/topo.cc ../../src/graph/rings.cc ../../src/graph/paths.cc ../../src/graph/trees.cc \ ../../src/graph/search.cc ../../src/graph/connect.cc ../../src/graph/tuning.cc ../../src/graph/xml.cc ../../src/misc/nvmlwrap_stub.cc diff --git a/projects/rccl/tools/topo_expl/models/topo_8p_rome_pcie.xml b/projects/rccl/tools/topo_expl/models/topo_8p_rome_pcie.xml new file mode 100644 index 0000000000..ff2521f48a --- /dev/null +++ b/projects/rccl/tools/topo_expl/models/topo_8p_rome_pcie.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/projects/rccl/tools/topo_expl/topo_expl.cpp b/projects/rccl/tools/topo_expl/topo_expl.cpp index 48709bb14a..ec10e0ed09 100644 --- a/projects/rccl/tools/topo_expl/topo_expl.cpp +++ b/projects/rccl/tools/topo_expl/topo_expl.cpp @@ -116,6 +116,7 @@ NodeModelDesc model_descs[] = { {4, "topo_4p3l_n2_1.xml", "4 nodes 8 gfx908 Rome"}, {1, "topo_8p_rome_n4_1.xml", "single node 8 gfx908 Rome NPS=4"}, {4, "topo_8p_rome_n4_1.xml", "4 nodes node 8 gfx908 Rome NPS=4"}, + {2, "topo_8p_rome_pcie.xml", "2 nodes node 8 VEGA20 PCIe"}, }; int main(int argc,char* argv[]) From d7ed44eb9a22c329148c5934514d73f259dfa631 Mon Sep 17 00:00:00 2001 From: Stanley Tsang Date: Fri, 15 Jan 2021 16:34:36 -0700 Subject: [PATCH 14/17] Adding multiprocess unit tests (#312) Adding multiprocess unit tests for collectives. To run, NCCL_COMM_ID=$HOSTNAME:12345 build/release/test/UnitTestsMultiProcess [ROCm/rccl commit: d3fa25768227c62c808a4d1f81472a926f3d77a9] --- projects/rccl/install.sh | 4 +- projects/rccl/test/CMakeLists.txt | 31 +- projects/rccl/test/CorrectnessTest.hpp | 538 +++++++++++++++++- .../rccl/test/test_AllGatherMultiProcess.cpp | 95 ++++ .../rccl/test/test_AllGatherMultiProcess.hpp | 69 +++ .../rccl/test/test_AllReduceMultiProcess.cpp | 96 ++++ .../rccl/test/test_AllReduceMultiProcess.hpp | 105 ++++ .../rccl/test/test_AllToAllMultiProcess.cpp | 95 ++++ .../rccl/test/test_AllToAllMultiProcess.hpp | 53 ++ .../rccl/test/test_BroadcastMultiProcess.cpp | 103 ++++ .../rccl/test/test_BroadcastMultiProcess.hpp | 68 +++ .../test/test_CombinedCallsMultiProcess.cpp | 112 ++++ .../test/test_CombinedCallsMultiProcess.hpp | 78 +++ .../rccl/test/test_GatherMultiProcess.cpp | 96 ++++ .../rccl/test/test_GatherMultiProcess.hpp | 59 ++ .../rccl/test/test_GroupCallsMultiProcess.cpp | 127 +++++ .../rccl/test/test_GroupCallsMultiProcess.hpp | 128 +++++ .../rccl/test/test_ReduceMultiProcess.cpp | 96 ++++ .../rccl/test/test_ReduceMultiProcess.hpp | 115 ++++ .../test/test_ReduceScatterMultiProcess.cpp | 96 ++++ .../test/test_ReduceScatterMultiProcess.hpp | 128 +++++ .../rccl/test/test_ScatterMultiProcess.cpp | 96 ++++ .../rccl/test/test_ScatterMultiProcess.hpp | 64 +++ 23 files changed, 2423 insertions(+), 29 deletions(-) create mode 100644 projects/rccl/test/test_AllGatherMultiProcess.cpp create mode 100644 projects/rccl/test/test_AllGatherMultiProcess.hpp create mode 100644 projects/rccl/test/test_AllReduceMultiProcess.cpp create mode 100644 projects/rccl/test/test_AllReduceMultiProcess.hpp create mode 100644 projects/rccl/test/test_AllToAllMultiProcess.cpp create mode 100644 projects/rccl/test/test_AllToAllMultiProcess.hpp create mode 100644 projects/rccl/test/test_BroadcastMultiProcess.cpp create mode 100644 projects/rccl/test/test_BroadcastMultiProcess.hpp create mode 100644 projects/rccl/test/test_CombinedCallsMultiProcess.cpp create mode 100644 projects/rccl/test/test_CombinedCallsMultiProcess.hpp create mode 100644 projects/rccl/test/test_GatherMultiProcess.cpp create mode 100644 projects/rccl/test/test_GatherMultiProcess.hpp create mode 100644 projects/rccl/test/test_GroupCallsMultiProcess.cpp create mode 100644 projects/rccl/test/test_GroupCallsMultiProcess.hpp create mode 100644 projects/rccl/test/test_ReduceMultiProcess.cpp create mode 100644 projects/rccl/test/test_ReduceMultiProcess.hpp create mode 100644 projects/rccl/test/test_ReduceScatterMultiProcess.cpp create mode 100644 projects/rccl/test/test_ReduceScatterMultiProcess.hpp create mode 100644 projects/rccl/test/test_ScatterMultiProcess.cpp create mode 100644 projects/rccl/test/test_ScatterMultiProcess.hpp diff --git a/projects/rccl/install.sh b/projects/rccl/install.sh index 8453ac53ed..9c95219e31 100755 --- a/projects/rccl/install.sh +++ b/projects/rccl/install.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. # ################################################# # helper functions @@ -200,8 +200,10 @@ if ($run_tests); then if (test -f "./test/UnitTests"); then if ($run_tests_all); then ./test/UnitTests + NCCL_COMM_ID=$HOSTNAME:55512 ./test/UnitTestsMultiProcess else ./test/UnitTests --gtest_filter="BroadcastCorrectnessSweep*:*float32*" + NCCL_COMM_ID=$HOSTNAME:55512 ./test/UnitTestsMultiProcess --gtest_filter="BroadcastMultiProcessCorrectnessSweep*:*float32*" fi else echo "Unit tests have not been built yet; please re-run script with -t to build unit tests." diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt index 617e5799cc..485f83311d 100644 --- a/projects/rccl/test/CMakeLists.txt +++ b/projects/rccl/test/CMakeLists.txt @@ -1,14 +1,14 @@ -# Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. cmake_minimum_required(VERSION 2.8.12) if(BUILD_TESTS) message("Going to build unit tests (Installed in /test/UnitTests)") - include_directories(${GTEST_INCLUDE_DIRS}) - + include_directories(${GTEST_INCLUDE_DIRS}) + # Collect source files for tests - set(TEST_SOURCES + set(TEST_SOURCES_SINGLE_PROCESS test_AllGather.cpp test_AllReduce.cpp test_Broadcast.cpp @@ -24,18 +24,39 @@ if(BUILD_TESTS) test_AllToAllv.cpp ) - add_executable(UnitTests ${TEST_SOURCES}) + set(TEST_SOURCES_MULTI_PROCESS + test_AllGatherMultiProcess.cpp + test_AllReduceMultiProcess.cpp + test_AllToAllMultiProcess.cpp + test_BroadcastMultiProcess.cpp + test_CombinedCallsMultiProcess.cpp + test_GatherMultiProcess.cpp + test_GroupCallsMultiProcess.cpp + test_ReduceMultiProcess.cpp + test_ReduceScatterMultiProcess.cpp + test_ScatterMultiProcess.cpp + ) + + add_executable(UnitTests ${TEST_SOURCES_SINGLE_PROCESS}) target_include_directories(UnitTests PRIVATE /opt/rocm ${GTEST_INCLUDE_DIRS}) target_link_libraries(UnitTests PRIVATE ${GTEST_BOTH_LIBRARIES}) + add_executable(UnitTestsMultiProcess ${TEST_SOURCES_MULTI_PROCESS}) + target_include_directories(UnitTestsMultiProcess PRIVATE /opt/rocm ${GTEST_INCLUDE_DIRS}) + target_link_libraries(UnitTestsMultiProcess PRIVATE ${GTEST_BOTH_LIBRARIES}) + # UnitTests using static library of rccl requires passing rccl # through -l and -L instead of command line input. if(BUILD_STATIC) add_dependencies(UnitTests rccl) target_link_libraries(UnitTests PRIVATE dl rt numa -lrccl -L${CMAKE_BINARY_DIR}) target_link_libraries(UnitTests PRIVATE amdhip64 amd_comgr hsa-runtime64::hsa-runtime64) + add_dependencies(UnitTestsMultiProcess rccl) + target_link_libraries(UnitTestsMultiProcess PRIVATE dl rt numa -lrccl -L${CMAKE_BINARY_DIR}) + target_link_libraries(UnitTestsMultiProcess PRIVATE amdhip64 amd_comgr hsa-runtime64::hsa-runtime64) else() target_link_libraries(UnitTests PRIVATE rccl) + target_link_libraries(UnitTestsMultiProcess PRIVATE rt rccl) endif() else() message("Not building unit tests") diff --git a/projects/rccl/test/CorrectnessTest.hpp b/projects/rccl/test/CorrectnessTest.hpp index 12c5fbe3f1..54f376d93e 100644 --- a/projects/rccl/test/CorrectnessTest.hpp +++ b/projects/rccl/test/CorrectnessTest.hpp @@ -1,15 +1,28 @@ /************************************************************************* - * Copyright (c) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. * * See LICENSE.txt for license information ************************************************************************/ #ifndef CORRECTNESSTEST_HPP #define CORRECTNESSTEST_HPP +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include + #include + #include "rccl.h" #include "../include/rccl_bfloat16.h" @@ -88,11 +101,12 @@ namespace CorrectnessTests return numElements * DataTypeToBytes(dataType); } - void Initialize(int const numDevices_, - size_t const numElements_, - ncclDataType_t const dataType_, - bool const inPlace_, - ncclFunc_t const func_ = ncclCollBroadcast) + // To be used in multi-process tests, in the parent process before forking children. + void InitializeRootProcess(int const numDevices_, + size_t const numElements_, + ncclDataType_t const dataType_, + bool const inPlace_, + ncclFunc_t const func_ = ncclCollBroadcast) { numDevices = numDevices_; numElements = numElements_; @@ -100,22 +114,68 @@ namespace CorrectnessTests inPlace = inPlace_; function = func_; - inputs.resize(numDevices); - outputs.resize(numDevices); - expected.resize(numDevices); + for (int i = 0; i < numDevices_; i++) + { + void* ptr = (void*)mmap(NULL, sizeof(void*), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + inputs.push_back(ptr); + } + for (int i = 0; i < numDevices_; i++) + { + void* ptr = (void*)mmap(NULL, sizeof(void*), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + outputs.push_back(ptr); + } + for (int i = 0; i < numDevices_; i++) + { + void* ptr = (void*)mmap(NULL, NumBytes(ncclOutputBuffer), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + expected.push_back(ptr); + } + } + + void Initialize(int const numDevices_, + size_t const numElements_, + ncclDataType_t const dataType_, + bool const inPlace_, + ncclFunc_t const func_ = ncclCollBroadcast, + int const multiProcessRank_ = -1) + { + numDevices = numDevices_; + numElements = numElements_; + dataType = dataType_; + inPlace = inPlace_; + function = func_; + + if (multiProcessRank_ == -1) + { + inputs.resize(numDevices); + outputs.resize(numDevices); + expected.resize(numDevices); + } // Allocate per-device memory - for (int i = 0; i < numDevices; i++) + if (multiProcessRank_ > -1) { - HIP_CALL(hipSetDevice(i)); - HIP_CALL(hipMalloc((void **)&inputs[i], NumBytes(ncclInputBuffer))); + HIP_CALL(hipSetDevice(multiProcessRank_)); + HIP_CALL(hipMalloc((void **)&inputs[multiProcessRank_], NumBytes(ncclInputBuffer))); if (inPlace) - outputs[i] = inputs[i]; + outputs[multiProcessRank_] = inputs[multiProcessRank_]; else - HIP_CALL(hipMalloc((void **)&outputs[i], NumBytes(ncclOutputBuffer))); - - expected[i] = malloc(NumBytes(ncclOutputBuffer)); + HIP_CALL(hipMalloc((void **)&outputs[multiProcessRank_], NumBytes(ncclOutputBuffer))); } + else + { + for (int i = 0; i < numDevices; i++) + { + HIP_CALL(hipSetDevice(i)); + HIP_CALL(hipMalloc((void **)&inputs[i], NumBytes(ncclInputBuffer))); + if (inPlace) + outputs[i] = inputs[i]; + else + HIP_CALL(hipMalloc((void **)&outputs[i], NumBytes(ncclOutputBuffer))); + + expected[i] = malloc(NumBytes(ncclOutputBuffer)); + } + } + } // Explicit memory release to avoid double-free from subDatasets @@ -131,11 +191,19 @@ namespace CorrectnessTests outputs.clear(); } + // Multi-process version of Release() where each process frees its own data + void Release(int rank) + { + if (!inPlace) hipFree(outputs[rank]); + hipFree(inputs[rank]); + } + // Creates a dataset by pointing to an existing dataset // Primarily to allow for testing with different starting byte-alignments void ExtractSubDataset(size_t const startElement, size_t const lastElement, - Dataset& subDataset) + Dataset& subDataset, + int const multiProcessRank = -1) { ASSERT_LE(startElement, lastElement); ASSERT_LT(lastElement, numElements); @@ -150,15 +218,182 @@ namespace CorrectnessTests subDataset.expected.resize(numDevices); size_t const byteOffset = (startElement * DataTypeToBytes(dataType)); - for (int i = 0; i < numDevices; i++) + if (multiProcessRank != -1) { - subDataset.inputs[i] = (int8_t *)inputs[i] + byteOffset; - subDataset.outputs[i] = (int8_t *)outputs[i] + byteOffset; - subDataset.expected[i] = (int8_t *)expected[i] + byteOffset; + subDataset.inputs[multiProcessRank] = (int8_t *)inputs[multiProcessRank] + byteOffset; + subDataset.outputs[multiProcessRank] = (int8_t *)outputs[multiProcessRank] + byteOffset; + subDataset.expected[multiProcessRank] = (int8_t *)expected[multiProcessRank] + byteOffset; + } + else + { + for (int i = 0; i < numDevices; i++) + { + subDataset.inputs[i] = (int8_t *)inputs[i] + byteOffset; + subDataset.outputs[i] = (int8_t *)outputs[i] + byteOffset; + subDataset.expected[i] = (int8_t *)expected[i] + byteOffset; + } } } }; + class Barrier + { + public: + Barrier(){}; + + Barrier(int rank, int numRanks, int uniqueId) + { + this->numRanks = numRanks; + std::string uniqueIdString = std::to_string(uniqueId); + mutexName = std::string("mutex").append(uniqueIdString); + turnstile1Name = std::string("turnstile1").append(uniqueIdString); + turnstile2Name = std::string("turnstile2").append(uniqueIdString); + counterName = std::string("counter").append(uniqueIdString); + tinyBarrierName = std::string("tinyBarrier").append(uniqueIdString); + + size_t smSize = sizeof(sem_t); + + if (rank == 0) + { + InitSemaphore(smSize, mutexName, 1, mutex); + InitSemaphore(smSize, turnstile1Name, 0, turnstile1); + InitSemaphore(smSize, turnstile2Name, 0, turnstile2); + OpenSharedMemoryVariable(sizeof(int), counterName, true, counter); + OpenSharedMemoryVariable(smSize, tinyBarrierName, true, tinyBarrier); + } + else + { + OpenSharedMemoryVariable(smSize, tinyBarrierName, false, tinyBarrier); + OpenSemaphore(smSize, mutexName, mutex); + OpenSemaphore(smSize, turnstile1Name, turnstile1); + OpenSemaphore(smSize, turnstile2Name, turnstile2); + OpenSharedMemoryVariable(sizeof(int), counterName, false, counter); + } + } + + void Wait() + { + Part1(); + Part2(); + } + + ~Barrier() + { + shm_unlink(mutexName.c_str()); + shm_unlink(turnstile1Name.c_str()); + shm_unlink(turnstile2Name.c_str()); + shm_unlink(counterName.c_str()); + shm_unlink(tinyBarrierName.c_str()); + } + + static void ClearShmFiles(int uniqueId) + { + std::string uniqueIdString = std::to_string(uniqueId); + std::vector names; + names.push_back(std::string("mutex").append(uniqueIdString)); + names.push_back(std::string("turnstile1").append(uniqueIdString)); + names.push_back(std::string("turnstile2").append(uniqueIdString)); + names.push_back(std::string("counter").append(uniqueIdString)); + names.push_back(std::string("tinyBarrier").append(uniqueIdString)); + + std::string shmDir = "/dev/shm/"; + for (auto it = names.begin(); it != names.end(); it++) + { + struct stat fileStatus; + std::string shmFullPath = shmDir + *it; + + // Check if shm file already exists; if so, unlink it + if (stat(shmFullPath.c_str(), &fileStatus) == 0) + { + shm_unlink(it->c_str()); + } + } + } + private: + template + void OpenSharedMemoryVariable(size_t size, std::string name, bool create, T& val) + { + int protection = PROT_READ | PROT_WRITE; + int visibility = MAP_SHARED; + int fd; + + if (create) + { + fd = shm_open(name.c_str(), O_CREAT | O_RDWR, S_IRUSR | S_IWUSR); + ftruncate(fd, size); + } + else + { + do + { + // TODO: Error checking so we don't just infinite loop + fd = shm_open(name.c_str(), O_RDWR, S_IRUSR | S_IWUSR); + } while (fd == -1 && errno == ENOENT); + } + val = (T)mmap(NULL, size, protection, visibility, fd, 0); + close(fd); + } + + void InitSemaphore(size_t size, std::string name, int semValue, sem_t*& semaphore) + { + OpenSharedMemoryVariable(size, name, true, semaphore); + sem_init(semaphore, 1, semValue); + } + + void OpenSemaphore(size_t size, std::string name, sem_t*& semaphore) + { + OpenSharedMemoryVariable(size, name, false, semaphore); + } + + void Part1() + { + sem_wait(mutex); + if (++(*counter) == numRanks) + { + sem_post_batch(turnstile1, numRanks); + } + sem_post(mutex); + sem_wait(turnstile1); + } + + void Part2() + { + sem_wait(mutex); + if (--(*counter) == 0) + { + sem_post_batch(turnstile2, numRanks); + } + sem_post(mutex); + sem_wait(turnstile2); + } + + int sem_post_batch(sem_t*& sem, int n) + { + int ret = 0; + for (int i = 0; i < n; i++) + { + ret = sem_post(sem); + if (ret != 0) break; + } + + return ret; + } + int numRanks; + + int* counter; + + sem_t* mutex; + sem_t* turnstile1; + sem_t* turnstile2; + sem_t* tinyBarrier; + + std::string mutexName; + std::string turnstile1Name; + std::string turnstile2Name; + std::string tinyBarrierName; + std::string counterName; + }; + typedef std::tuple dataTypeStrings { {ncclInt8, "int8"}, @@ -216,7 +450,6 @@ namespace CorrectnessTests }; }; protected: - // This code is called per test-tuple void SetUp() override { @@ -466,6 +699,265 @@ namespace CorrectnessTests char* savedEnv[MAX_ENV_TOKENS/2]; }; + class MultiProcessCorrectnessTest : public CorrectnessTest + { + protected: + void SetUp() override + { + // Check for NCCL_COMM_ID env variable (otherwise will not init) + if (!getenv("NCCL_COMM_ID")) + { + printf("Must set NCCL_COMM_ID prior to execution\n"); + exit(0); + } + + // Make the test tuple parameters accessible + std::tie(op, dataType, numElements, numDevices, inPlace, envVals) = GetParam(); + + envString = 0; + numTokens = 0; + if (strcmp(envVals, "")) { + // enable RCCL env vars testing + setenv("RCCL_TEST_ENV_VARS", "ENABLE", 1); + envString = strdup(envVals); + tokens[numTokens] = strtok(envString, "=, "); + numTokens++; + while (tokens[numTokens-1] != NULL && numTokens < MAX_ENV_TOKENS) + tokens[numTokens++] = strtok(NULL, "=, "); + for (int i = 0; i < numTokens/2; i++) { + char *val = getenv(tokens[i*2]); + if (val) + savedEnv[i] = strdup(val); + else + savedEnv[i] = 0; + setenv(tokens[i*2], tokens[i*2+1], 1); + fprintf(stdout, "[ ] setting environmental variable %s to %s\n", tokens[i*2], getenv(tokens[i*2])); + } + } + + comms.resize(numDevices); + streams.resize(numDevices); + } + + void TearDown() override + { + // Restore env vars after tests + for (int i = 0; i < numTokens/2; i++) { + if (savedEnv[i]) { + setenv(tokens[i*2], savedEnv[i], 1); + fprintf(stdout, "[ ] restored environmental variable %s to %s\n", tokens[i*2], getenv(tokens[i*2])); + free(savedEnv[i]); + } + else { + unsetenv(tokens[i*2]); + fprintf(stdout, "[ ] removed environmental variable %s\n", tokens[i*2]); + } + } + // Cleanup + unsetenv("RCCL_TEST_ENV_VARS"); + free(envString); + } + + void SetUpPerProcessHelper(int rank, ncclComm_t& comm, hipStream_t& stream) + { + // Check for NCCL_COMM_ID env variable (otherwise will not init) + if (!getenv("NCCL_COMM_ID")) + { + printf("Must set NCCL_COMM_ID prior to execution\n"); + exit(0); + } + + // Collect the number of available GPUs + HIP_CALL(hipGetDeviceCount(&numDevicesAvailable)); + + // Only proceed with testing if there are enough GPUs + if (numDevices > numDevicesAvailable) + { + fprintf(stdout, "[ SKIPPED ] Test requires %d devices (only %d available)\n", + numDevices, numDevicesAvailable); + + // Modify the number of devices so that tear-down doesn't occur + // This is temporary until GTEST_SKIP() becomes available + numDevices = 0; + numDevicesAvailable = -1; + return; + } + + HIP_CALL(hipSetDevice(rank)); + HIP_CALL(hipStreamCreate(&stream)); + + ncclUniqueId id; + NCCL_CALL(ncclGetUniqueId(&id)); + + ncclResult_t res; + res = ncclCommInitRank(&comm, numDevices, id, rank); // change to local comm and stream per process + + if (res != ncclSuccess) + { + printf("Test failure:%s %d '%s' numRanks:%d\n", __FILE__,__LINE__,ncclGetErrorString(res), numDevices); + ASSERT_EQ(res, hipSuccess); + } + } + + // To be called by each process individually + void SetUpPerProcess(int rank, ncclFunc_t const func, ncclComm_t& comm, hipStream_t& stream, Dataset& dataset) + { + SetUpPerProcessHelper(rank, comm, stream); + dataset.Initialize(numDevices, numElements, dataType, inPlace, func, rank); + } + + // To be called by each process/rank individually (see GroupCallsMultiProcess) + void SetUpPerProcess(int rank, std::vector const& func, ncclComm_t& comm, hipStream_t& stream, std::vector& datasets) + { + SetUpPerProcessHelper(rank, comm, stream); + + for (int i = 0; i < datasets.size(); i++) + { + datasets[i]->Initialize(numDevices, numElements, dataType, inPlace, func[i], rank); + } + } + + // Clean up per process + void TearDownPerProcess(ncclComm_t& comm, hipStream_t& stream) + { + NCCL_CALL(ncclCommDestroy(comm)); + HIP_CALL(hipStreamDestroy(stream)); + } + + void FillDatasetWithPattern(Dataset& dataset, int rank) + { + int8_t* arrayI1 = (int8_t *)malloc(dataset.NumBytes(ncclInputBuffer)); + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + rccl_bfloat16* arrayB2 = (rccl_bfloat16 *)arrayI1; + + // NOTE: Currently half-precision float tests are unsupported due to half being supported + // on GPU only and not host + + // Fills input data[i][j] with (i + j) % 6 + // - Keeping range small to reduce likelihood of overflow + // - Sticking with floating points values that are perfectly representable + + for (int j = 0; j < dataset.NumBytes(ncclInputBuffer)/DataTypeToBytes(dataset.dataType); j++) + { + int valueI = (rank + j) % 6; + float valueF = (float)valueI; + + switch (dataset.dataType) + { + case ncclInt8: arrayI1[j] = valueI; break; + case ncclUint8: arrayU1[j] = valueI; break; + case ncclInt32: arrayI4[j] = valueI; break; + case ncclUint32: arrayU4[j] = valueI; break; + case ncclInt64: arrayI8[j] = valueI; break; + case ncclUint64: arrayU8[j] = valueI; break; + case ncclFloat32: arrayF4[j] = valueF; break; + case ncclFloat64: arrayF8[j] = valueF; break; + case ncclBfloat16: arrayB2[j] = rccl_bfloat16(valueF); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + + HIP_CALL(hipSetDevice(rank)); + HIP_CALL(hipMemcpy(dataset.inputs[rank], arrayI1, dataset.NumBytes(ncclInputBuffer), hipMemcpyHostToDevice)); + + // Fills output data[i][j] with 0 (if not inplace) + if (!dataset.inPlace) + HIP_CALL(hipMemset(dataset.outputs[rank], 0, dataset.NumBytes(ncclOutputBuffer))); + + free(arrayI1); + } + + void ValidateResults(Dataset const& dataset, int rank, int root = 0) const + { + int8_t* outputI1 = (int8_t *)malloc(dataset.NumBytes(ncclOutputBuffer)); + uint8_t* outputU1 = (uint8_t *)outputI1; + int32_t* outputI4 = (int32_t *)outputI1; + uint32_t* outputU4 = (uint32_t *)outputI1; + int64_t* outputI8 = (int64_t *)outputI1; + uint64_t* outputU8 = (uint64_t *)outputI1; + float* outputF4 = (float *)outputI1; + double* outputF8 = (double *)outputI1; + rccl_bfloat16* outputB2 = (rccl_bfloat16 *)outputI1; + + bool isMatch = true; + + // Loop over each device's output and compare it to the expected output + // (Each collective operation computes its own expected results) + + // only output on root rank is valid for gather collective + if (dataset.function == ncclCollGather && rank != root) + return; + HIP_CALL(hipMemcpy(outputI1, dataset.outputs[rank], dataset.NumBytes(ncclOutputBuffer), hipMemcpyDeviceToHost)); + + int8_t* expectedI1 = (int8_t *)dataset.expected[rank]; + uint8_t* expectedU1 = (uint8_t *)expectedI1; + int32_t* expectedI4 = (int32_t *)expectedI1; + uint32_t* expectedU4 = (uint32_t *)expectedI1; + int64_t* expectedI8 = (int64_t *)expectedI1; + uint64_t* expectedU8 = (uint64_t *)expectedI1; + float* expectedF4 = (float *)expectedI1; + double* expectedF8 = (double *)expectedI1; + rccl_bfloat16* expectedB2 = (rccl_bfloat16 *)expectedI1; + + for (int j = 0; j < dataset.numElements && isMatch; j++) + { + switch (dataset.dataType) + { + case ncclInt8: isMatch &= (outputI1[j] == expectedI1[j]); break; + case ncclUint8: isMatch &= (outputU1[j] == expectedU1[j]); break; + case ncclInt32: isMatch &= (outputI4[j] == expectedI4[j]); break; + case ncclUint32: isMatch &= (outputU4[j] == expectedU4[j]); break; + case ncclInt64: isMatch &= (outputI8[j] == expectedI8[j]); break; + case ncclUint64: isMatch &= (outputU8[j] == expectedU8[j]); break; + case ncclFloat32: isMatch &= (outputF4[j] == expectedF4[j]); break; + case ncclFloat64: isMatch &= (outputF8[j] == expectedF8[j]); break; + case ncclBfloat16: isMatch &= (outputB2[j] == expectedB2[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + + if (!isMatch) + { + switch (dataset.dataType) + { + case ncclInt8: + printf("Output %d. Expected %d on device %d[%d]\n", outputI1[j], expectedI1[j], rank, j); break; + case ncclUint8: + printf("Output %u. Expected %u on device %d[%d]\n", outputU1[j], expectedU1[j], rank, j); break; + case ncclInt32: + printf("Output %d. Expected %d on device %d[%d]\n", outputI4[j], expectedI4[j], rank, j); break; + case ncclUint32: + printf("Output %u. Expected %u on device %d[%d]\n", outputU4[j], expectedU4[j], rank, j); break; + case ncclInt64: + printf("Output %ld. Expected %ld on device %d[%d]\n", outputI8[j], expectedI8[j], rank, j); break; + case ncclUint64: + printf("Output %lu. Expected %lu on device %d[%d]\n", outputU8[j], expectedU8[j], rank, j); break; + case ncclFloat32: + printf("Output %f. Expected %f on device %d[%d]\n", outputF4[j], expectedF4[j], rank, j); break; + case ncclFloat64: + printf("Output %lf. Expected %lf on device %d[%d]\n", outputF8[j], expectedF8[j], rank, j); break; + case ncclBfloat16: + printf("Output %f. Expected %f on device %d[%d]\n", (float)outputB2[j], (float)expectedB2[j], rank, j); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + ASSERT_EQ(isMatch, true); + } + }; + + std::string GenerateTestNameString(testing::TestParamInfo& info); } #endif diff --git a/projects/rccl/test/test_AllGatherMultiProcess.cpp b/projects/rccl/test/test_AllGatherMultiProcess.cpp new file mode 100644 index 0000000000..370f12cc71 --- /dev/null +++ b/projects/rccl/test/test_AllGatherMultiProcess.cpp @@ -0,0 +1,95 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_AllGatherMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(AllGatherMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollAllGather); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestAllGather(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestAllGather(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestAllGather(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestAllGather(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(AllGatherMultiProcessCorrectnessSweep, + AllGatherMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_AllGatherMultiProcess.hpp b/projects/rccl/test/test_AllGatherMultiProcess.hpp new file mode 100644 index 0000000000..b881d4b7b9 --- /dev/null +++ b/projects/rccl/test/test_AllGatherMultiProcess.hpp @@ -0,0 +1,69 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_ALLGATHER_MULTI_PROCESS_HPP +#define TEST_ALLGATHER_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class AllGatherMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, Barrier& barrier, int const rank, int const numDevices) + { + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + + HIP_CALL(hipMemcpy(static_cast(dataset.expected[0]) + rank * byteCount, (int8_t *)dataset.inputs[rank] + (rank * byteCount), + byteCount, hipMemcpyDeviceToHost)); + + barrier.Wait(); + // Rank 0 sends answer to other ranks + if (rank == 0) + { + for (int i = 0; i < dataset.numDevices; i++) + { + if (i == rank) continue; + memcpy(dataset.expected[i], dataset.expected[0], dataset.NumBytes()); + } + } + } + + void TestAllGather(int rank, Dataset& dataset) + { + // Prepare input / output / expected results + SetUpPerProcess(rank, ncclCollAllGather, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + if (numElements % numDevices != 0) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, barrier, rank, numDevices); + + size_t const byteCount = dataset.NumBytes() / numDevices; + size_t const sendCount = dataset.numElements / numDevices; + + // Launch the reduction (1 process per GPU) + ncclAllGather((int8_t *)dataset.inputs[rank] + (rank * byteCount), + dataset.outputs[rank], sendCount, + dataType, comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_AllReduceMultiProcess.cpp b/projects/rccl/test/test_AllReduceMultiProcess.cpp new file mode 100644 index 0000000000..2a5500e629 --- /dev/null +++ b/projects/rccl/test/test_AllReduceMultiProcess.cpp @@ -0,0 +1,96 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_AllReduceMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(AllReduceMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestAllReduce(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestAllReduce(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestAllReduce(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestAllReduce(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(AllReduceMultiProcessCorrectnessSweep, + AllReduceMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_AllReduceMultiProcess.hpp b/projects/rccl/test/test_AllReduceMultiProcess.hpp new file mode 100644 index 0000000000..2aa8e27087 --- /dev/null +++ b/projects/rccl/test/test_AllReduceMultiProcess.hpp @@ -0,0 +1,105 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_ALLREDUCE_MULTI_PROCESS_HPP +#define TEST_ALLREDUCE_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class AllReduceMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, Barrier& barrier, ncclRedOp_t const op, int const rank) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.inputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + barrier.Wait(); + // Allocate temporary host array to accumulate results + int8_t* resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + rccl_bfloat16* resultB2 = (rccl_bfloat16 *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + barrier.Wait(); + + // Perform reduction + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + rccl_bfloat16* arrayB2 = (rccl_bfloat16 *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + case ncclBfloat16: resultB2[j] = ReduceOp(op, resultB2[j], arrayB2[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + barrier.Wait(); + // Copy results into expected array + memcpy(dataset.expected[rank], resultI1, dataset.NumBytes()); + + free(resultI1); + } + + void TestAllReduce(int rank, Dataset& dataset) + { + SetUpPerProcess(rank, ncclCollAllReduce, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, barrier, op, rank); + + // Launch the reduction + ncclAllReduce(dataset.inputs[rank], dataset.outputs[rank], + numElements, dataType, op, comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_AllToAllMultiProcess.cpp b/projects/rccl/test/test_AllToAllMultiProcess.cpp new file mode 100644 index 0000000000..e607861833 --- /dev/null +++ b/projects/rccl/test/test_AllToAllMultiProcess.cpp @@ -0,0 +1,95 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_AllToAllMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(AllToAllMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollAllToAll); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestAllToAll(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestAllToAll(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestAllToAll(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestAllToAll(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(AllToAllMultiProcessCorrectnessSweep, + AllToAllMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false), + testing::Values("RCCL_ALLTOALL_KERNEL_DISABLE=0", "RCCL_ALLTOALL_KERNEL_DISABLE=1")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_AllToAllMultiProcess.hpp b/projects/rccl/test/test_AllToAllMultiProcess.hpp new file mode 100644 index 0000000000..8fbd56f8a0 --- /dev/null +++ b/projects/rccl/test/test_AllToAllMultiProcess.hpp @@ -0,0 +1,53 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_ALLTOALL_MULTI_PROCESS_HPP +#define TEST_ALLTOALL_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class AllToAllMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, int const rank) + { + for (int i = 0; i < dataset.numDevices; i++) + { + HIP_CALL(hipMemcpy((int8_t *)dataset.expected[i]+dataset.NumBytes()*rank, (int8_t *)dataset.inputs[rank]+dataset.NumBytes()*i, + dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + } + + void TestAllToAll(int rank, Dataset& dataset) + { + SetUpPerProcess(rank, ncclCollAllToAll, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, rank); + + // Launch the reduction + ncclAllToAll(dataset.inputs[rank], + dataset.outputs[rank], + numElements, dataType, + comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_BroadcastMultiProcess.cpp b/projects/rccl/test/test_BroadcastMultiProcess.cpp new file mode 100644 index 0000000000..1d43c6440a --- /dev/null +++ b/projects/rccl/test/test_BroadcastMultiProcess.cpp @@ -0,0 +1,103 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_BroadcastMultiProcess.hpp" + +#include +#include +#include +#include +#include +#include + +namespace CorrectnessTests +{ + TEST_P(BroadcastMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollBroadcast); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestBroadcast(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestBroadcast(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestBroadcast(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestBroadcast(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(BroadcastMultiProcessCorrectnessSweep, + BroadcastMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_BroadcastMultiProcess.hpp b/projects/rccl/test/test_BroadcastMultiProcess.hpp new file mode 100644 index 0000000000..0075faeeb7 --- /dev/null +++ b/projects/rccl/test/test_BroadcastMultiProcess.hpp @@ -0,0 +1,68 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_BROADCAST_MULTI_PROCESS_HPP +#define TEST_BROADCAST_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class BroadcastMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, int const root, int const rank) + { + // Root has the answer; share it via host memcpy's + if (rank == root) + { + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.inputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + for (int i = 0; i < dataset.numDevices; i++) + { + if (i == rank) continue; + memcpy(dataset.expected[i], dataset.expected[root], dataset.NumBytes()); + } + } + } + + void TestBroadcast(int rank, Dataset& dataset) + { + SetUpPerProcess(rank, ncclCollBroadcast, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, root, rank); + + // Launch the reduction (1 process per GPU) + ncclResult_t res = ncclBroadcast(dataset.inputs[rank], + dataset.outputs[rank], + numElements, dataType, + root, comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + // Ensure all processes have finished current iteration before proceeding + barrier.Wait(); + } + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_CombinedCallsMultiProcess.cpp b/projects/rccl/test/test_CombinedCallsMultiProcess.cpp new file mode 100644 index 0000000000..f7a4f10b4e --- /dev/null +++ b/projects/rccl/test/test_CombinedCallsMultiProcess.cpp @@ -0,0 +1,112 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_CombinedCallsMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(CombinedCallsMultiProcessCorrectnessTest, Correctness) + { + // Important: Make sure the order of ncclFunc_t's here match the order of ncclFunc_ts + // as they appear in TestCombinedCalls() + std::vector ncclFuncs; + ncclFuncs.push_back(ncclCollAllGather); + ncclFuncs.push_back(ncclCollAllReduce); + ncclFuncs.push_back(ncclCollBroadcast); + ncclFuncs.push_back(ncclCollReduce); + ncclFuncs.push_back(ncclCollReduceScatter); + + // Create multiple datasets for combined operation + std::vector datasets(ncclFuncs.size()); + for (int i = 0; i < datasets.size(); i++) + { + datasets[i] = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + datasets[i]->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclFuncs[i]); + } + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestCombinedCalls(0, datasets, ncclFuncs); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestCombinedCalls(1, datasets, ncclFuncs); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestCombinedCalls(2, datasets, ncclFuncs); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestCombinedCalls(3, datasets, ncclFuncs); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + for (int i = 0; i < datasets.size(); i++) + { + munmap(datasets[i], sizeof(Dataset)); + } + } + + INSTANTIATE_TEST_SUITE_P(CombinedCallsMultiProcessCorrectnessSweep, + CombinedCallsMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_CombinedCallsMultiProcess.hpp b/projects/rccl/test/test_CombinedCallsMultiProcess.hpp new file mode 100644 index 0000000000..e48902514f --- /dev/null +++ b/projects/rccl/test/test_CombinedCallsMultiProcess.hpp @@ -0,0 +1,78 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef TEST_COMBINEDCALLS_MULTI_PROCESS_HPP +#define TEST_COMBINEDCALLS_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +#include "test_AllGatherMultiProcess.hpp" +#include "test_AllReduceMultiProcess.hpp" +#include "test_BroadcastMultiProcess.hpp" +#include "test_ReduceMultiProcess.hpp" +#include "test_ReduceScatterMultiProcess.hpp" + +namespace CorrectnessTests +{ + class CombinedCallsMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + void TestCombinedCalls(int rank, std::vector& datasets, std::vector const& funcs) + { + SetUpPerProcess(rank, funcs, comms[rank], streams[rank], datasets); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Compute expected results for each dataset in combined + int const root = 0; + AllGatherMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[0], barrier, rank, numDevices); + AllReduceMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[1], barrier, op, rank); + BroadcastMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[2], root, rank); + ReduceMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[3], barrier, op, root, rank); + ReduceScatterMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[4], barrier, op, rank); + + size_t const byteCount = datasets[0]->NumBytes() / numDevices; + size_t const elemCount = numElements / numDevices; + + ncclAllGather((int8_t *)datasets[0]->inputs[rank] + (rank * byteCount), + datasets[0]->outputs[rank], elemCount, + dataType, comms[rank], streams[rank]); + + ncclAllReduce(datasets[1]->inputs[rank], datasets[1]->outputs[rank], + numElements, dataType, op, comms[rank], streams[rank]); + + ncclBroadcast(datasets[2]->inputs[rank], + datasets[2]->outputs[rank], + numElements, dataType, + root, comms[rank], streams[rank]); + + ncclReduce(datasets[3]->inputs[rank], + datasets[3]->outputs[rank], + numElements, dataType, op, + root, comms[rank], streams[rank]); + + ncclReduceScatter(datasets[4]->inputs[rank], + (int8_t *)datasets[4]->outputs[rank] + (rank * byteCount), + elemCount, dataType, op, + comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results for each collective in the combined + for (int i = 0; i < 5; i++) + { + ValidateResults(*datasets[i], rank); + barrier.Wait(); + datasets[i]->Release(rank); + } + } + }; +} + +#endif diff --git a/projects/rccl/test/test_GatherMultiProcess.cpp b/projects/rccl/test/test_GatherMultiProcess.cpp new file mode 100644 index 0000000000..7627a15e89 --- /dev/null +++ b/projects/rccl/test/test_GatherMultiProcess.cpp @@ -0,0 +1,96 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_GatherMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(GatherMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollGather); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestGather(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestGather(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestGather(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestGather(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(GatherMultiProcessCorrectnessSweep, + GatherMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false), + testing::Values("RCCL_ALLTOALL_KERNEL_DISABLE=0", "RCCL_ALLTOALL_KERNEL_DISABLE=1")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_GatherMultiProcess.hpp b/projects/rccl/test/test_GatherMultiProcess.hpp new file mode 100644 index 0000000000..e26c7f319b --- /dev/null +++ b/projects/rccl/test/test_GatherMultiProcess.hpp @@ -0,0 +1,59 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_GATHER_MULTI_PROCESS_HPP +#define TEST_GATHER_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class GatherMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, int const root, int const rank) + { + HIP_CALL(hipMemcpy((int8_t *)dataset.expected[root]+dataset.NumBytes()*rank, dataset.inputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + + void TestGather(int rank, Dataset& dataset) + { + SetUpPerProcess(rank, ncclCollGather, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, root, rank); + + // Launch the reduction (1 process per GPU) + ncclGather(dataset.inputs[rank], + dataset.outputs[rank], + numElements, dataType, + root, comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank, root); + + // Ensure all processes have finished current iteration before proceeding + barrier.Wait(); + } + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_GroupCallsMultiProcess.cpp b/projects/rccl/test/test_GroupCallsMultiProcess.cpp new file mode 100644 index 0000000000..ce7b7ecfa7 --- /dev/null +++ b/projects/rccl/test/test_GroupCallsMultiProcess.cpp @@ -0,0 +1,127 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#include "test_GroupCallsMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(GroupCallsMultiProcessCorrectnessTest, Correctness) + { + // Important: Make sure the order of ncclFunc_t's here match the order of ncclFunc_ts + // as they appear in TestGroupCalls() + std::vector ncclFuncs; + ncclFuncs.push_back(ncclCollAllGather); + ncclFuncs.push_back(ncclCollAllReduce); + ncclFuncs.push_back(ncclCollBroadcast); + ncclFuncs.push_back(ncclCollReduce); + ncclFuncs.push_back(ncclCollReduceScatter); + + // Create multiple datasets for combined operation + std::vector datasets(ncclFuncs.size()); + for (int i = 0; i < datasets.size(); i++) + { + datasets[i] = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + datasets[i]->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclFuncs[i]); + } + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 4) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 4) || (pid2 > 0 && pid3 > 0 && numDevices > 4)) + { + // Process 0 + std::vector ranks; + ranks.push_back(0); + ranks.push_back(1); + + TestGroupCalls(0, ranks, datasets, ncclFuncs); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 4) || (pid2 == 0 && pid3 > 0 && numDevices > 4)) + { + // Process 1 + std::vector ranks; + ranks.push_back(2); + ranks.push_back(3); + TestGroupCalls(1, ranks, datasets, ncclFuncs); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices == 8) + { + // Process 2 (available when numDevices == 8) + std::vector ranks; + ranks.push_back(4); + ranks.push_back(5); + + TestGroupCalls(2, ranks, datasets, ncclFuncs); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 8) + { + // Process 3 (available when numDevices == 8) + std::vector ranks; + ranks.push_back(6); + ranks.push_back(7); + + TestGroupCalls(3, ranks, datasets, ncclFuncs); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + for (int i = 0; i < datasets.size(); i++) + { + munmap(datasets[i], sizeof(Dataset)); + } + } + + INSTANTIATE_TEST_SUITE_P(GroupCallsMultiProcessCorrectnessSweep, + GroupCallsMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator (not used) + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_GroupCallsMultiProcess.hpp b/projects/rccl/test/test_GroupCallsMultiProcess.hpp new file mode 100644 index 0000000000..bc38688923 --- /dev/null +++ b/projects/rccl/test/test_GroupCallsMultiProcess.hpp @@ -0,0 +1,128 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef TEST_GROUPCALLS_MULTI_PROCESS_HPP +#define TEST_GROUPCALLS_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" +#include "test_AllGatherMultiProcess.hpp" +#include "test_AllReduceMultiProcess.hpp" +#include "test_BroadcastMultiProcess.hpp" +#include "test_ReduceMultiProcess.hpp" +#include "test_ReduceScatterMultiProcess.hpp" + +#include + +namespace CorrectnessTests +{ + class GroupCallsMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + void TestGroupCalls(int process, std::vector const& ranks, std::vector& datasets, std::vector const& funcs) + { + if (numDevices > numDevicesAvailable) return; + + for (int i = 0; i < ranks.size(); i++) + { + SetUpPerProcess(ranks[i], funcs, comms[ranks[i]], streams[ranks[i]], datasets); + } + + int numProcesses = numDevices / ranks.size(); + Barrier barrier(process, numProcesses, std::atoi(getenv("NCCL_COMM_ID"))); + + int const root = 0; + for (int i = 0; i < ranks.size(); i++) + { + AllGatherMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[0], barrier, numDevices, ranks[i]); + AllReduceMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[1], barrier, op, ranks[i]); + BroadcastMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[2], root, ranks[i]); + ReduceMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[3], barrier, op, root, ranks[i]); + ReduceScatterMultiProcessCorrectnessTest::ComputeExpectedResults(*datasets[4], barrier, op, ranks[i]); + } + barrier.Wait(); + + ncclGroupStart(); + + // AllGather + size_t const byteCount = datasets[0]->NumBytes() / numDevices; + size_t const elemCount = numElements / numDevices; + for (int i = 0; i < ranks.size(); i++) + { + int rank = ranks[i]; + ncclAllGather((int8_t *)datasets[0]->inputs[rank] + (rank * byteCount), + datasets[0]->outputs[rank], elemCount, + dataType, comms[rank], streams[rank]); + } + + // AllReduce + for (int i = 0; i < ranks.size(); i++) + { + int rank = ranks[i]; + ncclAllReduce(datasets[1]->inputs[rank], datasets[1]->outputs[rank], + numElements, dataType, op, comms[rank], streams[rank]); + } + + // Broadcast + for (int i = 0; i < ranks.size(); i++) + { + int rank = ranks[i]; + ncclBroadcast(datasets[2]->inputs[rank], + datasets[2]->outputs[rank], + numElements, dataType, + root, comms[rank], streams[rank]); + } + + // Reduce + for (int i = 0; i < ranks.size(); i++) + { + int rank = ranks[i]; + ncclReduce(datasets[3]->inputs[rank], + datasets[3]->outputs[rank], + numElements, dataType, op, + root, comms[rank], streams[rank]); + } + + // ReduceScatter + for (int i = 0; i < ranks.size(); i++) + { + int rank = ranks[i]; + ncclReduceScatter(datasets[4]->inputs[rank], + (int8_t *)datasets[4]->outputs[rank] + (i * byteCount), + elemCount, dataType, op, + comms[rank], streams[rank]); + } + + // Signal end of group call + ncclGroupEnd(); + + for (int i = 0; i < ranks.size(); i++) + { + HIP_CALL(hipSetDevice(ranks[i])); + HIP_CALL(hipStreamSynchronize(streams[ranks[i]])); + } + + for (int i = 0; i < funcs.size(); i++) + { + for (int j = 0; j < ranks.size(); j++) + { + ValidateResults(*datasets[i], ranks[j]); + } + barrier.Wait(); + for (int j = 0; j < ranks.size(); j++) + { + datasets[i]->Release(ranks[j]); + } + } + + for (int i = 0; i < ranks.size(); i++) + { + TearDownPerProcess(comms[ranks[i]], streams[ranks[i]]); + } + } + }; +} + +#endif diff --git a/projects/rccl/test/test_ReduceMultiProcess.cpp b/projects/rccl/test/test_ReduceMultiProcess.cpp new file mode 100644 index 0000000000..bf495d382a --- /dev/null +++ b/projects/rccl/test/test_ReduceMultiProcess.cpp @@ -0,0 +1,96 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_ReduceMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(ReduceMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollReduce); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestReduce(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestReduce(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestReduce(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestReduce(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(ReduceMultiProcessCorrectnessSweep, + ReduceMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_ReduceMultiProcess.hpp b/projects/rccl/test/test_ReduceMultiProcess.hpp new file mode 100644 index 0000000000..65977608b8 --- /dev/null +++ b/projects/rccl/test/test_ReduceMultiProcess.hpp @@ -0,0 +1,115 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_REDUCE_MULTI_PROCESS_HPP +#define TEST_REDUCE_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class ReduceMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, Barrier& barrier, ncclRedOp_t const op, int const root, int const rank) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.inputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + barrier.Wait(); + + if (rank == root) + { + // Allocate temporary host array to accumulate results + int8_t* resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + rccl_bfloat16* resultB2 = (rccl_bfloat16 *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + + // Perform reduction on the other device arrays + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + rccl_bfloat16* arrayB2 = (rccl_bfloat16 *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + case ncclBfloat16: resultB2[j] = ReduceOp(op, resultB2[j], arrayB2[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + memcpy(dataset.expected[root], resultI1, dataset.NumBytes()); + free(resultI1); + barrier.Wait(); + } + else + { + barrier.Wait(); + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.outputs[rank], dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + } + + void TestReduce(int rank, Dataset& dataset) + { + SetUpPerProcess(rank, ncclCollReduce, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, barrier, op, root, rank); + // Launch the reduction (1 process per GPU) + ncclResult_t res = ncclReduce(dataset.inputs[rank], + dataset.outputs[rank], + numElements, dataType, op, + root, comms[rank], streams[rank]); + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + // Check results + ValidateResults(dataset, rank); + // Ensure all processes have finished current iteration before proceeding + barrier.Wait(); + } + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_ReduceScatterMultiProcess.cpp b/projects/rccl/test/test_ReduceScatterMultiProcess.cpp new file mode 100644 index 0000000000..48c4b0cb9b --- /dev/null +++ b/projects/rccl/test/test_ReduceScatterMultiProcess.cpp @@ -0,0 +1,96 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_ReduceScatterMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(ReduceScatterMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollReduceScatter); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestReduceScatter(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestReduceScatter(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestReduceScatter(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestReduceScatter(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(ReduceScatterMultiProcessCorrectnessSweep, + ReduceScatterMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator + testing::Values(ncclSum, ncclProd, ncclMax, ncclMin), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(3072, 3145728), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false, true), + testing::Values("")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_ReduceScatterMultiProcess.hpp b/projects/rccl/test/test_ReduceScatterMultiProcess.hpp new file mode 100644 index 0000000000..a38ddf2148 --- /dev/null +++ b/projects/rccl/test/test_ReduceScatterMultiProcess.hpp @@ -0,0 +1,128 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_REDUCE_SCATTER_MULTI_PROCESS_HPP +#define TEST_REDUCE_SCATTER_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class ReduceScatterMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, Barrier& barrier, ncclRedOp_t const op, int const rank) + { + // Copy all inputs to expected arrays temporarily to perform reduction on host + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.inputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + barrier.Wait(); + + // Have rank 0 do the expected calculation, then send results to other processes + int8_t* resultI1; + if (rank == 0) + { + // Allocate temporary host array to accumulate results + resultI1 = (int8_t *)malloc(dataset.NumBytes()); + uint8_t* resultU1 = (uint8_t *)resultI1; + int32_t* resultI4 = (int32_t *)resultI1; + uint32_t* resultU4 = (uint32_t *)resultI1; + int64_t* resultI8 = (int64_t *)resultI1; + uint64_t* resultU8 = (uint64_t *)resultI1; + float* resultF4 = (float *)resultI1; + double* resultF8 = (double *)resultI1; + rccl_bfloat16* resultB2 = (rccl_bfloat16 *)resultI1; + + // Initialize the result with the first device's array + memcpy(resultI1, dataset.expected[0], dataset.NumBytes()); + + // Perform reduction on the other device arrays + for (int i = 1; i < dataset.numDevices; i++) + { + int8_t* arrayI1 = (int8_t *)dataset.expected[i]; + uint8_t* arrayU1 = (uint8_t *)arrayI1; + int32_t* arrayI4 = (int32_t *)arrayI1; + uint32_t* arrayU4 = (uint32_t *)arrayI1; + int64_t* arrayI8 = (int64_t *)arrayI1; + uint64_t* arrayU8 = (uint64_t *)arrayI1; + float* arrayF4 = (float *)arrayI1; + double* arrayF8 = (double *)arrayI1; + rccl_bfloat16* arrayB2 = (rccl_bfloat16 *)arrayI1; + + for (int j = 0; j < dataset.numElements; j++) + { + switch (dataset.dataType) + { + case ncclInt8: resultI1[j] = ReduceOp(op, resultI1[j], arrayI1[j]); break; + case ncclUint8: resultU1[j] = ReduceOp(op, resultU1[j], arrayU1[j]); break; + case ncclInt32: resultI4[j] = ReduceOp(op, resultI4[j], arrayI4[j]); break; + case ncclUint32: resultU4[j] = ReduceOp(op, resultU4[j], arrayU4[j]); break; + case ncclInt64: resultI8[j] = ReduceOp(op, resultI8[j], arrayI8[j]); break; + case ncclUint64: resultU8[j] = ReduceOp(op, resultU8[j], arrayU8[j]); break; + case ncclFloat32: resultF4[j] = ReduceOp(op, resultF4[j], arrayF4[j]); break; + case ncclFloat64: resultF8[j] = ReduceOp(op, resultF8[j], arrayF8[j]); break; + case ncclBfloat16: resultB2[j] = ReduceOp(op, resultB2[j], arrayB2[j]); break; + default: + fprintf(stderr, "[ERROR] Unsupported datatype\n"); + exit(0); + } + } + } + } + barrier.Wait(); + // Copy results into expected arrays + size_t const byteCount = dataset.NumBytes() / dataset.numDevices; + + HIP_CALL(hipMemcpy(dataset.expected[rank], dataset.outputs[rank], + dataset.NumBytes(), hipMemcpyDeviceToHost)); + + barrier.Wait(); + + if (rank == 0) + { + for (int i = 0; i < dataset.numDevices; i++) + memcpy((int8_t *)dataset.expected[i] + (i * byteCount), + resultI1 + (i * byteCount), byteCount); + + free(resultI1); + } + } + + void TestReduceScatter(int rank, Dataset& dataset) + { + // Prepare input / output / expected results + SetUpPerProcess(rank, ncclCollAllGather, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + if (numElements % numDevices != 0) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + ComputeExpectedResults(dataset, barrier, op, rank); + + size_t const byteCount = dataset.NumBytes() / numDevices; + size_t const recvCount = dataset.numElements / numDevices; + + // Launch the reduction (1 process per GPU) + ncclReduceScatter(dataset.inputs[rank], + (int8_t *)dataset.outputs[rank] + (rank * byteCount), + recvCount, dataType, op, + comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif diff --git a/projects/rccl/test/test_ScatterMultiProcess.cpp b/projects/rccl/test/test_ScatterMultiProcess.cpp new file mode 100644 index 0000000000..4211ace652 --- /dev/null +++ b/projects/rccl/test/test_ScatterMultiProcess.cpp @@ -0,0 +1,96 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#include "test_ScatterMultiProcess.hpp" + +namespace CorrectnessTests +{ + TEST_P(ScatterMultiProcessCorrectnessTest, Correctness) + { + Dataset* dataset = (Dataset*)mmap(NULL, sizeof(Dataset), PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANONYMOUS, -1, 0); + dataset->InitializeRootProcess(numDevices, numElements, dataType, inPlace, ncclCollScatter); + Barrier::ClearShmFiles(std::atoi(getenv("NCCL_COMM_ID"))); + + int pid1 = 0; + int pid2 = 0; + int pid3 = 0; + pid1 = fork(); + + // From this point on, ignore original process as we cannot have it create a HIP context + if (pid1 == 0) + { + pid2 = fork(); + if (numDevices > 2) + { + pid3 = fork(); + } + if ((pid2 > 0 && pid3 == 0 && numDevices == 2) || (pid2 > 0 && pid3 > 0 && numDevices > 2)) + { + // Process 0 + TestScatter(0, *dataset); + if (pid3 > 0) + { + waitpid(pid3, NULL, 0); + } + } + else if ((pid2 == 0 && pid3 == 0 && numDevices == 2) || (pid2 == 0 && pid3 > 0 && numDevices > 2)) + { + // Process 1 + TestScatter(1, *dataset); + if (numDevices > 2) + { + waitpid(pid3, NULL, 0); + } + exit(0); + } + else if (pid2 > 0 && pid3 == 0 && numDevices > 2) + { + // Process 2 (available when numDevices > 2) + TestScatter(2, *dataset); + exit(0); + } + else if (pid2 == 0 && pid3 == 0 && numDevices == 4) + { + // Process 3 (available when numDevices == 4) + TestScatter(3, *dataset); + exit(0); + } + else + { + exit(0); + } + waitpid(pid2, NULL, 0); + exit(0); + } + waitpid(pid1, NULL, 0); + munmap(dataset, sizeof(Dataset)); + } + + INSTANTIATE_TEST_SUITE_P(ScatterMultiProcessCorrectnessSweep, + ScatterMultiProcessCorrectnessTest, + testing::Combine( + // Reduction operator is not used + testing::Values(ncclSum), + // Data types + testing::Values(ncclInt8, + ncclUint8, + ncclInt32, + ncclUint32, + ncclInt64, + ncclUint64, + //ncclFloat16, + ncclFloat32, + ncclFloat64, + ncclBfloat16), + // Number of elements + testing::Values(1024, 1048576), + // Number of devices + testing::Values(2,3,4), + // In-place or not + testing::Values(false), + testing::Values("RCCL_ALLTOALL_KERNEL_DISABLE=0", "RCCL_ALLTOALL_KERNEL_DISABLE=1")), + CorrectnessTest::PrintToStringParamName()); +} // namespace diff --git a/projects/rccl/test/test_ScatterMultiProcess.hpp b/projects/rccl/test/test_ScatterMultiProcess.hpp new file mode 100644 index 0000000000..d48040246d --- /dev/null +++ b/projects/rccl/test/test_ScatterMultiProcess.hpp @@ -0,0 +1,64 @@ +/************************************************************************* + * Copyright (c) 2019-2021 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ +#ifndef TEST_SCATTER_MULTI_PROCESS_HPP +#define TEST_SCATTER_MULTI_PROCESS_HPP + +#include "CorrectnessTest.hpp" + +namespace CorrectnessTests +{ + class ScatterMultiProcessCorrectnessTest : public MultiProcessCorrectnessTest + { + public: + static void ComputeExpectedResults(Dataset& dataset, int const root, int const rank) + { + if (rank == root) + { + for (int i = 0; i < dataset.numDevices; i++) + HIP_CALL(hipMemcpy(dataset.expected[i], (int8_t *)dataset.inputs[root]+dataset.NumBytes()*i, + dataset.NumBytes(), hipMemcpyDeviceToHost)); + } + } + + void TestScatter(int rank, Dataset& dataset) + { + // Prepare input / output / expected results + SetUpPerProcess(rank, ncclCollScatter, comms[rank], streams[rank], dataset); + + if (numDevices > numDevicesAvailable) return; + + Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID"))); + + // Test each possible root + for (int root = 0; root < numDevices; root++) + { + // Prepare input / output / expected results + FillDatasetWithPattern(dataset, rank); + + ComputeExpectedResults(dataset, root, rank); + + // Launch the reduction (1 process per GPU) + ncclScatter(dataset.inputs[rank], + dataset.outputs[rank], + numElements, dataType, + root, comms[rank], streams[rank]); + + // Wait for reduction to complete + HIP_CALL(hipStreamSynchronize(streams[rank])); + + // Check results + ValidateResults(dataset, rank); + + barrier.Wait(); + } + + TearDownPerProcess(comms[rank], streams[rank]); + dataset.Release(rank); + } + }; +} + +#endif From 7f9c15b8430e054ccadc2d6cc09e7bfdfa3fd2ca Mon Sep 17 00:00:00 2001 From: Wenkai Du <43822138+wenkaidu@users.noreply.github.com> Date: Fri, 15 Jan 2021 17:48:10 -0800 Subject: [PATCH 15/17] Use less unroll for clique kernels (#313) [ROCm/rccl commit: 41e47a36e706597aa4271ca3243964b5adefde65] --- projects/rccl/src/clique/AllReduceCliqueKernel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl/src/clique/AllReduceCliqueKernel.h b/projects/rccl/src/clique/AllReduceCliqueKernel.h index 67a306d3c7..9889e9a9d5 100644 --- a/projects/rccl/src/clique/AllReduceCliqueKernel.h +++ b/projects/rccl/src/clique/AllReduceCliqueKernel.h @@ -67,7 +67,7 @@ __device__ void AllReduceCliqueSplitKernel(struct CollectiveArgs* args) } // Perform the reduction - #define ALL_REDUCE_CLIQUE_UNROLL 2 + #define ALL_REDUCE_CLIQUE_UNROLL 1 ReduceOrCopyMulti( threadIdx.x, blockDim.x, NUM_RANKS, srcs, NUM_RANKS, dsts, blockN); } From 60c74f63faf700c4e1d871a48afd7ba4274121bd Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Mon, 1 Feb 2021 09:48:09 -0700 Subject: [PATCH 16/17] [TransferBench] Restore some previous fixes - memory leak, PCIe address (#314) [ROCm/rccl commit: 62e0447e9a50553985f4f08549b9b146652daf52] --- projects/rccl/tools/TransferBench/TransferBench.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/projects/rccl/tools/TransferBench/TransferBench.cpp b/projects/rccl/tools/TransferBench/TransferBench.cpp index faed8dae44..a6ab78c46d 100644 --- a/projects/rccl/tools/TransferBench/TransferBench.cpp +++ b/projects/rccl/tools/TransferBench/TransferBench.cpp @@ -540,11 +540,12 @@ void DisplayTopology() printf(" |"); for (int j = 0; j < numGpuDevices; j++) printf(" GPU %02d |", j); - printf("\n"); + printf(" PCIe Bus ID\n"); for (int j = 0; j <= numGpuDevices; j++) printf("--------+"); - printf("\n"); + printf("-------------\n"); + char pciBusId[20]; for (int i = 0; i < numGpuDevices; i++) { printf(" GPU %02d |", i); @@ -565,7 +566,8 @@ void DisplayTopology() hopCount); } } - printf("\n"); + HIP_CALL(hipDeviceGetPCIBusId(pciBusId, 20, i)); + printf(" %s\n", pciBusId); } } @@ -865,6 +867,7 @@ void CheckOrFill(ModeType mode, int N, bool isMemset, bool isHipCall, float* ptr exit(1); } } + free(hostBuffer); } free(refBuffer); From 16d625ca27f00c55ee2b1bed8ec01bfe17847e09 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Wed, 3 Feb 2021 20:00:08 -0700 Subject: [PATCH 17/17] Tuning some clique-based kernel parameters (#315) [ROCm/rccl commit: 1990ffd76a7cc259018d8d674365b71c9125dbd0] --- projects/rccl/src/clique/CliqueManager.cc | 24 +++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/projects/rccl/src/clique/CliqueManager.cc b/projects/rccl/src/clique/CliqueManager.cc index dde0cf0085..91990d2f3b 100644 --- a/projects/rccl/src/clique/CliqueManager.cc +++ b/projects/rccl/src/clique/CliqueManager.cc @@ -46,8 +46,8 @@ int* CliqueManager::m_staticGpuBarrierMem = NULL; // Define some environment variables that affect clique-based kernels RCCL_PARAM(EnableClique, "ENABLE_CLIQUE", 0); // Opt-in environment variable for clique-based kernels -RCCL_PARAM(AllReduceCliqueByteLimit, "CLIQUE_ALLREDUCE_BYTE_LIMIT", 2097152); // Max number of bytes to use clique-based kernels for all reduce -RCCL_PARAM(AllReduceNumChannels, "CLIQUE_ALLREDUCE_NCHANNELS", 4); // Number of channels to use for all-reduce +RCCL_PARAM(AllReduceCliqueByteLimit, "CLIQUE_ALLREDUCE_BYTE_LIMIT", 16777216); // Max number of bytes to use clique-based kernels for all reduce +RCCL_PARAM(AllReduceNumChannels, "CLIQUE_ALLREDUCE_NCHANNELS", 0); // Number of channels to use for all-reduce. (0 for auto-select) RCCL_PARAM(CliqueDebug, "CLIQUE_DEBUG", 0); // Emit debug messages CliqueManager::CliqueManager(int const rank, @@ -321,7 +321,22 @@ ncclResult_t CliqueManager::GetNumChannelsToUse(ncclFunc_t const coll, *numChannelstoUse = 1; if (coll == ncclCollAllReduce) { - *numChannelstoUse = std::min((int)rcclParamAllReduceNumChannels(), totalNumChannels); + if (rcclParamAllReduceNumChannels() == 0) + { + // NOTE: These are currently based on collected data and not necessarily ideal for all hardware + int numChannels; + if (totalBytes <= 65536) numChannels = 1; + else if (totalBytes <= 262144) numChannels = 2; + else if (totalBytes <= 524288) numChannels = 4; + else if (totalBytes <= 2097152) numChannels = 8; + else numChannels = 11; + + *numChannelstoUse = std::min(numChannels, totalNumChannels); + } + else + { + *numChannelstoUse = std::min((int)rcclParamAllReduceNumChannels(), totalNumChannels); + } } return ncclSuccess; @@ -344,9 +359,6 @@ ncclResult_t CliqueManager::SetCliqueCollectiveArgs(CollectiveArgs* args) args->clique.ptrs = &m_pinnedCliquePtrs[opIndex]; args->clique.verbose = rcclParamCliqueDebug(); - // Determine number of channels to use for this collective - args->clique.nChannels = rcclParamAllReduceNumChannels(); - return ncclSuccess; }