From e43e1f1b3d975b6241b01cf8bbad23c4eb0946bc Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Thu, 31 Oct 2019 08:58:38 -0700 Subject: [PATCH] additional check for fine grain support in p2pCanConnect (#146) [ROCm/rccl commit: 5a502955c9a28e47098acee49a1b0ef3d79d63a8] --- projects/rccl/src/init.cc | 7 +++++-- projects/rccl/src/transport/p2p.cc | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/rccl/src/init.cc b/projects/rccl/src/init.cc index 3e9c793bbe..14825e1309 100644 --- a/projects/rccl/src/init.cc +++ b/projects/rccl/src/init.cc @@ -171,12 +171,15 @@ void parseHsaForceFineGrainVramPcie() { errno = 0; int64_t v = strtoll(str, NULL, 0); if (errno || (v != 0 && v != 1)) { - INFO(NCCL_ALL,"Invalid value %s for %s, using default %u.", str, "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); \ + INFO(NCCL_ALL,"Invalid value %s for %s, using default %u.", str, "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); } else { useFineGrainVramPcie = v; - INFO(NCCL_ALL,"%s set by environment to %u.", "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); \ + INFO(NCCL_ALL,"%s set by environment to %u.", "HSA_FORCE_FINE_GRAIN_PCIE", useFineGrainVramPcie); } } + else { + INFO(NCCL_ALL,"%s not set by environment.", "HSA_FORCE_FINE_GRAIN_PCIE"); + } } pthread_mutex_t initLock = PTHREAD_MUTEX_INITIALIZER; diff --git a/projects/rccl/src/transport/p2p.cc b/projects/rccl/src/transport/p2p.cc index 3ee052c0cf..bce7fd2370 100644 --- a/projects/rccl/src/transport/p2p.cc +++ b/projects/rccl/src/transport/p2p.cc @@ -112,6 +112,9 @@ ncclResult_t p2pCanConnect(ncclTvalue_t* ret, struct ncclPeerInfo* myInfo, struc // Do not detect topology if we're on the same GPU. Note this is not really supported. if (myInfo->cudaDev == peerCudaDev) { +#if defined(__HIP_PLATFORM_HCC__) || defined(__HCC__) || defined(__HIPCC__) + if (!useFineGrainVramPcie) return ncclSuccess; +#endif *ret = 1 + PATH_SYS; return ncclSuccess; }