Merge remote-tracking branch 'remotes/nccl/master' into rccl_2.5.6
Этот коммит содержится в:
Wenkai Du
+78
-246
@@ -5,39 +5,9 @@
|
||||
* See LICENSE.txt for license information
|
||||
************************************************************************/
|
||||
|
||||
#include "core.h"
|
||||
#include "transport.h"
|
||||
#include "nvmlwrap.h"
|
||||
#include "comm.h"
|
||||
#include "net.h"
|
||||
#include "param.h"
|
||||
#include "topo.h"
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <assert.h>
|
||||
|
||||
#define NET_MAX_IFS 16
|
||||
#define NET_MAX_GPUS 32
|
||||
|
||||
// Cache GPU-NIC distances to avoid re-computing them
|
||||
#define NET_TVALUE_UNKNOWN 0ULL
|
||||
static ncclTvalue_t ncclNetTvalues[NET_MAX_GPUS] = { NET_TVALUE_UNKNOWN };
|
||||
static int ncclNetNDev;
|
||||
|
||||
// We encode 3 bits of distance per interface into a ncclTvalue_t (64-bit)
|
||||
#define NET_BITS_PER_IF 3
|
||||
#define NET_BITS_PER_IF_MASK ((1<<NET_BITS_PER_IF)-1)
|
||||
static_assert(sizeof(ncclTvalue_t)*8 >= NET_MAX_IFS*NET_BITS_PER_IF, "NET_MAX_IFS*NET_BITS_PER_IF must fit in a ncclTvalue_t");
|
||||
static ncclTvalue_t getTvalue(short* distances, int ndev) {
|
||||
ncclTvalue_t tvalue = 0;
|
||||
for (int d=0; d<ndev; d++) {
|
||||
ncclTvalue_t score = 1 + PATH_SYS - distances[d];
|
||||
// Keep 3 bits of score info per dev
|
||||
tvalue |= ((score & NET_BITS_PER_IF_MASK)<<(NET_BITS_PER_IF*d));
|
||||
}
|
||||
return tvalue;
|
||||
}
|
||||
static int getScore(ncclTvalue_t tvalue, int dev) {
|
||||
return (tvalue >> (dev*NET_BITS_PER_IF)) & NET_BITS_PER_IF_MASK;
|
||||
}
|
||||
#include "graph.h"
|
||||
|
||||
struct netConnectInfo {
|
||||
ncclNetHandle_t netHandle;
|
||||
@@ -54,6 +24,7 @@ struct netSendResources {
|
||||
int buffSize;
|
||||
void* mhandle;
|
||||
void* llMhandle;
|
||||
void* ll128Mhandle;
|
||||
struct ncclRecvMem* devRecvMem;
|
||||
uint64_t step;
|
||||
uint64_t llLastCleaning;
|
||||
@@ -71,234 +42,62 @@ struct netRecvResources {
|
||||
int buffSize;
|
||||
void* mhandle;
|
||||
void* llMhandle;
|
||||
void* ll128Mhandle;
|
||||
struct ncclRecvMem* devRecvMem;
|
||||
uint64_t step;
|
||||
uint64_t llLastCleaning;
|
||||
uint32_t* curr_hdp_reg; // Curr GPU in ring (for rdma transport use only)
|
||||
};
|
||||
|
||||
static ncclResult_t netDistance(int cudaDev, int dev, short* distance) {
|
||||
char* cudaPath = NULL;
|
||||
char* nicPath = NULL;
|
||||
ncclResult_t err;
|
||||
NCCLCHECK(getCudaPath(cudaDev, &cudaPath));
|
||||
err = ncclNetPciPath(dev, &nicPath);
|
||||
*distance = (err != ncclSuccess || nicPath == NULL || cudaPath == NULL) ? PATH_SYS : pciDistance(nicPath, cudaPath);
|
||||
if (nicPath) free(nicPath);
|
||||
if (cudaPath) free(cudaPath);
|
||||
/* Determine if two peers can communicate with NET */
|
||||
ncclResult_t netCanConnect(int* ret, struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* info1, struct ncclPeerInfo* info2) {
|
||||
*ret = 1;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static ncclResult_t netDevices(int* ndev, short** distances) {
|
||||
NCCLCHECK(ncclNetDevices(ndev));
|
||||
if (*ndev == 0) {
|
||||
WARN("Error : Network returned 0 device");
|
||||
return ncclSystemError;
|
||||
}
|
||||
if (*ndev > NET_MAX_IFS) *ndev = NET_MAX_IFS;
|
||||
|
||||
*distances = (short*)malloc(*ndev*sizeof(short));
|
||||
if (*distances == NULL) return ncclSystemError;
|
||||
|
||||
// Find distance with current GPU
|
||||
int cudaDev, nvmlDev;
|
||||
CUDACHECK(hipGetDevice(&cudaDev));
|
||||
NCCLCHECK(getNvmlDevice(cudaDev, &nvmlDev))
|
||||
char line[1024];
|
||||
sprintf(line, "CUDA Dev %d[%d], %s NIC distance : ", cudaDev, nvmlDev, ncclNetName());
|
||||
for (int d=0; d<*ndev; d++) {
|
||||
NCCLCHECK(netDistance(cudaDev, d, *distances+d));
|
||||
sprintf(line+strlen(line), " %s", pathDists[(*distances)[d]]);
|
||||
}
|
||||
INFO(NCCL_INIT|NCCL_NET, "%s", line);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Determine if we can communicate with the peer */
|
||||
ncclResult_t netCanConnect(ncclTvalue_t* ret, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo) {
|
||||
int cudaDev;
|
||||
CUDACHECK(hipGetDevice(&cudaDev));
|
||||
ret[0] = ncclNetTvalues[cudaDev];
|
||||
if (ret[0] == NET_TVALUE_UNKNOWN) {
|
||||
if (cudaDev >= NET_MAX_GPUS) {
|
||||
WARN("CUDA device %d >= MAX %d\n", cudaDev, NET_MAX_GPUS);
|
||||
return ncclInternalError;
|
||||
}
|
||||
int nDev;
|
||||
short* distances;
|
||||
NCCLCHECK(netDevices(&nDev, &distances));
|
||||
ncclNetTvalues[cudaDev] = ret[0] = getTvalue(distances, nDev);
|
||||
ncclNetNDev = nDev;
|
||||
free(distances);
|
||||
}
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
static inline int groupBestStart(int nranks, int* groups, int group, ncclTvalue_t* values, int card, int minScore) {
|
||||
int bestRank = -1;
|
||||
int bestScore = 0;
|
||||
for (int rank=0; rank<nranks; rank++) {
|
||||
if (groups[rank] != group) continue;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
ncclTvalue_t netValue = values[rank*nranks+i];
|
||||
if (netValue != 0) {
|
||||
ncclTvalue_t score = (netValue>>(NET_BITS_PER_IF*card)) & NET_BITS_PER_IF_MASK;
|
||||
if (score >= minScore && score > bestScore) {
|
||||
bestScore = score;
|
||||
bestRank = rank;
|
||||
}
|
||||
// All other values should be the same, stop here for this rank
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return bestRank;
|
||||
}
|
||||
static inline int groupBestEnd(int nranks, int* groups, int group, int* subgroups, int startSubGroup, int startRank, ncclTvalue_t* values, int card, int minScore) {
|
||||
// For the last rank, we don't need the absolute best score, just to be within minScore.
|
||||
for (int rank=nranks-1; rank>=0; rank--) {
|
||||
if (groups[rank] != group) continue;
|
||||
if (startSubGroup != -1 && startSubGroup == subgroups[rank]) continue;
|
||||
if (startRank == rank) continue;
|
||||
for (int i=0; i<nranks; i++) {
|
||||
ncclTvalue_t netValue = values[rank*nranks+i];
|
||||
if (netValue != 0) {
|
||||
ncclTvalue_t score = (netValue>>(NET_BITS_PER_IF*card)) & NET_BITS_PER_IF_MASK;
|
||||
if (score >= minScore) {
|
||||
return rank;
|
||||
}
|
||||
// All other values should be the same, stop here for this rank
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
ncclResult_t netGetRings(int nranks, int* groups, int* subgroups, ncclTvalue_t* values, int* nringsRet, int* prev, int* next, int minScore, int* nthreads) {
|
||||
int nGroups = groups[nranks-1] + 1;
|
||||
int *cardUsed, *starts, *ends;
|
||||
NCCLCHECK(ncclCalloc(&cardUsed, NET_MAX_IFS*nGroups));
|
||||
NCCLCHECK(ncclCalloc(&starts, nGroups));
|
||||
NCCLCHECK(ncclCalloc(&ends, nGroups));
|
||||
|
||||
for (int ring = 0; ring<*nringsRet; ring++) {
|
||||
for (int group = 0; group<nGroups; group++) {
|
||||
int nranksInGroup = 0;
|
||||
int nsubGroups = 0;
|
||||
for (int rank=0; rank<nranks; rank++)
|
||||
if (groups[rank] == group) {
|
||||
nranksInGroup++;
|
||||
nsubGroups = std::max(subgroups[rank], nsubGroups);
|
||||
}
|
||||
starts[group] = ends[group] = -1;
|
||||
// Receive on the rank closest to the NIC
|
||||
for (int card=0; card<NET_MAX_IFS; card++) {
|
||||
if (cardUsed[group*NET_MAX_IFS+card] == 1) continue;
|
||||
int start = groupBestStart(nranks, groups, group, values, card, minScore);
|
||||
// Send from any rank, but best on a different subgroup and close to the NIC also.
|
||||
int end = (nranksInGroup == 1) ? start
|
||||
: groupBestEnd(nranks, groups, group, subgroups, nsubGroups ? subgroups[start] : -1, start, values, card, minScore);
|
||||
//printf("Ring %d, Minscore %d, Card %d, group %d, start = %d, end = %d\n", ring, minScore, card, group, start, end);
|
||||
if (start != -1 && end != -1) {
|
||||
cardUsed[group*NET_MAX_IFS+card] = 1;
|
||||
starts[group] = start;
|
||||
ends[group] = end;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (starts[group] == -1 || ends[group] == -1) {
|
||||
*nringsRet = ring;
|
||||
goto done;
|
||||
}
|
||||
}
|
||||
// Link groups together
|
||||
for (int group = 0; group<nGroups; group++) {
|
||||
int nextGroup = (group+1)%nGroups;
|
||||
next[ring*nranks+ends[group]] = starts[nextGroup];
|
||||
prev[ring*nranks+starts[nextGroup]] = ends[group];
|
||||
}
|
||||
}
|
||||
done:
|
||||
free(cardUsed);
|
||||
free(starts);
|
||||
free(ends);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
int getDev(int cudaDev, int ringId) {
|
||||
ncclTvalue_t tvalues = ncclNetTvalues[cudaDev];
|
||||
|
||||
int dev = 0;
|
||||
int maxScore = 0;
|
||||
for (int d=0; d<ncclNetNDev; d++) if (getScore(tvalues,d) > maxScore) maxScore = getScore(tvalues,d);
|
||||
int skip = ringId+1;
|
||||
while (skip) {
|
||||
for (int d=0; d<ncclNetNDev; d++) {
|
||||
if (getScore(tvalues, d) == maxScore) {
|
||||
skip--;
|
||||
if (skip == 0) { dev = d; goto end; }
|
||||
}
|
||||
}
|
||||
}
|
||||
end:
|
||||
return dev;
|
||||
}
|
||||
|
||||
NCCL_PARAM(NetGdrRead, "NET_GDR_READ", -2);
|
||||
NCCL_PARAM(NetGdrLevel, "NET_GDR_LEVEL", PATH_PHB);
|
||||
|
||||
static ncclResult_t netGetGdrSupport(int dev, int read, int* useGdr) {
|
||||
static ncclResult_t netGetGdrSupport(struct ncclTopoSystem* topo, int64_t busId, int netDev, int read, int* useGdr) {
|
||||
*useGdr = 0;
|
||||
|
||||
int cudaDev, nvmlDev;
|
||||
CUDACHECK(hipGetDevice(&cudaDev));
|
||||
NCCLCHECK(getNvmlDevice(cudaDev, &nvmlDev))
|
||||
|
||||
if (!hasFineGrainVramPcie()) {
|
||||
INFO(NCCL_INIT|NCCL_NET,"NET/%s : GPU Direct RDMA Disabled for GPU %d / Need Fine Grain VRAM over PCIe", ncclNetName(), cudaDev);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
if (read) { // For reads (sends) only enable under certain conditions
|
||||
int gdrReadParam = ncclParamNetGdrRead();
|
||||
if (gdrReadParam == 0) return ncclSuccess;
|
||||
if (gdrReadParam < 0) {
|
||||
int nvlink;
|
||||
NCCLCHECK(ncclNvlinkGpu(&nvlink));
|
||||
NCCLCHECK(ncclTopoHasNvlink(topo, busId, &nvlink));
|
||||
if (!nvlink) return ncclSuccess;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if we are close enough that it makes sense to enable GDR
|
||||
int netGdrLevel = ncclParamNetGdrLevel();
|
||||
short distance;
|
||||
NCCLCHECK(netDistance(cudaDev, dev, &distance));
|
||||
int distance;
|
||||
NCCLCHECK(ncclTopoNetDistance(topo, busId, netDev, &distance));
|
||||
if (distance >= netGdrLevel) {
|
||||
INFO(NCCL_NET,"NET/%s : GPU Direct RDMA Disabled for GPU %d[%d] / HCA %d (distance %d >= %d)", ncclNetName(), cudaDev, nvmlDev, dev, distance, netGdrLevel);
|
||||
INFO(NCCL_NET,"NET/%s : GPU Direct RDMA Disabled for GPU %lx / HCA %d (distance %d >= %d)", ncclNetName(), busId, netDev, distance, netGdrLevel);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Finally, check if the NIC supports it
|
||||
int flags;
|
||||
NCCLCHECK(ncclNetPtrSupport(dev, &flags));
|
||||
NCCLCHECK(ncclNetPtrSupport(netDev, &flags));
|
||||
if ((flags & NCCL_PTR_CUDA) == 0) return ncclSuccess;
|
||||
*useGdr = 1;
|
||||
INFO(NCCL_NET,"NET/%s : GPU Direct RDMA Enabled for GPU %d[%d] / HCA %d (distance %d < %d), read %d", ncclNetName(), cudaDev, nvmlDev, dev, distance, netGdrLevel, read);
|
||||
INFO(NCCL_NET,"NET/%s : GPU Direct RDMA Enabled for GPU %lx / HCA %d (distance %d < %d), read %d", ncclNetName(), busId, netDev, distance, netGdrLevel, read);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
/* Determine if we will use this transport for this peer and return connect
|
||||
* information for this peer */
|
||||
ncclResult_t netSendSetup(struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* send, int buffSize, int channelId) {
|
||||
ncclResult_t netSendSetup(struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* send, int buffSize, int channelId) {
|
||||
struct netSendResources* resources;
|
||||
NCCLCHECK(ncclCalloc(&resources, 1));
|
||||
send->transportResources = resources;
|
||||
|
||||
int cudaDev;
|
||||
CUDACHECK(hipGetDevice(&cudaDev));
|
||||
resources->netDev = getDev(cudaDev, channelId);
|
||||
NCCLCHECK(netGetGdrSupport(resources->netDev, 1, &resources->useGdr));
|
||||
NCCLCHECK(ncclTopoGetNetDev(graph, 1, channelId, &resources->netDev));
|
||||
NCCLCHECK(netGetGdrSupport(topo, myInfo->busId, resources->netDev, 1, &resources->useGdr));
|
||||
|
||||
int sendSize = sizeof(struct ncclSendMem);
|
||||
NCCLCHECK(ncclCudaHostAlloc((void**)&resources->hostSendMem, (void**)&resources->devHostSendMem, sendSize));
|
||||
@@ -310,20 +109,18 @@ ncclResult_t netSendSetup(struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peer
|
||||
NCCLCHECK(ncclCudaHostAlloc((void**)&resources->hostRecvMem, (void**)&resources->devHostRecvMem, recvSize));
|
||||
resources->buffSize = buffSize;
|
||||
|
||||
INFO(NCCL_INIT|NCCL_NET,"Ring %02d : %d -> %d [send] via NET/%s/%d%s", channelId, myInfo->rank, peerInfo->rank, ncclNetName(), resources->netDev,
|
||||
INFO(NCCL_INIT|NCCL_NET,"Ring %02d : %d[%lx] -> %d[%lx] [send] via NET/%s/%d%s", channelId, myInfo->rank, myInfo->busId, peerInfo->rank, peerInfo->busId, ncclNetName(), resources->netDev,
|
||||
resources->useGdr ? "/GDRDMA" : "");
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t netRecvSetup(struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* recv, int buffSize, int channelId) {
|
||||
ncclResult_t netRecvSetup(struct ncclTopoSystem* topo, struct ncclTopoGraph* graph, struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peerInfo, struct ncclConnect* connectInfo, struct ncclConnector* recv, int buffSize, int channelId) {
|
||||
struct netRecvResources* resources;
|
||||
NCCLCHECK(ncclCalloc(&resources, 1));
|
||||
recv->transportResources = resources;
|
||||
|
||||
int cudaDev;
|
||||
CUDACHECK(hipGetDevice(&cudaDev));
|
||||
resources->netDev = getDev(cudaDev, channelId);
|
||||
NCCLCHECK(netGetGdrSupport(resources->netDev, 0, &resources->useGdr));
|
||||
NCCLCHECK(ncclTopoGetNetDev(graph, 0, channelId, &resources->netDev));
|
||||
NCCLCHECK(netGetGdrSupport(topo, myInfo->busId, resources->netDev, 0, &resources->useGdr));
|
||||
|
||||
int sendSize = sizeof(struct ncclSendMem);
|
||||
NCCLCHECK(ncclCudaHostAlloc((void**)&resources->hostSendMem, (void**)&resources->devHostSendMem, sendSize));
|
||||
@@ -331,12 +128,11 @@ ncclResult_t netRecvSetup(struct ncclPeerInfo* myInfo, struct ncclPeerInfo* peer
|
||||
int recvSize = offsetof(struct ncclRecvMem, buff)+buffSize;
|
||||
if (resources->useGdr) {
|
||||
NCCLCHECK(ncclCudaCalloc((char**)(&resources->devRecvMem), recvSize, true));
|
||||
CUDACHECK(hipDeviceGetAttribute((int*)&resources->curr_hdp_reg, hipDeviceAttributeHdpMemFlushCntl, cudaDev));
|
||||
}
|
||||
NCCLCHECK(ncclCudaHostAlloc((void**)&resources->hostRecvMem, (void**)&resources->devHostRecvMem, recvSize));
|
||||
resources->buffSize = buffSize;
|
||||
|
||||
INFO(NCCL_INIT|NCCL_NET,"Ring %02d : %d -> %d [receive] via NET/%s/%d%s", channelId, peerInfo->rank, myInfo->rank, ncclNetName(), resources->netDev,
|
||||
INFO(NCCL_INIT|NCCL_NET,"Ring %02d : %d[%lx] -> %d[%lx] [receive] via NET/%s/%d%s", channelId, peerInfo->rank, peerInfo->busId, myInfo->rank, myInfo->busId, ncclNetName(), resources->netDev,
|
||||
resources->useGdr ? "/GDRDMA" : "");
|
||||
struct netConnectInfo* info = (struct netConnectInfo*) connectInfo;
|
||||
NCCLCHECK(ncclNetListen(resources->netDev, &info->netHandle, &resources->netListenComm));
|
||||
@@ -351,6 +147,7 @@ ncclResult_t netSendConnect(struct ncclConnect* connectInfo, struct ncclConnecto
|
||||
struct ncclRecvMem* recvMem = resources->useGdr ? resources->devRecvMem : resources->devHostRecvMem;
|
||||
send->conn.buff = recvMem->buff;
|
||||
send->conn.llBuff = resources->devHostRecvMem->llBuff;
|
||||
send->conn.ll128Buff = recvMem->ll128Buff;
|
||||
|
||||
// Head/Tail/Opcount/Fifos are always on host
|
||||
send->conn.tail = &resources->devHostRecvMem->tail;
|
||||
@@ -368,6 +165,8 @@ ncclResult_t netSendConnect(struct ncclConnect* connectInfo, struct ncclConnecto
|
||||
resources->useGdr ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->mhandle));
|
||||
NCCLCHECK(ncclNetRegMr(resources->netSendComm, resources->devHostRecvMem->llBuff,
|
||||
NCCL_LL_BUFF_SIZE, NCCL_PTR_HOST, &resources->llMhandle));
|
||||
NCCLCHECK(ncclNetRegMr(resources->netSendComm, recvMem->ll128Buff, NCCL_LL128_BUFF_SIZE,
|
||||
resources->useGdr ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->ll128Mhandle));
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -381,6 +180,7 @@ ncclResult_t netRecvConnect(struct ncclConnect* connectInfo, struct ncclConnecto
|
||||
struct ncclRecvMem* recvMem = resources->useGdr ? resources->devRecvMem : resources->devHostRecvMem;
|
||||
recv->conn.buff = recvMem->buff;
|
||||
recv->conn.llBuff = recvMem->llBuff;
|
||||
recv->conn.ll128Buff = recvMem->ll128Buff;
|
||||
|
||||
// Head/Tail/Opcount are always on host
|
||||
recv->conn.tail = &resources->devHostRecvMem->tail;
|
||||
@@ -396,6 +196,8 @@ ncclResult_t netRecvConnect(struct ncclConnect* connectInfo, struct ncclConnecto
|
||||
resources->useGdr ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->mhandle));
|
||||
NCCLCHECK(ncclNetRegMr(resources->netRecvComm, recvMem->llBuff, NCCL_LL_BUFF_SIZE,
|
||||
resources->useGdr ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->llMhandle));
|
||||
NCCLCHECK(ncclNetRegMr(resources->netRecvComm, recvMem->ll128Buff, NCCL_LL128_BUFF_SIZE,
|
||||
resources->useGdr ? NCCL_PTR_CUDA : NCCL_PTR_HOST, &resources->ll128Mhandle));
|
||||
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -405,6 +207,7 @@ ncclResult_t netSendFree(void* transportResources) {
|
||||
NCCLCHECK(ncclCudaHostFree(resources->hostSendMem));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netSendComm, resources->mhandle));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netSendComm, resources->llMhandle));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netSendComm, resources->ll128Mhandle));
|
||||
NCCLCHECK(ncclCudaHostFree(resources->hostRecvMem));
|
||||
if (resources->useGdr)
|
||||
CUDACHECK(hipFree(resources->devRecvMem));
|
||||
@@ -418,6 +221,7 @@ ncclResult_t netRecvFree(void* transportResources) {
|
||||
NCCLCHECK(ncclCudaHostFree(resources->hostSendMem));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netRecvComm, resources->mhandle));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netRecvComm, resources->llMhandle));
|
||||
NCCLCHECK(ncclNetDeregMr(resources->netRecvComm, resources->ll128Mhandle));
|
||||
NCCLCHECK(ncclCudaHostFree(resources->hostRecvMem));
|
||||
if (resources->useGdr)
|
||||
CUDACHECK(hipFree(resources->devRecvMem));
|
||||
@@ -445,7 +249,39 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) {
|
||||
if (args->tail < args->end && args->tail < args->head + NCCL_STEPS) {
|
||||
volatile int* sizesFifo = resources->hostRecvMem->sizesFifo;
|
||||
volatile uint64_t* recvTail = &resources->hostRecvMem->tail;
|
||||
if (args->llMode) {
|
||||
if (args->protocol == NCCL_PROTO_LL128) {
|
||||
int stepSize = NCCL_LL128_BUFF_SIZE/NCCL_STEPS;
|
||||
if (args->tail < *recvTail) {
|
||||
int buffSlot = args->tail%NCCL_STEPS;
|
||||
if (sizesFifo[buffSlot] != -1) {
|
||||
struct ncclRecvMem* localMem = resources->useGdr ? resources->devRecvMem : resources->hostRecvMem;
|
||||
char* localBuff = (char*)localMem->ll128Buff;
|
||||
int ready = resources->useGdr;
|
||||
if (!ready) {
|
||||
// When data is in sysmem, we need to wait until all flags are correct since the GPU only
|
||||
// called threadfence()
|
||||
uint64_t flag = args->tail + 1;
|
||||
int nFifoLines = DIVUP(sizesFifo[buffSlot], sizeof(uint64_t)*NCCL_LL128_LINEELEMS);
|
||||
volatile uint64_t* lines = (volatile uint64_t*)(localBuff+buffSlot*stepSize);
|
||||
ready = 1;
|
||||
for (int i=0; i<nFifoLines; i++) {
|
||||
if (lines[i*NCCL_LL128_LINEELEMS+NCCL_LL128_DATAELEMS] != flag) { ready = 0; break; }
|
||||
}
|
||||
}
|
||||
if (ready) {
|
||||
// Send through network
|
||||
NCCLCHECK(ncclNetIsend(resources->netSendComm, localBuff+buffSlot*stepSize, sizesFifo[buffSlot], resources->ll128Mhandle, args->requests+buffSlot));
|
||||
if (args->requests[buffSlot] != NULL) {
|
||||
sizesFifo[buffSlot] = -1;
|
||||
// Make sure size is reset to zero before we update the head.
|
||||
__sync_synchronize();
|
||||
args->tail += args->sliceSteps;
|
||||
args->idle = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (args->protocol == NCCL_PROTO_LL) {
|
||||
int buffSlot = args->tail%NCCL_STEPS;
|
||||
int size = LOAD(sizesFifo+buffSlot);
|
||||
if (size != -1) {
|
||||
@@ -471,17 +307,19 @@ ncclResult_t netSendProxy(struct ncclProxyArgs* args) {
|
||||
}
|
||||
}
|
||||
} else if (args->tail < LOAD(recvTail)) {
|
||||
struct ncclRecvMem* localMem = resources->useGdr ? resources->devRecvMem : resources->hostRecvMem;
|
||||
int stepSize = args->channel->buffSize/NCCL_STEPS;
|
||||
struct ncclRecvMem* localMem = resources->useGdr ? resources->devRecvMem : resources->hostRecvMem;
|
||||
// Send through network
|
||||
int buffSlot = args->tail%NCCL_STEPS;
|
||||
NCCLCHECK(ncclNetIsend(resources->netSendComm, localMem->buff+buffSlot*stepSize, LOAD(sizesFifo+buffSlot), resources->mhandle, args->requests+buffSlot));
|
||||
if (args->requests[buffSlot] != NULL) {
|
||||
STORE(sizesFifo+buffSlot, -1);
|
||||
// Make sure size is reset to zero before we update the head.
|
||||
__sync_synchronize();
|
||||
args->tail += args->sliceSteps;
|
||||
args->idle = 0;
|
||||
if (sizesFifo[buffSlot] != -1) {
|
||||
NCCLCHECK(ncclNetIsend(resources->netSendComm, localMem->buff+buffSlot*stepSize, LOAD(sizesFifo+buffSlot), resources->mhandle, args->requests+buffSlot));
|
||||
if (args->requests[buffSlot] != NULL) {
|
||||
STORE(sizesFifo+buffSlot, -1);
|
||||
// Make sure size is reset to zero before we update the head.
|
||||
__sync_synchronize();
|
||||
args->tail += args->sliceSteps;
|
||||
args->idle = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -520,11 +358,11 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) {
|
||||
}
|
||||
if (args->state == ncclProxyOpProgress) {
|
||||
args->idle = 1;
|
||||
int stepSize = ( args->llMode ? NCCL_LL_BUFF_SIZE : args->channel->buffSize ) / NCCL_STEPS;
|
||||
int stepSize = ( args->protocol == NCCL_PROTO_LL ? NCCL_LL_BUFF_SIZE : args->protocol == NCCL_PROTO_LL128 ? NCCL_LL128_BUFF_SIZE : args->channel->buffSize ) / NCCL_STEPS;
|
||||
if (args->head < args->end) {
|
||||
struct ncclRecvMem* localMem = resources->useGdr ? resources->devRecvMem : resources->hostRecvMem;
|
||||
char* localBuff = args->llMode ? (char*)localMem->llBuff : localMem->buff;
|
||||
void* mhandle = args->llMode ? resources->llMhandle : resources->mhandle;
|
||||
char* localBuff = args->protocol == NCCL_PROTO_LL ? (char*)localMem->llBuff : args->protocol == NCCL_PROTO_LL128 ? (char*)localMem->ll128Buff : localMem->buff;
|
||||
void* mhandle = args->protocol == NCCL_PROTO_LL ? resources->llMhandle : args->protocol == NCCL_PROTO_LL128 ? resources->ll128Mhandle : resources->mhandle;
|
||||
volatile uint64_t* sendHead = &resources->hostSendMem->head;
|
||||
if ((args->tail < args->head + NCCL_STEPS) && (args->tail < LOAD(sendHead) + NCCL_STEPS) && (args->tail < args->end)) {
|
||||
int buffSlot = args->tail%NCCL_STEPS;
|
||||
@@ -541,13 +379,8 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) {
|
||||
NCCLCHECK(ncclNetTest(args->requests[buffSlot], &done, &size));
|
||||
if (done) {
|
||||
args->head += args->sliceSteps;
|
||||
if (args->llMode == 0) {
|
||||
if (resources->useGdr) {
|
||||
ncclNetFlush(resources->netRecvComm, localBuff+buffSlot*stepSize, size, mhandle);
|
||||
// Flush local HDP register after local read-back finishes
|
||||
//STORE(resources->curr_hdp_reg, 0x1);
|
||||
//TRACE(NCCL_NET, "Flushing GPU memory via HDP %p", resources->curr_hdp_reg);
|
||||
}
|
||||
if (args->protocol == NCCL_PROTO_SIMPLE) {
|
||||
if (resources->useGdr) ncclNetFlush(resources->netRecvComm, localBuff+buffSlot*stepSize, size, mhandle);
|
||||
STORE(&resources->hostRecvMem->tail, args->head);
|
||||
}
|
||||
args->idle = 0;
|
||||
@@ -566,7 +399,6 @@ ncclResult_t netRecvProxy(struct ncclProxyArgs* args) {
|
||||
struct ncclTransport netTransport = {
|
||||
"NET",
|
||||
netCanConnect,
|
||||
netGetRings,
|
||||
{ netSendSetup, netSendConnect, netSendFree, netSendProxy },
|
||||
{ netRecvSetup, netRecvConnect, netRecvFree, netRecvProxy }
|
||||
};
|
||||
|
||||
Ссылка в новой задаче
Block a user