Merge remote-tracking branch 'nccl/master' into develop
Este commit está contenido en:
BertanDogancay
+62
-35
@@ -219,7 +219,7 @@ ncclResult_t ncclGetLevel(int* level, const char* disableEnv, const char* levelE
|
||||
const char* str = ncclGetEnv(disableEnv);
|
||||
if (str) {
|
||||
int disable = strtol(str, NULL, 0);
|
||||
if (disable == 1) l = 0;
|
||||
if (disable == 1) l = PATH_LOC;
|
||||
if (l >= 0) INFO(NCCL_ALL, "%s set by environment to %d", disableEnv, disable);
|
||||
}
|
||||
}
|
||||
@@ -252,7 +252,18 @@ ncclResult_t ncclGetLevel(int* level, const char* disableEnv, const char* levelE
|
||||
|
||||
NCCL_PARAM(IgnoreDisabledP2p, "IGNORE_DISABLED_P2P", 0);
|
||||
|
||||
int ncclTopoUserP2pLevel = -1;
|
||||
static int ncclTopoUserP2pLevel = -1; // Initially "uninitialized". When initialized but unset, changes to -2.
|
||||
|
||||
// Gets the user-provided value of NCCL_P2P_LEVEL/NCCL_P2P_DISABLE. If the user did not provide any, the value
|
||||
// of the "level" argument is left unchanged.
|
||||
ncclResult_t ncclGetUserP2pLevel(int* level) {
|
||||
if (ncclTopoUserP2pLevel == -1)
|
||||
NCCLCHECK(ncclGetLevel(&ncclTopoUserP2pLevel, "NCCL_P2P_DISABLE", "NCCL_P2P_LEVEL"));
|
||||
if (ncclTopoUserP2pLevel != -2)
|
||||
*level = ncclTopoUserP2pLevel;
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* system, int rank1, int rank2,
|
||||
int* p2p, int *read, int* intermediateRank) {
|
||||
int mnnvl = 0;
|
||||
@@ -280,9 +291,9 @@ ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* syst
|
||||
|
||||
// Get GPUs from topology
|
||||
int g1, g2;
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank1, &g1));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank1, &g1, /*showWarn=*/true));
|
||||
struct ncclTopoNode* gpu1 = system->nodes[GPU].nodes+g1;
|
||||
if (ncclTopoRankToIndex(system, rank2, &g2) == ncclInternalError) {
|
||||
if (ncclTopoRankToIndex(system, rank2, &g2, /*showWarn=*/false) == ncclInternalError) {
|
||||
// GPU not found, we can't use p2p.
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -305,12 +316,7 @@ ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* syst
|
||||
int p2pLevel = PATH_SYS;
|
||||
|
||||
// User override
|
||||
if (ncclTopoUserP2pLevel == -1)
|
||||
NCCLCHECK(ncclGetLevel(&ncclTopoUserP2pLevel, "NCCL_P2P_DISABLE", "NCCL_P2P_LEVEL"));
|
||||
if (ncclTopoUserP2pLevel != -2) {
|
||||
p2pLevel = ncclTopoUserP2pLevel;
|
||||
goto compare;
|
||||
}
|
||||
NCCLCHECK(ncclGetUserP2pLevel(&p2pLevel));
|
||||
|
||||
// Don't use P2P through ARM CPUs
|
||||
int arch, vendor, model;
|
||||
@@ -323,7 +329,6 @@ ncclResult_t ncclTopoCheckP2p(struct ncclComm* comm, struct ncclTopoSystem* syst
|
||||
p2pLevel = PATH_PXB;
|
||||
}
|
||||
|
||||
compare:
|
||||
// Compute the PCI distance and compare with the p2pLevel.
|
||||
if (path->type <= p2pLevel) *p2p = 1;
|
||||
|
||||
@@ -393,7 +398,8 @@ NCCL_PARAM(NetGdrRead, "NET_GDR_READ", -2);
|
||||
int ncclTopoUserGdrLevel = -1;
|
||||
const char* ncclTopoGdrModeStr[ncclTopoGdrModeNum] = { "Disabled", "Default", "PCI" };
|
||||
|
||||
NCCL_PARAM(NetGdrC2c, "NET_GDR_C2C", 0);
|
||||
// On C2C platforms use GDRDMA on NICs which are connected to the CPUs
|
||||
NCCL_PARAM(NetGdrC2c, "NET_GDR_C2C", 1);
|
||||
|
||||
ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t netId, int read, enum ncclTopoGdrMode* gdrMode) {
|
||||
*gdrMode = ncclTopoGdrModeDisable;
|
||||
@@ -402,7 +408,7 @@ ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t n
|
||||
int n, g;
|
||||
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n));
|
||||
struct ncclTopoNode* net = system->nodes[NET].nodes+n;
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true));
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
|
||||
// Check that both the NIC and GPUs support it
|
||||
@@ -459,29 +465,29 @@ ncclResult_t ncclTopoCheckGdr(struct ncclTopoSystem* system, int rank, int64_t n
|
||||
// In case of PXN, use the intermediate GPU distance instead
|
||||
int proxyRank;
|
||||
NCCLCHECK(ncclTopoGetIntermediateRank(system, gpu->gpu.rank, netId, &proxyRank));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, proxyRank, &g));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, proxyRank, &g, /*showWarn=*/true));
|
||||
gpu = system->nodes[GPU].nodes+g;
|
||||
distance = gpu->paths[NET][n].type;
|
||||
}
|
||||
|
||||
int c;
|
||||
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
|
||||
if (ncclParamNetGdrC2c() && distance == PATH_PHB && gpu->paths[CPU][c].type == PATH_C2C) {
|
||||
// On C2C platforms we can still use GDRDMA on NICs connected to the CPUs
|
||||
INFO(NCCL_NET, "GPU %d / HCA %lx connected to CPU %d via C2C link", rank, netId, c);
|
||||
// On C2C platforms we can still use GDRDMA on NICs connected to the CPUs
|
||||
if (ncclParamNetGdrC2c() && distance == PATH_P2C) {
|
||||
INFO(NCCL_GRAPH | NCCL_NET, "GPU %d / HCA %lx connected via C2C link", rank, netId);
|
||||
distance = PATH_C2C;
|
||||
}
|
||||
|
||||
if (distance > netGdrLevel) {
|
||||
INFO(NCCL_NET,"GPU Direct RDMA Disabled for GPU %d / HCA %lx (distance %d > %d)", rank, netId, distance, netGdrLevel);
|
||||
INFO(NCCL_GRAPH|NCCL_NET,"GPU Direct RDMA Disabled for GPU %d / HCA %lx (distance %d > %d)", rank, netId, distance, netGdrLevel);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
// Force PCIe mapping if path goes through PCI on a C2C system
|
||||
int c;
|
||||
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
|
||||
if (gpu->paths[CPU][c].type == PATH_C2C && distance != PATH_C2C) *gdrMode = ncclTopoGdrModePci;
|
||||
else *gdrMode = ncclTopoGdrModeDefault;
|
||||
|
||||
INFO(NCCL_NET,"GPU Direct RDMA Enabled for GPU %d / HCA %lx (distance %d <= %d), read %d mode %s", rank, netId, distance, netGdrLevel, read, ncclTopoGdrModeStr[*gdrMode]);
|
||||
INFO(NCCL_GRAPH|NCCL_NET,"GPU Direct RDMA Enabled for GPU %d / HCA %lx (distance %d <= %d), read %d mode %s", rank, netId, distance, netGdrLevel, read, ncclTopoGdrModeStr[*gdrMode]);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -516,7 +522,7 @@ ncclResult_t ncclTopoNeedFlush(struct ncclComm* comm, int64_t netId, int netDev,
|
||||
if (props.forceFlush == 1 || ncclParamNetForceFlush()) return ncclSuccess;
|
||||
int g;
|
||||
struct ncclTopoSystem* system = comm->topo;
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true));
|
||||
#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
|
||||
*flush = 1;
|
||||
#else
|
||||
@@ -546,8 +552,8 @@ ncclResult_t ncclTopoCheckNet(struct ncclTopoSystem* system, int rank1, int rank
|
||||
*net = 1;
|
||||
// First check the current GPU-to-GPU speed.
|
||||
int g1, g2;
|
||||
if (ncclTopoRankToIndex(system, rank1, &g1) != ncclSuccess ||
|
||||
ncclTopoRankToIndex(system, rank2, &g2) != ncclSuccess) {
|
||||
if (ncclTopoRankToIndex(system, rank1, &g1, /*showWarn=*/false) != ncclSuccess ||
|
||||
ncclTopoRankToIndex(system, rank2, &g2, /*showWarn=*/false) != ncclSuccess) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -573,7 +579,7 @@ ncclResult_t ncclTopoGetIntermediateRank(struct ncclTopoSystem* system, int rank
|
||||
// Get GPU and NET
|
||||
int n, g;
|
||||
NCCLCHECK(ncclTopoIdToIndex(system, NET, netId, &n));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g));
|
||||
NCCLCHECK(ncclTopoRankToIndex(system, rank, &g, /*showWarn=*/true));
|
||||
struct ncclTopoNode* gpu = system->nodes[GPU].nodes+g;
|
||||
struct ncclTopoLinkList* path = gpu->paths[NET]+n;
|
||||
if (path->type == PATH_PXN) {
|
||||
@@ -666,6 +672,8 @@ static bool rcclPathOverride(struct ncclTopoSystem* system, uint64_t distance) {
|
||||
}
|
||||
}
|
||||
|
||||
NCCL_PARAM(PxnC2c, "PXN_C2C", 0);
|
||||
|
||||
ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm* comm) {
|
||||
// Precompute paths between GPUs/NICs.
|
||||
|
||||
@@ -724,6 +732,20 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
|
||||
}
|
||||
}
|
||||
}
|
||||
// update the GPU -> NIC path in the case of C2C + PHB
|
||||
for (int n = 0; n < system->nodes[NET].count; n++) {
|
||||
struct ncclTopoNode* netNode = system->nodes[NET].nodes + n;
|
||||
for (int g = 0; g < system->nodes[GPU].count; g++) {
|
||||
struct ncclTopoNode* gpuNode = system->nodes[GPU].nodes + g;
|
||||
int c;
|
||||
NCCLCHECK(ncclGetLocalCpu(system, g, &c));
|
||||
if (c == -1) continue;
|
||||
if (gpuNode->paths[NET][n].type == PATH_PHB && gpuNode->paths[CPU][c].type == PATH_C2C) {
|
||||
gpuNode->paths[NET][n].type = PATH_P2C;
|
||||
netNode->paths[GPU][g].type = PATH_P2C;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Special handling of gfx94x and gfx950
|
||||
|
||||
@@ -759,15 +781,20 @@ ncclResult_t ncclTopoComputePaths(struct ncclTopoSystem* system, struct ncclComm
|
||||
// PXN = PCI + NVLink.
|
||||
struct ncclTopoNode* peerNode = system->nodes[GPU].nodes+localGpuIndex;
|
||||
// Only use PXN for NIC n if remote GPU p ...
|
||||
if (peerNode->paths[NET][n].type <= PATH_PXB && // Is connected to the NIC through PCI
|
||||
peerNode->paths[GPU][g].type <= PATH_NVL && // Is connected to us through NVLink
|
||||
NCCL_TOPO_ID_SYSTEM_ID(peerNode->id) == NCCL_TOPO_ID_SYSTEM_ID(gpu->id) && // Is on the same node as us
|
||||
(peerNode->paths[NET][n].bw > gpu->paths[NET][n].bw || // Has either higher BW to that NIC
|
||||
gpu->paths[NET][n].type > PATH_PXB)) // or avoids going through a CPU
|
||||
// We can use that GPU as relay to communicate with that NIC.
|
||||
// Only enabling it in the GPU->NIC direction for now to favor
|
||||
// receiving locally and sending remotely (consistent with net.cc)
|
||||
NCCLCHECK(addInterStep(system, GPU, localGpuIndex, GPU, g, NET, n));
|
||||
if (/* (1) is either connected to the NIC with PXB*/
|
||||
(peerNode->paths[NET][n].type <= PATH_PXB ||
|
||||
/* or with P2C and PxN over C2C is enabled */
|
||||
(ncclParamPxnC2c() && peerNode->paths[NET][n].type == PATH_P2C)) &&
|
||||
/* and (2) is connected to us through NVLink */
|
||||
peerNode->paths[GPU][g].type <= PATH_NVL &&
|
||||
/* and (3) is on the same node as us */
|
||||
NCCL_TOPO_ID_SYSTEM_ID(peerNode->id) == NCCL_TOPO_ID_SYSTEM_ID(gpu->id) &&
|
||||
/* and (4) has either higher bw to that NIC or avoid going through the CPU*/
|
||||
(peerNode->paths[NET][n].bw > gpu->paths[NET][n].bw || gpu->paths[NET][n].type > PATH_PXB))
|
||||
// We can use that GPU as relay to communicate with that NIC.
|
||||
// Only enabling it in the GPU->NIC direction for now to favor
|
||||
// receiving locally and sending remotely (consistent with net.cc)
|
||||
NCCLCHECK(addInterStep(system, GPU, localGpuIndex, GPU, g, NET, n));
|
||||
}
|
||||
}
|
||||
if (gpu->paths[NET][n].type < PATH_PHB) {
|
||||
@@ -904,7 +931,7 @@ static ncclResult_t ncclTopoGetNchannels(struct ncclComm* comm, int g /*local gp
|
||||
int peer;
|
||||
struct ncclTopoSystem* system = comm->topo;
|
||||
struct ncclTopoLinkList* path = NULL;
|
||||
if (ncclTopoRankToIndex(system, peerRank, &peer) == ncclSuccess) {
|
||||
if (ncclTopoRankToIndex(system, peerRank, &peer, /*showWarn=*/false) == ncclSuccess) {
|
||||
// Same rank
|
||||
if (g == peer) {
|
||||
*nChannels = -1;
|
||||
|
||||
Referencia en una nueva incidencia
Block a user