AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix crash when PXN is enabled on some platforms (#1369)

Browse Source
This commit is contained in:
Wenkai Du
2024-10-11 09:02:59 -07:00
committed by GitHub
parent 6160603d4c
commit b55b6be0cb
6 changed files with 28 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/graph/search.cc
+3 -3
View File
@@ -1252,10 +1252,10 @@ ncclResult_t ncclTopoGetNetDev(struct ncclComm* comm, int rank, struct ncclTopoG
int pxnLevel = ncclPxnDisable(comm) == 1 ? 0 : ncclParamP2pPxnLevel();
// See whether we can use the remote rank preferred device.
if (ncclParamCrossNic() == 0 || (pxnLevel != 0)) {
// Find local NIC number close to local cudaDev
int cudaDev = comm->peerInfo[peerRank].cudaDev;
// Find local NIC number close to local nvmlDev
int nvmlDev = comm->peerInfo[peerRank].nvmlDev;
int localRank;
if (ncclTopoDevToRank(comm->topo, cudaDev, &localRank) != ncclSuccess) return ncclSuccess;
if (ncclTopoDevToRank(comm->topo, nvmlDev, &localRank) != ncclSuccess) return ncclSuccess;
NCCLCHECK(ncclTopoGetLocalNet(comm->topo, localRank, channelId, &netId, &netDev));
// Check that device exists on our node
src/graph/xml.cc
+3 -12
View File
@@ -722,14 +722,7 @@ ncclResult_t ncclTopoGetXmlFromGpu(struct ncclXmlNode* pciNode, uint32_t rocmDev
int dev = -1;
NCCLCHECK(xmlGetAttrIndex(gpuNode, "dev", &index));
if (index == -1) {
if (rocmDev == -1) {
const char* busId;
NCCLCHECK(xmlGetAttr(pciNode, "busid", &busId));
if (busId == NULL || cudaDeviceGetByPCIBusId(&dev, busId) != cudaSuccess) dev = -1;
} else {
dev = rocmDev;
}
NCCLCHECK(xmlSetAttrInt(gpuNode, "dev", dev));
NCCLCHECK(xmlSetAttrInt(gpuNode, "dev", rocmDev));
}
NCCLCHECK(xmlGetAttrInt(gpuNode, "dev", &dev));
if (dev == -1) { *gpuNodeRet = NULL; return ncclSuccess; }
@@ -934,11 +927,9 @@ ncclResult_t ncclTopoFillGpu(struct ncclXml* xml, const char* busId, struct nccl
uint32_t devIndex = 0;
static int rocmsmiInit = 0;
if (rocmsmiInit == 0) {
rocmsmiInit = (rocm_smi_init() != ncclSuccess) ? 2 : 1;
}
if (rocmsmiInit == 1) {
if (rocm_smi_getDeviceIndexByPciBusId(busId, &devIndex) != ncclSuccess) devIndex = -1;
NCCLCHECK(rocm_smi_init());
}
NCCLCHECK(rocm_smi_getDeviceIndexByPciBusId(busId, &devIndex));
NCCLCHECK(ncclTopoGetXmlFromGpu(node, devIndex, xml, gpuNode));
#else
nvmlDevice_t nvmlDev;