diff --git a/projects/rccl/src/include/transport.h b/projects/rccl/src/include/transport.h index ed98f8caff..bc9b7779d8 100644 --- a/projects/rccl/src/include/transport.h +++ b/projects/rccl/src/include/transport.h @@ -110,8 +110,4 @@ inline void transportProxyIdle(int idle) { sched_yield(); } -// Function to get GPU's HDP_MEM_FLUSH_ADDR: HDP Memory Coherency Flush Control -// This register allows software to explicitly initiate a flush read to HDP memory -ncclResult_t getGpuHdpReg(int cudaDev, uint32_t** hdp); - #endif diff --git a/projects/rccl/src/transport.cu b/projects/rccl/src/transport.cu index 7d2003f267..240453cb3d 100644 --- a/projects/rccl/src/transport.cu +++ b/projects/rccl/src/transport.cu @@ -188,56 +188,3 @@ ncclResult_t transportDestroyProxy(struct ncclConnector* connector) { } return ncclSuccess; } - - -ncclResult_t getGpuHdpReg(int cudaDev, uint32_t** hdp) { - auto convert_bdf = [](const char *busId) { - char bdf[9]; - strncpy(bdf, busId, 4); - strncpy(bdf+4, busId+5, 2); - strncpy(bdf+6, busId+8, 2); - bdf[8] = '\0'; - uint16_t id = (uint16_t)strtol(bdf, NULL, 16); - return id; - }; - - union find_agent_args { - hsa_agent_t agent; - uint16_t id; - } args; - - const auto& find_agent = [](hsa_agent_t agent, void* arg) { - uint16_t id = ((union find_agent_args *)arg)->id; - hsa_device_type_t type; - hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, (void*)&type); - if(type == HSA_DEVICE_TYPE_GPU) { - uint16_t bdf_id = 1; - hsa_agent_get_info(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_BDFID, &bdf_id); - if(bdf_id == id) { - ((union find_agent_args *)arg)->agent=agent; - return HSA_STATUS_INFO_BREAK; - } - } - return HSA_STATUS_SUCCESS; - }; - -#define PCI_BUS_ID_BUFFER_SIZE 16 - char busId[PCI_BUS_ID_BUFFER_SIZE]; - *hdp = NULL; - CUDACHECK(hipDeviceGetPCIBusId(busId, PCI_BUS_ID_BUFFER_SIZE, cudaDev)); - args.id = convert_bdf(busId); - hsa_status_t err = hsa_iterate_agents(find_agent, (void*)&args); - if (err != HSA_STATUS_INFO_BREAK) { - WARN("failed to get locate HSA agent for GPU %d", cudaDev); - return ncclSystemError; - } - hsa_amd_hdp_flush_t hdpinfo; - err = hsa_agent_get_info(args.agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_HDP_FLUSH, &hdpinfo); - if ((err != HSA_STATUS_SUCCESS) && (err != HSA_STATUS_INFO_BREAK)) { - WARN("failed to get HSA_AMD_AGENT_INFO_HDP_FLUSH for GPU %d", cudaDev); - return ncclSystemError; - } - *hdp = hdpinfo.HDP_MEM_FLUSH_CNTL; - return ncclSuccess; -#undef PCI_BUS_ID_BUFFER_SIZE -} diff --git a/projects/rccl/src/transport/net.cu b/projects/rccl/src/transport/net.cu index b500cdb126..1c09c91378 100644 --- a/projects/rccl/src/transport/net.cu +++ b/projects/rccl/src/transport/net.cu @@ -294,7 +294,7 @@ ncclResult_t netRecvSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo // Collect HDR register for local GPU to initiate flush after receive int cudaDev; hipGetDevice(&cudaDev); - NCCLCHECK(getGpuHdpReg(cudaDev, &ring->curr_hdp_reg)); + CUDACHECK(hipDeviceGetAttribute((int*)&ring->curr_hdp_reg, hipDeviceAttributeHdpMemFlushCntl, cudaDev)); } int sendSize = sizeof(struct ncclSendMem); diff --git a/projects/rccl/src/transport/p2p.cu b/projects/rccl/src/transport/p2p.cu index 2e70f5b80c..f5ea1f1cbb 100644 --- a/projects/rccl/src/transport/p2p.cu +++ b/projects/rccl/src/transport/p2p.cu @@ -496,7 +496,7 @@ ncclResult_t p2pSendSetup(ncclTinfo_t* myOpaqueInfo, ncclTinfo_t* peerOpaqueInfo return ncclInternalError; } if (linktype != HSA_AMD_LINK_INFO_TYPE_XGMI) { - NCCLCHECK(getGpuHdpReg(peerInfo->cudaDev, &ring->next_hdp_reg)); + CUDACHECK(hipDeviceGetAttribute((int*)&ring->next_hdp_reg, hipDeviceAttributeHdpMemFlushCntl,peerInfo->cudaDev)); TRACE(NCCL_INIT|NCCL_P2P,"Ring %02d : %d -> %d HDP %p", ring->id, myInfo->rank, peerInfo->rank, ring->next_hdp_reg); } if (myInfo->pidHash == peerInfo->pidHash) {