AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move collective trace to HBM and fix log issue (#1542)

Browse Source 
[ROCm/rccl commit: f5b15f27a9]
This commit is contained in:
Wenkai Du
2025-02-11 11:40:14 -08:00
committed by GitHub
parent 689725fb9e
commit 7fdbcdfdec
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/init.cc
+5 -5
View File
@@ -254,7 +254,7 @@ void *ncclCommThreadMain(void *arg) {
} else { } else {
if (type & ncclCollTraceP2pElemType) if (type & ncclCollTraceP2pElemType)
sprintf(line, "## [%012.6f] [%02d:%02d:%02d] %06x-%06x", (double)(td->timeStamp)/vega_gpu_rtc_freq, comm->rank, td->bid, td->channelId, td->p2pOpCount[0], td->p2pOpCount[1]); sprintf(line, "## [%012.6f] [%02d:%02d:%02d] %06x-%06x", (double)(td->timeStamp)/vega_gpu_rtc_freq, comm->rank, td->bid, td->channelId, td->p2pOpCount[0], td->p2pOpCount[1]);
else if (type & ncclCollTraceCollElemType) else
sprintf(line, "## [%012.6f] [%02d:%02d:%02d] %06lx", (double)(td->timeStamp)/vega_gpu_rtc_freq, comm->rank, td->bid, td->channelId, td->opCount); sprintf(line, "## [%012.6f] [%02d:%02d:%02d] %06lx", (double)(td->timeStamp)/vega_gpu_rtc_freq, comm->rank, td->bid, td->channelId, td->opCount);
offset = strlen(line); offset = strlen(line);
if (type == ncclCollTraceCollElemType) { if (type == ncclCollTraceCollElemType) {
@@ -405,8 +405,8 @@ static ncclResult_t commFree(ncclComm_t comm) {
else else
ncclCommThreadMain((void *)comm); ncclCommThreadMain((void *)comm);
} }
NCCLCHECK(ncclCudaHostFree((void *)comm->collTrace)); NCCLCHECK(ncclCudaFree((void *)comm->collTrace));
NCCLCHECK(ncclCudaHostFree((void *)comm->collTraceTail)); NCCLCHECK(ncclCudaFree((void *)comm->collTraceTail));
#endif #endif
free(comm->peerInfo); free(comm->peerInfo);
@@ -591,8 +591,8 @@ static ncclResult_t commAlloc(struct ncclComm* comm, struct ncclComm* parent, in
comm->dmaBufSupport = (dmaBufSupported(comm) == ncclSuccess) ? true : false; comm->dmaBufSupport = (dmaBufSupported(comm) == ncclSuccess) ? true : false;
#ifdef ENABLE_COLLTRACE #ifdef ENABLE_COLLTRACE
NCCLCHECK(ncclCudaHostCalloc(&comm->collTraceTail, MAXCHANNELS)); NCCLCHECK(ncclCudaCalloc(&comm->collTraceTail, MAXCHANNELS));
NCCLCHECK(ncclCudaHostCalloc(&comm->collTrace, COLLTRACE_NUM_ITEMS*MAXCHANNELS)); NCCLCHECK(ncclCudaCalloc(&comm->collTrace, COLLTRACE_NUM_ITEMS*MAXCHANNELS));
comm->collTraceExit = 0; comm->collTraceExit = 0;
comm->collTraceEnabled = false; // we can enable colltrace without starting a thread comm->collTraceEnabled = false; // we can enable colltrace without starting a thread
if ((ncclDebugLevel >= NCCL_LOG_INFO) && rcclParamKernelCollTraceEnable()) { if ((ncclDebugLevel >= NCCL_LOG_INFO) && rcclParamKernelCollTraceEnable()) {