AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix MSCCL work FIFO allocation with HIP graph enabled (#967)

Browse Source 
[ROCm/rccl commit: 7fc891bc8d]
This commit is contained in:
Ziyue Yang
2023-11-16 08:43:28 +08:00
committed by GitHub
parent 9e8eb41337
commit 6ce074d92d
5 changed files with 80 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rccl/src/include/msccl/msccl_setup.h
+4
View File
@@ -27,4 +27,8 @@ ncclResult_t mscclSetupKernel(const void* sendBuff, void* recvBuff, size_t count
ncclDataType_t dataType, ncclRedOp_t op, struct mscclAlgo* hostAlgo, struct mscclAlgo* devAlgo,
ncclComm_t comm, hipStream_t stream);
ncclResult_t mscclInitWorkFifoStatus(mscclWorkFifoStatus* workFifoStatus);
ncclResult_t mscclDestroyWorkFifoStatus(mscclWorkFifoStatus* workFifoStatus);
#endif
projects/rccl/src/include/msccl/msccl_struct.h
+13 -8
View File
@@ -36,8 +36,6 @@
#define MSCCL_LOCAL_COPY 6
#define MSCCL_REDUCE 7
#define MSCCL_WORK_FIFO_DEPTH (64 << 10)
struct mscclTransmission {
int16_t dependencePointer; // index to the first dependence
int16_t numDependencies; // dependencePointer+numDependencies indicate the last dependence
@@ -189,6 +187,17 @@ struct mscclThreadLocalStatus {
hipGraph_t graph;
};
struct mscclWorkFifoStatus {
uint64_t workFifoDepth;
struct mscclWork* workFifo;
uint32_t* workFifoDone;
uint32_t workFifoSent;
uint32_t workFifoSentPerThreadBlock[MSCCL_MAX_NUM_THREAD_BLOCKS];
uint32_t workFifoAckdMin;
};
typedef std::map<unsigned long long, mscclWorkFifoStatus> mscclSavedGraphWorkFifoStatus;
struct mscclStatus {
std::vector<mscclAlgoHandle_t> freeAlgoHandles;
std::map<mscclAlgoHandle_t, mscclAlgo *> hostAlgos;
@@ -214,12 +223,8 @@ struct mscclStatus {
bool graphEnabled;
bool graphFirstKernel;
bool needsProxy;
uint64_t workFifoDepth;
struct mscclWork* workFifo;
uint32_t* workFifoDone;
uint32_t workFifoSent;
uint32_t workFifoSentPerThreadBlock[MSCCL_MAX_NUM_THREAD_BLOCKS];
uint32_t workFifoAckdMin;
mscclWorkFifoStatus defaultWorkFifoStatus;
mscclSavedGraphWorkFifoStatus graphWorkFifoStatus;
};
#pragma pack(push)