From bd6351894489206c1c3f64e817dcc83e2a6a41e5 Mon Sep 17 00:00:00 2001 From: Rahul Vaidya Date: Mon, 21 Jul 2025 11:50:45 -0500 Subject: [PATCH] Add datatype validation for MSCCLPP AllGather (#1816) Signed-off-by: rahulvaidya20 [ROCm/rccl commit: c28d3d26a3e64b4ca1a8858ae5d9e4f16c1ad7db] --- projects/rccl/src/misc/msccl/msccl_lifecycle.cc | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/projects/rccl/src/misc/msccl/msccl_lifecycle.cc b/projects/rccl/src/misc/msccl/msccl_lifecycle.cc index 38d2661b42..1fa16c3849 100644 --- a/projects/rccl/src/misc/msccl/msccl_lifecycle.cc +++ b/projects/rccl/src/misc/msccl/msccl_lifecycle.cc @@ -562,6 +562,18 @@ static inline bool isMscclppAllReduceSupported(ncclDataType_t dataType, ncclRedO return (op == ncclSum); } + +static inline bool isMscclppAllGatherSupported(ncclDataType_t dataType) { + switch (dataType) { +#ifdef RCCL_FLOAT8 + case ncclFloat8e4m3: + case ncclFloat8e5m2: + return false; +#endif + default: + return true; + } +} #endif ncclResult_t mscclEnqueueCheck( @@ -606,7 +618,7 @@ ncclResult_t mscclEnqueueCheck( threadLocalStatus.savedSchedulerParams.clear(); break; } - else if (func == mscclFuncAllGather && nBytes * comm->nRanks <= comm->mscclpp_threshold) { + else if (func == mscclFuncAllGather && nBytes * comm->nRanks <= comm->mscclpp_threshold && isMscclppAllGatherSupported(dataType)) { INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p graphMode %d ubr %d", "mscclpp_ncclAllGather", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream, graphMode, buffsRegistered); @@ -652,7 +664,7 @@ ncclResult_t mscclEnqueueCheck( threadLocalStatus.savedSchedulerParams.clear(); break; } - else if (func == mscclFuncAllGather && nBytes * comm->nRanks <= comm->mscclpp_threshold) { + else if (func == mscclFuncAllGather && nBytes * comm->nRanks <= comm->mscclpp_threshold && isMscclppAllGatherSupported(dataType)) { INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d root %d comm %p [nranks=%d] stream %p graphMode %d ubr %d" , "mscclpp_ncclAllGather", comm->opCount, sendBuff, recvBuff, count, dataType, op, root, comm, comm->nRanks, stream, graphMode, buffsRegistered);