diff --git a/projects/rccl/src/enqueue.cc b/projects/rccl/src/enqueue.cc index 22ec0abd6c..3946b32e37 100644 --- a/projects/rccl/src/enqueue.cc +++ b/projects/rccl/src/enqueue.cc @@ -1197,6 +1197,8 @@ static ncclResult_t scheduleP2pTasksToPlan( // Spin until its safe to increase comm->workFifoProduced to desiredProduced. static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredProduced) { bool hasRoom = (desiredProduced - comm->workFifoConsumedLeast) <= comm->workFifoBytes; + uint64_t count = 0; + int warned = 0; if (hasRoom) return; while (true) { // We have to poll for notifications from device. @@ -1235,6 +1237,17 @@ static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredProduce hasRoom = (desiredProduced - comm->workFifoConsumedLeast) <= comm->workFifoBytes; if (hasRoom) break; sched_yield(); + + /* Warn if we get stuck waiting for workFifo. */ + count++; + if (warned == 0 && count == 100000 && comm->rank == 0) { + warned = 1; + WARN("Waiting for work FIFO to become available. " + "Work fifo exhaustion can happen in large scale/high iteration count of alltoall. " + "In order to increase work FIFO size, set NCCL_WORK_FIFO_BYTES to higher number (current: %ld).\n\n" + "RCCL continues to retry...", comm->workFifoBytes); + } + } }