From c3b110f9e9170675122ebb0f57d7efdd836971d8 Mon Sep 17 00:00:00 2001 From: Arm Patinyasakdikul Date: Thu, 26 Jun 2025 19:49:52 -0500 Subject: [PATCH] add warning if workFIFO is not available after multiple retries. (#1772) * add warning if workFIFO is not available after multiple retries. * Update src/enqueue.cc Co-authored-by: corey-derochie-amd <161367113+corey-derochie-amd@users.noreply.github.com> --------- Co-authored-by: corey-derochie-amd <161367113+corey-derochie-amd@users.noreply.github.com> [ROCm/rccl commit: 265b1b37750e7bababf19927664bf28f3173eb36] --- projects/rccl/src/enqueue.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/projects/rccl/src/enqueue.cc b/projects/rccl/src/enqueue.cc index 22ec0abd6c..3946b32e37 100644 --- a/projects/rccl/src/enqueue.cc +++ b/projects/rccl/src/enqueue.cc @@ -1197,6 +1197,8 @@ static ncclResult_t scheduleP2pTasksToPlan( // Spin until its safe to increase comm->workFifoProduced to desiredProduced. static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredProduced) { bool hasRoom = (desiredProduced - comm->workFifoConsumedLeast) <= comm->workFifoBytes; + uint64_t count = 0; + int warned = 0; if (hasRoom) return; while (true) { // We have to poll for notifications from device. @@ -1235,6 +1237,17 @@ static void waitWorkFifoAvailable(struct ncclComm* comm, uint32_t desiredProduce hasRoom = (desiredProduced - comm->workFifoConsumedLeast) <= comm->workFifoBytes; if (hasRoom) break; sched_yield(); + + /* Warn if we get stuck waiting for workFifo. */ + count++; + if (warned == 0 && count == 100000 && comm->rank == 0) { + warned = 1; + WARN("Waiting for work FIFO to become available. " + "Work fifo exhaustion can happen in large scale/high iteration count of alltoall. " + "In order to increase work FIFO size, set NCCL_WORK_FIFO_BYTES to higher number (current: %ld).\n\n" + "RCCL continues to retry...", comm->workFifoBytes); + } + } }