Rework kernel launch code (#449)

[ROCm/rccl commit: d221fb672a]

projects/rccl/src/collectives/device/common.h

@@ -348,28 +348,31 @@ __device__ void ncclKernel(ncclWorkElem first)  {
   turn = copyToShmem(&shmem.channel, channel, turn);
 
   // To optimize for latency, (only) the first operation is passed as argument.
-  struct ncclWorkElem* elems = NULL;
+  if (bid == 0 && first.active != 0)
-  bool firstLaunch = true;
+    turn = copyToShmem(&shmem.work.elems[0], &first, turn);
-  if (bid == 0 && first.funcIndex != FUNC_INDEX_P2P) elems = &first;
 
-  ncclWork *workFifoHost = channel->workFifo;
+  struct ncclWorkElem* elems = shmem.work.elems;
-  ncclWork *workFifoDev = channel->workFifoDev;
+  __syncthreads(); // publish shmem
-  int workFifoIx = channel->index;
 
-  while (1) {
+  ncclWork *workFifoHost = shmem.channel.workFifo;
-    if (elems == NULL) {
+  ncclWork *workFifoDev = shmem.channel.workFifoDev;
-      elems = shmem.work.elems;
+  int workFifoIx = shmem.channel.index;
-      __syncthreads();
+
-      copyToShmem(&shmem.work, &workFifoDev[workFifoIx]);
+  bool skipLoadWork = false, firstLaunch = true;
-      { // Check whether the last operation was aborted and make sure all threads exit
+  if (bid == 0 && first.active != 0)
-        int aborted = tid == 0 ? *shmem.comm.abortFlag : 0;
+    skipLoadWork = true;
-        if (barrierReduceAny(aborted, &abortCount)) { // publish ncclShmem->work
+
-          if (COLLTRACE && tid == 0) traceAbort(0xffff);
+  while (true) {
-          break;
+    if (!skipLoadWork) {
-        }
+      copyToShmem(&shmem.work, &workFifoDev[workFifoIx]); // turn no longer helps
-        if (tid == 0)
+      // Check whether the last operation was aborted and make sure all threads exit
-          workFifoHost[workFifoIx].elems[0].active = 0;
+      int aborted = tid == 0 ? *shmem.comm.abortFlag : 0;
+      if (barrierReduceAny(aborted, &abortCount)) { // publish shmem.work
+        if (COLLTRACE && tid == 0) traceAbort(elems->funcIndex);
+        break;
       }
+      if (tid == 0)
+        workFifoHost[workFifoIx].elems[0].active = 0;
       if (COLLTRACE && tid == 0) {
         if (firstLaunch) traceKernelLaunch(elems->funcIndex);
         if (!firstLaunch) traceCollEnd(elems->funcIndex);
@@ -379,21 +382,22 @@ __device__ void ncclKernel(ncclWorkElem first)  {
         traceKernelLaunch(elems->funcIndex);
         firstLaunch = false;
     }
+
     workFifoIx = (workFifoIx + 1)%NCCL_MAX_OPS;
     if (tid == 0)
       channel->index = workFifoIx; // write back to real channel, not shmem shadow
-    if (tid < elems->nThreads && elems->active != 0) {
+
-      if (elems->funcIndex == FnIndex) {
+    if (shmem.work.elems[0].funcIndex == FnIndex)
-        RunWork<Fn, T, RedOp, Algo, Proto>().run(&shmem.work);
+      RunWork<Fn, T, RedOp, Algo, Proto>().run(&shmem.work);
-      } else {
+    else
-        NCCL_CALL_FUNCTIONS(elems);
+      NCCL_CALL_FUNCTIONS(&elems[0]);
-      }
+
+    if (shmem.work.elems[0].active == 2) {
+      if (COLLTRACE && tid == 0) traceCollEnd(0xffff)
+      break;
     }
-    if (elems->active == 2) {
+    __syncthreads();
-      if (COLLTRACE && tid == 0) traceCollEnd(0xffff);
+    skipLoadWork = false;
-      return;
-    }
-    elems = NULL;
   }
 }