Support fused all reduce and elementwise operations (#1729)

* Support fused all reduce and elementwise operations

Add additional "acc" parameter to RCCL Replayer logs

Add flag which indicates availability of new API

* Fix Recorder json parsing

* Remove unreachable code

* Remove extra acc pointer check

* .

* Revert "[DEVICE] Adding ability to choose unroll factor at runtime (#1734)"

This reverts commit 4cadf3597c.

* Use noinline to reduce kernels linking time

* Don't use noinline for gfx942 and gfx950 to avoid perf regression

---------

Co-authored-by: AtlantaPepsi <timhu102@amd.com>
Co-authored-by: BertanDogancay <bertan.dogancay@gmail.com>

[ROCm/rccl commit: 9a4213356d]

projects/rccl/tools/RcclReplayer/rcclReplayer.cpp

@@ -487,6 +487,13 @@ void Replayer::replay()
       NCCL_CALL(ncclAllReduce(sbuffer, rbuffer, call.count, call.datatype, call.op, commMap[call.comm], streams[call.stream].first));
       break;
     }
+    case rrAllReduceWithBias:
+    {
+      std::vector<char> acc(call.count * ncclTypeSize(call.datatype));
+      NCCL_CALL(ncclAllReduceWithBias(sbuffer, rbuffer, call.count, call.datatype, call.op, commMap[call.comm], streams[call.stream].first, acc.data()));
+      HIP_CALL(hipStreamSynchronize(streams[call.stream].first)); // TODO: remove, and further verify behavior of fused AR
+      break;
+    }
     // a2av
     case rrAllToAllv:
     {
@@ -670,4 +677,4 @@ int main(int argc, char **argv)
   replayer.replay();
   MPI_Finalize();
   return 0;
-}
+}