Added signal management which passes stress tests

Change-Id: I7e1660a8ca2c5ee580a91f76eae9a58ca49f0457


[ROCm/clr commit: 5845a2d8c6]

projects/clr/hipamd/include/hcc_detail/hip_hcc.h

@@ -431,6 +431,9 @@ public:
     SIGSEQNUM                   _streamSigId;      // Monotonically increasing unique signal id.
 
     hc::accelerator_view        _av;
+
+		std::vector<hc::completion_future*> _cfs;
+
 };
 
 
@@ -468,6 +471,9 @@ typedef uint64_t SeqNum_t ;
     void                 locked_waitEvent(hipEvent_t event);
     void                 locked_recordEvent(hipEvent_t event);
 
+    void                 addCFtoStream(LockedAccessor_StreamCrit_t &crit, hc::completion_future* cf);
+    void                 waitOnAllCFs(LockedAccessor_StreamCrit_t &crit);
+
     //---
 
     // Use this if we already have the stream critical data mutex:

projects/clr/hipamd/src/hip_hcc.cpp

@@ -250,7 +250,8 @@ void ihipStream_t::wait(LockedAccessor_StreamCrit_t &crit, bool assertQueueEmpty
 {
     if (! assertQueueEmpty) {
         tprintf (DB_SYNC, "stream %p wait for queue-empty..\n", this);
-        crit->_av.wait();
+//        crit->_av.wait();
+        waitOnAllCFs(crit);
     }
 
     if (crit->_last_copy_signal) {
@@ -266,6 +267,21 @@ void ihipStream_t::wait(LockedAccessor_StreamCrit_t &crit, bool assertQueueEmpty
 //    crit->_signalCnt = 0;
 }
 
+void ihipStream_t::addCFtoStream(LockedAccessor_StreamCrit_t &crit, hc::completion_future *cf)
+{
+    crit->_cfs.push_back(cf);
+}
+
+void ihipStream_t::waitOnAllCFs(LockedAccessor_StreamCrit_t &crit)
+{
+    for(uint32_t i=0;i<crit->_cfs.size();i++){
+        if(crit->_cfs[i] != NULL){
+            crit->_cfs[i]->wait();
+            delete crit->_cfs[i];
+        }
+    }
+    crit->_cfs.clear();
+}
 
 //---
 //Wait for all kernel and data copy commands in this stream to complete.
@@ -1439,7 +1455,7 @@ hipStream_t ihipPreLaunchKernel(hipStream_t stream, dim3 grid, dim3 block, grid_
     auto crit = stream->lockopen_preKernelCommand();
     lp->av = &(crit->_av);
     lp->cf = new hc::completion_future;
-
+    stream->addCFtoStream(crit, lp->cf);
     ihipPrintKernelLaunch(kernelNameStr, lp, stream);
 
     return (stream);
@@ -1462,7 +1478,7 @@ hipStream_t ihipPreLaunchKernel(hipStream_t stream, size_t grid, dim3 block, gri
     auto crit = stream->lockopen_preKernelCommand();
     lp->av = &(crit->_av);
     lp->cf = new hc::completion_future;
-
+    stream->addCFtoStream(crit, lp->cf);
     ihipPrintKernelLaunch(kernelNameStr, lp, stream);
     return (stream);
 }
@@ -1484,6 +1500,7 @@ hipStream_t ihipPreLaunchKernel(hipStream_t stream, dim3 grid, size_t block, gri
     auto crit = stream->lockopen_preKernelCommand();
     lp->av = &(crit->_av);
     lp->cf = new hc::completion_future;
+    stream->addCFtoStream(crit, lp->cf);
     ihipPrintKernelLaunch(kernelNameStr, lp, stream);
     return (stream);
 }
@@ -1505,6 +1522,9 @@ hipStream_t ihipPreLaunchKernel(hipStream_t stream, size_t grid, size_t block, g
     auto crit = stream->lockopen_preKernelCommand();
     lp->av = &(crit->_av);
     lp->cf = new hc::completion_future; // TODO, is this necessary?
+
+    stream->addCFtoStream(crit, lp->cf);
+
     ihipPrintKernelLaunch(kernelNameStr, lp, stream);
     return (stream);
 }