hipDeviceSynchronize needs to sync NonBlocking streams as well

SWDEV-237167

Change-Id: Ie916d8f03ce91e8ef05a2b4edc580a7021520f6f


[ROCm/clr commit: 9611b5a8b4]

projects/clr/hipamd/rocclr/hip_device_runtime.cpp

@@ -442,6 +442,9 @@ hipError_t hipDeviceSynchronize ( void ) {
   }
 
   queue->finish();
+
+  hip::Stream::syncNonBlockingStreams();
+
   HIP_RETURN(hipSuccess);
 }
 

projects/clr/hipamd/rocclr/hip_internal.hpp

@@ -109,6 +109,9 @@ namespace hip {
     amd::Monitor& Lock() const { return lock_; }
     /// Returns the creation flags for the current stream
     unsigned int Flags() const { return flags_; }
+
+    /// Sync all non-blocking streams
+    static void syncNonBlockingStreams();
   };
 
   /// HIP Device class

projects/clr/hipamd/rocclr/hip_stream.cpp

@@ -57,10 +57,8 @@ bool Stream::Create() {
   bool result = (queue_ != nullptr) ? queue_->create() : false;
   // Insert just created stream into the list of the blocking queues
   if (result) {
-    if (!(flags_ & hipStreamNonBlocking)) {
-      amd::ScopedLock lock(streamSetLock);
-      streamSet.insert(this);
-    }
+    amd::ScopedLock lock(streamSetLock);
+    streamSet.insert(this);
   } else {
     Destroy();
   }
@@ -104,6 +102,15 @@ int Stream::DeviceId() const {
   return device_->deviceId();
 }
 
+void Stream::syncNonBlockingStreams() {
+  amd::ScopedLock lock(streamSetLock);
+  for (auto& it : streamSet) {
+    if (it->Flags() & hipStreamNonBlocking) {
+      it->asHostQueue()->finish();
+    }
+  }
+}
+
 };
 
 // ================================================================================================
@@ -116,6 +123,8 @@ void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream
       amd::HostQueue* active_queue = stream->asHostQueue();
       // If it's the current device
       if ((&active_queue->device() == &blocking_queue->device()) &&
+          // Make sure it's a default stream
+          ((stream->Flags() & hipStreamNonBlocking) == 0) &&
           // and it's not the current stream
           (active_queue != blocking_queue) &&
           // check for a wait on the null stream