From 20dadcac1e1f4d0d32807be4408124eeabcc4dd6 Mon Sep 17 00:00:00 2001
From: Christophe Paquot <christophe.paquot@amd.com>
Date: Tue, 26 May 2020 12:54:27 -0700
Subject: [PATCH] hipDeviceSynchronize needs to sync NonBlocking streams as
 well

SWDEV-237167

Change-Id: Ie916d8f03ce91e8ef05a2b4edc580a7021520f6f


[ROCm/clr commit: 9611b5a8b43a7b1462bbd409e43c0187da7d7cf7]
---
 .../clr/hipamd/rocclr/hip_device_runtime.cpp    |  3 +++
 projects/clr/hipamd/rocclr/hip_internal.hpp     |  3 +++
 projects/clr/hipamd/rocclr/hip_stream.cpp       | 17 +++++++++++++----
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/projects/clr/hipamd/rocclr/hip_device_runtime.cpp b/projects/clr/hipamd/rocclr/hip_device_runtime.cpp
index 3823af5206..9ed1c64145 100644
--- a/projects/clr/hipamd/rocclr/hip_device_runtime.cpp
+++ b/projects/clr/hipamd/rocclr/hip_device_runtime.cpp
@@ -442,6 +442,9 @@ hipError_t hipDeviceSynchronize ( void ) {
   }
 
   queue->finish();
+
+  hip::Stream::syncNonBlockingStreams();
+
   HIP_RETURN(hipSuccess);
 }
 
diff --git a/projects/clr/hipamd/rocclr/hip_internal.hpp b/projects/clr/hipamd/rocclr/hip_internal.hpp
index 492842c32b..400aa8b99a 100755
--- a/projects/clr/hipamd/rocclr/hip_internal.hpp
+++ b/projects/clr/hipamd/rocclr/hip_internal.hpp
@@ -109,6 +109,9 @@ namespace hip {
     amd::Monitor& Lock() const { return lock_; }
     /// Returns the creation flags for the current stream
     unsigned int Flags() const { return flags_; }
+
+    /// Sync all non-blocking streams
+    static void syncNonBlockingStreams();
   };
 
   /// HIP Device class
diff --git a/projects/clr/hipamd/rocclr/hip_stream.cpp b/projects/clr/hipamd/rocclr/hip_stream.cpp
index 23b89e71ca..0f9b782066 100644
--- a/projects/clr/hipamd/rocclr/hip_stream.cpp
+++ b/projects/clr/hipamd/rocclr/hip_stream.cpp
@@ -57,10 +57,8 @@ bool Stream::Create() {
   bool result = (queue_ != nullptr) ? queue_->create() : false;
   // Insert just created stream into the list of the blocking queues
   if (result) {
-    if (!(flags_ & hipStreamNonBlocking)) {
-      amd::ScopedLock lock(streamSetLock);
-      streamSet.insert(this);
-    }
+    amd::ScopedLock lock(streamSetLock);
+    streamSet.insert(this);
   } else {
     Destroy();
   }
@@ -104,6 +102,15 @@ int Stream::DeviceId() const {
   return device_->deviceId();
 }
 
+void Stream::syncNonBlockingStreams() {
+  amd::ScopedLock lock(streamSetLock);
+  for (auto& it : streamSet) {
+    if (it->Flags() & hipStreamNonBlocking) {
+      it->asHostQueue()->finish();
+    }
+  }
+}
+
 };
 
 // ================================================================================================
@@ -116,6 +123,8 @@ void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream
       amd::HostQueue* active_queue = stream->asHostQueue();
       // If it's the current device
       if ((&active_queue->device() == &blocking_queue->device()) &&
+          // Make sure it's a default stream
+          ((stream->Flags() & hipStreamNonBlocking) == 0) &&
           // and it's not the current stream
           (active_queue != blocking_queue) &&
           // check for a wait on the null stream