SWDEV-486602 - Add tracking of HSA handlers

Add an atomic counter to track the outstanding HSA handlers.
Wait on CPU for the callbacks if the number exceeds the value
in DEBUG_HIP_BLOCK_SYNC env variable.

Change-Id: I95dc8c4bf0258c7e59411b7504220709ed6898c5


[ROCm/clr commit: 403f624bf8]

projects/clr/rocclr/platform/commandqueue.cpp

@@ -141,6 +141,11 @@ void HostQueue::finish(bool cpu_wait) {
         "Can't claim the queue is finished with the active batch!");
       return;
     }
+    // Force blocking wait if requested. That allows to avoid a build up of unreleased CPU commands
+    if ((DEBUG_HIP_BLOCK_SYNC > 0) &&
+        (vdev()->QueuedAsyncHandlers().load() > DEBUG_HIP_BLOCK_SYNC)) {
+      cpu_wait = true;
+    }
   }
   // Force marker if the batch wasn't sent for CPU update or fence is dirty
   if (nullptr == command || (GetSubmissionBatch() != nullptr) || vdev()->isFenceDirty()) {