SWDEV-269654 - Fix HIP stream busy query

- Avoid GPU wait on the marker submission and update the command
batch after HSA signal callback upon HSA barrier completion.

Change-Id: I5c1c97212aefc2ae4b99aa9e2a81627ee9a38c1c

rocclr/platform/command.cpp

@@ -243,14 +243,14 @@ Command* Event::notifyCmdQueue(bool retain) {
 
 const Event::EventWaitList Event::nullWaitList(0);
 
+// ================================================================================================
 Command::Command(HostQueue& queue, cl_command_type type,
                  const EventWaitList& eventWaitList, uint32_t commandWaitBits)
     : Event(queue),
       queue_(&queue),
-      next_(NULL),
+      next_(nullptr),
       type_(type),
-      exception_(0),
-      data_(NULL),
+      data_(nullptr),
       eventWaitList_(eventWaitList),
       commandWaitBits_(commandWaitBits) {
   // Retain the commands from the event wait list.
@@ -258,6 +258,7 @@ Command::Command(HostQueue& queue, cl_command_type type,
   if (type != 0) activity_.Initialize(type, queue.vdev()->index(), queue.device().index());
 }
 
+// ================================================================================================
 void Command::releaseResources() {
   const Command::EventWaitList& events = eventWaitList();
 
@@ -265,6 +266,7 @@ void Command::releaseResources() {
   std::for_each(events.begin(), events.end(), std::mem_fun(&Command::release));
 }
 
+// ================================================================================================
 void Command::enqueue() {
   assert(queue_ != NULL && "Cannot be enqueued");
 
@@ -277,16 +279,28 @@ void Command::enqueue() {
   // Direct dispatch logic below will submit the command immediately, but the command status
   // update will occur later after flush() with a wait
   if (AMD_DIRECT_DISPATCH) {
+    setStatus(CL_QUEUED);
+    // The wait should be performed before the lock,
+    // otherwise signal handler may have a deadlock, but awaitCompletion() is thread safe itself
+    std::for_each(eventWaitList().begin(), eventWaitList().end(),
+        std::mem_fun(&Command::awaitCompletion));
+
     // The batch update must be lock protected to avoid a race condition
     // when multiple threads submit/flush/update the batch at the same time
-    ScopedLock sl(queue_->lock());
+    ScopedLock sl(queue_->vdev()->execution());
     queue_->FormSubmissionBatch(this);
     if ((type() == CL_COMMAND_MARKER || type() == 0) && !profilingInfo().marker_ts_) {
-      // Flush the current batch and wait for the results, since it's a marker.
-      // @todo: The condition requires a reevaluation to determine if any marker needs a wait.
+      // The current HSA signal tracking logic requires profiling enabled for the markers
+      EnableProfiling();
+      // Update batch head for the current marker. Hence the status of all commands can be
+      // updated upon the marker completion
+      SetBatchHead(queue_->GetSubmittionBatch());
+      // Flush the current batch, but skip the wait on CPU if possible to avoid a stall
       queue_->vdev()->flush(queue_->GetSubmittionBatch());
+      // The batch will be tracked with the marker now
       queue_->ResetSubmissionBatch();
     } else {
+      setStatus(CL_SUBMITTED);
       submit(*queue_->vdev());
     }
   } else {
@@ -299,6 +313,7 @@ void Command::enqueue() {
   }
 }
 
+// ================================================================================================
 const Context& Command::context() const { return queue_->context(); }
 
 NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList& eventWaitList,