SWDEV-486602 - Optimize HSA callback performance

- Don't generate callbacks for HIP events
- Don't process profiling info in the callback for HIP events
- Wait for CPU status update of the submitted commands
every 50 calls. That will allow to drain the commands and
destroy HSA signals.

Change-Id: Ib601a350e7e7c2b6c6209a172385389baccf73a9


[ROCm/clr commit: 364dfb0ed1]

projects/clr/rocclr/platform/command.cpp

@@ -366,7 +366,9 @@ void Command::enqueue() {
     ScopedLock sl(queue_->vdev()->execution());
     queue_->FormSubmissionBatch(this);
 
-    if (type() == CL_COMMAND_MARKER || type() == 0 || type() == CL_COMMAND_TASK) {
+    // Enqueue flushes, except profiling markers to avoid frequent expensive callbacks
+    if (((type() == 0) && profilingInfo().batch_flush_) ||
+        (type() == CL_COMMAND_MARKER) || (type() == CL_COMMAND_TASK)) {
       // The current HSA signal tracking logic requires profiling enabled for the markers
       EnableProfiling();
       // Update batch head for the current marker. Hence the status of all commands can be

projects/clr/rocclr/platform/command.hpp

@@ -114,6 +114,7 @@ class Event : public RuntimeObject {
     uint64_t correlation_id_;
     bool enabled_;        //!< Profiling enabled for the wave limiter
     bool marker_ts_;      //!< TS marker
+    bool batch_flush_ = true; //!< Command can flush the batch in direct dispatch mode
 
    void clear() {
       queued_ = 0ULL;

projects/clr/rocclr/platform/commandqueue.cpp

@@ -65,6 +65,15 @@ bool HostQueue::terminate() {
       // destroyed.
       Command* lastCommand = getLastQueuedCommand(true);
       if (lastCommand != nullptr) {
+        // Check if CPU batch wasn't flushed for completion with the last command
+        if (GetSubmittionBatch() != nullptr) {
+            auto command = new Marker(*this, false);
+            if (command != nullptr) {
+              ClPrint(LOG_DEBUG, LOG_CMD, "Marker queued to ensure finish");
+              command->enqueue();
+              lastCommand = command;
+            }
+        }
         lastCommand->awaitCompletion();
         // Note that if lastCommand isn't a marker, it may not be lastEnqueueCommand_ now
         // after lastCommand->awaitCompletion() is called.
@@ -128,19 +137,13 @@ void HostQueue::finish(bool cpu_wait) {
   if (IS_HIP) {
     command = getLastQueuedCommand(true);
     if (command == nullptr) {
+      assert(GetSubmittionBatch() == nullptr &&
+        "Can't claim the queue is finished with the active batch!");
       return;
     }
   }
-  // If command doesn't contain HW event and runtime didn't request CPU wait,
-  // then force marker submit
-  bool force_marker = false;
-  if (AMD_DIRECT_DISPATCH && (command != nullptr) && !cpu_wait) {
-    void* hw_event =
-      (command->NotifyEvent() != nullptr) ? command->NotifyEvent()->HwEvent() : command->HwEvent();
-    force_marker = (hw_event == nullptr);
-  }
-  if (nullptr == command || force_marker ||
-      vdev()->isHandlerPending() || vdev()->isFenceDirty()) {
+  // Force marker if the batch wasn't sent for CPU update or fence is dirty
+  if (nullptr == command || (GetSubmittionBatch() != nullptr) || vdev()->isFenceDirty()) {
     if (nullptr != command) {
       command->release();
     }