From e4f51e063b32d30699beea337444ebb25565d65f Mon Sep 17 00:00:00 2001
From: German Andryeyev <German.Andryeyev@amd.com>
Date: Mon, 30 Nov 2020 17:49:25 -0500
Subject: [PATCH] Disable worker thread creation for direct dispatch

Change-Id: I28f08ab9352310c9bf843fcb803a48f95ddf4676
---
 rocclr/platform/commandqueue.cpp | 65 ++++++++++++++++++--------------
 rocclr/platform/commandqueue.hpp | 13 ++++++-
 2 files changed, 48 insertions(+), 30 deletions(-)

diff --git a/rocclr/platform/commandqueue.cpp b/rocclr/platform/commandqueue.cpp
index cb0fb9c6b1..6f8d44393b 100644
--- a/rocclr/platform/commandqueue.cpp
+++ b/rocclr/platform/commandqueue.cpp
@@ -38,43 +38,52 @@ HostQueue::HostQueue(Context& context, Device& device, cl_command_queue_properti
     : CommandQueue(context, device, properties, device.info().queueProperties_, queueRTCUs,
                    priority, cuMask),
       lastEnqueueCommand_(nullptr) {
-  if (thread_.state() >= Thread::INITIALIZED) {
-    ScopedLock sl(queueLock_);
-    thread_.start(this);
-    queueLock_.wait();
+  if (AMD_DIRECT_DISPATCH) {
+    // Initialize the queue 
+    thread_.Init(this);
+  } else {
+    if (thread_.state() >= Thread::INITIALIZED) {
+      ScopedLock sl(queueLock_);
+      thread_.start(this);
+      queueLock_.wait();
+    }
   }
 }
 
 bool HostQueue::terminate() {
-  if (Os::isThreadAlive(thread_)) {
-    Command* marker = nullptr;
+  if (AMD_DIRECT_DISPATCH) {
+    thread_.Release();
+  } else {
+    if (Os::isThreadAlive(thread_)) {
+      Command* marker = nullptr;
 
-    // Send a finish if the queue is still accepting commands.
-    {
-      ScopedLock sl(queueLock_);
-      if (thread_.acceptingCommands_) {
-        marker = new Marker(*this, false);
-        if (marker != nullptr) {
-          append(*marker);
-          queueLock_.notify();
+      // Send a finish if the queue is still accepting commands.
+      {
+        ScopedLock sl(queueLock_);
+        if (thread_.acceptingCommands_) {
+          marker = new Marker(*this, false);
+          if (marker != nullptr) {
+            append(*marker);
+            queueLock_.notify();
+          }
         }
       }
-    }
-    if (marker != nullptr) {
-      marker->awaitCompletion();
-      marker->release();
-    }
+      if (marker != nullptr) {
+        marker->awaitCompletion();
+        marker->release();
+      }
 
-    // Wake-up the command loop, so it can exit
-    {
-      ScopedLock sl(queueLock_);
-      thread_.acceptingCommands_ = false;
-      queueLock_.notify();
-    }
+      // Wake-up the command loop, so it can exit
+      {
+        ScopedLock sl(queueLock_);
+        thread_.acceptingCommands_ = false;
+        queueLock_.notify();
+      }
 
-    // FIXME_lmoriche: fix termination handshake
-    while (thread_.state() < Thread::FINISHED) {
-      Os::yield();
+      // FIXME_lmoriche: fix termination handshake
+      while (thread_.state() < Thread::FINISHED) {
+        Os::yield();
+      }
     }
   }
 
diff --git a/rocclr/platform/commandqueue.hpp b/rocclr/platform/commandqueue.hpp
index fc83dacb56..588485b2fe 100644
--- a/rocclr/platform/commandqueue.hpp
+++ b/rocclr/platform/commandqueue.hpp
@@ -157,7 +157,7 @@ class HostQueue : public CommandQueue {
 
     //! Create a new thread
     Thread()
-        : amd::Thread("Command Queue Thread", CQ_THREAD_STACK_SIZE),
+        : amd::Thread("Command Queue Thread", CQ_THREAD_STACK_SIZE, !AMD_DIRECT_DISPATCH),
           acceptingCommands_(false),
           virtualDevice_(NULL) {}
 
@@ -167,13 +167,22 @@ class HostQueue : public CommandQueue {
       virtualDevice_ = queue->device().createVirtualDevice(queue);
       if (virtualDevice_ != NULL) {
         queue->loop(virtualDevice_);
-        delete virtualDevice_;
+        Release();
       } else {
         acceptingCommands_ = false;
         queue->flush();
       }
     }
 
+    void Init(HostQueue* queue) {
+      virtualDevice_ = queue->device().createVirtualDevice(queue);
+      if (virtualDevice_ != nullptr) {
+        acceptingCommands_ = true;
+      }
+    }
+
+    void Release() const { delete virtualDevice_; }
+
     //! Get virtual device for the current thread
     device::VirtualDevice* vdev() const { return virtualDevice_; }