concurrent: improve concurrent profiling

This patch adds barrier packets, together with extra signals,
to enforce the completion order of read packets w.r.t dispatch.
And, PmcStopper is added to stop the profiling finally.

Change-Id: I8e8d3a41d86e42be1d9e5afd44c247be876cf1a5


[ROCm/rocprofiler commit: e26210d9d9]

projects/rocprofiler/src/core/rocprofiler.cpp

@@ -150,20 +150,6 @@ void RestoreHsaApi() {
   table->amd_ext_->hsa_amd_queue_intercept_register_fn = hsa_amd_queue_intercept_register_fn;
 }
 
-void PmcStarter(Context* context) {
-  hsa_agent_t agent = context->GetAgent();
-  // Create queue
-  hsa_queue_t* queue;
-  hsa_status_t status = rocprofiler::CreateQueuePro(agent, 1,
-    HSA_QUEUE_TYPE_MULTI, NULL, NULL, UINT32_MAX, UINT32_MAX, &queue);
-  if (status != HSA_STATUS_SUCCESS) EXC_RAISING(status, "CreateQueuePro");
-  HsaQueue hsa_queue(NULL, queue);
-  context->Start(0, &hsa_queue);
-  context->Read(0, &hsa_queue);
-  context->GetData(0);
-  hsa_queue_destroy(queue);
-}
-
 void StandaloneIntercept() {
   ::HsaApiTable* table = kHsaApiTable;
   table->core_->hsa_queue_create_fn = rocprofiler::CreateQueuePro;
@@ -241,9 +227,40 @@ uint32_t LoadTool() {
   return intercept_mode;
 }
 
+void PmcStopper() {
+  rocprofiler::util::HsaRsrcFactory* rsrc = &rocprofiler::util::HsaRsrcFactory::Instance();
+
+  const uint32_t gpu_count = rsrc->GetCountOfGpuAgents();
+  for (uint32_t gpu_id = 0; gpu_id < gpu_count; gpu_id++) {
+    // Get agent info
+    const rocprofiler::util::AgentInfo* agent_info;
+    if (rsrc->GetGpuAgentInfo(gpu_id, &agent_info) == false) {
+      fprintf(stderr, "Error: GetGpuAgentInfo(%u) \n", gpu_id);
+      abort();
+    }
+
+    // Create queue
+    hsa_queue_t* queue;
+    hsa_status_t status = rocprofiler::CreateQueuePro(agent_info->dev_id, 1,
+            HSA_QUEUE_TYPE_MULTI, NULL, NULL, UINT32_MAX, UINT32_MAX, &queue);
+    if (status != HSA_STATUS_SUCCESS) EXC_RAISING(status, "CreateQueuePro ("
+            << gpu_id << ") " << std::hex << status);
+
+    // Submit packets
+    for (auto& pkt: Context::stop_packets_) {
+      rsrc->Submit(queue, &pkt);
+      // Wait for stop packet to complete
+      rsrc->SignalWaitRestore(pkt.completion_signal, 1);
+    }
+
+    hsa_queue_destroy(queue);
+  }
+}
+
 // Unload profiling tool librray
 void UnloadTool() {
   ONLOAD_TRACE("tool handle(" << tool_handle << ")");
+  //if (Context::k_concurrent_) PmcStopper();
   if (tool_handle) {
     tool_handler_t handler = reinterpret_cast<tool_handler_t>(dlsym(tool_handle, "OnUnloadTool"));
     if (handler == NULL) {