Fixing Multiple Profiler Issues:

1- Fixing Output Buffer Issues
2- Metrics Instances created for all GPUs in Init
3- Multi CPU/GPU/KernArg Pools are initialized for every Agent in Init
4- Lowering OverHead in the Packet WriteInterceptor in the ProxyQueue for both Profile Counting & ATT

Change-Id: Ic3d78156af8405bb134d01584574c339237d265f


[ROCm/rocprofiler commit: 00ecca25c7]

projects/rocprofiler/src/core/hardware/hsa_info.cpp

@@ -27,7 +27,7 @@
     if ((status) != HSA_STATUS_SUCCESS) {                                                          \
       const char* emsg = 0;                                                                        \
       hsa_status_string(status, &emsg);                                                            \
-      throw(ROCPROFILER_STATUS_ERROR_HSA_SUPPORT,                                                    \
+      throw(ROCPROFILER_STATUS_ERROR_HSA_SUPPORT,                                                  \
             "Error: " << msg << ": " << emsg ? emsg : "<unknown error>");                          \
     }                                                                                              \
   } while (0)
@@ -62,32 +62,27 @@ AgentInfo::AgentInfo(const hsa_agent_t agent, ::CoreApiTable* table) : handle_(a
   table->hsa_agent_get_info_fn(
       agent, static_cast<hsa_agent_info_t>(HSA_AMD_AGENT_INFO_NUM_SHADER_ENGINES), &se_num_);
 
-  if (table->hsa_agent_get_info_fn(
-        agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_NUM_SHADER_ARRAYS_PER_SE,
-        &shader_arrays_per_se_) != HSA_STATUS_SUCCESS ||
-      table->hsa_agent_get_info_fn(
-        agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU,
-        &waves_per_cu_) != HSA_STATUS_SUCCESS)
-  {
-      rocmtools::fatal("hsa_agent_get_info for gfxip hardware configuration failed");
+  if (table->hsa_agent_get_info_fn(agent,
+                                   (hsa_agent_info_t)HSA_AMD_AGENT_INFO_NUM_SHADER_ARRAYS_PER_SE,
+                                   &shader_arrays_per_se_) != HSA_STATUS_SUCCESS ||
+      table->hsa_agent_get_info_fn(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MAX_WAVES_PER_CU,
+                                   &waves_per_cu_) != HSA_STATUS_SUCCESS) {
+    rocmtools::fatal("hsa_agent_get_info for gfxip hardware configuration failed");
   }
 
   compute_units_per_sh_ = cu_num_ / (se_num_ * shader_arrays_per_se_);
   wave_slots_per_simd_ = waves_per_cu_ / simds_per_cu_;
 
-  if (table->hsa_agent_get_info_fn(
-        agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_DOMAIN,
-        &pci_domain_) != HSA_STATUS_SUCCESS ||
-      table->hsa_agent_get_info_fn(
-        agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_BDFID,
-        &pci_location_id_) != HSA_STATUS_SUCCESS)
-  {
-      rocmtools::fatal("hsa_agent_get_info for PCI info failed");
+  if (table->hsa_agent_get_info_fn(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_DOMAIN,
+                                   &pci_domain_) != HSA_STATUS_SUCCESS ||
+      table->hsa_agent_get_info_fn(agent, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_BDFID,
+                                   &pci_location_id_) != HSA_STATUS_SUCCESS) {
+    rocmtools::fatal("hsa_agent_get_info for PCI info failed");
   }
 
-  // TODO: (sauverma) use hsa_agent_get_info_fn(HSA_AMD_AGENT_INFO_NUM_XCC) 
+  // TODO: (sauverma) use hsa_agent_get_info_fn(HSA_AMD_AGENT_INFO_NUM_XCC)
   // to get xcc_num once hsa headers are updated from rocr/hsa
-  std::string gpu_name = std::string(name_).substr(0,6);
+  std::string gpu_name = std::string(name_).substr(0, 6);
   if (gpu_name == "gfx940")
     xcc_num_ = 6;
   else
@@ -117,6 +112,12 @@ void AgentInfo::setType(hsa_device_type_t type) { type_ = type; }
 void AgentInfo::setHandle(uint64_t handle) { handle_ = handle; }
 void AgentInfo::setName(const std::string& name) { strcpy(name_, name.c_str()); }
 
+void AgentInfo::setNumaNode(uint32_t numa_node) { numa_node_ = numa_node; }
+uint32_t AgentInfo::getNumaNode() { return numa_node_; }
+
+void AgentInfo::setNearCpuAgent(hsa_agent_t near_cpu_agent) { near_cpu_agent_ = near_cpu_agent; }
+hsa_agent_t AgentInfo::getNearCpuAgent() { return near_cpu_agent_; }
+
 // CounterHardwareInfo Class
 
 CounterHardwareInfo::CounterHardwareInfo(uint64_t event_id, const char* block_id)

projects/rocprofiler/src/core/hardware/hsa_info.h

@@ -23,6 +23,7 @@
 
 #include <hsa/hsa.h>
 #include <hsa/hsa_api_trace.h>
+#include <hsa/hsa_ext_amd.h>
 
 #include <atomic>
 #include <map>
@@ -66,6 +67,16 @@ class AgentInfo {
   void setHandle(uint64_t handle);
   void setName(const std::string& name);
 
+  void setNumaNode(uint32_t numa_node);
+  uint32_t getNumaNode();
+
+  void setNearCpuAgent(hsa_agent_t near_cpu_agent);
+  hsa_agent_t getNearCpuAgent();
+
+  hsa_amd_memory_pool_t cpu_pool;
+  hsa_amd_memory_pool_t kernarg_pool;
+  hsa_amd_memory_pool_t gpu_pool;
+
  private:
   int index_;
   hsa_device_type_t type_;  // Agent type - Cpu = 0, Gpu = 1 or Dsp = 2
@@ -87,6 +98,9 @@ class AgentInfo {
 
   uint32_t pci_domain_;
   uint32_t pci_location_id_;
+
+  uint32_t numa_node_;
+  hsa_agent_t near_cpu_agent_;
 };
 
 // XXX TODO: This should be moved somewhere else so this file can be deleted