supporting original hsa api table

Change-Id: Ia98554c137a3ec6433d8234ba17ca4dc2f2be940


[ROCm/rocprofiler commit: df860838a2]

projects/rocprofiler/src/core/rocprofiler.cpp

@@ -90,6 +90,8 @@ decltype(hsa_amd_memory_async_copy_rect)* hsa_amd_memory_async_copy_rect_fn;
 ::HsaApiTable* kHsaApiTable;
 
 void SaveHsaApi(::HsaApiTable* table) {
+  util::HsaRsrcFactory::InitHsaApiTable(table);
+
   kHsaApiTable = table;
   hsa_queue_create_fn = table->core_->hsa_queue_create_fn;
   hsa_queue_destroy_fn = table->core_->hsa_queue_destroy_fn;

projects/rocprofiler/src/util/hsa_rsrc_factory.cpp

@@ -117,7 +117,7 @@ HsaRsrcFactory::HsaRsrcFactory(bool initialize_hsa) : initialize_hsa_(initialize
   cpu_pool_ = NULL;
   kern_arg_pool_ = NULL;
 
-  InitHsaApiTable();
+  InitHsaApiTable(NULL);
 
   // Initialize the Hsa Runtime
   if (initialize_hsa_) {
@@ -165,42 +165,86 @@ HsaRsrcFactory::~HsaRsrcFactory() {
   }
 }
 
-void HsaRsrcFactory::InitHsaApiTable() {
-  hsa_api_.hsa_init = hsa_init;
-  hsa_api_.hsa_shut_down = hsa_shut_down;
-  hsa_api_.hsa_agent_get_info = hsa_agent_get_info;
+void HsaRsrcFactory::InitHsaApiTable(HsaApiTable* table) {
+  std::lock_guard<mutex_t> lck(mutex_);
 
-  hsa_api_.hsa_iterate_agents = hsa_iterate_agents;
+  if (hsa_api_.hsa_init == NULL) {
+    if (table != NULL) {
+      hsa_api_.hsa_init = table->core_->hsa_init_fn;
+      hsa_api_.hsa_shut_down = table->core_->hsa_shut_down_fn;
+      hsa_api_.hsa_agent_get_info = table->core_->hsa_agent_get_info_fn;
 
-  hsa_api_.hsa_queue_create = hsa_queue_create;
-  hsa_api_.hsa_queue_load_write_index_relaxed = hsa_queue_load_write_index_relaxed;
-  hsa_api_.hsa_queue_store_write_index_relaxed = hsa_queue_store_write_index_relaxed;
-  hsa_api_.hsa_queue_load_read_index_relaxed = hsa_queue_load_read_index_relaxed;
-  hsa_api_.hsa_signal_create = hsa_signal_create;
-  hsa_api_.hsa_signal_destroy = hsa_signal_destroy;
-  hsa_api_.hsa_signal_store_relaxed = hsa_signal_store_relaxed;
-  hsa_api_.hsa_signal_wait_scacquire = hsa_signal_wait_scacquire;
+      hsa_api_.hsa_iterate_agents = table->core_->hsa_iterate_agents_fn;
 
-  hsa_api_.hsa_amd_agent_iterate_memory_pools = hsa_amd_agent_iterate_memory_pools;
-  hsa_api_.hsa_amd_memory_pool_get_info = hsa_amd_memory_pool_get_info;
-  hsa_api_.hsa_amd_memory_pool_allocate = hsa_amd_memory_pool_allocate;
-  hsa_api_.hsa_amd_agents_allow_access = hsa_amd_agents_allow_access;
+      hsa_api_.hsa_queue_create = table->core_->hsa_queue_create_fn;
+      hsa_api_.hsa_queue_destroy = table->core_->hsa_queue_destroy_fn;
+      hsa_api_.hsa_queue_load_write_index_relaxed = table->core_->hsa_queue_load_write_index_relaxed_fn;
+      hsa_api_.hsa_queue_store_write_index_relaxed = table->core_->hsa_queue_store_write_index_relaxed_fn;
+      hsa_api_.hsa_queue_load_read_index_relaxed = table->core_->hsa_queue_load_read_index_relaxed_fn;
+      hsa_api_.hsa_signal_create = table->core_->hsa_signal_create_fn;
+      hsa_api_.hsa_signal_destroy = table->core_->hsa_signal_destroy_fn;
+      hsa_api_.hsa_signal_load_relaxed = table->core_->hsa_signal_load_relaxed_fn;
+      hsa_api_.hsa_signal_store_relaxed = table->core_->hsa_signal_store_relaxed_fn;
+      hsa_api_.hsa_signal_store_screlease = table->core_->hsa_signal_store_screlease_fn;
+      hsa_api_.hsa_signal_wait_scacquire = table->core_->hsa_signal_wait_scacquire_fn;
 
-  hsa_api_.hsa_amd_memory_async_copy = hsa_amd_memory_async_copy;
+      hsa_api_.hsa_system_get_major_extension_table = table->core_->hsa_system_get_major_extension_table_fn;
 
-  hsa_api_.hsa_system_get_major_extension_table = hsa_system_get_major_extension_table;
+      hsa_api_.hsa_code_object_reader_create_from_file = table->core_->hsa_code_object_reader_create_from_file_fn;
+      hsa_api_.hsa_executable_create_alt = table->core_->hsa_executable_create_alt_fn;
+      hsa_api_.hsa_executable_load_agent_code_object = table->core_->hsa_executable_load_agent_code_object_fn;
+      hsa_api_.hsa_executable_freeze = table->core_->hsa_executable_freeze_fn;
+      hsa_api_.hsa_executable_get_symbol = table->core_->hsa_executable_get_symbol_fn;
 
-  hsa_api_.hsa_code_object_reader_create_from_file = hsa_code_object_reader_create_from_file;
-  hsa_api_.hsa_executable_create_alt = hsa_executable_create_alt;
-  hsa_api_.hsa_executable_load_agent_code_object = hsa_executable_load_agent_code_object;
-  hsa_api_.hsa_executable_freeze = hsa_executable_freeze;
-  hsa_api_.hsa_executable_get_symbol = hsa_executable_get_symbol;
+      hsa_api_.hsa_amd_agent_iterate_memory_pools = table->amd_ext_->hsa_amd_agent_iterate_memory_pools_fn;
+      hsa_api_.hsa_amd_memory_pool_get_info = table->amd_ext_->hsa_amd_memory_pool_get_info_fn;
+      hsa_api_.hsa_amd_memory_pool_allocate = table->amd_ext_->hsa_amd_memory_pool_allocate_fn;
+      hsa_api_.hsa_amd_agents_allow_access = table->amd_ext_->hsa_amd_agents_allow_access_fn;
 
-  hsa_api_.hsa_amd_signal_async_handler = hsa_amd_signal_async_handler;
-  hsa_api_.hsa_amd_profiling_get_async_copy_time = hsa_amd_profiling_get_async_copy_time;
-  hsa_api_.hsa_amd_profiling_get_dispatch_time = hsa_amd_profiling_get_dispatch_time;
-  hsa_api_.hsa_signal_load_relaxed = hsa_signal_load_relaxed;
-  hsa_api_.hsa_signal_store_screlease = hsa_signal_store_screlease;
+      hsa_api_.hsa_amd_memory_async_copy = table->amd_ext_->hsa_amd_memory_async_copy_fn;
+
+      hsa_api_.hsa_amd_signal_async_handler = table->amd_ext_->hsa_amd_signal_async_handler_fn;
+      hsa_api_.hsa_amd_profiling_get_async_copy_time = table->amd_ext_->hsa_amd_profiling_get_async_copy_time_fn;
+      hsa_api_.hsa_amd_profiling_get_dispatch_time = table->amd_ext_->hsa_amd_profiling_get_dispatch_time_fn;
+    } else {
+      hsa_api_.hsa_init = hsa_init;
+      hsa_api_.hsa_shut_down = hsa_shut_down;
+      hsa_api_.hsa_agent_get_info = hsa_agent_get_info;
+
+      hsa_api_.hsa_iterate_agents = hsa_iterate_agents;
+
+      hsa_api_.hsa_queue_create = hsa_queue_create;
+      hsa_api_.hsa_queue_destroy = hsa_queue_destroy;
+      hsa_api_.hsa_queue_load_write_index_relaxed = hsa_queue_load_write_index_relaxed;
+      hsa_api_.hsa_queue_store_write_index_relaxed = hsa_queue_store_write_index_relaxed;
+      hsa_api_.hsa_queue_load_read_index_relaxed = hsa_queue_load_read_index_relaxed;
+      hsa_api_.hsa_signal_create = hsa_signal_create;
+      hsa_api_.hsa_signal_destroy = hsa_signal_destroy;
+      hsa_api_.hsa_signal_store_relaxed = hsa_signal_store_relaxed;
+      hsa_api_.hsa_signal_wait_scacquire = hsa_signal_wait_scacquire;
+
+      hsa_api_.hsa_amd_agent_iterate_memory_pools = hsa_amd_agent_iterate_memory_pools;
+      hsa_api_.hsa_amd_memory_pool_get_info = hsa_amd_memory_pool_get_info;
+      hsa_api_.hsa_amd_memory_pool_allocate = hsa_amd_memory_pool_allocate;
+      hsa_api_.hsa_amd_agents_allow_access = hsa_amd_agents_allow_access;
+
+      hsa_api_.hsa_amd_memory_async_copy = hsa_amd_memory_async_copy;
+
+      hsa_api_.hsa_system_get_major_extension_table = hsa_system_get_major_extension_table;
+
+      hsa_api_.hsa_code_object_reader_create_from_file = hsa_code_object_reader_create_from_file;
+      hsa_api_.hsa_executable_create_alt = hsa_executable_create_alt;
+      hsa_api_.hsa_executable_load_agent_code_object = hsa_executable_load_agent_code_object;
+      hsa_api_.hsa_executable_freeze = hsa_executable_freeze;
+      hsa_api_.hsa_executable_get_symbol = hsa_executable_get_symbol;
+
+      hsa_api_.hsa_amd_signal_async_handler = hsa_amd_signal_async_handler;
+      hsa_api_.hsa_amd_profiling_get_async_copy_time = hsa_amd_profiling_get_async_copy_time;
+      hsa_api_.hsa_amd_profiling_get_dispatch_time = hsa_amd_profiling_get_dispatch_time;
+      hsa_api_.hsa_signal_load_relaxed = hsa_signal_load_relaxed;
+      hsa_api_.hsa_signal_store_screlease = hsa_signal_store_screlease;
+    }
+  }
 }
 
 hsa_status_t HsaRsrcFactory::LoadAqlProfileLib(aqlprofile_pfn_t* api) {
@@ -635,7 +679,7 @@ uint64_t HsaRsrcFactory::Submit(hsa_queue_t* queue, const void* packet, size_t s
   return write_idx;
 }
 
-HsaRsrcFactory* HsaRsrcFactory::instance_ = NULL;
+std::atomic<HsaRsrcFactory*> HsaRsrcFactory::instance_{};
 HsaRsrcFactory::mutex_t HsaRsrcFactory::mutex_;
 HsaRsrcFactory::timestamp_t HsaRsrcFactory::timeout_ns_ = HsaTimer::TIMESTAMP_MAX;
 hsa_pfn_t HsaRsrcFactory::hsa_api_{};

projects/rocprofiler/src/util/hsa_rsrc_factory.h

@@ -26,6 +26,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #define SRC_UTIL_HSA_RSRC_FACTORY_H_
 
 #include <hsa.h>
+#include <hsa_api_trace.h>
 #include <hsa_ext_amd.h>
 #include <hsa_ext_finalize.h>
 #include <hsa_ven_amd_aqlprofile.h>
@@ -35,6 +36,7 @@ POSSIBILITY OF SUCH DAMAGE.
 #include <stdlib.h>
 #include <string.h>
 
+#include <atomic>
 #include <iostream>
 #include <mutex>
 #include <map>
@@ -77,6 +79,7 @@ struct hsa_pfn_t {
   decltype(hsa_iterate_agents)* hsa_iterate_agents;
 
   decltype(hsa_queue_create)* hsa_queue_create;
+  decltype(hsa_queue_destroy)* hsa_queue_destroy;
   decltype(hsa_queue_load_write_index_relaxed)* hsa_queue_load_write_index_relaxed;
   decltype(hsa_queue_store_write_index_relaxed)* hsa_queue_store_write_index_relaxed;
   decltype(hsa_queue_load_read_index_relaxed)* hsa_queue_load_read_index_relaxed;
@@ -201,17 +204,20 @@ class HsaRsrcFactory {
 
   static HsaRsrcFactory* Create(bool initialize_hsa = true) {
     std::lock_guard<mutex_t> lck(mutex_);
-    if (instance_ == NULL) {
-      instance_ = new HsaRsrcFactory(initialize_hsa);
+    HsaRsrcFactory* obj = instance_.load(std::memory_order_relaxed);
+    if (obj == NULL) {
+      obj = new HsaRsrcFactory(initialize_hsa);
+      instance_.store(obj, std::memory_order_release);
     }
-    return instance_;
+    return obj;
   }
 
   static HsaRsrcFactory& Instance() {
-    if (instance_ == NULL) instance_ = Create(false);
-    hsa_status_t status = (instance_ != NULL) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR;
+    HsaRsrcFactory* obj = instance_.load(std::memory_order_acquire);
+    if (obj == NULL) obj = Create(false);
+    hsa_status_t status = (obj != NULL) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR;
     CHECK_STATUS("HsaRsrcFactory::Instance() failed", status);
-    return *instance_;
+    return *obj;
   }
 
   static void Destroy() {
@@ -314,7 +320,7 @@ class HsaRsrcFactory {
   static uint64_t Submit(hsa_queue_t* queue, const void* packet, size_t size_bytes);
 
   // Initialize HSA API table
-  void static InitHsaApiTable();
+  void static InitHsaApiTable(HsaApiTable* table);
   static const hsa_pfn_t* HsaApi() { return &hsa_api_; }
 
   // Return AqlProfile API table
@@ -334,7 +340,7 @@ class HsaRsrcFactory {
   static void SetTimeoutNs(const timestamp_t& time) {
     std::lock_guard<mutex_t> lck(mutex_);
     timeout_ns_ = time;
-    if (instance_ != NULL) instance_->timeout_ = instance_->timer_->ns_to_sysclock(time);
+    if (instance_ != NULL) Instance().timeout_ = Instance().timer_->ns_to_sysclock(time);
   }
 
  private:
@@ -363,7 +369,7 @@ class HsaRsrcFactory {
   // HSA was initialized
   const bool initialize_hsa_;
 
-  static HsaRsrcFactory* instance_;
+  static std::atomic<HsaRsrcFactory*> instance_;
   static mutex_t mutex_;
 
   // Used to maintain a list of Hsa Gpu Agent Info

projects/rocprofiler/test/run.sh

@@ -88,6 +88,13 @@ export ROCP_THRS=1
 export ROCP_INPUT=input.xml
 eval_test "'rocprof' libtool test" ./test/ctrl
 
+export ROCP_KITER=10
+export ROCP_DITER=10
+export ROCP_AGENTS=1
+export ROCP_THRS=10
+export ROCP_INPUT=input1.xml
+eval_test "'rocprof' libtool test n-threads" ./test/ctrl
+
 ## Libtool test, counter sets
 
 # Memcopies tracking
@@ -95,7 +102,7 @@ export ROCP_MCOPY_TRACKING=1
 
 export ROCP_KITER=1
 export ROCP_DITER=4
-export ROCP_INPUT=input1.xml
+export ROCP_INPUT=input2.xml
 eval_test "libtool test, counter sets" ./test/ctrl
 
 #valgrind --leak-check=full $tbin

projects/rocprofiler/test/tool/input1.xml

@@ -1,5 +1,14 @@
+# Filter by dispatches range, GPU index and kernel names
+<metric
+  # range format "3:9"
+  range=""
+  # list of gpu indexes "0,1,2,3"
+  gpu_index=""
+  # list of matched sub-strings "Simple1,Conv1,SimpleConvolution"
+  kernel=""
+></metric>
+
 # List of metrics
 <metric
-  name="SQ:4,SQ:32,SQ:33,SQ:81,SQ:89, SQ:28,SQ:89,SQ:81,SQ:33,SQ:32,SQ:4,SQ:27, SQ:27,SQ:28, SQ:28,SQ:89,SQ:81,SQ:33"
-  set="5,7,2"
+  name=SQ:4,SQ_WAVES,SQ_INSTS_SMEM,SQ_INSTS_VALU,TA_FLAT_WRITE_WAVEFRONTS[0],TA_FLAT_WRITE_WAVEFRONTS[1],GPUBusy,VALUBusy,SALUBusy,MemUnitBusy,SFetchInsts,FetchSize,VWriteInsts,WriteSize
 ></metric>

projects/rocprofiler/test/tool/input2.xml

@@ -0,0 +1,5 @@
+# List of metrics
+<metric
+  name="SQ:4,SQ:32,SQ:33,SQ:81,SQ:89, SQ:28,SQ:89,SQ:81,SQ:33,SQ:32,SQ:4,SQ:27, SQ:27,SQ:28, SQ:28,SQ:89,SQ:81,SQ:33"
+  set="5,7,2"
+></metric>