From 222ef52c5d8c69aa8e4f2877901580fef167a7b6 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Wed, 6 Mar 2019 13:41:54 -0600 Subject: [PATCH] supporting original hsa api table Change-Id: Ia98554c137a3ec6433d8234ba17ca4dc2f2be940 [ROCm/rocprofiler commit: df860838a29ecf2489269f21b20647b7e3459cb3] --- projects/rocprofiler/src/core/rocprofiler.cpp | 2 + .../rocprofiler/src/util/hsa_rsrc_factory.cpp | 106 +++++++++++++----- .../rocprofiler/src/util/hsa_rsrc_factory.h | 24 ++-- projects/rocprofiler/test/run.sh | 9 +- projects/rocprofiler/test/tool/input1.xml | 13 ++- projects/rocprofiler/test/tool/input2.xml | 5 + 6 files changed, 116 insertions(+), 43 deletions(-) create mode 100644 projects/rocprofiler/test/tool/input2.xml diff --git a/projects/rocprofiler/src/core/rocprofiler.cpp b/projects/rocprofiler/src/core/rocprofiler.cpp index 8eed0fcda7..c49a301885 100644 --- a/projects/rocprofiler/src/core/rocprofiler.cpp +++ b/projects/rocprofiler/src/core/rocprofiler.cpp @@ -90,6 +90,8 @@ decltype(hsa_amd_memory_async_copy_rect)* hsa_amd_memory_async_copy_rect_fn; ::HsaApiTable* kHsaApiTable; void SaveHsaApi(::HsaApiTable* table) { + util::HsaRsrcFactory::InitHsaApiTable(table); + kHsaApiTable = table; hsa_queue_create_fn = table->core_->hsa_queue_create_fn; hsa_queue_destroy_fn = table->core_->hsa_queue_destroy_fn; diff --git a/projects/rocprofiler/src/util/hsa_rsrc_factory.cpp b/projects/rocprofiler/src/util/hsa_rsrc_factory.cpp index a3ad50a987..a141fe04c8 100644 --- a/projects/rocprofiler/src/util/hsa_rsrc_factory.cpp +++ b/projects/rocprofiler/src/util/hsa_rsrc_factory.cpp @@ -117,7 +117,7 @@ HsaRsrcFactory::HsaRsrcFactory(bool initialize_hsa) : initialize_hsa_(initialize cpu_pool_ = NULL; kern_arg_pool_ = NULL; - InitHsaApiTable(); + InitHsaApiTable(NULL); // Initialize the Hsa Runtime if (initialize_hsa_) { @@ -165,42 +165,86 @@ HsaRsrcFactory::~HsaRsrcFactory() { } } -void HsaRsrcFactory::InitHsaApiTable() { - hsa_api_.hsa_init = hsa_init; - hsa_api_.hsa_shut_down = hsa_shut_down; - hsa_api_.hsa_agent_get_info = hsa_agent_get_info; +void HsaRsrcFactory::InitHsaApiTable(HsaApiTable* table) { + std::lock_guard lck(mutex_); - hsa_api_.hsa_iterate_agents = hsa_iterate_agents; + if (hsa_api_.hsa_init == NULL) { + if (table != NULL) { + hsa_api_.hsa_init = table->core_->hsa_init_fn; + hsa_api_.hsa_shut_down = table->core_->hsa_shut_down_fn; + hsa_api_.hsa_agent_get_info = table->core_->hsa_agent_get_info_fn; - hsa_api_.hsa_queue_create = hsa_queue_create; - hsa_api_.hsa_queue_load_write_index_relaxed = hsa_queue_load_write_index_relaxed; - hsa_api_.hsa_queue_store_write_index_relaxed = hsa_queue_store_write_index_relaxed; - hsa_api_.hsa_queue_load_read_index_relaxed = hsa_queue_load_read_index_relaxed; - hsa_api_.hsa_signal_create = hsa_signal_create; - hsa_api_.hsa_signal_destroy = hsa_signal_destroy; - hsa_api_.hsa_signal_store_relaxed = hsa_signal_store_relaxed; - hsa_api_.hsa_signal_wait_scacquire = hsa_signal_wait_scacquire; + hsa_api_.hsa_iterate_agents = table->core_->hsa_iterate_agents_fn; - hsa_api_.hsa_amd_agent_iterate_memory_pools = hsa_amd_agent_iterate_memory_pools; - hsa_api_.hsa_amd_memory_pool_get_info = hsa_amd_memory_pool_get_info; - hsa_api_.hsa_amd_memory_pool_allocate = hsa_amd_memory_pool_allocate; - hsa_api_.hsa_amd_agents_allow_access = hsa_amd_agents_allow_access; + hsa_api_.hsa_queue_create = table->core_->hsa_queue_create_fn; + hsa_api_.hsa_queue_destroy = table->core_->hsa_queue_destroy_fn; + hsa_api_.hsa_queue_load_write_index_relaxed = table->core_->hsa_queue_load_write_index_relaxed_fn; + hsa_api_.hsa_queue_store_write_index_relaxed = table->core_->hsa_queue_store_write_index_relaxed_fn; + hsa_api_.hsa_queue_load_read_index_relaxed = table->core_->hsa_queue_load_read_index_relaxed_fn; + hsa_api_.hsa_signal_create = table->core_->hsa_signal_create_fn; + hsa_api_.hsa_signal_destroy = table->core_->hsa_signal_destroy_fn; + hsa_api_.hsa_signal_load_relaxed = table->core_->hsa_signal_load_relaxed_fn; + hsa_api_.hsa_signal_store_relaxed = table->core_->hsa_signal_store_relaxed_fn; + hsa_api_.hsa_signal_store_screlease = table->core_->hsa_signal_store_screlease_fn; + hsa_api_.hsa_signal_wait_scacquire = table->core_->hsa_signal_wait_scacquire_fn; - hsa_api_.hsa_amd_memory_async_copy = hsa_amd_memory_async_copy; + hsa_api_.hsa_system_get_major_extension_table = table->core_->hsa_system_get_major_extension_table_fn; - hsa_api_.hsa_system_get_major_extension_table = hsa_system_get_major_extension_table; + hsa_api_.hsa_code_object_reader_create_from_file = table->core_->hsa_code_object_reader_create_from_file_fn; + hsa_api_.hsa_executable_create_alt = table->core_->hsa_executable_create_alt_fn; + hsa_api_.hsa_executable_load_agent_code_object = table->core_->hsa_executable_load_agent_code_object_fn; + hsa_api_.hsa_executable_freeze = table->core_->hsa_executable_freeze_fn; + hsa_api_.hsa_executable_get_symbol = table->core_->hsa_executable_get_symbol_fn; - hsa_api_.hsa_code_object_reader_create_from_file = hsa_code_object_reader_create_from_file; - hsa_api_.hsa_executable_create_alt = hsa_executable_create_alt; - hsa_api_.hsa_executable_load_agent_code_object = hsa_executable_load_agent_code_object; - hsa_api_.hsa_executable_freeze = hsa_executable_freeze; - hsa_api_.hsa_executable_get_symbol = hsa_executable_get_symbol; + hsa_api_.hsa_amd_agent_iterate_memory_pools = table->amd_ext_->hsa_amd_agent_iterate_memory_pools_fn; + hsa_api_.hsa_amd_memory_pool_get_info = table->amd_ext_->hsa_amd_memory_pool_get_info_fn; + hsa_api_.hsa_amd_memory_pool_allocate = table->amd_ext_->hsa_amd_memory_pool_allocate_fn; + hsa_api_.hsa_amd_agents_allow_access = table->amd_ext_->hsa_amd_agents_allow_access_fn; - hsa_api_.hsa_amd_signal_async_handler = hsa_amd_signal_async_handler; - hsa_api_.hsa_amd_profiling_get_async_copy_time = hsa_amd_profiling_get_async_copy_time; - hsa_api_.hsa_amd_profiling_get_dispatch_time = hsa_amd_profiling_get_dispatch_time; - hsa_api_.hsa_signal_load_relaxed = hsa_signal_load_relaxed; - hsa_api_.hsa_signal_store_screlease = hsa_signal_store_screlease; + hsa_api_.hsa_amd_memory_async_copy = table->amd_ext_->hsa_amd_memory_async_copy_fn; + + hsa_api_.hsa_amd_signal_async_handler = table->amd_ext_->hsa_amd_signal_async_handler_fn; + hsa_api_.hsa_amd_profiling_get_async_copy_time = table->amd_ext_->hsa_amd_profiling_get_async_copy_time_fn; + hsa_api_.hsa_amd_profiling_get_dispatch_time = table->amd_ext_->hsa_amd_profiling_get_dispatch_time_fn; + } else { + hsa_api_.hsa_init = hsa_init; + hsa_api_.hsa_shut_down = hsa_shut_down; + hsa_api_.hsa_agent_get_info = hsa_agent_get_info; + + hsa_api_.hsa_iterate_agents = hsa_iterate_agents; + + hsa_api_.hsa_queue_create = hsa_queue_create; + hsa_api_.hsa_queue_destroy = hsa_queue_destroy; + hsa_api_.hsa_queue_load_write_index_relaxed = hsa_queue_load_write_index_relaxed; + hsa_api_.hsa_queue_store_write_index_relaxed = hsa_queue_store_write_index_relaxed; + hsa_api_.hsa_queue_load_read_index_relaxed = hsa_queue_load_read_index_relaxed; + hsa_api_.hsa_signal_create = hsa_signal_create; + hsa_api_.hsa_signal_destroy = hsa_signal_destroy; + hsa_api_.hsa_signal_store_relaxed = hsa_signal_store_relaxed; + hsa_api_.hsa_signal_wait_scacquire = hsa_signal_wait_scacquire; + + hsa_api_.hsa_amd_agent_iterate_memory_pools = hsa_amd_agent_iterate_memory_pools; + hsa_api_.hsa_amd_memory_pool_get_info = hsa_amd_memory_pool_get_info; + hsa_api_.hsa_amd_memory_pool_allocate = hsa_amd_memory_pool_allocate; + hsa_api_.hsa_amd_agents_allow_access = hsa_amd_agents_allow_access; + + hsa_api_.hsa_amd_memory_async_copy = hsa_amd_memory_async_copy; + + hsa_api_.hsa_system_get_major_extension_table = hsa_system_get_major_extension_table; + + hsa_api_.hsa_code_object_reader_create_from_file = hsa_code_object_reader_create_from_file; + hsa_api_.hsa_executable_create_alt = hsa_executable_create_alt; + hsa_api_.hsa_executable_load_agent_code_object = hsa_executable_load_agent_code_object; + hsa_api_.hsa_executable_freeze = hsa_executable_freeze; + hsa_api_.hsa_executable_get_symbol = hsa_executable_get_symbol; + + hsa_api_.hsa_amd_signal_async_handler = hsa_amd_signal_async_handler; + hsa_api_.hsa_amd_profiling_get_async_copy_time = hsa_amd_profiling_get_async_copy_time; + hsa_api_.hsa_amd_profiling_get_dispatch_time = hsa_amd_profiling_get_dispatch_time; + hsa_api_.hsa_signal_load_relaxed = hsa_signal_load_relaxed; + hsa_api_.hsa_signal_store_screlease = hsa_signal_store_screlease; + } + } } hsa_status_t HsaRsrcFactory::LoadAqlProfileLib(aqlprofile_pfn_t* api) { @@ -635,7 +679,7 @@ uint64_t HsaRsrcFactory::Submit(hsa_queue_t* queue, const void* packet, size_t s return write_idx; } -HsaRsrcFactory* HsaRsrcFactory::instance_ = NULL; +std::atomic HsaRsrcFactory::instance_{}; HsaRsrcFactory::mutex_t HsaRsrcFactory::mutex_; HsaRsrcFactory::timestamp_t HsaRsrcFactory::timeout_ns_ = HsaTimer::TIMESTAMP_MAX; hsa_pfn_t HsaRsrcFactory::hsa_api_{}; diff --git a/projects/rocprofiler/src/util/hsa_rsrc_factory.h b/projects/rocprofiler/src/util/hsa_rsrc_factory.h index faea67b850..a1bc4b2eb5 100644 --- a/projects/rocprofiler/src/util/hsa_rsrc_factory.h +++ b/projects/rocprofiler/src/util/hsa_rsrc_factory.h @@ -26,6 +26,7 @@ POSSIBILITY OF SUCH DAMAGE. #define SRC_UTIL_HSA_RSRC_FACTORY_H_ #include +#include #include #include #include @@ -35,6 +36,7 @@ POSSIBILITY OF SUCH DAMAGE. #include #include +#include #include #include #include @@ -77,6 +79,7 @@ struct hsa_pfn_t { decltype(hsa_iterate_agents)* hsa_iterate_agents; decltype(hsa_queue_create)* hsa_queue_create; + decltype(hsa_queue_destroy)* hsa_queue_destroy; decltype(hsa_queue_load_write_index_relaxed)* hsa_queue_load_write_index_relaxed; decltype(hsa_queue_store_write_index_relaxed)* hsa_queue_store_write_index_relaxed; decltype(hsa_queue_load_read_index_relaxed)* hsa_queue_load_read_index_relaxed; @@ -201,17 +204,20 @@ class HsaRsrcFactory { static HsaRsrcFactory* Create(bool initialize_hsa = true) { std::lock_guard lck(mutex_); - if (instance_ == NULL) { - instance_ = new HsaRsrcFactory(initialize_hsa); + HsaRsrcFactory* obj = instance_.load(std::memory_order_relaxed); + if (obj == NULL) { + obj = new HsaRsrcFactory(initialize_hsa); + instance_.store(obj, std::memory_order_release); } - return instance_; + return obj; } static HsaRsrcFactory& Instance() { - if (instance_ == NULL) instance_ = Create(false); - hsa_status_t status = (instance_ != NULL) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; + HsaRsrcFactory* obj = instance_.load(std::memory_order_acquire); + if (obj == NULL) obj = Create(false); + hsa_status_t status = (obj != NULL) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; CHECK_STATUS("HsaRsrcFactory::Instance() failed", status); - return *instance_; + return *obj; } static void Destroy() { @@ -314,7 +320,7 @@ class HsaRsrcFactory { static uint64_t Submit(hsa_queue_t* queue, const void* packet, size_t size_bytes); // Initialize HSA API table - void static InitHsaApiTable(); + void static InitHsaApiTable(HsaApiTable* table); static const hsa_pfn_t* HsaApi() { return &hsa_api_; } // Return AqlProfile API table @@ -334,7 +340,7 @@ class HsaRsrcFactory { static void SetTimeoutNs(const timestamp_t& time) { std::lock_guard lck(mutex_); timeout_ns_ = time; - if (instance_ != NULL) instance_->timeout_ = instance_->timer_->ns_to_sysclock(time); + if (instance_ != NULL) Instance().timeout_ = Instance().timer_->ns_to_sysclock(time); } private: @@ -363,7 +369,7 @@ class HsaRsrcFactory { // HSA was initialized const bool initialize_hsa_; - static HsaRsrcFactory* instance_; + static std::atomic instance_; static mutex_t mutex_; // Used to maintain a list of Hsa Gpu Agent Info diff --git a/projects/rocprofiler/test/run.sh b/projects/rocprofiler/test/run.sh index 6f6d1f88e5..543625d95c 100755 --- a/projects/rocprofiler/test/run.sh +++ b/projects/rocprofiler/test/run.sh @@ -88,6 +88,13 @@ export ROCP_THRS=1 export ROCP_INPUT=input.xml eval_test "'rocprof' libtool test" ./test/ctrl +export ROCP_KITER=10 +export ROCP_DITER=10 +export ROCP_AGENTS=1 +export ROCP_THRS=10 +export ROCP_INPUT=input1.xml +eval_test "'rocprof' libtool test n-threads" ./test/ctrl + ## Libtool test, counter sets # Memcopies tracking @@ -95,7 +102,7 @@ export ROCP_MCOPY_TRACKING=1 export ROCP_KITER=1 export ROCP_DITER=4 -export ROCP_INPUT=input1.xml +export ROCP_INPUT=input2.xml eval_test "libtool test, counter sets" ./test/ctrl #valgrind --leak-check=full $tbin diff --git a/projects/rocprofiler/test/tool/input1.xml b/projects/rocprofiler/test/tool/input1.xml index 254c83dc90..f4ecd1786f 100644 --- a/projects/rocprofiler/test/tool/input1.xml +++ b/projects/rocprofiler/test/tool/input1.xml @@ -1,5 +1,14 @@ +# Filter by dispatches range, GPU index and kernel names + + # List of metrics diff --git a/projects/rocprofiler/test/tool/input2.xml b/projects/rocprofiler/test/tool/input2.xml new file mode 100644 index 0000000000..254c83dc90 --- /dev/null +++ b/projects/rocprofiler/test/tool/input2.xml @@ -0,0 +1,5 @@ +# List of metrics +