From 162c77d34974f52abb4adc764196299292ffb812 Mon Sep 17 00:00:00 2001 From: "Jonathan R. Madsen" Date: Thu, 19 Oct 2023 11:43:15 -0500 Subject: [PATCH] Conditional Queue Interception (#141) - only intercept queue if there is a context that requires it - make copy of HSA API table after internal modifications to the table to prevent those changes being lost when HSA API tracing is enabled [ROCm/rocprofiler-sdk commit: 5819ca589f431cabe9410b674d4b17c52708b407] --- .../lib/rocprofiler/hsa/queue_controller.cpp | 35 +++++++++++++++++-- .../source/lib/rocprofiler/registration.cpp | 8 +++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler/hsa/queue_controller.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler/hsa/queue_controller.cpp index 18ea5730a1..ee82557d0c 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler/hsa/queue_controller.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler/hsa/queue_controller.cpp @@ -19,6 +19,7 @@ // THE SOFTWARE. #include "lib/rocprofiler/hsa/queue_controller.hpp" +#include "lib/rocprofiler/context/context.hpp" #include @@ -139,9 +140,6 @@ QueueController::init(CoreApiTable& core_table, AmdExtTable& ext_table) _core_table = core_table; _ext_table = ext_table; - core_table.hsa_queue_create_fn = create_queue; - core_table.hsa_queue_destroy_fn = destroy_queue; - // Generate supported agents rocprofiler_query_available_agents( [](const rocprofiler_agent_t** agents, size_t num_agents, void* user_data) { @@ -167,6 +165,37 @@ QueueController::init(CoreApiTable& core_table, AmdExtTable& ext_table) }, sizeof(rocprofiler_agent_t), this); + + auto enable_intercepter = false; + for(const auto& itr : context::get_registered_contexts()) + { + constexpr auto expected_context_size = 160UL; + static_assert( + sizeof(context::context) == expected_context_size, + "If you added a new field to context struct, make sure there is a check here if it " + "requires queue interception. Once you have done so, increment expected_context_size"); + + if(itr->counter_collection) + { + enable_intercepter = true; + break; + } + else if(itr->buffered_tracer) + { + if(itr->buffered_tracer->domains(ROCPROFILER_SERVICE_BUFFER_TRACING_KERNEL_DISPATCH) || + itr->buffered_tracer->domains(ROCPROFILER_SERVICE_BUFFER_TRACING_MEMORY_COPY)) + { + enable_intercepter = true; + break; + } + } + } + + if(enable_intercepter) + { + core_table.hsa_queue_create_fn = create_queue; + core_table.hsa_queue_destroy_fn = destroy_queue; + } } QueueController& diff --git a/projects/rocprofiler-sdk/source/lib/rocprofiler/registration.cpp b/projects/rocprofiler-sdk/source/lib/rocprofiler/registration.cpp index 28272d90fc..800270b5b0 100644 --- a/projects/rocprofiler-sdk/source/lib/rocprofiler/registration.cpp +++ b/projects/rocprofiler-sdk/source/lib/rocprofiler/registration.cpp @@ -540,11 +540,15 @@ rocprofiler_set_api_table(const char* name, LOG_IF(ERROR, num_tables > 1) << " rocprofiler expected HSA library to pass 1 API table, not " << num_tables; - auto* hsa_api_table = static_cast(*tables); + auto* hsa_api_table = static_cast(*tables); + rocprofiler::hsa::queue_controller_init(hsa_api_table); + + // any internal modifications to the HsaApiTable need to be done before we make the + // copy or else those modifications will be lost when HSA API tracing is enabled + // because the HSA API tracing invokes the function pointers from the copy below auto& saved_hsa_api_table = rocprofiler::hsa::get_table(); ::copyTables(hsa_api_table, &saved_hsa_api_table); - rocprofiler::hsa::queue_controller_init(hsa_api_table); rocprofiler::hsa::update_table(hsa_api_table); } else if(std::string_view{name} == "roctx")