Counter API and Samples Updates (#410)

* Update include/rocprofiler-sdk/{counters,profile_config}.h

- use rocprofiler_agent_id_t instead of rocprofiler_agent_t

* Update samples

- use rocprofiler-sdk::rocprofiler-sdk instead of rocprofiler::rocprofiler in cmake
- api_callback_tracing sample roctxProfiler{Pause,Resume}
- api_callback_tracing sample uses ROCTx
- updates to use rocprofiler_agent_id_t

* Update run-ci.py

- exclude rocprofiler-sdk-tool from samples (no sample uses that code)

* Update lib/rocprofiler-sdk-tool/tool.cpp

- Update rocprofiler_iterate_agent_supported_counters to use agent ID

* Update lib/rocprofiler-sdk/counters/core.*

- profile_config has pointer to agent instead of copy

* Update lib/rocprofiler-sdk/agent.*

- provide get_agent(...) func via rocp agent id

* Update lib/rocprofiler-sdk/{buffer,callback}_tracing.cpp

- return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED for enums missing implementation

* Update lib/rocprofiler-sdk/counters.cpp

- update to use rocprofiler_agent_id_t instead of rocprofiler_agent_t

* Update lib/rocprofiler-sdk/profile_config.cpp

- update to use rocprofiler_agent_id_t instead of rocprofiler_agent_t

* Update source/docs

- requirements.txt + install reqs in cmake

* Bump version to 0.1.0

* Update samples/api_callback_tracing/CMakeLists.txt

- LD_LIBRARY_PATH for test

* Update test/rocprofv3/tracing/CMakeLists.txt

- reorder validation files so memory copy comes first

* Update lib/rocprofiler-sdk-tool/tool.cpp

- logging for flushing buffers
- variables for buffer_size and buffer_watermark
  - increase the watermark to a full buffer
- use dedicated threads for each buffer

* Update lib/rocprofiler-sdk-tool/CMakeLists.txt

- test sets ROCPROF_LOG_LEVEL and ROCPROFILER_LOG_LEVEL to info

* Remove lib/rocprofiler-sdk-tool/trace_buffer.hpp

* Update lib/rocprofiler-sdk-tool/CMakeLists.txt

- drop log level to warning when leak sanitizer is enabled (produces small memory leak)

source/lib/rocprofiler-sdk/agent.cpp

@@ -671,6 +671,16 @@ get_agents()
     return pointers;
 }
 
+const rocprofiler_agent_t*
+get_agent(rocprofiler_agent_id_t id)
+{
+    for(const auto& itr : get_agents())
+    {
+        if(itr && itr->id.handle == id.handle) return itr;
+    }
+    return nullptr;
+}
+
 void
 construct_agent_cache(::HsaApiTable* table)
 {

source/lib/rocprofiler-sdk/agent.hpp

@@ -39,6 +39,9 @@ namespace agent
 std::vector<const rocprofiler_agent_t*>
 get_agents();
 
+const rocprofiler_agent_t*
+get_agent(rocprofiler_agent_id_t id);
+
 void
 construct_agent_cache(::HsaApiTable* table);
 

source/lib/rocprofiler-sdk/buffer_tracing.cpp

@@ -96,6 +96,12 @@ rocprofiler_configure_buffer_tracing_service(rocprofiler_context_id_t          c
     if(rocprofiler::registration::get_init_status() > -1)
         return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED;
 
+    static auto unsupported = std::unordered_set<rocprofiler_buffer_tracing_kind_t>{
+        ROCPROFILER_BUFFER_TRACING_PAGE_MIGRATION,
+        ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY,
+        ROCPROFILER_BUFFER_TRACING_EXTERNAL_CORRELATION};
+    if(unsupported.count(kind) > 0) return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
+
     auto* ctx = rocprofiler::context::get_mutable_registered_context(context_id);
 
     if(!ctx) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;

source/lib/rocprofiler-sdk/callback_tracing.cpp

@@ -92,6 +92,10 @@ rocprofiler_configure_callback_tracing_service(rocprofiler_context_id_t
     if(rocprofiler::registration::get_init_status() > -1)
         return ROCPROFILER_STATUS_ERROR_CONFIGURATION_LOCKED;
 
+    static auto unsupported = std::unordered_set<rocprofiler_callback_tracing_kind_t>{
+        ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH};
+    if(unsupported.count(kind) > 0) return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
+
     auto* ctx = rocprofiler::context::get_mutable_registered_context(context_id);
 
     if(!ctx) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;

source/lib/rocprofiler-sdk/counters.cpp

@@ -26,6 +26,7 @@
 #include <fmt/core.h>
 
 #include "lib/common/synchronized.hpp"
+#include "lib/rocprofiler-sdk/agent.hpp"
 #include "lib/rocprofiler-sdk/aql/helpers.hpp"
 #include "lib/rocprofiler-sdk/counters/evaluate_ast.hpp"
 #include "lib/rocprofiler-sdk/counters/id_decode.hpp"
@@ -43,7 +44,7 @@ extern "C" {
  * @param [out] size
  * @return ::rocprofiler_status_t
  */
-rocprofiler_status_t ROCPROFILER_API
+rocprofiler_status_t
 rocprofiler_query_counter_name(rocprofiler_counter_id_t counter_id, const char** name, size_t* size)
 {
     const auto& id_map = *CHECK_NOTNULL(rocprofiler::counters::getMetricIdMap());
@@ -71,11 +72,16 @@ rocprofiler_query_counter_name(rocprofiler_counter_id_t counter_id, const char**
  * @param [out] instance_count number of instances the counter has
  * @return rocprofiler_status_t
  */
-rocprofiler_status_t ROCPROFILER_API
-rocprofiler_query_counter_instance_count(rocprofiler_agent_t      agent,
+rocprofiler_status_t
+rocprofiler_query_counter_instance_count(rocprofiler_agent_id_t   agent_id,
                                          rocprofiler_counter_id_t counter_id,
                                          size_t*                  instance_count)
 {
+    const rocprofiler_agent_t* agent = rocprofiler::agent::get_agent(agent_id);
+
+    if(!agent) return ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND;
+    if(agent->type != ROCPROFILER_AGENT_TYPE_GPU) return ROCPROFILER_STATUS_ERROR;
+
     const auto& id_map     = *CHECK_NOTNULL(rocprofiler::counters::getMetricIdMap());
     const auto* metric_ptr = rocprofiler::common::get_val(id_map, counter_id.handle);
     if(!metric_ptr) return ROCPROFILER_STATUS_ERROR_COUNTER_NOT_FOUND;
@@ -93,19 +99,9 @@ rocprofiler_query_counter_instance_count(rocprofiler_agent_t      agent,
     // For derived metrics, this can be more than one counter. In that case,
     // we return the maximum instance count among all underlying counters.
     auto req_counters = rocprofiler::counters::get_required_hardware_counters(
-        rocprofiler::counters::get_ast_map(), std::string(agent.name), *metric_ptr);
+        rocprofiler::counters::get_ast_map(), std::string(agent->name), *metric_ptr);
     if(!req_counters) return ROCPROFILER_STATUS_ERROR_COUNTER_NOT_FOUND;
 
-    // NOTE: to look up instance information, we require HSA be init'd. Reason
-    // for this is the call to get instance information is an HSA call.
-    const auto* maybe_agent = rocprofiler::common::get_val(
-        rocprofiler::hsa::get_queue_controller().get_supported_agents(), agent.id.handle);
-    if(!maybe_agent)
-    {
-        LOG(ERROR) << "HSA must be loaded to obtain instance information.";
-        return ROCPROFILER_STATUS_ERROR;
-    }
-
     for(const auto& counter : *req_counters)
     {
         if(!counter.special().empty())
@@ -116,7 +112,7 @@ rocprofiler_query_counter_instance_count(rocprofiler_agent_t      agent,
 
         try
         {
-            auto dims = rocprofiler::counters::getBlockDimensions(maybe_agent->name(), counter);
+            auto dims = rocprofiler::counters::getBlockDimensions(agent->name, counter);
             for(const auto& dim : dims)
             {
                 *instance_count = std::max(static_cast<size_t>(dim.size()), *instance_count);
@@ -138,12 +134,15 @@ rocprofiler_query_counter_instance_count(rocprofiler_agent_t      agent,
  * @param [out] counters_count
  * @return ::rocprofiler_status_t
  */
-rocprofiler_status_t ROCPROFILER_API
-rocprofiler_iterate_agent_supported_counters(rocprofiler_agent_t                 agent,
+rocprofiler_status_t
+rocprofiler_iterate_agent_supported_counters(rocprofiler_agent_id_t              agent_id,
                                              rocprofiler_available_counters_cb_t cb,
                                              void*                               user_data)
 {
-    auto metrics = rocprofiler::counters::getMetricsForAgent(agent.name);
+    const auto* agent = rocprofiler::agent::get_agent(agent_id);
+    if(!agent) return ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND;
+
+    auto metrics = rocprofiler::counters::getMetricsForAgent(agent->name);
     std::vector<rocprofiler_counter_id_t> ids;
     ids.reserve(metrics.size());
     for(const auto& metric : metrics)
@@ -151,7 +150,7 @@ rocprofiler_iterate_agent_supported_counters(rocprofiler_agent_t
         ids.push_back({.handle = metric.id()});
     }
 
-    return cb(ids.data(), ids.size(), user_data);
+    return cb(agent_id, ids.data(), ids.size(), user_data);
 }
 
 /**
@@ -161,7 +160,7 @@ rocprofiler_iterate_agent_supported_counters(rocprofiler_agent_t
  * @param [out] counter_id counter id associated with the record
  * @return ::rocprofiler_status_t
  */
-rocprofiler_status_t ROCPROFILER_API
+rocprofiler_status_t
 rocprofiler_query_record_counter_id(rocprofiler_counter_instance_id_t id,
                                     rocprofiler_counter_id_t*         counter_id)
 {
@@ -170,7 +169,7 @@ rocprofiler_query_record_counter_id(rocprofiler_counter_instance_id_t id,
     return ROCPROFILER_STATUS_SUCCESS;
 }
 
-rocprofiler_status_t ROCPROFILER_API
+rocprofiler_status_t
 rocprofiler_query_record_dimension_position(rocprofiler_counter_instance_id_t  id,
                                             rocprofiler_counter_dimension_id_t dim,
                                             size_t*                            pos)
@@ -180,7 +179,7 @@ rocprofiler_query_record_dimension_position(rocprofiler_counter_instance_id_t  i
     return ROCPROFILER_STATUS_SUCCESS;
 }
 
-rocprofiler_status_t ROCPROFILER_API
+rocprofiler_status_t
 rocprofiler_query_record_dimension_info(rocprofiler_counter_id_t,
                                         rocprofiler_counter_dimension_id_t   dim,
                                         rocprofiler_record_dimension_info_t* info)

source/lib/rocprofiler-sdk/counters/core.cpp

@@ -183,7 +183,7 @@ queue_cb(const std::shared_ptr<counter_callback_info>& info,
             if(prof_config->reqired_hw_counters.empty())
             {
                 auto& config     = *prof_config;
-                auto  agent_name = std::string(config.agent.name);
+                auto  agent_name = std::string(config.agent->name);
                 for(const auto& metric : config.metrics)
                 {
                     auto req_counters =
@@ -273,7 +273,7 @@ completed_cb(const std::shared_ptr<counter_callback_info>& info,
 
     auto decoded_pkt = EvaluateAST::read_pkt(prof_config->pkt_generator.get(), *pkt);
     EvaluateAST::read_special_counters(
-        prof_config->agent, prof_config->required_special_counters, decoded_pkt);
+        *prof_config->agent, prof_config->required_special_counters, decoded_pkt);
 
     prof_config->packets.wlock([&](auto& pkt_vector) {
         if(pkt)

source/lib/rocprofiler-sdk/counters/core.hpp

@@ -45,7 +45,7 @@ namespace counters
 // This profile can be shared among many rocprof contexts.
 struct profile_config
 {
-    rocprofiler_agent_t           agent{};
+    const rocprofiler_agent_t*    agent = nullptr;
     std::vector<counters::Metric> metrics{};
     // HW counters that must be collected to compute the above
     // metrics (derived metrics are broken down into hw counters

source/lib/rocprofiler-sdk/profile_config.cpp

@@ -25,11 +25,13 @@
 
 #include "lib/common/synchronized.hpp"
 #include "lib/common/utility.hpp"
+#include "lib/rocprofiler-sdk/agent.hpp"
 #include "lib/rocprofiler-sdk/aql/helpers.hpp"
 #include "lib/rocprofiler-sdk/counters/core.hpp"
 #include "lib/rocprofiler-sdk/counters/evaluate_ast.hpp"
 #include "lib/rocprofiler-sdk/counters/metrics.hpp"
 #include "lib/rocprofiler-sdk/hsa/agent_cache.hpp"
+#include "rocprofiler-sdk/fwd.h"
 
 extern "C" {
 /**
@@ -41,12 +43,15 @@ extern "C" {
  * @param [out] config_id Identifier for GPU counters group
  * @return ::rocprofiler_status_t
  */
-rocprofiler_status_t ROCPROFILER_API
-rocprofiler_create_profile_config(rocprofiler_agent_t              agent,
+rocprofiler_status_t
+rocprofiler_create_profile_config(rocprofiler_agent_id_t           agent_id,
                                   rocprofiler_counter_id_t*        counters_list,
                                   size_t                           counters_count,
                                   rocprofiler_profile_config_id_t* config_id)
 {
+    const auto* agent = ::rocprofiler::agent::get_agent(agent_id);
+    if(!agent) return ROCPROFILER_STATUS_ERROR_AGENT_NOT_FOUND;
+
     std::shared_ptr<rocprofiler::counters::profile_config> config =
         std::make_shared<rocprofiler::counters::profile_config>();
 
@@ -57,7 +62,7 @@ rocprofiler_create_profile_config(rocprofiler_agent_t              agent,
 
         const auto* metric_ptr = rocprofiler::common::get_val(id_map, counter_id.handle);
         if(!metric_ptr) return ROCPROFILER_STATUS_ERROR_COUNTER_NOT_FOUND;
-        if(!rocprofiler::counters::checkValidMetric(std::string(agent.name), *metric_ptr))
+        if(!rocprofiler::counters::checkValidMetric(std::string(agent->name), *metric_ptr))
         {
             return ROCPROFILER_STATUS_ERROR_METRIC_NOT_VALID_FOR_AGENT;
         }
@@ -70,7 +75,7 @@ rocprofiler_create_profile_config(rocprofiler_agent_t              agent,
     return ROCPROFILER_STATUS_SUCCESS;
 }
 
-rocprofiler_status_t ROCPROFILER_API
+rocprofiler_status_t
 rocprofiler_destroy_profile_config(rocprofiler_profile_config_id_t config_id)
 {
     rocprofiler::counters::destroy_counter_profile(config_id.handle);