Add rocprofiler_load_counter_definition (#1193)

Adds rocprofiler_load_counter_definition. This function allows a counter definition file to be supplied to rocprofiler-sdk directly. Takes in a string containing the counter definition YAML, its size (in bytes), and a flag value to state whether this is an append operation or not.

---------

Co-authored-by: Benjamin Welton <ben@amd.com>
Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com>
Co-authored-by: usrihari123 <srihari.u@amd.com>

[ROCm/rocprofiler-sdk commit: 7ddc72ad45]

projects/rocprofiler-sdk/source/lib/output/generateJSON.hpp

@@ -56,9 +56,9 @@ struct json_output
         return (*archive)(std::forward<Args>(args)...);
     }
 
-    decltype(auto) startNode() { return archive->startNode(); }
-    decltype(auto) finishNode() { return archive->finishNode(); }
-    decltype(auto) makeArray() { return archive->makeArray(); }
+    void           startNode() { archive->startNode(); }
+    void           finishNode() { archive->finishNode(); }
+    void           makeArray() { archive->makeArray(); }
     decltype(auto) setNextName(const char* name) { archive->setNextName(name); }
 
     void start_process();

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/config.hpp

@@ -96,8 +96,9 @@ struct config : output_config
     int mpi_size = get_mpi_size();
     int mpi_rank = get_mpi_rank();
 
-    std::string kernel_filter_include = get_env("ROCPROF_KERNEL_FILTER_INCLUDE_REGEX", ".*");
-    std::string kernel_filter_exclude = get_env("ROCPROF_KERNEL_FILTER_EXCLUDE_REGEX", "");
+    std::string kernel_filter_include   = get_env("ROCPROF_KERNEL_FILTER_INCLUDE_REGEX", ".*");
+    std::string kernel_filter_exclude   = get_env("ROCPROF_KERNEL_FILTER_EXCLUDE_REGEX", "");
+    std::string extra_counters_contents = get_env("ROCPROF_EXTRA_COUNTERS_CONTENTS", "");
 
     std::unordered_set<uint32_t> kernel_filter_range = {};
     std::set<std::string>        counters            = {};

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk-tool/tool.cpp

@@ -49,6 +49,7 @@
 #include <rocprofiler-sdk/agent.h>
 #include <rocprofiler-sdk/buffer_tracing.h>
 #include <rocprofiler-sdk/callback_tracing.h>
+#include <rocprofiler-sdk/experimental/counters.h>
 #include <rocprofiler-sdk/external_correlation.h>
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/internal_threading.h>
@@ -1597,6 +1598,15 @@ rocprofiler_configure(uint32_t                 version,
         return compose_tmp_file_name(tool::get_config(), type);
     };
 
+    if(!tool::get_config().extra_counters_contents.empty())
+    {
+        std::string contents(tool::get_config().extra_counters_contents);
+        size_t      length = contents.size();
+        ROCPROFILER_CALL(rocprofiler_load_counter_definition(
+                             contents.c_str(), length, ROCPROFILER_COUNTER_FLAG_APPEND_DEFINITION),
+                         "Loading extra counters");
+    }
+
     if(tool::get_config().list_metrics)
     {
         tool_metadata->init(tool::metadata::inprocess{});

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters.cpp

@@ -20,6 +20,7 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 // SOFTWARE.
 
+#include <rocprofiler-sdk/experimental/counters.h>
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/rocprofiler.h>
 
@@ -191,4 +192,15 @@ rocprofiler_iterate_counter_dimensions(rocprofiler_counter_id_t              id,
 
     return ROCPROFILER_STATUS_SUCCESS;
 }
+
+rocprofiler_status_t
+rocprofiler_load_counter_definition(const char* yaml, size_t size, rocprofiler_counter_flag_t flags)
+{
+    rocprofiler::counters::CustomCounterDefinition def;
+    if(yaml == nullptr && size != 0) return ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT;
+    def.data   = std::string(yaml, size);
+    def.append = (flags == ROCPROFILER_COUNTER_FLAG_APPEND_DEFINITION ? true : false);
+    def.loaded = false;
+    return rocprofiler::counters::setCustomCounterDefinition(def);
+}
 }

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/metrics.cpp

@@ -26,11 +26,13 @@
 
 #include "lib/common/filesystem.hpp"
 #include "lib/common/static_object.hpp"
+#include "lib/common/synchronized.hpp"
 #include "lib/common/utility.hpp"
 #include "lib/rocprofiler-sdk/agent.hpp"
 
 #include "glog/logging.h"
 
+#include "rocprofiler-sdk/fwd.h"
 #include "yaml-cpp/exceptions.h"
 #include "yaml-cpp/node/convert.h"
 #include "yaml-cpp/node/detail/impl.h"
@@ -50,6 +52,13 @@ namespace counters
 {
 namespace
 {
+common::Synchronized<CustomCounterDefinition>&
+getCustomCounterDefinition()
+{
+    static common::Synchronized<CustomCounterDefinition> def = {};
+    return def;
+}
+
 uint64_t&
 current_id()
 {
@@ -104,12 +113,31 @@ MetricMap
 loadYAML(const std::string& filename, bool load_constants = false, bool load_derived = false)
 {
     MetricMap ret;
-    ROCP_INFO << "Loading Counter Config: " << filename;
-    auto yaml = YAML::LoadFile(filename);
+    auto      override = getCustomCounterDefinition().wlock([&](auto& data) {
+        data.loaded = true;
+        return data;
+    });
+
+    std::stringstream counter_data;
+    if(override.data.empty() || override.append)
+    {
+        ROCP_INFO << "Loading Counter Config: " << filename;
+        std::ifstream file(filename);
+        counter_data << file.rdbuf();
+    }
+
+    if(!override.data.empty())
+    {
+        ROCP_INFO << "Adding Override Config Data: " << override.data;
+        counter_data << override.data;
+    }
+
+    auto yaml = YAML::Load(counter_data.str());
 
     for(auto it = yaml.begin(); it != yaml.end(); ++it)
     {
         auto counter_name = it->first.as<std::string>();
+        if(counter_name == "schema-version") continue;
         auto counter_def  = it->second;
         auto def_iterator = counter_def["architectures"];
 
@@ -189,6 +217,18 @@ findViaEnvironment(const std::string& filename)
 
 }  // namespace
 
+rocprofiler_status_t
+setCustomCounterDefinition(const CustomCounterDefinition& def)
+{
+    return getCustomCounterDefinition().wlock([&](auto& data) {
+        // Counter definition already loaded, cannot override anymore
+        if(data.loaded) return ROCPROFILER_STATUS_ERROR;
+        data.data   = def.data;
+        data.append = def.append;
+        return ROCPROFILER_STATUS_SUCCESS;
+    });
+}
+
 MetricMap
 getDerivedHardwareMetrics()
 {

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/metrics.hpp

@@ -26,6 +26,7 @@
 #include <string>
 #include <unordered_map>
 #include <vector>
+#include "rocprofiler-sdk/fwd.h"
 
 #include <fmt/core.h>
 #include <fmt/ranges.h>
@@ -84,6 +85,13 @@ private:
     uint32_t    flags_       = 0;
 };
 
+struct CustomCounterDefinition
+{
+    std::string data   = {};
+    bool        append = {false};
+    bool        loaded = {false};
+};
+
 using MetricMap   = std::unordered_map<std::string, std::vector<Metric>>;
 using MetricIdMap = std::unordered_map<uint64_t, Metric>;
 
@@ -130,6 +138,12 @@ getPerfCountersIdMap();
  **/
 bool
 checkValidMetric(const std::string& agent, const Metric& metric);
+
+/**
+ * Set a custom counter definition
+ */
+rocprofiler_status_t
+setCustomCounterDefinition(const CustomCounterDefinition& def);
 }  // namespace counters
 }  // namespace rocprofiler
 

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/tests/core.cpp

@@ -34,6 +34,7 @@
 #include "lib/rocprofiler-sdk/registration.hpp"
 
 #include <rocprofiler-sdk/dispatch_counting_service.h>
+#include <rocprofiler-sdk/experimental/counters.h>
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/registration.h>
 #include <rocprofiler-sdk/rocprofiler.h>
@@ -746,3 +747,65 @@ TEST(core, public_api_iterate_agents)
         EXPECT_TRUE(from_api.empty());
     }
 }
+
+TEST(core, check_load_counter_def_append)
+{
+    const std::string test_yaml = R"(
+TEST_YAML_LOAD:
+  architectures:
+    gfx942/gfx10/gfx1010/gfx1030/gfx1031/gfx11/gfx1032/gfx1102/gfx906/gfx1100/gfx1101/gfx908/gfx90a/gfx9:
+      expression: reduce(GRBM_GUI_ACTIVE,max)*CU_NUM
+  description: 'Unit: cycles'
+    )";
+    ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS);
+    test_init();
+
+    registration::init_logging();
+    registration::set_init_status(-1);
+    context::push_client(1);
+
+    ROCPROFILER_CALL(
+        rocprofiler_load_counter_definition(
+            test_yaml.c_str(), test_yaml.size(), ROCPROFILER_COUNTER_FLAG_APPEND_DEFINITION),
+        "Could not load counter definition");
+    auto agents = hsa::get_queue_controller()->get_supported_agents();
+    for(const auto& [_, agent] : agents)
+    {
+        EXPECT_EQ(findDeviceMetrics(agent, {"TEST_YAML_LOAD"}).size(), 1);
+    }
+}
+
+TEST(core, check_load_counter_def)
+{
+    const std::string test_yaml = R"(
+GRBM_GUI_ACTIVE:
+  architectures:
+    gfx942/gfx941/gfx10/gfx1010/gfx1030/gfx1031/gfx11/gfx1032/gfx1102/gfx906/gfx1100/gfx1101/gfx940/gfx908/gfx900/gfx90a/gfx9:
+      block: GRBM
+      event: 2
+  description: The GUI is Active
+TEST_YAML_LOAD:
+  architectures:
+    gfx942/gfx10/gfx1010/gfx1030/gfx1031/gfx11/gfx1032/gfx1102/gfx906/gfx1100/gfx1101/gfx908/gfx90a/gfx9:
+      expression: reduce(GRBM_GUI_ACTIVE,max)
+  description: cycles
+    )";
+    ASSERT_EQ(hsa_init(), HSA_STATUS_SUCCESS);
+    test_init();
+
+    registration::init_logging();
+    registration::set_init_status(-1);
+    context::push_client(1);
+
+    ROCPROFILER_CALL(rocprofiler_load_counter_definition(
+                         test_yaml.c_str(), test_yaml.size(), ROCPROFILER_COUNTER_FLAG_NONE),
+                     "Could not load counter definition");
+    auto agents = hsa::get_queue_controller()->get_supported_agents();
+    for(const auto& [_, agent] : agents)
+    {
+        // MAX_WAVE_SIZE should not be present
+        EXPECT_EQ(
+            findDeviceMetrics(agent, {"TEST_YAML_LOAD", "GRBM_GUI_ACTIVE", "MAX_WAVE_SIZE"}).size(),
+            2);
+    }
+}

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/counters/yaml/counter_defs.yaml

@@ -1,3 +1,4 @@
+schema-version: 1
 ALUStalledByLDS:
   architectures:
     gfx10/gfx1010/gfx1030/gfx1031/gfx11/gfx1032/gfx1102/gfx906/gfx1100/gfx1101/gfx908/gfx90a/gfx9: