From 73948f95e2ccb7911855e86352e838cd3d8e5e56 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Mon, 10 Jun 2024 15:10:06 -0500 Subject: [PATCH] Rewrite rocprofiler plugin Change-Id: Ic7dd967cc60cacd2b16a465180505ea2a342fccf Signed-off-by: Galantsev, Dmitrii [ROCm/rdc commit: 3514225b83cdce8267a81cd15a3cc1b53f6808b6] --- .../rdc_modules/rdc_rocp/RdcRocpBase.h | 40 +++-- .../rdc_modules/rdc_rocp/RdcRocpBase.cc | 147 +++++++++++------- .../rdc_modules/rdc_rocp/RdcTelemetryLib.cc | 5 +- 3 files changed, 126 insertions(+), 66 deletions(-) diff --git a/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h b/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h index 25d1908d91..5ed43342fc 100644 --- a/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h +++ b/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h @@ -24,13 +24,14 @@ THE SOFTWARE. #define RDC_MODULES_RDC_ROCP_RDCROCPBASE_H_ #include -#include #include #include #include +#include #include #include "rdc/rdc.h" +#include "rdc_lib/RdcTelemetryLibInterface.h" namespace amd { namespace rdc { @@ -43,8 +44,6 @@ typedef struct { /// Common interface for RocP tests and samples class RdcRocpBase { - // typedef const char* rocp_metric_name_t; - public: RdcRocpBase(); RdcRocpBase(const RdcRocpBase&) = default; @@ -56,32 +55,49 @@ class RdcRocpBase { /** * @brief Lookup ROCProfiler counter * - * @param[in] field An existing field already added to sessions dictionary + * @param[in] gpu_field GPU_ID and FIELD_ID of requested metric * @param[out] value A pointer that will be populated with returned value * * @retval ::ROCMTOOLS_STATUS_SUCCESS The function has been executed * successfully. */ - rdc_status_t rocp_lookup(uint32_t gpu_index, rdc_field_t field, double* value); + rdc_status_t rocp_lookup(rdc_gpu_field_t gpu_field, double* value); const char* get_field_id_from_name(rdc_field_t); const std::vector get_field_ids(); + /** + * @brief Reset average metrics for gpu_field + */ + void reset_average(rdc_gpu_field_t gpu_field); + protected: private: - std::map metric_to_value = {}; - // array of features for each device - std::map feature; - void read_feature(rocprofiler_t* context, const unsigned feature_count, uint32_t gpu_index); - int run_profiler(uint32_t gpu_index, rdc_field_t field); - std::vector queues; + typedef struct { + std::vector buffer; + uint32_t index; + } rdc_average_t; + typedef std::pair rdc_field_pair_t; + static const size_t buffer_length_k = 5; + /** + * @brief Tweak this to change for how long each metric is collected + */ + static const uint32_t collection_duration_us_k = 10000; + + double read_feature(rocprofiler_t* context, uint32_t gpu_index); + double run_profiler(uint32_t gpu_index, rdc_field_t field); + double get_average(rdc_field_pair_t field_pair, double raw_value); + hsa_agent_arr_t agent_arr = {}; + std::vector queues; + std::map gpuid_to_feature; std::map field_to_metric = {}; + std::map average = {}; + // these fields must be divided by time passed std::unordered_set eval_fields = { RDC_FI_PROF_EVAL_MEM_R_BW, RDC_FI_PROF_EVAL_MEM_W_BW, RDC_FI_PROF_EVAL_FLOPS_16, RDC_FI_PROF_EVAL_FLOPS_32, RDC_FI_PROF_EVAL_FLOPS_64, }; - std::chrono::time_point start_time; /** * @brief Convert from rocmtools status into RDC status diff --git a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc index 299beb8301..b73a453720 100644 --- a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc +++ b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc @@ -29,18 +29,18 @@ THE SOFTWARE. #include #include #include -#include +#include #include #include #include -#include +#include #include #include // #include "hsa.h" #include "rdc/rdc.h" #include "rdc_lib/RdcLogger.h" -#include "rdc_lib/rdc_common.h" +#include "rdc_lib/RdcTelemetryLibInterface.h" namespace amd { namespace rdc { @@ -72,27 +72,31 @@ static hsa_status_t get_agent_handle_cb(hsa_agent_t agent, void* agent_arr) { return HSA_STATUS_SUCCESS; } -void RdcRocpBase::read_feature(rocprofiler_t* context, const unsigned feature_count, - uint32_t gpu_index) { +double RdcRocpBase::read_feature(rocprofiler_t* context, uint32_t gpu_index) { hsa_status_t status = rocprofiler_read(context, 0); assert(status == HSA_STATUS_SUCCESS); status = rocprofiler_get_data(context, 0); assert(status == HSA_STATUS_SUCCESS); status = rocprofiler_get_metrics(context); assert(status == HSA_STATUS_SUCCESS); - switch (feature[gpu_index].data.kind) { + switch (gpuid_to_feature[gpu_index].data.kind) { case ROCPROFILER_DATA_KIND_DOUBLE: - metric_to_value[feature[gpu_index].name] = feature[gpu_index].data.result_double; + return gpuid_to_feature[gpu_index].data.result_double; break; case ROCPROFILER_DATA_KIND_INT32: - metric_to_value[feature[gpu_index].name] = feature[gpu_index].data.result_int32; + return static_cast(gpuid_to_feature[gpu_index].data.result_int32); break; case ROCPROFILER_DATA_KIND_INT64: - metric_to_value[feature[gpu_index].name] = feature[gpu_index].data.result_int64; + return static_cast(gpuid_to_feature[gpu_index].data.result_int64); + break; + case ROCPROFILER_DATA_KIND_FLOAT: + return static_cast(gpuid_to_feature[gpu_index].data.result_float); break; default: - RDC_LOG(RDC_ERROR, "ERROR: Unexpected feature kind: " << feature[gpu_index].data.kind); + RDC_LOG(RDC_ERROR, + "ERROR: Unexpected feature kind: " << gpuid_to_feature[gpu_index].data.kind); } + return 0.0; } static int get_agents(hsa_agent_arr_t* agent_arr) { @@ -132,12 +136,12 @@ bool createHsaQueue(hsa_queue_t** queue, hsa_agent_t gpu_agent) { return (status == HSA_STATUS_SUCCESS); } -int RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) { +double RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) { // initialize hsa. hsa_init() will also load the profiler libs under the hood hsa_status_t status = HSA_STATUS_SUCCESS; - feature[gpu_index].kind = (rocprofiler_feature_kind_t)ROCPROFILER_FEATURE_KIND_METRIC; - feature[gpu_index].name = field_to_metric[field]; + gpuid_to_feature[gpu_index].kind = (rocprofiler_feature_kind_t)ROCPROFILER_FEATURE_KIND_METRIC; + gpuid_to_feature[gpu_index].name = field_to_metric[field]; // rocprofiler_t* contexts[agent_arr.count] = {0}; std::vector contexts; @@ -149,7 +153,7 @@ int RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) { NULL, }; int mode = (ROCPROFILER_MODE_STANDALONE | ROCPROFILER_MODE_SINGLEGROUP); - status = rocprofiler_open(agent_arr.agents[gpu_index], &feature[gpu_index], 1, + status = rocprofiler_open(agent_arr.agents[gpu_index], &gpuid_to_feature[gpu_index], 1, &contexts[gpu_index], mode, &properties); const char* error_string = nullptr; rocprofiler_error_string(&error_string); @@ -165,19 +169,19 @@ int RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) { // this is the duration for which the counter increments from zero. // TODO: Return error if sampling interval is lower than this value - usleep(10000); + usleep(collection_duration_us_k); status = rocprofiler_stop(contexts[gpu_index], 0); assert(status == HSA_STATUS_SUCCESS); - read_feature(contexts[gpu_index], 1, gpu_index); + double raw_value = read_feature(contexts[gpu_index], gpu_index); usleep(100); status = rocprofiler_close(contexts[gpu_index]); assert(status == HSA_STATUS_SUCCESS); - return 0; + return raw_value; } const char* RdcRocpBase::get_field_id_from_name(rdc_field_t field) { @@ -233,7 +237,6 @@ const std::vector RdcRocpBase::get_field_ids() { } RdcRocpBase::RdcRocpBase() { - start_time = std::chrono::high_resolution_clock::now(); hsa_status_t status = hsa_init(); if (status != HSA_STATUS_SUCCESS) { const char* errstr = nullptr; @@ -256,11 +259,11 @@ RdcRocpBase::RdcRocpBase() { {RDC_FI_PROF_EVAL_FLOPS_64, "TOTAL_64_OPS"}, }; - std::vector unchecked_fields; + std::vector all_fields; std::vector checked_fields; for (auto& [k, v] : temp_field_map_k) { - unchecked_fields.push_back(v); + all_fields.push_back(v); } // populate list of agents @@ -282,33 +285,28 @@ RdcRocpBase::RdcRocpBase() { } else { RDC_LOG(RDC_DEBUG, "gpu_index[" << gpu_index << "] = node_id[" << driver_node_id << "]"); } - } - // only check metrics for the last GPU - // TODO: add support for different metrics per GPU - // currently it's assumed that all GPUs are the same - check_metrics_supported(driver_node_id, unchecked_fields, checked_fields); + check_metrics_supported(driver_node_id, all_fields, checked_fields); - for (auto& [k, v] : temp_field_map_k) { - auto found = std::find(checked_fields.begin(), checked_fields.end(), v); - if (found != checked_fields.end()) { - field_to_metric.insert({k, v}); + for (auto& [k, v] : temp_field_map_k) { + auto found = std::find(checked_fields.begin(), checked_fields.end(), v); + if (found != checked_fields.end()) { + field_to_metric.insert({k, v}); + // initialize the buffer + const rdc_field_pair_t field_pair = {gpu_index, k}; + const rdc_average_t avg = {std::vector(), 0}; + average.insert({field_pair, avg}); + } } } RDC_LOG(RDC_DEBUG, "Rocprofiler supports " << field_to_metric.size() << " fields"); - for (auto& [k, v] : field_to_metric) { - const char* str = v; - metric_to_value.insert({str, 0.0}); - } - assert(metric_to_value.size() == field_to_metric.size()); - for (uint32_t gpu_index = 0; gpu_index < agent_arr.count; gpu_index++) { - for (auto& metric : metric_to_value) { + for (const auto& [k, v] : field_to_metric) { rocprofiler_feature_t temp_feature; temp_feature.kind = (rocprofiler_feature_kind_t)ROCPROFILER_FEATURE_KIND_METRIC; - temp_feature.name = metric.first; - feature.insert({gpu_index, temp_feature}); + temp_feature.name = v; + gpuid_to_feature.insert({gpu_index, temp_feature}); } } @@ -328,7 +326,53 @@ RdcRocpBase::~RdcRocpBase() { assert(status == HSA_STATUS_ERROR_NOT_INITIALIZED); } -rdc_status_t RdcRocpBase::rocp_lookup(uint32_t gpu_index, rdc_field_t field, double* value) { +double RdcRocpBase::get_average(rdc_field_pair_t field_pair, double raw_value) { + // check if vector exists + if (average.find(field_pair) == average.end()) { + RDC_LOG(RDC_ERROR, + "gpu[" << field_pair.first << "]field[" << field_pair.second << "] not found"); + return NAN; + } + + if (average[field_pair].buffer.size() < buffer_length_k) { + // buffer not yet filled up + average[field_pair].buffer.push_back(raw_value); + } else { + // buffer is filled up + average[field_pair].buffer[average[field_pair].index] = raw_value; + } + + // RDC_LOG(RDC_DEBUG, "gpu[" << field_pair.first << "]field[" << field_pair.second << + // "]avg_index[" + // << average[field_pair].index << "] = " << raw_value); + + average[field_pair].index++; + // cap index at buffer_length_k + average[field_pair].index = average[field_pair].index % buffer_length_k; + + double sum = + std::accumulate(average[field_pair].buffer.begin(), average[field_pair].buffer.end(), 0.0); + double avg = sum / static_cast(average[field_pair].buffer.size()); + + return avg; +} + +void RdcRocpBase::reset_average(rdc_gpu_field_t gpu_field) { + rdc_field_pair_t pair = {gpu_field.gpu_index, gpu_field.field_id}; + // check if vector exists + if (average.find(pair) == average.end()) { + RDC_LOG(RDC_ERROR, "gpu[" << pair.first << "]field[" << pair.second << "] not found"); + return; + } + + average[pair].buffer.clear(); + average[pair].index = 0; +} + +rdc_status_t RdcRocpBase::rocp_lookup(rdc_gpu_field_t gpu_field, double* value) { + const auto& gpu_index = gpu_field.gpu_index; + const auto& field = gpu_field.field_id; + if (value == nullptr) { return RDC_ST_BAD_PARAMETER; } @@ -337,20 +381,17 @@ rdc_status_t RdcRocpBase::rocp_lookup(uint32_t gpu_index, rdc_field_t field, dou if (status != HSA_STATUS_SUCCESS) { return Rocp2RdcError(status); } - switch (field) { - default: - const auto stop_time = std::chrono::high_resolution_clock::now(); - run_profiler(gpu_index, field); - *value = metric_to_value[field_to_metric[field]]; - // extra processing required - if (eval_fields.find(field) != eval_fields.end()) { - const auto elapsed = - std::chrono::duration_cast(stop_time - start_time).count(); - RDC_LOG(RDC_DEBUG, "INDEX: " << gpu_index << " before[" << *value << "] after[" - << (*value / elapsed) << "]"); - *value = *value / elapsed; - } - break; + const auto start_time = std::chrono::high_resolution_clock::now(); + double raw_value = run_profiler(gpu_index, field); + const auto stop_time = std::chrono::high_resolution_clock::now(); + *value = get_average({gpu_index, field}, raw_value); + // extra processing required + if (eval_fields.find(field) != eval_fields.end()) { + const auto elapsed = + std::chrono::duration_cast(stop_time - start_time).count(); + // RDC_LOG(RDC_DEBUG, "INDEX: " << gpu_index << " before[" << *value << "] after[" + // << (*value / elapsed) << "]"); + *value = *value / elapsed; } return Rocp2RdcError(status); } diff --git a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc index bb24ca0c1a..41a6a50da0 100644 --- a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc +++ b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc @@ -94,7 +94,7 @@ rdc_status_t rdc_telemetry_fields_value_get(rdc_gpu_field_t* fields, const uint3 bulk_count = 0; } - status = rocp_p->rocp_lookup(fields[i].gpu_index, fields[i].field_id, &data); + status = rocp_p->rocp_lookup(fields[i], &data); // get value values[bulk_count].gpu_index = fields[i].gpu_index; values[bulk_count].field_value.type = DOUBLE; @@ -131,6 +131,9 @@ rdc_status_t rdc_telemetry_fields_unwatch(rdc_gpu_field_t* fields, uint32_t fiel rdc_status_t status = RDC_ST_OK; for (uint32_t i = 0; i < fields_count; i++) { RDC_LOG(RDC_DEBUG, "UNWATCH: " << fields[i].field_id); + if (rocp_p != nullptr) { + rocp_p->reset_average(fields[i]); + } const rdc_status_t temp_status = RDC_ST_OK; // return last non-ok status if (temp_status != RDC_ST_OK) {