From d9b13912c647f81765a93b1d4fbd85002f4a8a97 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Wed, 11 Dec 2024 11:11:57 -0600 Subject: [PATCH] Profiler - Remove averaging Averaging happens very slowly and only confuses people... Change-Id: I60754d3b896b6ffeb6104bb1c2fcc54e9869b331 Signed-off-by: Galantsev, Dmitrii [ROCm/rdc commit: 2c61dfe2ce02a6b6ad8b4b395d633cad07f30464] --- .../rdc_modules/rdc_rocp/RdcRocpBase.h | 11 ---- .../rdc_modules/rdc_rocp/RdcRocpBase.cc | 54 ++----------------- .../rdc_modules/rdc_rocp/RdcTelemetryLib.cc | 5 +- 3 files changed, 4 insertions(+), 66 deletions(-) diff --git a/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h b/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h index 5ed43342fc..49b9c91282 100644 --- a/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h +++ b/projects/rdc/include/rdc_modules/rdc_rocp/RdcRocpBase.h @@ -65,17 +65,8 @@ class RdcRocpBase { const char* get_field_id_from_name(rdc_field_t); const std::vector get_field_ids(); - /** - * @brief Reset average metrics for gpu_field - */ - void reset_average(rdc_gpu_field_t gpu_field); - protected: private: - typedef struct { - std::vector buffer; - uint32_t index; - } rdc_average_t; typedef std::pair rdc_field_pair_t; static const size_t buffer_length_k = 5; /** @@ -85,13 +76,11 @@ class RdcRocpBase { double read_feature(rocprofiler_t* context, uint32_t gpu_index); double run_profiler(uint32_t gpu_index, rdc_field_t field); - double get_average(rdc_field_pair_t field_pair, double raw_value); hsa_agent_arr_t agent_arr = {}; std::vector queues; std::map gpuid_to_feature; std::map field_to_metric = {}; - std::map average = {}; // these fields must be divided by time passed std::unordered_set eval_fields = { diff --git a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc index c59eee55f2..bd45af7d6f 100644 --- a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc +++ b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcRocpBase.cc @@ -174,14 +174,14 @@ double RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) { status = rocprofiler_stop(contexts[gpu_index], 0); assert(status == HSA_STATUS_SUCCESS); - double raw_value = read_feature(contexts[gpu_index], gpu_index); + double value = read_feature(contexts[gpu_index], gpu_index); usleep(100); status = rocprofiler_close(contexts[gpu_index]); assert(status == HSA_STATUS_SUCCESS); - return raw_value; + return value; } const char* RdcRocpBase::get_field_id_from_name(rdc_field_t field) { @@ -293,10 +293,6 @@ RdcRocpBase::RdcRocpBase() { auto found = std::find(checked_fields.begin(), checked_fields.end(), v); if (found != checked_fields.end()) { field_to_metric.insert({k, v}); - // initialize the buffer - const rdc_field_pair_t field_pair = {gpu_index, k}; - const rdc_average_t avg = {std::vector(), 0}; - average.insert({field_pair, avg}); } } } @@ -328,49 +324,6 @@ RdcRocpBase::~RdcRocpBase() { assert(status == HSA_STATUS_ERROR_NOT_INITIALIZED); } -double RdcRocpBase::get_average(rdc_field_pair_t field_pair, double raw_value) { - // check if vector exists - if (average.find(field_pair) == average.end()) { - RDC_LOG(RDC_ERROR, - "gpu[" << field_pair.first << "]field[" << field_pair.second << "] not found"); - return NAN; - } - - if (average[field_pair].buffer.size() < buffer_length_k) { - // buffer not yet filled up - average[field_pair].buffer.push_back(raw_value); - } else { - // buffer is filled up - average[field_pair].buffer[average[field_pair].index] = raw_value; - } - - // RDC_LOG(RDC_DEBUG, "gpu[" << field_pair.first << "]field[" << field_pair.second << - // "]avg_index[" - // << average[field_pair].index << "] = " << raw_value); - - average[field_pair].index++; - // cap index at buffer_length_k - average[field_pair].index = average[field_pair].index % buffer_length_k; - - double sum = - std::accumulate(average[field_pair].buffer.begin(), average[field_pair].buffer.end(), 0.0); - double avg = sum / static_cast(average[field_pair].buffer.size()); - - return avg; -} - -void RdcRocpBase::reset_average(rdc_gpu_field_t gpu_field) { - rdc_field_pair_t pair = {gpu_field.gpu_index, gpu_field.field_id}; - // check if vector exists - if (average.find(pair) == average.end()) { - RDC_LOG(RDC_ERROR, "gpu[" << pair.first << "]field[" << pair.second << "] not found"); - return; - } - - average[pair].buffer.clear(); - average[pair].index = 0; -} - rdc_status_t RdcRocpBase::rocp_lookup(rdc_gpu_field_t gpu_field, double* value) { const auto& gpu_index = gpu_field.gpu_index; const auto& field = gpu_field.field_id; @@ -384,9 +337,8 @@ rdc_status_t RdcRocpBase::rocp_lookup(rdc_gpu_field_t gpu_field, double* value) return Rocp2RdcError(status); } const auto start_time = std::chrono::high_resolution_clock::now(); - double raw_value = run_profiler(gpu_index, field); + *value = run_profiler(gpu_index, field); const auto stop_time = std::chrono::high_resolution_clock::now(); - *value = get_average({gpu_index, field}, raw_value); // extra processing required if (eval_fields.find(field) != eval_fields.end()) { const auto elapsed = diff --git a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc index 919468bf2e..4db8e97d31 100644 --- a/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc +++ b/projects/rdc/rdc_libs/rdc_modules/rdc_rocp/RdcTelemetryLib.cc @@ -52,7 +52,7 @@ bool is_rocp_disabled() { [&value_str](const char* val) { return value_str == val; }); } -rdc_status_t rdc_module_init(uint64_t flags) { +rdc_status_t rdc_module_init(uint64_t /*flags*/) { if (is_rocp_disabled()) { // rocprofiler does NOT work in gtest. // GTest starts up multiple instances of the progam under test, @@ -156,9 +156,6 @@ rdc_status_t rdc_telemetry_fields_unwatch(rdc_gpu_field_t* fields, uint32_t fiel rdc_status_t status = RDC_ST_OK; for (uint32_t i = 0; i < fields_count; i++) { RDC_LOG(RDC_DEBUG, "UNWATCH: " << fields[i].field_id); - if (rocp_p != nullptr) { - rocp_p->reset_average(fields[i]); - } const rdc_status_t temp_status = RDC_ST_OK; // return last non-ok status if (temp_status != RDC_ST_OK) {