Averaging happens very slowly and only confuses people...

Change-Id: I60754d3b896b6ffeb6104bb1c2fcc54e9869b331
Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>


[ROCm/rdc commit: 2c61dfe2ce]
Этот коммит содержится в:
Galantsev, Dmitrii
2024-12-11 11:11:57 -06:00
коммит произвёл Galantsev, Dmitrii
родитель fc83179a9d
Коммит d9b13912c6
3 изменённых файлов: 4 добавлений и 66 удалений
-11
Просмотреть файл
@@ -65,17 +65,8 @@ class RdcRocpBase {
const char* get_field_id_from_name(rdc_field_t);
const std::vector<rdc_field_t> get_field_ids();
/**
* @brief Reset average metrics for gpu_field
*/
void reset_average(rdc_gpu_field_t gpu_field);
protected:
private:
typedef struct {
std::vector<double> buffer;
uint32_t index;
} rdc_average_t;
typedef std::pair<uint32_t, rdc_field_t> rdc_field_pair_t;
static const size_t buffer_length_k = 5;
/**
@@ -85,13 +76,11 @@ class RdcRocpBase {
double read_feature(rocprofiler_t* context, uint32_t gpu_index);
double run_profiler(uint32_t gpu_index, rdc_field_t field);
double get_average(rdc_field_pair_t field_pair, double raw_value);
hsa_agent_arr_t agent_arr = {};
std::vector<hsa_queue_t*> queues;
std::map<uint32_t, rocprofiler_feature_t> gpuid_to_feature;
std::map<rdc_field_t, const char*> field_to_metric = {};
std::map<rdc_field_pair_t, rdc_average_t> average = {};
// these fields must be divided by time passed
std::unordered_set<rdc_field_t> eval_fields = {
+3 -51
Просмотреть файл
@@ -174,14 +174,14 @@ double RdcRocpBase::run_profiler(uint32_t gpu_index, rdc_field_t field) {
status = rocprofiler_stop(contexts[gpu_index], 0);
assert(status == HSA_STATUS_SUCCESS);
double raw_value = read_feature(contexts[gpu_index], gpu_index);
double value = read_feature(contexts[gpu_index], gpu_index);
usleep(100);
status = rocprofiler_close(contexts[gpu_index]);
assert(status == HSA_STATUS_SUCCESS);
return raw_value;
return value;
}
const char* RdcRocpBase::get_field_id_from_name(rdc_field_t field) {
@@ -293,10 +293,6 @@ RdcRocpBase::RdcRocpBase() {
auto found = std::find(checked_fields.begin(), checked_fields.end(), v);
if (found != checked_fields.end()) {
field_to_metric.insert({k, v});
// initialize the buffer
const rdc_field_pair_t field_pair = {gpu_index, k};
const rdc_average_t avg = {std::vector<double>(), 0};
average.insert({field_pair, avg});
}
}
}
@@ -328,49 +324,6 @@ RdcRocpBase::~RdcRocpBase() {
assert(status == HSA_STATUS_ERROR_NOT_INITIALIZED);
}
double RdcRocpBase::get_average(rdc_field_pair_t field_pair, double raw_value) {
// check if vector exists
if (average.find(field_pair) == average.end()) {
RDC_LOG(RDC_ERROR,
"gpu[" << field_pair.first << "]field[" << field_pair.second << "] not found");
return NAN;
}
if (average[field_pair].buffer.size() < buffer_length_k) {
// buffer not yet filled up
average[field_pair].buffer.push_back(raw_value);
} else {
// buffer is filled up
average[field_pair].buffer[average[field_pair].index] = raw_value;
}
// RDC_LOG(RDC_DEBUG, "gpu[" << field_pair.first << "]field[" << field_pair.second <<
// "]avg_index["
// << average[field_pair].index << "] = " << raw_value);
average[field_pair].index++;
// cap index at buffer_length_k
average[field_pair].index = average[field_pair].index % buffer_length_k;
double sum =
std::accumulate(average[field_pair].buffer.begin(), average[field_pair].buffer.end(), 0.0);
double avg = sum / static_cast<double>(average[field_pair].buffer.size());
return avg;
}
void RdcRocpBase::reset_average(rdc_gpu_field_t gpu_field) {
rdc_field_pair_t pair = {gpu_field.gpu_index, gpu_field.field_id};
// check if vector exists
if (average.find(pair) == average.end()) {
RDC_LOG(RDC_ERROR, "gpu[" << pair.first << "]field[" << pair.second << "] not found");
return;
}
average[pair].buffer.clear();
average[pair].index = 0;
}
rdc_status_t RdcRocpBase::rocp_lookup(rdc_gpu_field_t gpu_field, double* value) {
const auto& gpu_index = gpu_field.gpu_index;
const auto& field = gpu_field.field_id;
@@ -384,9 +337,8 @@ rdc_status_t RdcRocpBase::rocp_lookup(rdc_gpu_field_t gpu_field, double* value)
return Rocp2RdcError(status);
}
const auto start_time = std::chrono::high_resolution_clock::now();
double raw_value = run_profiler(gpu_index, field);
*value = run_profiler(gpu_index, field);
const auto stop_time = std::chrono::high_resolution_clock::now();
*value = get_average({gpu_index, field}, raw_value);
// extra processing required
if (eval_fields.find(field) != eval_fields.end()) {
const auto elapsed =
+1 -4
Просмотреть файл
@@ -52,7 +52,7 @@ bool is_rocp_disabled() {
[&value_str](const char* val) { return value_str == val; });
}
rdc_status_t rdc_module_init(uint64_t flags) {
rdc_status_t rdc_module_init(uint64_t /*flags*/) {
if (is_rocp_disabled()) {
// rocprofiler does NOT work in gtest.
// GTest starts up multiple instances of the progam under test,
@@ -156,9 +156,6 @@ rdc_status_t rdc_telemetry_fields_unwatch(rdc_gpu_field_t* fields, uint32_t fiel
rdc_status_t status = RDC_ST_OK;
for (uint32_t i = 0; i < fields_count; i++) {
RDC_LOG(RDC_DEBUG, "UNWATCH: " << fields[i].field_id);
if (rocp_p != nullptr) {
rocp_p->reset_average(fields[i]);
}
const rdc_status_t temp_status = RDC_ST_OK;
// return last non-ok status
if (temp_status != RDC_ST_OK) {