Bulk fetch metrics from rocm_smi_lib
The RDC provides a wrapper to bulk fetch metrics from rocm_smi_lib. If the video card does not support bulk fetch or the metrics cannot be bulk fetched, it will fallback to fetch them one by one. Change-Id: I8852ba1ed67e0fabc805c93b1080f74c233516e1
Этот коммит содержится в:
Bill(Shuzhou) Liu
коммит произвёл
Shuzhou Liu
Shuzhou Liu
родитель
ceb562d630
Коммит
51efe26442
@@ -23,7 +23,9 @@ THE SOFTWARE.
|
||||
#define INCLUDE_RDC_LIB_RDCMETRICFETCHER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include "rdc_lib/rdc_common.h"
|
||||
#include "rdc_lib/RdcTelemetryLibInterface.h"
|
||||
#include "rdc/rdc.h"
|
||||
|
||||
|
||||
@@ -37,6 +39,10 @@ class RdcMetricFetcher {
|
||||
|
||||
virtual rdc_status_t fetch_smi_field(uint32_t gpu_index,
|
||||
rdc_field_t field_id, rdc_field_value* value) = 0;
|
||||
|
||||
virtual rdc_status_t bulk_fetch_smi_fields(
|
||||
rdc_gpu_field_t* fields, uint32_t fields_count,
|
||||
std::vector<rdc_gpu_field_value_t>& results) = 0; // NOLINT
|
||||
virtual ~RdcMetricFetcher() {}
|
||||
};
|
||||
|
||||
|
||||
@@ -70,6 +70,9 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher {
|
||||
public:
|
||||
rdc_status_t fetch_smi_field(uint32_t gpu_index,
|
||||
rdc_field_t field_id, rdc_field_value* value) override;
|
||||
rdc_status_t bulk_fetch_smi_fields(
|
||||
rdc_gpu_field_t* fields, uint32_t fields_count,
|
||||
std::vector<rdc_gpu_field_value_t>& results) override; // NOLINT
|
||||
RdcMetricFetcherImpl();
|
||||
~RdcMetricFetcherImpl();
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ THE SOFTWARE.
|
||||
#include <chrono> //NOLINT
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include "rdc_lib/rdc_common.h"
|
||||
#include "common/rdc_fields_supported.h"
|
||||
#include "rdc_lib/RdcLogger.h"
|
||||
@@ -227,6 +228,86 @@ void RdcMetricFetcherImpl::get_pcie_throughput(const RdcFieldKey& key) {
|
||||
} while (0);
|
||||
}
|
||||
|
||||
rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields(
|
||||
rdc_gpu_field_t* fields, uint32_t fields_count,
|
||||
std::vector<rdc_gpu_field_value_t>& results) { // NOLINT
|
||||
const std::set<rdc_field_t> rdc_bulk_fields = {
|
||||
RDC_FI_GPU_CLOCK, // current_gfxclk * 1000000
|
||||
RDC_FI_MEMORY_TEMP, // temperature_mem * 1000
|
||||
RDC_FI_GPU_TEMP, // temperature_edge * 1000
|
||||
RDC_FI_POWER_USAGE, // average_socket_power * 1000000
|
||||
RDC_FI_GPU_UTIL // average_gfx_activity
|
||||
};
|
||||
|
||||
// To prevent always call the bulk API even if it is not supported,
|
||||
// the static is used to cache last try.
|
||||
static rsmi_status_t rs = RSMI_STATUS_SUCCESS;
|
||||
if (rs != RSMI_STATUS_SUCCESS) {
|
||||
results.clear();
|
||||
return RDC_ST_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
// Organize the fields per GPU
|
||||
std::map<uint32_t, std::vector<rdc_field_t>> bulk_fields;
|
||||
for (uint32_t i = 0; i < fields_count; i++) {
|
||||
if (rdc_bulk_fields.find(fields[i].field_id) != rdc_bulk_fields.end()) {
|
||||
bulk_fields[fields[i].gpu_index].push_back(fields[i].field_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Call the rocm_smi_lib API to bulk fetch the data
|
||||
auto cur_time = now();
|
||||
auto ite = bulk_fields.begin();
|
||||
for (; ite != bulk_fields.end(); ite++) {
|
||||
rsmi_gpu_metrics_t gpu_metrics;
|
||||
rs = rsmi_dev_gpu_metrics_info_get(ite->first, &gpu_metrics);
|
||||
if (rs != RSMI_STATUS_SUCCESS) {
|
||||
results.clear();
|
||||
return RDC_ST_NOT_SUPPORTED;
|
||||
}
|
||||
for (uint32_t j=0; j < ite->second.size(); j++) {
|
||||
auto field_id = ite->second[j];
|
||||
rdc_gpu_field_value_t value;
|
||||
value.gpu_index = ite->first;
|
||||
value.field_value.field_id = field_id;
|
||||
value.field_value.type = INTEGER;
|
||||
value.field_value.status = RSMI_STATUS_SUCCESS;
|
||||
value.field_value.ts = cur_time;
|
||||
|
||||
switch (field_id) {
|
||||
case RDC_FI_GPU_CLOCK: // current_gfxclk * 1000000
|
||||
value.field_value.value.l_int =
|
||||
static_cast<int64_t>(gpu_metrics.current_gfxclk * 1000000);
|
||||
break;
|
||||
case RDC_FI_MEMORY_TEMP: // temperature_mem * 1000
|
||||
value.field_value.value.l_int =
|
||||
static_cast<int64_t>(gpu_metrics.temperature_mem * 1000);
|
||||
break;
|
||||
case RDC_FI_GPU_TEMP: // temperature_edge * 1000
|
||||
value.field_value.value.l_int =
|
||||
static_cast<int64_t>(gpu_metrics.temperature_edge * 1000);
|
||||
break;
|
||||
case RDC_FI_POWER_USAGE: // average_socket_power * 1000000
|
||||
value.field_value.value.l_int =
|
||||
static_cast<int64_t>(gpu_metrics.average_socket_power * 1000000);
|
||||
break;
|
||||
case RDC_FI_GPU_UTIL: // average_gfx_activity
|
||||
value.field_value.value.l_int =
|
||||
static_cast<int64_t>(gpu_metrics.average_gfx_activity);
|
||||
break;
|
||||
default:
|
||||
value.field_value.status = RSMI_STATUS_NOT_SUPPORTED;
|
||||
break;
|
||||
}
|
||||
if (value.field_value.status == RSMI_STATUS_SUCCESS) {
|
||||
results.push_back(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return RDC_ST_OK;
|
||||
}
|
||||
|
||||
static const uint64_t kGig = 1000000000;
|
||||
|
||||
rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index,
|
||||
|
||||
@@ -39,13 +39,37 @@ rdc_status_t RdcSmiLib::rdc_telemetry_fields_value_get(rdc_gpu_field_t* fields,
|
||||
return RDC_ST_BAD_PARAMETER;
|
||||
}
|
||||
|
||||
RDC_LOG(RDC_DEBUG, "Bulk fetch " << fields_count
|
||||
RDC_LOG(RDC_DEBUG, "Fetch " << fields_count
|
||||
<< " fields from rocm_smi_lib.");
|
||||
|
||||
// Bulk fetch fields
|
||||
std::vector<rdc_gpu_field_value_t> bulk_results;
|
||||
rdc_status_t status = metric_fetcher_->bulk_fetch_smi_fields(
|
||||
fields, fields_count, bulk_results);
|
||||
RDC_LOG(RDC_DEBUG, "Bulk fetched " << bulk_results.size()
|
||||
<< " fields from rocm_smi_lib which return " << status);
|
||||
if (bulk_results.size() > 0) {
|
||||
rdc_status_t status = callback(&bulk_results[0],
|
||||
bulk_results.size(), user_data);
|
||||
if (status != RDC_ST_OK) {
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch it one by one for left fields
|
||||
const int BULK_FIELDS_MAX = 16;
|
||||
rdc_gpu_field_value_t values[BULK_FIELDS_MAX];
|
||||
uint32_t bulk_count = 0;
|
||||
for (uint32_t i = 0; i < fields_count; i++) {
|
||||
bool is_fetched = false;
|
||||
for (std::size_t j = 0; j < bulk_results.size(); j++) {
|
||||
if (bulk_results[j].gpu_index == fields[i].gpu_index &&
|
||||
bulk_results[j].field_value.field_id == fields[i].field_id) {
|
||||
is_fetched = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (is_fetched) continue;
|
||||
if (bulk_count >= BULK_FIELDS_MAX) {
|
||||
rdc_status_t status = callback(values, bulk_count, user_data);
|
||||
// When the callback returns errors, stop processing and return.
|
||||
|
||||
@@ -507,7 +507,7 @@ void RdciDmonSubSystem::process() {
|
||||
amd::rdc::fld_id2name_map_t &field_id_to_descript =
|
||||
amd::rdc::get_field_id_description_from_id();
|
||||
|
||||
if (notif_fields.size()>0) {
|
||||
if (notif_fields.size() > 0) {
|
||||
ss << "Listening for events: ";
|
||||
uint32_t i;
|
||||
for (i = 0; i < notif_fields.size() - 1; ++i) {
|
||||
|
||||
Ссылка в новой задаче
Block a user