From 51efe264420f72e62480c9dcbb0201a6809538c7 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Wed, 6 Jan 2021 09:12:13 -0500 Subject: [PATCH] Bulk fetch metrics from rocm_smi_lib The RDC provides a wrapper to bulk fetch metrics from rocm_smi_lib. If the video card does not support bulk fetch or the metrics cannot be bulk fetched, it will fallback to fetch them one by one. Change-Id: I8852ba1ed67e0fabc805c93b1080f74c233516e1 --- include/rdc_lib/RdcMetricFetcher.h | 6 ++ include/rdc_lib/impl/RdcMetricFetcherImpl.h | 3 + rdc_libs/rdc/src/RdcMetricFetcherImpl.cc | 81 +++++++++++++++++++++ rdc_libs/rdc/src/RdcSmiLib.cc | 26 ++++++- rdci/src/RdciDmonSubSystem.cc | 2 +- 5 files changed, 116 insertions(+), 2 deletions(-) diff --git a/include/rdc_lib/RdcMetricFetcher.h b/include/rdc_lib/RdcMetricFetcher.h index b7ba515893..829e93d749 100644 --- a/include/rdc_lib/RdcMetricFetcher.h +++ b/include/rdc_lib/RdcMetricFetcher.h @@ -23,7 +23,9 @@ THE SOFTWARE. #define INCLUDE_RDC_LIB_RDCMETRICFETCHER_H_ #include +#include #include "rdc_lib/rdc_common.h" +#include "rdc_lib/RdcTelemetryLibInterface.h" #include "rdc/rdc.h" @@ -37,6 +39,10 @@ class RdcMetricFetcher { virtual rdc_status_t fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) = 0; + + virtual rdc_status_t bulk_fetch_smi_fields( + rdc_gpu_field_t* fields, uint32_t fields_count, + std::vector& results) = 0; // NOLINT virtual ~RdcMetricFetcher() {} }; diff --git a/include/rdc_lib/impl/RdcMetricFetcherImpl.h b/include/rdc_lib/impl/RdcMetricFetcherImpl.h index 2274c382f4..e188f2bd65 100644 --- a/include/rdc_lib/impl/RdcMetricFetcherImpl.h +++ b/include/rdc_lib/impl/RdcMetricFetcherImpl.h @@ -70,6 +70,9 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { public: rdc_status_t fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) override; + rdc_status_t bulk_fetch_smi_fields( + rdc_gpu_field_t* fields, uint32_t fields_count, + std::vector& results) override; // NOLINT RdcMetricFetcherImpl(); ~RdcMetricFetcherImpl(); diff --git a/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index 8a13217b04..9cd6ee8274 100644 --- a/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -27,6 +27,7 @@ THE SOFTWARE. #include //NOLINT #include #include +#include #include "rdc_lib/rdc_common.h" #include "common/rdc_fields_supported.h" #include "rdc_lib/RdcLogger.h" @@ -227,6 +228,86 @@ void RdcMetricFetcherImpl::get_pcie_throughput(const RdcFieldKey& key) { } while (0); } +rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields( + rdc_gpu_field_t* fields, uint32_t fields_count, + std::vector& results) { // NOLINT + const std::set rdc_bulk_fields = { + RDC_FI_GPU_CLOCK, // current_gfxclk * 1000000 + RDC_FI_MEMORY_TEMP, // temperature_mem * 1000 + RDC_FI_GPU_TEMP, // temperature_edge * 1000 + RDC_FI_POWER_USAGE, // average_socket_power * 1000000 + RDC_FI_GPU_UTIL // average_gfx_activity + }; + + // To prevent always call the bulk API even if it is not supported, + // the static is used to cache last try. + static rsmi_status_t rs = RSMI_STATUS_SUCCESS; + if (rs != RSMI_STATUS_SUCCESS) { + results.clear(); + return RDC_ST_NOT_SUPPORTED; + } + + // Organize the fields per GPU + std::map> bulk_fields; + for (uint32_t i = 0; i < fields_count; i++) { + if (rdc_bulk_fields.find(fields[i].field_id) != rdc_bulk_fields.end()) { + bulk_fields[fields[i].gpu_index].push_back(fields[i].field_id); + } + } + + // Call the rocm_smi_lib API to bulk fetch the data + auto cur_time = now(); + auto ite = bulk_fields.begin(); + for (; ite != bulk_fields.end(); ite++) { + rsmi_gpu_metrics_t gpu_metrics; + rs = rsmi_dev_gpu_metrics_info_get(ite->first, &gpu_metrics); + if (rs != RSMI_STATUS_SUCCESS) { + results.clear(); + return RDC_ST_NOT_SUPPORTED; + } + for (uint32_t j=0; j < ite->second.size(); j++) { + auto field_id = ite->second[j]; + rdc_gpu_field_value_t value; + value.gpu_index = ite->first; + value.field_value.field_id = field_id; + value.field_value.type = INTEGER; + value.field_value.status = RSMI_STATUS_SUCCESS; + value.field_value.ts = cur_time; + + switch (field_id) { + case RDC_FI_GPU_CLOCK: // current_gfxclk * 1000000 + value.field_value.value.l_int = + static_cast(gpu_metrics.current_gfxclk * 1000000); + break; + case RDC_FI_MEMORY_TEMP: // temperature_mem * 1000 + value.field_value.value.l_int = + static_cast(gpu_metrics.temperature_mem * 1000); + break; + case RDC_FI_GPU_TEMP: // temperature_edge * 1000 + value.field_value.value.l_int = + static_cast(gpu_metrics.temperature_edge * 1000); + break; + case RDC_FI_POWER_USAGE: // average_socket_power * 1000000 + value.field_value.value.l_int = + static_cast(gpu_metrics.average_socket_power * 1000000); + break; + case RDC_FI_GPU_UTIL: // average_gfx_activity + value.field_value.value.l_int = + static_cast(gpu_metrics.average_gfx_activity); + break; + default: + value.field_value.status = RSMI_STATUS_NOT_SUPPORTED; + break; + } + if (value.field_value.status == RSMI_STATUS_SUCCESS) { + results.push_back(value); + } + } + } + + return RDC_ST_OK; +} + static const uint64_t kGig = 1000000000; rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, diff --git a/rdc_libs/rdc/src/RdcSmiLib.cc b/rdc_libs/rdc/src/RdcSmiLib.cc index c9649f451b..4214eeed77 100644 --- a/rdc_libs/rdc/src/RdcSmiLib.cc +++ b/rdc_libs/rdc/src/RdcSmiLib.cc @@ -39,13 +39,37 @@ rdc_status_t RdcSmiLib::rdc_telemetry_fields_value_get(rdc_gpu_field_t* fields, return RDC_ST_BAD_PARAMETER; } - RDC_LOG(RDC_DEBUG, "Bulk fetch " << fields_count + RDC_LOG(RDC_DEBUG, "Fetch " << fields_count << " fields from rocm_smi_lib."); + // Bulk fetch fields + std::vector bulk_results; + rdc_status_t status = metric_fetcher_->bulk_fetch_smi_fields( + fields, fields_count, bulk_results); + RDC_LOG(RDC_DEBUG, "Bulk fetched " << bulk_results.size() + << " fields from rocm_smi_lib which return " << status); + if (bulk_results.size() > 0) { + rdc_status_t status = callback(&bulk_results[0], + bulk_results.size(), user_data); + if (status != RDC_ST_OK) { + return status; + } + } + + // Fetch it one by one for left fields const int BULK_FIELDS_MAX = 16; rdc_gpu_field_value_t values[BULK_FIELDS_MAX]; uint32_t bulk_count = 0; for (uint32_t i = 0; i < fields_count; i++) { + bool is_fetched = false; + for (std::size_t j = 0; j < bulk_results.size(); j++) { + if (bulk_results[j].gpu_index == fields[i].gpu_index && + bulk_results[j].field_value.field_id == fields[i].field_id) { + is_fetched = true; + break; + } + } + if (is_fetched) continue; if (bulk_count >= BULK_FIELDS_MAX) { rdc_status_t status = callback(values, bulk_count, user_data); // When the callback returns errors, stop processing and return. diff --git a/rdci/src/RdciDmonSubSystem.cc b/rdci/src/RdciDmonSubSystem.cc index 163a5d2a19..ddb582c70d 100644 --- a/rdci/src/RdciDmonSubSystem.cc +++ b/rdci/src/RdciDmonSubSystem.cc @@ -507,7 +507,7 @@ void RdciDmonSubSystem::process() { amd::rdc::fld_id2name_map_t &field_id_to_descript = amd::rdc::get_field_id_description_from_id(); - if (notif_fields.size()>0) { + if (notif_fields.size() > 0) { ss << "Listening for events: "; uint32_t i; for (i = 0; i < notif_fields.size() - 1; ++i) {