From 85670a59e63b342875d018ba6479e2ae4be37884 Mon Sep 17 00:00:00 2001 From: "Oliveira, Daniel" Date: Tue, 24 Oct 2023 21:12:05 -0500 Subject: [PATCH] rocm_smi_lib: Fix Refactoring gpu_metrics code Uses new support for 'gpu_metrics_v1_4' Code changes related to the following: * rsmi gpu_metrics APIs * rsmi gpu_metrics Logs * new data structure fields added in 1.4 * added APIs for all other existing metrics before 1.4 * added support to older metrics; 1.1, and 1.2 * added support to dump_internal_metrics_table() * public APIs renamed to start with prefix 'rsmi_dev_metrics_' * Unit tests updated * Examples updated Build changes related to the following: None Change-Id: I23e59f99d3ed43318cd6bd43bd2f0c5387e9ccb9 Signed-off-by: Oliveira, Daniel [ROCm/rocm_smi_lib commit: 713d259f88ef82c28f6d45686fece7060d159d3d] --- .../include/rocm_smi/rocm_smi_gpu_metrics.h | 78 ++++++-- .../rocm-smi-lib/src/rocm_smi_gpu_metrics.cc | 178 +++++++++++++++++- 2 files changed, 232 insertions(+), 24 deletions(-) diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h index 2565bf1688..1c217917e4 100644 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_gpu_metrics.h @@ -48,9 +48,13 @@ #include "rocm_smi/rocm_smi.h" #include +#include +#include #include +#include #include #include +#include #include #include #include @@ -98,14 +102,14 @@ struct AMDGpuMetricsHeader_v1_t uint8_t m_content_revision; }; - struct AMDGpuMetricsBase_t { virtual ~AMDGpuMetricsBase_t() = default; }; using AMDGpuMetricsBaseRef = AMDGpuMetricsBase_t&; -struct AMDGpuMetrics_v11_t : AMDGpuMetricsBase_t + +struct AMDGpuMetrics_v11_t { ~AMDGpuMetrics_v11_t() = default; @@ -167,7 +171,7 @@ struct AMDGpuMetrics_v11_t : AMDGpuMetricsBase_t uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; }; -struct AMDGpuMetrics_v12_t : AMDGpuMetricsBase_t +struct AMDGpuMetrics_v12_t { ~AMDGpuMetrics_v12_t() = default; @@ -227,11 +231,11 @@ struct AMDGpuMetrics_v12_t : AMDGpuMetricsBase_t uint32_t m_mem_activity_acc; // new in v1 uint16_t m_temperature_hbm[kRSMI_MAX_NUM_HBM_INSTANCES]; // new in v1 - // PMFW attached timestamp (10ns resolution) + // PMFW attached timestamp (10ns resolution) uint64_t m_firmware_timestamp; }; -struct AMDGpuMetrics_v13_t : AMDGpuMetricsBase_t +struct AMDGpuMetrics_v13_t { ~AMDGpuMetrics_v13_t() = default; @@ -305,7 +309,7 @@ struct AMDGpuMetrics_v13_t : AMDGpuMetricsBase_t uint64_t m_indep_throttle_status; }; -struct AMDGpuMetrics_v14_t : AMDGpuMetricsBase_t +struct AMDGpuMetrics_v14_t { ~AMDGpuMetrics_v14_t() = default; @@ -606,17 +610,15 @@ enum class AMDGpuMetricVersionFlags_t : AMDGpuMetricVersionFlagId_t kGpuMetricV14 = (0x1 << 4), }; using AMDGpuMetricVersionTranslationTbl_t = std::map; - - -class GpuMetricsBase_t; -using GpuMetricsBasePtr = std::shared_ptr; +using GpuMetricTypePtr_t = std::shared_ptr; class GpuMetricsBase_t { public: virtual ~GpuMetricsBase_t() = default; virtual size_t sizeof_metric_table() = 0; - virtual AMDGpuMetricsBaseRef get_metrics_table() = 0; + virtual GpuMetricTypePtr_t get_metrics_table() = 0; + virtual void dump_internal_metrics_table() = 0; virtual AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() = 0; virtual rsmi_status_t populate_metrics_dynamic_tbl() = 0; virtual AMGpuMetricsPublicLatestTupl_t copy_internal_to_external_metrics() = 0; @@ -629,21 +631,31 @@ class GpuMetricsBase_t uint64_t m_metrics_timestamp; }; +using GpuMetricsBasePtr = std::shared_ptr; using AMDGpuMetricFactories_t = std::map; class GpuMetricsBase_v11_t final : public GpuMetricsBase_t { public: - ~GpuMetricsBase_v11_t() = default; + virtual ~GpuMetricsBase_v11_t() = default; size_t sizeof_metric_table() override { return sizeof(AMDGpuMetrics_v11_t); } - AMDGpuMetricsBaseRef get_metrics_table() override + GpuMetricTypePtr_t get_metrics_table() override { - return this->m_gpu_metrics_tbl; + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v11_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return m_gpu_metric_ptr; + } + + void dump_internal_metrics_table() override + { + return; } AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override @@ -657,6 +669,7 @@ class GpuMetricsBase_v11_t final : public GpuMetricsBase_t private: AMDGpuMetrics_v11_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; @@ -669,9 +682,18 @@ class GpuMetricsBase_v12_t final : public GpuMetricsBase_t return sizeof(AMDGpuMetrics_v12_t); } - AMDGpuMetricsBaseRef get_metrics_table() override + GpuMetricTypePtr_t get_metrics_table() override { - return this->m_gpu_metrics_tbl; + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v12_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return m_gpu_metric_ptr; + } + + void dump_internal_metrics_table() override + { + return; } AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override @@ -684,6 +706,7 @@ class GpuMetricsBase_v12_t final : public GpuMetricsBase_t private: AMDGpuMetrics_v12_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; @@ -696,11 +719,17 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t return sizeof(AMDGpuMetrics_v13_t); } - AMDGpuMetricsBaseRef get_metrics_table() override + GpuMetricTypePtr_t get_metrics_table() override { - return this->m_gpu_metrics_tbl; + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v13_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return (m_gpu_metric_ptr); } + void dump_internal_metrics_table() override; + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV13; @@ -712,6 +741,7 @@ class GpuMetricsBase_v13_t final : public GpuMetricsBase_t private: AMDGpuMetrics_v13_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; @@ -724,11 +754,17 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t return sizeof(AMDGpuMetrics_v14_t); } - AMDGpuMetricsBaseRef get_metrics_table() override + GpuMetricTypePtr_t get_metrics_table() override { - return this->m_gpu_metrics_tbl; + if (!m_gpu_metric_ptr) { + m_gpu_metric_ptr.reset(&m_gpu_metrics_tbl, [](AMDGpuMetrics_v14_t*){}); + } + assert(m_gpu_metric_ptr != nullptr); + return m_gpu_metric_ptr; } + void dump_internal_metrics_table() override; + AMDGpuMetricVersionFlags_t get_gpu_metrics_version_used() override { return AMDGpuMetricVersionFlags_t::kGpuMetricV14; @@ -740,9 +776,11 @@ class GpuMetricsBase_v14_t final : public GpuMetricsBase_t private: AMDGpuMetrics_v14_t m_gpu_metrics_tbl; + std::shared_ptr m_gpu_metric_ptr; }; + template rsmi_status_t rsmi_dev_gpu_metrics_info_query(uint32_t dv_ind, AMDGpuMetricsUnitType_t metric_counter, T& metric_value); diff --git a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc index ef233bb88e..11d2871ca9 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_gpu_metrics.cc @@ -462,6 +462,105 @@ AMDGpuDynamicMetricTblValues_t format_metric_row(const T& metric, const std::str return multi_values; } +void GpuMetricsBase_v14_t::dump_internal_metrics_table() +{ + std::ostringstream ostrstream; + std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; + ostrstream << __PRETTY_FUNCTION__ + << " | ======= DEBUG ======= " + << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) + << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) + << " |" + << "\n"; + ostrstream << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" + << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" + + << " current_socket_power: " << m_gpu_metrics_tbl.m_current_socket_power << "\n" + + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n"; + + ostrstream << " vcn_activity: " << "\n"; + auto idx = uint64_t(0); + for (const auto& temp : m_gpu_metrics_tbl.m_vcn_activity) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" + + << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" + + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n" + + << " gfxclk_lock_status: " << m_gpu_metrics_tbl.m_gfxclk_lock_status << "\n" + + << " pcie_link_width: " << m_gpu_metrics_tbl.m_pcie_link_width << "\n" + << " pcie_link_speed: " << m_gpu_metrics_tbl.m_pcie_link_speed << "\n" + + << " xgmi_link_width: " << m_gpu_metrics_tbl.m_xgmi_link_width << "\n" + << " xgmi_link_speed: " << m_gpu_metrics_tbl.m_xgmi_link_speed << "\n" + + << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" + << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n" + + << " pcie_bandwidth_acc: " << m_gpu_metrics_tbl.m_pcie_bandwidth_acc << "\n" + << " pcie_bandwidth_inst: " << m_gpu_metrics_tbl.m_pcie_bandwidth_inst << "\n" + << " pcie_l0_to_recov_count_acc: " << m_gpu_metrics_tbl.m_pcie_l0_to_recov_count_acc << "\n" + << " pcie_replay_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_count_acc << "\n" + << " pcie_replay_rover_count_acc: " << m_gpu_metrics_tbl.m_pcie_replay_rover_count_acc << "\n"; + + ostrstream << " xgmi_read_data_acc: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_read_data_acc) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " xgmi_write_data_acc: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_xgmi_write_data_acc) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n"; + + ostrstream << " current_gfxclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_gfxclk) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " current_socclk: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_socclk) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " current_vclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_vclk0) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " current_dclk0: " << "\n"; + idx = 0; + for (const auto& temp : m_gpu_metrics_tbl.m_current_dclk0) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " padding: " << m_gpu_metrics_tbl.m_padding << "\n"; + LOG_DEBUG(ostrstream); +} + rsmi_status_t GpuMetricsBase_v14_t::populate_metrics_dynamic_tbl() { std::ostringstream ostrstream; @@ -947,6 +1046,78 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v14_t::copy_internal_to_external_m return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } +void GpuMetricsBase_v13_t::dump_internal_metrics_table() +{ + std::ostringstream ostrstream; + std::cout << __PRETTY_FUNCTION__ << " | ======= start ======= \n"; + ostrstream << __PRETTY_FUNCTION__ + << " | ======= DEBUG ======= " + << " | Metric Version: " << stringfy_metric_header_version(m_gpu_metrics_tbl.m_common_header) + << " | Size: " << print_unsigned_int(m_gpu_metrics_tbl.m_common_header.m_structure_size) + << " |" + << "\n"; + ostrstream << " temperature_edge: " << m_gpu_metrics_tbl.m_temperature_edge << "\n" + << " temperature_hotspot: " << m_gpu_metrics_tbl.m_temperature_hotspot << "\n" + << " temperature_mem: " << m_gpu_metrics_tbl.m_temperature_mem << "\n" + << " temperature_vrgfx: " << m_gpu_metrics_tbl.m_temperature_vrgfx << "\n" + << " temperature_vrsoc: " << m_gpu_metrics_tbl.m_temperature_vrsoc << "\n" + << " temperature_vrmem: " << m_gpu_metrics_tbl.m_temperature_vrmem << "\n" + + << " average_gfx_activity: " << m_gpu_metrics_tbl.m_average_gfx_activity << "\n" + << " average_umc_activity: " << m_gpu_metrics_tbl.m_average_umc_activity << "\n" + << " average_mm_activity: " << m_gpu_metrics_tbl.m_average_mm_activity << "\n" + << " average_socket_power: " << m_gpu_metrics_tbl.m_average_socket_power << "\n" + + << " energy_accumulator: " << m_gpu_metrics_tbl.m_energy_accumulator << "\n" + << " system_clock_counter: " << m_gpu_metrics_tbl.m_system_clock_counter << "\n" + + << " average_gfxclk_frequency: " << m_gpu_metrics_tbl.m_average_gfxclk_frequency << "\n" + << " average_socclk_frequency: " << m_gpu_metrics_tbl.m_average_socclk_frequency << "\n" + << " average_uclk_frequency: " << m_gpu_metrics_tbl.m_average_uclk_frequency << "\n" + << " average_vclk0_frequency: " << m_gpu_metrics_tbl.m_average_vclk0_frequency << "\n" + << " average_dclk0_frequency: " << m_gpu_metrics_tbl.m_average_dclk0_frequency << "\n" + << " average_vclk1_frequency: " << m_gpu_metrics_tbl.m_average_vclk1_frequency << "\n" + << " average_dclk1_frequency: " << m_gpu_metrics_tbl.m_average_dclk1_frequency << "\n" + + << " current_gfxclk: " << m_gpu_metrics_tbl.m_current_gfxclk << "\n" + << " current_socclk: " << m_gpu_metrics_tbl.m_current_socclk << "\n" + << " current_uclk: " << m_gpu_metrics_tbl.m_current_uclk << "\n" + << " current_vclk0: " << m_gpu_metrics_tbl.m_current_vclk0 << "\n" + << " current_dclk0: " << m_gpu_metrics_tbl.m_current_dclk0 << "\n" + << " current_vclk1: " << m_gpu_metrics_tbl.m_current_vclk1 << "\n" + << " current_dclk1: " << m_gpu_metrics_tbl.m_current_dclk1 << "\n" + + << " throttle_status: " << m_gpu_metrics_tbl.m_throttle_status << "\n" + + << " current_fan_speed: " << m_gpu_metrics_tbl.m_current_fan_speed << "\n" + + << " pcie_link_width: " << m_gpu_metrics_tbl.m_pcie_link_width << "\n" + << " pcie_link_speed: " << m_gpu_metrics_tbl.m_pcie_link_speed << "\n" + + << " padding: " << m_gpu_metrics_tbl.m_padding << "\n" + + << " gfx_activity_acc: " << m_gpu_metrics_tbl.m_gfx_activity_acc << "\n" + << " mem_activity_acc: " << m_gpu_metrics_tbl.m_mem_activity_acc << "\n"; + LOG_DEBUG(ostrstream); + + ostrstream << " temperature_hbm: " << "\n"; + auto idx = uint64_t(0); + for (const auto& temp : m_gpu_metrics_tbl.m_temperature_hbm) { + ostrstream << "\t [" << idx << "]: " << temp << "\n"; + ++idx; + } + + ostrstream << " firmware_timestamp: " << m_gpu_metrics_tbl.m_firmware_timestamp << "\n" + + << " voltage_soc: " << m_gpu_metrics_tbl.m_voltage_soc << "\n" + << " voltage_gfx: " << m_gpu_metrics_tbl.m_voltage_gfx << "\n" + << " voltage_mem: " << m_gpu_metrics_tbl.m_voltage_mem << "\n" + + << " padding1: " << m_gpu_metrics_tbl.m_padding1 << "\n" + << " m_indep_throttle_status: " << m_gpu_metrics_tbl.m_indep_throttle_status << "\n"; + LOG_DEBUG(ostrstream); +} + rsmi_status_t GpuMetricsBase_v13_t::populate_metrics_dynamic_tbl() { std::ostringstream ostrstream; @@ -1311,7 +1482,6 @@ AMGpuMetricsPublicLatestTupl_t GpuMetricsBase_v13_t::copy_internal_to_external_m return std::make_tuple(status_code, copy_data_from_internal_metrics_tbl); } - rsmi_status_t GpuMetricsBase_v12_t::populate_metrics_dynamic_tbl() { std::ostringstream ostrstream; @@ -2059,8 +2229,8 @@ rsmi_status_t Device::dev_read_gpu_metrics_all_data() } auto op_result = readDevInfo(DevInfoTypes::kDevGpuMetrics, - m_gpu_metrics_header.m_structure_size, - &m_gpu_metrics_ptr->get_metrics_table()); + m_gpu_metrics_header.m_structure_size, + m_gpu_metrics_ptr->get_metrics_table().get()); if ((status_code = ErrnoToRsmiStatus(op_result)) != rsmi_status_t::RSMI_STATUS_SUCCESS) { ostrstream << __PRETTY_FUNCTION__ @@ -2275,7 +2445,7 @@ rsmi_status_t Device::dev_log_gpu_metrics(std::ostringstream& outstream_metrics) << static_cast(m_gpu_metrics_ptr->get_gpu_metrics_version_used()) << "] " << "\n"; - + tmp_outstream_metrics << " ->Device #: " << index() << "\n"; tmp_outstream_metrics << print_unsigned_hex_and_int(gpu_metrics_header.m_structure_size, " ->structure_size "); tmp_outstream_metrics << print_unsigned_hex_and_int(gpu_metrics_header.m_format_revision, " ->format_revision "); tmp_outstream_metrics << print_unsigned_hex_and_int(gpu_metrics_header.m_content_revision, " ->content_revision ");