AI/rocm-systems
2
0
Fork 0
Código Incidencias Pull Requests Acciones Paquetes Proyectos Lanzamientos Wiki Actividad

Fix metric type error output + re-align with ROCm SMI metrics

Explorar el Código 
Changes:
* [CLI] Provide fix for "/opt/rocm/bin/amd-smi metric
TypeError: '>' not supported between instances of 'str' and 'i"
--> Python API was updated, CLI needed to reflect these changes
* [API] Updated amdsmi.h's with ROCm SMI
--> Incorrectly added mem_bandwidth_acc & mem_max_bandwidth
--> Realigned wrapper with updates
* [Test] Added metrics not shown in gpu_metrics_read.cc

Change-Id: Ia3a172377fd5a582254dd5a46d81dbec7e763cd9
Signed-off-by: Charis Poag <Charis.Poag@amd.com>
Este commit está contenido en:
Charis Poag
2024-01-24 20:43:47 -06:00
cometido por Maisam Arif
padre 0b67c2ccc4
commit 34bd26c68e
Se han modificado 7 ficheros con 109 adiciones y 88 borrados
Descargar archivo de parche Descargar archivo de diferencias Expandir todos los archivos Contraer todos los archivos
amdsmi_cli/amdsmi_commands.py
+2 -6
Ver fichero
@@ -1121,8 +1121,8 @@ class AMDSMICommands():
# Put the metrics table in the debug logs
try:
gpu_metric_debug_output = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu)
gpu_metric_str = json.dumps(gpu_metric_debug_output, indent=4)
gpu_metric_debug_info = amdsmi_interface.amdsmi_get_gpu_metrics_info(args.gpu)
gpu_metric_str = json.dumps(gpu_metric_debug_info, indent=4)
logging.debug("GPU Metrics table for %s | %s", gpu_id, gpu_metric_str)
except amdsmi_exception.AmdSmiLibraryException as e:
logging.debug("Unabled to load GPU Metrics table for %s | %s", gpu_id, e.err_info)
@@ -1152,10 +1152,6 @@ class AMDSMICommands():
engine_usage['jpeg_activity'] = gpu_metric_info.pop('jpeg_activity')
for key, value in engine_usage.items():
if not isinstance(value, list) and value > 100:
engine_usage[key] = "N/A"
elif isinstance(value, list):
engine_usage[key] = ["N/A" if v > 100 else v for v in value]
if self.logger.is_human_readable_format():
unit = '%'
include/amd_smi/amdsmi.h
-6
Ver fichero
@@ -1385,12 +1385,6 @@ typedef struct {
// JPEG activity % per AID
uint16_t jpeg_activity[AMDSMI_MAX_NUM_JPEG];
// Memory Bandwidth Usage Accumulated (GB/sec)
uint64_t mem_bandwidth_acc;
// Memory Bandwidth Maximum (GB/sec)
uint32_t mem_max_bandwidth;
// PCIE NAK sent accumulated count
uint32_t pcie_nak_sent_count_acc;
py-interface/README.md
-2
Ver fichero
@@ -2136,8 +2136,6 @@ Output: Dictionary with fields
`current_socclks` | List of current soc clock frequencies | MHz
`current_vclk0s` | List of current v0 clock frequencies | MHz
`current_dclk0s` | List of current d0 clock frequencies | MHz
`mem_bandwidth_acc` | Memory bandwidth usage accumulated | GB/s
`mem_max_bandwidth` | Maximum memory bandwidth usage accumulated | GB/s
`pcie_nak_sent_count_acc` | PCIe NAC sent count accumulated |
`pcie_nak_rcvd_count_acc` | PCIe NAC received count accumulated |
`jpeg_activity` | List of JPEG engine activity | %
py-interface/amdsmi_interface.py
+4 -7
Ver fichero
@@ -3290,8 +3290,6 @@ def amdsmi_get_gpu_metrics_info(
"current_socclks": list(gpu_metrics.current_socclks),
"current_vclk0s": list(gpu_metrics.current_vclk0s),
"current_dclk0s": list(gpu_metrics.current_dclk0s),
"mem_bandwidth_acc": gpu_metrics.mem_bandwidth_acc,
"mem_max_bandwidth": gpu_metrics.mem_max_bandwidth,
"pcie_nak_sent_count_acc": gpu_metrics.pcie_nak_sent_count_acc,
"pcie_nak_rcvd_count_acc": gpu_metrics.pcie_nak_rcvd_count_acc,
"jpeg_activity": list(gpu_metrics.jpeg_activity),
@@ -3313,8 +3311,7 @@ def amdsmi_get_gpu_metrics_info(
if gpu_metrics_output[value] == 0xFFFF:
gpu_metrics_output[value] = "N/A"
uint_32_values = ['gfx_activity_acc', 'mem_activity_acc', 'mem_max_bandwidth',
'pcie_nak_sent_count_acc', 'pcie_nak_rcvd_count_acc']
uint_32_values = ['gfx_activity_acc','mem_activity_acc', 'pcie_nak_sent_count_acc', 'pcie_nak_rcvd_count_acc']
for value in uint_32_values:
if gpu_metrics_output[value] == 0xFFFFFFFF:
@@ -3323,7 +3320,7 @@ def amdsmi_get_gpu_metrics_info(
uint_64_values = ['energy_accumulator', 'system_clock_counter', 'firmware_timestamp',
'pcie_bandwidth_acc', 'pcie_bandwidth_inst',
'pcie_l0_to_recov_count_acc', 'pcie_replay_count_acc',
'pcie_replay_rover_count_acc', 'mem_bandwidth_acc']
'pcie_replay_rover_count_acc']
for value in uint_64_values:
if gpu_metrics_output[value] == 0xFFFFFFFFFFFFFFFF:
@@ -3345,7 +3342,7 @@ def amdsmi_get_gpu_metrics_info(
gpu_metrics_output['indep_throttle_status'] = bool(gpu_metrics_output['indep_throttle_status'])
for idx, activity in enumerate(gpu_metrics_output['vcn_activity']):
if activity == 0xFFFF:
if activity == 0xFFFF or activity > 100:
gpu_metrics_output['vcn_activity'][idx] = "N/A"
if gpu_metrics_output['gfxclk_lock_status'] == 0xFFFFFFFF:
@@ -3378,7 +3375,7 @@ def amdsmi_get_gpu_metrics_info(
gpu_metrics_output['current_dclk0s'][idx] = "N/A"
for idx, activity in enumerate(gpu_metrics_output['jpeg_activity']):
if activity == 0xFFFF:
if activity == 0xFFFF or activity > 100:
gpu_metrics_output['jpeg_activity'][idx] = "N/A"
return gpu_metrics_output
py-interface/amdsmi_wrapper.py
-3
Ver fichero
@@ -1620,11 +1620,8 @@ struct_amdsmi_gpu_metrics_t._fields_ = [
('current_vclk0s', ctypes.c_uint16 * 4),
('current_dclk0s', ctypes.c_uint16 * 4),
('jpeg_activity', ctypes.c_uint16 * 32),
('mem_bandwidth_acc', ctypes.c_uint64),
('mem_max_bandwidth', ctypes.c_uint32),
('pcie_nak_sent_count_acc', ctypes.c_uint32),
('pcie_nak_rcvd_count_acc', ctypes.c_uint32),
('PADDING_4', ctypes.c_ubyte * 4),
]
amdsmi_gpu_metrics_t = struct_amdsmi_gpu_metrics_t
src/amd_smi/amd_smi.cc
-4
Ver fichero
@@ -1140,10 +1140,6 @@ amdsmi_status_t amdsmi_get_gpu_metrics_info(
(sizeof(pgpu_metrics->jpeg_activity) /
sizeof(pgpu_metrics->jpeg_activity[0])),
std::numeric_limits<uint16_t>::max());
pgpu_metrics->mem_bandwidth_acc =
static_cast<uint64_t>(std::numeric_limits<uint64_t>::max());
pgpu_metrics->mem_max_bandwidth =
static_cast<uint32_t>(std::numeric_limits<uint32_t>::max());
pgpu_metrics->pcie_nak_sent_count_acc =
static_cast<uint32_t>(std::numeric_limits<uint32_t>::max());
pgpu_metrics->pcie_nak_rcvd_count_acc =
tests/amd_smi_test/functional/gpu_metrics_read.cc
+103 -60
Ver fichero
@@ -135,65 +135,77 @@ void TestGpuMetricsRead::Run(void) {
<< smu.firmware_timestamp << '\n';
std::cout << "\n";
std::cout << "TEMPERATURES (C):\n";
std::cout << std::dec << "temperature_edge="
<< smu.temperature_edge << '\n';
std::cout << std::dec << "temperature_hotspot="
<< smu.temperature_hotspot << '\n';
std::cout << std::dec << "temperature_mem="
<< smu.temperature_mem << '\n';
std::cout << std::dec << "temperature_vrgfx="
<< smu.temperature_vrgfx << '\n';
std::cout << std::dec << "temperature_vrsoc="
<< smu.temperature_vrsoc << '\n';
std::cout << std::dec << "temperature_vrmem="
<< smu.temperature_vrmem << '\n';
std::cout << std::dec << "temperature_edge= "
<< static_cast<uint16_t>(smu.temperature_edge) << '\n';
std::cout << std::dec << "temperature_hotspot= "
<< static_cast<uint16_t>(smu.temperature_hotspot) << '\n';
std::cout << std::dec << "temperature_mem= "
<< static_cast<uint16_t>(smu.temperature_mem) << '\n';
std::cout << std::dec << "temperature_vrgfx= "
<< static_cast<uint16_t>(smu.temperature_vrgfx) << '\n';
std::cout << std::dec << "temperature_vrsoc= "
<< static_cast<uint16_t>(smu.temperature_vrsoc) << '\n';
std::cout << std::dec << "temperature_vrmem= "
<< static_cast<uint16_t>(smu.temperature_vrmem) << '\n';
for (int i = 0; i < AMDSMI_NUM_HBM_INSTANCES; ++i) {
std::cout << "temperature_hbm[" << i << "]=" << std::dec <<
smu.temperature_hbm[i] << '\n';
std::cout << "temperature_hbm[" << i << "]= " << std::dec
<< static_cast<uint16_t>(smu.temperature_hbm[i]) << '\n';
}
std::cout << "\n";
std::cout << "UTILIZATION (%):\n";
std::cout << std::dec << "average_gfx_activity="
<< smu.average_gfx_activity << '\n';
<< static_cast<uint16_t>(smu.average_gfx_activity) << '\n';
std::cout << std::dec << "average_umc_activity="
<< smu.average_umc_activity << '\n';
<< static_cast<uint16_t>(smu.average_umc_activity) << '\n';
std::cout << std::dec << "average_mm_activity="
<< smu.average_mm_activity << '\n';
std::cout << std::dec << "jpeg_activity= [";
<< static_cast<uint16_t>(smu.average_mm_activity) << '\n';
std::cout << std::dec << "vcn_activity= [";
uint16_t size = static_cast<uint16_t>(
sizeof(smu.vcn_activity)/sizeof(smu.vcn_activity[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << static_cast<uint16_t>(smu.vcn_activity[i]) << ", ";
} else {
std::cout << std::dec << static_cast<uint16_t>(smu.vcn_activity[i]);
}
}
std::cout << std::dec << "]\n";
std::cout << "\n";
std::cout << std::dec << "jpeg_activity= [";
size = static_cast<uint16_t>(
sizeof(smu.jpeg_activity)/sizeof(smu.jpeg_activity[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.jpeg_activity[i] << ", ";
std::cout << std::dec << static_cast<uint16_t>(smu.jpeg_activity[i]) << ", ";
} else {
std::cout << std::dec << smu.jpeg_activity[i];
std::cout << std::dec << static_cast<uint16_t>(smu.jpeg_activity[i]);
}
}
std::cout << std::dec << "]\n";
std::cout << "\n";
std::cout << "POWER (W)/ENERGY (15.259uJ per 1ns):\n";
std::cout << std::dec << "average_socket_power="
<< smu.average_socket_power << '\n';
<< static_cast<uint16_t>(smu.average_socket_power) << '\n';
std::cout << std::dec << "current_socket_power="
<< smu.current_socket_power << '\n';
<< static_cast<uint16_t>(smu.current_socket_power) << '\n';
std::cout << std::dec << "energy_accumulator="
<< smu.energy_accumulator << '\n';
<< static_cast<uint16_t>(smu.energy_accumulator) << '\n';
std::cout << "\n";
std::cout << "AVG CLOCKS (MHz):\n";
std::cout << std::dec << "average_gfxclk_frequency="
<< smu.average_gfxclk_frequency << '\n';
<< static_cast<uint16_t>(smu.average_gfxclk_frequency) << '\n';
std::cout << std::dec << "average_gfxclk_frequency="
<< smu.average_gfxclk_frequency << '\n';
<< static_cast<uint16_t>(smu.average_gfxclk_frequency) << '\n';
std::cout << std::dec << "average_uclk_frequency="
<< smu.average_uclk_frequency << '\n';
<< static_cast<uint16_t>(smu.average_uclk_frequency) << '\n';
std::cout << std::dec << "average_vclk0_frequency="
<< smu.average_vclk0_frequency << '\n';
<< static_cast<uint16_t>(smu.average_vclk0_frequency) << '\n';
std::cout << std::dec << "average_dclk0_frequency="
<< smu.average_dclk0_frequency << '\n';
<< static_cast<uint16_t>(smu.average_dclk0_frequency) << '\n';
std::cout << std::dec << "average_vclk1_frequency="
<< smu.average_vclk1_frequency << '\n';
<< static_cast<uint16_t>(smu.average_vclk1_frequency) << '\n';
std::cout << std::dec << "average_dclk1_frequency="
<< smu.average_dclk1_frequency << '\n';
<< static_cast<uint16_t>(smu.average_dclk1_frequency) << '\n';
std::cout << "\n";
std::cout << "CURRENT CLOCKS (MHz):\n";
std::cout << std::dec << "current_gfxclk="
@@ -203,9 +215,9 @@ void TestGpuMetricsRead::Run(void) {
sizeof(smu.current_gfxclks)/sizeof(smu.current_gfxclks[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.current_gfxclks[i] << ", ";
std::cout << std::dec << static_cast<uint16_t>(smu.current_gfxclks[i]) << ", ";
} else {
std::cout << std::dec << smu.current_gfxclks[i];
std::cout << std::dec << static_cast<uint16_t>(smu.current_gfxclks[i]);
}
}
std::cout << std::dec << "]\n";
@@ -216,24 +228,24 @@ void TestGpuMetricsRead::Run(void) {
sizeof(smu.current_socclks)/sizeof(smu.current_socclks[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.current_socclks[i] << ", ";
std::cout << std::dec << static_cast<uint16_t>(smu.current_socclks[i]) << ", ";
} else {
std::cout << std::dec << smu.current_socclks[i];
std::cout << std::dec << static_cast<uint16_t>(smu.current_socclks[i]);
}
}
std::cout << std::dec << "]\n";
std::cout << std::dec << "current_uclk="
<< smu.current_uclk << '\n';
<< static_cast<uint16_t>(smu.current_uclk) << '\n';
std::cout << std::dec << "current_vclk0="
<< smu.current_vclk0 << '\n';
<< static_cast<uint16_t>(smu.current_vclk0) << '\n';
std::cout << std::dec << "current_vclk0s= [";
size = static_cast<uint16_t>(
sizeof(smu.current_vclk0s)/sizeof(smu.current_vclk0s[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.current_vclk0s[i] << ", ";
std::cout << std::dec << static_cast<uint16_t>(smu.current_vclk0s[i]) << ", ";
} else {
std::cout << std::dec << smu.current_vclk0s[i];
std::cout << std::dec << static_cast<uint16_t>(smu.current_vclk0s[i]);
}
}
std::cout << std::dec << "]\n";
@@ -244,24 +256,24 @@ void TestGpuMetricsRead::Run(void) {
sizeof(smu.current_dclk0s)/sizeof(smu.current_dclk0s[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.current_dclk0s[i] << ", ";
std::cout << std::dec << static_cast<uint16_t>(smu.current_dclk0s[i]) << ", ";
} else {
std::cout << std::dec << smu.current_dclk0s[i];
std::cout << std::dec << static_cast<uint16_t>(smu.current_dclk0s[i]);
}
}
std::cout << std::dec << "]\n";
std::cout << std::dec << "current_vclk1="
<< smu.current_vclk1 << '\n';
<< static_cast<uint16_t>(smu.current_vclk1) << '\n';
std::cout << std::dec << "current_dclk1="
<< smu.current_dclk1 << '\n';
<< static_cast<uint16_t>(smu.current_dclk1) << '\n';
std::cout << "\n";
std::cout << "TROTTLE STATUS:\n";
std::cout << std::dec << "throttle_status="
<< smu.throttle_status << '\n';
<< static_cast<uint32_t>(smu.throttle_status) << '\n';
std::cout << "\n";
std::cout << "FAN SPEED:\n";
std::cout << std::dec << "current_fan_speed="
<< smu.current_fan_speed << '\n';
<< static_cast<uint16_t>(smu.current_fan_speed) << '\n';
std::cout << "\n";
std::cout << "LINK WIDTH (number of lanes) /SPEED (0.1 GT/s):\n";
std::cout << "pcie_link_width="
@@ -276,9 +288,9 @@ void TestGpuMetricsRead::Run(void) {
std::cout << "\n";
std::cout << "Utilization Accumulated(%):\n";
std::cout << "gfx_activity_acc="
<< std::dec << smu.gfx_activity_acc << '\n';
<< std::dec << static_cast<uint32_t>(smu.gfx_activity_acc) << '\n';
std::cout << "mem_activity_acc="
<< std::dec << smu.mem_activity_acc << '\n';
<< std::dec << static_cast<uint32_t>(smu.mem_activity_acc) << '\n';
std::cout << "\n";
std::cout << "XGMI ACCUMULATED DATA TRANSFER SIZE (KB):\n";
@@ -287,9 +299,9 @@ void TestGpuMetricsRead::Run(void) {
sizeof(smu.xgmi_read_data_acc)/sizeof(smu.xgmi_read_data_acc[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.xgmi_read_data_acc[i] << ", ";
std::cout << std::dec << static_cast<uint64_t>(smu.xgmi_read_data_acc[i]) << ", ";
} else {
std::cout << std::dec << smu.xgmi_read_data_acc[i];
std::cout << std::dec << static_cast<uint64_t>(smu.xgmi_read_data_acc[i]);
}
}
std::cout << std::dec << "]\n";
@@ -298,31 +310,62 @@ void TestGpuMetricsRead::Run(void) {
sizeof(smu.xgmi_write_data_acc)/sizeof(smu.xgmi_write_data_acc[0]));
for (uint16_t i= 0; i < size; i++) {
if (i+1 < size) {
std::cout << std::dec << smu.xgmi_write_data_acc[i] << ", ";
std::cout << std::dec << static_cast<uint64_t>(smu.xgmi_write_data_acc[i]) << ", ";
} else {
std::cout << std::dec << smu.xgmi_write_data_acc[i];
std::cout << std::dec << static_cast<uint64_t>(smu.xgmi_write_data_acc[i]);
}
}
std::cout << std::dec << "]\n";
std::cout << "mem_bandwidth_acc=" << std::dec
<< smu.mem_bandwidth_acc << "\n";
std::cout << "mem_max_bandwidth=" << std::dec
<< smu.mem_max_bandwidth << "\n";
std::cout << "pcie_nak_sent_count_acc=" << std::dec
<< smu.pcie_nak_sent_count_acc << "\n";
std::cout << "pcie_nak_rcvd_count_acc=" << std::dec
<< smu.pcie_nak_rcvd_count_acc << "\n";
// Voltage (mV)
std::cout << "voltage_soc = "
<< std::dec << static_cast<uint16_t>(smu.voltage_soc) << "\n";
std::cout << "voltage_soc = "
<< std::dec << static_cast<uint16_t>(smu.voltage_gfx) << "\n";
std::cout << "voltage_mem = "
<< std::dec << static_cast<uint16_t>(smu.voltage_mem) << "\n";
std::cout << "indep_throttle_status = "
<< std::dec << static_cast<uint64_t>(smu.indep_throttle_status) << "\n";
// Clock Lock Status. Each bit corresponds to clock instance
std::cout << "gfxclk_lock_status (in hex) = "
<< std::hex << static_cast<uint32_t>(smu.gfxclk_lock_status) << std::dec <<"\n";
// Bandwidth (GB/sec)
std::cout << "pcie_bandwidth_acc=" << std::dec
<< static_cast<uint64_t>(smu.pcie_bandwidth_acc) << "\n";
std::cout << "pcie_bandwidth_inst=" << std::dec
<< static_cast<uint64_t>(smu.pcie_bandwidth_inst) << "\n";
// Counts
std::cout << "pcie_l0_to_recov_count_acc= " << std::dec
<< static_cast<uint64_t>(smu.pcie_l0_to_recov_count_acc) << "\n";
std::cout << "pcie_replay_count_acc= " << std::dec
<< static_cast<uint64_t>(smu.pcie_replay_count_acc) << "\n";
std::cout << "pcie_replay_rover_count_acc= " << std::dec
<< static_cast<uint64_t>(smu.pcie_replay_rover_count_acc) << "\n";
std::cout << "pcie_nak_rcvd_count_acc= " << std::dec
<< static_cast<uint32_t>(smu.pcie_nak_rcvd_count_acc) << "\n";
std::cout << "pcie_replay_rover_count_acc= " << std::dec
<< static_cast<uint64_t>(smu.pcie_replay_rover_count_acc) << "\n";
}
}
// Verify api support checking functionality is working
err = amdsmi_get_gpu_metrics_info(processor_handles_[i], nullptr);
DISPLAY_AMDSMI_ERR(err);
if (err !=AMDSMI_STATUS_INVAL) {
DISPLAY_AMDSMI_ERR(err);
}
amdsmi_status_code_to_string(err, &status_string);
std::cout << "\t\t** amdsmi_get_gpu_metrics_info(nullptr check): " << status_string << "\n";
ASSERT_EQ(err, AMDSMI_STATUS_INVAL);
}
/**
* START OF INDIVIDUAL METRIC CALLS
*/
auto val_ui16 = uint16_t(0);
auto val_ui32 = uint32_t(0);
auto val_ui64 = uint64_t(0);