AI/rocm-systems
2
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

SWDEV-397381 - Corrected ECC per block reporting

Ver Código-Fonte 
Change-Id: If666e2f1a4b13dd08b6a19f08f3c01297515712c
Signed-off-by: Maisam Arif <Maisam.Arif@amd.com>


[ROCm/amdsmi commit: fe64f426d8]
Esse commit está contido em:
Maisam Arif
2023-05-18 15:53:48 -05:00
pai 66f933eeb3
commit 800af9bf55
4 arquivos alterados com 52 adições e 49 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
projects/amdsmi/amdsmi_cli/amdsmi_commands.py
+27 -24
Ver Arquivo
@@ -263,16 +263,15 @@ class AMDSMICommands():
static_dict['vbios'] = e.get_error_info()
if not self.all_arguments:
raise e
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if args.board:
try:
board_info = amdsmi_interface.amdsmi_get_gpu_board_info(args.gpu)
board_info['serial_number'] = hex(board_info['serial_number'])
board_info['model_number'] = board_info['model_number'].strip()
board_info['product_serial'] = '0x' + board_info['product_serial']
if self.logger.is_gpuvsmi_compatibility():
board_info['product_number'] = board_info.pop('product_serial')
board_info['product_name'] = board_info.pop('product_name')
board_info['product_name'] = board_info['product_name'].strip()
board_info['manufacturer_name'] = board_info['manufacturer_name'].strip()
static_dict['board'] = board_info
except amdsmi_exception.AmdSmiLibraryException as e:
@@ -337,7 +336,7 @@ class AMDSMICommands():
static_dict['driver'] = e.get_error_info()
if not self.all_arguments:
raise e
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if args.ras:
try:
static_dict['ras'] = amdsmi_interface.amdsmi_get_gpu_ras_block_features_enabled(args.gpu)
@@ -373,7 +372,7 @@ class AMDSMICommands():
# Convert and store output by pid for csv format
if self.logger.is_csv_format():
# expand if ras blocks are populated
if (self.helpers.is_linux() and self.helpers.is_baremetal() and args.ras):
if self.helpers.is_linux() and self.helpers.is_baremetal() and args.ras:
if isinstance(static_dict['ras'], list):
ras_dicts = static_dict.pop('ras')
multiple_devices_csv_override = True
@@ -385,7 +384,7 @@ class AMDSMICommands():
else:
# Store values if ras has an error
self.logger.store_output(args.gpu, 'values', static_dict)
if (self.helpers.is_linux() and self.helpers.is_virtual_os()):
if self.helpers.is_linux() and self.helpers.is_virtual_os():
self.logger.store_output(args.gpu, 'values', static_dict)
else:
self.logger.store_output(args.gpu, 'values', static_dict)
@@ -663,7 +662,7 @@ class AMDSMICommands():
if mem_usage:
args.mem_usage = mem_usage
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if usage:
args.usage = usage
if power:
@@ -736,11 +735,11 @@ class AMDSMICommands():
raise IndexError("args.gpu should not be an empty list")
# Check if any of the options have been set, if not then set them all to true
if (self.helpers.is_linux() and self.helpers.is_virtual_os()):
if self.helpers.is_linux() and self.helpers.is_virtual_os():
if not any([args.fb_usage, args.replay_count, args.mem_usage]):
args.fb_usage = args.replay_count = args.mem_usage = self.all_arguments = True
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if not any([args.usage, args.fb_usage, args.power, args.clock, args.temperature, args.ecc, args.ecc_block, args.pcie, args.voltage, args.fan,
args.voltage_curve, args.overdrive, args.mem_overdrive, args.perf_level,
args.replay_count, args.xgmi_err, args.energy, args.mem_usage]):
@@ -750,7 +749,7 @@ class AMDSMICommands():
# Add timestamp and store values for specified arguments
values_dict = {}
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if args.usage:
try:
engine_usage = amdsmi_interface.amdsmi_get_gpu_activity(args.gpu)
@@ -789,7 +788,7 @@ class AMDSMICommands():
if not self.all_arguments:
raise e
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if args.power:
power_dict = {}
try:
@@ -880,6 +879,7 @@ class AMDSMICommands():
if not self.all_arguments:
raise e
if args.ecc:
ecc_count = {}
try:
ecc_count = amdsmi_interface.amdsmi_get_gpu_ecc_error_count(args.gpu)
ecc_count['correctable'] = ecc_count.pop('correctable_count')
@@ -899,18 +899,21 @@ class AMDSMICommands():
if args.ecc_block:
ecc_dict = {}
try:
if self.helpers.has_ras_support(args.gpu):
ras_states = amdsmi_interface.amdsmi_get_gpu_ras_block_features_enabled(args.gpu)
for state in ras_states:
if state['status'] == amdsmi_interface.AmdSmiRasErrState.ENABLED:
gpu_block = amdsmi_interface.AmdSmiGpuBlock[state['block']]
ras_states = amdsmi_interface.amdsmi_get_ras_block_features_enabled(args.gpu)
for state in ras_states:
if state['status'] == amdsmi_interface.AmdSmiRasErrState.ENABLED.name:
gpu_block = amdsmi_interface.AmdSmiGpuBlock[state['block']]
try:
ecc_count = amdsmi_interface.amdsmi_get_gpu_ecc_count(args.gpu, gpu_block)
ecc_dict[state['block']] = {'correctable' : ecc_count['correctable_count'],
'uncorrectable': ecc_count['uncorrectable_count']}
if not ecc_dict:
ecc_dict['correctable_per_block'] = 'N/A'
ecc_dict['uncorrectable_per_block'] = 'N/A'
'uncorrectable': ecc_count['uncorrectable_count']}
except amdsmi_exception.AmdSmiLibraryException as e:
ecc_count = e.get_error_info()
if self.logger.is_gpuvsmi_compatibility():
ecc_count = "N/A"
ecc_dict[state['block']] = {'correctable' : ecc_count,
'uncorrectable': ecc_count}
values_dict['ecc_block'] = ecc_dict
except amdsmi_exception.AmdSmiLibraryException as e:
values_dict['ecc_block'] = e.get_error_info()
@@ -989,7 +992,7 @@ class AMDSMICommands():
else:
frequency = 0
voltage = 0
voltage_point_dict[f'voltage_point_{point}'] = f"{frequency}Mhz {voltage}mV"
voltage_point_dict[f'voltage_point_{point}'] = f"{frequency} Mhz {voltage} mV"
values_dict['voltage_curve'] = voltage_point_dict
except amdsmi_exception.AmdSmiLibraryException as e:
@@ -1027,7 +1030,7 @@ class AMDSMICommands():
values_dict['replay_count'] = e.get_error_info()
if not self.all_arguments:
raise e
if (self.helpers.is_linux() and self.helpers.is_baremetal()):
if self.helpers.is_linux() and self.helpers.is_baremetal():
if args.xgmi_err:
try:
values_dict['xgmi_err'] = amdsmi_interface.amdsmi_gpu_xgmi_error_status(args.gpu)
projects/amdsmi/amdsmi_cli/amdsmi_helpers.py
+7 -10
Ver Arquivo
@@ -61,6 +61,7 @@ class AMDSMIHelpers():
else:
self._is_virtual_os = True
def os_info(self, string_format=True):
"""Return operating_system and type information ex. (Linux, Baremetal)
params:
@@ -386,13 +387,9 @@ class AMDSMIHelpers():
return False, profile_presets.values()
def has_ras_support(self, device_handle):
try:
caps_info = amdsmi_interface.amdsmi_get_caps_info(device_handle)
if caps_info['ras_supported']:
return True
else:
return False
except amdsmi_exception.AmdSmiLibraryException:
return False
def convert_bytes_to_readable(self, bytes_input):
for unit in ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"]:
if abs(bytes_input) < 1024:
return f"{bytes_input:3.1f} {unit}"
bytes_input /= 1024
return f"{bytes_input:.1f} YB"
projects/amdsmi/amdsmi_cli/amdsmi_logger.py
+15 -14
Ver Arquivo
@@ -48,6 +48,7 @@ class AMDSMILogger():
csv = 'csv'
human_readable = 'human_readable'
class LoggerCompatibility(Enum):
"""Enum for logger compatibility"""
amdsmi = 'amdsmi'
@@ -55,6 +56,17 @@ class AMDSMILogger():
gpuvsmi = 'gpuvsmi'
class CsvStdoutBuilder(object):
def __init__(self):
self.csv_string = []
def write(self, row):
self.csv_string.append(row)
def __str__(self):
return ''.join(self.csv_string)
def is_json_format(self):
return self.format == self.LoggerFormat.json.value
@@ -79,17 +91,6 @@ class AMDSMILogger():
return self.compatibility == self.LoggerCompatibility.gpuvsmi.value
class CsvStdoutBuilder(object):
def __init__(self):
self.csv_string = []
def write(self, row):
self.csv_string.append(row)
def __str__(self):
return ''.join(self.csv_string)
def _capitalize_keys(self, input_dict):
output_dict = {}
for key in input_dict.keys():
@@ -239,7 +240,6 @@ class AMDSMILogger():
else:
self.output[argument] = data
elif self.is_csv_format():
# New way is in gpuvsmi func
self.output['gpu'] = int(gpu_id)
if argument == 'values' or isinstance(data, dict):
@@ -416,6 +416,7 @@ class AMDSMILogger():
human_readable_output = ''
for output in self.multiple_device_output:
human_readable_output += self._convert_json_to_human_readable(output)
human_readable_output += '\n'
else:
human_readable_output = self._convert_json_to_human_readable(self.output)
@@ -432,7 +433,7 @@ class AMDSMILogger():
human_readable_output = ''
for output in self.watch_output:
human_readable_output += self._convert_json_to_human_readable(output)
output_file.write(human_readable_output)
output_file.write(human_readable_output + '\n')
else:
with self.destination.open('a') as output_file:
output_file.write(human_readable_output)
output_file.write(human_readable_output + '\n')
projects/amdsmi/py-interface/amdsmi_interface.py
+3 -1
Ver Arquivo
@@ -787,8 +787,10 @@ def amdsmi_get_gpu_board_info(
return {
"serial_number": board_info.serial_number,
"model_number": board_info.model_number.decode("utf-8"),
"product_serial": board_info.product_serial.decode("utf-8"),
"product_name": board_info.product_name.decode("utf-8"),
"manufacturer_name" : board_info.product_name.decode("utf-8")
}
@@ -803,7 +805,7 @@ def amdsmi_get_gpu_ras_block_features_enabled(
ras_state = amdsmi_wrapper.amdsmi_ras_err_state_t()
ras_states = []
for gpu_block in AmdSmiGpuBlock:
if gpu_block.name == "RESERVED":
if gpu_block.name == "RESERVED" or gpu_block.name == "INVALID":
continue
if gpu_block.name == "LAST":
gpu_block.name = "FUSE"