From 800af9bf556f70c06296bb5c6b23b1206b4a2e86 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Thu, 18 May 2023 15:53:48 -0500 Subject: [PATCH] SWDEV-397381 - Corrected ECC per block reporting Change-Id: If666e2f1a4b13dd08b6a19f08f3c01297515712c Signed-off-by: Maisam Arif [ROCm/amdsmi commit: fe64f426d8109075037b516f42bff8e05e0e6ccd] --- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 51 ++++++++++--------- projects/amdsmi/amdsmi_cli/amdsmi_helpers.py | 17 +++---- projects/amdsmi/amdsmi_cli/amdsmi_logger.py | 29 ++++++----- .../amdsmi/py-interface/amdsmi_interface.py | 4 +- 4 files changed, 52 insertions(+), 49 deletions(-) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index e44ab1aede..1669590aa7 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -263,16 +263,15 @@ class AMDSMICommands(): static_dict['vbios'] = e.get_error_info() if not self.all_arguments: raise e - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.board: try: board_info = amdsmi_interface.amdsmi_get_gpu_board_info(args.gpu) board_info['serial_number'] = hex(board_info['serial_number']) + board_info['model_number'] = board_info['model_number'].strip() board_info['product_serial'] = '0x' + board_info['product_serial'] - - if self.logger.is_gpuvsmi_compatibility(): - board_info['product_number'] = board_info.pop('product_serial') - board_info['product_name'] = board_info.pop('product_name') + board_info['product_name'] = board_info['product_name'].strip() + board_info['manufacturer_name'] = board_info['manufacturer_name'].strip() static_dict['board'] = board_info except amdsmi_exception.AmdSmiLibraryException as e: @@ -337,7 +336,7 @@ class AMDSMICommands(): static_dict['driver'] = e.get_error_info() if not self.all_arguments: raise e - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.ras: try: static_dict['ras'] = amdsmi_interface.amdsmi_get_gpu_ras_block_features_enabled(args.gpu) @@ -373,7 +372,7 @@ class AMDSMICommands(): # Convert and store output by pid for csv format if self.logger.is_csv_format(): # expand if ras blocks are populated - if (self.helpers.is_linux() and self.helpers.is_baremetal() and args.ras): + if self.helpers.is_linux() and self.helpers.is_baremetal() and args.ras: if isinstance(static_dict['ras'], list): ras_dicts = static_dict.pop('ras') multiple_devices_csv_override = True @@ -385,7 +384,7 @@ class AMDSMICommands(): else: # Store values if ras has an error self.logger.store_output(args.gpu, 'values', static_dict) - if (self.helpers.is_linux() and self.helpers.is_virtual_os()): + if self.helpers.is_linux() and self.helpers.is_virtual_os(): self.logger.store_output(args.gpu, 'values', static_dict) else: self.logger.store_output(args.gpu, 'values', static_dict) @@ -663,7 +662,7 @@ class AMDSMICommands(): if mem_usage: args.mem_usage = mem_usage - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if usage: args.usage = usage if power: @@ -736,11 +735,11 @@ class AMDSMICommands(): raise IndexError("args.gpu should not be an empty list") # Check if any of the options have been set, if not then set them all to true - if (self.helpers.is_linux() and self.helpers.is_virtual_os()): + if self.helpers.is_linux() and self.helpers.is_virtual_os(): if not any([args.fb_usage, args.replay_count, args.mem_usage]): args.fb_usage = args.replay_count = args.mem_usage = self.all_arguments = True - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if not any([args.usage, args.fb_usage, args.power, args.clock, args.temperature, args.ecc, args.ecc_block, args.pcie, args.voltage, args.fan, args.voltage_curve, args.overdrive, args.mem_overdrive, args.perf_level, args.replay_count, args.xgmi_err, args.energy, args.mem_usage]): @@ -750,7 +749,7 @@ class AMDSMICommands(): # Add timestamp and store values for specified arguments values_dict = {} - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.usage: try: engine_usage = amdsmi_interface.amdsmi_get_gpu_activity(args.gpu) @@ -789,7 +788,7 @@ class AMDSMICommands(): if not self.all_arguments: raise e - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.power: power_dict = {} try: @@ -880,6 +879,7 @@ class AMDSMICommands(): if not self.all_arguments: raise e if args.ecc: + ecc_count = {} try: ecc_count = amdsmi_interface.amdsmi_get_gpu_ecc_error_count(args.gpu) ecc_count['correctable'] = ecc_count.pop('correctable_count') @@ -899,18 +899,21 @@ class AMDSMICommands(): if args.ecc_block: ecc_dict = {} try: - if self.helpers.has_ras_support(args.gpu): - ras_states = amdsmi_interface.amdsmi_get_gpu_ras_block_features_enabled(args.gpu) - for state in ras_states: - if state['status'] == amdsmi_interface.AmdSmiRasErrState.ENABLED: - gpu_block = amdsmi_interface.AmdSmiGpuBlock[state['block']] + ras_states = amdsmi_interface.amdsmi_get_ras_block_features_enabled(args.gpu) + for state in ras_states: + if state['status'] == amdsmi_interface.AmdSmiRasErrState.ENABLED.name: + gpu_block = amdsmi_interface.AmdSmiGpuBlock[state['block']] + try: ecc_count = amdsmi_interface.amdsmi_get_gpu_ecc_count(args.gpu, gpu_block) ecc_dict[state['block']] = {'correctable' : ecc_count['correctable_count'], - 'uncorrectable': ecc_count['uncorrectable_count']} - if not ecc_dict: - ecc_dict['correctable_per_block'] = 'N/A' - ecc_dict['uncorrectable_per_block'] = 'N/A' + 'uncorrectable': ecc_count['uncorrectable_count']} + except amdsmi_exception.AmdSmiLibraryException as e: + ecc_count = e.get_error_info() + if self.logger.is_gpuvsmi_compatibility(): + ecc_count = "N/A" + ecc_dict[state['block']] = {'correctable' : ecc_count, + 'uncorrectable': ecc_count} values_dict['ecc_block'] = ecc_dict except amdsmi_exception.AmdSmiLibraryException as e: values_dict['ecc_block'] = e.get_error_info() @@ -989,7 +992,7 @@ class AMDSMICommands(): else: frequency = 0 voltage = 0 - voltage_point_dict[f'voltage_point_{point}'] = f"{frequency}Mhz {voltage}mV" + voltage_point_dict[f'voltage_point_{point}'] = f"{frequency} Mhz {voltage} mV" values_dict['voltage_curve'] = voltage_point_dict except amdsmi_exception.AmdSmiLibraryException as e: @@ -1027,7 +1030,7 @@ class AMDSMICommands(): values_dict['replay_count'] = e.get_error_info() if not self.all_arguments: raise e - if (self.helpers.is_linux() and self.helpers.is_baremetal()): + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.xgmi_err: try: values_dict['xgmi_err'] = amdsmi_interface.amdsmi_gpu_xgmi_error_status(args.gpu) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py b/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py index f97417fa90..f5df6c72db 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_helpers.py @@ -61,6 +61,7 @@ class AMDSMIHelpers(): else: self._is_virtual_os = True + def os_info(self, string_format=True): """Return operating_system and type information ex. (Linux, Baremetal) params: @@ -386,13 +387,9 @@ class AMDSMIHelpers(): return False, profile_presets.values() - def has_ras_support(self, device_handle): - try: - caps_info = amdsmi_interface.amdsmi_get_caps_info(device_handle) - - if caps_info['ras_supported']: - return True - else: - return False - except amdsmi_exception.AmdSmiLibraryException: - return False + def convert_bytes_to_readable(self, bytes_input): + for unit in ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB"]: + if abs(bytes_input) < 1024: + return f"{bytes_input:3.1f} {unit}" + bytes_input /= 1024 + return f"{bytes_input:.1f} YB" diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py index e8e2e7fd8c..100747bb91 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py @@ -48,6 +48,7 @@ class AMDSMILogger(): csv = 'csv' human_readable = 'human_readable' + class LoggerCompatibility(Enum): """Enum for logger compatibility""" amdsmi = 'amdsmi' @@ -55,6 +56,17 @@ class AMDSMILogger(): gpuvsmi = 'gpuvsmi' + class CsvStdoutBuilder(object): + def __init__(self): + self.csv_string = [] + + def write(self, row): + self.csv_string.append(row) + + def __str__(self): + return ''.join(self.csv_string) + + def is_json_format(self): return self.format == self.LoggerFormat.json.value @@ -79,17 +91,6 @@ class AMDSMILogger(): return self.compatibility == self.LoggerCompatibility.gpuvsmi.value - class CsvStdoutBuilder(object): - def __init__(self): - self.csv_string = [] - - def write(self, row): - self.csv_string.append(row) - - def __str__(self): - return ''.join(self.csv_string) - - def _capitalize_keys(self, input_dict): output_dict = {} for key in input_dict.keys(): @@ -239,7 +240,6 @@ class AMDSMILogger(): else: self.output[argument] = data elif self.is_csv_format(): - # New way is in gpuvsmi func self.output['gpu'] = int(gpu_id) if argument == 'values' or isinstance(data, dict): @@ -416,6 +416,7 @@ class AMDSMILogger(): human_readable_output = '' for output in self.multiple_device_output: human_readable_output += self._convert_json_to_human_readable(output) + human_readable_output += '\n' else: human_readable_output = self._convert_json_to_human_readable(self.output) @@ -432,7 +433,7 @@ class AMDSMILogger(): human_readable_output = '' for output in self.watch_output: human_readable_output += self._convert_json_to_human_readable(output) - output_file.write(human_readable_output) + output_file.write(human_readable_output + '\n') else: with self.destination.open('a') as output_file: - output_file.write(human_readable_output) + output_file.write(human_readable_output + '\n') diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index 700f3b543b..cafe277edb 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -787,8 +787,10 @@ def amdsmi_get_gpu_board_info( return { "serial_number": board_info.serial_number, + "model_number": board_info.model_number.decode("utf-8"), "product_serial": board_info.product_serial.decode("utf-8"), "product_name": board_info.product_name.decode("utf-8"), + "manufacturer_name" : board_info.product_name.decode("utf-8") } @@ -803,7 +805,7 @@ def amdsmi_get_gpu_ras_block_features_enabled( ras_state = amdsmi_wrapper.amdsmi_ras_err_state_t() ras_states = [] for gpu_block in AmdSmiGpuBlock: - if gpu_block.name == "RESERVED": + if gpu_block.name == "RESERVED" or gpu_block.name == "INVALID": continue if gpu_block.name == "LAST": gpu_block.name = "FUSE"