diff --git a/.gitignore b/.gitignore index d87e07ee63..5b157b6f4b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,8 @@ DEBIAN/postinst DEBIAN/prerm RPM/ docs/*.pdf + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*.egg-* diff --git a/README.md b/README.md index 40141df9f2..25f8225bec 100755 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ After the AMD SMI library git repository has been cloned to a local Linux machin ```bash mkdir -p build cd build -cmake +cmake .. make -j $(nproc) # Install library file and header; default location is /opt/rocm make install``` diff --git a/amdsmi_cli/BDF.py b/amdsmi_cli/BDF.py new file mode 100644 index 0000000000..f9ffe338e5 --- /dev/null +++ b/amdsmi_cli/BDF.py @@ -0,0 +1,130 @@ +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import logging +import re + + +class BDF(): + """ BDF Class to cast and compare BDF objects using built-in python comparators + + Useful for validating a BDF string and converting it to a BDF object + This allows us to handle BDF objects in a pythonic way + + Attributes: + __eq__: The equals comparator + __: An integer count of the eggs we have laid. + """ + def __init__(self, bdf): + """Init a BDF object""" + if isinstance(bdf, BDF): + self.segment, self.bus, self.device, self.function = tuple(bdf) + else: + if bdf.startswith("BDF("): + bdf = bdf.replace('BDF(', '').replace(')', '') + + try: + bdf_components = [int(x, 16) for x in re.split('[:.]', bdf)] + except self.BDFError as e: + logging.error(f"Invalid string passed: {bdf}") + raise e + + self.segment = bdf_components[0] if len(bdf_components) == 4 else 0 + self.bus, self.device, self.function = bdf_components[-3:] + if self.segment > 65535: + raise self.BDFError("Segment can't be greater than 65535") + if self.bus > 255: + raise self.BDFError("Bus can't be greater than 255") + if self.device > 31: + raise self.BDFError("Device can't be greater than 31") + if self.function > 7: + raise self.BDFError("Function can't be greater than 7") + + + class BDFError(Exception): + """BDF Class Error""" + + + def __eq__(self, passed_bdf): + """Overrides the == operator and allows for BDF objects to be compared to BDF strings""" + + # Only accept strings and BDF objects + if isinstance(passed_bdf, str): + if passed_bdf == '': + return False + passed_bdf = BDF(passed_bdf) + elif not isinstance(passed_bdf, BDF): + return False + + if self.segment == passed_bdf.segment and \ + self.bus == passed_bdf.bus and \ + self.device == passed_bdf.device and \ + self.function == passed_bdf.function: + return True + else: + return False + + + def __ne__(self, passed_bdf): + """Overrides the != operator and allows for BDF objects to be compared to BDF strings""" + # Since we overrided the == operator we can use that to make this simple + return not self == passed_bdf + + + def __add__(self, passed_bdf): + """Overrides the + operator and allows for string concatenation""" + return str(self) + passed_bdf + + + def __radd__(self, passed_bdf): + """Overrides the + operator and allows for string concatenation""" + return passed_bdf + str(self) + + + def __str__(self): + """Cast BDF object to a string""" + return "{:04X}:{:02X}:{:02X}:{}".format(self.segment, self.bus, self.device, self.function) + + + def __repr__(self): + """How the BDF object is represented""" + return f"BDF({self})" + + + def __hash__(self): + """Allow the BDF object to be hashable""" + return hash(str(self)) + + + def __iter__(self): + """Make the BDF object iterable over its 4 values""" + yield from (self.segment, self.bus, self.device, self.function) + + + def __contains__(self, passed_bdf): + """Overrided the 'in' comparator in python""" + passed_bdf = str(BDF(passed_bdf)) + + bdf_regex = "(?:[0-6]?[0-9a-fA-F]{1,4}:)?[0-2]?[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}\.[0-7]" + for match in re.findall(bdf_regex, passed_bdf): + if self == match: + return True + return False diff --git a/amdsmi_cli/_version.py b/amdsmi_cli/_version.py new file mode 100644 index 0000000000..b3c06d4883 --- /dev/null +++ b/amdsmi_cli/_version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" \ No newline at end of file diff --git a/amdsmi_cli/amdsmi_cli.py b/amdsmi_cli/amdsmi_cli.py new file mode 100755 index 0000000000..38e346fb9e --- /dev/null +++ b/amdsmi_cli/amdsmi_cli.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import logging +import sys + +from amdsmi_commands import AMDSMICommands +from amdsmi_parser import AMDSMIParser + + +if __name__ == "__main__": + amd_smi_commands = AMDSMICommands() + amd_smi_parser = AMDSMIParser(amd_smi_commands.version, + amd_smi_commands.discovery, + amd_smi_commands.static, + amd_smi_commands.firmware, + amd_smi_commands.bad_pages, + amd_smi_commands.metric, + amd_smi_commands.process, + amd_smi_commands.profile, + amd_smi_commands.event, + amd_smi_commands.topology, + amd_smi_commands.set_value, + amd_smi_commands.reset, + amd_smi_commands.rocm_smi) + + args = amd_smi_parser.parse_args(args=None if sys.argv[1:] else ['--help']) + + # Handle command modifiers before subcommand execution + if args.json: + amd_smi_commands.logger.format = amd_smi_commands.logger.LoggerFormat.json.value + if args.csv: + amd_smi_commands.logger.format = amd_smi_commands.logger.LoggerFormat.csv.value + if args.file: + amd_smi_commands.logger.destination = args.file + if args.loglevel: + logging_dict = {'DEBUG' : logging.DEBUG, + 'INFO' : logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL} + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_dict[args.loglevel]) + if args.compatibility: + amd_smi_commands.logger.compatibility = args.compatibility + + # Execute subcommands + args.func(args) diff --git a/amdsmi_cli/amdsmi_commands.py b/amdsmi_cli/amdsmi_commands.py new file mode 100644 index 0000000000..5a8129e1bb --- /dev/null +++ b/amdsmi_cli/amdsmi_commands.py @@ -0,0 +1,1200 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +"""AMDSMICommands + +This class contains all the commands corresponding to AMDSMIParser +Each command function will interact with AMDSMILogger to handle +displaying the output to the specified compatibility, format, and +destination. + +""" + +from _version import __version__ + +from amdsmi_helpers import * +from amdsmi_logger import AMDSMILogger + + +class AMDSMICommands(): + def __init__(self, compatibility='amdsmi', + format='human_readable', + destination='stdout') -> None: + self.helpers = AMDSMIHelpers() + self.logger = AMDSMILogger(compatibility=compatibility, + format=format, + destination=destination) + self.device_handles = amdsmi_interface.amdsmi_get_device_handles() + + + def version(self, args): + """Print Version String + + Args: + args (Namespace): Namespace containing the parsed CLI args + """ + kernel_component = amdsmi_interface.AmdSmiSwComponent.DRIVER + kernel_version = amdsmi_interface.amdsmi_get_version_str(sw_component=kernel_component) + amdsmi_lib_version = amdsmi_interface.amdsmi_get_version() + major = amdsmi_lib_version["major"] + minor = amdsmi_lib_version["minor"] + patch = amdsmi_lib_version["patch"] + amdsmi_lib_version_str = f'{major}.{minor}.{patch}' + + self.logger.output['tool'] = 'AMDSMI Tool' + self.logger.output['version'] = f'{__version__}' + self.logger.output['amdsmi_library_version'] = f'{amdsmi_lib_version_str}' + self.logger.output['kernel_version'] = f'{kernel_version}' + + if self.logger.is_human_readable_format(): + print(f'AMDSMI Tool: {__version__} | '\ + f'AMDSMI Library version: {amdsmi_lib_version_str} | '\ + f'Kernel version: {kernel_version}') + elif self.logger.is_json_format() or self.logger.is_csv_format(): + self.logger.print_output() + + + def discovery(self, args, multiple_devices=False, gpu=None): + """Get Discovery information for target gpu + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.discovery(args, multiple_devices=True, gpu=device_handle) + self.logger.print_output(multiple_device_output=True) + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + bdf = amdsmi_interface.amdsmi_get_device_bdf(args.gpu) + uuid = amdsmi_interface.amdsmi_get_device_uuid(args.gpu) + + # Store values based on format + if self.logger.is_human_readable_format(): + self.logger.store_output(args.gpu, 'AMDSMI_SPACING_REMOVAL', {'bdf':bdf, 'uuid':uuid}) + else: + self.logger.store_output(args.gpu, 'bdf', bdf) + self.logger.store_output(args.gpu, 'uuid', uuid) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + # compatibility with gpuvsmi needs a list for single gpu + if self.logger.is_gpuvsmi_compatibility() and not multiple_devices: + self.logger.store_multiple_device_output() + self.logger.print_output(multiple_device_output=True) + else: + self.logger.print_output() + + + def static(self, args, multiple_devices=False, gpu=None, asic=None, + bus=None, vbios=None, limit=None, driver=None, caps=None, + ras=None, board=None): + """Get Static information for target gpu + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + asic (bool, optional): Value override for args.asic. Defaults to None. + bus (bool, optional): Value override for args.bus. Defaults to None. + vbios (bool, optional): Value override for args.vbios. Defaults to None. + limit (bool, optional): Value override for args.limit. Defaults to None. + driver (bool, optional): Value override for args.driver. Defaults to None. + caps (bool, optional): Value override for args.caps. Defaults to None. + ras (bool, optional): Value override for args.ras. Defaults to None. + board (bool, optional): Value override for args.board. Defaults to None. + + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + if asic: + args.asic = asic + if bus: + args.bus = bus + if vbios: + args.vbios = vbios + if limit: + args.limit = limit + if driver: + args.driver = driver + if caps: + args.caps = caps + if ras: + args.ras = ras + if board: + args.board = board + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.static(args, multiple_devices=True, gpu=device_handle) + self.logger.print_output(multiple_device_output=True) + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + # If all arguments are False, it means that no argument was passed and the entire static should be printed + if not any([args.asic, args.bus, args.vbios, args.limit, args.driver, args.caps, args.ras, args.board]): + args.asic = args.bus = args.vbios = args.limit = args.driver = args.caps = args.ras = args.board = True + + values_dict = {} + + if args.asic: + asic_info = amdsmi_interface.amdsmi_get_asic_info(args.gpu) + asic_info['family'] = hex(asic_info['family']) + asic_info['vendor_id'] = hex(asic_info['vendor_id']) + asic_info['device_id'] = hex(asic_info['device_id']) + asic_info['rev_id'] = hex(asic_info['rev_id']) + if asic_info['asic_serial'] != '': + asic_info['asic_serial'] = '0x' + asic_info['asic_serial'] + + values_dict['asic'] = asic_info + if args.bus: + bus_info = amdsmi_interface.amdsmi_get_pcie_link_caps(args.gpu) + + if self.logger.is_human_readable_format(): + unit ='MT/s' + bus_info['pcie_speed'] = f"{bus_info['pcie_speed']} {unit}" + + bus_output_info = {} + bus_output_info['bdf'] = amdsmi_interface.amdsmi_get_device_bdf(args.gpu) + bus_output_info.update(bus_info) + + values_dict['bus'] = bus_output_info + if args.vbios: + vbios_info = amdsmi_interface.amdsmi_get_vbios_info(args.gpu) + + if self.logger.is_gpuvsmi_compatibility(): + vbios_info['version'] = vbios_info.pop('vbios_version_string') + vbios_info['build_date'] = vbios_info.pop('build_date') + vbios_info['part_number'] = vbios_info.pop('part_number') + vbios_info['vbios_version'] = vbios_info.pop('vbios_version') + + values_dict['vbios'] = vbios_info + if args.board: + board_info = amdsmi_interface.amdsmi_get_board_info(args.gpu) + board_info['serial_number'] = hex(board_info['serial_number']) + board_info['product_serial'] = '0x' + board_info['product_serial'] + + if self.logger.is_gpuvsmi_compatibility(): + board_info['product_number'] = board_info.pop('product_serial') + board_info['product_name'] = board_info.pop('product_name') + + values_dict['board'] = board_info + if args.limit: + power_limit = amdsmi_interface.amdsmi_get_power_measure(args.gpu)['power_limit'] + temp_edge_limit = amdsmi_interface.amdsmi_dev_get_temp_metric(args.gpu, + amdsmi_interface.AmdSmiTemperatureType.EDGE, amdsmi_interface.AmdSmiTemperatureMetric.CRITICAL) + temp_junction_limit = amdsmi_interface.amdsmi_dev_get_temp_metric(args.gpu, + amdsmi_interface.AmdSmiTemperatureType.JUNCTION, amdsmi_interface.AmdSmiTemperatureMetric.CRITICAL) + temp_vram_limit = amdsmi_interface.amdsmi_dev_get_temp_metric(args.gpu, + amdsmi_interface.AmdSmiTemperatureType.VRAM, amdsmi_interface.AmdSmiTemperatureMetric.CRITICAL) + + if self.logger.is_human_readable_format(): + unit = 'W' + power_limit = f"{power_limit} {unit}" + + unit = 'C' + temp_edge_limit = f"{temp_edge_limit} {unit}" + temp_junction_limit = f"{temp_junction_limit} {unit}" + temp_vram_limit = f"{temp_vram_limit} {unit}" + + limit_info = {} + limit_info['power'] = power_limit + limit_info['temperature_edge'] = temp_edge_limit + limit_info['temperature_junction'] = temp_junction_limit + limit_info['temperature_vram'] = temp_vram_limit + + values_dict['limit'] = limit_info + if args.driver: + driver_info = {} + driver_info['driver_version'] = amdsmi_interface.amdsmi_get_driver_version(args.gpu) + + values_dict['driver'] = driver_info + if args.ras: + try: + ras_info = amdsmi_interface.amdsmi_get_ras_block_features_enabled(args.gpu) + values_dict['ras'] = ras_info + except amdsmi_interface.AmdSmiLibraryException as err: + values_dict['ras'] = err.get_error_info() + if args.caps: + caps_info = amdsmi_interface.amdsmi_get_caps_info(args.gpu) + + if self.logger.is_gpuvsmi_compatibility(): + del caps_info['ras_supported'] + caps_info['gfx'] = caps_info.pop('gfx') + + if self.logger.is_human_readable_format(): + for capability_name, capability_value in caps_info.items(): + if isinstance(capability_value, list): + caps_info[capability_name] = f"{capability_value}" + + values_dict['caps'] = caps_info + + # Store values in logger.output + self.logger.store_output(args.gpu, 'values', values_dict) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + + def firmware(self, args, multiple_devices=False, gpu=None, fw_list=True): + """ Get Firmware information for target gpu + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + fw_list (bool, optional): True to get list of all firmware information + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + if gpu: + args.gpu = gpu + if fw_list: # Currently a compatiblity option of gpuv-smi + args.fw_list = fw_list + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.firmware(args, multiple_devices=True, gpu=device_handle) + self.logger.print_output(multiple_device_output=True) + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + values_dict = {} + + if args.fw_list: + fw_info = amdsmi_interface.amdsmi_get_fw_info(args.gpu) + + for fw_index, fw_entry in enumerate(fw_info['fw_list']): + # Change fw_name to fw_id + fw_entry['fw_id'] = fw_entry.pop('fw_name').name.strip('FW_ID_') + fw_entry['fw_version'] = fw_entry.pop('fw_version') + firmware_identifier = 'FW' + + if self.logger.is_gpuvsmi_compatibility(): + firmware_identifier = 'UCODE' + fw_entry['name'] = fw_entry.pop('fw_id') + fw_entry['version'] = fw_entry.pop('fw_version') + + # Add custom human readable formatting + if self.logger.is_human_readable_format(): + fw_info['fw_list'][fw_index] = {f'{firmware_identifier} {fw_index}': fw_entry} + else: + fw_info['fw_list'][fw_index] = fw_entry + + if self.logger.is_gpuvsmi_compatibility(): + fw_info['ucode_list'] = fw_info.pop('fw_list') + + values_dict.update(fw_info) + + # Store values in logger.output + self.logger.store_output(args.gpu, 'values', values_dict) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + + def bad_pages(self, args, multiple_devices=False, gpu=None, retired=None, pending=None, un_res=None): + """ Get bad pages information for target gpu + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + retired (bool, optional) - Value override for args.retired + pending (bool, optional) - Value override for args.pending + un_res (bool, optional) - Value override for args.un_res + + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + if retired: + args.retired = retired + if pending: + args.pending = pending + if un_res: + args.un_res = un_res + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.bad_pages(args, multiple_devices=True, gpu=device_handle) + self.logger.print_output(multiple_device_output=True) + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + # If all arguments are False, the print all bad_page information + if not any([args.retired, args.pending, args.un_res]): + args.retired = args.pending = args.un_res = True + + values_dict = {} + bad_page_err_output = '' + + try: + bad_page_info = amdsmi_interface.amdsmi_get_bad_page_info(args.gpu) + bad_page_error = False + except amdsmi_interface.AmdSmiLibraryException as err: + bad_page_err_output = err.get_error_info() + bad_page_error = True + + if isinstance(bad_page_info, str): + pass + else: + if args.retired: + if bad_page_error: + bad_page_info_output = bad_page_err_output + else: + bad_page_info_output = [] + for bad_page in bad_page_info: + if bad_page["status"] == amdsmi_interface.AmdSmiMemoryPageStatus.RESERVED: + bad_page_info_entry = {} + bad_page_info_entry["page_address"] = bad_page["page_address"] + bad_page_info_entry["page_size"] = bad_page["page_size"] + bad_page_info_entry["status"] = bad_page["status"].name + + bad_page_info_output.append(bad_page_info_entry) + # Remove brackets if there is only one value + if len(bad_page_info_output) == 1: + bad_page_info_output = bad_page_info_output[0] + + values_dict['retired'] = bad_page_info_output + + if args.pending: + if bad_page_error: + bad_page_info_output = bad_page_err_output + else: + bad_page_info_output = [] + for bad_page in bad_page_info: + if bad_page["status"] == amdsmi_interface.AmdSmiMemoryPageStatus.PENDING: + bad_page_info_entry = {} + bad_page_info_entry["page_address"] = bad_page["page_address"] + bad_page_info_entry["page_size"] = bad_page["page_size"] + bad_page_info_entry["status"] = bad_page["status"].name + + bad_page_info_output.append(bad_page_info_entry) + # Remove brackets if there is only one value + if len(bad_page_info_output) == 1: + bad_page_info_output = bad_page_info_output[0] + + values_dict['pending'] = bad_page_info_output + + if args.un_res: + if bad_page_error: + bad_page_info_output = bad_page_err_output + else: + bad_page_info_output = [] + for bad_page in bad_page_info: + if bad_page["status"] == amdsmi_interface.AmdSmiMemoryPageStatus.UNRESERVABLE: + bad_page_info_entry = {} + bad_page_info_entry["page_address"] = bad_page["page_address"] + bad_page_info_entry["page_size"] = bad_page["page_size"] + bad_page_info_entry["status"] = bad_page["status"].name + + bad_page_info_output.append(bad_page_info_entry) + # Remove brackets if there is only one value + if len(bad_page_info_output) == 1: + bad_page_info_output = bad_page_info_output[0] + + values_dict['un_res'] = bad_page_info_output + + # Store values in logger.output + self.logger.store_output(args.gpu, 'values', values_dict) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + + def metric(self, args, multiple_devices=False, watching_output=False, gpu=None, + usage=None, watch=None, watch_time=None, iterations=None, fb_usage=None, power=None, + clock=None, temperature=None, ecc=None, pcie=None, voltage=None, fan=None, + pcie_usage=None, voltage_curve=None, overdrive=None, mem_overdrive=None, + perf_level=None, replay_count=None, xgmi_err=None, energy=None, mem_usage=None): + """Get Metric information for target gpu + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + watching_output (bool, optional): True if watch option has been set. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + usage (bool, optional): Value override for args.usage. Defaults to None. + watch (Positive int, optional): Value override for args.watch. Defaults to None. + watch_time (Positive int, optional): Value override for args.watch_time. Defaults to None. + iterations (Positive int, optional): Value override for args.iterations. Defaults to None. + fb_usage (bool, optional): Value override for args.fb_usage. Defaults to None. + power (bool, optional): Value override for args.power. Defaults to None. + clock (bool, optional): Value override for args.clock. Defaults to None. + temperature (bool, optional): Value override for args.temperature. Defaults to None. + ecc (bool, optional): Value override for args.ecc. Defaults to None. + pcie (bool, optional): Value override for args.pcie. Defaults to None. + voltage (bool, optional): Value override for args.voltage. Defaults to None. + fan (bool, optional): Value override for args.fan. Defaults to None. + pcie_usage (bool, optional): Value override for args.pcie_usage. Defaults to None. + voltage_curve (bool, optional): Value override for args.voltage_curve. Defaults to None. + overdrive (bool, optional): Value override for args.overdrive. Defaults to None. + mem_overdrive (bool, optional): Value override for args.mem_overdrive. Defaults to None. + perf_level (bool, optional): Value override for args.perf_level. Defaults to None. + replay_count (bool, optional): Value override for args.replay_count. Defaults to None. + xgmi_err (bool, optional): Value override for args.xgmi_err. Defaults to None. + energy (bool, optional): Value override for args.energy. Defaults to None. + mem_usage (bool, optional): Value override for args.mem_usage. Defaults to None. + + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + if usage: + args.usage = usage + if watch: + args.watch = watch + if watch_time: + args.watch_time = watch_time + if iterations: + args.iterations = iterations + if fb_usage: + args.fb_usage = fb_usage + if power: + args.power = power + if clock: + args.clock = clock + if temperature: + args.temperature = temperature + if ecc: + args.ecc = ecc + if pcie: + args.pcie = pcie + if voltage: + args.voltage = voltage + if fan: + args.fan = fan + if pcie_usage: + args.pcie_usage = pcie_usage + if voltage_curve: + args.voltage_curve = voltage_curve + if overdrive: + args.overdrive = overdrive + if mem_overdrive: + args.mem_overdrive = mem_overdrive + if perf_level: + args.perf_level = perf_level + if replay_count: + args.replay_count = replay_count + if xgmi_err: + args.xgmi_err = xgmi_err + if energy: + args.energy = energy + if mem_usage: + args.mem_usage = mem_usage + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle watch logic, will only enter this block once + if args.watch: + args = self.helpers.handle_watch(args=args, subcommand=self.metric) + self.logger.print_output(watch_output=True) # Print at the end of watch ( final flush ) + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.metric(args, multiple_devices=True, watching_output=False, gpu=device_handle) + + self.logger.print_output(multiple_device_output=True) + # End of multiple gpus add to watch_output + if watching_output: + self.logger.store_watch_output(multiple_devices=True) + + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + # Check if any of the options have been set, if not then set them all to true + if not any([args.usage, args.fb_usage, args.power, args.clock, args.temperature, args.ecc, args.pcie, args.voltage, args.fan, + args.pcie_usage, args.voltage_curve, args.overdrive, args.mem_overdrive, args.perf_level, + args.replay_count, args.xgmi_err, args.energy, mem_usage]): + args.usage = args.fb_usage = args.power = args.clock = args.temperature = args.ecc = args.pcie = args.voltage = args.fan = \ + args.pcie_usage = args.voltage_curve = args.overdrive = args.mem_overdrive = args.perf_level = \ + args.replay_count = args.xgmi_err = args.energy = mem_usage = True + + # Add timestamp and store values for specified arguments + values_dict = {} + if args.usage: + engine_usage = amdsmi_interface.amdsmi_get_gpu_activity(args.gpu) + + if self.logger.is_gpuvsmi_compatibility(): + engine_usage['gfx_usage'] = engine_usage.pop('gfx_activity') + engine_usage['mem_usage'] = engine_usage.pop('umc_activity') + engine_usage['mm_usage_list'] = engine_usage.pop('mm_activity') + + if self.logger.is_human_readable_format(): + unit = '%' + for usage_name, usage_value in engine_usage.items(): + engine_usage[usage_name] = f"{usage_value} {unit}" + + values_dict['usage'] = engine_usage + if args.fb_usage: + vram_usage = amdsmi_interface.amdsmi_get_vram_usage(args.gpu) + + if self.logger.is_gpuvsmi_compatibility(): + vram_usage['fb_total'] = vram_usage.pop('vram_total') + vram_usage['fb_used'] = vram_usage.pop('vram_used') + + if self.logger.is_human_readable_format(): + unit = 'MB' + for vram_name, vram_value in vram_usage.items(): + vram_usage[vram_name] = f"{vram_value} {unit}" + + values_dict['fb_usage'] = vram_usage + if args.power: + average_socket_power = amdsmi_interface.amdsmi_get_power_measure(args.gpu)['average_socket_power'] + + if self.logger.is_gpuvsmi_compatibility(): + pass + + if self.logger.is_human_readable_format(): + unit = 'W' + average_socket_power = f"{average_socket_power} {unit}" + + values_dict['power'] = average_socket_power + if args.clock: + clock_gfx = amdsmi_interface.amdsmi_get_clock_measure(args.gpu, amdsmi_interface.AmdSmiClkType.GFX) + clock_mem = amdsmi_interface.amdsmi_get_clock_measure(args.gpu, amdsmi_interface.AmdSmiClkType.MEM) + + clocks = {'gfx': clock_gfx, + 'mem': clock_mem} + + if self.logger.is_human_readable_format(): + unit = 'MHz' + for clock_target, clock_metric_values in clocks.items(): + for clock_type, clock_value in clock_metric_values.items(): + clocks[clock_target][clock_type] = f"{clock_value} {unit}" + + values_dict['clock'] = clocks + if args.temperature: + temperature_edge_current = amdsmi_interface.amdsmi_dev_get_temp_metric( + args.gpu, amdsmi_interface.AmdSmiTemperatureType.EDGE, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) + temperature_junction_current = amdsmi_interface.amdsmi_dev_get_temp_metric( + args.gpu, amdsmi_interface.AmdSmiTemperatureType.JUNCTION, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) + temperature_vram_current = amdsmi_interface.amdsmi_dev_get_temp_metric( + args.gpu, amdsmi_interface.AmdSmiTemperatureType.VRAM, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) + + temperatures = { 'edge': temperature_edge_current, + 'hotspot': temperature_junction_current, + 'mem': temperature_vram_current} + + if self.logger.is_gpuvsmi_compatibility(): + temperatures = { 'edge_temperature': temperature_edge_current, + 'hotspot_temperature': temperature_junction_current, + 'mem_temperature': temperature_vram_current} + + if self.logger.is_human_readable_format(): + unit = '\N{DEGREE SIGN}C' + for temperature_value in temperatures: + temperatures[temperature_value] = f"{temperatures[temperature_value]} {unit}" + + values_dict['temperature'] = temperatures + if args.ecc: + try: + values_dict['ecc'] = amdsmi_interface.amdsmi_get_ecc_error_count(args.gpu) + except amdsmi_interface.AmdSmiLibraryException as err: + values_dict['ecc'] = err.get_error_info() + + if args.pcie: + pcie_link_status = amdsmi_interface.amdsmi_get_pcie_link_caps(args.gpu) + + if self.logger.is_human_readable_format(): + unit ='MT/s' + pcie_link_status['pcie_speed'] = f"{pcie_link_status['pcie_speed']} {unit}" + + if self.logger.is_gpuvsmi_compatibility(): + pcie_link_status['current_width'] = pcie_link_status.pop('pcie_lanes') + pcie_link_status['current_speed'] = pcie_link_status.pop('pcie_speed') + + values_dict['pcie'] = pcie_link_status + if args.voltage: + volt_metric = amdsmi_interface.amdsmi_dev_get_volt_metric( + args.gpu, amdsmi_interface.AmdSmiVoltageType.VDDGFX, amdsmi_interface.AmdSmiVoltageMetric.CURRENT) + + if self.logger.is_human_readable_format(): + unit = 'mV' + volt_metric = f"{volt_metric} {unit}" + + values_dict['voltage'] = volt_metric + if args.fan: + fan_speed = amdsmi_interface.amdsmi_dev_get_fan_speed(args.gpu, 0) + fan_max = amdsmi_interface.amdsmi_dev_get_fan_speed_max(args.gpu, 0) + fan_rpm = amdsmi_interface.amdsmi_dev_get_fan_rpms(args.gpu, 0) + + if fan_max <= 0: + fan_percent = 'Unable to detect fan speed' + else: + fan_percent = round((float(fan_speed) / float(fan_max)) * 100, 2) + + if self.logger.is_human_readable_format(): + unit = '%' + fan_percent = f"{fan_percent} {unit}" + + values_dict['fan'] = {'speed': fan_speed, + 'max' : fan_max, + 'rpm' : fan_rpm, + 'usage' : fan_percent} + if args.pcie_usage: + pcie_link_status = amdsmi_interface.amdsmi_get_pcie_link_status(args.gpu) + + if self.logger.is_human_readable_format(): + unit ='MT/s' + pcie_link_status['pcie_speed'] = f"{pcie_link_status['pcie_speed']} {unit}" + + values_dict['pcie_usage'] = pcie_link_status + if args.voltage_curve: + try: + od_volt = amdsmi_interface.amdsmi_dev_get_od_volt_info(args.gpu) + voltage_curve_error = False + except amdsmi_interface.AmdSmiLibraryException as err: + values_dict["voltage_curve"] = err.get_error_info() + voltage_curve_error = True + + if not voltage_curve_error: + voltage_point_dict = {} + + for point in range(3): + frequency = int(od_volt["curve.vc_points"][point].frequency / 1000000) + voltage = int(od_volt["curve.vc_points"][point].voltage) + voltage_point_dict[f'voltage_point_{point}'] = f"{frequency}Mhz {voltage}mV" + + values_dict['voltage_curve'] = voltage_point_dict + if args.overdrive: + overdrive_level = amdsmi_interface.amdsmi_dev_get_overdrive_level(args.gpu) + + if self.logger.is_human_readable_format(): + unit = '%' + overdrive_level = f"{overdrive_level} {unit}" + + values_dict['overdrive'] = overdrive_level + if args.mem_overdrive: + values_dict['mem_overdrive'] = amdsmi_interface.AmdSmiRetCode.NOT_IMPLEMENTED + if args.perf_level: + perf_level = amdsmi_interface.amdsmi_dev_get_perf_level(args.gpu) + values_dict['perf_level'] = perf_level + if args.replay_count: + pci_replay_counter = amdsmi_interface.amdsmi_dev_get_pci_replay_counter(args.gpu) + values_dict['replay_count'] = pci_replay_counter + if args.xgmi_err: + try: + values_dict['xgmi_err'] = amdsmi_interface.amdsmi_dev_xgmi_error_status(args.gpu) + except amdsmi_interface.AmdSmiLibraryException as err: + values_dict['xgmi_err'] = err.get_error_info() + if args.energy: + energy = amdsmi_interface.amdsmi_get_power_measure(args.gpu)['energy_accumulator'] + + if self.logger.is_human_readable_format(): + unit = 'J' + energy = f"{energy} {unit}" + + values_dict['energy'] = energy + if args.mem_usage: + memory_total_vram = amdsmi_interface.amdsmi_dev_get_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.VRAM) + memory_total_vis_vram = amdsmi_interface.amdsmi_dev_get_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.VIS_VRAM) + memory_total_gtt = amdsmi_interface.amdsmi_dev_get_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.GTT) + + memory_total = {} + # Convert mem_usage to megabytes + memory_total['vram'] = memory_total_vram // (1024*1024) + memory_total['vis_vram'] = memory_total_vis_vram // (1024*1024) + memory_total['gtt'] = memory_total_gtt // (1024*1024) + + if self.logger.is_human_readable_format(): + unit = 'MB' + energy = f"{energy} {unit}" + memory_total['vram'] = f"{memory_total['vram']} {unit}" + memory_total['vis_vram'] = f"{memory_total['vis_vram']} {unit}" + memory_total['gtt'] = f"{memory_total['gtt']} {unit}" + + + values_dict['mem_usage'] = memory_total + + # Store values in logger.output + self.logger.store_output(args.gpu, 'values', values_dict) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + if watching_output: # End of single gpu add to watch_output + self.logger.store_watch_output(multiple_devices=False) + + + def process(self, args, multiple_devices=False, watching_output=False, + gpu=None, general=None, engine=None, pid=None, name=None, + watch=None, watch_time=None, iterations=None): + """Get Process Information from the target GPU + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + watching_output (bool, optional): True if watch option has been set. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + general (bool, optional): Value override for args.general. Defaults to None. + engine (bool, optional): Value override for args.engine. Defaults to None. + pid (Positive int, optional): Value override for args.pid. Defaults to None. + name (str, optional): Value override for args.name. Defaults to None. + watch (Positive int, optional): Value override for args.watch. Defaults to None. + watch_time (Positive int, optional): Value override for args.watch_time. Defaults to None. + iterations (Positive int, optional): Value override for args.iterations. Defaults to None. + + Raises: + IndexError: Index error if gpu list is empty + + Returns: + None: Print output via AMDSMILogger to destination + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + if general: + args.general = general + if engine: + args.engine = engine + if pid: + args.pid = pid + if name: + args.name = name + if watch: + args.watch = watch + if watch_time: + args.watch_time = watch_time + if iterations: + args.iterations = iterations + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle watch logic, will only enter this block once + if args.watch: + args = self.helpers.handle_watch(args=args, subcommand=self.process) + self.logger.print_output(watch_output=True) # Print at the end of watch ( final flush ) + return + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.process(args, multiple_devices=True, watching_output=False, gpu=device_handle) + + self.logger.print_output(multiple_device_output=True) + # End of multiple gpus add to watch_output + if watching_output: + self.logger.store_watch_output(multiple_devices=True) + + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + # Populate initial processes + process_list = amdsmi_interface.amdsmi_get_process_list(args.gpu) + filtered_process_values = [] + for process_handle in process_list: + process_info = amdsmi_interface.amdsmi_get_process_info(args.gpu, process_handle) + process_info['mem_usage'] = process_info.pop('mem') + process_info['usage'] = process_info.pop('engine_usage') + + # Convert mem_usage to megabytes + + mem_usage_mb = (process_info['mem_usage']//1024) // 1024 + if mem_usage_mb < 0: + process_info['mem_usage'] = (process_info['mem_usage']//1024) + mem_usage_unit = 'B' + else: + process_info['mem_usage'] = mem_usage_mb + + if self.logger.is_human_readable_format(): + mem_usage_unit = 'MB' + engine_usage_unit = '%' + process_info['mem_usage'] = f"{process_info['mem_usage']} {mem_usage_unit}" + + for usage_metric in process_info['usage']: + process_info['usage'][usage_metric] = f"{process_info['usage'][usage_metric]} {engine_usage_unit}" + for usage_metric in process_info['memory_usage']: + process_info['memory_usage'][usage_metric] = f"{process_info['memory_usage'][usage_metric]} {engine_usage_unit}" + + filtered_process_values.append({'process_info': process_info}) + + # Arguments will filter the populated processes + # General and Engine to expose process_info values + if args.general or args.engine: + for process_info in filtered_process_values: + if args.general and args.engine: + del process_info['process_info']['memory_usage'] + elif args.general: + del process_info['process_info']['memory_usage'] + del process_info['process_info']['usage'] # Used in engine + elif args.engine: + del process_info['process_info']['memory_usage'] + del process_info['process_info']['mem_usage'] # Used in general + + # Filter out non specified pids + if args.pid: + process_pids = [] + for process_info in filtered_process_values: + pid = str(process_info['process_info']['pid']) + if str(args.pid) == pid: + process_pids.append(process_info) + filtered_process_values = process_pids + + # Filter out non specified process names + if args.name: + process_names = [] + for process_info in filtered_process_values: + process_name = str(process_info['process_info']['name']).lower() + if str(args.name).lower() == process_name: + process_names.append(process_info) + filtered_process_values = process_names + + # Remove brackets if there is only one value + if len(filtered_process_values) == 1: + filtered_process_values = filtered_process_values[0] + + # Store values in logger.output + if filtered_process_values == []: + self.logger.store_output(args.gpu, 'values', {'process_info': 'Not Found'}) + else: + self.logger.store_output(args.gpu, 'values', filtered_process_values) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + if watching_output: # End of single gpu add to watch_output + self.logger.store_watch_output(multiple_devices=False) + + + def profile(self, args): + """Not applicable to linux baremetal""" + print('Profile test') + + + def event(self, args): + print('event test') + + + def topology(self, args, multiple_devices=False, gpu=None, topo_access=None, + topo_weight=None, topo_hops=None, topo_type=None, topo_numa=None): + """ Get topology information for target gpus + The compatibility mode for this will only be in amdsmi & rocm-smi + params: + args - argparser args to pass to subcommand + multiple_devices (bool) - True if checking for multiple devices + gpu (device_handle) - device_handle for target device + topo_access (bool) - Value override for args.topo_access + topo_weight (bool) - Value override for args.topo_weight + topo_hops (bool) - Value override for args.topo_hops + topo_type (bool) - Value override for args.topo_type + topo_numa (bool) - Value override for args.topo_numa + return: + Nothing + """ + # Set args.* to passed in arguments + if gpu: + args.gpu = gpu + if topo_access: args.topo_access = topo_access + if topo_weight: args.topo_weight = topo_weight + if topo_hops: args.topo_hops = topo_hops + if topo_type: args.topo_type = topo_type + if topo_numa: args.topo_numa = topo_numa + + # Handle No GPU passed + if args.gpu is None: + args.gpu = self.device_handles + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + pass + if len(args.gpu) == 1: + # args.gpu = args.gpu[0] + pass + else: + raise IndexError("args.gpu should not be an empty list") + + # Handle all args being false + + # If all arguments are False, it means that no argument was passed and the entire topology should be printed + # if not any([args.asic, args.bus, args.vbios, args.limit, args.driver, args.caps, args.ras, args.board]): + # args.asic = args.bus = args.vbios = args.limit = args.driver = args.caps = args.ras = args.board = True + + topo_json = {} + topo_table = [] + + if args.topo_access: + pass + if args.topo_weight: + pass + if args.topo_hops: + pass + if args.topo_type: + pass + if args.topo_numa: + pass + + + def set_value(self, args, multiple_devices=False, gpu=None, clk=None, sclk=None, mclk=None, + pcie=None, slevel=None, mlevel=None, vc=None, srange=None, mrange=None, + fan=None, perflevel=None, overdrive=None, memoverdrive=None, + poweroverdrive=None, profile=None, perfdet=None, rasenable=None, + rasdisable=None): + pass + + + def reset(self, args, multiple_devices=False, gpu=None, gpureset=None, + resetclocks=None, resetfans=None, resetprofile=None, + resetpoweroverdrive=None, resetxgmierr=None, resetperfdet=None): + """Issue reset commands to target gpu(s) + + Args: + args (Namespace): Namespace containing the parsed CLI args + multiple_devices (bool, optional): True if checking for multiple devices. Defaults to False. + gpu (device_handle, optional): device_handle for target device. Defaults to None. + gpureset (bool, optional): Value over ride for args.gpureset. Defaults to None. + resetclocks (bool, optional): Value over ride for args.resetclocks. Defaults to None. + resetfans (bool, optional): Value over ride for args.resetfans. Defaults to None. + resetprofile (bool, optional): Value over ride for args.resetprofile. Defaults to None. + resetpoweroverdrive (bool, optional): Value over ride for args.resetpoweroverdrive. Defaults to None. + resetxgmierr (bool, optional): Value over ride for args.resetxgmierr. Defaults to None. + resetperfdet (bool, optional): Value over ride for args.resetperfdet. Defaults to None. + + Raises: + ValueError: Value error if no gpu value is provided + IndexError: Index error if gpu list is empty + + Return: + Nothing + """ + # Set args.* to passed in arguments + if gpu: args.gpu = gpu + if gpureset: args.gpureset = gpureset + if resetclocks: args.resetclocks = resetclocks + if resetfans: args.resetfans = resetfans + if resetprofile: args.resetprofile = resetprofile + if resetpoweroverdrive: args.resetpoweroverdrive = resetpoweroverdrive + if resetxgmierr: args.resetxgmierr = resetxgmierr + if resetperfdet: args.resetperfdet = resetperfdet + + # Handle No GPU passed + if args.gpu is None: + raise ValueError('No GPU provided, specific GPU target(s) are needed') + + # Handle multiple GPUs + if isinstance(args.gpu, list): + if len(args.gpu) > 1: + for device_handle in args.gpu: + # Handle multiple_devices to print all output at once + self.bad_pages(args, multiple_devices=True, gpu=device_handle) + self.logger.print_output(multiple_device_output=True) + return + if len(args.gpu) == 1: + args.gpu = args.gpu[0] + else: + raise IndexError("args.gpu should not be an empty list") + + if args.gpureset: + if self.helpers.is_amd_device(args.gpu): + try: + amdsmi_interface.amdsmi_dev_reset_gpu(args.gpu) + result = 'Successfully reset GPU' + except amdsmi_interface.AmdSmiLibraryException as err: + result = err.get_error_info() + else: + result = 'Unable to reset non-amd GPU' + + self.logger.store_output(args.gpu, 'gpu_reset', result) + if args.resetclocks: + # rsmi_string = ' Reset Clocks ' + reset_clocks_results = {'overdrive' : '', + 'clocks' : '', + 'performance': ''} + try: + amdsmi_interface.amdsmi_dev_set_overdrive_level_v1(args.gpu, 0) + reset_clocks_results['overdrive'] = 'Overdrive set to 0' + except amdsmi_interface.AmdSmiLibraryException as err: + reset_clocks_results['overdrive'] = err.get_error_info() + + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_dev_set_perf_level_v1(args.gpu, level_auto) + reset_clocks_results['clocks'] = 'Successfully reset clocks' + except amdsmi_interface.AmdSmiLibraryException as err: + reset_clocks_results['clocks'] = err.get_error_info() + + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_dev_set_perf_level_v1(args.gpu, level_auto) + reset_clocks_results['performance'] = 'Performance level reset to auto' + except amdsmi_interface.AmdSmiLibraryException as err: + reset_clocks_results['performance'] = err.get_error_info() + + self.logger.store_output(args.gpu, 'reset_clocks', reset_clocks_results) + if args.resetfans: + try: + amdsmi_interface.amdsmi_dev_reset_fan(args.gpu, 0) + result = 'Successfully reset fan speed to driver control' + except amdsmi_interface.AmdSmiLibraryException as err: + result = err.get_error_info() + + self.logger.store_output(args.gpu, 'reset_fans', result) + if args.resetprofile: + reset_profile_results = {'power_profile' : '', + 'performance_level': ''} + try: + power_profile_mask = amdsmi_interface.AmdSmiPowerProfilePresetMasks.BOOTUP_DEFAULT + amdsmi_interface.amdsmi_dev_set_power_profile(args.gpu, 0, power_profile_mask) + result = 'Successfully reset Power Profile' + except amdsmi_interface.AmdSmiLibraryException as err: + result = err.get_error_info() + + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_dev_set_perf_level_v1(args.gpu, level_auto) + reset_clocks_results['performance'] = 'Successfully reset Performance Level' + except amdsmi_interface.AmdSmiLibraryException as err: + reset_clocks_results['performance'] = err.get_error_info() + + self.logger.store_output(args.gpu, 'reset_profile', reset_profile_results) + if args.resetxgmierr: + try: + amdsmi_interface.amdsmi_dev_reset_xgmi_error(args.gpu) + result = 'Successfully reset XGMI Error count' + except amdsmi_interface.AmdSmiLibraryException as err: + result = err.get_error_info() + self.logger.store_output(args.gpu, 'reset_xgmi_err', result) + if args.resetprefdet: + try: + level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO + amdsmi_interface.amdsmi_dev_set_perf_level_v1(args.gpu, level_auto) + result = 'Successfully disabled performance determinism' + except amdsmi_interface.AmdSmiLibraryException as err: + result = err.get_error_info() + + self.logger.store_output(args.gpu, 'reset_pref_det', result) + + if multiple_devices: + self.logger.store_multiple_device_output() + return # Skip printing when there are multiple devices + + self.logger.print_output() + + + def rocm_smi(self, args): + print("rocmsmi test") diff --git a/amdsmi_cli/amdsmi_helpers.py b/amdsmi_cli/amdsmi_helpers.py new file mode 100644 index 0000000000..0821f7a19a --- /dev/null +++ b/amdsmi_cli/amdsmi_helpers.py @@ -0,0 +1,280 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import logging +import platform +import time + +from pathlib import Path + +from amdsmi_init import * +from BDF import BDF + + +class AMDSMIHelpers(): + """Helper functions that aren't apart of the AMDSMI API + Useful for determining platform and device identifiers + + Functions: + os_info: tuple () + """ + + def __init__(self) -> None: + self.operating_system = platform.system() + + self._is_hypervisor = False + self._is_virtual_os = False + self._is_baremetal = False + + self._is_linux = False + self._is_windows = False + + if self.operating_system.startswith("Linux"): + self._is_linux = True + logging.debug(f"AMDSMIHelpers: Platform is linux:{self._is_linux}") + + product_name = "" + product_name_path = Path("/sys/class/dmi/id/product_name") + if product_name_path.exists(): + product_name = product_name_path.read_text().strip() + + if product_name == "": + # Unable to determine product_name default to baremetal + self._is_baremetal = True + + # Determine if a system is baremetal by deduction + self._is_baremetal = not self._is_hypervisor and not self._is_virtual_os + + + def os_info(self, string_format=True): + """Return operating_system and type information ex. (Linux, Baremetal) + params: + string_format (bool) True to return in string format, False to return Tuple + returns: + str or (str, str) + """ + operating_system = "" + if self._is_linux: + operating_system = "Linux" + elif self._is_windows: + operating_system = "Windows" + else: + operating_system = "Unknown" + + operating_system_type = "" + if self._is_baremetal: + operating_system_type = "Baremetal" + elif self._is_virtual_os: + operating_system_type = "Guest" + elif self._is_hypervisor: + operating_system_type = "Hypervisor" + else: + operating_system_type = "Unknown" + + if string_format: + return f"{operating_system} {operating_system_type}" + else: + return (operating_system, operating_system_type) + + + def is_virtual_os(self): + return self._is_virtual_os + + + def is_hypervisor(self): + # Returns True if hypervisor is enabled on the system + return self._is_hypervisor + + + def is_baremetal(self): + # Returns True if system is baremetal, if system is hypervisor this should return False + return self._is_baremetal + + + def is_linux(self): + return self._is_linux + + + def is_windows(self): + return self._is_windows + + + def get_gpu_choices(self): + """Return dictionary of possible GPU choices and string of the output: + Dictionary will be in format: gpus[ID] : (BDF, UUID, Device Handle) + String output will be in format: + "ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-0000-1000-0000-000000000000" + params: + None + return: + (dict, str) : (gpu_choices, gpu_choices_str) + """ + gpu_choices = {} + gpu_choices_str = "" + + # amdsmi_get_device_handles returns the device_handles storted by gpu_id + device_handles = amdsmi_interface.amdsmi_get_device_handles() + for gpu_id, device_handle in enumerate(device_handles): + bdf = amdsmi_interface.amdsmi_get_device_bdf(device_handle) + uuid = amdsmi_interface.amdsmi_get_device_uuid(device_handle) + gpu_choices[str(gpu_id)] = { + "BDF": bdf, + "UUID": uuid, + "Device Handle": device_handle, + } + gpu_choices_str += f"ID:{gpu_id:<2} | BDF:{bdf} | UUID:{uuid}" + + return (gpu_choices, gpu_choices_str) + + + def get_device_handles_from_gpu_selections(self, gpu_selections, gpu_choices=None): + """Convert provided gpu_selections to device_handles + + Args: + gpu_selections (list[str]): This will be the GPU ID, BDF, or UUID: + ex: ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-0000-1000-0000-000000000000 + gpu_choices (dict{gpu_choices}): This is a dictionary of the possible gpu_choices + Returns: + (True, list[device_handles]): Returns a list of all the gpu_selections converted to + amdsmi device_handles + (False, str): Return False, and the first input that failed to be converted + """ + if isinstance(gpu_selections, str): + gpu_selections = [gpu_selections] + + if gpu_choices is None: + gpu_choices = self.get_gpu_choices()[0] + + selected_device_handles = [] + for gpu_selection in gpu_selections: + valid_gpu_choice = False + + for gpu_id, gpu_info in gpu_choices.items(): + bdf = gpu_info['BDF'] + uuid = gpu_info['UUID'] + device_handle = gpu_info['Device Handle'] + + # Check if passed gpu is a gpu ID or UUID + if gpu_selection == gpu_id or gpu_selection.lower() == uuid: + selected_device_handles.append(device_handle) + valid_gpu_choice = True + break + else: # Check if gpu passed is a BDF object + try: + if BDF(gpu_selection) == BDF(bdf): + selected_device_handles.append(device_handle) + valid_gpu_choice = True + break + except Exception: + # Ignore exception when checking if the gpu_choice is a BDF + pass + + if not valid_gpu_choice: + logging.debug(f"AMDSMIHelpers.get_device_handles_from_gpu_selections - Unable to convert {gpu_selection}") + return False, gpu_selection + return True, selected_device_handles + + + def handle_watch(self, args, subcommand): + """This function will run the subcommand multiple times based + on the passed watch, watch_time, and iterations passed in. + params: + args - argparser args to pass to subcommand + subcommand (AMDSMICommands) - Function that can handle + watching output (Currently: metric & process) + return: + Nothing + """ + # Set the values for watching as the args will cleared + watch = args.watch + watch_time = args.watch_time + iterations = args.iterations + + # Set the args values to None so we don't loop recursively + args.watch = None + args.watch_time = None + args.iterations = None + + if watch_time: # Run for set amount of time + iterations_ran = 0 + end_time = time.time() + watch_time + while time.time() <= end_time: + subcommand(args, watching_output=True) + # Handle iterations limit + iterations_ran += 1 + if iterations: + if iterations >= iterations_ran: + break + time.sleep(watch) + elif iterations: # Run for a set amount of iterations + for iteration in range(iterations): + subcommand(args, watching_output=True) + if iteration == iterations - 1: # Break on iteration completion + break + time.sleep(watch) + else: # Run indefinitely as watch_time and iterations are not set + while True: + subcommand(args, watching_output=True) + time.sleep(watch) + + return 1 + + + def get_gpu_id_from_device_handle(self, input_device_handle): + """Get the gpu index from the device_handle. + amdsmi_get_device_handles() returns the list of device_handles in order of gpu_index + """ + device_handles = amdsmi_interface.amdsmi_get_device_handles() + for gpu_index, device_handle in enumerate(device_handles): + if input_device_handle.value == device_handle.value: + return gpu_index + raise IndexError("Unable to find gpu ID from device_handle") + + + def get_amd_gpu_bdfs(self): + """Return a list of GPU BDFs visibile to amdsmi + + Returns: + list[BDF]: List of GPU BDFs + """ + gpu_bdfs = [] + device_handles = amdsmi_interface.amdsmi_get_device_handles() + + for device_handle in device_handles: + bdf = amdsmi_interface.amdsmi_get_device_bdf(device_handle) + gpu_bdfs.append(bdf) + + return gpu_bdfs + + + # def get_amd_cpu_bdfs(self): + # pass + + + def is_amd_device(self, device_handle): + """ Return whether the specified device is an AMD device or not + + param device: DRM device identifier + """ + # Get card vendor id + asic_info = amdsmi_interface.amdsmi_get_asic_info(device_handle) + return asic_info['vendor_id'] == AMD_VENDOR_ID diff --git a/amdsmi_cli/amdsmi_init.py b/amdsmi_cli/amdsmi_init.py new file mode 100644 index 0000000000..0c1c941e51 --- /dev/null +++ b/amdsmi_cli/amdsmi_init.py @@ -0,0 +1,95 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +### Handle safe initialization for amdsmi + +import atexit +import logging +import signal +import sys + +from pathlib import Path + +sys.path.insert( 0, "../build/python_package/amdsmi") +import amdsmi_interface + +# Using basic python logging for user errors and development +logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.ERROR) # User level logging + +# On initial import set initialized variable +AMDSMI_INITIALIZED = False +AMD_VENDOR_ID = 4098 + +def check_amdgpu_driver(): + """ Returns true if amdgpu is found in the list of initialized modules """ + amd_gpu_status_file = Path("/sys/module/amdgpu/initstate") + + if amd_gpu_status_file.exists(): + if amd_gpu_status_file.read_text(encoding='ascii').strip() == 'live': + return True + + return False + + +def init_amdsmi(flag=amdsmi_interface.AmdSmiInitFlags.AMD_GPUS): + """ Initializes AMDSMI + + Raises: + err: AmdSmiLibraryException if not successful + """ + # Check if amdgpu driver is up & Handle error gracefully + if check_amdgpu_driver(): + # Only init AMD GPUs for now, waiting for future support for AMD CPUs + try: + amdsmi_interface.amdsmi_init(flag) + except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as err: + raise err + + logging.info('AMDSMI initialized successfully') # without errors really + else: + logging.error('Driver not initialized (amdgpu not found in modules)') + exit(-1) + + +def shut_down_amdsmi(): + """Shutdown AMDSMI instance + + Raises: + err: AmdSmiLibraryException if not successful + """ + try: + amdsmi_interface.amdsmi_shut_down() + except amdsmi_interface.AmdSmiLibraryException as err: + raise err + + +def signal_handler(sig, frame): + logging.debug(f'Handling signal: {sig}') + sys.exit(0) + + +if not AMDSMI_INITIALIZED: + init_amdsmi() + AMDSMI_INITIALIZED = True + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) + atexit.register(shut_down_amdsmi) diff --git a/amdsmi_cli/amdsmi_logger.py b/amdsmi_cli/amdsmi_logger.py new file mode 100644 index 0000000000..ec95b8feaa --- /dev/null +++ b/amdsmi_cli/amdsmi_logger.py @@ -0,0 +1,384 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +import json +import time +import yaml +import re +from enum import Enum + +from amdsmi_helpers import AMDSMIHelpers + +class AMDSMILogger(): + def __init__(self, compatibility='amdsmi', format='human_readable', + destination='stdout') -> None: + self.output = {} + self.multiple_device_output = [] + self.watch_output = [] + self.compatibility = compatibility # amdsmi, gpuvsmi, or rocmsmi + self.format = format # csv, json, or human_readable + self.destination = destination # stdout, path to a file (append) + self.amd_smi_helpers = AMDSMIHelpers() + + + class LoggerFormat(Enum): + """Enum for logger formats""" + json = 'json' + csv = 'csv' + human_readable = 'human_readable' + + class Loggercompatibility(Enum): + """Enum for logger compatibility""" + amdsmi = 'amdsmi' + rocmsmi = 'rocmsmi' + gpuvsmi = 'gpuvsmi' + + + def is_json_format(self): + return self.format == self.LoggerFormat.json.value + + + def is_csv_format(self): + return self.format == self.LoggerFormat.csv.value + + + def is_human_readable_format(self): + return self.format == self.LoggerFormat.human_readable.value + + + def is_amdsmi_compatibility(self): + return self.compatibility == self.Loggercompatibility.amdsmi.value + + + def is_rocmsmi_compatibility(self): + return self.compatibility == self.Loggercompatibility.rocmsmi.value + + + def is_gpuvsmi_compatibility(self): + return self.compatibility == self.Loggercompatibility.gpuvsmi.value + + + def store_output(self, device_handle, argument, data): + """ Store the argument and device handle according to the compatibility. + Each compatibility function will handle the output format and + populate the output + params: + device_handle - device handle object to the target device output + argument (str) - key to store data + data (dict | list) - Data store against argument + return: + Nothing + """ + gpu_id = self.amd_smi_helpers.get_gpu_id_from_device_handle(device_handle) + if self.is_amdsmi_compatibility(): + self._store_output_amdsmi(gpu_id=gpu_id, argument=argument, data=data) + elif self.is_rocmsmi_compatibility(): + self._store_output_rocmsmi(gpu_id=gpu_id, argument=argument, data=data) + elif self.is_gpuvsmi_compatibility(): + self._store_output_gpuvsmi(gpu_id=gpu_id, argument=argument, data=data) + + + def _store_output_amdsmi(self, gpu_id, argument, data): + if self.is_json_format() or self.is_human_readable_format(): + self.output['gpu'] = int(gpu_id) + if argument == 'values' and isinstance(data, dict): + self.output.update(data) + else: + self.output[argument] = data + + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def _store_output_rocmsmi(self, gpu_id, argument, data): + if self.is_json_format(): + # put output into self.json_output + pass + elif self.is_csv_format(): + # put output into self.csv_output + pass + elif self.is_human_readable_format(): + # put output into self.human_readable_output + pass + else: + raise "err" + + + def _store_output_gpuvsmi(self, gpu_id, argument, data): + if self.is_json_format() or self.is_human_readable_format(): + self.output['gpu'] = int(gpu_id) + self.output[argument] = data + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def store_multiple_device_output(self): + """ Store the current output into the multiple_device_output + then clear the current output + params: + None + return: + Nothing + """ + if self.is_amdsmi_compatibility(): + self._store_multiple_device_output_amdsmi() + elif self.is_rocmsmi_compatibility(): + self._store_multiple_device_output_rocmsmi() + elif self.is_gpuvsmi_compatibility(): + self._store_multiple_device_output_gpuvsmi() + + + def _store_multiple_device_output_amdsmi(self): + if self.is_json_format() or self.is_human_readable_format(): + self.multiple_device_output.append(self.output) + self.output = {} + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def _store_multiple_device_output_rocmsmi(self): + if self.is_json_format(): + # put output into self.json_output + pass + elif self.is_csv_format(): + # put output into self.csv_output + pass + elif self.is_human_readable_format(): + # put output into self.human_readable_output + pass + else: + raise "err" + + + def _store_multiple_device_output_gpuvsmi(self): + if self.is_json_format() or self.is_human_readable_format(): + self.multiple_device_output.append(self.output) + self.output = {} + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def store_watch_output(self, multiple_devices=False): + """ Add the current output or multiple_devices_output + params: + multiple_devices (bool) - True if watching multiple devices + return: + Nothing + """ + if self.is_amdsmi_compatibility(): + self._store_watch_output_amdsmi(multiple_devices=multiple_devices) + elif self.is_rocmsmi_compatibility(): + self._store_watch_output_rocmsmi(multiple_devices=multiple_devices) + elif self.is_gpuvsmi_compatibility(): + self._store_watch_output_gpuvsmi(multiple_devices=multiple_devices) + + + def _store_watch_output_amdsmi(self, multiple_devices): + if self.is_json_format() or self.is_human_readable_format(): + values = self.output + if multiple_devices: + values = self.multiple_device_output + + self.watch_output.append({'timestamp': int(time.time()), + 'values': values}) + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def _store_watch_output_rocmsmi(self, multiple_devices): + if self.is_json_format(): + # put output into self.json_output + pass + elif self.is_csv_format(): + # put output into self.csv_output + pass + elif self.is_human_readable_format(): + # put output into self.human_readable_output + pass + else: + raise "err" + + + def _store_watch_output_gpuvsmi(self, multiple_devices): + if self.is_json_format() or self.is_human_readable_format(): + values = self.output + if multiple_devices: + values = self.multiple_device_output + + self.watch_output.append({'timestamp': int(time.time()), + 'values': values}) + elif self.is_csv_format(): + # put output into self.csv_output + pass + else: + raise "err" + + + def capitalize_keys(self, input_dict): + output_dict = {} + for key in input_dict.keys(): + # Capitalize key if it is a string + if isinstance(key, str): + cap_key = key.upper() + else: + cap_key = key + + if isinstance(input_dict[key], dict): + output_dict[cap_key] = self.capitalize_keys(input_dict[key]) + elif isinstance(input_dict[key], list): + cap_key_list = [] + for data in input_dict[key]: + if isinstance(data, dict): + cap_key_list.append(self.capitalize_keys(data)) + else: + cap_key_list.append(data) + output_dict[cap_key] = cap_key_list + else: + output_dict[cap_key] = input_dict[key] + + return output_dict + + + def convert_json_to_human_readable(self, json_object): + # First Capitalize all keys in the json object + capitalized_json = self.capitalize_keys(json_object) + json_string = json.dumps(capitalized_json, indent=4) + yaml_data = yaml.safe_load(json_string) + yaml_output = yaml.dump(yaml_data, sort_keys=False, allow_unicode=True) + + if self.is_gpuvsmi_compatibility(): + # Convert from GPU: 0 to GPU 0: + yaml_output = re.sub('GPU: ([0-9]+)', 'GPU \\1:', yaml_output) + + # Remove a key line if it is a spacer + yaml_output = yaml_output.replace("AMDSMI_SPACING_REMOVAL:\n", "") + yaml_output = yaml_output.replace("'", "") # Remove '' + + clean_yaml_output = '' + for line in yaml_output.splitlines(): + line = line.split(':') + + # Remove dashes and increase tabbing split key + line[0] = line[0].replace("-", " ", 1) + line[0] = line[0].replace(" ", " ") + + # Join cleaned output + line = ':'.join(line) + '\n' + clean_yaml_output += line + + return clean_yaml_output + + + def print_output(self, multiple_device_output=False, watch_output=False): + """ Print current output acording to format and then destination + params: + multiple_device_output (bool) - True if printing output from + multiple devices + watch_output (bool) - True if printing watch output + return: + Nothing + """ + if self.is_json_format(): + self._print_json_output(multiple_device_output=multiple_device_output, + watch_output=watch_output) + elif self.is_csv_format(): + self._print_csv_output(multiple_device_output=multiple_device_output, + watch_output=watch_output) + elif self.is_human_readable_format(): + self._print_human_readable_output(multiple_device_output=multiple_device_output, + watch_output=watch_output) + + + def _print_json_output(self, multiple_device_output=False, watch_output=False): + json_output = json.dumps(self.output, indent = 4) + json_multiple_device_output = json.dumps(self.multiple_device_output, indent = 4) + if self.destination == 'stdout': + if watch_output: + return # We don't need to print to stdout at the end of watch + elif multiple_device_output: + print(json_multiple_device_output) + else: + print(json_output) + else: # Write output to file + if watch_output: + with self.destination.open('w') as output_file: + json.dump(self.watch_output, output_file, indent=4) + elif multiple_device_output: + with self.destination.open('a') as output_file: + json.dump(self.multiple_device_output, output_file, indent=4) + else: + with self.destination.open('a') as output_file: + json.dump(self.output, output_file, indent=4) + + + def _print_csv_output(self, multiple_device_output=False, watch_output=False): + if self.destination == 'stdout': + if watch_output: + return # We don't need to print to stdout at the end of watch + elif multiple_device_output: + pass + else: + pass + else: # Write output to file + if watch_output: + pass + elif multiple_device_output: + pass + else: + pass + + + def _print_human_readable_output(self, multiple_device_output=False, watch_output=False): + if multiple_device_output: + human_readable = '' + for output in self.multiple_device_output: + human_readable += (self.convert_json_to_human_readable(output)) + else: + human_readable = self.convert_json_to_human_readable(self.output) + + if self.destination == 'stdout': + if watch_output: + return + # print_output may need another value: flush_output vs watch_output + print(human_readable) + else: + if watch_output: + return + with self.destination.open('a') as output_file: + output_file.write(human_readable) + + return diff --git a/amdsmi_cli/amdsmi_parser.py b/amdsmi_cli/amdsmi_parser.py new file mode 100644 index 0000000000..03885d903e --- /dev/null +++ b/amdsmi_cli/amdsmi_parser.py @@ -0,0 +1,724 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + + +import argparse +import errno +import os +import time +from pathlib import Path + +from _version import __version__ +from amdsmi_helpers import AMDSMIHelpers + + +class AMDSMIParser(argparse.ArgumentParser): + def __init__(self, version, discovery, static, firmware, bad_pages, metric, + process, profile, event, topology, set_value, reset, rocmsmi): + + # Helper variables + self.amd_smi_helpers = AMDSMIHelpers() + self.gpu_choices, self.gpu_choices_str = self.amd_smi_helpers.get_gpu_choices() + self.vf_choices = ['3', '2', '1'] + + version_string = f"Version: {__version__}" + platform_string = f"Platform: {self.amd_smi_helpers.os_info()}" + + # Adjust argument parser options + super().__init__( + formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, + max_help_position=80, + width=90), + description=f"AMD System Management Interface | {version_string} | {platform_string}", + add_help=True, + prog="amdsmi_cli") + + # Setup subparsers + subparsers = self.add_subparsers( + title="AMD-SMI Commands", + parser_class=argparse.ArgumentParser, + required=True, + help="Descriptions:", + # dest='cmd', + metavar="") + + # Add all subparsers + self._add_version_parser(subparsers, version) + self._add_discovery_parser(subparsers, discovery) + self._add_static_parser(subparsers, static) + self._add_firmware_parser(subparsers, firmware) + self._add_bad_pages_parser(subparsers, bad_pages) + self._add_metric_parser(subparsers, metric) + self._add_process_parser(subparsers, process) + self._add_profile_parser(subparsers, profile) + self._add_event_parser(subparsers, event) + self._add_topology_parser(subparsers, topology) + self._add_set_value_parser(subparsers, set_value) + self._add_reset_parser(subparsers, reset) + self._add_rocm_smi_parser(subparsers, rocmsmi) + + + def _positive_int(self, int_value): + # Argument type validator + if int_value.isdigit(): # Is digit works only on positive numbers + return int(int_value) + else: + raise argparse.ArgumentTypeError( + f"invalid input:{int_value} integer provided must be positive") + + + def _check_output_file_path(self): + """ Argument action validator: + Returns a path to a file from the output file path provided. + If the path is a directory then create a file within it and return that file path + If the path is a file and it exists return the file path + If the path is a file and it doesn't exist create and return the file path + """ + class CheckOutputFilePath(argparse.Action): + # Checks the values + def __call__(self, parser, args, values, option_string=None): + path = Path(values) + if not path.exists(): + if path.parent.is_dir(): + path.touch() + else: + raise argparse.ArgumentTypeError( + f"Invalid path:{path} Could not find parent directory of given path") + + if path.is_dir(): + path = path / f"{int(time.time())}-amdsmi-output.txt" + path.touch() + setattr(args, self.dest, path) + elif path.is_file(): + setattr(args, self.dest, path) + else: + raise argparse.ArgumentTypeError( + f"Invalid path:{path} Could not determine if value given is a valid path") + return CheckOutputFilePath + + + def _check_input_file_path(self): + """ Argument action validator: + Returns a path to a file from the input file path provided. + If the file doesn't exist or is empty raise error + """ + class _CheckInputFilePath(argparse.Action): + # Checks the values + def __call__(self, parser, args, values, option_string=None): + path = Path(values) + if not path.exists(): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), values) + + if path.is_dir(): + raise argparse.ArgumentTypeError( + f"Invalid Path: {path} is directory when it needs to be a specific file") + + if path.is_file(): + if os.stat(values).st_size == 0: + raise argparse.ArgumentTypeError( + f"Invalid Path: {path} Input file is empty") + setattr(args, self.dest, path) + else: + raise argparse.ArgumentTypeError( + f"Invalid path:{path} Could not determine if value given is a valid path") + return _CheckInputFilePath + + + def _check_watch_selected(self): + """ Argument action validator: + Validate that the -w/--watch argument was selected + This is because -W/--watch_time and -i/--iterations are dependent on watch + """ + class _WatchSelectedAction(argparse.Action): + # Checks the values + def __call__(self, parser, args, values, option_string=None): + if args.watch is None: + raise argparse.ArgumentError(self, f"Invalid argument: '{self.dest}' needs to be paired with -w/--watch") + setattr(args, self.dest, values) + return _WatchSelectedAction + + + def _gpu_select(self, gpu_choices): + """ Argument action validator: + Custom argparse action to return the device handle(s) for the gpu(s) selected + This will set the destination (args.gpu) to a list of 1 or more device handles + If 1 or more device handles are not found then raise an ArgumentError for the first invalid gpu seen + """ + amd_smi_helpers = self.amd_smi_helpers + class _GPUSelectAction(argparse.Action): + # Checks the values + def __call__(self, parser, args, values, option_string=None): + status, selected_device_handles = amd_smi_helpers.get_device_handles_from_gpu_selections(gpu_selections=values, + gpu_choices=gpu_choices) + if status: + setattr(args, self.dest, selected_device_handles) + else: + invalid_selection = selected_device_handles + raise argparse.ArgumentError(self, f"invalid choice: '{invalid_selection}' (see available choices with -h)") + return _GPUSelectAction + + + def _add_command_modifiers(self, subcommand_parser, rocm_smi=True, gpuv_smi=True): + json_help = "Displays output in JSON format (human readable by default)." + csv_help = "Displays output in CSV format (human readable by default)." + file_help = "Saves output into a file on the provided path (stdout by default)." + loglevel_help = "Set the logging level for the parser commands" + compatibility_help = "Display output in gpuvsmi or rocmsmi legacy format if possible" + + command_modifier_group = subcommand_parser.add_argument_group('Command Modifiers') + + # Output Format options + logging_args = command_modifier_group.add_mutually_exclusive_group() + logging_args.add_argument('--json', action='store_true', required=False, help=json_help) + logging_args.add_argument('--csv', action='store_true', required=False, help=csv_help) + + command_modifier_group.add_argument('--file', action=self._check_output_file_path(), type=str, required=False, help=file_help) + # Placing loglevel outside the subcommands so it can be used with any subcommand + command_modifier_group.add_argument('--loglevel', action='store', required=False, help=loglevel_help, default='ERROR', + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) + + # Output Compatibility options + command_modifier_group.add_argument('--compatibility', action='store', required=False, help=compatibility_help, + choices=["amdsmi", "gpuvsmi", "rocmsmi"], default='amdsmi') + + + def _add_device_arguments(self, subcommand_parser, required=False): + # Device arguments help text + gpu_help = f"Select a GPU ID, BDF, or UUID from the possible choices:\n{self.gpu_choices_str}" + vf_help = "Gets general information about the specified VF (timeslice, fb info, …).\ + \nAvailable only on virtualization OSs" + + # Mutually Exclusive Args within the subparser + device_args = subcommand_parser.add_mutually_exclusive_group(required=required) + device_args.add_argument('-g', '--gpu', action=self._gpu_select(self.gpu_choices), + nargs='+', help=gpu_help) + + if self.amd_smi_helpers.is_hypervisor(): + device_args.add_argument('-v', '--vf', action='store', nargs='+', + help=vf_help, choices=self.vf_choices) + + + def _add_version_parser(self, subparsers, func): + # Subparser help text + version_help = "Display version information" + + # Create version subparser + version_parser = subparsers.add_parser('version', help=version_help, description=None) + version_parser._optionals.title = None + version_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + version_parser.set_defaults(func=func) + self._add_command_modifiers(version_parser) + + + def _add_discovery_parser(self, subparsers, func): + # Subparser help text + discovery_help = "Display discovery information" + discovery_subcommand_help = "Lists all the devices on the system and the links between devices.\ + \nLists all the sockets and for each socket, GPUs and/or CPUs associated to\ + \nthat socket alongside some basic information for each device.\ + \nIn virtualization environment, it can also list VFs associated to each\ + \nGPU with some basic information for each VF." + + # Create discovery subparser + discovery_parser = subparsers.add_parser('discovery', help=discovery_help, description=discovery_subcommand_help, aliases=['list']) + discovery_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + discovery_parser.set_defaults(func=func) + + # Add Command Modifiers + self._add_command_modifiers(discovery_parser) + + # Add Device args + self._add_device_arguments(discovery_parser, required=False) + + + def _add_static_parser(self, subparsers, func): + # Subparser help text + static_help = "Gets static information about the specified GPU" + static_subcommand_help = "If no argument is provided, return static information for all GPUs on the system.\ + \nIf no static argument is specified all static information will be displayed." + static_optionals_title = "Static Arguments" + + # Optional arguments help text + asic_help = "All asic information" + bus_help = "All bus information" + vbios_help = "All video bios information (if available)" + limit_help = "All limit metric values (i.e. power and thermal limits)" + driver_help = "Displays driver version" + caps_help = "All caps information" + + # Options arguments help text for Hypervisors and Baremetal + ras_help = "Displays RAS features information" + board_help = "All board information" # Linux Baremetal only + + # Options arguments help text for Hypervisors + dfc_help = "All DFC FW table information" + fb_help = "Displays Frame Buffer information" + num_vf_help = "Displays number of supported and enabled VFs" + + # Create static subparser + static_parser = subparsers.add_parser('static', help=static_help, description=static_subcommand_help) + static_parser._optionals.title = static_optionals_title + static_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + static_parser.set_defaults(func=func) + self._add_command_modifiers(static_parser) + + # Add Device args + self._add_device_arguments(static_parser, required=False) + + # Optional Args + static_parser.add_argument('-a', '--asic', action='store_true', required=False, help=asic_help) + static_parser.add_argument('-b', '--bus', action='store_true', required=False, help=bus_help) + static_parser.add_argument('-V', '--vbios', action='store_true', required=False, help=vbios_help) + static_parser.add_argument('-l', '--limit', action='store_true', required=False, help=limit_help) + static_parser.add_argument('-d', '--driver', action='store_true', required=False, help=driver_help) + static_parser.add_argument('-c', '--caps', action='store_true', required=False, help=caps_help) + + # Options to display on Hypervisors and Baremetal + if self.amd_smi_helpers.is_hypervisor() or self.amd_smi_helpers.is_baremetal(): + static_parser.add_argument('-r', '--ras', action='store_true', required=False, help=ras_help) + if self.amd_smi_helpers.is_linux(): + static_parser.add_argument('-B', '--board', action='store_true', required=False, help=board_help) + + # Options to only display on a Hypervisor + if self.amd_smi_helpers.is_hypervisor(): + static_parser.add_argument('-d', '--dfc-ucode', action='store_true', required=False, help=dfc_help) + static_parser.add_argument('-f', '--fb-info', action='store_true', required=False, help=fb_help) + static_parser.add_argument('-n', '--num-vf', action='store_true', required=False, help=num_vf_help) + + + def _add_firmware_parser(self, subparsers, func): + # Subparser help text + firmware_help = "Gets firmware information about the specified GPU" + firmware_subcommand_help = "If no argument is provided, return firmware information for all GPUs on the system." + firmware_optionals_title = "Firmware Arguments" + + # Optional arguments help text + fw_list_help = "All FW list information" + err_records_help = "All error records information" + + # Create firmware subparser + firmware_parser = subparsers.add_parser('firmware', help=firmware_help, description=firmware_subcommand_help, aliases=['ucode']) + firmware_parser._optionals.title = firmware_optionals_title + firmware_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + firmware_parser.set_defaults(func=func) + self._add_command_modifiers(firmware_parser) + + # Add Device args + self._add_device_arguments(firmware_parser, required=False) + + # Optional Args + firmware_parser.add_argument('-f', '--ucode-list', '--fw-list', dest='fw_list', action='store_true', required=False, help=fw_list_help, default=True) + + # Options to only display on a Hypervisor + if self.amd_smi_helpers.is_hypervisor(): + firmware_parser.add_argument('-e', '--error-records', action='store_true', required=False, help=err_records_help) + + + def _add_bad_pages_parser(self, subparsers, func): + if not (self.amd_smi_helpers.is_baremetal() and self.amd_smi_helpers.is_linux()): + # The bad_pages subcommand is only applicable to Linux Baremetal systems + return + + # Subparser help text + bad_pages_help = "Gets bad page information about the specified GPU" + bad_pages_subcommand_help = "If no argument is provided, return bad page information for all GPUs on the system." + bad_pages_optionals_title = "Bad pages Arguments" + + # Optional arguments help text + pending_help = "Displays all pending retired pages" + retired_help = "Displays retired pages" + un_res_help = "Displays unreservable pages" + + # Create bad_pages subparser + bad_pages_parser = subparsers.add_parser('bad-pages', help=bad_pages_help, description=bad_pages_subcommand_help, aliases=['bad_pages']) + bad_pages_parser._optionals.title = bad_pages_optionals_title + bad_pages_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + bad_pages_parser.set_defaults(func=func) + self._add_command_modifiers(bad_pages_parser) + + # Add Device args + self._add_device_arguments(bad_pages_parser, required=False) + + # Optional Args + bad_pages_parser.add_argument('-p', '--pending', action='store_true', required=False, help=pending_help) + bad_pages_parser.add_argument('-r', '--retired', action='store_true', required=False, help=retired_help) + bad_pages_parser.add_argument('-u', '--un-res', action='store_true', required=False, help=un_res_help) + + + def _add_metric_parser(self, subparsers, func): + # Subparser help text + metric_help = "Gets metric/performance information about the specified GPU" + metric_subcommand_help = "If no argument is provided, return metric information for all GPUs on the system.\ + \nIf no metric argument is specified all metric information will be displayed." + metric_optionals_title = "Metric arguments" + + # Optional arguments help text + usage_help = "Displays engine usage information" + watch_help = "Reprint the command in a loop of Interval seconds" + watch_time_help = "The total time to watch the given command" + iterations_help = "Total number of iterations to loop on the given command" + + # Help text for Arguments only Available on Virtual OS and Baremetal platforms + fb_usage_help = "Total and used framebuffer" + + # Help text for Arguments only on Hypervisor and Baremetal platforms + power_help = "Current power usage" + clock_help = "Average, max, and current clock frequencies" + temperature_help = "Current temperatures" + ecc_help = "Number of ECC errors" + pcie_help = "Current PCIe speed and width" + voltage_help = "Current GPU voltages" + + # Help text for Arguments only on Linux Baremetal platforms + fan_help = "Current fan speed" + pcie_usage_help = "Estimated PCIe link usage" + vc_help = "Display voltage curve" + overdrive_help = "Current GPU clock overdrive level" + mo_help = "Current memory clock overdrive level" + perf_level_help = "Current DPM performance level" + replay_count_help = "PCIe replay count" + xgmi_err_help = "XGMI error information since last read" + energy_help = "Amount of energy consumed" + mem_usage_help = "Memory usage per block" + + # Help text for Arguments only on Hypervisors + schedule_help = "All scheduling information" + guard_help = "All guard information" + guest_help = "All guest data information" + + # Create metric subparser + metric_parser = subparsers.add_parser('metric', help=metric_help, description=metric_subcommand_help) + metric_parser._optionals.title = metric_optionals_title + metric_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + metric_parser.set_defaults(func=func) + self._add_command_modifiers(metric_parser) + + # Add Device args + self._add_device_arguments(metric_parser, required=False) + + # Optional Args + metric_parser.add_argument('-u', '--usage', action='store_true', required=False, help=usage_help) + metric_parser.add_argument('-w', '--watch', action='store', metavar='Interval', + type=self._positive_int, required=False, help=watch_help) + metric_parser.add_argument('-W', '--watch_time', action=self._check_watch_selected(), metavar='Duration', + type=self._positive_int, required=False, help=watch_time_help) + metric_parser.add_argument('-i', '--iterations', action=self._check_watch_selected(), metavar='Iterations', + type=self._positive_int, required=False, help=iterations_help) + + # Optional Args for Virtual OS and Baremetal systems + if self.amd_smi_helpers.is_virtual_os() or self.amd_smi_helpers.is_baremetal(): + metric_parser.add_argument('-b', '--fb-usage', action='store_true', required=False, help=fb_usage_help) + + # Optional Args for Hypervisors and Baremetal systems + if self.amd_smi_helpers.is_hypervisor() or self.amd_smi_helpers.is_baremetal(): + metric_parser.add_argument('-p', '--power', action='store_true', required=False, help=power_help) + metric_parser.add_argument('-c', '--clock', action='store_true', required=False, help=clock_help) + metric_parser.add_argument('-t', '--temperature', action='store_true', required=False, help=temperature_help) + metric_parser.add_argument('-e', '--ecc', action='store_true', required=False, help=ecc_help) + metric_parser.add_argument('-P', '--pcie', action='store_true', required=False, help=pcie_help) + metric_parser.add_argument('-V', '--voltage', action='store_true', required=False, help=voltage_help) + + # Optional Args for Linux Baremetal Systems + if self.amd_smi_helpers.is_baremetal() and self.amd_smi_helpers.is_linux(): + metric_parser.add_argument('-f', '--fan', action='store_true', required=False, help=fan_help) + metric_parser.add_argument('-s', '--pcie-usage', action='store_true', required=False, help=pcie_usage_help) + metric_parser.add_argument('-C', '--voltage-curve', action='store_true', required=False, help=vc_help) + metric_parser.add_argument('-o', '--overdrive', action='store_true', required=False, help=overdrive_help) + metric_parser.add_argument('-M', '--mem-overdrive', action='store_true', required=False, help=mo_help) + metric_parser.add_argument('-l', '--perf-level', action='store_true', required=False, help=perf_level_help) + metric_parser.add_argument('-r', '--replay-count', action='store_true', required=False, help=replay_count_help) + metric_parser.add_argument('-x', '--xgmi-err', action='store_true', required=False, help=xgmi_err_help) + metric_parser.add_argument('-E', '--energy', action='store_true', required=False, help=energy_help) + metric_parser.add_argument('-m', '--mem-usage', action='store_true', required=False, help=mem_usage_help) + + # Options to only display to Hypervisors + if self.amd_smi_helpers.is_hypervisor(): + metric_parser.add_argument('-s', '--schedule', action='store_true', required=False, help=schedule_help) + metric_parser.add_argument('-G', '--guard', action='store_true', required=False, help=guard_help) + metric_parser.add_argument('-u', '--guest', action='store_true', required=False, help=guest_help) + + + def _add_process_parser(self, subparsers, func): + if self.amd_smi_helpers.is_hypervisor(): + # Don't add this subparser on Hypervisors + # This subparser is only available to Guest and Baremetal systems + return + + # Subparser help text + process_help = "Lists general process information running on the specified GPU" + process_subcommand_help = "If no argument is provided, returns information for all GPUs on the system.\ + \nIf no argument is provided all process information will be displayed." + process_optionals_title = "Process arguments" + + # Optional Arguments help text + general_help = "pid, process name, memory usage" + engine_help = "All engine usages" + pid_help = "Gets all process information about the specified process based on Process ID" + name_help = "Gets all process information about the specified process based on Process Name.\ + \nIf multiple processes have the same name information is returned for all of them." + watch_help = "Reprint the command in a loop of Interval seconds" + watch_time_help = "The total time to watch the given command" + iterations_help = "Total number of iterations to loop on the given command" + + # Create process subparser + process_parser = subparsers.add_parser('process', help=process_help, description=process_subcommand_help) + process_parser._optionals.title = process_optionals_title + process_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + process_parser.set_defaults(func=func) + self._add_command_modifiers(process_parser) + + # Add Device args + self._add_device_arguments(process_parser, required=False) + + # Optional Args + process_parser.add_argument('-G', '--general', action='store_true', required=False, help=general_help) + process_parser.add_argument('-e', '--engine', action='store_true', required=False, help=engine_help) + process_parser.add_argument('-p', '--pid', action='store', type=self._positive_int, required=False, help=pid_help) + process_parser.add_argument('-n', '--name', action='store', required=False, help=name_help) + process_parser.add_argument('-w', '--watch', action='store', metavar='Interval', + type=self._positive_int, required=False, help=watch_help) + process_parser.add_argument('-W', '--watch_time', action=self._check_watch_selected(), metavar='Duration', + type=self._positive_int, required=False, help=watch_time_help) + process_parser.add_argument('-i', '--iterations', action=self._check_watch_selected(), metavar='Iterations', + type=self._positive_int, required=False, help=iterations_help) + + + def _add_profile_parser(self, subparsers, func): + if not (self.amd_smi_helpers.is_windows() and self.amd_smi_helpers.is_hypervisor()): + # This subparser only applies to Hypervisors + return + + # Subparser help text + profile_help = "Displays information about all profiles and current profile" + profile_subcommand_help = "If no argument is provided, returns information for all GPUs on the system." + profile_optionals_title = "Profile Arguments" + + # Create profile subparser + profile_parser = subparsers.add_parser('profile', help=profile_help, description=profile_subcommand_help) + profile_parser._optionals.title = profile_optionals_title + profile_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + profile_parser.set_defaults(func=func) + self._add_command_modifiers(profile_parser) + + # Add Device args + self._add_device_arguments(profile_parser, required=False) + + + def _add_event_parser(self, subparsers, func): + if self.amd_smi_helpers.is_linux() and not self.amd_smi_helpers.is_virtual_os(): + # This subparser only applies to Linux BareMetal & Linux Hypervisors + return + + # Subparser help text + event_help = "Displays event information for the given GPU" + event_subcommand_help = "If no argument is provided, returns event information for all GPUs on the system." + event_optionals_title = "Event Arguments" + + # Create event subparser + event_parser = subparsers.add_parser('event', help=event_help, description=event_subcommand_help) + event_parser._optionals.title = event_optionals_title + event_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + event_parser.set_defaults(func=func) + self._add_command_modifiers(event_parser) + + # Add Device args + self._add_device_arguments(event_parser, required=False) + + + def _add_topology_parser(self, subparsers, func): + if not(self.amd_smi_helpers.is_baremetal() and self.amd_smi_helpers.is_linux()): + # This subparser is only applicable to Baremetal Linux + return + + # Subparser help text + topology_help = "Displays topology information of the devices." + topology_subcommand_help = "If no argument is provided, returns information for all GPUs on the system." + topology_optionals_title = "Topology arguments" + + # Help text for Arguments only on Guest and BM platforms + topo_access_help = "Displays link accessibility between GPUs" + topo_weight_help = "Displays relative weight between GPUs" + topo_hops_help = "Displays the number of hops between GPUs" + topo_type_help = "Displays the link type between GPUs." + topo_numa_help = "Displays the numa nodes." + + # Create topology subparser + topology_parser = subparsers.add_parser('topology', help=topology_help, description=topology_subcommand_help) + topology_parser._optionals.title = topology_optionals_title + topology_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + topology_parser.set_defaults(func=func) + self._add_command_modifiers(topology_parser) + + # Add Device args + self._add_device_arguments(topology_parser, required=False) + + # Optional Args + topology_parser.add_argument('-a', '--topo-access', action='store_true', required=False, help=topo_access_help) + topology_parser.add_argument('-w', '--topo-weight', action='store_true', required=False, help=topo_weight_help) + topology_parser.add_argument('-o', '--topo-hops', action='store_true', required=False, help=topo_hops_help) + topology_parser.add_argument('-t', '--topo-type', action='store_true', required=False, help=topo_type_help) + topology_parser.add_argument('-n', '--topo-numa', action='store_true', required=False, help=topo_numa_help) + + + def _add_set_value_parser(self, subparsers, func): + if not(self.amd_smi_helpers.is_baremetal() and self.amd_smi_helpers.is_linux()): + # This subparser is only applicable to Baremetal Linux + return + + # Subparser help text + set_value_help = "Set options for devices." + set_value_subcommand_help = "The user must specify one of the options for the set configuration." + set_value_optionals_title = "Set Arguments" + + # Help text for Arguments only on Guest and BM platforms + set_clk_help = "Sets clock frequency levels for specified clocks" + set_sclk_help = "Sets GPU clock frequency levels" + set_mclk_help = "Sets memory clock frequency levels" + set_pcie_help = "Sets PCIe Bandwith " + set_slevel_help = "Change GPU clock frequency and voltage for a specific level" + set_mlevel_help = "Change GPU memory frequency and voltage for a specific level" + set_vc_help = "Change SCLK voltage curve for a specified point" + set_srange_help = "Sets min and max SCLK speed" + set_mrange_help = "Sets min and max MCLK speed" + set_fan_help = "Sets GPU fan speed (level or %)" + set_perf_level_help = "Sets performance level" + set_overdrive_help = "Set GPU overdrive level" + set_mem_overdrive_help = "Set memory overclock overdrive level" + set_power_overdrive_help = "Set the maximum GPU power using power overdrive in Watts" + set_profile_help = "Set power profile level (#) or a quoted string of custom profile attributes" + set_perf_det_help = "Set GPU clock frequency limit to get minimal performance variation" + ras_enable_help = "Enable RAS for specified block and error type" + ras_disable_help = "Disable RAS for specified block and error type." + + # Create set_value subparser + set_value_parser = subparsers.add_parser('set', help=set_value_help, description=set_value_subcommand_help) + set_value_parser._optionals.title = set_value_optionals_title + set_value_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + set_value_parser.set_defaults(func=func) + self._add_command_modifiers(set_value_parser) + + # Device args are required as safeguard from the user applying the operation to all gpus unintentionally + self._add_device_arguments(set_value_parser, required=True) + + # Optional Args + set_value_parser.add_argument('-c', '--clk', action='store', required=False, help=set_clk_help) + set_value_parser.add_argument('-s', '--sclk', action='store', required=False, help=set_sclk_help) + set_value_parser.add_argument('-m', '--mclk', action='store', required=False, help=set_mclk_help) + set_value_parser.add_argument('-p', '--pcie', action='store', required=False, help=set_pcie_help) + set_value_parser.add_argument('-S', '--slevel', action='store', required=False, help=set_slevel_help) + set_value_parser.add_argument('-M', '--mlevel', action='store', required=False, help=set_mlevel_help) + set_value_parser.add_argument('-V', '--vc', action='store', required=False, help=set_vc_help) + set_value_parser.add_argument('-r', '--srange', action='store', required=False, help=set_srange_help) + set_value_parser.add_argument('-R', '--mrange', action='store', required=False, help=set_mrange_help) + set_value_parser.add_argument('-f', '--fan', action='store', required=False, help=set_fan_help) + set_value_parser.add_argument('-l', '--perflevel', action='store', required=False, help=set_perf_level_help) + set_value_parser.add_argument('-o', '--overdrive', action='store', required=False, help=set_overdrive_help) + set_value_parser.add_argument('-O', '--memoverdrive', action='store', required=False, help=set_mem_overdrive_help) + set_value_parser.add_argument('-w', '--poweroverdrive', action='store', required=False, help=set_power_overdrive_help) + set_value_parser.add_argument('-P', '--profile', action='store', required=False, help=set_profile_help) + set_value_parser.add_argument('-d', '--perfdet', action='store', required=False, help=set_perf_det_help) + set_value_parser.add_argument('-e', '--rasenable', action='store', required=False, help=ras_enable_help) + set_value_parser.add_argument('-D', '--rasdisable', action='store', required=False, help=ras_disable_help) + + + def _add_reset_parser(self, subparsers, func): + if not(self.amd_smi_helpers.is_baremetal() and self.amd_smi_helpers.is_linux()): + # This subparser is only applicable to Baremetal Linux + return + + # Subparser help text + reset_help = "Reset options for devices." + reset_subcommand_help = "The user must specify one of the options to reset devices." + reset_optionals_title = "Reset Arguments" + + # Help text for Arguments only on Guest and BM platforms + gpureset_help = "Reset the specified GPU" + resetclocks_help = "Reset clocks and overdrive to default" + resetfans_help = "Reset fans to automatic (driver) control" + resetprofile_help = "Reset power profile back to default" + resetpoweroverdrive_help = "Set the maximum GPU power back to the device default state" + resetxgmierr_help = "Reset XGMI error counts" + resetperfdet_help = "Disable performance determinism" + + # Create reset subparser + reset_parser = subparsers.add_parser('reset', help=reset_help, description=reset_subcommand_help) + reset_parser._optionals.title = reset_optionals_title + reset_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + reset_parser.set_defaults(func=func) + self._add_command_modifiers(reset_parser) + + # Device args are required as safeguard from the user applying the operation to all gpus unintentionally + self._add_device_arguments(reset_parser, required=True) + + # Optional Args + reset_parser.add_argument('-G', '--gpureset', action='store_true', required=False, help=gpureset_help) + reset_parser.add_argument('-c', '--resetclocks', action='store_true', required=False, help=resetclocks_help) + reset_parser.add_argument('-f', '--resetfans', action='store_true', required=False, help=resetfans_help) + reset_parser.add_argument('-p', '--resetprofile', action='store_true', required=False, help=resetprofile_help) + reset_parser.add_argument('-o', '--resetpoweroverdrive', action='store_true', required=False, help=resetpoweroverdrive_help) + reset_parser.add_argument('-x', '--resetxgmierr', action='store_true', required=False, help=resetxgmierr_help) + reset_parser.add_argument('-d', '--resetperfdet', action='store_true', required=False, help=resetperfdet_help) + + + def _add_rocm_smi_parser(self, subparsers, func): + # Subparser help text + rocm_smi_help = "Legacy rocm_smi commands ported for backward compatibility" + rocm_smi_subcommand_help = "If no argument is provided, return showall and print the information for all\ + GPUs on the system." + rocm_smi_optionals_title = "rocm_smi Arguments" + + # Optional arguments help text + load_help = "Load clock, fan, performance, and profile settings from a given file." + save_help = "Save clock, fan, performance, and profile settings to a given file." + showpidgpus_help = "Display's all the pids in a table sorted by gpu's" + showtopo_help = "Show combinded table to individual topo info" + showallinfo_help = "Show Temperature, Fan and Clock values" + showcompactview_help = "Show main points of interest" + showuse_help = "Show gpu usage" + showmemuse_help = "Show usage of gpu and memory" + + # Create rocm_smi subparser + rocm_smi_parser = subparsers.add_parser('rocm-smi', help=rocm_smi_help, description=rocm_smi_subcommand_help, aliases=['rocm_smi']) + rocm_smi_parser._optionals.title = rocm_smi_optionals_title + rocm_smi_parser.formatter_class=lambda prog: argparse.RawTextHelpFormatter(prog, max_help_position=80, width=90) + rocm_smi_parser.set_defaults(func=func) + self._add_command_modifiers(rocm_smi_parser) + + # Add Device args + self._add_device_arguments(rocm_smi_parser, required=False) + + # Optional Args + rocm_smi_parser.add_argument('-l', '--load', action=self._check_input_file_path(), type=str, required=False, help=load_help) + rocm_smi_parser.add_argument('-s', '--save', action=self._check_output_file_path(), type=str, required=False, help=save_help) + + rocm_smi_parser.add_argument('-T', '--showtempgraph', action='store_true', required=False, help=showpidgpus_help) + rocm_smi_parser.add_argument('-P', '--showprofile', action='store_true', required=False, help=showpidgpus_help) + rocm_smi_parser.add_argument('-M', '--showmaxpower', action='store_true', required=False, help=showpidgpus_help) + + rocm_smi_parser.add_argument('-p', '--showpidgpus', action='store_true', required=False, help=showpidgpus_help) + rocm_smi_parser.add_argument('-t', '--showtopo', action='store_true', required=False, help=showtopo_help) + rocm_smi_parser.add_argument('-a', '--showallinfo', action='store_true', required=False, help=showallinfo_help) + + rocm_smi_parser.add_argument('-c', '--showcompactview', action='store_true', required=False, help=showcompactview_help) + rocm_smi_parser.add_argument('-u', '--showuse', action='store_true', required=False, help=showuse_help) + rocm_smi_parser.add_argument('-m', '--showmemuse', action='store_true', required=False, help=showmemuse_help) diff --git a/py-interface/__init__.py b/py-interface/__init__.py index 82c8dd1d8c..d95f571307 100644 --- a/py-interface/__init__.py +++ b/py-interface/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -177,7 +177,6 @@ from .amdsmi_interface import amdsmi_is_P2P_accessible from .amdsmi_interface import amdsmi_get_xgmi_info # # Enums - from .amdsmi_interface import AmdSmiInitFlags from .amdsmi_interface import AmdSmiContainerTypes from .amdsmi_interface import AmdSmiDeviceType @@ -205,8 +204,8 @@ from .amdsmi_interface import AmdSmiIoLinkType from .amdsmi_interface import AmdSmiUtilizationCounterType from .amdsmi_interface import AmdSmiSwComponent from .amdsmi_interface import AmdSmiIoLinkType -# Exceptions +# Exceptions from .amdsmi_exception import AmdSmiLibraryException from .amdsmi_exception import AmdSmiRetryException from .amdsmi_exception import AmdSmiParameterException diff --git a/py-interface/_version.py b/py-interface/_version.py new file mode 100644 index 0000000000..b3c06d4883 --- /dev/null +++ b/py-interface/_version.py @@ -0,0 +1 @@ +__version__ = "0.0.1" \ No newline at end of file diff --git a/py-interface/amdsmi_exception.py b/py-interface/amdsmi_exception.py index 2659a44fe0..8dcf3fa625 100644 --- a/py-interface/amdsmi_exception.py +++ b/py-interface/amdsmi_exception.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -22,7 +22,6 @@ from enum import IntEnum from . import amdsmi_wrapper - class AmdSmiRetCode(IntEnum): SUCCESS = amdsmi_wrapper.AMDSMI_STATUS_SUCCESS ERR_INVAL = amdsmi_wrapper.AMDSMI_STATUS_INVAL @@ -72,6 +71,9 @@ class AmdSmiLibraryException(AmdSmiException): err_code=self.err_code, err_info=self.err_info ) + def get_error_info(self): + return self.err_info + def get_error_code(self): return self.err_code diff --git a/py-interface/amdsmi_interface.py b/py-interface/amdsmi_interface.py index c48aedf752..55ade08abb 100644 --- a/py-interface/amdsmi_interface.py +++ b/py-interface/amdsmi_interface.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -26,7 +26,7 @@ from enum import IntEnum from collections.abc import Iterable from . import amdsmi_wrapper -from .amdsmi_exception import * +from amdsmi_exception import * class AmdSmiInitFlags(IntEnum): @@ -292,18 +292,6 @@ class AmdSmiUtilizationCounterType(IntEnum): COARSE_GRAIN_MEM_ACTIVITY = amdsmi_wrapper.AMDSMI_COARSE_GRAIN_MEM_ACTIVITY -class AmdSmiSwComponent(IntEnum): - DRIVER = amdsmi_wrapper.AMDSMI_SW_COMP_DRIVER - - -class AmdSmiIoLinkType(IntEnum): - UNDEFINED = amdsmi_wrapper.AMDSMI_IOLINK_TYPE_UNDEFINED - PCIEXPRESS = amdsmi_wrapper.AMDSMI_IOLINK_TYPE_PCIEXPRESS - XGMI = amdsmi_wrapper.AMDSMI_IOLINK_TYPE_XGMI - NUMIOLINKTYPES = amdsmi_wrapper.AMDSMI_IOLINK_TYPE_NUMIOLINKTYPES - SIZE = amdsmi_wrapper.AMDSMI_IOLINK_TYPE_SIZE - - class AmdSmiEventReader: def __init__( self, device_handle: amdsmi_wrapper.amdsmi_device_handle, *event_types @@ -498,8 +486,7 @@ def amdsmi_get_socket_handles() -> List[amdsmi_wrapper.amdsmi_socket_handle]: socket_count.value)() _check_res( amdsmi_wrapper.amdsmi_get_socket_handles( - ctypes.byref(socket_count), socket_handles - ) + ctypes.byref(socket_count), socket_handles) ) sockets = [ amdsmi_wrapper.amdsmi_socket_handle(socket_handles[sock_idx]) @@ -513,11 +500,14 @@ def amdsmi_get_socket_info(socket_handle): if not isinstance(socket_handle, amdsmi_wrapper.amdsmi_socket_handle): raise AmdSmiParameterException( socket_handle, amdsmi_wrapper.amdsmi_socket_handle) + socket_info = ctypes.create_string_buffer(128) - return { - "name": "" - } + _check_res( + amdsmi_wrapper.amdsmi_get_socket_info( + socket_handle, ctypes.byref(socket_info), ctypes.c_size_t(128)) + ) + return socket_info.value.decode() def amdsmi_get_device_handles() -> List[amdsmi_wrapper.amdsmi_device_handle]: socket_handles = amdsmi_get_socket_handles() @@ -677,10 +667,10 @@ def amdsmi_get_vbios_info( return { "name": vbios_info.name.decode("utf-8"), - "vbios_version": vbios_info.vbios_version, + "vbios_version_string": vbios_info.vbios_version_string.decode("utf-8"), "build_date": vbios_info.build_date.decode("utf-8"), "part_number": vbios_info.part_number.decode("utf-8"), - "vbios_version_string": vbios_info.vbios_version_string.decode("utf-8"), + "vbios_version": vbios_info.vbios_version, } @@ -2261,20 +2251,6 @@ def amdsmi_dev_get_perf_level( return result -def amdsmi_set_perf_determinism_mode( - device_handle: amdsmi_wrapper.amdsmi_device_handle, clkvalue: int -) -> None: - if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): - raise AmdSmiParameterException( - device_handle, amdsmi_wrapper.amdsmi_device_handle - ) - if not isinstance(clkvalue, int): - raise AmdSmiParameterException(clkvalue, int) - - _check_res(amdsmi_wrapper.amdsmi_set_perf_determinism_mode( - device_handle, clkvalue)) - - def amdsmi_dev_get_overdrive_level( device_handle: amdsmi_wrapper.amdsmi_device_handle, ) -> int: @@ -2446,30 +2422,6 @@ def amdsmi_dev_get_od_volt_curve_regions( return result -def amdsmi_dev_get_power_profile_presets( - device_handle: amdsmi_wrapper.amdsmi_device_handle, sensor_idx: int -) -> Dict[str, Any]: - if not isinstance(device_handle, amdsmi_wrapper.amdsmi_device_handle): - raise AmdSmiParameterException( - device_handle, amdsmi_wrapper.amdsmi_device_handle - ) - if not isinstance(sensor_idx, int): - raise AmdSmiParameterException(sensor_idx, int) - - status = amdsmi_wrapper.amdsmi_power_profile_status_t() - _check_res( - amdsmi_wrapper. amdsmi_dev_get_power_profile_presets( - device_handle, sensor_idx, ctypes.byref(status) - ) - ) - - return { - "available_profiles": status.available_profiles, - "current": status.current, - "num_profiles": status.num_profiles, - } - - def amdsmi_dev_get_ecc_count( device_handle: amdsmi_wrapper.amdsmi_device_handle, block: AmdSmiGpuBlock ) -> Dict[str, int]: diff --git a/py-interface/amdsmi_wrapper.py b/py-interface/amdsmi_wrapper.py index 5ca5474cd8..19fde95fcf 100644 --- a/py-interface/amdsmi_wrapper.py +++ b/py-interface/amdsmi_wrapper.py @@ -1,6 +1,6 @@ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in diff --git a/py-interface/rocm_smi_tool.py b/py-interface/rocm_smi_tool.py index c6670a693d..9df8409ca0 100644 --- a/py-interface/rocm_smi_tool.py +++ b/py-interface/rocm_smi_tool.py @@ -159,14 +159,14 @@ class CmdLineParser: def get_device_handle(self): device_handles = [] - def parse_device_handle(possition): - self._check_if_arg_exists("Device identifier", "device bdf or device id", possition) + def parse_device_handle(position): + self._check_if_arg_exists("Device identifier", "device bdf or device id", position) ids = self._parse_vf_id_if_exists() if ids["gpu_id"] is not None and ids["vf_id"] is not None: device_handle = self._get_device_handle_from_id(ids["gpu_id"], ids["vf_id"]) else: - gpu_arg = self.cmd_args[possition] + gpu_arg = self.cmd_args[position] if gpu_arg.isdigit(): device_handle = self._get_device_handle_from_id(int(gpu_arg)) else: diff --git a/tools/generator.py b/tools/generator.py index 39dc5d8a6f..6eb0f335ed 100644 --- a/tools/generator.py +++ b/tools/generator.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in @@ -32,7 +32,7 @@ from ctypeslib.clang2py import main as clangToPy HEADER = \ """ # -# Copyright (C) 2022 Advanced Micro Devices. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy of # this software and associated documentation files (the "Software"), to deal in