From 63cc7eafbe17dac7b6efe9fe4a76ddabbc56dcb3 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Fri, 22 Sep 2023 19:13:33 -0500 Subject: [PATCH 01/19] PY: Remove f-strings from rocm_smi.py Change-Id: I0a422e8f66473af837460ecb2450e5be329163b0 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: 1683245ecfcdc9fb4c7d21ec11c7a71fcc16b2cc] --- projects/amdsmi/python_smi_tools/rocm_smi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/amdsmi/python_smi_tools/rocm_smi.py b/projects/amdsmi/python_smi_tools/rocm_smi.py index 2a0a4655d7..568484899b 100755 --- a/projects/amdsmi/python_smi_tools/rocm_smi.py +++ b/projects/amdsmi/python_smi_tools/rocm_smi.py @@ -3525,7 +3525,7 @@ def save(deviceList, savefilepath): # The code below is for when this script is run as an executable instead of when imported as a module if __name__ == '__main__': parser = argparse.ArgumentParser( - description=f'AMD ROCm System Management Interface | ROCM-SMI version: {__version__}', + description='AMD ROCm System Management Interface | ROCM-SMI version: %s' % __version__, formatter_class=lambda prog: argparse.HelpFormatter(prog, max_help_position=90, width=120)) groupDev = parser.add_argument_group() groupDisplayOpt = parser.add_argument_group('Display Options') From ab545816ba3b0c9fa1cce63fd203ae779fe65605 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Fri, 22 Sep 2023 20:56:21 -0500 Subject: [PATCH 02/19] SWDEV-423672 - Always compile and install gtest This commit makes sure GTest is always compiled with rocm_smi_lib_tests. GTest installation was inconsistent outside of AMD CI environment. libgtest.so wouldn't get installed with rocm_smi_lib_tests if gtest existed on the build machine. Which is undesirable when packaging. Change-Id: I607df6c67c81480e3b6487b28f14924e8bf56ad4 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: 0c662611e9f593e248dcee864fe5d3ebad6c94a2] --- .../amdsmi/tests/rocm_smi_test/CMakeLists.txt | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/projects/amdsmi/tests/rocm_smi_test/CMakeLists.txt b/projects/amdsmi/tests/rocm_smi_test/CMakeLists.txt index b2347e0bff..efdbe96469 100755 --- a/projects/amdsmi/tests/rocm_smi_test/CMakeLists.txt +++ b/projects/amdsmi/tests/rocm_smi_test/CMakeLists.txt @@ -26,16 +26,19 @@ set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(RSMITST "rsmitst") # Hack to find libraries after installation +# $ORIGIN is needed for libgtest.so # /opt/rocm/share/rocm-smi/rsmitst_tests/../../../ = /opt/rocm set(RSMITST_RPATH - "\$ORIGIN/../../../lib") + "\$ORIGIN/../../../lib" + "\$ORIGIN") # combine lists set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${RSMITST_RPATH}) -# Try to find googletest -find_package(GTest 1.12.0) +# TODO: Try to find googletest +# DISABLED because we want to install gtest with rocm_smi_lib ourselves +#find_package(GTest 1.12.0) # GTest_FOUND is set to TRUE if ANY version is found # GTest_VERSION is set if 1.12.0 or newer version is found @@ -47,6 +50,9 @@ if(NOT GTest_FOUND STREQUAL "TRUE" OR NOT DEFINED GTest_VERSION) GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-1.12.0) FetchContent_MakeAvailable(googletest) + install(TARGETS gtest gtest_main + DESTINATION ${SHARE_INSTALL_PREFIX}/rsmitst_tests + COMPONENT ${TESTS_COMPONENT}) endif() # Other source directories @@ -64,13 +70,14 @@ target_include_directories(${RSMITST} PUBLIC ${SRC_DIR}/..) target_link_libraries( ${RSMITST} PUBLIC ${ROCM_SMI_TARGET} - PUBLIC GTest::gtest_main + PUBLIC gtest + PUBLIC gtest_main PUBLIC c PUBLIC stdc++ PUBLIC pthread PUBLIC dl) -install(TARGETS ${RSMITST} gtest gtest_main +install(TARGETS ${RSMITST} DESTINATION ${SHARE_INSTALL_PREFIX}/rsmitst_tests COMPONENT ${TESTS_COMPONENT}) From b6630e02002a150bd6745f7c276e4e9a8b885f17 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Fri, 22 Sep 2023 03:15:38 -0500 Subject: [PATCH 03/19] Change uuid_info file read to uuid generation Signed-off-by: Maisam Arif Change-Id: I77f2c6b44efd6faae3c2466989e1caeaad16669d [ROCm/amdsmi commit: 8a9f862dbf28cba77f5033072084c84d0e4f30bd] --- projects/amdsmi/src/amd_smi/amd_smi.cc | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/projects/amdsmi/src/amd_smi/amd_smi.cc b/projects/amdsmi/src/amd_smi/amd_smi.cc index 170a2f1576..9e376dd912 100644 --- a/projects/amdsmi/src/amd_smi/amd_smi.cc +++ b/projects/amdsmi/src/amd_smi/amd_smi.cc @@ -1736,7 +1736,7 @@ amdsmi_status_t amdsmi_get_gpu_device_uuid(amdsmi_processor_handle processor_handle, unsigned int *uuid_length, char *uuid) { AMDSMI_CHECK_INIT(); - if (uuid_length == nullptr || uuid == nullptr) { + if (uuid_length == nullptr || uuid == nullptr || uuid_length == nullptr || *uuid_length < AMDSMI_GPU_UUID_SIZE) { return AMDSMI_STATUS_INVAL; } @@ -1748,35 +1748,20 @@ amdsmi_get_gpu_device_uuid(amdsmi_processor_handle processor_handle, unsigned in amdsmi_status_t status = AMDSMI_STATUS_SUCCESS; SMIGPUDEVICE_MUTEX(gpu_device->get_mutex()) - FILE *fp; size_t len = AMDSMI_GPU_UUID_SIZE; - ssize_t nread; amdsmi_asic_info_t asic_info = {}; const uint8_t fcn = 0xff; - std::string path = "/sys/class/drm/" + gpu_device->get_gpu_path() + "/device/uuid_info"; status = amdsmi_get_gpu_asic_info(processor_handle, &asic_info); if (status != AMDSMI_STATUS_SUCCESS) { printf("Getting asic info failed. Return code: %d", status); return status; } - fp = fopen(path.c_str(), "rb"); - if (!fp) { - /* generate random UUID */ - status = amdsmi_uuid_gen(uuid, strtoul(asic_info.asic_serial, nullptr, AMDSMI_NORMAL_STRING_LENGTH), (uint16_t)asic_info.device_id, fcn); - return status; - } - - nread = getline(&uuid, &len, fp); - if (nread <= 0) { - /* generate random UUID */ - status = amdsmi_uuid_gen(uuid, strtoul(asic_info.asic_serial, nullptr, AMDSMI_NORMAL_STRING_LENGTH), (uint16_t)asic_info.device_id, fcn); - fclose(fp); - return status; - } - - fclose(fp); + /* generate random UUID */ + status = amdsmi_uuid_gen(uuid, + strtoull(asic_info.asic_serial, nullptr, 16), + (uint16_t)asic_info.device_id, fcn); return status; } From 260fd7faae543fad9d5f04f395a86b3e95b0de1f Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 02:03:51 -0500 Subject: [PATCH 04/19] Removed compatibility modes Removed gpuv-smi and rocm-smi compatibility options Updated py-interface README Signed-off-by: Maisam Arif Change-Id: I1cad3e65e0928e7041727c6d5e7e7bdc39d667d0 Signed-off-by: Maisam Arif [ROCm/amdsmi commit: 5c52f82328aec8986a5855c2522fc43c73dac0a0] --- projects/amdsmi/amdsmi_cli/CMakeLists.txt | 7 +- projects/amdsmi/amdsmi_cli/amdsmi_cli.py | 10 +-- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 49 ++--------- projects/amdsmi/amdsmi_cli/amdsmi_logger.py | 81 +------------------ projects/amdsmi/amdsmi_cli/amdsmi_parser.py | 4 +- projects/amdsmi/py-interface/README.md | 22 ++--- 6 files changed, 23 insertions(+), 150 deletions(-) diff --git a/projects/amdsmi/amdsmi_cli/CMakeLists.txt b/projects/amdsmi/amdsmi_cli/CMakeLists.txt index 34626a24ee..3cd72a9fb1 100644 --- a/projects/amdsmi/amdsmi_cli/CMakeLists.txt +++ b/projects/amdsmi/amdsmi_cli/CMakeLists.txt @@ -72,17 +72,12 @@ add_custom_target( link_amdsmi_cli ALL DEPENDS amdsmi_cli BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/bin/amd-smi - ${CMAKE_CURRENT_BINARY_DIR}/bin/gpuv-smi WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND ${CMAKE_COMMAND} -E create_symlink ../${PY_CLI_INSTALL_DIR}/${PY_PACKAGE_DIR}/amdsmi_cli.py - ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/amd-smi - COMMAND ${CMAKE_COMMAND} -E create_symlink - amd-smi - ${CMAKE_CURRENT_BINARY_DIR}/bin/gpuv-smi) + ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_BINDIR}/amd-smi) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/bin/amd-smi - ${CMAKE_CURRENT_BINARY_DIR}/bin/gpuv-smi DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT dev) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py index e07db5457e..a696a774a8 100755 --- a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py @@ -42,15 +42,7 @@ def _print_error(e, destination): if __name__ == "__main__": - # Set compatability mode based on which cli mapping user selects - if 'gpuv-smi' in sys.argv[0]: - compatibility = AMDSMILogger.LoggerCompatibility.gpuvsmi.value - elif 'rocm-smi' in sys.argv[0]: - compatibility = AMDSMILogger.LoggerCompatibility.rocmsmi.value - else: - compatibility = AMDSMILogger.LoggerCompatibility.amdsmi.value - - amd_smi_commands = AMDSMICommands(compatibility=compatibility) + amd_smi_commands = AMDSMICommands() amd_smi_parser = AMDSMIParser(amd_smi_commands.version, amd_smi_commands.list, amd_smi_commands.static, diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index b54768bd4d..41ccfc4fe1 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -34,16 +34,11 @@ from amdsmi import amdsmi_exception class AMDSMICommands(): """This class contains all the commands corresponding to AMDSMIParser Each command function will interact with AMDSMILogger to handle - displaying the output to the specified compatibility, format, and - destination. + displaying the output to the specified format and destination. """ - def __init__(self, compatibility='amdsmi', - format='human_readable', - destination='stdout') -> None: + def __init__(self, format='human_readable', destination='stdout') -> None: self.helpers = AMDSMIHelpers() - self.logger = AMDSMILogger(compatibility=compatibility, - format=format, - destination=destination) + self.logger = AMDSMILogger(format=format, destination=destination) try: self.device_handles = amdsmi_interface.amdsmi_get_processor_handles() except amdsmi_exception.AmdSmiLibraryException as e: @@ -125,12 +120,7 @@ class AMDSMICommands(): self.logger.store_multiple_device_output() return # Skip printing when there are multiple devices - # compatibility with gpuvsmi needs a list for single gpu - if self.logger.is_gpuvsmi_compatibility() and not multiple_devices: - self.logger.store_multiple_device_output() - self.logger.print_output(multiple_device_enabled=True) - else: - self.logger.print_output() + self.logger.print_output() def static(self, args, multiple_devices=False, gpu=None, asic=None, @@ -262,11 +252,6 @@ class AMDSMICommands(): if args.vbios: try: vbios_info = amdsmi_interface.amdsmi_get_gpu_vbios_info(args.gpu) - if self.logger.is_gpuvsmi_compatibility(): - vbios_info['version'] = vbios_info.pop('version') - vbios_info['build_date'] = vbios_info.pop('build_date') - vbios_info['part_number'] = vbios_info.pop('part_number') - static_dict['vbios'] = vbios_info except amdsmi_exception.AmdSmiLibraryException as e: static_dict['vbios'] = "N/A" @@ -500,7 +485,7 @@ class AMDSMICommands(): """ if gpu: args.gpu = gpu - if fw_list: # Currently a compatiblity option of gpuv-smi + if fw_list: args.fw_list = fw_list # Handle No GPU passed @@ -525,20 +510,12 @@ class AMDSMICommands(): fw_entry['fw_version'] = fw_entry.pop('fw_version') firmware_identifier = 'FW' - if self.logger.is_gpuvsmi_compatibility(): - firmware_identifier = 'UCODE' - fw_entry['name'] = fw_entry.pop('fw_id') - fw_entry['version'] = fw_entry.pop('fw_version') - # Add custom human readable formatting if self.logger.is_human_readable_format(): fw_info['fw_list'][fw_index] = {f'{firmware_identifier} {fw_index}': fw_entry} else: fw_info['fw_list'][fw_index] = fw_entry - if self.logger.is_gpuvsmi_compatibility(): - fw_info['ucode_list'] = fw_info.pop('fw_list') - fw_list.update(fw_info) except amdsmi_exception.AmdSmiLibraryException as e: fw_list['fw_list'] = "N/A" @@ -547,11 +524,7 @@ class AMDSMICommands(): multiple_devices_csv_override = False # Convert and store output by pid for csv format if self.logger.is_csv_format(): - if self.logger.is_gpuvsmi_compatibility(): - fw_key = 'ucode_list' - else: - fw_key = 'fw_list' - + fw_key = 'fw_list' for fw_info_dict in fw_list[fw_key]: for key, value in fw_info_dict.items(): multiple_devices_csv_override = True @@ -961,15 +934,8 @@ class AMDSMICommands(): 'hotspot': temperature_hotspot_current, 'mem': temperature_vram_current} - if self.logger.is_gpuvsmi_compatibility(): - temperatures = {'edge_temperature': temperature_edge_current, - 'hotspot_temperature': temperature_hotspot_current, - 'mem_temperature': temperature_vram_current} - if self.logger.is_human_readable_format(): unit = '\N{DEGREE SIGN}C' - if self.logger.is_gpuvsmi_compatibility(): - unit = 'C' for temperature_key, temperature_value in temperatures.items(): if 'AMD_SMI_STATUS' not in str(temperature_value): temperatures[temperature_key] = f"{temperature_value} {unit}" @@ -1001,8 +967,6 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: ecc_count = "N/A" logging.debug("Failed to get ecc count for gpu %s at block %s | %s", args.gpu, gpu_block, e.get_error_info()) - if self.logger.is_gpuvsmi_compatibility(): - ecc_count = "N/A" ecc_dict[state['block']] = {'correctable' : ecc_count, 'uncorrectable': ecc_count} @@ -1434,7 +1398,6 @@ class AMDSMICommands(): def topology(self, args, multiple_devices=False, gpu=None, access=None, weight=None, hops=None, link_type=None, numa_bw=None): """ Get topology information for target gpus - The compatibility mode for this will only be in amdsmi & rocm-smi params: args - argparser args to pass to subcommand multiple_devices (bool) - True if checking for multiple devices diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py index 100747bb91..563798a486 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py @@ -31,12 +31,10 @@ from amdsmi_helpers import AMDSMIHelpers import amdsmi_cli_exceptions class AMDSMILogger(): - def __init__(self, compatibility='amdsmi', format='human_readable', - destination='stdout') -> None: + def __init__(self, format='human_readable', destination='stdout') -> None: self.output = {} self.multiple_device_output = [] self.watch_output = [] - self.compatibility = compatibility # amd-smi, gpuv-smi, or rocm-smi self.format = format # csv, json, or human_readable self.destination = destination # stdout, path to a file (append) self.helpers = AMDSMIHelpers() @@ -49,13 +47,6 @@ class AMDSMILogger(): human_readable = 'human_readable' - class LoggerCompatibility(Enum): - """Enum for logger compatibility""" - amdsmi = 'amdsmi' - rocmsmi = 'rocmsmi' - gpuvsmi = 'gpuvsmi' - - class CsvStdoutBuilder(object): def __init__(self): self.csv_string = [] @@ -79,18 +70,6 @@ class AMDSMILogger(): return self.format == self.LoggerFormat.human_readable.value - def is_amdsmi_compatibility(self): - return self.compatibility == self.LoggerCompatibility.amdsmi.value - - - def is_rocmsmi_compatibility(self): - return self.compatibility == self.LoggerCompatibility.rocmsmi.value - - - def is_gpuvsmi_compatibility(self): - return self.compatibility == self.LoggerCompatibility.gpuvsmi.value - - def _capitalize_keys(self, input_dict): output_dict = {} for key in input_dict.keys(): @@ -123,10 +102,6 @@ class AMDSMILogger(): yaml_data = yaml.safe_load(json_string) yaml_output = yaml.dump(yaml_data, sort_keys=False, allow_unicode=True) - if self.is_gpuvsmi_compatibility(): - # Convert from GPU: 0 to GPU 0: - yaml_output = re.sub('GPU: ([0-9]+)', 'GPU \\1:', yaml_output) - # Remove a key line if it is a spacer yaml_output = yaml_output.replace("AMDSMI_SPACING_REMOVAL:\n", "") yaml_output = yaml_output.replace("'", "") # Remove '' @@ -196,13 +171,6 @@ class AMDSMILogger(): value_with_parent_key[parent_key] = child_dict value = value_with_parent_key - if self.is_gpuvsmi_compatibility(): - if key in ('asic', 'bus', 'pcie', 'vbios','board', 'limit'): - value_with_parent_key = {} - for child_key, child_value in value.items(): - value_with_parent_key[key + '_' + child_key] = child_value - value = value_with_parent_key - output_dict.update(self.flatten_dict(value).items()) else: output_dict[key] = value @@ -210,9 +178,7 @@ class AMDSMILogger(): def store_output(self, device_handle, argument, data): - """ Store the argument and device handle according to the compatibility. - Each compatibility function will handle the output format and - populate the output + """ Convert device handle to gpu id and store output params: device_handle - device handle object to the target device output argument (str) - key to store data @@ -221,12 +187,7 @@ class AMDSMILogger(): Nothing """ gpu_id = self.helpers.get_gpu_id_from_device_handle(device_handle) - if self.is_amdsmi_compatibility(): - self._store_output_amdsmi(gpu_id=gpu_id, argument=argument, data=data) - elif self.is_rocmsmi_compatibility(): - self._store_output_rocmsmi(gpu_id=gpu_id, argument=argument, data=data) - elif self.is_gpuvsmi_compatibility(): - self._store_output_gpuvsmi(gpu_id=gpu_id, argument=argument, data=data) + self._store_output_amdsmi(gpu_id=gpu_id, argument=argument, data=data) def _store_output_amdsmi(self, gpu_id, argument, data): @@ -265,42 +226,6 @@ class AMDSMILogger(): raise amdsmi_cli_exceptions(self, "Invalid output format given, only json, csv, and human_readable supported") - def _store_output_gpuvsmi(self, gpu_id, argument, data): - if argument == 'timestamp': # Make sure timestamp is the first element in the output - self.output['timestamp'] = int(time.time()) - - if self.is_json_format() or self.is_human_readable_format(): - self.output['gpu'] = int(gpu_id) - self.output[argument] = data - elif self.is_csv_format(): - self.output['gpu'] = int(gpu_id) - - if argument == 'values' or isinstance(data, dict): - flat_dict = self.flatten_dict(data) - self.output.update(flat_dict) - else: - self.output[argument] = data - - gpuv_flat_dict = {} - for key, value in self.output.items(): - gpuv_flat_dict[key] = value - - # Change AMDSMI_STATUS strings to N/A for gpuv compatability - if isinstance(value, str): - if 'AMDSMI_STATUS' in value: - gpuv_flat_dict[key] = 'N/A' - - # Change bdf and uuid keys for gpuv compatability - if isinstance(key, str): - if key in ('bdf','uuid'): - gpuv_flat_dict['gpu_' + key] = gpuv_flat_dict.pop(key) - - self.output = gpuv_flat_dict - - else: - raise amdsmi_cli_exceptions(self, "Invalid output format given, only json, csv, and human_readable supported") - - def store_multiple_device_output(self): """ Store the current output into the multiple_device_output then clear the current output diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py index cbfcc87172..e58344598a 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py @@ -53,8 +53,6 @@ class AMDSMIParser(argparse.ArgumentParser): platform_string = f"Platform: {self.helpers.os_info()}" program_name = 'amd-smi' - if 'gpuv-smi' in sys.argv[0]: - program_name = 'gpuv-smi' # Adjust argument parser options super().__init__( @@ -207,7 +205,7 @@ class AMDSMIParser(argparse.ArgumentParser): return _GPUSelectAction - def _add_command_modifiers(self, subcommand_parser, rocm_smi=True, gpuv_smi=True): + def _add_command_modifiers(self, subcommand_parser): json_help = "Displays output in JSON format (human readable by default)." csv_help = "Displays output in CSV format (human readable by default)." file_help = "Saves output into a file on the provided path (stdout by default)." diff --git a/projects/amdsmi/py-interface/README.md b/projects/amdsmi/py-interface/README.md index 09eb158b49..ca39ab110c 100644 --- a/projects/amdsmi/py-interface/README.md +++ b/projects/amdsmi/py-interface/README.md @@ -1208,7 +1208,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1253,7 +1253,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1290,7 +1290,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1322,7 +1322,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1353,7 +1353,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1391,7 +1391,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1423,7 +1423,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1495,7 +1495,7 @@ Example: ```python try: - devices = gpuvsmi_get_devices() + devices = amdsmi_get_processor_handles() if len(devices) == 0: print("No GPUs on machine") else: @@ -1658,11 +1658,11 @@ Exceptions that can be thrown by `amdsmi_is_gpu_power_management_enabled` functi Example: ```python try: - processors = amdsmi_get_processor_handles() - if len(processors) == 0: + devices = amdsmi_get_processor_handles() + if len(devices) == 0: print("No GPUs on machine") else: - for processor in processors: + for processor in devices: is_power_management_enabled = amdsmi_is_gpu_power_management_enabled(processor) print(is_power_management_enabled) except AmdSmiException as e: From e408c46775f740966ddd5375d90be3eff182d966 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 01:17:54 -0500 Subject: [PATCH 05/19] Added energy counter in metric --energy Change-Id: I8ea09af79a1a37f53038d06a89eeac0882c47a1d Signed-off-by: Maisam Arif [ROCm/amdsmi commit: 842b290a446dea71e9051462bfe5859d8a52266f] --- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index 41ccfc4fe1..179296ff40 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -1106,7 +1106,21 @@ class AMDSMICommands(): values_dict['xgmi_err'] = "N/A" logging.debug("Failed to get xgmi error status for gpu %s | %s", args.gpu, e.get_error_info()) if args.energy: - pass + try: + energy_dict = amdsmi_interface.amdsmi_get_energy_count(args.gpu) + + energy = energy_dict['power'] * round(energy_dict['counter_resolution'], 1) + energy /= 1000000 + energy = round(energy, 3) + + if self.logger.is_human_readable_format(): + unit = 'J' + energy = f"{energy} {unit}" + + values_dict['energy'] = {"total_energy_consumption" : energy} + except amdsmi_interface.AmdSmiLibraryException as e: + values_dict['energy'] = "N/A" + logging.debug("Failed to get energy usage for gpu %s | %s", args.gpu, e.get_error_info()) if self.helpers.is_linux() and (self.helpers.is_baremetal() or self.helpers.is_virtual_os()): if args.mem_usage: From 137fd1b6280b74ed1d7bebae841a20f93d6cada9 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Fri, 22 Sep 2023 08:58:27 -0500 Subject: [PATCH 06/19] Handle the memory frequency with only one line Change the code to handle the memory frequency if it is only one line. Change-Id: I09e6ee78a2b9c12c861243dc89296e4e7862da49 [ROCm/amdsmi commit: 85df5676d438824b0463558bee12a8432fd00085] --- projects/amdsmi/src/rocm_smi.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index bfe427fcba..5fc777f6a8 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -1262,6 +1262,11 @@ static rsmi_status_t get_od_clk_volt_info(uint32_t dv_ind, return RSMI_STATUS_UNEXPECTED_DATA; } + // find last_item but skip empty lines + int last_item = val_vec.size()-1; + while (val_vec[last_item].empty() || val_vec[last_item][0] == 0) + last_item--; + p->curr_sclk_range.lower_bound = freq_string_to_int(val_vec, nullptr, nullptr, kOD_SCLK_label_array_index + 1); p->curr_sclk_range.upper_bound = freq_string_to_int(val_vec, nullptr, @@ -1275,16 +1280,18 @@ static rsmi_status_t get_od_clk_volt_info(uint32_t dv_ind, } else if (val_vec[kOD_MCLK_label_array_index] == "MCLK:") { p->curr_mclk_range.lower_bound = freq_string_to_int(val_vec, nullptr, nullptr, kOD_MCLK_label_array_index + 1); + // the upper memory frequency is the last p->curr_mclk_range.upper_bound = freq_string_to_int(val_vec, nullptr, - nullptr, kOD_MCLK_label_array_index + 4); + nullptr, last_item); return RSMI_STATUS_SUCCESS; } else if (val_vec[kOD_MCLK_label_array_index + 1] == "MCLK:") { p->curr_sclk_range.upper_bound = freq_string_to_int(val_vec, nullptr, nullptr, kOD_SCLK_label_array_index + 3); p->curr_mclk_range.lower_bound = freq_string_to_int(val_vec, nullptr, nullptr, kOD_MCLK_label_array_index + 2); + // the upper memory frequency is the last p->curr_mclk_range.upper_bound = freq_string_to_int(val_vec, nullptr, - nullptr, kOD_MCLK_label_array_index + 5); + nullptr, last_item); return RSMI_STATUS_SUCCESS; } else { return RSMI_STATUS_NOT_YET_IMPLEMENTED; From 8c9d2f4e20ae452614cf81240ea1e583d60a6221 Mon Sep 17 00:00:00 2001 From: "Bill(Shuzhou) Liu" Date: Fri, 22 Sep 2023 15:38:23 -0500 Subject: [PATCH 07/19] Change the python tool id output label Change the label from GPU to Device as we call rsmi_dev_id_get(). Change-Id: I8ffe3673d434e5291ebd5cc909afb7d18154ecb6 [ROCm/amdsmi commit: 2247c4b46c7bc1f8cdf2d109d25a7c242c308a9d] --- projects/amdsmi/python_smi_tools/README.md | 2 +- projects/amdsmi/python_smi_tools/rocm_smi.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/amdsmi/python_smi_tools/README.md b/projects/amdsmi/python_smi_tools/README.md index e33e093548..5c65f21f4c 100644 --- a/projects/amdsmi/python_smi_tools/README.md +++ b/projects/amdsmi/python_smi_tools/README.md @@ -69,7 +69,7 @@ Display Options: -a, --showallinfo Show Temperature, Fan and Clock values Topology: - -i, --showid Show GPU ID + -i, --showid Show DEVICE ID -v, --showvbios Show VBIOS version --showdriverversion Show kernel driver version --showfwinfo [BLOCK [BLOCK ...]] Show FW information diff --git a/projects/amdsmi/python_smi_tools/rocm_smi.py b/projects/amdsmi/python_smi_tools/rocm_smi.py index 568484899b..cc9bc1f695 100755 --- a/projects/amdsmi/python_smi_tools/rocm_smi.py +++ b/projects/amdsmi/python_smi_tools/rocm_smi.py @@ -2108,8 +2108,8 @@ def showId(deviceList): """ printLogSpacer(' ID ') for device in deviceList: - printLog(device, 'GPU ID', getId(device)) - printLog(device, 'GPU Rev', getRev(device)) + printLog(device, 'Device ID', getId(device)) + printLog(device, 'Device Rev', getRev(device)) printLogSpacer() @@ -3545,7 +3545,7 @@ if __name__ == '__main__': groupDisplayOpt.add_argument('--showhw', help='Show Hardware details', action='store_true') groupDisplayOpt.add_argument('-a', '--showallinfo', help='Show Temperature, Fan and Clock values', action='store_true') - groupDisplayTop.add_argument('-i', '--showid', help='Show GPU ID', action='store_true') + groupDisplayTop.add_argument('-i', '--showid', help='Show DEVICE ID', action='store_true') groupDisplayTop.add_argument('-v', '--showvbios', help='Show VBIOS version', action='store_true') groupDisplayTop.add_argument('-e', '--showevents', help='Show event list', metavar='EVENT', type=str, nargs='*') groupDisplayTop.add_argument('--showdriverversion', help='Show kernel driver version', action='store_true') From 0d1ac5edac481cdf6b8766c07ed73d178121c6d3 Mon Sep 17 00:00:00 2001 From: Ori Messinger Date: Wed, 13 Sep 2023 16:23:57 -0400 Subject: [PATCH 08/19] ROCm SMI LIB: Add Missing Firmware Blocks The purpose of this patch is to add the following missing firmware blocks to the SMI LIB: -RSMI_FW_BLOCK_MES -RSMI_FW_BLOCK_MES_KIQ Signed-off-by: Ori Messinger Change-Id: I5d4d37d883878dd02ef8533d4eb8891d54d70630 [ROCm/amdsmi commit: d44a6ef52307f2926d57edb06d3d4356d12b8254] --- projects/amdsmi/include/rocm_smi/rocm_smi.h | 2 ++ projects/amdsmi/include/rocm_smi/rocm_smi_device.h | 2 ++ projects/amdsmi/src/rocm_smi.cc | 2 ++ projects/amdsmi/src/rocm_smi_device.cc | 10 ++++++++++ projects/amdsmi/src/rocm_smi_main.cc | 2 ++ projects/amdsmi/tests/rocm_smi_test/test_utils.cc | 2 ++ 6 files changed, 20 insertions(+) diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/include/rocm_smi/rocm_smi.h index 14e4db0d58..6b755ef154 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi.h @@ -625,6 +625,8 @@ typedef enum { RSMI_FW_BLOCK_ME, RSMI_FW_BLOCK_MEC, RSMI_FW_BLOCK_MEC2, + RSMI_FW_BLOCK_MES, + RSMI_FW_BLOCK_MES_KIQ, RSMI_FW_BLOCK_PFP, RSMI_FW_BLOCK_RLC, RSMI_FW_BLOCK_RLC_SRLC, diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_device.h b/projects/amdsmi/include/rocm_smi/rocm_smi_device.h index a1b2809457..7a28e1327c 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_device.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_device.h @@ -146,6 +146,8 @@ enum DevInfoTypes { kDevFwVersionMe, kDevFwVersionMec, kDevFwVersionMec2, + kDevFwVersionMes, + kDevFwVersionMesKiq, kDevFwVersionPfp, kDevFwVersionRlc, kDevFwVersionRlcSrlc, diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index 5fc777f6a8..b05f37dbb8 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -1655,6 +1655,8 @@ rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block, { RSMI_FW_BLOCK_ME, amd::smi::kDevFwVersionMe }, { RSMI_FW_BLOCK_MEC, amd::smi::kDevFwVersionMec }, { RSMI_FW_BLOCK_MEC2, amd::smi::kDevFwVersionMec2 }, + { RSMI_FW_BLOCK_MES, amd::smi::kDevFwVersionMes }, + { RSMI_FW_BLOCK_MES_KIQ, amd::smi::kDevFwVersionMesKiq }, { RSMI_FW_BLOCK_PFP, amd::smi::kDevFwVersionPfp }, { RSMI_FW_BLOCK_RLC, amd::smi::kDevFwVersionRlc }, { RSMI_FW_BLOCK_RLC_SRLC, amd::smi::kDevFwVersionRlcSrlc }, diff --git a/projects/amdsmi/src/rocm_smi_device.cc b/projects/amdsmi/src/rocm_smi_device.cc index 03a38dc749..12f69dcec0 100755 --- a/projects/amdsmi/src/rocm_smi_device.cc +++ b/projects/amdsmi/src/rocm_smi_device.cc @@ -138,6 +138,8 @@ static const char *kDevFwVersionMcFName = "fw_version/mc_fw_version"; static const char *kDevFwVersionMeFName = "fw_version/me_fw_version"; static const char *kDevFwVersionMecFName = "fw_version/mec_fw_version"; static const char *kDevFwVersionMec2FName = "fw_version/mec2_fw_version"; +static const char *kDevFwVersionMesFName = "fw_version/mes_fw_version"; +static const char *kDevFwVersionMesKiqFName = "fw_version/mes_kiq_fw_version"; static const char *kDevFwVersionPfpFName = "fw_version/pfp_fw_version"; static const char *kDevFwVersionRlcFName = "fw_version/rlc_fw_version"; static const char *kDevFwVersionRlcSrlcFName = "fw_version/rlc_srlc_fw_version"; @@ -280,6 +282,8 @@ static const std::map kDevAttribNameMap = { {kDevFwVersionMe, kDevFwVersionMeFName}, {kDevFwVersionMec, kDevFwVersionMecFName}, {kDevFwVersionMec2, kDevFwVersionMec2FName}, + {kDevFwVersionMes, kDevFwVersionMesFName}, + {kDevFwVersionMesKiq, kDevFwVersionMesKiqFName}, {kDevFwVersionPfp, kDevFwVersionPfpFName}, {kDevFwVersionRlc, kDevFwVersionRlcFName}, {kDevFwVersionRlcSrlc, kDevFwVersionRlcSrlcFName}, @@ -343,6 +347,8 @@ static std::map kDevInfoVarTypeToRSMIVariant = { {kDevFwVersionMe, RSMI_FW_BLOCK_ME}, {kDevFwVersionMec, RSMI_FW_BLOCK_MEC}, {kDevFwVersionMec2, RSMI_FW_BLOCK_MEC2}, + {kDevFwVersionMes, RSMI_FW_BLOCK_MES}, + {kDevFwVersionMesKiq, RSMI_FW_BLOCK_MES_KIQ}, {kDevFwVersionPfp, RSMI_FW_BLOCK_PFP}, {kDevFwVersionRlc, RSMI_FW_BLOCK_RLC}, {kDevFwVersionRlcSrlc, RSMI_FW_BLOCK_RLC_SRLC}, @@ -478,6 +484,8 @@ static const std::map kDevFuncDependsMap = { kDevFwVersionMe, kDevFwVersionMec, kDevFwVersionMec2, + kDevFwVersionMes, + kDevFwVersionMesKiq, kDevFwVersionPfp, kDevFwVersionRlc, kDevFwVersionRlcSrlc, @@ -943,6 +951,8 @@ int Device::readDevInfo(DevInfoTypes type, uint64_t *val) { case kDevFwVersionMe: case kDevFwVersionMec: case kDevFwVersionMec2: + case kDevFwVersionMes: + case kDevFwVersionMesKiq: case kDevFwVersionPfp: case kDevFwVersionRlc: case kDevFwVersionRlcSrlc: diff --git a/projects/amdsmi/src/rocm_smi_main.cc b/projects/amdsmi/src/rocm_smi_main.cc index 831e382b93..ab0ee435d2 100755 --- a/projects/amdsmi/src/rocm_smi_main.cc +++ b/projects/amdsmi/src/rocm_smi_main.cc @@ -129,6 +129,8 @@ amd::smi::RocmSMI::devInfoTypesStrings = { {amd::smi::kDevFwVersionMe, amdSMI + "kDevFwVersionMe"}, {amd::smi::kDevFwVersionMec, amdSMI + "kDevFwVersionMec"}, {amd::smi::kDevFwVersionMec2, amdSMI + "kDevFwVersionMec2"}, + {amd::smi::kDevFwVersionMes, amdSMI + "kDevFwVersionMes"}, + {amd::smi::kDevFwVersionMesKiq, amdSMI + "kDevFwVersionMesKiq"}, {amd::smi::kDevFwVersionPfp, amdSMI + "kDevFwVersionPfp"}, {amd::smi::kDevFwVersionRlc, amdSMI + "kDevFwVersionRlc"}, {amd::smi::kDevFwVersionRlcSrlc, amdSMI + "kDevFwVersionRlcSrlc"}, diff --git a/projects/amdsmi/tests/rocm_smi_test/test_utils.cc b/projects/amdsmi/tests/rocm_smi_test/test_utils.cc index c1b9a8de86..d27f185bba 100755 --- a/projects/amdsmi/tests/rocm_smi_test/test_utils.cc +++ b/projects/amdsmi/tests/rocm_smi_test/test_utils.cc @@ -56,6 +56,8 @@ static const std::map kDevFWNameMap = { {RSMI_FW_BLOCK_ME, "me"}, {RSMI_FW_BLOCK_MEC, "mec"}, {RSMI_FW_BLOCK_MEC2, "mec2"}, + {RSMI_FW_BLOCK_MES, "mes"}, + {RSMI_FW_BLOCK_MES_KIQ, "mes_kiq"}, {RSMI_FW_BLOCK_PFP, "pfp"}, {RSMI_FW_BLOCK_RLC, "rlc"}, {RSMI_FW_BLOCK_RLC_SRLC, "rlc_srlc"}, From e9addd72cced2350aab0b0002ac3086ff6cb75d1 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Sun, 24 Sep 2023 23:01:54 -0500 Subject: [PATCH 09/19] SWDEV-422836 - Add sleep frequency support Change-Id: I0bde403b010bf036ce44ed0600cc7eb03742c6b6 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: 3d40c4bb2c1108e5d6a181295f8970fe775c1007] --- projects/amdsmi/include/rocm_smi/rocm_smi.h | 8 ++- projects/amdsmi/python_smi_tools/rocm_smi.py | 54 +++++++++++++------ .../python_smi_tools/rsmiBindings.py.in | 5 +- projects/amdsmi/src/rocm_smi.cc | 26 ++++++--- 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/include/rocm_smi/rocm_smi.h index 6b755ef154..4a15c6178c 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi.h @@ -70,7 +70,8 @@ extern "C" { */ //! Guaranteed maximum possible number of supported frequencies -#define RSMI_MAX_NUM_FREQUENCIES 32 +//! (32 normal + 1 sleep frequency) +#define RSMI_MAX_NUM_FREQUENCIES 33 //! Maximum possible value for fan speed. Should be used as the denominator //! when determining fan speed percentage. @@ -747,6 +748,11 @@ typedef rsmi_power_profile_status_t rsmi_power_profile_status; * @brief This structure holds information about clock frequencies. */ typedef struct { + /** + * Deep Sleep frequency is only supported by some GPUs + */ + bool has_deep_sleep; + /** * The number of supported frequencies */ diff --git a/projects/amdsmi/python_smi_tools/rocm_smi.py b/projects/amdsmi/python_smi_tools/rocm_smi.py index cc9bc1f695..f705ad22a9 100755 --- a/projects/amdsmi/python_smi_tools/rocm_smi.py +++ b/projects/amdsmi/python_smi_tools/rocm_smi.py @@ -1808,12 +1808,21 @@ def showClocks(deviceList): if not rsmi_ret_ok(ret, device, 'get_clk_freq_' + clk_type, True): continue printLog(device, 'Supported %s frequencies on GPU%s' % (clk_type, str(device)), None) - for x in range(freq.num_supported): - fr = '{:>.0f}Mhz'.format(freq.frequency[x] / 1000000) - if x == freq.current: - printLog(device, str(x), str(fr) + ' *') - else: - printLog(device, str(x), str(fr)) + for i in range(freq.num_supported): + freq_string = '{:>.0f}Mhz'.format(freq.frequency[i] / 1000000) + if i == freq.current: + freq_string += ' *' + freq_index = i + # Deep Sleep frequency is only supported by some GPUs + # It is indicated by letter 'S' instead of the index number + if freq.has_deep_sleep: + # sleep state + if i == 0: + freq_index = 'S' + # all indices are offset by 1 because Deep Sleep occupies index 0 + else: + freq_index = i - 1 + printLog(device, str(freq_index), freq_string) printLog(device, '', None) else: logging.debug('{} frequency is unsupported on device[{}]'.format(clk_type, device)) @@ -1822,12 +1831,11 @@ def showClocks(deviceList): ret = rocmsmi.rsmi_dev_pci_bandwidth_get(device, byref(bw)) if rsmi_ret_ok(ret, device, 'get_PCIe_bandwidth', True): printLog(device, 'Supported %s frequencies on GPU%s' % ('PCIe', str(device)), None) - for x in range(bw.transfer_rate.num_supported): - fr = '{:>.1f}GT/s x{}'.format(bw.transfer_rate.frequency[x] / 1000000000, bw.lanes[x]) - if x == bw.transfer_rate.current: - printLog(device, str(x), str(fr) + ' *') - else: - printLog(device, str(x), str(fr)) + for i in range(bw.transfer_rate.num_supported): + freq_string = '{:>.1f}GT/s x{}'.format(bw.transfer_rate.frequency[i] / 1000000000, bw.lanes[i]) + if i == bw.transfer_rate.current: + freq_string += ' *' + printLog(device, str(i), str(freq_string)) printLog(device, '', None) else: logging.debug('PCIe frequency is unsupported on device [{}]'.format(device)) @@ -1857,9 +1865,17 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): printLog(device, '%s current clock frequency not found' % (clk_defined), None) continue fr = freq.frequency[levl] / 1000000 + freq_index = levl + if freq.has_deep_sleep: + # sleep state + if levl == 0: + freq_index = 'S' + # all indices are offset by 1 because Deep Sleep occupies index 0 + else: + freq_index = levl - 1 if concise: # in case function is used for concise output, no need to print. return '{:.0f}Mhz'.format(fr) - printLog(device, '{} clock level'.format(clk_defined), '{} ({:.0f}Mhz)'.format(levl, fr)) + printLog(device, '{} clock level'.format(clk_defined), '{} ({:.0f}Mhz)'.format(freq_index, fr)) elif not concise: logging.debug('{} clock is unsupported on device[{}]'.format(clk_defined, device)) @@ -1872,12 +1888,20 @@ def showCurrentClocks(deviceList, clk_defined=None, concise=False): if levl >= freq.num_supported: printLog(device, '%s current clock frequency not found' % (clk_type), None) continue + freq_index = levl + if freq.has_deep_sleep: + # sleep state + if levl == 0: + freq_index = 'S' + # all indices are offset by 1 because Deep Sleep occupies index 0 + else: + freq_index = levl - 1 fr = freq.frequency[levl] / 1000000 if PRINT_JSON: printLog(device, '%s clock speed:' % (clk_type), '(%sMhz)' % (str(fr)[:-2])) - printLog(device, '%s clock level:' % (clk_type), levl) + printLog(device, '%s clock level:' % (clk_type), freq_index) else: - printLog(device, '%s clock level: %s' % (clk_type, levl), '(%sMhz)' % (str(fr)[:-2])) + printLog(device, '%s clock level: %s' % (clk_type, freq_index), '(%sMhz)' % (str(fr)[:-2])) elif not concise: logging.debug('{} clock is unsupported on device[{}]'.format(clk_type, device)) # pcie clocks diff --git a/projects/amdsmi/python_smi_tools/rsmiBindings.py.in b/projects/amdsmi/python_smi_tools/rsmiBindings.py.in index e6b141889f..284319e10f 100644 --- a/projects/amdsmi/python_smi_tools/rsmiBindings.py.in +++ b/projects/amdsmi/python_smi_tools/rsmiBindings.py.in @@ -59,7 +59,7 @@ gpu_id = c_uint32(0) # Policy enums -RSMI_MAX_NUM_FREQUENCIES = 32 +RSMI_MAX_NUM_FREQUENCIES = 33 RSMI_MAX_FAN_SPEED = 255 RSMI_NUM_VOLTAGE_CURVE_POINTS = 3 @@ -492,7 +492,8 @@ rsmi_power_profile_status = rsmi_power_profile_status_t class rsmi_frequencies_t(Structure): - _fields_ = [('num_supported', c_int32), + _fields_ = [('has_deep_sleep', c_bool), + ('num_supported', c_int32), ('current', c_uint32), ('frequency', c_uint64 * RSMI_MAX_NUM_FREQUENCIES)] diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index b05f37dbb8..f35ab6a193 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -147,14 +147,21 @@ static uint64_t freq_string_to_int(const std::vector &freq_lines, std::istringstream fs(freq_lines[i]); - uint32_t ind; + char junk_ch; + int ind; float freq; - std::string junk; + std::string junk_str; std::string units_str; std::string star_str; - fs >> ind; - fs >> junk; // colon + if (fs.peek() == 'S') { + // Deep Sleep frequency is only supported by some GPUs + fs >> junk_ch; + } else { + // All other frequency indices are numbers + fs >> ind; + } + fs >> junk_str; // colon fs >> freq; fs >> units_str; fs >> star_str; @@ -1084,9 +1091,14 @@ static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, rsmi_clk_type_ } f->num_supported = static_cast(val_vec.size()); - bool current = false; f->current = RSMI_MAX_NUM_FREQUENCIES + 1; // init to an invalid value + // Deep Sleep frequency is only supported by some GPUs + // It is indicated by letter 'S' instead of the index number + f->has_deep_sleep = (val_vec[0][0] == 'S'); + + bool current = false; + for (uint32_t i = 0; i < f->num_supported; ++i) { f->frequency[i] = freq_string_to_int(val_vec, ¤t, lanes, i); @@ -1113,9 +1125,9 @@ static rsmi_status_t get_frequencies(amd::smi::DevInfoTypes type, rsmi_clk_type_ sysvalue += " Previous Value"; sysvalue += ' ' + std::to_string(f->frequency[f->current]); DEBUG_LOG("More than one current clock. ", sysvalue); - } - else + } else { f->current = i; + } } } From 6e81bbcf16cfd46f53bf4d9285f590324f6d4925 Mon Sep 17 00:00:00 2001 From: Charis Poag Date: Sun, 24 Sep 2023 02:29:07 -0500 Subject: [PATCH 10/19] Add Current (Instant) Socket Power * Updates: - rocm_smi_logger: General cleanup & Aligned to cpplint rules for usage - rocm_smi_monitor: Fixed MonitorTypes from not displaying properly in logs & Added socket power label + current socket power MonitorTypes - rocm_smi API: Added rsmi_dev_current_socket_power_get API - rocm_smi CLI: General cleanup, Concise info now displays device data in variable width (see printLogSpacer's new field), printLogSpacer now as an adjustable variable that overrides appWidth, Added Socket Power to base rocm-smi + --showpower CLI calls, --showpower & base rocm-smi CLI defaults to printing socket power (if not available, displays average power) - Cleaned up temp label references - power_read gtests: Added current socket power to testing Change-Id: Ica57e6f98ad96e2584e7c7955e188f68d2dab89d Signed-off-by: Charis Poag [ROCm/amdsmi commit: f07837535082c2556bb451e050b70d7b9416949e] --- projects/amdsmi/include/rocm_smi/rocm_smi.h | 24 +++ .../amdsmi/include/rocm_smi/rocm_smi_logger.h | 6 +- .../include/rocm_smi/rocm_smi_monitor.h | 80 ++++---- projects/amdsmi/python_smi_tools/rocm_smi.py | 173 ++++++++++++------ .../python_smi_tools/rsmiBindings.py.in | 5 + projects/amdsmi/src/rocm_smi.cc | 109 +++++++++-- projects/amdsmi/src/rocm_smi_device.cc | 2 - projects/amdsmi/src/rocm_smi_gpu_metrics.cc | 1 - projects/amdsmi/src/rocm_smi_logger.cc | 111 +++++------ projects/amdsmi/src/rocm_smi_main.cc | 5 +- projects/amdsmi/src/rocm_smi_monitor.cc | 13 +- projects/amdsmi/src/rocm_smi_utils.cc | 1 - .../rocm_smi_test/functional/power_read.cc | 52 ++++-- 13 files changed, 387 insertions(+), 195 deletions(-) diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi.h b/projects/amdsmi/include/rocm_smi/rocm_smi.h index 4a15c6178c..764a459814 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi.h @@ -1707,6 +1707,30 @@ rsmi_status_t rsmi_dev_pci_bandwidth_set(uint32_t dv_ind, uint64_t bw_bitmask); rsmi_status_t rsmi_dev_power_ave_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power); +/** + * @brief Get the current socket power (also known as instant + * power) of the device index provided. + * + * @details Given a device index @p dv_ind and a pointer to a uint64_t + * @p socket_power, this function will write the current socket power + * (in microwatts) to the uint64_t pointed to by @p socket_power. + * + * @param[in] dv_ind a device index + * + * @param[inout] socket_power a pointer to uint64_t to which the current + * socket power will be written to. If this parameter is nullptr, + * this function will return ::RSMI_STATUS_INVALID_ARGS if the function is + * supported with the provided, arguments and ::RSMI_STATUS_NOT_SUPPORTED + * if it is not supported with the provided arguments. + * + * @retval ::RSMI_STATUS_SUCCESS call was successful + * @retval ::RSMI_STATUS_NOT_SUPPORTED installed software or hardware does not + * support this function with the given arguments + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + */ +rsmi_status_t +rsmi_dev_current_socket_power_get(uint32_t dv_ind, uint64_t *socket_power); + /** * @brief Get the energy accumulator counter of the device with provided * device index. diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_logger.h b/projects/amdsmi/include/rocm_smi/rocm_smi_logger.h index bd2608db58..f83240fbf4 100644 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_logger.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_logger.h @@ -130,18 +130,18 @@ class Logger { break; } return *getInstance(); - }; + } Logger &operator<<(const char* s) { return operator<<(std::string(s)); - }; + } template Logger &operator<<(const T &v) { std::ostringstream s; s << v; std::string str = s.str(); return operator<<(str); - }; + } // Interface for Error Log void error(const char* text) throw(); diff --git a/projects/amdsmi/include/rocm_smi/rocm_smi_monitor.h b/projects/amdsmi/include/rocm_smi/rocm_smi_monitor.h index ea639eae35..ad284646b3 100755 --- a/projects/amdsmi/include/rocm_smi/rocm_smi_monitor.h +++ b/projects/amdsmi/include/rocm_smi/rocm_smi_monitor.h @@ -5,7 +5,7 @@ * The University of Illinois/NCSA * Open Source License (NCSA) * - * Copyright (c) 2017, Advanced Micro Devices, Inc. + * Copyright (c) 2017-2023, Advanced Micro Devices, Inc. * All rights reserved. * * Developed by: @@ -67,6 +67,8 @@ enum MonitorTypes { kMonPowerCapMax, kMonPowerCapMin, kMonPowerAve, + kMonPowerInput, + kMonPowerLabel, kMonTempMax, kMonTempMin, kMonTempMaxHyst, @@ -94,45 +96,47 @@ enum MonitorTypes { kMonInvalid = 0xFFFFFFFF, }; -const std::map monitorTypesToString { - {MonitorTypes::kMonName, "amd::smi::kMonName"}, - {MonitorTypes::kMonTemp, "amd::smi::kMonName"}, - {MonitorTypes::kMonFanSpeed, "amd::smi::kMonName"}, - {MonitorTypes::kMonMaxFanSpeed, "amd::smi::kMonName"}, - {MonitorTypes::kMonFanRPMs, "amd::smi::kMonName"}, - {MonitorTypes::kMonFanCntrlEnable, "amd::smi::kMonName"}, - {MonitorTypes::kMonPowerCap, "amd::smi::kMonName"}, - {MonitorTypes::kMonPowerCapDefault, "amd::smi::kMonName"}, - {MonitorTypes::kMonPowerCapMax, "amd::smi::kMonName"}, - {MonitorTypes::kMonPowerCapMin, "amd::smi::kMonName"}, - {MonitorTypes::kMonPowerAve, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempMax, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempMin, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempMaxHyst, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempMinHyst, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempCritical, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempCriticalHyst, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempEmergency, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempEmergencyHyst, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempCritMin, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempCritMinHyst, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempOffset, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempLowest, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempHighest, "amd::smi::kMonName"}, - {MonitorTypes::kMonTempLabel, "amd::smi::kMonName"}, - {MonitorTypes::kMonVolt, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltMax, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltMinCrit, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltMin, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltMaxCrit, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltAverage, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltLowest, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltHighest, "amd::smi::kMonName"}, - {MonitorTypes::kMonVoltLabel, "amd::smi::kMonName"}, - {MonitorTypes::kMonInvalid, "amd::smi::kMonName"}, +const std::map monitorTypesToString{ + {MonitorTypes::kMonName, "MonitorTypes::kMonName"}, + {MonitorTypes::kMonTemp, "MonitorTypes::kMonTemp"}, + {MonitorTypes::kMonFanSpeed, "MonitorTypes::kMonFanSpeed"}, + {MonitorTypes::kMonMaxFanSpeed, "MonitorTypes::kMonMaxFanSpeed"}, + {MonitorTypes::kMonFanRPMs, "MonitorTypes::kMonFanRPMs"}, + {MonitorTypes::kMonFanCntrlEnable, "MonitorTypes::kMonFanCntrlEnable"}, + {MonitorTypes::kMonPowerCap, "MonitorTypes::kMonPowerCap"}, + {MonitorTypes::kMonPowerCapDefault, "MonitorTypes::kMonPowerCapDefault"}, + {MonitorTypes::kMonPowerCapMax, "MonitorTypes::kMonPowerCapMax"}, + {MonitorTypes::kMonPowerCapMin, "MonitorTypes::kMonPowerCapMin"}, + {MonitorTypes::kMonPowerAve, "MonitorTypes::kMonPowerAve"}, + {MonitorTypes::kMonPowerInput, "MonitorTypes::kMonPowerInput"}, + {MonitorTypes::kMonPowerLabel, "MonitorTypes::kMonPowerLabel"}, + {MonitorTypes::kMonTempMax, "MonitorTypes::kMonTempMax"}, + {MonitorTypes::kMonTempMin, "MonitorTypes::kMonTempMin"}, + {MonitorTypes::kMonTempMaxHyst, "MonitorTypes::kMonTempMaxHyst"}, + {MonitorTypes::kMonTempMinHyst, "MonitorTypes::kMonTempMinHyst"}, + {MonitorTypes::kMonTempCritical, "MonitorTypes::kMonTempCritical"}, + {MonitorTypes::kMonTempCriticalHyst, "MonitorTypes::kMonTempCriticalHyst"}, + {MonitorTypes::kMonTempEmergency, "MonitorTypes::kMonTempEmergency"}, + {MonitorTypes::kMonTempEmergencyHyst, + "MonitorTypes::kMonTempEmergencyHyst"}, + {MonitorTypes::kMonTempCritMin, "MonitorTypes::kMonTempCritMin"}, + {MonitorTypes::kMonTempCritMinHyst, "MonitorTypes::kMonTempCritMinHyst"}, + {MonitorTypes::kMonTempOffset, "MonitorTypes::kMonTempOffset"}, + {MonitorTypes::kMonTempLowest, "MonitorTypes::kMonTempLowest"}, + {MonitorTypes::kMonTempHighest, "MonitorTypes::kMonTempHighest"}, + {MonitorTypes::kMonTempLabel, "MonitorTypes::kMonTempLabel"}, + {MonitorTypes::kMonVolt, "MonitorTypes::kMonVolt"}, + {MonitorTypes::kMonVoltMax, "MonitorTypes::kMonVoltMax"}, + {MonitorTypes::kMonVoltMinCrit, "MonitorTypes::kMonVoltMinCrit"}, + {MonitorTypes::kMonVoltMin, "MonitorTypes::kMonVoltMin"}, + {MonitorTypes::kMonVoltMaxCrit, "MonitorTypes::kMonVoltMaxCrit"}, + {MonitorTypes::kMonVoltAverage, "MonitorTypes::kMonVoltAverage"}, + {MonitorTypes::kMonVoltLowest, "MonitorTypes::kMonVoltLowest"}, + {MonitorTypes::kMonVoltHighest, "MonitorTypes::kMonVoltHighest"}, + {MonitorTypes::kMonVoltLabel, "MonitorTypes::kMonVoltLabel"}, + {MonitorTypes::kMonInvalid, "MonitorTypes::kMonInvalid"}, }; - class Monitor { public: explicit Monitor(std::string path, RocmSMI_env_vars const *e); diff --git a/projects/amdsmi/python_smi_tools/rocm_smi.py b/projects/amdsmi/python_smi_tools/rocm_smi.py index f705ad22a9..6f7ba1a8e0 100755 --- a/projects/amdsmi/python_smi_tools/rocm_smi.py +++ b/projects/amdsmi/python_smi_tools/rocm_smi.py @@ -45,9 +45,8 @@ CLOCK_JSON_VERSION = 1 headerString = ' ROCm System Management Interface ' footerString = ' End of ROCm SMI Log ' - # Output formatting -appWidth = 100 +appWidth = 90 deviceList = [] # Enable or disable serialized format @@ -383,8 +382,8 @@ def getPidList(): return -def getPower(device, silent=False): - """ Return the current power level of a given device +def getAvgPower(device, silent=False): + """ Return the average power level of a given device @param device: DRM device identifier @param silent=Turn on to silence error output @@ -393,7 +392,21 @@ def getPower(device, silent=False): power = c_uint32() ret = rocmsmi.rsmi_dev_power_ave_get(device, 0, byref(power)) if rsmi_ret_ok(ret, device, 'get_power_avg', silent): - return power.value / 1000000 + return str(power.value / 1000000) + return 'N/A' + +def getCurrentSocketPower(device, silent=False): + """ Return the current (also known as instant) + socket power of a given device + + @param device: DRM device identifier + @param silent=Turn on to silence error output + (you plan to handle manually). Default is off. + """ + power = c_uint32() + ret = rocmsmi.rsmi_dev_current_socket_power_get(device, byref(power)) + if rsmi_ret_ok(ret, device, 'get_socket_power', silent): + return str(power.value / 1000000) return 'N/A' @@ -437,7 +450,7 @@ def findFirstAvailableTemp(device): temp = c_int64(0) metric = rsmi_temperature_metric_t.RSMI_TEMP_CURRENT ret_temp = "N/A" - ret_temp_type = "(Unknown)" + ret_temp_type = temp_type_lst[0] for i, templist_val in enumerate(temp_type_lst): ret = rocmsmi.rsmi_dev_temp_metric_get(c_uint32(device), i, metric, byref(temp)) if rsmi_ret_ok(ret, device, 'get_temp_metric_' + templist_val, silent=True): @@ -448,6 +461,37 @@ def findFirstAvailableTemp(device): continue return (ret_temp_type, ret_temp) +def getTemperatureLabel(deviceList): + """ Discovers the the first identified power label + + Returns a string label value + @param device: DRM device identifier + """ + # Default label is Edge + tempLabel = temp_type_lst[0].lower() + if len(deviceList) < 1: + return tempLabel + (temp_type, _) = findFirstAvailableTemp(deviceList[0]) + tempLabel = temp_type.lower().replace('(', '').replace(')', '') + return tempLabel + +def getPowerLabel(deviceList): + """ Discovers the the first identified power label + + Returns a string label value + @param device: DRM device identifier + """ + power = c_int64(0) + # Default label is AvgPower + powerLabel = rsmi_power_label.AVG_POWER + if len(deviceList) < 1: + return powerLabel + device=deviceList[0] + power = getCurrentSocketPower(device, True) + if power != '0.0' and power != 'N/A': + powerLabel = rsmi_power_label.CURRENT_SOCKET_POWER + return powerLabel + def getVbiosVersion(device, silent=False): """ Returns the VBIOS version for a given device @@ -679,23 +723,35 @@ def printListLog(metricName, valuesList): print(listStr + line) -def printLogSpacer(displayString=None, fill='='): +def printLogSpacer(displayString=None, fill='=', contentSizeToFit=0): """ Prints [name of the option]/[name of the program] in the spacer to explain data below If no parameters are given, a default fill of the '=' string is used in the spacer @param displayString: name of item to be displayed inside of the log spacer @param fill: padding string which surrounds the given display string + @param contentSizeToFit: providing an integer > 0 allows + ability to dynamically change output padding/fill based on this value + instead of appWidth. Handy for concise info output. """ global appWidth, PRINT_JSON + resizeValue = appWidth + if contentSizeToFit != 0: + resizeValue = contentSizeToFit + if resizeValue % 2: # if odd -> make even + resizeValue += 1 + # leaving below to check if resizing works properly + # print("resizeVal=" +str(resizeValue) + "; appWidth=" + str(appWidth) + + # "; contentSizeToFit=" + str(contentSizeToFit) + "; fill=" + fill) + if not PRINT_JSON: if displayString: if len(displayString) % 2: displayString += fill - logSpacer = fill * int((appWidth - (len(displayString))) / 2) + displayString + fill * int( - (appWidth - (len(displayString))) / 2) + logSpacer = fill * int((resizeValue - (len(displayString))) / 2) + displayString + fill * int( + (resizeValue - (len(displayString))) / 2) else: - logSpacer = fill * appWidth + logSpacer = fill * resizeValue print(logSpacer) @@ -1630,22 +1686,15 @@ def showAllConcise(deviceList): print('ERROR: Cannot print JSON/CSV output for concise output') sys.exit(1) - """ Place holder for the actual max size """ - MAX_ALL_CONCISE_WIDTH = 100 - appWidth_temp = appWidth - appWidth = MAX_ALL_CONCISE_WIDTH silent = True - printLogSpacer(' Concise Info ') deviceList.sort() - temp_type = '(' + temp_type_lst[0] + ')' - if len(deviceList) >= 1: - (temp_type, _) = findFirstAvailableTemp(deviceList[0]) - available_temp_type = temp_type.lower() - available_temp_type = available_temp_type.replace('(', '') - available_temp_type = available_temp_type.replace(')', '') - header = ['GPU', '[Model : Revision]', 'Temp', 'AvgPwr', 'Partitions', 'SCLK', 'MCLK', 'Fan', 'Perf', 'PwrCap', 'VRAM%', 'GPU%'] - subheader = ['', 'Name (20 chars)', temp_type, '', '(Mem, Compute)', '', '', '', '', '', '', ''] + available_temp_type = getTemperatureLabel(deviceList) + temp_type = "(" + available_temp_type.capitalize() + ")" + header=['Device', '[Model : Revision]', 'Temp', 'Power', 'Partitions', + 'SCLK', 'MCLK', 'Fan', 'Perf', 'PwrCap', 'VRAM%', 'GPU%'] + subheader = ['', 'Name (20 chars)', temp_type, getPowerLabel(deviceList), + '(Mem, Compute)', '', '', '', '', '', '', ''] # add additional spaces to match header for idx, item in enumerate(subheader): header_size = len(header[idx]) @@ -1667,11 +1716,17 @@ def showAllConcise(deviceList): temp_val = str(getTemp(device, available_temp_type, silent)) if temp_val != 'N/A': temp_val += degree_sign + 'C' - avgPwr = str(getPower(device)) - if avgPwr != '0.0' and avgPwr != 'N/A': + socketPwr = getCurrentSocketPower(device, True) + avgPwr = getAvgPower(device, True) + powerVal = 'N/A' + if socketPwr != '0.0' and socketPwr != 'N/A': + socketPwr += 'W' + powerVal=socketPwr + elif avgPwr != '0.0' and avgPwr != 'N/A': avgPwr += 'W' + powerVal=avgPwr else: - avgPwr = 'N/A' + powerVal = 'N/A' combined_partition = (getMemoryPartition(device, silent) + ", " + getComputePartition(device, silent)) sclk = showCurrentClocks([device], 'sclk', concise=silent) @@ -1704,10 +1759,10 @@ def showAllConcise(deviceList): '', '', '', ''] gpu_dev_product_info_top_name = gpu_dev_product_info_names[1] - values['card%s' % (str(device))] = [device, gpu_dev_product_info_top_name, temp_val, avgPwr, - combined_partition, sclk, mclk, - fan, str(perf).lower(), pwrCap, - mem_use_pct, gpu_busy] + values['card%s' % (str(device))] = [device, gpu_dev_product_info_top_name, temp_val, + powerVal, combined_partition, sclk, mclk, + fan, str(perf).lower(), pwrCap, mem_use_pct, + gpu_busy] val_widths = {} for device in deviceList: @@ -1716,10 +1771,17 @@ def showAllConcise(deviceList): for device in deviceList: for col in range(len(val_widths[device])): max_widths[col] = max(max_widths[col], val_widths[device][col]) - printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)), None) - printLog(None, "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), subheader)), - None, useItalics=True) - printLogSpacer(fill='=') + + ######################## + # Display concise info # + ######################## + header_output = "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), header)) + subheader_output = "".join(word.ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), subheader)) + printLogSpacer(headerString, contentSizeToFit=len(header_output)) + printLogSpacer(' Concise Info ', contentSizeToFit=len(header_output)) + printLog(None, header_output, None) + printLog(None, subheader_output, None, useItalics=True) + printLogSpacer(fill='=', contentSizeToFit=len(header_output)) for device in deviceList: printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in @@ -1730,9 +1792,8 @@ def showAllConcise(deviceList): printLog(None, "".join(str(word).ljust(max_widths[col]) for col, word in zip(range(len(max_widths)), values['card%s_Info' % (str(device))])), None) - printLogSpacer() - """ Restore original max size """ - appWidth = appWidth_temp + printLogSpacer(contentSizeToFit=len(header_output)) + printLogSpacer(footerString, contentSizeToFit=len(header_output)) def showAllConciseHw(deviceList): @@ -2360,23 +2421,25 @@ def showPids(verbose): def showPower(deviceList): - """ Display current Average Graphics Package Power Consumption for a list of devices + """ Display Current (also known as instant) Socket or Average + Graphics Package Power Consumption for a list of devices @param deviceList: List of DRM devices (can be a single-item list) """ secondaryPresent=False printLogSpacer(' Power Consumption ') for device in deviceList: - if checkIfSecondaryDie(device): + if str(getCurrentSocketPower(device, True)) != 'N/A': + printLog(device, 'Current Socket Graphics Package Power (W)', getCurrentSocketPower(device)) + elif checkIfSecondaryDie(device): printLog(device, 'Average Graphics Package Power (W)', "N/A (Secondary die)") secondaryPresent=True - elif str(getPower(device)) != '0.0': - printLog(device, 'Average Graphics Package Power (W)', getPower(device)) + elif str(getAvgPower(device)) != '0.0': + printLog(device, 'Average Graphics Package Power (W)', getAvgPower(device)) else: - printErrLog(device, 'Unable to get Average Graphics Package Power Consumption') + printErrLog(device, 'Unable to get Average or Current Socket Graphics Package Power Consumption') if secondaryPresent: printLog(None, "\n\t\tPrimary die (usually one above or below the secondary) shows total (primary + secondary) socket power information", None) - printLogSpacer() @@ -2872,13 +2935,8 @@ def getGraphColor(percentage): def showTempGraph(deviceList): deviceList.sort() - temp_type = '(' + temp_type_lst[0] + ')' - if len(deviceList) >= 1: - (temp_type, _) = findFirstAvailableTemp(deviceList[0]) - printLogSpacer(' Temperature Graph ' + temp_type + ' ') - temp_type = temp_type.lower() - temp_type = temp_type.replace('(', '') - temp_type = temp_type.replace(')', '') + temp_type = getTemperatureLabel(deviceList) + printLogSpacer(' Temperature Graph ' + temp_type.capitalize() + ' ') # Start a thread for constantly printing try: # Create a thread (call print function, devices, delay in ms) @@ -3547,6 +3605,11 @@ def save(deviceList, savefilepath): # The code below is for when this script is run as an executable instead of when imported as a module +def isConciseInfoRequested(args): + return len(sys.argv) == 1 or \ + len(sys.argv) == 2 and (args.alldevices or (args.json or args.csv)) or \ + len(sys.argv) == 3 and (args.alldevices and (args.json or args.csv)) + if __name__ == '__main__': parser = argparse.ArgumentParser( description='AMD ROCm System Management Interface | ROCM-SMI version: %s' % __version__, @@ -3755,7 +3818,8 @@ if __name__ == '__main__': if not PRINT_JSON: print('\n') - printLogSpacer(headerString) + if not isConciseInfoRequested(args): + printLogSpacer(headerString) if args.showallinfo: args.list = True @@ -3809,9 +3873,7 @@ if __name__ == '__main__': if not checkAmdGpus(deviceList): logging.warning('No AMD GPUs specified') - if len(sys.argv) == 1 or \ - len(sys.argv) == 2 and (args.alldevices or (args.json or args.csv)) or \ - len(sys.argv) == 3 and (args.alldevices and (args.json or args.csv)): + if isConciseInfoRequested(args): showAllConcise(deviceList) if args.showhw: showAllConciseHw(deviceList) @@ -4018,7 +4080,8 @@ if __name__ == '__main__': devCsv = formatCsv(deviceList) print(devCsv) - printLogSpacer(footerString) + if not isConciseInfoRequested(args): + printLogSpacer(footerString) rsmi_ret_ok(rocmsmi.rsmi_shut_down()) exit(RETCODE) diff --git a/projects/amdsmi/python_smi_tools/rsmiBindings.py.in b/projects/amdsmi/python_smi_tools/rsmiBindings.py.in index 284319e10f..36dbb6e5ff 100644 --- a/projects/amdsmi/python_smi_tools/rsmiBindings.py.in +++ b/projects/amdsmi/python_smi_tools/rsmiBindings.py.in @@ -655,3 +655,8 @@ rsmi_nps_mode_type = rsmi_nps_mode_type_t # nps_mode_type_l[rsmi_nps_mode_type_t.RSMI_MEMORY_PARTITION_NPS2] # will return string 'NPS2' nps_mode_type_l = ['NPS1', 'NPS2', 'NPS4', 'NPS8'] + +class rsmi_power_label(str, Enum): + AVG_POWER = '(Avg)' + CURRENT_SOCKET_POWER = '(Socket)' + diff --git a/projects/amdsmi/src/rocm_smi.cc b/projects/amdsmi/src/rocm_smi.cc index f35ab6a193..a0061ddd9b 100755 --- a/projects/amdsmi/src/rocm_smi.cc +++ b/projects/amdsmi/src/rocm_smi.cc @@ -77,7 +77,6 @@ #include "rocm_smi/rocm_smi64Config.h" #include "rocm_smi/rocm_smi_logger.h" -using namespace ROCmLogging; using namespace amd::smi; static const uint32_t kMaxOverdriveLevel = 20; @@ -2386,21 +2385,22 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, amd::smi::MonitorTypes mon_type = amd::smi::kMonInvalid; uint16_t val_ui16; - static const std::map kMetricTypeMap = { - { RSMI_TEMP_CURRENT, amd::smi::kMonTemp }, - { RSMI_TEMP_MAX, amd::smi::kMonTempMax }, - { RSMI_TEMP_MIN, amd::smi::kMonTempMin }, - { RSMI_TEMP_MAX_HYST, amd::smi::kMonTempMaxHyst }, - { RSMI_TEMP_MIN_HYST, amd::smi::kMonTempMinHyst }, - { RSMI_TEMP_CRITICAL, amd::smi::kMonTempCritical }, - { RSMI_TEMP_CRITICAL_HYST, amd::smi::kMonTempCriticalHyst }, - { RSMI_TEMP_EMERGENCY, amd::smi::kMonTempEmergency }, - { RSMI_TEMP_EMERGENCY_HYST, amd::smi::kMonTempEmergencyHyst }, - { RSMI_TEMP_CRIT_MIN, amd::smi::kMonTempCritMin }, - { RSMI_TEMP_CRIT_MIN_HYST, amd::smi::kMonTempCritMinHyst }, - { RSMI_TEMP_OFFSET, amd::smi::kMonTempOffset }, - { RSMI_TEMP_LOWEST, amd::smi::kMonTempLowest }, - { RSMI_TEMP_HIGHEST, amd::smi::kMonTempHighest }, + static const std::map + kMetricTypeMap = { + { RSMI_TEMP_CURRENT, amd::smi::kMonTemp }, + { RSMI_TEMP_MAX, amd::smi::kMonTempMax }, + { RSMI_TEMP_MIN, amd::smi::kMonTempMin }, + { RSMI_TEMP_MAX_HYST, amd::smi::kMonTempMaxHyst }, + { RSMI_TEMP_MIN_HYST, amd::smi::kMonTempMinHyst }, + { RSMI_TEMP_CRITICAL, amd::smi::kMonTempCritical }, + { RSMI_TEMP_CRITICAL_HYST, amd::smi::kMonTempCriticalHyst }, + { RSMI_TEMP_EMERGENCY, amd::smi::kMonTempEmergency }, + { RSMI_TEMP_EMERGENCY_HYST, amd::smi::kMonTempEmergencyHyst }, + { RSMI_TEMP_CRIT_MIN, amd::smi::kMonTempCritMin }, + { RSMI_TEMP_CRIT_MIN_HYST, amd::smi::kMonTempCritMinHyst }, + { RSMI_TEMP_OFFSET, amd::smi::kMonTempOffset }, + { RSMI_TEMP_LOWEST, amd::smi::kMonTempLowest }, + { RSMI_TEMP_HIGHEST, amd::smi::kMonTempHighest }, }; const auto mon_type_it = kMetricTypeMap.find(metric); @@ -2485,7 +2485,8 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type, return RSMI_STATUS_NOT_SUPPORTED; } - *temperature = static_cast(val_ui16) * CENTRIGRADE_TO_MILLI_CENTIGRADE; + *temperature = + static_cast(val_ui16) * CENTRIGRADE_TO_MILLI_CENTIGRADE; ss << __PRETTY_FUNCTION__ << " | ======= end ======= " << " | Success " @@ -2815,6 +2816,80 @@ rsmi_dev_power_ave_get(uint32_t dv_ind, uint32_t sensor_ind, uint64_t *power) { CATCH } +rsmi_status_t +rsmi_dev_current_socket_power_get(uint32_t dv_ind, uint64_t *socket_power) { + TRY + std::ostringstream ss; + rsmi_status_t rsmiReturn = RSMI_STATUS_NOT_SUPPORTED; + std::string val_str; + uint32_t sensor_ind = 1; // socket_power sysfs files have 1-based indices + MonitorTypes mon_type = amd::smi::kMonPowerInput; + ss << __PRETTY_FUNCTION__ << " | ======= start =======, dv_ind=" + << std::to_string(dv_ind); + LOG_TRACE(ss); + if (socket_power == nullptr) { + rsmiReturn = RSMI_STATUS_INVALID_ARGS; + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Fail " + << " | Device #: " << dv_ind + << " | Type: " << monitorTypesToString.at(mon_type) + << " | Cause: socket_power was a null ptr reference" + << " | Returning = " + << getRSMIStatusString(rsmiReturn) << " |"; + LOG_ERROR(ss); + return RSMI_STATUS_INVALID_ARGS; + } + CHK_SUPPORT_SUBVAR_ONLY(socket_power, sensor_ind) + DEVICE_MUTEX + + if (dev->monitor() == nullptr) { + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Fail " + << " | Device #: " << dv_ind + << " | Type: " << monitorTypesToString.at(mon_type) + << " | Cause: hwmon monitor was a null ptr reference" + << " | Returning = " + << getRSMIStatusString(rsmiReturn) << " |"; + LOG_ERROR(ss); + return rsmiReturn; + } + + int ret = dev->monitor()->readMonitor(amd::smi::kMonPowerLabel, + sensor_ind, &val_str); + if (ret || val_str != "PPT" || val_str.size() != 3) { + if (ret != 0) { + rsmiReturn = amd::smi::ErrnoToRsmiStatus(ret); + } + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Fail " + << " | Device #: " << dv_ind + << " | Type: " << monitorTypesToString.at(mon_type) + << " | Cause: readMonitor() returned an error status" + << " or Socket Power label did not show PPT or size of label data was" + << " unexpected" + << " | Returning = " + << getRSMIStatusString(rsmiReturn) << " |"; + LOG_ERROR(ss); + return rsmiReturn; + } + rsmiReturn = get_dev_mon_value(mon_type, dv_ind, sensor_ind, + socket_power); + ss << __PRETTY_FUNCTION__ + << " | ======= end ======= " + << " | Success " + << " | Device #: " << dv_ind + << " | Type: " << monitorTypesToString.at(mon_type) + << " | Data: " << *socket_power + << " | Returning = " + << getRSMIStatusString(rsmiReturn) << " |"; + LOG_TRACE(ss); + return rsmiReturn; + CATCH +} + rsmi_status_t rsmi_dev_energy_count_get(uint32_t dv_ind, uint64_t *power, float *counter_resolution, uint64_t *timestamp) { diff --git a/projects/amdsmi/src/rocm_smi_device.cc b/projects/amdsmi/src/rocm_smi_device.cc index 12f69dcec0..33d35bed0a 100755 --- a/projects/amdsmi/src/rocm_smi_device.cc +++ b/projects/amdsmi/src/rocm_smi_device.cc @@ -68,8 +68,6 @@ #include "rocm_smi/rocm_smi_logger.h" #include "shared_mutex.h" // NOLINT -using namespace ROCmLogging; - namespace amd { namespace smi { diff --git a/projects/amdsmi/src/rocm_smi_gpu_metrics.cc b/projects/amdsmi/src/rocm_smi_gpu_metrics.cc index d7aab133c3..16d629895d 100755 --- a/projects/amdsmi/src/rocm_smi_gpu_metrics.cc +++ b/projects/amdsmi/src/rocm_smi_gpu_metrics.cc @@ -61,7 +61,6 @@ #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_logger.h" -using namespace ROCmLogging; using namespace amd::smi; #define TRY try { diff --git a/projects/amdsmi/src/rocm_smi_logger.cc b/projects/amdsmi/src/rocm_smi_logger.cc index ccbb12c29a..c7fcce537f 100644 --- a/projects/amdsmi/src/rocm_smi_logger.cc +++ b/projects/amdsmi/src/rocm_smi_logger.cc @@ -71,9 +71,8 @@ #include "rocm_smi/rocm_smi_logger.h" #include "rocm_smi/rocm_smi_main.h" -using namespace ROCmLogging; -Logger* Logger::m_Instance = nullptr; +ROCmLogging::Logger *ROCmLogging::Logger::m_Instance = nullptr; // Log file name // WARNING: File name should be changed here and @@ -81,39 +80,39 @@ Logger* Logger::m_Instance = nullptr; // in one place will cause a mismatch in these scripts, // files may not have proper permissions, and logrotate // would not function properly. -const std::string logPath = "/var/log/rocm_smi_lib/"; -const std::string logBaseFName = "ROCm-SMI-lib"; -const std::string logExtension = ".log"; -const std::string logFileName = logPath + logBaseFName + logExtension; +#define LOGPATH "/var/log/rocm_smi_lib/" +#define LOGBASE_FNAME "ROCm-SMI-lib" +#define LOGEXTENSION ".log" +const char *logFileName = LOGPATH LOGBASE_FNAME LOGEXTENSION; -Logger::Logger() { +ROCmLogging::Logger::Logger() { initialize_resources(); } -Logger::~Logger() { +ROCmLogging::Logger::~Logger() { if (m_loggingIsOn) { destroy_resources(); } } -Logger* Logger::getInstance() throw() { +ROCmLogging::Logger* ROCmLogging::Logger::getInstance() throw() { if (m_Instance == nullptr) { - m_Instance = new Logger(); + m_Instance = new ROCmLogging::Logger(); } return m_Instance; } -void Logger::lock() { +void ROCmLogging::Logger::lock() { m_Lock.lock(); } -void Logger::unlock() { +void ROCmLogging::Logger::unlock() { m_Lock.unlock(); } -void Logger::logIntoFile(std::string& data) { +void ROCmLogging::Logger::logIntoFile(std::string& data) { lock(); - if(!m_File.is_open()) { + if (!m_File.is_open()) { initialize_resources(); if (!m_File.is_open()) { std::cout << "WARNING: re-initializing resources was unsuccessful." @@ -127,24 +126,24 @@ void Logger::logIntoFile(std::string& data) { unlock(); } -void Logger::logOnConsole(std::string& data) { +void ROCmLogging::Logger::logOnConsole(std::string& data) { std::cout << getCurrentTime() << " " << data << std::endl; } // Returns: In string format, YY-MM-DD HH:MM:SS.microseconds -std::string Logger::getCurrentTime(void) { - using namespace std::chrono; +std::string ROCmLogging::Logger::getCurrentTime(void) { std::string currentTime; // get current time - auto now = system_clock::now(); + auto now = std::chrono::system_clock::now(); // get number of milliseconds for the current second // (remainder after division into seconds) - auto ms = duration_cast(now.time_since_epoch()) % 1000000; + auto ms = std::chrono::duration_cast( + now.time_since_epoch()) % 1000000; // convert to std::time_t in order to convert to std::tm (broken time) - auto timer = system_clock::to_time_t(now); + auto timer = std::chrono::system_clock::to_time_t(now); // convert to broken time std::tm bt = *std::localtime(&timer); @@ -159,7 +158,7 @@ std::string Logger::getCurrentTime(void) { } // Interface for Error Log -void Logger::error(const char* text) throw() { +void ROCmLogging::Logger::error(const char* text) throw() { // By default, logging is disabled // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -182,18 +181,18 @@ void Logger::error(const char* text) throw() { } } -void Logger::error(std::string& text) throw() { +void ROCmLogging::Logger::error(std::string& text) throw() { error(text.data()); } -void Logger::error(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::error(std::ostringstream& stream) throw() { std::string text = stream.str(); error(text.data()); stream.str(""); } // Interface for Alarm Log -void Logger::alarm(const char* text) throw() { +void ROCmLogging::Logger::alarm(const char* text) throw() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -216,18 +215,18 @@ void Logger::alarm(const char* text) throw() { } } -void Logger::alarm(std::string& text) throw() { +void ROCmLogging::Logger::alarm(std::string& text) throw() { alarm(text.data()); } -void Logger::alarm(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::alarm(std::ostringstream& stream) throw() { std::string text = stream.str(); alarm(text.data()); stream.str(""); } // Interface for Always Log -void Logger::always(const char* text) throw() { +void ROCmLogging::Logger::always(const char* text) throw() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -250,18 +249,18 @@ void Logger::always(const char* text) throw() { } } -void Logger::always(std::string& text) throw() { +void ROCmLogging::Logger::always(std::string& text) throw() { always(text.data()); } -void Logger::always(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::always(std::ostringstream& stream) throw() { std::string text = stream.str(); always(text.data()); stream.str(""); } // Interface for Buffer Log -void Logger::buffer(const char* text) throw() { +void ROCmLogging::Logger::buffer(const char* text) throw() { // Buffer is the special case. So don't add log level // and timestamp in the buffer message. Just log the raw bytes. if ((m_LogType == FILE_LOG) && (m_LogLevel >= LOG_LEVEL_BUFFER)) { @@ -284,18 +283,18 @@ void Logger::buffer(const char* text) throw() { } } -void Logger::buffer(std::string& text) throw() { +void ROCmLogging::Logger::buffer(std::string& text) throw() { buffer(text.data()); } -void Logger::buffer(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::buffer(std::ostringstream& stream) throw() { std::string text = stream.str(); buffer(text.data()); stream.str(""); } // Interface for Info Log -void Logger::info(const char* text) throw() { +void ROCmLogging::Logger::info(const char* text) throw() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -318,18 +317,18 @@ void Logger::info(const char* text) throw() { } } -void Logger::info(std::string& text) throw() { +void ROCmLogging::Logger::info(std::string& text) throw() { info(text.data()); } -void Logger::info(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::info(std::ostringstream& stream) throw() { std::string text = stream.str(); info(text.data()); stream.str(""); } // Interface for Trace Log -void Logger::trace(const char* text) throw() { +void ROCmLogging::Logger::trace(const char* text) throw() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -352,18 +351,18 @@ void Logger::trace(const char* text) throw() { } } -void Logger::trace(std::string& text) throw() { +void ROCmLogging::Logger::trace(std::string& text) throw() { trace(text.data()); } -void Logger::trace(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::trace(std::ostringstream& stream) throw() { std::string text = stream.str(); trace(text.data()); stream.str(""); } // Interface for Debug Log -void Logger::debug(const char* text) throw() { +void ROCmLogging::Logger::debug(const char* text) throw() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -386,51 +385,53 @@ void Logger::debug(const char* text) throw() { } } -void Logger::debug(std::string& text) throw() { +void ROCmLogging::Logger::debug(std::string& text) throw() { debug(text.data()); } -void Logger::debug(std::ostringstream& stream) throw() { +void ROCmLogging::Logger::debug(std::ostringstream& stream) throw() { std::string text = stream.str(); debug(text.data()); stream.str(""); } // Interfaces to control log levels -void Logger::updateLogLevel(LogLevel logLevel) { +void ROCmLogging::Logger::updateLogLevel(LogLevel logLevel) { m_LogLevel = logLevel; } -void Logger::enableAllLogLevels() { +void ROCmLogging::Logger::enableAllLogLevels() { m_LogLevel = ENABLE_LOG; } // Disable all log levels, except error and alarm -void Logger::disableLog() { +void ROCmLogging::Logger::disableLog() { m_LogLevel = DISABLE_LOG; } // Interfaces to control log Types -void Logger::updateLogType(LogType logType) { +void ROCmLogging::Logger::updateLogType(LogType logType) { m_LogType = logType; } -void Logger::enableConsoleLogging() { +void ROCmLogging::Logger::enableConsoleLogging() { m_LogType = CONSOLE; } -void Logger::enableFileLogging() { +void ROCmLogging::Logger::enableFileLogging() { m_LogType = FILE_LOG; } // Returns a string of details on current log settings -std::string Logger::getLogSettings() { +std::string ROCmLogging::Logger::getLogSettings() { std::string logSettings; if (m_File.is_open()) { - logSettings += "OpenStatus = File (" + logFileName + ") is open"; + logSettings += "OpenStatus = File (" + std::string(logFileName) + + ") is open"; } else { - logSettings += "OpenStatus = File (" + logFileName + ") is not open"; + logSettings += "OpenStatus = File (" + std::string(logFileName) + + ") is not open"; } logSettings += ", "; @@ -480,11 +481,11 @@ std::string Logger::getLogSettings() { // Returns current reported enabled logging state. State is controlled by // user's environment variable RSMI_LOGGING. -bool Logger::isLoggerEnabled() { +bool ROCmLogging::Logger::isLoggerEnabled() { return m_loggingIsOn; } -void Logger::initialize_resources() { +void ROCmLogging::Logger::initialize_resources() { // By default, logging is disabled (ie. no RSMI_LOGGING) // The check below allows us to toggle logging through RSMI_LOGGING // set or unset @@ -492,7 +493,7 @@ void Logger::initialize_resources() { if (!m_loggingIsOn) { return; } - m_File.open(logFileName.c_str(), std::ios::out | std::ios::app); + m_File.open(logFileName, std::ios::out | std::ios::app); m_LogLevel = LOG_LEVEL_TRACE; // RSMI_LOGGING = 1, output to logs only // RSMI_LOGGING = 2, output to console only @@ -521,9 +522,9 @@ void Logger::initialize_resources() { if (m_File.fail()) { std::cout << "WARNING: Failed opening log file." << std::endl; } - chmod(logFileName.c_str(), S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH); + chmod(logFileName, S_IRUSR|S_IRGRP|S_IROTH|S_IWUSR|S_IWGRP|S_IWOTH); } -void Logger::destroy_resources() { +void ROCmLogging::Logger::destroy_resources() { m_File.close(); } diff --git a/projects/amdsmi/src/rocm_smi_main.cc b/projects/amdsmi/src/rocm_smi_main.cc index ab0ee435d2..d180253211 100755 --- a/projects/amdsmi/src/rocm_smi_main.cc +++ b/projects/amdsmi/src/rocm_smi_main.cc @@ -68,7 +68,6 @@ #include "rocm_smi/rocm_smi_kfd.h" #include "rocm_smi/rocm_smi_logger.h" -using namespace ROCmLogging; static const char *kPathDRMRoot = "/sys/class/drm"; static const char *kPathHWMonRoot = "/sys/class/hwmon"; @@ -314,12 +313,12 @@ RocmSMI::Initialize(uint64_t flags) { int i_ret; LOG_ALWAYS("=============== ROCM SMI initialize ================"); - Logger::getInstance()->enableAllLogLevels(); + ROCmLogging::Logger::getInstance()->enableAllLogLevels(); // Leaving below to allow developers to check current log settings // std::string logSettings = Logger::getInstance()->getLogSettings(); // std::cout << "Current log settings:\n" << logSettings << std::endl; - if (Logger::getInstance()->isLoggerEnabled()) { + if (ROCmLogging::Logger::getInstance()->isLoggerEnabled()) { logSystemDetails(); } diff --git a/projects/amdsmi/src/rocm_smi_monitor.cc b/projects/amdsmi/src/rocm_smi_monitor.cc index f0fe75fdbc..adcb2361cb 100755 --- a/projects/amdsmi/src/rocm_smi_monitor.cc +++ b/projects/amdsmi/src/rocm_smi_monitor.cc @@ -3,7 +3,7 @@ * The University of Illinois/NCSA * Open Source License (NCSA) * - * Copyright (c) 2017, Advanced Micro Devices, Inc. + * Copyright (c) 2017-2023, Advanced Micro Devices, Inc. * All rights reserved. * * Developed by: @@ -58,8 +58,6 @@ #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_logger.h" -using namespace ROCmLogging; - namespace amd { namespace smi { @@ -80,6 +78,8 @@ static const char *kMonPowerCapName = "power#_cap"; static const char *kMonPowerCapMaxName = "power#_cap_max"; static const char *kMonPowerCapMinName = "power#_cap_min"; static const char *kMonPowerAveName = "power#_average"; +static const char *kMonPowerInputName = "power#_input"; +static const char *kMonPowerLabelName = "power#_label"; static const char *kMonTempMaxName = "temp#_max"; static const char *kMonTempMinName = "temp#_min"; static const char *kMonTempMaxHystName = "temp#_max_hyst"; @@ -135,6 +135,8 @@ static const std::map kMonitorNameMap = { {kMonPowerCapMax, kMonPowerCapMaxName}, {kMonPowerCapMin, kMonPowerCapMinName}, {kMonPowerAve, kMonPowerAveName}, + {kMonPowerInput, kMonPowerInputName}, + {kMonPowerLabel, kMonPowerLabelName}, {kMonTempMax, kMonTempMaxName}, {kMonTempMin, kMonTempMinName}, {kMonTempMaxHyst, kMonTempMaxHystName}, @@ -202,7 +204,8 @@ static const std::map kMonFuncDependsMap = { .variants = {kMonInvalid}, } }, - {"rsmi_dev_power_cap_default_get", { .mandatory_depends = {kMonPowerCapDefaultName}, + {"rsmi_dev_power_cap_default_get", { .mandatory_depends = + {kMonPowerCapDefaultName}, .variants = {kMonInvalid}, } }, @@ -613,7 +616,7 @@ void Monitor::fillSupportedFuncs(SupportedFuncMap *supported_funcs) { supported_monitors = intersect; } if (!supported_monitors.empty()) { - for (unsigned long & supported_monitor : supported_monitors) { + for (uint64_t &supported_monitor : supported_monitors) { if (m_type == eDefaultMonitor) { assert(supported_monitor > 0); supported_monitor |= diff --git a/projects/amdsmi/src/rocm_smi_utils.cc b/projects/amdsmi/src/rocm_smi_utils.cc index 4b8f61a842..66944ba524 100755 --- a/projects/amdsmi/src/rocm_smi_utils.cc +++ b/projects/amdsmi/src/rocm_smi_utils.cc @@ -73,7 +73,6 @@ #include "rocm_smi/rocm_smi_device.h" #include "rocm_smi/rocm_smi_logger.h" -using namespace ROCmLogging; namespace amd { namespace smi { diff --git a/projects/amdsmi/tests/rocm_smi_test/functional/power_read.cc b/projects/amdsmi/tests/rocm_smi_test/functional/power_read.cc index a18cd70676..641ddde00e 100755 --- a/projects/amdsmi/tests/rocm_smi_test/functional/power_read.cc +++ b/projects/amdsmi/tests/rocm_smi_test/functional/power_read.cc @@ -53,6 +53,7 @@ #include "rocm_smi/rocm_smi.h" #include "rocm_smi_test/functional/power_read.h" #include "rocm_smi_test/test_common.h" +#include "rocm_smi/rocm_smi_utils.h" TestPowerRead::TestPowerRead() : TestBase() { set_title("RSMI Power Read Test"); @@ -116,27 +117,48 @@ void TestPowerRead::Run(void) { val_ui64 << " uW" << std::endl; } + /* Average Power */ err = rsmi_dev_power_ave_get(i, 0, &val_ui64); if (err == RSMI_STATUS_NOT_SUPPORTED) { std::cout << - "\t**Power average information is not supported for this device" + "\t**Average Power Usage: not supported on this device" << std::endl; - - // Verify api support checking functionality is working - err = rsmi_dev_power_ave_get(i, 0, nullptr); - ASSERT_EQ(err, RSMI_STATUS_NOT_SUPPORTED); - continue; - } - IF_VERB(STANDARD) { - std::cout << "\t**Average Power Usage: "; - CHK_RSMI_PERM_ERR(err) - if (err == RSMI_STATUS_SUCCESS) { - std::cout << static_cast(val_ui64)/1000 << " mW" << std::endl; + } else { + IF_VERB(STANDARD) { + std::cout << "\t**Average Power Usage: "; + CHK_RSMI_PERM_ERR(err) + if (err == RSMI_STATUS_SUCCESS) { + std::cout << static_cast(val_ui64) / 1000 << " mW" + << std::endl; + } + // Verify api support checking functionality is working + err = rsmi_dev_power_ave_get(i, 0, nullptr); + ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); } - // Verify api support checking functionality is working - err = rsmi_dev_power_ave_get(i, 0, nullptr); - ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); } + + /* Current Socket Power */ + err = rsmi_dev_current_socket_power_get(i, &val_ui64); + + if (err == RSMI_STATUS_NOT_SUPPORTED) { + std::cout << + "\t**Current Socket Power: not supported" + " on this device" << std::endl; + } else { + IF_VERB(STANDARD) { + std::cout << "\t**Current Socket Power: "; + CHK_RSMI_PERM_ERR(err) + if (err == RSMI_STATUS_SUCCESS) { + std::cout << static_cast(val_ui64) / 1000 << " mW" + << std::endl; + } + // Verify api support checking functionality is working + err = rsmi_dev_current_socket_power_get(i, nullptr); + // std::cout << "err = " << amd::smi::getRSMIStatusString(err); + ASSERT_EQ(err, RSMI_STATUS_INVALID_ARGS); + } + } + std::cout << "\n"; } } } From 10207f0a3afd692365a02480c8c7fe4afc1dba1a Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Sun, 24 Sep 2023 20:16:06 -0500 Subject: [PATCH 11/19] Remove old gpuv-smi symlink Change-Id: I39d9aa96eb8ed2dd6a4a222c3c4da9ff2b2de007 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: db3e119f7d7a653342cde07804d7f584d6be633a] --- projects/amdsmi/DEBIAN/prerm.in | 2 ++ projects/amdsmi/RPM/preun.in | 2 ++ 2 files changed, 4 insertions(+) diff --git a/projects/amdsmi/DEBIAN/prerm.in b/projects/amdsmi/DEBIAN/prerm.in index e0aae8f396..3a4ab7ff26 100755 --- a/projects/amdsmi/DEBIAN/prerm.in +++ b/projects/amdsmi/DEBIAN/prerm.in @@ -36,6 +36,8 @@ return_logrotateToOrigConfig() { case "$1" in ( remove | upgrade) + # remove old gpuv-smi symlink + rm -f @CPACK_PACKAGING_INSTALL_PREFIX@/bin/gpuv-smi &> /dev/null rm_ldconfig rm_pyc rm_logFolder diff --git a/projects/amdsmi/RPM/preun.in b/projects/amdsmi/RPM/preun.in index 2b8654046e..f607a4ac7c 100755 --- a/projects/amdsmi/RPM/preun.in +++ b/projects/amdsmi/RPM/preun.in @@ -27,6 +27,8 @@ return_logrotateToOrigConfig() { if [ "$1" -le 1 ]; then # perform the below actions for rpm remove($1=0) or upgrade($1=1) operations + # remove old gpuv-smi symlink + rm -f @CPACK_PACKAGING_INSTALL_PREFIX@/bin/gpuv-smi &> /dev/null rm_pyc rm_logFolder return_logrotateToOrigConfig From 821994b869161b3c732f6551b2099a3600698c80 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 16:12:30 -0500 Subject: [PATCH 12/19] Fixed log handling and exceptions Updated exceptions Added driver load exception Fixed logging override by removing previous log handlers Updated debug output to use gpu_id vs C-pointer Removed AmdSmiRetcode class in favor of using the wrapper directly Added traceback limits for clean errors (Not in debug) Change-Id: Ia02bb842b8f60d9ab4b68b7f8b1afda30b1c021c Signed-off-by: Maisam Arif [ROCm/amdsmi commit: 9281bfbbfaf255266c843edce0fcbf9983315125] --- projects/amdsmi/amdsmi_cli/README.md | 18 +- projects/amdsmi/amdsmi_cli/amdsmi_cli.py | 24 ++- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 189 +++++++++++------- projects/amdsmi/amdsmi_cli/amdsmi_init.py | 5 +- projects/amdsmi/amdsmi_cli/amdsmi_parser.py | 2 +- projects/amdsmi/py-interface/README.md | 2 +- projects/amdsmi/py-interface/__init__.py | 1 - .../amdsmi/py-interface/amdsmi_exception.py | 104 ++++------ 8 files changed, 188 insertions(+), 157 deletions(-) diff --git a/projects/amdsmi/amdsmi_cli/README.md b/projects/amdsmi/amdsmi_cli/README.md index 0805064c99..f687cd63b8 100644 --- a/projects/amdsmi/amdsmi_cli/README.md +++ b/projects/amdsmi/amdsmi_cli/README.md @@ -102,7 +102,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ``` bash @@ -123,7 +123,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -153,7 +153,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -176,7 +176,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -215,7 +215,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -245,7 +245,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -271,7 +271,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -296,7 +296,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -323,7 +323,7 @@ Command Modifiers: --json Displays output in JSON format (human readable by default). --csv Displays output in CSV format (human readable by default). --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ## Disclaimer diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py index a696a774a8..d1232b2b26 100755 --- a/projects/amdsmi/amdsmi_cli/amdsmi_cli.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_cli.py @@ -66,14 +66,22 @@ if __name__ == "__main__": amd_smi_commands.logger.format = amd_smi_commands.logger.LoggerFormat.csv.value if args.file: amd_smi_commands.logger.destination = args.file - if args.loglevel: - logging_dict = {'DEBUG' : logging.DEBUG, - 'INFO' : logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL} - # Enable debug logs on amdsmi library ie. RSMI_LOGGING = 1 in environment or otherwise - logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_dict[args.loglevel]) + + # Remove previous log handlers + for handler in logging.root.handlers[:]: + logging.root.removeHandler(handler) + + logging_dict = {'DEBUG' : logging.DEBUG, + 'INFO' : logging.INFO, + 'WARNING': logging.WARNING, + 'ERROR': logging.ERROR, + 'CRITICAL': logging.CRITICAL} + # To enable debug logs on rocm-smi library set RSMI_LOGGING = 1 in environment + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_dict[args.loglevel]) + + # Disable traceback for non-debug log levels + if args.loglevel != "DEBUG": + sys.tracebacklimit = -1 # Execute subcommands args.func(args) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index 179296ff40..f5020d708f 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -191,6 +191,9 @@ class AMDSMICommands(): static_dict = {} + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + if args.asic: try: asic_info = amdsmi_interface.amdsmi_get_gpu_asic_info(args.gpu) @@ -203,7 +206,7 @@ class AMDSMICommands(): static_dict['asic'] = asic_info except amdsmi_exception.AmdSmiLibraryException as e: static_dict['asic'] = "N/A" - logging.debug("Failed to get asic info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get asic info for gpu %s | %s", gpu_id, e.get_error_info()) if args.bus: bus_output_info = {} @@ -239,13 +242,13 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: bus_info = "N/A" - logging.debug("Failed to get bus info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get bus info for gpu %s | %s", gpu_id, e.get_error_info()) try: bus_output_info['bdf'] = amdsmi_interface.amdsmi_get_gpu_device_bdf(args.gpu) except amdsmi_exception.AmdSmiLibraryException as e: bus_output_info['bdf'] = "N/A" - logging.debug("Failed to get bdf for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get bdf for gpu %s | %s", gpu_id, e.get_error_info()) bus_output_info.update(bus_info) static_dict['bus'] = bus_output_info @@ -255,7 +258,7 @@ class AMDSMICommands(): static_dict['vbios'] = vbios_info except amdsmi_exception.AmdSmiLibraryException as e: static_dict['vbios'] = "N/A" - logging.debug("Failed to get vbios info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get vbios info for gpu %s | %s", gpu_id, e.get_error_info()) if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.board: @@ -271,7 +274,7 @@ class AMDSMICommands(): static_dict['board'] = board_info except amdsmi_exception.AmdSmiLibraryException as e: static_dict['board'] = "N/A" - logging.debug("Failed to get board info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get board info for gpu %s | %s", gpu_id, e.get_error_info()) if args.limit: # Power limits try: @@ -283,7 +286,7 @@ class AMDSMICommands(): power_limit_error = True max_power_limit = "N/A" current_power_limit = "N/A" - logging.debug("Failed to get power cap info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get power cap info for gpu %s | %s", gpu_id, e.get_error_info()) # Edge temperature limits try: @@ -293,7 +296,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: slowdown_temp_edge_limit_error = True slowdown_temp_edge_limit = "N/A" - logging.debug("Failed to get edge temperature slowdown metric for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get edge temperature slowdown metric for gpu %s | %s", gpu_id, e.get_error_info()) if slowdown_temp_edge_limit == 0: slowdown_temp_edge_limit_error = True @@ -306,7 +309,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: shutdown_temp_edge_limit_error = True shutdown_temp_edge_limit = "N/A" - logging.debug("Failed to get edge temperature shutdown metrics for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get edge temperature shutdown metrics for gpu %s | %s", gpu_id, e.get_error_info()) if shutdown_temp_edge_limit == 0: shutdown_temp_edge_limit_error = True @@ -320,7 +323,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: slowdown_temp_hotspot_limit_error = True slowdown_temp_hotspot_limit = "N/A" - logging.debug("Failed to get hotspot temperature slowdown metrics for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get hotspot temperature slowdown metrics for gpu %s | %s", gpu_id, e.get_error_info()) try: shutdown_temp_hotspot_limit_error = False @@ -329,7 +332,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: shutdown_temp_hotspot_limit_error = True shutdown_temp_hotspot_limit = "N/A" - logging.debug("Failed to get hotspot temperature shutdown metrics for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get hotspot temperature shutdown metrics for gpu %s | %s", gpu_id, e.get_error_info()) # VRAM temperature limits @@ -340,7 +343,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: slowdown_temp_vram_limit_error = True slowdown_temp_vram_limit = "N/A" - logging.debug("Failed to get vram temperature slowdown metrics for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get vram temperature slowdown metrics for gpu %s | %s", gpu_id, e.get_error_info()) try: shutdown_temp_vram_limit_error = False @@ -349,7 +352,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: shutdown_temp_vram_limit_error = True shutdown_temp_vram_limit = "N/A" - logging.debug("Failed to get vram temperature shutdown metrics for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get vram temperature shutdown metrics for gpu %s | %s", gpu_id, e.get_error_info()) if self.logger.is_human_readable_format(): unit = 'W' @@ -392,7 +395,7 @@ class AMDSMICommands(): static_dict['driver'] = driver_info except amdsmi_exception.AmdSmiLibraryException as e: static_dict['driver'] = "N/A" - logging.debug("Failed to get driver info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get driver info for gpu %s | %s", gpu_id, e.get_error_info()) if self.helpers.is_hypervisor() or self.helpers.is_baremetal(): if args.ras: @@ -400,7 +403,7 @@ class AMDSMICommands(): static_dict['ras'] = amdsmi_interface.amdsmi_get_gpu_ras_block_features_enabled(args.gpu) except amdsmi_exception.AmdSmiLibraryException as e: static_dict['ras'] = "N/A" - logging.debug("Failed to get ras block features for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get ras block features for gpu %s | %s", gpu_id, e.get_error_info()) if args.vram: try: vram_info = amdsmi_interface.amdsmi_get_gpu_vram_info(args.gpu) @@ -417,7 +420,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: vram_info = "N/A" - logging.debug("Failed to get vram info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get vram info for gpu %s | %s", gpu_id, e.get_error_info()) static_dict['vram'] = vram_info @@ -427,13 +430,13 @@ class AMDSMICommands(): numa_node_number = amdsmi_interface.amdsmi_topo_get_numa_node_number(args.gpu) except amdsmi_exception.AmdSmiLibraryException as e: numa_node_number = "N/A" - logging.debug("Failed to get numa node number for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get numa node number for gpu %s | %s", gpu_id, e.get_error_info()) try: numa_affinity = amdsmi_interface.amdsmi_get_gpu_topo_numa_affinity(args.gpu) except amdsmi_exception.AmdSmiLibraryException as e: numa_affinity = "N/A" - logging.debug("Failed to get numa affinity for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get numa affinity for gpu %s | %s", gpu_id, e.get_error_info()) static_dict['numa'] = {'node' : numa_node_number, 'affinity' : numa_affinity} @@ -500,6 +503,10 @@ class AMDSMICommands(): args.gpu = device_handle fw_list = {} + + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + if args.fw_list: try: fw_info = amdsmi_interface.amdsmi_get_fw_info(args.gpu) @@ -519,7 +526,7 @@ class AMDSMICommands(): fw_list.update(fw_info) except amdsmi_exception.AmdSmiLibraryException as e: fw_list['fw_list'] = "N/A" - logging.debug("Failed to get firmware info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get firmware info for gpu %s | %s", gpu_id, e.get_error_info()) multiple_devices_csv_override = False # Convert and store output by pid for csv format @@ -586,13 +593,16 @@ class AMDSMICommands(): values_dict = {} bad_page_err_output = '' + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + try: bad_page_info = amdsmi_interface.amdsmi_get_gpu_bad_page_info(args.gpu) bad_page_error = False except amdsmi_exception.AmdSmiLibraryException as e: bad_page_error = True bad_page_err_output = "N/A" - logging.debug("Failed to get bad page info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get bad page info for gpu %s | %s", gpu_id, e.get_error_info()) if bad_page_info == "No bad pages found.": bad_page_error = True @@ -798,6 +808,10 @@ class AMDSMICommands(): # Add timestamp and store values for specified arguments values_dict = {} + + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.usage: try: @@ -818,7 +832,7 @@ class AMDSMICommands(): values_dict['usage'] = engine_usage except amdsmi_exception.AmdSmiLibraryException as e: values_dict['usage'] = "N/A" - logging.debug("Failed to get gpu activity for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get gpu activity for gpu %s | %s", gpu_id, e.get_error_info()) if args.power: power_dict = {'current_power': "N/A", 'current_gfx_voltage': "N/A", @@ -845,7 +859,7 @@ class AMDSMICommands(): power_dict['power_limit'] = power_info['power_limit'] except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get power info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get power info for gpu %s | %s", gpu_id, e.get_error_info()) try: is_power_management_enabled = amdsmi_interface.amdsmi_is_gpu_power_management_enabled(args.gpu) @@ -854,7 +868,7 @@ class AMDSMICommands(): else: power_dict['power_management'] = "DISABLED" except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get power management status for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get power management status for gpu %s | %s", gpu_id, e.get_error_info()) values_dict['power'] = power_dict if args.clock: @@ -870,20 +884,20 @@ class AMDSMICommands(): clocks['gfx'] = gfx_clock except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get gfx clock info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get gfx clock info for gpu %s | %s", gpu_id, e.get_error_info()) try: # is_clk_locked = amdsmi_interface.amdsmi_is_clk_locked(args.gpu, amdsmi_interface.AmdSmiClkType.GFX) is_clk_locked = "N/A" except amdsmi_exception.AmdSmiLibraryException as e: is_clk_locked = "N/A" - logging.debug("Failed to get gfx clock lock status info for gpu %s | %s", args.gpu, e.get_error_info()) - + logging.debug("Failed to get gfx clock lock status info for gpu %s | %s", gpu_id, e.get_error_info()) + if isinstance(clocks['gfx'], dict): clocks['gfx']['is_clk_locked'] = is_clk_locked else: clocks['gfx'] = {'is_clk_locked': is_clk_locked} - + try: mem_clock = amdsmi_interface.amdsmi_get_clock_info(args.gpu, amdsmi_interface.AmdSmiClkType.MEM) @@ -894,7 +908,7 @@ class AMDSMICommands(): clocks['mem'] = mem_clock except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get mem clock info for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get mem clock info for gpu %s | %s", gpu_id, e.get_error_info()) values_dict['clock'] = clocks if args.temperature: @@ -903,14 +917,14 @@ class AMDSMICommands(): args.gpu, amdsmi_interface.AmdSmiTemperatureType.EDGE, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) except amdsmi_exception.AmdSmiLibraryException as e: temperature_edge_current = "N/A" - logging.debug("Failed to get current edge temperature for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get current edge temperature for gpu %s | %s", gpu_id, e.get_error_info()) try: temperature_edge_limit = amdsmi_interface.amdsmi_get_temp_metric( args.gpu, amdsmi_interface.AmdSmiTemperatureType.EDGE, amdsmi_interface.AmdSmiTemperatureMetric.CRITICAL) except amdsmi_exception.AmdSmiLibraryException as e: temperature_edge_limit = "N/A" - logging.debug("Failed to get edge temperature limit for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get edge temperature limit for gpu %s | %s", gpu_id, e.get_error_info()) # If edge limit is reporting 0 then set the current edge temp to N/A if temperature_edge_limit == 0: @@ -921,14 +935,14 @@ class AMDSMICommands(): args.gpu, amdsmi_interface.AmdSmiTemperatureType.HOTSPOT, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) except amdsmi_exception.AmdSmiLibraryException as e: temperature_hotspot_current = "N/A" - logging.debug("Failed to get current hotspot temperature for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get current hotspot temperature for gpu %s | %s", gpu_id, e.get_error_info()) try: temperature_vram_current = amdsmi_interface.amdsmi_get_temp_metric( args.gpu, amdsmi_interface.AmdSmiTemperatureType.VRAM, amdsmi_interface.AmdSmiTemperatureMetric.CURRENT) except amdsmi_exception.AmdSmiLibraryException as e: temperature_vram_current = "N/A" - logging.debug("Failed to get current vram temperature for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get current vram temperature for gpu %s | %s", gpu_id, e.get_error_info()) temperatures = {'edge': temperature_edge_current, 'hotspot': temperature_hotspot_current, @@ -950,7 +964,7 @@ class AMDSMICommands(): except amdsmi_exception.AmdSmiLibraryException as e: ecc_count['correctable'] = "N/A" ecc_count['uncorrectable'] = "N/A" - logging.debug("Failed to get ecc count for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get ecc count for gpu %s | %s", gpu_id, e.get_error_info()) values_dict['ecc'] = ecc_count if args.ecc_block: @@ -966,14 +980,14 @@ class AMDSMICommands(): 'uncorrectable': ecc_count['uncorrectable_count']} except amdsmi_exception.AmdSmiLibraryException as e: ecc_count = "N/A" - logging.debug("Failed to get ecc count for gpu %s at block %s | %s", args.gpu, gpu_block, e.get_error_info()) + logging.debug("Failed to get ecc count for gpu %s at block %s | %s", gpu_id, gpu_block, e.get_error_info()) ecc_dict[state['block']] = {'correctable' : ecc_count, 'uncorrectable': ecc_count} values_dict['ecc_block'] = ecc_dict except amdsmi_exception.AmdSmiLibraryException as e: values_dict['ecc_block'] = "N/A" - logging.debug("Failed to get ecc block features for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get ecc block features for gpu %s | %s", gpu_id, e.get_error_info()) if args.pcie: pcie_dict = {'current_width': "N/A", 'current_speed': "N/A", @@ -997,13 +1011,13 @@ class AMDSMICommands(): unit = 'GT/s' pcie_link_status['current_speed'] = f"{pcie_link_status['pcie_speed']} {unit}" except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get pcie link status for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get pcie link status for gpu %s | %s", gpu_id, e.get_error_info()) try: pci_replay_counter = amdsmi_interface.amdsmi_get_gpu_pci_replay_counter(args.gpu) pcie_dict['replay_count'] = pci_replay_counter except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get pci replay counter for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get pci replay counter for gpu %s | %s", gpu_id, e.get_error_info()) try: pcie_bw = amdsmi_interface.amdsmi_get_gpu_pci_throughput(args.gpu) @@ -1024,7 +1038,7 @@ class AMDSMICommands(): pcie_dict['current_bandwith_received'] = received pcie_dict['max_packet_size'] = pcie_bw['max_pkt_sz'] except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get pcie bandwidth for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get pcie bandwidth for gpu %s | %s", gpu_id, e.get_error_info()) values_dict['pcie'] = pcie_dict if args.fan: @@ -1077,7 +1091,7 @@ class AMDSMICommands(): values_dict['voltage_curve'] = voltage_point_dict except amdsmi_exception.AmdSmiLibraryException as e: values_dict['voltage_curve'] = "N/A" - logging.debug("Failed to get voltage curve for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get voltage curve for gpu %s | %s", gpu_id, e.get_error_info()) if args.overdrive: try: overdrive_level = amdsmi_interface.amdsmi_get_gpu_overdrive_level(args.gpu) @@ -1089,14 +1103,14 @@ class AMDSMICommands(): values_dict['overdrive'] = overdrive_level except amdsmi_exception.AmdSmiLibraryException as e: values_dict['overdrive'] = "N/A" - logging.debug("Failed to get overdrive level for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get overdrive level for gpu %s | %s", gpu_id, e.get_error_info()) if args.perf_level: try: perf_level = amdsmi_interface.amdsmi_get_gpu_perf_level(args.gpu) values_dict['perf_level'] = perf_level except amdsmi_exception.AmdSmiLibraryException as e: values_dict['perf_level'] = "N/A" - logging.debug("Failed to get perf level for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get perf level for gpu %s | %s", gpu_id, e.get_error_info()) if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.xgmi_err: @@ -1104,7 +1118,7 @@ class AMDSMICommands(): values_dict['xgmi_err'] = amdsmi_interface.amdsmi_gpu_xgmi_error_status(args.gpu) except amdsmi_interface.AmdSmiLibraryException as e: values_dict['xgmi_err'] = "N/A" - logging.debug("Failed to get xgmi error status for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get xgmi error status for gpu %s | %s", gpu_id, e.get_error_info()) if args.energy: try: energy_dict = amdsmi_interface.amdsmi_get_energy_count(args.gpu) @@ -1140,19 +1154,19 @@ class AMDSMICommands(): total_vram = amdsmi_interface.amdsmi_get_gpu_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.VRAM) memory_usage['total_vram'] = total_vram // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get total VRAM memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get total VRAM memory for gpu %s | %s", gpu_id, e.get_error_info()) try: total_visible_vram = amdsmi_interface.amdsmi_get_gpu_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.VIS_VRAM) memory_usage['total_visible_vram'] = total_visible_vram // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get total VIS VRAM memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get total VIS VRAM memory for gpu %s | %s", gpu_id, e.get_error_info()) try: total_gtt = amdsmi_interface.amdsmi_get_gpu_memory_total(args.gpu, amdsmi_interface.AmdSmiMemoryType.GTT) memory_usage['total_gtt'] = total_gtt // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get total GTT memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get total GTT memory for gpu %s | %s", gpu_id, e.get_error_info()) # Used VRAM try: @@ -1160,19 +1174,19 @@ class AMDSMICommands(): memory_usage['used_vram'] = used_vram // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get used VRAM memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get used VRAM memory for gpu %s | %s", gpu_id, e.get_error_info()) try: used_visible_vram = amdsmi_interface.amdsmi_get_gpu_memory_usage(args.gpu, amdsmi_interface.AmdSmiMemoryType.VIS_VRAM) memory_usage['used_visible_vram'] = used_visible_vram // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get used VIS VRAM memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get used VIS VRAM memory for gpu %s | %s", gpu_id, e.get_error_info()) try: used_gtt = amdsmi_interface.amdsmi_get_gpu_memory_usage(args.gpu, amdsmi_interface.AmdSmiMemoryType.GTT) memory_usage['used_gtt'] = used_gtt // (1024*1024) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get used GTT memory for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get used GTT memory for gpu %s | %s", gpu_id, e.get_error_info()) # Free VRAM if memory_usage['total_vram'] != "N/A" and memory_usage['used_vram'] != "N/A": @@ -1288,11 +1302,14 @@ class AMDSMICommands(): else: raise IndexError("args.gpu should not be an empty list") + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + # Populate initial processes try: process_list = amdsmi_interface.amdsmi_get_gpu_process_list(args.gpu) except amdsmi_exception.AmdSmiLibraryException as e: - logging.debug("Failed to get process list for gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to get process list for gpu %s | %s", gpu_id, e.get_error_info()) raise e filtered_process_values = [] @@ -1301,7 +1318,7 @@ class AMDSMICommands(): process_info = amdsmi_interface.amdsmi_get_gpu_process_info(args.gpu, process_handle) except amdsmi_exception.AmdSmiLibraryException as e: process_info = "N/A" - logging.debug("Failed to get process info for gpu %s on process_handle %s | %s", args.gpu, process_handle, e.get_error_info()) + logging.debug("Failed to get process info for gpu %s on process_handle %s | %s", gpu_id, process_handle, e.get_error_info()) filtered_process_values.append({'process_info': process_info}) continue @@ -1467,7 +1484,10 @@ class AMDSMICommands(): src_gpu_links[dest_gpu_key] = bool(dest_gpu_link_status) except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_links[dest_gpu_key] = "N/A" - logging.debug("Failed to get link status for %s to %s | %s", src_gpu, dest_gpu, e.get_error_info()) + logging.debug("Failed to get link status for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) topo_values[src_gpu_index]['link_accessibility'] = src_gpu_links @@ -1487,7 +1507,10 @@ class AMDSMICommands(): src_gpu_weight[dest_gpu_key] = dest_gpu_link_weight except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_weight[dest_gpu_key] = "N/A" - logging.debug("Failed to get link weight for %s to %s | %s", src_gpu, dest_gpu, e.get_error_info()) + logging.debug("Failed to get link weight for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) topo_values[src_gpu_index]['weight'] = src_gpu_weight @@ -1507,7 +1530,10 @@ class AMDSMICommands(): src_gpu_hops[dest_gpu_key] = dest_gpu_hops except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_hops[dest_gpu_key] = "N/A" - logging.debug("Failed to get link hops for %s to %s | %s", src_gpu, dest_gpu, e.get_error_info()) + logging.debug("Failed to get link hops for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) topo_values[src_gpu_index]['hops'] = src_gpu_hops @@ -1532,7 +1558,10 @@ class AMDSMICommands(): src_gpu_link_type[dest_gpu_key] = "XGMI" except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = "N/A" - logging.debug("Failed to get link type for %s to %s | %s", src_gpu, dest_gpu, e.get_error_info()) + logging.debug("Failed to get link type for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) topo_values[src_gpu_index]['link_type'] = src_gpu_link_type @@ -1556,7 +1585,10 @@ class AMDSMICommands(): continue except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = "N/A" - logging.debug("Failed to get link type for %s to %s | %s", src_gpu, dest_gpu, e.get_error_info()) + logging.debug("Failed to get link type for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) try: min_bw = amdsmi_interface.amdsmi_get_minmax_bandwidth_between_processors(src_gpu, dest_gpu)['min_bandwidth'] @@ -1565,6 +1597,10 @@ class AMDSMICommands(): src_gpu_link_type[dest_gpu_key] = f'{min_bw}-{max_bw}' except amdsmi_exception.AmdSmiLibraryException as e: src_gpu_link_type[dest_gpu_key] = e.get_error_info() + logging.debug("Failed to get min max bandwidth for %s to %s | %s", + self.helpers.get_gpu_id_from_device_handle(src_gpu), + self.helpers.get_gpu_id_from_device_handle(dest_gpu), + e.get_error_info()) topo_values[src_gpu_index]['numa_bandwidth'] = src_gpu_link_type @@ -1638,7 +1674,7 @@ class AMDSMICommands(): try: amdsmi_interface.amdsmi_set_gpu_fan_speed(args.gpu, 0, args.fan) except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e raise ValueError(f"Unable to set fan speed {args.fan} on {gpu_string}") from e @@ -1648,7 +1684,7 @@ class AMDSMICommands(): try: amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, perf_level) except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e raise ValueError(f"Unable to set performance level {args.perflevel} on {gpu_string}") from e @@ -1659,7 +1695,7 @@ class AMDSMICommands(): try: amdsmi_interface.amdsmi_set_gpu_perf_determinism_mode(args.gpu, args.perfdeterminism) except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e raise ValueError(f"Unable to set performance determinism and clock frequency to {args.perfdeterminism} on {gpu_string}") from e @@ -1721,13 +1757,16 @@ class AMDSMICommands(): args.gpu = device_handle + # Get gpu_id for logging + gpu_id = self.helpers.get_gpu_id_from_device_handle(args.gpu) + if args.gpureset: if self.helpers.is_amd_device(args.gpu): try: amdsmi_interface.amdsmi_reset_gpu(args.gpu) result = 'Successfully reset GPU' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e result = "Failed to reset GPU" else: @@ -1742,30 +1781,30 @@ class AMDSMICommands(): amdsmi_interface.amdsmi_set_gpu_overdrive_level(args.gpu, 0) reset_clocks_results['overdrive'] = 'Overdrive set to 0' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e reset_clocks_results['overdrive'] = "N/A" - logging.debug("Failed to reset overdrive on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset overdrive on gpu %s | %s", gpu_id, e.get_error_info()) try: level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) reset_clocks_results['clocks'] = 'Successfully reset clocks' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e reset_clocks_results['clocks'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) try: level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) reset_clocks_results['performance'] = 'Performance level reset to auto' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e reset_clocks_results['performance'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) self.logger.store_output(args.gpu, 'reset_clocks', reset_clocks_results) if args.fans: @@ -1773,10 +1812,10 @@ class AMDSMICommands(): amdsmi_interface.amdsmi_reset_gpu_fan(args.gpu, 0) result = 'Successfully reset fan speed to driver control' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e result = "N/A" - logging.debug("Failed to reset fans on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset fans on gpu %s | %s", gpu_id, e.get_error_info()) self.logger.store_output(args.gpu, 'reset_fans', result) if args.profile: @@ -1787,20 +1826,20 @@ class AMDSMICommands(): amdsmi_interface.amdsmi_set_gpu_power_profile(args.gpu, 0, power_profile_mask) reset_profile_results['power_profile'] = 'Successfully reset Power Profile' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e reset_profile_results['power_profile'] = "N/A" - logging.debug("Failed to reset power profile on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset power profile on gpu %s | %s", gpu_id, e.get_error_info()) try: level_auto = amdsmi_interface.AmdSmiDevPerfLevel.AUTO amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) reset_profile_results['performance_level'] = 'Successfully reset Performance Level' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e reset_profile_results['performance_level'] = "N/A" - logging.debug("Failed to reset perf level on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset perf level on gpu %s | %s", gpu_id, e.get_error_info()) self.logger.store_output(args.gpu, 'reset_profile', reset_profile_results) if args.xgmierr: @@ -1808,10 +1847,10 @@ class AMDSMICommands(): amdsmi_interface.amdsmi_reset_gpu_xgmi_error(args.gpu) result = 'Successfully reset XGMI Error count' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e result = "N/A" - logging.debug("Failed to reset xgmi error count on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to reset xgmi error count on gpu %s | %s", gpu_id, e.get_error_info()) self.logger.store_output(args.gpu, 'reset_xgmi_err', result) if args.perfdeterminism: try: @@ -1819,10 +1858,10 @@ class AMDSMICommands(): amdsmi_interface.amdsmi_set_gpu_perf_level(args.gpu, level_auto) result = 'Successfully disabled performance determinism' except amdsmi_exception.AmdSmiLibraryException as e: - if e.get_error_code() == amdsmi_exception.AmdSmiRetCode.STATUS_NO_PERM: + if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM: raise PermissionError('Command requires elevation') from e result = "N/A" - logging.debug("Failed to set perf level on gpu %s | %s", args.gpu, e.get_error_info()) + logging.debug("Failed to set perf level on gpu %s | %s", gpu_id, e.get_error_info()) self.logger.store_output(args.gpu, 'reset_perf_determinism', result) @@ -1857,7 +1896,7 @@ class AMDSMICommands(): commands.logger.store_output(device, 'values', values_dict) commands.logger.print_output() except amdsmi_exception.AmdSmiLibraryException as e: - if e.err_code != amdsmi_exception.AmdSmiRetCode.STATUS_NO_DATA: + if e.err_code != amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_DATA: print(e) except Exception as e: print(e) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_init.py b/projects/amdsmi/amdsmi_cli/amdsmi_init.py index 5f201d7d15..7e837026cc 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_init.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_init.py @@ -37,6 +37,8 @@ from amdsmi import amdsmi_exception # Using basic python logging for user errors and development logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.ERROR) # User level logging +# This traceback limit only affects this file, once the code hit's the cli portion it get's reset to the user's preference +sys.tracebacklimit = -1 # Disable traceback for user errors # On initial import set initialized variable AMDSMI_INITIALIZED = False @@ -66,8 +68,7 @@ def init_amdsmi(flag=amdsmi_interface.AmdSmiInitFlags.INIT_AMD_GPUS): amdsmi_interface.amdsmi_init(flag) except (amdsmi_interface.AmdSmiLibraryException, amdsmi_interface.AmdSmiParameterException) as err: raise err - - logging.info('AMDSMI initialized successfully') # without errors really + logging.debug('AMDSMI initialized successfully') else: logging.error('Driver not initialized (amdgpu not found in modules)') exit(-1) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py index e58344598a..65cb7bd85c 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py @@ -209,7 +209,7 @@ class AMDSMIParser(argparse.ArgumentParser): json_help = "Displays output in JSON format (human readable by default)." csv_help = "Displays output in CSV format (human readable by default)." file_help = "Saves output into a file on the provided path (stdout by default)." - loglevel_help = "Set the logging level for the parser commands" + loglevel_help = "Set the logging level for the parser commands (ERROR by default)." command_modifier_group = subcommand_parser.add_argument_group('Command Modifiers') diff --git a/projects/amdsmi/py-interface/README.md b/projects/amdsmi/py-interface/README.md index ca39ab110c..6efba3b16d 100644 --- a/projects/amdsmi/py-interface/README.md +++ b/projects/amdsmi/py-interface/README.md @@ -60,7 +60,7 @@ try: print("No GPUs on machine") except AmdSmiException as e: print("Error code: {}".format(e.err_code)) - if e.err_code == AmdSmiRetCode.STATUS_RETRY: + if e.err_code == amdsmi_wrapper.AMDSMI_STATUS_RETRY: print("Error info: {}".format(e.err_info)) ``` diff --git a/projects/amdsmi/py-interface/__init__.py b/projects/amdsmi/py-interface/__init__.py index 050a25a2c1..6be5f36436 100644 --- a/projects/amdsmi/py-interface/__init__.py +++ b/projects/amdsmi/py-interface/__init__.py @@ -194,4 +194,3 @@ from .amdsmi_exception import AmdSmiKeyException from .amdsmi_exception import AmdSmiBdfFormatException from .amdsmi_exception import AmdSmiTimeoutException from .amdsmi_exception import AmdSmiException -from .amdsmi_exception import AmdSmiRetCode diff --git a/projects/amdsmi/py-interface/amdsmi_exception.py b/projects/amdsmi/py-interface/amdsmi_exception.py index 046659e459..162e4f7a00 100644 --- a/projects/amdsmi/py-interface/amdsmi_exception.py +++ b/projects/amdsmi/py-interface/amdsmi_exception.py @@ -22,40 +22,9 @@ from enum import IntEnum from . import amdsmi_wrapper -class AmdSmiRetCode(IntEnum): - SUCCESS = amdsmi_wrapper.AMDSMI_STATUS_SUCCESS - STATUS_INVAL = amdsmi_wrapper.AMDSMI_STATUS_INVAL - STATUS_NOT_SUPPORTED = amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED - STATUS_FILE_ERROR = amdsmi_wrapper.AMDSMI_STATUS_FILE_ERROR - STATUS_NO_PERM = amdsmi_wrapper.AMDSMI_STATUS_NO_PERM - STATUS_OUT_OF_RESOURCES = amdsmi_wrapper.AMDSMI_STATUS_OUT_OF_RESOURCES - STATUS_INTERNAL_EXCEPTION = amdsmi_wrapper.AMDSMI_STATUS_INTERNAL_EXCEPTION - STATUS_INPUT_OUT_OF_BOUNDS = amdsmi_wrapper.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS - STATUS_INIT_ERROR = amdsmi_wrapper.AMDSMI_STATUS_INIT_ERROR - STATUS_NOT_YET_IMPLEMENTED = amdsmi_wrapper.AMDSMI_STATUS_NOT_YET_IMPLEMENTED - STATUS_NOT_FOUND = amdsmi_wrapper.AMDSMI_STATUS_NOT_FOUND - STATUS_INSUFFICIENT_SIZE = amdsmi_wrapper.AMDSMI_STATUS_INSUFFICIENT_SIZE - STATUS_INTERRUPT = amdsmi_wrapper.AMDSMI_STATUS_INTERRUPT - STATUS_UNEXPECTED_SIZE = amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_SIZE - STATUS_NO_DATA = amdsmi_wrapper.AMDSMI_STATUS_NO_DATA - STATUS_UNEXPECTED_DATA = amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA - STATUS_BUSY = amdsmi_wrapper.AMDSMI_STATUS_BUSY - STATUS_REFCOUNT_OVERFLOW = amdsmi_wrapper.AMDSMI_STATUS_REFCOUNT_OVERFLOW - STATUS_FAIL_LOAD_MODULE = amdsmi_wrapper.AMDSMI_STATUS_FAIL_LOAD_MODULE - STATUS_FAIL_LOAD_SYMBOL = amdsmi_wrapper.AMDSMI_STATUS_FAIL_LOAD_SYMBOL - STATUS_DRM_ERROR = amdsmi_wrapper.AMDSMI_STATUS_DRM_ERROR - STATUS_IO = amdsmi_wrapper.AMDSMI_STATUS_IO - STATUS_API_FAILED = amdsmi_wrapper.AMDSMI_STATUS_API_FAILED - STATUS_TIMEOUT = amdsmi_wrapper.AMDSMI_STATUS_TIMEOUT - STATUS_NO_SLOT = amdsmi_wrapper.AMDSMI_STATUS_NO_SLOT - STATUS_RETRY = amdsmi_wrapper.AMDSMI_STATUS_RETRY - STATUS_NOT_INIT = amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT - UNKNOWN_ERROR = amdsmi_wrapper.AMDSMI_STATUS_UNKNOWN_ERROR - class AmdSmiException(Exception): """Base smi exception class""" - pass @@ -67,7 +36,7 @@ class AmdSmiLibraryException(AmdSmiException): self.set_err_info() def __str__(self): - return "An error occured with code: {err_code}({err_info})".format( + return "Error code:\n\t{err_code} | {err_info}".format( err_code=self.err_code, err_info=self.err_info ) @@ -77,34 +46,49 @@ class AmdSmiLibraryException(AmdSmiException): def get_error_code(self): return self.err_code + # Translate error codes to error strings def set_err_info(self): switch = { - AmdSmiRetCode.STATUS_INVAL: "AMDSMI_STATUS_INVAL - Invalid parameters", - AmdSmiRetCode.STATUS_NOT_SUPPORTED: "AMDSMI_STATUS_NOT_SUPPORTED - Feature not supported", - AmdSmiRetCode.STATUS_FILE_ERROR: "AMDSMI_STATUS_FILE_ERROR - Error opening file", - AmdSmiRetCode.STATUS_OUT_OF_RESOURCES: "AMDSMI_STATUS_OUT_OF_RESOURCES - Not enough memory", - AmdSmiRetCode.STATUS_INTERNAL_EXCEPTION: "AMDSMI_STATUS_INTERNAL_EXCEPTION - Internal error", - AmdSmiRetCode.STATUS_NO_PERM: "AMDSMI_STATUS_NO_PERM - Permission Denied", - AmdSmiRetCode.STATUS_INPUT_OUT_OF_BOUNDS: "AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS - Out of bounds", - AmdSmiRetCode.STATUS_INIT_ERROR: "AMDSMI_STATUS_INIT_ERROR - Initialization error", - AmdSmiRetCode.STATUS_BUSY: "AMDSMI_STATUS_BUSY - Device busy", - AmdSmiRetCode.STATUS_NOT_FOUND: "AMDSMI_STATUS_NOT_FOUND - Device Not found", - AmdSmiRetCode.STATUS_IO: "AMDSMI_STATUS_IO - I/O Error", - AmdSmiRetCode.STATUS_NOT_YET_IMPLEMENTED: "AMDSMI_STATUS_NOT_YET_IMPLEMENTED - Feature not yet implemented", - AmdSmiRetCode.STATUS_INSUFFICIENT_SIZE: "AMDSMI_STATUS_INSUFFICIENT_SIZE - Insufficient size for operation", - AmdSmiRetCode.STATUS_INTERRUPT: "AMDSMI_STATUS_INTERRUPT - Interrupt ocurred during execution", - AmdSmiRetCode.STATUS_UNEXPECTED_SIZE: "AMDSMI_STATUS_UNEXPECTED_SIZE - unexpected size of data was read", - AmdSmiRetCode.STATUS_NO_DATA: "AMDSMI_STATUS_NO_DATA - No data was found for given input", - AmdSmiRetCode.STATUS_UNEXPECTED_DATA: "AMDSMI_STATUS_UNEXPECTED_DATA - The data read or provided was unexpected", - AmdSmiRetCode.STATUS_REFCOUNT_OVERFLOW: "AMDSMI_STATUS_REFCOUNT_OVERFLOW - Internal reference counter exceeded INT32_MAX", - AmdSmiRetCode.STATUS_FAIL_LOAD_MODULE: "AMDSMI_STATUS_FAIL_LOAD_MODULE - Fail to load lib", - AmdSmiRetCode.STATUS_FAIL_LOAD_SYMBOL: "AMDSMI_STATUS_FAIL_LOAD_SYMBOL - Fail to load symbol", - AmdSmiRetCode.STATUS_DRM_ERROR: "AMDSMI_STATUS_DRM_ERROR - Error when called libdrm", - AmdSmiRetCode.STATUS_API_FAILED: "AMDSMI_STATUS_API_FAILED - API call failed", - AmdSmiRetCode.STATUS_TIMEOUT: "AMDSMI_STATUS_TIMEOUT - Timeout in API call", - AmdSmiRetCode.STATUS_NO_SLOT: "AMDSMI_STATUS_NO_SLOT - No more free slot", - AmdSmiRetCode.STATUS_RETRY: "AMDSMI_STATUS_RETRY - Retry operation", - AmdSmiRetCode.STATUS_NOT_INIT: "AMDSMI_STATUS_NOT_INIT - Device not initialized", + amdsmi_wrapper.AMDSMI_STATUS_INVAL : "AMDSMI_STATUS_INVAL - Invalid parameters", + amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED : "AMDSMI_STATUS_NOT_SUPPORTED - Feature not supported", + amdsmi_wrapper.AMDSMI_STATUS_NOT_YET_IMPLEMENTED : "AMDSMI_STATUS_NOT_YET_IMPLEMENTED - Feature not yet implemented", + amdsmi_wrapper.AMDSMI_STATUS_FAIL_LOAD_MODULE : "AMDSMI_STATUS_FAIL_LOAD_MODULE - Fail to load lib", + amdsmi_wrapper.AMDSMI_STATUS_FAIL_LOAD_SYMBOL : "AMDSMI_STATUS_FAIL_LOAD_SYMBOL - Fail to load symbol", + amdsmi_wrapper.AMDSMI_STATUS_DRM_ERROR : "AMDSMI_STATUS_DRM_ERROR - Error when called libdrm", + amdsmi_wrapper.AMDSMI_STATUS_API_FAILED : "AMDSMI_STATUS_API_FAILED - API call failed", + amdsmi_wrapper.AMDSMI_STATUS_TIMEOUT : "AMDSMI_STATUS_TIMEOUT - Timeout in API call", + amdsmi_wrapper.AMDSMI_STATUS_RETRY : "AMDSMI_STATUS_RETRY - Retry operation", + amdsmi_wrapper.AMDSMI_STATUS_NO_PERM : "AMDSMI_STATUS_NO_PERM - Permission Denied", + amdsmi_wrapper.AMDSMI_STATUS_INTERRUPT : "AMDSMI_STATUS_INTERRUPT - Interrupt ocurred during execution", + amdsmi_wrapper.AMDSMI_STATUS_IO : "AMDSMI_STATUS_IO - I/O Error", + amdsmi_wrapper.AMDSMI_STATUS_ADDRESS_FAULT : "AMDSMI_STATUS_ADDRESS_FAULT - Bad address", + amdsmi_wrapper.AMDSMI_STATUS_FILE_ERROR : "AMDSMI_STATUS_FILE_ERROR - Error opening file", + amdsmi_wrapper.AMDSMI_STATUS_OUT_OF_RESOURCES : "AMDSMI_STATUS_OUT_OF_RESOURCES - Not enough memory", + amdsmi_wrapper.AMDSMI_STATUS_INTERNAL_EXCEPTION : "AMDSMI_STATUS_INTERNAL_EXCEPTION - Internal error", + amdsmi_wrapper.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS : "AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS - Out of bounds", + amdsmi_wrapper.AMDSMI_STATUS_INIT_ERROR : "AMDSMI_STATUS_INIT_ERROR - Initialization error", + amdsmi_wrapper.AMDSMI_STATUS_REFCOUNT_OVERFLOW : "AMDSMI_STATUS_REFCOUNT_OVERFLOW - Internal reference counter exceeded INT32_MAX", + amdsmi_wrapper.AMDSMI_STATUS_BUSY : "AMDSMI_STATUS_BUSY - Device busy", + amdsmi_wrapper.AMDSMI_STATUS_NOT_FOUND : "AMDSMI_STATUS_NOT_FOUND - Device Not found", + amdsmi_wrapper.AMDSMI_STATUS_NOT_INIT : "AMDSMI_STATUS_NOT_INIT - Device not initialized", + amdsmi_wrapper.AMDSMI_STATUS_NO_SLOT : "AMDSMI_STATUS_NO_SLOT - No more free slot", + amdsmi_wrapper.AMDSMI_STATUS_DRIVER_NOT_LOADED : "AMDSMI_STATUS_DRIVER_NOT_LOADED - Processor driver not loaded", + amdsmi_wrapper.AMDSMI_STATUS_NO_DATA : "AMDSMI_STATUS_NO_DATA - No data was found for given input", + amdsmi_wrapper.AMDSMI_STATUS_INSUFFICIENT_SIZE : "AMDSMI_STATUS_INSUFFICIENT_SIZE - Insufficient size for operation", + amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_SIZE : "AMDSMI_STATUS_UNEXPECTED_SIZE - unexpected size of data was read", + amdsmi_wrapper.AMDSMI_STATUS_UNEXPECTED_DATA : "AMDSMI_STATUS_UNEXPECTED_DATA - The data read or provided was unexpected", + amdsmi_wrapper.AMDSMI_STATUS_NON_AMD_CPU : "AMDSMI_STATUS_NON_AMD_CPU - System has non-AMD CPU", + amdsmi_wrapper.AMDSMI_NO_ENERGY_DRV : "AMD_SMI_NO_ENERGY_DRV - Energy driver not found", + amdsmi_wrapper.AMDSMI_NO_MSR_DRV : "AMDSMI_NO_MSR_DRV - MSR driver not found", + amdsmi_wrapper.AMDSMI_NO_HSMP_DRV : "AMD_SMI_NO_HSMP_DRV - HSMP driver not found", + amdsmi_wrapper.AMDSMI_NO_HSMP_SUP : "AMD_SMI_NO_HSMP_SUP - HSMP not supported", + amdsmi_wrapper.AMDSMI_NO_HSMP_MSG_SUP : "AMD_SMI_NO_HSMP_MSG_SUP - HSMP message/feature not supported", + amdsmi_wrapper.AMDSMI_HSMP_TIMEOUT : "AMD_SMI_HSMP_TIMEOUT - HSMP message timeout", + amdsmi_wrapper.AMDSMI_NO_DRV : "AMDSMI_NO_DRV - No Energy and HSMP driver present", + amdsmi_wrapper.AMDSMI_FILE_NOT_FOUND : "AMDSMI_FILE_NOT_FOUND - File or directory not found", + amdsmi_wrapper.AMDSMI_ARG_PTR_NULL : "AMDSMI_ARG_PTR_NULL - Parsed argument is invalid", + amdsmi_wrapper.AMDSMI_STATUS_MAP_ERROR : "AMDSMI_STATUS_MAP_ERROR - The internal library error did not map to a status code", + amdsmi_wrapper.AMDSMI_STATUS_UNKNOWN_ERROR : "AMDSMI_STATUS_UNKNOWN_ERROR - An unknown error occurred" } self.err_info = switch.get(self.err_code, "AMDSMI_STATUS_UNKNOWN_ERROR - An unknown error occurred") @@ -112,12 +96,12 @@ class AmdSmiLibraryException(AmdSmiException): class AmdSmiRetryException(AmdSmiLibraryException): def __init__(self): - super().__init__(AmdSmiRetCode.RETRY) + super().__init__(amdsmi_wrapper.AMDSMI_STATUS_RETRY) class AmdSmiTimeoutException(AmdSmiLibraryException): def __init__(self): - super().__init__(AmdSmiRetCode.TIMEOUT) + super().__init__(amdsmi_wrapper.AMDSMI_STATUS_TIMEOUT) class AmdSmiParameterException(AmdSmiException): From f87489b7358f3398fe2f82067ba73794f03c75f0 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 21:08:01 -0500 Subject: [PATCH 13/19] Updated metric --xgmi_err to display string instead of object Change-Id: I4f67384cdf65b5fa3cb04e1012ba79a53ae72cd7 Signed-off-by: Maisam Arif [ROCm/amdsmi commit: 9c314dfd5c8d0593a1d0b1b020c7133381c8f57d] --- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 5 +++-- projects/amdsmi/py-interface/amdsmi_interface.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index f5020d708f..739234c8a6 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -1115,8 +1115,9 @@ class AMDSMICommands(): if self.helpers.is_linux() and self.helpers.is_baremetal(): if args.xgmi_err: try: - values_dict['xgmi_err'] = amdsmi_interface.amdsmi_gpu_xgmi_error_status(args.gpu) - except amdsmi_interface.AmdSmiLibraryException as e: + xgmi_err_status = amdsmi_interface.amdsmi_gpu_xgmi_error_status(args.gpu) + values_dict['xgmi_err'] = amdsmi_interface.amdsmi_wrapper.amdsmi_xgmi_status_t__enumvalues[xgmi_err_status] + except amdsmi_exception.AmdSmiLibraryException as e: values_dict['xgmi_err'] = "N/A" logging.debug("Failed to get xgmi error status for gpu %s | %s", gpu_id, e.get_error_info()) if args.energy: diff --git a/projects/amdsmi/py-interface/amdsmi_interface.py b/projects/amdsmi/py-interface/amdsmi_interface.py index dfe52effbd..526aa8a5db 100644 --- a/projects/amdsmi/py-interface/amdsmi_interface.py +++ b/projects/amdsmi/py-interface/amdsmi_interface.py @@ -2393,7 +2393,7 @@ def amdsmi_gpu_xgmi_error_status( processor_handle, ctypes.byref(status)) ) - return AmdSmiXgmiStatus(status.value) + return AmdSmiXgmiStatus(status.value).value def amdsmi_reset_gpu_xgmi_error( From 29e9a4eec101d5401712c21350d691579bea5b29 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 18:45:24 -0500 Subject: [PATCH 14/19] Remove adding .json, .csv, & .txt file extensions Change-Id: I35172115932a1363c23aedf877151e87b669bca4 Signed-off-by: Maisam Arif [ROCm/amdsmi commit: b90dff83eae610c3f9529d49e313329243f12310] --- projects/amdsmi/amdsmi_cli/amdsmi_parser.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py index 65cb7bd85c..93f0eb658b 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py @@ -125,14 +125,6 @@ class AMDSMIParser(argparse.ArgumentParser): path.touch() setattr(args, self.dest, path) elif path.is_file(): - file_name = str(path) - if args.json and str(path).split('.')[-1].lower() != 'json': - file_name += ".json" - elif args.csv and str(path).split('.')[-1].lower() != 'csv': - file_name += ".csv" - elif str(path).split('.')[-1].lower() != 'txt': - file_name += ".txt" - path = Path(file_name) path.touch() setattr(args, self.dest, path) else: From 75689a3c7e5c32a76528a135215287411bdfc1f0 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 21:25:35 -0500 Subject: [PATCH 15/19] Added N/A for mixed gpu values with --csv Signed-off-by: Maisam Arif Change-Id: Iab730ae61d2c0fb08785e90f05f83bab755b5d94 [ROCm/amdsmi commit: 5ab82ad4e6b9e2e4146a811c13cd1d1152b8586d] --- projects/amdsmi/amdsmi_cli/amdsmi_logger.py | 24 +++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py index 563798a486..3ebf11981c 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_logger.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_logger.py @@ -312,8 +312,20 @@ class AMDSMILogger(): if not isinstance(self.output, list): stored_csv_output = [self.output] + if stored_csv_output: + csv_keys = set() + for output in stored_csv_output: + for key in output: + csv_keys.add(key) + + for index, output_dict in enumerate(stored_csv_output): + remaining_keys = csv_keys - set(output_dict.keys()) + for key in remaining_keys: + stored_csv_output[index][key] = "N/A" + if self.destination == 'stdout': if stored_csv_output: + # Get the header as a list of the first element to maintain order csv_header = stored_csv_output[0].keys() csv_stdout_output = self.CsvStdoutBuilder() writer = csv.DictWriter(csv_stdout_output, csv_header) @@ -324,12 +336,24 @@ class AMDSMILogger(): if watching_output: with self.destination.open('w', newline = '') as output_file: if self.watch_output: + csv_keys = set() + for output in self.watch_output: + for key in output: + csv_keys.add(key) + + for index, output_dict in enumerate(self.watch_output): + remaining_keys = csv_keys - set(output_dict.keys()) + for key in remaining_keys: + self.watch_output[index][key] = "N/A" + + # Get the header as a list of the first element to maintain order csv_header = self.watch_output[0].keys() writer = csv.DictWriter(output_file, csv_header) writer.writeheader() writer.writerows(self.watch_output) else: with self.destination.open('a', newline = '') as output_file: + # Get the header as a list of the first element to maintain order csv_header = stored_csv_output[0].keys() writer = csv.DictWriter(output_file, csv_header) writer.writeheader() From af638612c1adf706294684e7e5daa328d6f9a76e Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 23:11:37 -0500 Subject: [PATCH 16/19] Fixed vram_type reference to vram_vendor Signed-off-by: Maisam Arif Change-Id: I3519992f1cc59c411c2b045f0532be4de63090a2 [ROCm/amdsmi commit: 053c5d0c53dd5496844cd9d2476375bcfd0acebf] --- projects/amdsmi/amdsmi_cli/amdsmi_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py index 739234c8a6..9b456fb725 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_commands.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_commands.py @@ -414,7 +414,7 @@ class AMDSMICommands(): # Remove amdsmi enum prefix vram_info['vram_type'] = vram_type.replace('VRAM_TYPE_', '').replace('_', '') - vram_info['vram_vendor'] = vram_type.replace('AMDSMI_VRAM_VENDOR__', '') + vram_info['vram_vendor'] = vram_vendor.replace('AMDSMI_VRAM_VENDOR__', '') if self.logger.is_human_readable_format(): vram_info['vram_size_mb'] = f"{vram_info['vram_size_mb']} MB" From d0656df4cae720ad46cf1cbc940144fb7f4db432 Mon Sep 17 00:00:00 2001 From: Maisam Arif Date: Sun, 24 Sep 2023 19:08:37 -0500 Subject: [PATCH 17/19] Updated tool & lib versions & README.md Signed-off-by: Maisam Arif Change-Id: Ic41a36bcfa988ce9c8304157593012752857e919 [ROCm/amdsmi commit: 25b055014d0a297ce4da0d21382aff0e7d338465] --- projects/amdsmi/README.md | 59 +++++- projects/amdsmi/amdsmi_cli/README.md | 215 ++++++++++++++------ projects/amdsmi/amdsmi_cli/Release_Notes.md | 7 +- projects/amdsmi/amdsmi_cli/__init__.py | 2 +- projects/amdsmi/amdsmi_cli/_version.py | 2 +- projects/amdsmi/amdsmi_cli/amdsmi_parser.py | 6 +- projects/amdsmi/docs/doxygen/Doxyfile | 2 +- projects/amdsmi/include/amd_smi/amdsmi.h | 2 +- projects/amdsmi/py-interface/pyproject.toml | 2 +- 9 files changed, 215 insertions(+), 82 deletions(-) diff --git a/projects/amdsmi/README.md b/projects/amdsmi/README.md index dd277cb414..7de1330296 100755 --- a/projects/amdsmi/README.md +++ b/projects/amdsmi/README.md @@ -127,35 +127,74 @@ The output will be in `docs/_build/html`. For additional details, see the [ROCm Contributing Guide](https://rocm.docs.amd.com/en/latest/contributing.html#building-documentation) -## Install Python Library and CLI Tool +## Install CLI Tool and Python Library ### Requirements * python 3.7+ 64-bit * driver must be loaded for amdsmi_init() to pass -### Installation +### CLI Installation -Follow user permissions best practices if installing AMDSMI as any user than root. +Before amd-smi install, ensure previous versions of amdsmi library are uninstalled using pip: + +```bash +python3 -m pip list | grep amd +python3 -m pip uninstall amdsmi +``` * Install amdgpu driver * Install amd-smi-lib package through package manager +* amd-smi --help -Before amd-smi install, uninstall current versions of amdsmi using pip: +#### Install Example for Ubuntu 22.04 -```bash -pip3 list | grep amd -pip3 uninstall amdsmi +``` bash +python3 -m pip list | grep amd +python3 -m pip uninstall amdsmi +apt install amd-smi-lib +amd-smi --help ``` -```bash +### Python Library Installation + +This option is for users who want to develop their own scripts using amd-smi's python library + +Verify that your python version is 3.7+ to install the python library + +* Install amdgpu driver +* Install amd-smi-lib package through package manager +* cd /opt/rocm/share/amd_smi +* python3 -m pip install --upgrade pip +* python3 -m pip install --user . +* import amdsmi in python to start development + +Warning: this will take precedence over the cli tool's library install, to avoid issues run these steps after every amd-smi-lib update. + +#### RHEL 8 & SLES 15 + +The default python versions in RHEL 8 and SLES 15 are 3.6.8 and 3.6.15 + +While the CLI will work with these python versions, to install the python library you need to upgrade to python 3.7+ + +#### Python Library Install Example for Ubuntu 22.04 + +``` bash +apt install amd-smi-lib +amd-smi --help cd /opt/rocm/share/amd_smi python3 -m pip install --upgrade pip python3 -m pip install --user . -/opt/rocm/bin/amd-smi --help ``` -after installing amd-smi-lib, amd-smi is also available as a binary in /opt/rocm/bin +``` bash +python3 +Python 3.8.10 (default, May 26 2023, 14:05:08) +[GCC 9.4.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import amdsmi +>>> +``` ### Rebuilding Python wrapper diff --git a/projects/amdsmi/amdsmi_cli/README.md b/projects/amdsmi/amdsmi_cli/README.md index f687cd63b8..ccc75b416a 100644 --- a/projects/amdsmi/amdsmi_cli/README.md +++ b/projects/amdsmi/amdsmi_cli/README.md @@ -6,44 +6,73 @@ and deprecate the existing rocm_smi CLI tool & gpuv-smi tool. It uses Ctypes to call the amd_smi_lib API. Recommended: At least one AMD GPU with AMD driver installed -## Requirements +## Install CLI Tool and Python Library + +### Requirements * python 3.7+ 64-bit -* driver must be loaded for amdsmi_init() to pass +* amdgpu driver must be loaded for amdsmi_init() to pass -## Installation +### CLI Installation + +Before amd-smi install, ensure previous versions of amdsmi library are uninstalled using pip: + +```bash +python3 -m pip list | grep amd +python3 -m pip uninstall amdsmi +``` + +* Install amdgpu driver +* Install amd-smi-lib package through package manager +* amd-smi --help + +#### Install Example for Ubuntu 22.04 + +``` bash +python3 -m pip list | grep amd +python3 -m pip uninstall amdsmi +apt install amd-smi-lib +amd-smi --help +``` + +### Python Library Installation + +This option is for users who want to develop their own scripts using amd-smi's python library + +Verify that your python version is 3.7+ to install the python library * Install amdgpu driver * Install amd-smi-lib package through package manager * cd /opt/rocm/share/amd_smi * python3 -m pip install --upgrade pip * python3 -m pip install --user . -* /opt/rocm/bin/amd-smi --help +* import amdsmi in python to start development -Add /opt/rocm/bin to your shell's path to access amd-smi via the cmdline +Warning: this will take precedence over the cli tool's library install, to avoid issues run these steps after every amd-smi-lib update. -### RHEL 8 & SLES 15 +#### RHEL 8 & SLES 15 The default python versions in RHEL 8 and SLES 15 are 3.6.8 and 3.6.15 -While the CLI may work with these python versions, to install the python library you need to upgrade to python 3.7+ +While the CLI will work with these python versions, to install the python library you need to upgrade to python 3.7+ -Verify that your python version is 3.7+ to install the python library - -### Install Example for Ubuntu 22.04 +#### Python Library Install Example for Ubuntu 22.04 ``` bash apt install amd-smi-lib +amd-smi --help cd /opt/rocm/share/amd_smi python3 -m pip install --upgrade pip python3 -m pip install --user . -/opt/rocm/bin/amd-smi ``` -Add /opt/rocm/bin to your shell's path to access amd-smi via the cmdline - ``` bash -export PATH=$PATH:/opt/rocm/bin +python3 +Python 3.8.10 (default, May 26 2023, 14:05:08) +[GCC 9.4.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import amdsmi +>>> ``` ## Usage @@ -54,26 +83,27 @@ amd-smi will report the version and current platform detected when running the c amd-smi usage: amd-smi [-h] ... -AMD System Management Interface | Version: 23.2.1.0 | Platform: Linux Baremetal +AMD System Management Interface | Version: 23.3.1.0 | Platform: Linux Baremetal optional arguments: - -h, --help show this help message and exit + -h, --help show this help message and exit AMD-SMI Commands: - Descriptions: - version Display version information - list List GPU information - static Gets static information about the specified GPU - firmware Gets ucode/firmware information about the specified GPU - bad-pages Gets bad page information about the specified GPU - metric Gets metric/performance information about the specified GPU - process Lists general process information running on the specified GPU - topology Displays topology information of the devices. - set Set options for devices. - reset Reset options for devices. + Descriptions: + version Display version information + list List GPU information + static Gets static information about the specified GPU + firmware Gets firmware information about the specified GPU + bad-pages + Gets bad page information about the specified GPU + metric Gets metric/performance information about the specified GPU + process Lists general process information running on the specified GPU + topology Displays topology information of the devices + set Set options for devices + reset Reset options for devices ``` -More detailed verison information can be give when running `amd-smi version` +More detailed verison information is available from `amd-smi version` Each command will have detailed information via `amd-smi [command] --help` @@ -82,10 +112,8 @@ Each command will have detailed information via `amd-smi [command] --help` For convenience, here is the help output for each command ``` bash -amd-smi list --help usage: amd-smi list [-h] [--json | --csv] [--file FILE] - [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] - [-g GPU [GPU ...]] + [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [-g GPU [GPU ...]] Lists all the devices on the system and the links between devices. Lists all the sockets and for each socket, GPUs and/or CPUs associated to @@ -96,7 +124,8 @@ GPU with some basic information for each VF. optional arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices Command Modifiers: --json Displays output in JSON format (human readable by default). @@ -116,7 +145,8 @@ If no GPU is specified, return firmware information for all GPUs on the system. Firmware Arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -f, --ucode-list, --fw-list All FW list information Command Modifiers: @@ -130,7 +160,7 @@ Command Modifiers: amd-smi static --help usage: amd-smi static [-h] [--json | --csv] [--file FILE] [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [-g GPU [GPU ...]] - [-a] [-b] [-V] [-l] [-d] [-c] [-r] [-B] [-u] + [-a] [-b] [-V] [-d] [-r] [-v] [-B] [-l] [-u] If no GPU is specified, returns static information for all GPUs on the system. If no static argument is provided, all static information will be displayed. @@ -138,16 +168,17 @@ If no static argument is provided, all static information will be displayed. Static Arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -a, --asic All asic information -b, --bus All bus information -V, --vbios All video bios information (if available) - -l, --limit All limit metric values (i.e. power and thermal limits) -d, --driver Displays driver version -r, --ras Displays RAS features information - -B, --board All board information - -u, --numa All numa node information -v, --vram All vram information + -B, --board All board information + -l, --limit All limit metric values (i.e. power and thermal limits) + -u, --numa All numa node information Command Modifiers: --json Displays output in JSON format (human readable by default). @@ -167,7 +198,8 @@ If no GPU is specified, return bad page information for all GPUs on the system. Bad Pages Arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -p, --pending Displays all pending retired pages -r, --retired Displays retired pages -u, --un-res Displays unreservable pages @@ -183,9 +215,8 @@ Command Modifiers: amd-smi metric --help usage: amd-smi metric [-h] [--json | --csv] [--file FILE] [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [-g GPU [GPU ...]] - [-w loop_time] [-W total_loop_time] [-i number_of_iterations] [-u] - [-b] [-p] [-c] [-t] [-e] [-P] [-f] [-C] [-o] [-l] [-r] [-x] - [-E] [-m] + [-w loop_time] [-W total_loop_time] [-i number_of_iterations] [-m] + [-u] [-p] [-c] [-t] [-e] [-k] [-P] [-f] [-C] [-o] [-l] [-x] [-E] If no GPU is specified, returns metric information for all GPUs on the system. If no metric argument is provided all metric information will be displayed. @@ -193,15 +224,18 @@ If no metric argument is provided all metric information will be displayed. Metric arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -w loop_time, --watch loop_time Reprint the command in a loop of Interval seconds -W total_loop_time, --watch_time total_loop_time The total time to watch the given command -i number_of_iterations, --iterations number_of_iterations Total number of iterations to loop on the given command + -m, --mem-usage Memory usage per block -u, --usage Displays engine usage information -p, --power Current power usage -c, --clock Average, max, and current clock frequencies -t, --temperature Current temperatures -e, --ecc Number of ECC errors + -k, --ecc-block Number of ECC errors per block -P, --pcie Current PCIe speed, width, and replay count -f, --fan Current fan speed -C, --voltage-curve Display voltage curve @@ -209,7 +243,6 @@ Metric arguments: -l, --perf-level Current DPM performance level -x, --xgmi-err XGMI error information since last read -E, --energy Amount of energy consumed - -m, --mem-usage Memory usage per block Command Modifiers: --json Displays output in JSON format (human readable by default). @@ -225,20 +258,21 @@ usage: amd-smi process [-h] [--json | --csv] [--file FILE] [-w loop_time] [-W total_loop_time] [-i number_of_iterations] [-G] [-e] [-p PID] [-n NAME] -If no GPU is specified, returns information for all GPUs on the system. +If no GPU is specified, returns information for all GPUs on the system. If no process argument is provided all process information will be displayed. Process arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -w loop_time, --watch loop_time Reprint the command in a loop of Interval seconds -W total_loop_time, --watch_time total_loop_time The total time to watch the given command -i number_of_iterations, --iterations number_of_iterations Total number of iterations to loop on the given command -G, --general pid, process name, memory usage -e, --engine All engine usages -p PID, --pid PID Gets all process information about the specified process based on Process ID - -n NAME, --name NAME Gets all process information about the specified process based on Process Name. + -n NAME, --name NAME Gets all process information about the specified process based on Process Name. If multiple processes have the same name information is returned for all of them. Command Modifiers: @@ -254,13 +288,14 @@ usage: amd-smi topology [-h] [--json | --csv] [--file FILE] [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] [-g GPU [GPU ...]] [-a] [-w] [-o] [-t] [-b] -If no GPU is specified, returns information for all GPUs on the system. +If no GPU is specified, returns information for all GPUs on the system. If no topology argument is provided all topology information will be displayed. Topology arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -a, --access Displays link accessibility between GPUs -w, --weight Displays relative weight between GPUs -o, --hops Displays the number of hops between GPUs @@ -280,23 +315,24 @@ usage: amd-smi set [-h] [--json | --csv] [--file FILE] [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] -g GPU [GPU ...] [-f %] [-l LEVEL] [-P SETPROFILE] [-d SCLKMAX] -A GPU must be specified to set a configuration. +A GPU must be specified to set a configuration. A set argument must be provided; Multiple set arguments are accepted Set Arguments: - -h, --help show this help message and exit - -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff - -f %, --fan % Sets GPU fan speed (0-255 or 0-100%) - -l LEVEL, --perflevel LEVEL Sets performance level - -P SETPROFILE, --profile SETPROFILE Set power profile level (#) or a quoted string of custom profile attributes - -d SCLKMAX, --perfdeterminism SCLKMAX Sets GPU clock frequency limit and performance level to determinism to get minimal performance variation + -h, --help show this help message and exit + -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices + -f %, --fan % Sets GPU fan speed (0-255 or 0-100%) + -l LEVEL, --perflevel LEVEL Sets performance level + -P SETPROFILE, --profile SETPROFILE Set power profile level (#) or a quoted string of custom profile attributes + -d SCLKMAX, --perfdeterminism SCLKMAX Sets GPU clock frequency limit and performance level to determinism to get minimal performance variation Command Modifiers: - --json Displays output in JSON format (human readable by default). - --csv Displays output in CSV format (human readable by default). - --file FILE Saves output into a file on the provided path (stdout by default). - --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). + --json Displays output in JSON format (human readable by default). + --csv Displays output in CSV format (human readable by default). + --file FILE Saves output into a file on the provided path (stdout by default). + --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` ```bash @@ -305,13 +341,14 @@ usage: amd-smi reset [-h] [--json | --csv] [--file FILE] [--loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL}] -g GPU [GPU ...] [-G] [-c] [-f] [-p] [-x] [-d] -A GPU must be specified to reset a configuration. +A GPU must be specified to reset a configuration. A reset argument must be provided; Multiple reset arguments are accepted Reset Arguments: -h, --help show this help message and exit -g GPU [GPU ...], --gpu GPU [GPU ...] Select a GPU ID, BDF, or UUID from the possible choices: - ID:0 | BDF:0000:23:00.0 | UUID:ffffffff-ffff-ffff-ffff-ffffffffffff + ID:0 | BDF:0000:23:00.0 | UUID:c4ff73bf-0000-1000-80ff-ffffffffffff + all | Selects all devices -G, --gpureset Reset the specified GPU -c, --clocks Reset clocks and overdrive to default -f, --fans Reset fans to automatic (driver) control @@ -326,6 +363,58 @@ Command Modifiers: --loglevel {DEBUG,INFO,WARNING,ERROR,CRITICAL} Set the logging level for the parser commands (ERROR by default). ``` +### Example output from amd-smi static + +Here is some example output from the tool: + +```bash +amd-smi static +GPU: 0 +ASIC: + MARKET_NAME: 0x73bf + VENDOR_ID: 0x1002 + VENDOR_NAME: Advanced Micro Devices, Inc. [AMD/ATI] + SUBVENDOR_ID: 0 + DEVICE_ID: 0x73bf + REV_ID: 0xc3 + ASIC_SERIAL: 0xffffffffffffffff +BUS: + BDF: 0000:23:00.0 + MAX_PCIE_SPEED: 16 GT/s + MAX_PCIE_LANES: 16 + PCIE_INTERFACE_VERSION: Gen 4 + SLOT_TYPE: PCIE +VBIOS: + NAME: NAVI21 Gaming XL D41209 + BUILD_DATE: 2020/10/29 13:30 + PART_NUMBER: 113-D4120900-101 + VERSION: 020.001.000.038.015720 +BOARD: + SERIAL_NUMBER: 0xffffffffffffffff + MODEL_NUMBER: ffffffffffffffff + PRODUCT_NAME: ffffffffffffffff +LIMIT: + MAX_POWER: 203 W + CURRENT_POWER: 203 W + SLOWDOWN_EDGE_TEMPERATURE: 100 °C + SLOWDOWN_HOTSPOT_TEMPERATURE: 110 °C + SLOWDOWN_VRAM_TEMPERATURE: 100 °C + SHUTDOWN_EDGE_TEMPERATURE: 105 °C + SHUTDOWN_HOTSPOT_TEMPERATURE: 115 °C + SHUTDOWN_VRAM_TEMPERATURE: 105 °C +DRIVER: + DRIVER_VERSION: 6.1.10 + DRIVER_DATE: 2015/01/01 00:00 +RAS: N/A +VRAM: + VRAM_TYPE: MAX + VRAM_VENDOR: SAMSUNG + VRAM_SIZE_MB: 16368 MB +NUMA: + NODE: 0 + AFFINITY: -1 +``` + ## Disclaimer The information contained herein is for informational purposes only, and is subject to change without notice. While every precaution has been taken in the preparation of this document, it may contain technical inaccuracies, omissions and typographical errors, and AMD is under no obligation to update or otherwise correct this information. Advanced Micro Devices, Inc. makes no representations or warranties with respect to the accuracy or completeness of the contents of this document, and assumes no liability of any kind, including the implied warranties of noninfringement, merchantability or fitness for particular purposes, with respect to the operation or use of AMD hardware, software or other products described herein. diff --git a/projects/amdsmi/amdsmi_cli/Release_Notes.md b/projects/amdsmi/amdsmi_cli/Release_Notes.md index 1d405bd4ed..ce0dca0faa 100644 --- a/projects/amdsmi/amdsmi_cli/Release_Notes.md +++ b/projects/amdsmi/amdsmi_cli/Release_Notes.md @@ -4,6 +4,11 @@ Documentation for AMDSMI-CLI is available post install in /opt//libexec/amdsmi_cli/README.md +## AMDSMI-CLI 23.3.1.0 + +- not all ecc fields are currently supported +- RHEL 8 & SLES 15 may have extra install steps + ## AMDSMI-CLI 23.0.1.1 ### Known Issues @@ -44,4 +49,4 @@ Documentation for AMDSMI-CLI is available post install in /opt//l - csv modifier does not work - topology information is not yet enabled - watch modifier not fully enabled -- limited guest support \ No newline at end of file +- limited guest support diff --git a/projects/amdsmi/amdsmi_cli/__init__.py b/projects/amdsmi/amdsmi_cli/__init__.py index 06eafbb554..457ded273b 100644 --- a/projects/amdsmi/amdsmi_cli/__init__.py +++ b/projects/amdsmi/amdsmi_cli/__init__.py @@ -1 +1 @@ -__version__ = "23.2.1.0" +__version__ = "23.3.1.0" diff --git a/projects/amdsmi/amdsmi_cli/_version.py b/projects/amdsmi/amdsmi_cli/_version.py index 06eafbb554..457ded273b 100644 --- a/projects/amdsmi/amdsmi_cli/_version.py +++ b/projects/amdsmi/amdsmi_cli/_version.py @@ -1 +1 @@ -__version__ = "23.2.1.0" +__version__ = "23.3.1.0" diff --git a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py index 93f0eb658b..54cba3081e 100644 --- a/projects/amdsmi/amdsmi_cli/amdsmi_parser.py +++ b/projects/amdsmi/amdsmi_cli/amdsmi_parser.py @@ -561,7 +561,7 @@ class AMDSMIParser(argparse.ArgumentParser): return # Subparser help text - topology_help = "Displays topology information of the devices." + topology_help = "Displays topology information of the devices" topology_subcommand_help = "If no GPU is specified, returns information for all GPUs on the system.\ \nIf no topology argument is provided all topology information will be displayed." topology_optionals_title = "Topology arguments" @@ -597,7 +597,7 @@ class AMDSMIParser(argparse.ArgumentParser): return # Subparser help text - set_value_help = "Set options for devices." + set_value_help = "Set options for devices" set_value_subcommand_help = "A GPU must be specified to set a configuration.\ \nA set argument must be provided; Multiple set arguments are accepted" set_value_optionals_title = "Set Arguments" @@ -724,7 +724,7 @@ class AMDSMIParser(argparse.ArgumentParser): return # Subparser help text - reset_help = "Reset options for devices." + reset_help = "Reset options for devices" reset_subcommand_help = "A GPU must be specified to reset a configuration.\ \nA reset argument must be provided; Multiple reset arguments are accepted" reset_optionals_title = "Reset Arguments" diff --git a/projects/amdsmi/docs/doxygen/Doxyfile b/projects/amdsmi/docs/doxygen/Doxyfile index 8e7f69c1f7..b90d176126 100644 --- a/projects/amdsmi/docs/doxygen/Doxyfile +++ b/projects/amdsmi/docs/doxygen/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = AMD SMI # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "23.2.1.0" +PROJECT_NUMBER = "23.3.1.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 010744c10e..b2bd4c82fb 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -99,7 +99,7 @@ typedef enum { #define AMDSMI_LIB_VERSION_YEAR 23 //! Major version should be changed for every header change (adding/deleting APIs, changing names, fields of structures, etc.) -#define AMDSMI_LIB_VERSION_MAJOR 2 +#define AMDSMI_LIB_VERSION_MAJOR 3 //! Minor version should be updated for each API change, but without changing headers #define AMDSMI_LIB_VERSION_MINOR 1 diff --git a/projects/amdsmi/py-interface/pyproject.toml b/projects/amdsmi/py-interface/pyproject.toml index 87624e2d05..17ba948c13 100644 --- a/projects/amdsmi/py-interface/pyproject.toml +++ b/projects/amdsmi/py-interface/pyproject.toml @@ -10,7 +10,7 @@ name = "amdsmi" authors = [ {name = "AMD", email = "amd-smi.support@amd.com"}, ] -version = "23.2.1.0" +version = "23.3.1.0" license = {file = "amdsmi/LICENSE"} readme = {file = "amdsmi/README.md", content-type = "text/markdown"} description = "AMDSMI Python LIB - AMD GPU Monitoring Library" From 82a44740fe086e7e2d0cb8dd23d5097cec16d636 Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Mon, 25 Sep 2023 01:53:42 -0500 Subject: [PATCH 18/19] CMAKE - Fix --break-system-packages Change-Id: I4a46859124018b6574d2b53e6ef9e72482784f39 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: ac8bde5978201d76f029552231d279ef9fd6ae81] --- projects/amdsmi/py-interface/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/amdsmi/py-interface/CMakeLists.txt b/projects/amdsmi/py-interface/CMakeLists.txt index dd0516c908..dc1d498103 100644 --- a/projects/amdsmi/py-interface/CMakeLists.txt +++ b/projects/amdsmi/py-interface/CMakeLists.txt @@ -47,9 +47,12 @@ else() find_package(Python3 3.7 COMPONENTS Interpreter Development REQUIRED) # --break-system-packages is needed for python 3.11 # see: https://peps.python.org/pep-0668/ + if (NOT Python3_VERSION VERSION_LESS "3.11") + set(Python3_BREAK_SYSTEM_PACKAGES "--break-system-packages") + endif() add_custom_target( python_pre_reqs - COMMAND ${Python3_EXECUTABLE} -m pip install --break-system-packages clang==${clang_ver} ctypeslib2==${ctypeslib_ver}) + COMMAND ${Python3_EXECUTABLE} -m pip install ${Python3_BREAK_SYSTEM_PACKAGES} clang==${clang_ver} ctypeslib2==${ctypeslib_ver}) # generate new wrapper configure_file(${PROJECT_SOURCE_DIR}/tools/generator.py generator.py @ONLY COPYONLY) add_custom_command( From 07e65d05d4e4589cd70a5332466ccd987d84de7f Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Mon, 25 Sep 2023 11:24:52 -0500 Subject: [PATCH 19/19] SWDEV-423796 - Resolve stack smashing issue Inconsistency between struct fields caused stack smashing Change-Id: Ib06d67723e062d4306420854ba7ab45fb252ffe3 Signed-off-by: Galantsev, Dmitrii [ROCm/amdsmi commit: 21dcf6d66c0ade7a08b9885d498acdd876e2683c] --- projects/amdsmi/include/amd_smi/amdsmi.h | 7 ++++- .../amdsmi/py-interface/amdsmi_wrapper.py | 28 +++++++++++-------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/projects/amdsmi/include/amd_smi/amdsmi.h b/projects/amdsmi/include/amd_smi/amdsmi.h index 124746421b..f7b8d7ee5e 100644 --- a/projects/amdsmi/include/amd_smi/amdsmi.h +++ b/projects/amdsmi/include/amd_smi/amdsmi.h @@ -503,7 +503,7 @@ typedef struct { } amdsmi_proc_info_t; //! Guaranteed maximum possible number of supported frequencies -#define AMDSMI_MAX_NUM_FREQUENCIES 32 +#define AMDSMI_MAX_NUM_FREQUENCIES 33 //! Maximum possible value for fan speed. Should be used as the denominator //! when determining fan speed percentage. @@ -944,6 +944,11 @@ typedef struct { * @brief This structure holds information about clock frequencies. */ typedef struct { + /** + * Deep Sleep frequency is only supported by some GPUs + */ + bool has_deep_sleep; + /** * The number of supported frequencies */ diff --git a/projects/amdsmi/py-interface/amdsmi_wrapper.py b/projects/amdsmi/py-interface/amdsmi_wrapper.py index 360780bd3e..9c49d42b1a 100644 --- a/projects/amdsmi/py-interface/amdsmi_wrapper.py +++ b/projects/amdsmi/py-interface/amdsmi_wrapper.py @@ -837,16 +837,6 @@ amdsmi_process_handle_t = ctypes.c_uint32 class struct_amdsmi_proc_info_t(Structure): pass -class struct_engine_usage_(Structure): - pass - -struct_engine_usage_._pack_ = 1 # source:False -struct_engine_usage_._fields_ = [ - ('gfx', ctypes.c_uint64), - ('enc', ctypes.c_uint64), - ('reserved', ctypes.c_uint32 * 12), -] - class struct_memory_usage_(Structure): pass @@ -858,6 +848,16 @@ struct_memory_usage_._fields_ = [ ('reserved', ctypes.c_uint32 * 10), ] +class struct_engine_usage_(Structure): + pass + +struct_engine_usage_._pack_ = 1 # source:False +struct_engine_usage_._fields_ = [ + ('gfx', ctypes.c_uint64), + ('enc', ctypes.c_uint64), + ('reserved', ctypes.c_uint32 * 12), +] + struct_amdsmi_proc_info_t._pack_ = 1 # source:False struct_amdsmi_proc_info_t._fields_ = [ ('name', ctypes.c_char * 32), @@ -1285,9 +1285,12 @@ class struct_amdsmi_frequencies_t(Structure): struct_amdsmi_frequencies_t._pack_ = 1 # source:False struct_amdsmi_frequencies_t._fields_ = [ + ('has_deep_sleep', ctypes.c_bool), + ('PADDING_0', ctypes.c_ubyte * 3), ('num_supported', ctypes.c_uint32), ('current', ctypes.c_uint32), - ('frequency', ctypes.c_uint64 * 32), + ('PADDING_1', ctypes.c_ubyte * 4), + ('frequency', ctypes.c_uint64 * 33), ] amdsmi_frequencies_t = struct_amdsmi_frequencies_t @@ -1297,7 +1300,8 @@ class struct_amdsmi_pcie_bandwidth_t(Structure): struct_amdsmi_pcie_bandwidth_t._pack_ = 1 # source:False struct_amdsmi_pcie_bandwidth_t._fields_ = [ ('transfer_rate', amdsmi_frequencies_t), - ('lanes', ctypes.c_uint32 * 32), + ('lanes', ctypes.c_uint32 * 33), + ('PADDING_0', ctypes.c_ubyte * 4), ] amdsmi_pcie_bandwidth_t = struct_amdsmi_pcie_bandwidth_t