[SWDEV-488276/SWDEV-497613] Update memory partition set functionality

Changes:
  - [CLI] Added warning screen to AMD SMI users
    setting memory partition
  - [CLI] Added a progress bar time-bar for CLI sets display to 40 seconds
  - [API] Updated to wait until the driver reloads with SYSFS files active
  - [CLI] Now users can set or reset without providing:
    amd-smi set -g all <set arguments>
    or amd-smi reset -g all <set arguments>
    now can directly call -> sudo amd-smi set <set arguments>
    or sudo amd-smi reset <set arguments>
  - [SWDEV-475712][CLI/API] Fixed target_graphics_version field
    not properly displaying for older MI or Navi ASICs.
  - [All APIs] Added a catch for the driver to report invalid arguments
    now these APIs will show AMDSMI_STATUS_INVAL
    (ex. changing to NPS8 if the device does not support it)
  - [Install] Modified paths for Python install commands to support
    multi-ROCm installs

Change-Id: Id11f25d68a82d23c6b2d77ccb30b51e860dd0ca7
Signed-off-by: Charis Poag <Charis.Poag@amd.com>

amdsmi_cli/amdsmi_cli_exceptions.py

@@ -49,11 +49,13 @@ AMDSMI_ERROR_MESSAGES = {
     31: "Device Not found",
     32: "Device not initialized",
     33: "No more free slot",
+    34: "Driver not loaded",
     # Reserved for future error messages
     40: "No data was found for given input",
     41: "Insufficient size for operation",
     42: "Unexpected size of data was read",
     43: "The data read or provided was unexpected",
+    54: "AMDGPU restart error",
 }
 
 def _get_error_message(error_code):

amdsmi_cli/amdsmi_commands.py

@@ -25,6 +25,9 @@ import sys
 import threading
 import time
 import json
+import multiprocessing
+import threading
+import os
 
 from _version import __version__
 from amdsmi_helpers import AMDSMIHelpers
@@ -3890,10 +3893,10 @@ class AMDSMICommands():
             args.process_isolation = process_isolation
         if clk_limit:
             args.clk_limit = clk_limit
-
+ 
         # Handle No GPU passed
         if args.gpu == None:
-            raise ValueError('No GPU provided, specific GPU target(s) are needed')
+            args.gpu = self.device_handles
 
         # Handle multiple GPUs
         handled_multiple_gpus, device_handle = self.helpers.handle_gpus(args, self.logger, self.set_gpu)
@@ -3975,13 +3978,101 @@ class AMDSMICommands():
                     raise ValueError(f"Unable to set compute partition to {args.compute_partition} on {gpu_string}") from e
                 self.logger.store_output(args.gpu, 'computepartition', f"Successfully set compute partition to {args.compute_partition}")
             if args.memory_partition:
+                ####################################################################
+                # Get current and available memory partition modes                 #
+                # Info used if AMDSMI_STATUS_INVAL is caught & to set progress bar #
+                ####################################################################
+                try:
+                    memory_partition = amdsmi_interface.amdsmi_get_gpu_memory_partition(gpu) # this info likely actually comes from different apis than used here
+                except amdsmi_exception.AmdSmiLibraryException as e:
+                    memory_partition = "N/A"
+                    logging.debug("Failed to get current memory partition for GPU %s | %s", gpu_id, e.get_error_info())
+                try:
+                    mem_caps_str = "N/A"
+                    partition_dict = amdsmi_interface.amdsmi_get_gpu_accelerator_partition_profile(gpu)
+                    temp_mem_caps = partition_dict['partition_profile']['memory_caps']
+                    mem_caps = temp_mem_caps.nps_cap_mask
+                    if temp_mem_caps.amdsmi_nps_flags_t == None:
+                        mem_caps_list = []
+                        if mem_caps & 1 == 1:
+                            mem_caps_list.append("NPS1")
+                        if mem_caps & 2 == 2:
+                            mem_caps_list.append("NPS2")
+                        if mem_caps & 4 == 4:
+                            mem_caps_list.append("NPS4")
+                        if mem_caps & 8 == 8:
+                            mem_caps_list.append("NPS8")
+                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "")
+                    else:
+                        mem_caps = temp_mem_caps.amdsmi_nps_flags_t
+                        mem_caps_list = []
+                        if mem_caps.nps1_cap == 1:
+                            mem_caps_list.append("NPS1")
+                        if mem_caps.nps2_cap == 1:
+                            mem_caps_list.append("NPS2")
+                        if mem_caps.nps4_cap == 1:
+                            mem_caps_list.append("NPS4")
+                        if mem_caps.nps8_cap == 1:
+                            mem_caps_list.append("NPS8")
+                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "").replace("\'", "")
+                    if mem_caps_str == "":
+                        mem_caps_str = "N/A"
+                except amdsmi_exception.AmdSmiLibraryException as e:
+                    logging.debug("Failed to get accelerator partition profile for GPU %s | %s", gpu_id, e.get_error_info())
+                memory_dict = {'caps': mem_caps_str, 'current': memory_partition}
+
+                ###############################################################
+                # memory partition set starts here                            #
+                ###############################################################
+                showProgressBar = False
+                if ((str(memory_dict['current']) != "N/A") and (str(args.memory_partition) in mem_caps_str)
+                   and ((str(memory_dict['current']) != str(args.memory_partition)))):
+                    showProgressBar = True  # Only show progress bar if
+                                            # 1) Device can set memory partition modes
+                                            # 2) Requested mode is a valid mode to set
+                                            # 3) Current is not already the requested mode
+                                            # otherwise function will return fast
+                threads = []
+                kTimeWait = 40
+                self.helpers.increment_set_count()
+                set_count = self.helpers.get_set_count()
+                if set_count == 1: # only show reload warning on 1st set
+                    self.helpers.confirm_changing_memory_partition_gpu_reload_warning()
                 memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[args.memory_partition]
                 try:
+                    if set_count == 1 and showProgressBar: # only show reload warning on 1st set
+                        string_out = f"Updating memory partition for gpu {gpu_id}"
+                        t1 = multiprocessing.Process(target=self.helpers.showProgressbar,
+                                                    args=(string_out, kTimeWait,))
+                        threads.append(t1)
+                        t1.start()
                     amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition)
+                    for thread in threads:
+                        thread.terminate()
+                        thread.join()
+
                 except amdsmi_exception.AmdSmiLibraryException as e:
+                    f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
+                    print("\n\n", end='\r', flush=True, file=f)
+                    for thread in threads:
+                        thread.join()
+                        thread.terminate()
+
                     if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM:
                         raise PermissionError('Command requires elevation') from e
+                    if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL:
+                        print(f"[amdsmi_wrapper.AMDSMI_STATUS_INVAL] Unable to set memory partition to {args.memory_partition} on {gpu_string}")
+                        print(f"Valid Memory partition Modes: {mem_caps_str}\n")
+                        # fall through for value error
+
+                    f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
+                    print("\n\n", end='\r', flush=True, file=f)
                     raise ValueError(f"Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e
+                except Exception as e:
+                    for thread in threads:
+                        thread.join()
+                        thread.terminate()
+                    raise ValueError(f"Generic error found | Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e
                 self.logger.store_output(args.gpu, 'memorypartition', f"Successfully set memory partition to {args.memory_partition}")
             if isinstance(args.power_cap, int):
                 try:
@@ -4226,7 +4317,7 @@ class AMDSMICommands():
                 self.set_core(args, multiple_devices, core, core_boost_limit)
         elif self.helpers.is_amdgpu_initialized(): # Only GPU is initialized
             if args.gpu == None:
-                raise ValueError('No GPU provided, specific GPU target(s) are needed')
+                args.gpu = self.device_handles
             self.logger.clear_multiple_devices_ouput()
             self.set_gpu(args, multiple_devices, gpu, fan, perf_level,
                             profile, perf_determinism, compute_partition,
@@ -4281,7 +4372,7 @@ class AMDSMICommands():
 
         # Handle No GPU passed
         if args.gpu == None:
-            raise ValueError('No GPU provided, specific GPU target(s) are needed')
+            args.gpu = self.device_handles
 
         # Handle multiple GPUs
         handled_multiple_gpus, device_handle = self.helpers.handle_gpus(args, self.logger, self.reset)
@@ -5299,7 +5390,7 @@ class AMDSMICommands():
                             mem_caps_list.append("NPS4")
                         if mem_caps.nps8_cap == 1:
                             mem_caps_list.append("NPS8")
-                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "")
+                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "").replace("\'", "")
                     if mem_caps_str == "":
                         mem_caps_str = "N/A"
                 except amdsmi_exception.AmdSmiLibraryException as e:
@@ -5350,7 +5441,7 @@ class AMDSMICommands():
                             mem_caps_list.append("NPS4")
                         if mem_caps & 8 == 8:
                             mem_caps_list.append("NPS8")
-                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "")
+                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "").replace("\'", "")
                     else:
                         mem_caps = temp_mem_caps.amdsmi_nps_flags_t
                         mem_caps_list = []
@@ -5362,7 +5453,7 @@ class AMDSMICommands():
                             mem_caps_list.append("NPS4")
                         if mem_caps.nps8_cap == 1:
                             mem_caps_list.append("NPS8")
-                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "")
+                        mem_caps_str = str(mem_caps_list).replace("]", "").replace("[", "").replace("\'", "")
                     if mem_caps_str == "":
                         mem_caps_str = "N/A"
                 except amdsmi_exception.AmdSmiLibraryException as e:

amdsmi_cli/amdsmi_helpers.py

@@ -27,6 +27,7 @@ import platform
 import sys
 import time
 import re
+import multiprocessing
 
 from typing import List, Union
 from enum import Enum
@@ -54,6 +55,7 @@ class AMDSMIHelpers():
 
         self._is_linux = False
         self._is_windows = False
+        self._count_of_sets_called = 0
 
         if self.operating_system.startswith("Linux"):
             self._is_linux = True
@@ -77,6 +79,11 @@ class AMDSMIHelpers():
                     self._is_virtual_os = False
                     self._is_passthrough = True
 
+    def increment_set_count(self):
+        self._count_of_sets_called += 1
+
+    def get_set_count(self):
+        return self._count_of_sets_called
 
     def is_virtual_os(self):
         return self._is_virtual_os
@@ -740,6 +747,30 @@ class AMDSMIHelpers():
         else:
             sys.exit('Confirmation not given. Exiting without setting value')
 
+    def confirm_changing_memory_partition_gpu_reload_warning(self, auto_respond=False):
+        """ Print the warning for running outside of specification and prompt user to accept the terms.
+
+        :param autoRespond: Response to automatically provide for all prompts
+        """
+        print('''
+          ****** WARNING ******\n
+          Setting Dynamic Memory (NPS) partition modes require users to quit all GPU workloads.
+          AMD SMI will then attempt to change memory (NPS) partition mode.
+          Upon a successful set, AMD SMI will then initiate an action to restart amdgpu driver.
+          This action will change all GPU's in the hive to the requested memory (NPS) partition mode.
+
+          Please use this utility with caution.
+          ''')
+        if not auto_respond:
+            user_input = input('Do you accept these terms? [Y/N] ')
+        else:
+            user_input = auto_respond
+        if user_input in ['Yes', 'yes', 'y', 'Y', 'YES']:
+            print('')
+            return
+        else:
+            print('Confirmation not given. Exiting without setting value')
+            sys.exit(1)
 
     def is_valid_profile(self, profile):
         profile_presets = amdsmi_interface.amdsmi_wrapper.amdsmi_power_profile_preset_masks_t__enumvalues
@@ -818,3 +849,21 @@ class AMDSMIHelpers():
             except Exception as _:
                 continue
         return pci_devices
+
+    def progressbar(self, it, prefix="", size=60, out=sys.stdout):
+        count = len(it)
+        def show(j):
+            x = int(size*j/count)
+            print("{}[{}{}] {}/{} secs remain".format(prefix, u"█"*x, "."*(size-x), j, count),
+                    end='\r', file=out, flush=True)
+        show(0)
+        for i, item in enumerate(it):
+            yield item
+            show(i+1)
+        print("\n\n", end='\r', flush=True, file=out)
+
+    def showProgressbar(self, title="", timeInSeconds=13):
+        if title != "":
+            title += ": "
+        for i in self.progressbar(range(timeInSeconds), title, 40):
+            time.sleep(1)

amdsmi_cli/amdsmi_init.py

@@ -26,6 +26,7 @@ import atexit
 import logging
 import signal
 import sys
+import os
 
 from pathlib import Path
 
@@ -134,8 +135,11 @@ def amdsmi_cli_shutdown():
 
 def signal_handler(sig, frame):
     logging.debug(f"Handling signal: {sig}")
-    sys.exit(0)
-
+    try:
+       sys.exit(0)
+    except Exception as e:
+        logging.error("Unable to cleanly shut down amd-smi-lib, exception: %s", str(type(e).__name__))
+        os._exit(0)
 
 if not AMDSMI_INITIALIZED:
     AMDSMI_INIT_FLAG = amdsmi_cli_init()

amdsmi_cli/amdsmi_parser.py

@@ -1032,7 +1032,7 @@ class AMDSMIParser(argparse.ArgumentParser):
 
         # Subparser help text
         set_value_help = "Set options for devices"
-        set_value_subcommand_help = "A GPU must be specified to set a configuration.\
+        set_value_subcommand_help = "If no GPU is specified, will select all GPUs on the system.\
                                     \nA set argument must be provided; Multiple set arguments are accepted"
         set_value_optionals_title = "Set Arguments"
 
@@ -1073,8 +1073,8 @@ class AMDSMIParser(argparse.ArgumentParser):
         set_value_parser.formatter_class=lambda prog: AMDSMISubparserHelpFormatter(prog)
         set_value_parser.set_defaults(func=func)
 
-        # Device args are required as safeguard from the user applying the operation to all gpus unintentionally
-        self._add_device_arguments(set_value_parser, required=True)
+        # Providing no -g 0 or -g all, is not required
+        self._add_device_arguments(set_value_parser, required=False)
 
         if self.helpers.is_amdgpu_initialized():
             if self.helpers.is_baremetal():
@@ -1126,7 +1126,7 @@ class AMDSMIParser(argparse.ArgumentParser):
 
         # Subparser help text
         reset_help = "Reset options for devices"
-        reset_subcommand_help = "A GPU must be specified to reset a configuration.\
+        reset_subcommand_help = "If no GPU is specified, will select all GPUs on the system.\
                                 \nA reset argument must be provided; Multiple reset arguments are accepted"
         reset_optionals_title = "Reset Arguments"
 
@@ -1148,8 +1148,8 @@ class AMDSMIParser(argparse.ArgumentParser):
 
         # Add Universal Arguments
         self._add_command_modifiers(reset_parser)
-        # Device args are required as safeguard from the user applying the operation to all gpus unintentionally
-        self._add_device_arguments(reset_parser, required=True)
+        # Providing no -g 0 or -g all, is not required
+        self._add_device_arguments(reset_parser, required=False)
 
         if self.helpers.is_baremetal():
             # Add Baremetal reset arguments