[SWDEV-499029] Fix unable to change memory partition modes
Changes:
* [API] Removed checking board name, fixes for other MI ASICs
* [API] Fixed unable to restart AMD GPU, libdrm blocked
doing this operation
* [API] Added ability to unload/reload libdrm
from within AMD SMI APIs
* [CLI] Increased progress bar to change memory partition modes
to 140 seconds, since driver reload is variable per system
Change-Id: I52f227f2ab850c4a6332ff3ecdc899903b1080f1
Signed-off-by: Charis Poag <Charis.Poag@amd.com>
Tento commit je obsažen v:
Charis Poag committed
1 rodič
35d8e827b9
revize
7d061f9ae4
8 files changed
+207
-92
No files matched your search
@@ -3978,18 +3978,20 @@ class AMDSMICommands():
|
||||
raise ValueError(f"Unable to set compute partition to {args.compute_partition} on {gpu_string}") from e
|
||||
self.logger.store_output(args.gpu, 'computepartition', f"Successfully set compute partition to {args.compute_partition}")
|
||||
if args.memory_partition:
|
||||
lock = multiprocessing.Lock()
|
||||
lock.acquire()
|
||||
####################################################################
|
||||
# Get current and available memory partition modes #
|
||||
# Info used if AMDSMI_STATUS_INVAL is caught & to set progress bar #
|
||||
####################################################################
|
||||
try:
|
||||
memory_partition = amdsmi_interface.amdsmi_get_gpu_memory_partition(gpu) # this info likely actually comes from different apis than used here
|
||||
memory_partition = amdsmi_interface.amdsmi_get_gpu_memory_partition(args.gpu) # this info likely actually comes from different apis than used here
|
||||
except amdsmi_exception.AmdSmiLibraryException as e:
|
||||
memory_partition = "N/A"
|
||||
logging.debug("Failed to get current memory partition for GPU %s | %s", gpu_id, e.get_error_info())
|
||||
try:
|
||||
mem_caps_str = "N/A"
|
||||
partition_dict = amdsmi_interface.amdsmi_get_gpu_accelerator_partition_profile(gpu)
|
||||
partition_dict = amdsmi_interface.amdsmi_get_gpu_accelerator_partition_profile(args.gpu)
|
||||
temp_mem_caps = partition_dict['partition_profile']['memory_caps']
|
||||
mem_caps = temp_mem_caps.nps_cap_mask
|
||||
if temp_mem_caps.amdsmi_nps_flags_t == None:
|
||||
@@ -4032,48 +4034,92 @@ class AMDSMICommands():
|
||||
# 2) Requested mode is a valid mode to set
|
||||
# 3) Current is not already the requested mode
|
||||
# otherwise function will return fast
|
||||
else:
|
||||
showProgressBar = False
|
||||
|
||||
threads = []
|
||||
kTimeWait = 40
|
||||
k140secs = 140
|
||||
string_out = f"Updating memory partition for gpu {gpu_id}"
|
||||
timesToRetryRestartErr = 1
|
||||
|
||||
self.helpers.increment_set_count()
|
||||
set_count = self.helpers.get_set_count()
|
||||
if set_count == 1: # only show reload warning on 1st set
|
||||
self.helpers.confirm_changing_memory_partition_gpu_reload_warning()
|
||||
memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[args.memory_partition]
|
||||
try:
|
||||
if set_count == 1 and showProgressBar: # only show reload warning on 1st set
|
||||
string_out = f"Updating memory partition for gpu {gpu_id}"
|
||||
t1 = multiprocessing.Process(target=self.helpers.showProgressbar,
|
||||
args=(string_out, kTimeWait,))
|
||||
threads.append(t1)
|
||||
t1.start()
|
||||
amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition)
|
||||
for thread in threads:
|
||||
thread.terminate()
|
||||
thread.join()
|
||||
|
||||
except amdsmi_exception.AmdSmiLibraryException as e:
|
||||
f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
|
||||
print("\n\n", end='\r', flush=True, file=f)
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
thread.terminate()
|
||||
while timesToRetryRestartErr >= 0:
|
||||
timesToRetryRestartErr -= 1
|
||||
try:
|
||||
if showProgressBar: # only show reload warning on 1st set
|
||||
t1 = multiprocessing.Process(target=self.helpers.showProgressbar,
|
||||
args=(string_out, k140secs,))
|
||||
threads.append(t1)
|
||||
t1.start()
|
||||
memory_partition = amdsmi_interface.AmdSmiMemoryPartitionType[args.memory_partition]
|
||||
amdsmi_interface.amdsmi_set_gpu_memory_partition(args.gpu, memory_partition)
|
||||
for thread in threads:
|
||||
thread.terminate()
|
||||
print("")
|
||||
break # successful case
|
||||
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM:
|
||||
raise PermissionError('Command requires elevation') from e
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL:
|
||||
print(f"[amdsmi_wrapper.AMDSMI_STATUS_INVAL] Unable to set memory partition to {args.memory_partition} on {gpu_string}")
|
||||
print(f"Valid Memory partition Modes: {mem_caps_str}\n")
|
||||
# fall through for value error
|
||||
except amdsmi_exception.AmdSmiLibraryException as e:
|
||||
f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
|
||||
print("\n\n", end='\r', flush=True, file=f)
|
||||
for thread in threads:
|
||||
thread.terminate()
|
||||
|
||||
f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
|
||||
print("\n\n", end='\r', flush=True, file=f)
|
||||
raise ValueError(f"Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e
|
||||
except Exception as e:
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
thread.terminate()
|
||||
raise ValueError(f"Generic error found | Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e
|
||||
self.logger.store_output(args.gpu, 'memorypartition', f"Successfully set memory partition to {args.memory_partition}")
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NO_PERM:
|
||||
raise PermissionError('Command requires elevation') from e
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_INVAL:
|
||||
out = f"[AMDSMI_STATUS_INVAL] Unable to set memory partition to {args.memory_partition} on {gpu_string}"
|
||||
print(f"Valid Memory partition Modes: {mem_caps_str}\n")
|
||||
self.logger.store_output(args.gpu, 'memory_partition', out)
|
||||
self.logger.print_output()
|
||||
self.logger.clear_multiple_devices_ouput()
|
||||
lock.release()
|
||||
return
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_NOT_SUPPORTED:
|
||||
out = f"[AMDSMI_STATUS_NOT_SUPPORTED] Device does not support setting memory partition to {args.memory_partition} on {gpu_string}"
|
||||
self.logger.store_output(args.gpu, 'memory_partition', out)
|
||||
self.logger.print_output()
|
||||
self.logger.clear_multiple_devices_ouput()
|
||||
lock.release()
|
||||
return
|
||||
if e.get_error_code() == amdsmi_interface.amdsmi_wrapper.AMDSMI_STATUS_AMDGPU_RESTART_ERR:
|
||||
# Try again on a failure -> work around for not being able to close libdrm
|
||||
string_out = f"Trying again - Updating memory partition for gpu {gpu_id}"
|
||||
for thread in threads:
|
||||
thread.terminate()
|
||||
thread.join()
|
||||
if timesToRetryRestartErr < 0:
|
||||
out = f"[AMDSMI_STATUS_AMDGPU_RESTART_ERR] Could not successfully restart driver after applying {args.memory_partition} on {gpu_string}"
|
||||
self.logger.store_output(args.gpu, 'memory_partition', out)
|
||||
self.logger.print_output()
|
||||
self.logger.clear_multiple_devices_ouput()
|
||||
return
|
||||
continue
|
||||
|
||||
f = open(os.devnull, 'w') #redirect to /dev/null (crossplatform)
|
||||
print("\n\n", end='\r', flush=True, file=f)
|
||||
out = f"Unable to set memory partition to {args.memory_partition} on {gpu_string}"
|
||||
print(out)
|
||||
self.logger.store_output(args.gpu, 'memorypartition', out)
|
||||
self.logger.print_output()
|
||||
self.logger.clear_multiple_devices_ouput()
|
||||
lock.release()
|
||||
return
|
||||
except Exception as e:
|
||||
for thread in threads:
|
||||
thread.terminate()
|
||||
out = f"Generic error found | Unable to set memory partition to {args.memory_partition} on {gpu_string}"
|
||||
print(out)
|
||||
lock.release()
|
||||
raise ValueError(f"Generic error found | Unable to set memory partition to {args.memory_partition} on {gpu_string}") from e
|
||||
self.logger.store_output(args.gpu, 'memory_partition', f"Successfully set memory partition to {args.memory_partition}")
|
||||
self.logger.print_output()
|
||||
self.logger.clear_multiple_devices_ouput()
|
||||
lock.release()
|
||||
return
|
||||
if isinstance(args.power_cap, int):
|
||||
try:
|
||||
power_cap_info = amdsmi_interface.amdsmi_get_power_cap_info(args.gpu)
|
||||
|
||||
@@ -755,7 +755,7 @@ class AMDSMIHelpers():
|
||||
****** WARNING ******\n
|
||||
Setting Dynamic Memory (NPS) partition modes require users to quit all GPU workloads.
|
||||
AMD SMI will then attempt to change memory (NPS) partition mode.
|
||||
Upon a successful set, AMD SMI will then initiate an action to restart amdgpu driver.
|
||||
Upon a successful set, AMD SMI will then initiate an action to restart AMD GPU driver.
|
||||
This action will change all GPU's in the hive to the requested memory (NPS) partition mode.
|
||||
|
||||
Please use this utility with caution.
|
||||
|
||||
Reference in new issue
Zablokovat Uživatele