AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[SWDEV-357472] Add evicted_ms metric (#620)

Browse Source 
- **Added evicted_time metric for kfd processes**.  
  - Time that queues are evicted on a GPU in milliseconds
  - Added to CLI in `amd-smi monitor -q` and `amd-smi process`
  - Added to C API and Python API:
    - amdsmi_get_gpu_process_list()
    - amdsmi_get_gpu_compute_process_info()
    - amdsmi_get_gpu_compute_process_info_by_pid()

---------

Signed-off-by: Pryor, Adam <Adam.Pryor@amd.com>
This commit is contained in:
Pryor, Adam
2025-10-28 14:49:03 -05:00
committed by GitHub
parent 6f222c11a6
commit 2144cfbba4
11 changed files with 95 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
amdsmi_cli/amdsmi_commands.py
+16 -3
View File
@@ -3400,11 +3400,13 @@ class AMDSMICommands():
"gfx": process_info["engine_usage"]["gfx"],
"enc": process_info["engine_usage"]["enc"],
},
"cu_occupancy": process_info["cu_occupancy"]
"cu_occupancy": process_info["cu_occupancy"],
"evicted_time": process_info["evicted_time"]
}
engine_usage_unit = "ns"
memory_usage_unit = "B"
evicted_time_unit = "ms"
if self.logger.is_human_readable_format():
process_info['mem_usage'] = self.helpers.convert_bytes_to_readable(process_info['mem_usage'])
@@ -3415,6 +3417,10 @@ class AMDSMICommands():
process_info['mem_usage'] = self.helpers.unit_format(self.logger,
process_info['mem_usage'],
memory_usage_unit)
process_info['evicted_time'] = self.helpers.unit_format(self.logger,
process_info['evicted_time'],
evicted_time_unit)
for usage_metric in process_info['usage']:
process_info['usage'][usage_metric] = self.helpers.unit_format(self.logger,
@@ -6130,8 +6136,10 @@ class AMDSMICommands():
process_info.pop('engine_usage') # Remove 'engine_usage' value
process_info['mem_usage'] = process_info.pop('mem')
process_info['cu_occupancy'] = process_info.pop('cu_occupancy')
process_info['evicted_time'] = process_info.pop('evicted_time')
memory_usage_unit = "B"
evicted_time_unit = "ms"
if self.logger.is_human_readable_format():
process_info['mem_usage'] = self.helpers.convert_bytes_to_readable(process_info['mem_usage'])
@@ -6143,6 +6151,10 @@ class AMDSMICommands():
process_info['mem_usage'],
memory_usage_unit)
process_info['evicted_time'] = self.helpers.unit_format(self.logger,
process_info['evicted_time'],
evicted_time_unit)
for usage_metric in process_info['memory_usage']:
process_info['memory_usage'][usage_metric] = self.helpers.unit_format(self.logger,
process_info['memory_usage'][usage_metric],
@@ -6176,7 +6188,7 @@ class AMDSMICommands():
# Build the process table's title and header
self.logger.secondary_table_title = "PROCESS INFO"
self.logger.secondary_table_header = 'GPU'.rjust(3) + "NAME".rjust(19) + "PID".rjust(9) + "GTT_MEM".rjust(10) + \
"CPU_MEM".rjust(10) + "VRAM_MEM".rjust(10) + "MEM_USG".rjust(10) + "CU%".rjust(9)
"CPU_MEM".rjust(10) + "VRAM_MEM".rjust(10) + "MEM_USG".rjust(10) + "CU%".rjust(9) + "EVICT".rjust(10)
if watching_output:
self.logger.secondary_table_header = 'TIMESTAMP'.rjust(10) + ' ' + self.logger.secondary_table_header
@@ -7334,7 +7346,7 @@ class AMDSMICommands():
try:
raw_process_list = amdsmi_interface.amdsmi_get_gpu_process_list(processor)
for proc in raw_process_list:
proc_info_dict = {"gpu": "N/A", "pid": "N/A", "name": "N/A","gtt": "N/A", "vram": "N/A", "mem_usage": "N/A", "cu_occupancy": "N/A"}
proc_info_dict = {"gpu": "N/A", "pid": "N/A", "name": "N/A","gtt": "N/A", "vram": "N/A", "mem_usage": "N/A", "cu_occupancy": "N/A", "evicted_time" : "N/A"}
proc_info_dict['gpu'] = gpu_id
proc_info_dict['pid'] = proc['pid']
proc_info_dict['name'] = proc['name']
@@ -7350,6 +7362,7 @@ class AMDSMICommands():
proc_info_dict['cu_occupancy'] = {"current_cu": "N/A", "total_num_cu": total_num_cu}
except (ValueError, TypeError):
proc_info_dict['cu_occupancy'] = {"current_cu": "N/A", "total_num_cu": total_num_cu}
proc_info_dict['evicted_time'] = proc['evicted_time']
all_process_list.append(proc_info_dict)
except amdsmi_exception.AmdSmiLibraryException as e:
amdsmi_cli/amdsmi_logger.py
+6 -3
View File
@@ -230,7 +230,7 @@ class AMDSMILogger():
# Add N/A for empty process_info
table_values += "N/A".rjust(17) + "N/A".rjust(9) + "N/A".rjust(10) + \
"N/A".rjust(10) + "N/A".rjust(10) + "N/A".rjust(10) + \
"N/A".rjust(9) + '\n'
"N/A".rjust(9) + "N/A".rjust(10) + '\n'
else:
#Fix this herre
for process_key, process_value in process_dict['process_info'].items():
@@ -251,6 +251,8 @@ class AMDSMILogger():
table_values += string_process_value.rjust(10)
elif process_key == "cu_occupancy":
table_values += string_process_value.rjust(9)
elif process_key == "evicted_time":
table_values += string_process_value.rjust(9)
# Add the stored gpu and stored timestamp to the next line
table_values += '\n'
if stored_timestamp:
@@ -1124,8 +1126,9 @@ class AMDSMILogger():
cu_occupancy = (str(round(process['cu_occupancy']['current_cu'] / process['cu_occupancy']['total_num_cu'] * 100, 1)) + " %").rjust(7)
else:
cu_occupancy = "N/A"
print("| {0:4.4s} {1:9.9s} {2:19.19s} {3:8.8s} {4:8.8s} {5:9.9s} {6:7.7s} |".format(
gpu_id, pid, process_name, gtt_mem, vram_mem, mem_usage, cu_occupancy))
evicted_time = str(process['evicted_time']).rjust(9)
print("| {0:4.4s} {1:9.9s} {2:19.19s} {3:8.8s} {4:8.8s} {5:9.9s} {6:7.7s} {7:9.9s} |".format(
gpu_id, pid, process_name, gtt_mem, vram_mem, mem_usage, cu_occupancy, evicted_time))
if process['name'] == "N/A":
elevated_permission_error = True
else: