Expand showpids to provide more details

Provide details of GPU usage by an application.

Change-Id: I0f36df7d358754c2c8a60432b736d98f667ee99c

include/rocm_smi/rocm_smi.h

@@ -3315,7 +3315,30 @@ rsmi_compute_process_info_by_pid_get(uint32_t pid, rsmi_process_info_t *proc);
  */
 rsmi_status_t
 rsmi_compute_process_gpus_get(uint32_t pid, uint32_t *dv_indices,
-                                                       uint32_t *num_devices);
+                                            uint32_t *num_devices);
+
+/**
+ *  @brief Get the info of a process on a specific device.
+ *
+ *  @details Given a process id @p pid, a @p dv_ind, this function will
+ *  write the process information for @p pid on the device, if available, to
+ *  the memory pointed to by @p proc.
+ *
+ *  @param[in] pid The process id of the process for which the gpu
+ *  currently being used is requested.
+ *
+ *  @param[in] dv_ind a device index where the process running on.
+ *
+ *  @param[inout] procs a pointer to memory provided by the caller to which
+ *  process information will be written.
+ *
+ *  @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
+ *
+ */
+rsmi_status_t
+rsmi_compute_process_info_by_device_get(uint32_t pid, uint32_t dv_ind,
+                                                rsmi_process_info_t *proc);
 
 /** @} */  // end of SysInfo
 

include/rocm_smi/rocm_smi_logger.h

@@ -214,7 +214,7 @@ class Logger {
 
   void logIntoFile(std::string& data);
   void logOnConsole(std::string& data);
-  void operator=(const Logger& obj) {}
+  void operator=(const Logger&) {}
   void initialize_resources();
   void destroy_resources();
 };

python_smi_tools/rocm_smi.py

@@ -2115,11 +2115,15 @@ def showPerformanceLevel(deviceList):
     printLogSpacer()
 
 
-def showPids():
+def showPids(verbose):
     """ Show Information for PIDs created in a KFD (Compute) context """
     printLogSpacer(' KFD Processes ')
     dataArray = []
-    dataArray.append(['PID', 'PROCESS NAME', 'GPU(s)', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY'])
+    if verbose == "details":
+        dataArray.append(['PID', 'PROCESS NAME', 'GPU', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY'])
+    else:
+        dataArray.append(['PID', 'PROCESS NAME', 'GPU(s)', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY'])
+
     pidList = getPidList()
     if not pidList:
         printLog(None, 'No KFD PIDs currently running', None)
@@ -2133,6 +2137,7 @@ def showPids():
         vramUsage = 'UNKNOWN'
         sdmaUsage = 'UNKNOWN'
         cuOccupancy = 'UNKNOWN'
+        dv_indices = (c_uint32 * num_devices.value)()
         ret = rocmsmi.rsmi_compute_process_gpus_get(int(pid), None, byref(num_devices))
         if rsmi_ret_ok(ret, metric='get_gpu_compute_process'):
             dv_indices = (c_uint32 * num_devices.value)()
@@ -2141,14 +2146,25 @@ def showPids():
                 gpuNumber = str(num_devices.value)
             else:
                 logging.debug('Unable to fetch GPU number by PID')
-        ret = rocmsmi.rsmi_compute_process_info_by_pid_get(int(pid), byref(proc))
-        if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'):
-            vramUsage = proc.vram_usage
-            sdmaUsage = proc.sdma_usage
-            cuOccupancy = proc.cu_occupancy
+        if verbose == "details":
+            for dv_ind in dv_indices:
+                ret = rocmsmi.rsmi_compute_process_info_by_device_get(int(pid), dv_ind, byref(proc))
+                if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'):
+                    vramUsage = proc.vram_usage
+                    sdmaUsage = proc.sdma_usage
+                    cuOccupancy = proc.cu_occupancy
+                else:
+                    logging.debug('Unable to fetch process info by PID')
+                dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)])
         else:
-            logging.debug('Unable to fetch process info by PID')
-        dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)])
+            ret = rocmsmi.rsmi_compute_process_info_by_pid_get(int(pid), byref(proc))
+            if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'):
+                vramUsage = proc.vram_usage
+                sdmaUsage = proc.sdma_usage
+                cuOccupancy = proc.cu_occupancy
+            else:
+                logging.debug('Unable to fetch process info by PID')
+            dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)])
     printLog(None, 'KFD process information:', None)
     print2DArray(dataArray)
     printLogSpacer()
@@ -3340,7 +3356,8 @@ if __name__ == '__main__':
     groupDisplay.add_argument('-s', '--showclkfrq', help='Show supported GPU and Memory Clock', action='store_true')
     groupDisplay.add_argument('--showmeminfo', help='Show Memory usage information for given block(s) TYPE',
                               metavar='TYPE', type=str, nargs='+')
-    groupDisplay.add_argument('--showpids', help='Show current running KFD PIDs', action='store_true')
+    groupDisplay.add_argument('--showpids', help='Show current running KFD PIDs (pass details to VERBOSE for detailed information)',
+                              metavar='VERBOSE', const="summary", type=str, nargs='?')
     groupDisplay.add_argument('--showpidgpus', help='Show GPUs used by specified KFD PIDs (all if no arg given)',
                               nargs='*')
     groupDisplay.add_argument('--showreplaycount', help='Show PCIe Replay Count', action='store_true')
@@ -3508,7 +3525,7 @@ if __name__ == '__main__':
         args.showmemoverdrive = True
         args.showoverdrive = True
         args.showperflevel = True
-        args.showpids = True
+        args.showpids = "summary"
         args.showpidgpus = []
         args.showreplaycount = True
         args.showvc = True
@@ -3587,8 +3604,8 @@ if __name__ == '__main__':
         showPcieReplayCount(deviceList)
     if args.showserial:
         showSerialNumber(deviceList)
-    if args.showpids:
-        showPids()
+    if args.showpids != None:
+        showPids(args.showpids)
     if args.showpidgpus or str(args.showpidgpus) == '[]':
         showGpusByPid(args.showpidgpus)
     if args.showclkvolt:

src/rocm_smi.cc

@@ -3768,6 +3768,29 @@ rsmi_compute_process_info_by_pid_get(uint32_t pid,
   CATCH
 }
 
+rsmi_status_t
+rsmi_compute_process_info_by_device_get(uint32_t pid, uint32_t dv_ind,
+                                          rsmi_process_info_t *proc) {
+  TRY
+  if (proc == nullptr) {
+    return RSMI_STATUS_INVALID_ARGS;
+  }
+  // Check the device and kfdnode exist
+  GET_DEV_AND_KFDNODE_FROM_INDX
+
+  std::unordered_set<uint64_t> gpu_set;
+  gpu_set.insert(dev->kfd_gpu_id());
+  int err = amd::smi::GetProcessInfoForPID(pid, proc, &gpu_set);
+
+  if (err) {
+    return amd::smi::ErrnoToRsmiStatus(err);
+  }
+
+  return RSMI_STATUS_SUCCESS;
+
+  CATCH
+}
+
 rsmi_status_t
 rsmi_dev_xgmi_error_status(uint32_t dv_ind, rsmi_xgmi_status_t *status) {
   TRY

tests/rocm_smi_test/functional/process_info_read.cc

@@ -184,6 +184,21 @@ void TestProcInfoRead::Run(void) {
         std::cout << dev_inds[i];
       }
       std::cout << std::endl;
+
+      // Get details of the resource by the process on a specific device
+      // if any process running on devices
+      for (i = 0; i < amt_allocd; ++i) {
+        rsmi_process_info_t proc_info;
+        err = rsmi_compute_process_info_by_device_get(
+            procs[j].process_id, dev_inds[i], &proc_info);
+        CHK_ERR_ASRT(err)
+        ASSERT_EQ(proc_info.process_id, procs[j].process_id);
+        ASSERT_EQ(proc_info.pasid, procs[j].pasid);
+        std::cout << "\t** Process ID: " << procs[j].process_id
+              << " on device " << dev_inds[i] << " VRAM Usage: "
+              << proc_info.vram_usage << " SDMA Usage: " << proc_info.sdma_usage
+              << " Compute Unit Usage: " << proc_info.cu_occupancy << std::endl;
+      }
       // Reset amt_allocd back to the amount acutally allocated
       amt_allocd = num_devices;
     }