diff --git a/include/rocm_smi/rocm_smi.h b/include/rocm_smi/rocm_smi.h index d690050783..f0a531f154 100755 --- a/include/rocm_smi/rocm_smi.h +++ b/include/rocm_smi/rocm_smi.h @@ -3315,7 +3315,30 @@ rsmi_compute_process_info_by_pid_get(uint32_t pid, rsmi_process_info_t *proc); */ rsmi_status_t rsmi_compute_process_gpus_get(uint32_t pid, uint32_t *dv_indices, - uint32_t *num_devices); + uint32_t *num_devices); + +/** + * @brief Get the info of a process on a specific device. + * + * @details Given a process id @p pid, a @p dv_ind, this function will + * write the process information for @p pid on the device, if available, to + * the memory pointed to by @p proc. + * + * @param[in] pid The process id of the process for which the gpu + * currently being used is requested. + * + * @param[in] dv_ind a device index where the process running on. + * + * @param[inout] procs a pointer to memory provided by the caller to which + * process information will be written. + * + * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call + * @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid + * + */ +rsmi_status_t +rsmi_compute_process_info_by_device_get(uint32_t pid, uint32_t dv_ind, + rsmi_process_info_t *proc); /** @} */ // end of SysInfo diff --git a/include/rocm_smi/rocm_smi_logger.h b/include/rocm_smi/rocm_smi_logger.h index 5997a6621d..3ff1070418 100644 --- a/include/rocm_smi/rocm_smi_logger.h +++ b/include/rocm_smi/rocm_smi_logger.h @@ -214,7 +214,7 @@ class Logger { void logIntoFile(std::string& data); void logOnConsole(std::string& data); - void operator=(const Logger& obj) {} + void operator=(const Logger&) {} void initialize_resources(); void destroy_resources(); }; diff --git a/python_smi_tools/rocm_smi.py b/python_smi_tools/rocm_smi.py index b0f530cc41..5c12624142 100755 --- a/python_smi_tools/rocm_smi.py +++ b/python_smi_tools/rocm_smi.py @@ -2115,11 +2115,15 @@ def showPerformanceLevel(deviceList): printLogSpacer() -def showPids(): +def showPids(verbose): """ Show Information for PIDs created in a KFD (Compute) context """ printLogSpacer(' KFD Processes ') dataArray = [] - dataArray.append(['PID', 'PROCESS NAME', 'GPU(s)', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY']) + if verbose == "details": + dataArray.append(['PID', 'PROCESS NAME', 'GPU', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY']) + else: + dataArray.append(['PID', 'PROCESS NAME', 'GPU(s)', 'VRAM USED', 'SDMA USED', 'CU OCCUPANCY']) + pidList = getPidList() if not pidList: printLog(None, 'No KFD PIDs currently running', None) @@ -2133,6 +2137,7 @@ def showPids(): vramUsage = 'UNKNOWN' sdmaUsage = 'UNKNOWN' cuOccupancy = 'UNKNOWN' + dv_indices = (c_uint32 * num_devices.value)() ret = rocmsmi.rsmi_compute_process_gpus_get(int(pid), None, byref(num_devices)) if rsmi_ret_ok(ret, metric='get_gpu_compute_process'): dv_indices = (c_uint32 * num_devices.value)() @@ -2141,14 +2146,25 @@ def showPids(): gpuNumber = str(num_devices.value) else: logging.debug('Unable to fetch GPU number by PID') - ret = rocmsmi.rsmi_compute_process_info_by_pid_get(int(pid), byref(proc)) - if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'): - vramUsage = proc.vram_usage - sdmaUsage = proc.sdma_usage - cuOccupancy = proc.cu_occupancy + if verbose == "details": + for dv_ind in dv_indices: + ret = rocmsmi.rsmi_compute_process_info_by_device_get(int(pid), dv_ind, byref(proc)) + if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'): + vramUsage = proc.vram_usage + sdmaUsage = proc.sdma_usage + cuOccupancy = proc.cu_occupancy + else: + logging.debug('Unable to fetch process info by PID') + dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)]) else: - logging.debug('Unable to fetch process info by PID') - dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)]) + ret = rocmsmi.rsmi_compute_process_info_by_pid_get(int(pid), byref(proc)) + if rsmi_ret_ok(ret, metric='get_compute_process_info_by_pid'): + vramUsage = proc.vram_usage + sdmaUsage = proc.sdma_usage + cuOccupancy = proc.cu_occupancy + else: + logging.debug('Unable to fetch process info by PID') + dataArray.append([pid, getProcessName(pid), str(gpuNumber), str(vramUsage), str(sdmaUsage), str(cuOccupancy)]) printLog(None, 'KFD process information:', None) print2DArray(dataArray) printLogSpacer() @@ -3340,7 +3356,8 @@ if __name__ == '__main__': groupDisplay.add_argument('-s', '--showclkfrq', help='Show supported GPU and Memory Clock', action='store_true') groupDisplay.add_argument('--showmeminfo', help='Show Memory usage information for given block(s) TYPE', metavar='TYPE', type=str, nargs='+') - groupDisplay.add_argument('--showpids', help='Show current running KFD PIDs', action='store_true') + groupDisplay.add_argument('--showpids', help='Show current running KFD PIDs (pass details to VERBOSE for detailed information)', + metavar='VERBOSE', const="summary", type=str, nargs='?') groupDisplay.add_argument('--showpidgpus', help='Show GPUs used by specified KFD PIDs (all if no arg given)', nargs='*') groupDisplay.add_argument('--showreplaycount', help='Show PCIe Replay Count', action='store_true') @@ -3508,7 +3525,7 @@ if __name__ == '__main__': args.showmemoverdrive = True args.showoverdrive = True args.showperflevel = True - args.showpids = True + args.showpids = "summary" args.showpidgpus = [] args.showreplaycount = True args.showvc = True @@ -3587,8 +3604,8 @@ if __name__ == '__main__': showPcieReplayCount(deviceList) if args.showserial: showSerialNumber(deviceList) - if args.showpids: - showPids() + if args.showpids != None: + showPids(args.showpids) if args.showpidgpus or str(args.showpidgpus) == '[]': showGpusByPid(args.showpidgpus) if args.showclkvolt: diff --git a/src/rocm_smi.cc b/src/rocm_smi.cc index 6b5fdfedb3..12f6d2c665 100755 --- a/src/rocm_smi.cc +++ b/src/rocm_smi.cc @@ -3768,6 +3768,29 @@ rsmi_compute_process_info_by_pid_get(uint32_t pid, CATCH } +rsmi_status_t +rsmi_compute_process_info_by_device_get(uint32_t pid, uint32_t dv_ind, + rsmi_process_info_t *proc) { + TRY + if (proc == nullptr) { + return RSMI_STATUS_INVALID_ARGS; + } + // Check the device and kfdnode exist + GET_DEV_AND_KFDNODE_FROM_INDX + + std::unordered_set gpu_set; + gpu_set.insert(dev->kfd_gpu_id()); + int err = amd::smi::GetProcessInfoForPID(pid, proc, &gpu_set); + + if (err) { + return amd::smi::ErrnoToRsmiStatus(err); + } + + return RSMI_STATUS_SUCCESS; + + CATCH +} + rsmi_status_t rsmi_dev_xgmi_error_status(uint32_t dv_ind, rsmi_xgmi_status_t *status) { TRY diff --git a/tests/rocm_smi_test/functional/process_info_read.cc b/tests/rocm_smi_test/functional/process_info_read.cc index f82d8ff402..0f31ac77a9 100755 --- a/tests/rocm_smi_test/functional/process_info_read.cc +++ b/tests/rocm_smi_test/functional/process_info_read.cc @@ -184,6 +184,21 @@ void TestProcInfoRead::Run(void) { std::cout << dev_inds[i]; } std::cout << std::endl; + + // Get details of the resource by the process on a specific device + // if any process running on devices + for (i = 0; i < amt_allocd; ++i) { + rsmi_process_info_t proc_info; + err = rsmi_compute_process_info_by_device_get( + procs[j].process_id, dev_inds[i], &proc_info); + CHK_ERR_ASRT(err) + ASSERT_EQ(proc_info.process_id, procs[j].process_id); + ASSERT_EQ(proc_info.pasid, procs[j].pasid); + std::cout << "\t** Process ID: " << procs[j].process_id + << " on device " << dev_inds[i] << " VRAM Usage: " + << proc_info.vram_usage << " SDMA Usage: " << proc_info.sdma_usage + << " Compute Unit Usage: " << proc_info.cu_occupancy << std::endl; + } // Reset amt_allocd back to the amount acutally allocated amt_allocd = num_devices; }