Add rsmi_compute_process_gpus_get()

Given a process ID, give the device indices that process is
currently using.

Also:
* made corrections to how RSMI, amdgpu (ie, "card#") and
  KFD indicies translate from one another
* add a few missing error codes to rsmi_status_string()
* fix some formatting

Change-Id: Icd2cae66bb4fec768da96af7cf9cf8b8b66ec7f9


[ROCm/amdsmi commit: 2d6e15190c]

projects/amdsmi/include/rocm_smi/rocm_smi.h

@@ -2301,7 +2301,7 @@ rsmi_dev_counter_group_supported(uint32_t dv_ind, rsmi_event_group_t group);
  *
  *  @param[in] dv_ind a device index
  *
- *  @param[in] type the type of performance event to create
+ *  @param[in] type the ::rsmi_event_type_t of performance event to create
  *
  *  @param[inout] evnt_handle A pointer to a ::rsmi_event_handle_t which will be
  *  associated with a newly allocated counter
@@ -2448,24 +2448,64 @@ rsmi_compute_process_info_get(rsmi_process_info_t *procs, uint32_t *num_items);
 /**
  *  @brief Get process information about a specific process
  *
- *  @details Given a pointer to an ::rsmi_process_info_t @p proc and a process id
+ *  @details Given a pointer to an ::rsmi_process_info_t @p proc and a process
+ *  id
  *  @p pid, this function will write the process information for @p pid, if
  *  available, to the memory pointed to by @p proc.
  *
- *  @param[in] pid The process ID for which process information is being requested
+ *  @param[in] pid The process ID for which process information is being
+ *  requested
  *
  *  @param[inout] proc a pointer to a ::rsmi_process_info_t to which
  *  process information for @p pid will be written if it is found.
  *
  *  @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
  *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
- *  @retval ::RSMI_STATUS_NOT_FOUND is returned if there was no process information
+ *  @retval ::RSMI_STATUS_NOT_FOUND is returned if there was no process
+ *  information
  *  found for the provided @p pid
  *
  */
 rsmi_status_t
 rsmi_compute_process_info_by_pid_get(uint32_t pid, rsmi_process_info_t *proc);
 
+/**
+ *  @brief Get the device indices currently being used by a process
+ *
+ *  @details Given a process id @p pid, a non-NULL pointer to an array of
+ *  uint32_t's @p dv_indices of length *@p num_devices, this function will
+ *  write up to @p num_devices device indices to the memory pointed to by
+ *  @p dv_indices. If @p dv_indices is not NULL, @p num_devices will be
+ *  updated with the number of gpu's currently being used by process @p pid.
+ *  If @p dv_indices is NULL, @p dv_indices will be updated with the number of
+ *  gpus currently being used by @p pid. Calling this function with @p
+ *  dv_indices being NULL is a way to determine how much memory is required
+ *  for when @p dv_indices is not NULL.
+ *
+ *  @param[in] pid The process id of the process for which the number of gpus
+ *  currently being used is requested
+ *
+ *  @param[inout] dv_indices a pointer to memory provided by the caller to
+ *  which indices of devices currently being used by the process will be
+ *  written. This may be NULL in which case only @p num_devices will be
+ *  updated with the number of devices being used.
+ *
+ *  @param[inout] num_devices A pointer to a uint32_t, which on input, should
+ *  contain the amount of memory in uint32_t's which have been provided by the
+ *  @p dv_indices argument. On output, if @p dv_indices is non-NULL, this will
+ *  be updated with the number uint32_t's actually written. If @p dv_indices is
+ *  NULL, this argument will be updated with the number devices being used.
+ *
+ *  @retval ::RSMI_STATUS_SUCCESS is returned upon successful call
+ *  @retval ::RSMI_STATUS_INVALID_ARGS the provided arguments are not valid
+ *  @retval ::RSMI_STATUS_INSUFFICIENT_SIZE is returned if there were more
+ *  gpu indices that could have been written, but not enough space was
+ *  provided as indicated by @p dv_indices and @p num_devices, on input.
+ *
+ */
+rsmi_status_t
+rsmi_compute_process_gpus_get(uint32_t pid, uint32_t *dv_indices,
+                                                       uint32_t *num_devices);
 
 /** @} */  // end of SysInfo
 

projects/amdsmi/include/rocm_smi/rocm_smi_device.h

@@ -178,11 +178,9 @@ class Device {
     int readDevInfo(DevInfoTypes type, std::vector<std::string> *retVec);
     int writeDevInfo(DevInfoTypes type, uint64_t val);
     int writeDevInfo(DevInfoTypes type, std::string val);
-    int populateKFDNodeProperties(bool force_update = false);
-    int getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val);
 
-    uint32_t index(void) const {return index_;}
-    void set_index(uint32_t index) {index_ = index;}
+    uint32_t index(void) const {return card_indx_;}
+    void set_card_index(uint32_t index) {card_indx_ = index;}
     uint32_t drm_render_minor(void) const {return drm_render_minor_;}
     void set_drm_render_minor(uint32_t minor) {drm_render_minor_ = minor;}
     static rsmi_dev_perf_level perfLvlStrToEnum(std::string s);
@@ -192,6 +190,8 @@ class Device {
     evt::dev_evt_grp_set_t* supported_event_groups(void) {
                                              return &supported_event_groups_;}
     SupportedFuncMap *supported_funcs(void) {return &supported_funcs_;}
+    uint64_t kfd_gpu_id(void) const {return kfd_gpu_id_;}
+    void set_kfd_gpu_id(uint64_t id) {kfd_gpu_id_ = id;}
     void fillSupportedFuncs(void);
     void DumpSupportedFunctions(void);
     bool DeviceAPISupported(std::string name, uint64_t variant,
@@ -202,20 +202,20 @@ class Device {
     std::shared_ptr<PowerMon> power_monitor_;
     std::string path_;
     shared_mutex_t mutex_;
-    uint32_t index_;
+    uint32_t card_indx_;  // This index corresponds to the drm index (ie, card#)
     uint32_t drm_render_minor_;
     const RocmSMI_env_vars *env_;
     template <typename T> int openSysfsFileStream(DevInfoTypes type, T *fs,
                                                    const char *str = nullptr);
-
     int readDevInfoStr(DevInfoTypes type, std::string *retStr);
     int readDevInfoMultiLineStr(DevInfoTypes type,
                                             std::vector<std::string> *retVec);
     int writeDevInfoStr(DevInfoTypes type, std::string valStr);
     uint64_t bdfid_;
+    uint64_t kfd_gpu_id_;
     std::unordered_set<rsmi_event_group_t,
                        evt::RSMIEventGrpHashFunction> supported_event_groups_;
-    std::map<std::string, uint64_t> kfdNodePropMap_;
+    // std::map<std::string, uint64_t> kfdNodePropMap_;
     SupportedFuncMap supported_funcs_;
 };
 

projects/amdsmi/include/rocm_smi/rocm_smi_kfd.h

@@ -45,18 +45,50 @@
 
 #include <string>
 #include <vector>
+#include <unordered_set>
+#include <memory>
+#include <map>
 
 #include "rocm_smi/rocm_smi.h"
+#include "rocm_smi/rocm_smi_device.h"
 
 namespace amd {
 namespace smi {
 
+class KFDNode {
+ public:
+    explicit KFDNode(uint32_t node_ind) : node_indx_(node_ind) {}
+    ~KFDNode();
+
+    int Initialize();
+    int ReadProperties(void);
+    int get_property_value(std::string property, uint64_t *value);
+    uint64_t gpu_id(void) const {return gpu_id_;}
+    std::string name(void) const {return name_;}
+    std::shared_ptr<Device> amdgpu_device(void) const {return amdgpu_device_;}
+    uint32_t amdgpu_dev_index(void) const {return amdgpu_dev_index_;}
+    void set_amdgpu_dev_index(uint32_t val) {amdgpu_dev_index_ = val;}
+
+ private:
+    uint32_t node_indx_;
+    uint32_t amdgpu_dev_index_;
+    uint64_t gpu_id_;
+    std::string name_;
+    std::map<std::string, uint64_t> properties_;
+    std::shared_ptr<Device> amdgpu_device_;
+};
+
+int
+DiscoverKFDNodes(std::map<uint64_t, std::shared_ptr<KFDNode>> *nodes);
+
 int
 GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
                                                    uint32_t *num_procs_found);
 int
 GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc);
 
+int
+GetProcessGPUs(uint32_t pid, std::unordered_set<uint64_t> *gpu_count);
 int
 ReadKFDDeviceProperties(uint32_t dev_id, std::vector<std::string> *retVec);
 

projects/amdsmi/include/rocm_smi/rocm_smi_main.h

@@ -51,7 +51,10 @@
 #include <set>
 #include <string>
 #include <cstdint>
+#include <unordered_map>
+#include <map>
 
+#include "rocm_smi/rocm_smi_kfd.h"
 #include "rocm_smi/rocm_smi_device.h"
 #include "rocm_smi/rocm_smi_monitor.h"
 #include "rocm_smi/rocm_smi_power_mon.h"
@@ -71,7 +74,7 @@ class RocmSMI {
 
     static std::vector<std::shared_ptr<amd::smi::Device>>&
                                   monitor_devices() {return s_monitor_devices;}
-    uint32_t DiscoverDevices(void);
+    uint32_t DiscoverAmdgpuDevices(void);
     uint32_t DiscoverAMDPowerMonitors(bool force_update = false);
 
     // Will execute "func" for every Device object known about, or until func
@@ -84,17 +87,21 @@ class RocmSMI {
 
     uint32_t euid() const {return euid_;}
 
+    std::map<uint64_t, std::shared_ptr<KFDNode>> & kfd_node_map(void) {
+      return kfd_node_map_;}
+
  private:
     std::vector<std::shared_ptr<Device>> devices_;
+    std::map<uint64_t, std::shared_ptr<KFDNode>> kfd_node_map_;
     std::vector<std::shared_ptr<Monitor>> monitors_;
     std::vector<std::shared_ptr<PowerMon>> power_mons_;
-
     std::set<std::string> amd_monitor_types_;
     void AddToDeviceList(std::string dev_name);
     void GetEnvVariables(void);
     uint32_t DiscoverAMDMonitors(void);
 
     static std::vector<std::shared_ptr<amd::smi::Device>> s_monitor_devices;
+
     RocmSMI_env_vars env_vars_;
     uint64_t init_options_;
     uint32_t euid_;