diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h index 28a5f52ce9..a3e9d65903 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi.h @@ -328,7 +328,7 @@ typedef enum { /** * @brief Pre-set Profile Selections. These bitmasks can be AND'd with the * ::rsmi_power_profile_status_t.available_profiles returned from - * ::rsmi_dev_power_profile_presets_get() to determine which power profiles + * ::rsmi_dev_power_profile_presets_get to determine which power profiles * are supported by the system. */ typedef enum { @@ -996,6 +996,19 @@ rsmi_dev_pci_bandwidth_get(uint32_t dv_ind, rsmi_pcie_bandwidth_t *bandwidth); * (BDFID) associated with device @p dv_ind to the value pointed to by * @p bdfid. * + * The format of @p bdfid will be as follows: + * + * BDFID = ((DOMAIN & 0xffffffff) << 32) | ((BUS & 0xff) << 8) | + * ((DEVICE & 0x1f) <<3 ) | (FUNCTION & 0x7) + * + * | Name | Field | + * ---------- | ------- | + * | Domain | [64:32] | + * | Reserved | [31:16] | + * | Bus | [15: 8] | + * | Device | [ 7: 3] | + * | Function | [ 2: 0] | + * * @param[in] dv_ind a device index * * @param[inout] bdfid a pointer to uint64_t to which the device bdfid value @@ -1851,9 +1864,9 @@ rsmi_dev_firmware_version_get(uint32_t dv_ind, rsmi_fw_block_t block, * * @retval ::RSMI_STATUS_SUCCESS is returned upon successful call. * - * ::RSMI_NOT_SUPPORTED will be returned if either ECC is not enabled for the - * specified block @p block, or if there is no kernel support for ECC for - * that block. + * ::RSMI_STATUS_NOT_SUPPORTED will be returned if either ECC is not enabled + * for the specified block @p block, or if there is no kernel support for ECC + * for that block. * */ rsmi_status_t rsmi_dev_ecc_count_get(uint32_t dv_ind, diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h index 73ec27de69..6803cfe3ef 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_device.h @@ -51,6 +51,7 @@ #include #include #include +#include #include "rocm_smi/rocm_smi_monitor.h" #include "rocm_smi/rocm_smi_power_mon.h" @@ -64,6 +65,38 @@ extern "C" { namespace amd { namespace smi { +enum DevKFDNodePropTypes { + kDevKFDNodePropCachesCnt, + kDevKFDNodePropIoLinksCnt, + kDevKFDNodePropCPUCoreIdBase, + kDevKFDNodePropSimdIdBase, + kDevKFDNodePropMaxWavePerSimd, + kDevKFDNodePropLdsSz, + kDevKFDNodePropGdsSz, + kDevKFDNodePropNumGWS, + kDevKFDNodePropWaveFrontSize, + kDevKFDNodePropArrCnt, + kDevKFDNodePropSimdArrPerEng, + kDevKFDNodePropCuPerSimdArr, + kDevKFDNodePropSimdPerCU, + kDevKFDNodePropMaxSlotsScratchCu, + kDevKFDNodePropVendorId, + kDevKFDNodePropDeviceId, + kDevKFDNodePropLocationId, + kDevKFDNodePropDrmRenderMinor, + kDevKFDNodePropHiveId, + kDevKFDNodePropNumSdmaEngines, + kDevKFDNodePropNumSdmaXgmiEngs, + kDevKFDNodePropMaxEngClkFComp, + kDevKFDNodePropLocMemSz, + kDevKFDNodePropFwVer, + kDevKFDNodePropCapability, + kDevKFDNodePropDbgProp, + kDevKFDNodePropSdmaFwVer, + kDevKFDNodePropMaxEngClkCComp, + kDevKFDNodePropDomain, +}; + enum DevInfoTypes { kDevPerfLevel, kDevOverDriveLevel, @@ -139,6 +172,9 @@ class Device { int readDevInfo(DevInfoTypes type, std::vector *retVec); int writeDevInfo(DevInfoTypes type, uint64_t val); int writeDevInfo(DevInfoTypes type, std::string val); + int populateKFDNodeProperties(bool force_update = false); + int getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val); + uint32_t index(void) const {return index_;} void set_index(uint32_t index) {index_ = index;} uint32_t drm_render_minor(void) const {return drm_render_minor_;} @@ -146,7 +182,6 @@ class Device { static rsmi_dev_perf_level perfLvlStrToEnum(std::string s); uint64_t bdfid(void) const {return bdfid_;} void set_bdfid(uint64_t val) {bdfid_ = val;} - uint64_t get_bdfid(void) const {return bdfid_;} pthread_mutex_t *mutex(void) {return mutex_.ptr;} evt::dev_evt_grp_set_t* supported_event_groups(void) { return &supported_event_groups_;} @@ -169,6 +204,7 @@ class Device { uint64_t bdfid_; std::unordered_set supported_event_groups_; + std::map kfdNodePropMap_; }; } // namespace smi diff --git a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h index f9b6440c86..0ae429d5a7 100755 --- a/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h +++ b/projects/rocm-smi-lib/include/rocm_smi/rocm_smi_kfd.h @@ -43,6 +43,9 @@ #ifndef INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_ #define INCLUDE_ROCM_SMI_ROCM_SMI_KFD_H_ +#include +#include + #include "rocm_smi/rocm_smi.h" namespace amd { @@ -54,6 +57,9 @@ GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated, int GetProcessInfoForPID(uint32_t pid, rsmi_process_info_t *proc); +int +ReadKFDDeviceProperties(uint32_t dev_id, std::vector *retVec); + } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/src/rocm_smi.cc b/projects/rocm-smi-lib/src/rocm_smi.cc index 742379d210..d2ac0f9a49 100755 --- a/projects/rocm-smi-lib/src/rocm_smi.cc +++ b/projects/rocm-smi-lib/src/rocm_smi.cc @@ -621,7 +621,35 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) { DEVICE_MUTEX - *bdfid = dev->get_bdfid(); + *bdfid = dev->bdfid(); + + int32_t ret = dev->populateKFDNodeProperties(); + + if (ret) { + return errno_to_rsmi_status(errno); + } + + uint64_t domain = 0; + + ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain); + + if (ret == EINVAL) { + // "domain" is not found in properties file; just go with the 16 bit + // domain already found + return RSMI_STATUS_SUCCESS; + } + + // Replace the 16 bit domain originally set like this: + // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | + // ((device& 0x1f) <<3 ) | (function & 0x7) + // with this: + // BDFID = (( & 0xffffffff) << 32) | (( & 0xff) << 8) | + // ((device& 0x1f) <<3 ) | (function & 0x7) + + assert((domain & 0xFFFFFFFF00000000) == 0); + (*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain + *bdfid |= (domain & 0xFFFFFFFF) << 32; + return RSMI_STATUS_SUCCESS; CATCH } diff --git a/projects/rocm-smi-lib/src/rocm_smi_device.cc b/projects/rocm-smi-lib/src/rocm_smi_device.cc index dc6c0a5b53..1caae50516 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_device.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_device.cc @@ -61,6 +61,7 @@ #include "rocm_smi/rocm_smi.h" #include "rocm_smi/rocm_smi_exception.h" #include "rocm_smi/rocm_smi_utils.h" +#include "rocm_smi/rocm_smi_kfd.h" extern "C" { #include "shared_mutex.h" // NOLINT @@ -139,7 +140,71 @@ static const char *kDevFwVersionUvdFName = "fw_version/uvd_fw_version"; static const char *kDevFwVersionVceFName = "fw_version/vce_fw_version"; static const char *kDevFwVersionVcnFName = "fw_version/vcn_fw_version"; +static const char *kDevKFDNodePropCachesCntSName = "caches_count"; +static const char *kDevKFDNodePropIoLinksCntSName = "io_links_count"; +static const char *kDevKFDNodePropCPUCoreIdBaseSName = "cpu_core_id_base"; +static const char *kDevKFDNodePropSimdIdBaseSName = "simd_id_base"; +static const char *kDevKFDNodePropMaxWavePerSimdSName = "max_waves_per_simd"; +static const char *kDevKFDNodePropLdsSzSName = "lds_size_in_kb"; +static const char *kDevKFDNodePropGdsSzSName = "gds_size_in_kb"; +static const char *kDevKFDNodePropNumGWSSName = "num_gws"; +static const char *kDevKFDNodePropWaveFrontSizeSName = "wave_front_size"; +static const char *kDevKFDNodePropArrCntSName = "array_count"; +static const char *kDevKFDNodePropSimdArrPerEngSName = "simd_arrays_per_engine"; +static const char *kDevKFDNodePropCuPerSimdArrSName = "cu_per_simd_array"; +static const char *kDevKFDNodePropSimdPerCUSName = "simd_per_cu"; +static const char *kDevKFDNodePropMaxSlotsScratchCuSName = + "max_slots_scratch_cu"; +static const char *kDevKFDNodePropVendorIdSName = "vendor_id"; +static const char *kDevKFDNodePropDeviceIdSName = "device_id"; +static const char *kDevKFDNodePropLocationIdSName = "location_id"; +static const char *kDevKFDNodePropDrmRenderMinorSName = "drm_render_minor"; +static const char *kDevKFDNodePropHiveIdSName = "hive_id"; +static const char *kDevKFDNodePropNumSdmaEnginesSName = "num_sdma_engines"; +static const char *kDevKFDNodePropNumSdmaXgmiEngsSName = + "num_sdma_xgmi_engines"; +static const char *kDevKFDNodePropMaxEngClkFCompSName = + "max_engine_clk_fcompute"; +static const char *kDevKFDNodePropLocMemSzSName = "local_mem_size"; +static const char *kDevKFDNodePropFwVerSName = "fw_version"; +static const char *kDevKFDNodePropCapabilitySName = "capability"; +static const char *kDevKFDNodePropDbgPropSName = "debug_prop"; +static const char *kDevKFDNodePropSdmaFwVerSName = "sdma_fw_version"; +static const char *kDevKFDNodePropMaxEngClkCCompSName = + "max_engine_clk_ccompute"; +static const char *kDevKFDNodePropDomainSName = "domain"; +static const std::map kDevKFDPropNameMap = { + {kDevKFDNodePropCachesCnt, kDevKFDNodePropCachesCntSName}, + {kDevKFDNodePropIoLinksCnt, kDevKFDNodePropIoLinksCntSName}, + {kDevKFDNodePropCPUCoreIdBase, kDevKFDNodePropCPUCoreIdBaseSName}, + {kDevKFDNodePropSimdIdBase, kDevKFDNodePropSimdIdBaseSName}, + {kDevKFDNodePropMaxWavePerSimd, kDevKFDNodePropMaxWavePerSimdSName}, + {kDevKFDNodePropLdsSz, kDevKFDNodePropLdsSzSName}, + {kDevKFDNodePropGdsSz, kDevKFDNodePropGdsSzSName}, + {kDevKFDNodePropNumGWS, kDevKFDNodePropNumGWSSName}, + {kDevKFDNodePropWaveFrontSize, kDevKFDNodePropWaveFrontSizeSName}, + {kDevKFDNodePropArrCnt, kDevKFDNodePropArrCntSName}, + {kDevKFDNodePropSimdArrPerEng, kDevKFDNodePropSimdArrPerEngSName}, + {kDevKFDNodePropCuPerSimdArr, kDevKFDNodePropCuPerSimdArrSName}, + {kDevKFDNodePropSimdPerCU, kDevKFDNodePropSimdPerCUSName}, + {kDevKFDNodePropMaxSlotsScratchCu, kDevKFDNodePropMaxSlotsScratchCuSName}, + {kDevKFDNodePropVendorId, kDevKFDNodePropVendorIdSName}, + {kDevKFDNodePropDeviceId, kDevKFDNodePropDeviceIdSName}, + {kDevKFDNodePropLocationId, kDevKFDNodePropLocationIdSName}, + {kDevKFDNodePropDrmRenderMinor, kDevKFDNodePropDrmRenderMinorSName}, + {kDevKFDNodePropHiveId, kDevKFDNodePropHiveIdSName}, + {kDevKFDNodePropNumSdmaEngines, kDevKFDNodePropNumSdmaEnginesSName}, + {kDevKFDNodePropNumSdmaXgmiEngs, kDevKFDNodePropNumSdmaXgmiEngsSName}, + {kDevKFDNodePropMaxEngClkFComp, kDevKFDNodePropMaxEngClkFCompSName}, + {kDevKFDNodePropLocMemSz, kDevKFDNodePropLocMemSzSName}, + {kDevKFDNodePropFwVer, kDevKFDNodePropFwVerSName}, + {kDevKFDNodePropCapability, kDevKFDNodePropCapabilitySName}, + {kDevKFDNodePropDbgProp, kDevKFDNodePropDbgPropSName}, + {kDevKFDNodePropSdmaFwVer, kDevKFDNodePropSdmaFwVerSName}, + {kDevKFDNodePropMaxEngClkCComp, kDevKFDNodePropMaxEngClkCCompSName}, + {kDevKFDNodePropDomain, kDevKFDNodePropDomainSName}, +}; static const std::map kDevAttribNameMap = { {kDevPerfLevel, kDevPerfLevelFName}, {kDevOverDriveLevel, kDevOverDriveLevelFName}, @@ -520,6 +585,53 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) { return 0; } +int Device::populateKFDNodeProperties(bool force_update) { + int ret; + + std::vector propVec; + + if (kfdNodePropMap_.size() > 0 && !force_update) { + return 0; + } + + ret = ReadKFDDeviceProperties(index_, &propVec); + + if (ret) { + return ret; + } + + std::string key_str; + // std::string val_str; + uint64_t val_int; // Assume all properties are unsigned integers for now + std::istringstream fs; + + for (uint32_t i = 0; i < propVec.size(); ++i) { + fs.str(propVec[i]); + fs >> key_str; + fs >> val_int; + + kfdNodePropMap_[key_str] = val_int; + + fs.str(""); + fs.clear(); + } + + return 0; +} + +int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) { + assert(val != nullptr); + assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end()); + + const char *prop_name = kDevKFDPropNameMap.at(prop); + if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) { + return EINVAL; + } + + *val = kfdNodePropMap_.at(prop_name); + return 0; +} + #undef RET_IF_NONZERO } // namespace smi } // namespace amd diff --git a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc index a118b86f73..32149afebd 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_kfd.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_kfd.cc @@ -62,7 +62,7 @@ namespace amd { namespace smi { static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc"; - +static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes"; // Sysfs file names static const char *kKFDPasidFName = "pasid"; @@ -70,6 +70,50 @@ static bool is_number(const std::string &s) { return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit); } +int ReadKFDDeviceProperties(uint32_t dev_id, + std::vector *retVec) { + std::string line; + int ret; + std::ifstream fs; + std::string properties_path = kKFDNodesPathRoot; + bool reg_file; + + assert(retVec != nullptr); + + properties_path += '/'; + properties_path += std::to_string(dev_id); + properties_path += "/properties"; + + ret = isRegularFile(properties_path, ®_file); + + if (ret != 0) { + return ret; + } + if (!reg_file) { + return ENOENT; + } + + fs.open(properties_path); + + if (!fs.is_open()) { + return errno; + } + + while (std::getline(fs, line)) { + retVec->push_back(line); + } + + if (retVec->size() == 0) { + return 0; + } + // Remove any *trailing* empty (whitespace) lines + while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) { + retVec->pop_back(); + } + return 0; +} + + int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated, uint32_t *num_procs_found) { assert(num_procs_found != nullptr); diff --git a/projects/rocm-smi-lib/src/rocm_smi_main.cc b/projects/rocm-smi-lib/src/rocm_smi_main.cc index f5d3e63b0a..08d35dda5a 100755 --- a/projects/rocm-smi-lib/src/rocm_smi_main.cc +++ b/projects/rocm-smi-lib/src/rocm_smi_main.cc @@ -73,7 +73,6 @@ static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""}; namespace amd { namespace smi { - static uint32_t GetDeviceIndex(const std::string s) { std::string t = s; size_t tmp = t.find_last_not_of("0123456789"); @@ -123,12 +122,13 @@ static int SameDevice(const std::string fileA, const std::string fileB) { // XXXX:XX:XX.X, // domain:bus:device.function // -// where X is a hex integer (lower case is expected) -static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { +// where X is a hex integer (lower case is expected). If so, write the value +// to bdfid +static bool bdfid_from_path(const std::string in_name, uint64_t *bdfid) { char *p = nullptr; char *name_start; char name[13] = {'\0'}; - uint32_t tmp; + uint64_t tmp; assert(bdfid != nullptr); @@ -139,7 +139,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { tmp = in_name.copy(name, 12); assert(tmp == 12); - // BDFID = (( & 0xffff) << 13) | (( & 0x1f) << 8) | + // BDFID = (( & 0xffff) << 32) | (( & 0xff) << 8) | // ((device& 0x1f) <<3 ) | (function & 0x7) *bdfid = 0; name_start = name; @@ -150,7 +150,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) { if (*p != ':' || p - name_start != 4) { return false; } - *bdfid |= tmp << 13; + *bdfid |= tmp << 32; // Match this: xxxx:XX:xx.x p++; // Skip past ':' @@ -205,7 +205,7 @@ static uint32_t ConstructBDFID(std::string path, uint64_t *bdfid) { slash_i = tpath_str.find_last_of('/', end_i); tmp = tpath_str.substr(slash_i + 1, end_i - slash_i); - if (is_bdfid_path_str(tmp, bdfid)) { + if (bdfid_from_path(tmp, bdfid)) { return 0; } end_i = slash_i - 1; @@ -335,7 +335,6 @@ RocmSMI::AddToDeviceList(std::string dev_name) { uint32_t d_index = GetDeviceIndex(d_name); dev->set_drm_render_minor(GetDrmRenderMinor(dev_path)); dev->set_index(d_index); - GetSupportedEventGroups(d_index, dev->supported_event_groups()); devices_.push_back(dev);