AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make bdfid use 32 bit domain if possible

Browse Source 
If the 32-bit domain is found in the kfd node properties for
a device, then it will be used when constructing the bdfid.
If it's not present, it will continue to use the 16 bit version.

Also, whether or not 32b or 16b are used for the domain, the
domain will now be placed in the upper 32b of the 64b bdfid.

* Fixed some unrelated doxygen issues

Change-Id: Icb5116daa1ab45ee305bdbe6cd5df5736dd3ffa3


[ROCm/rocm_smi_lib commit: 469af303d6]
This commit is contained in:
Chris Freehill
2019-08-25 19:31:46 -05:00
parent e35a20518c
commit 0bcdfe553d
7 changed files with 253 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
projects/rocm-smi-lib/src/rocm_smi.cc
+29 -1
View File
@@ -621,7 +621,35 @@ rsmi_dev_pci_id_get(uint32_t dv_ind, uint64_t *bdfid) {
DEVICE_MUTEX
*bdfid = dev->get_bdfid();
*bdfid = dev->bdfid();
int32_t ret = dev->populateKFDNodeProperties();
if (ret) {
return errno_to_rsmi_status(errno);
}
uint64_t domain = 0;
ret = dev->getKFDNodeProperty(amd::smi::kDevKFDNodePropDomain, &domain);
if (ret == EINVAL) {
// "domain" is not found in properties file; just go with the 16 bit
// domain already found
return RSMI_STATUS_SUCCESS;
}
// Replace the 16 bit domain originally set like this:
// BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
// with this:
// BDFID = ((<DOMAIN> & 0xffffffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
assert((domain & 0xFFFFFFFF00000000) == 0);
(*bdfid) &= 0xFFFF; // Clear out the old 16 bit domain
*bdfid |= (domain & 0xFFFFFFFF) << 32;
return RSMI_STATUS_SUCCESS;
CATCH
}
projects/rocm-smi-lib/src/rocm_smi_device.cc
+112
View File
@@ -61,6 +61,7 @@
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi_exception.h"
#include "rocm_smi/rocm_smi_utils.h"
#include "rocm_smi/rocm_smi_kfd.h"
extern "C" {
#include "shared_mutex.h" // NOLINT
@@ -139,7 +140,71 @@ static const char *kDevFwVersionUvdFName = "fw_version/uvd_fw_version";
static const char *kDevFwVersionVceFName = "fw_version/vce_fw_version";
static const char *kDevFwVersionVcnFName = "fw_version/vcn_fw_version";
static const char *kDevKFDNodePropCachesCntSName = "caches_count";
static const char *kDevKFDNodePropIoLinksCntSName = "io_links_count";
static const char *kDevKFDNodePropCPUCoreIdBaseSName = "cpu_core_id_base";
static const char *kDevKFDNodePropSimdIdBaseSName = "simd_id_base";
static const char *kDevKFDNodePropMaxWavePerSimdSName = "max_waves_per_simd";
static const char *kDevKFDNodePropLdsSzSName = "lds_size_in_kb";
static const char *kDevKFDNodePropGdsSzSName = "gds_size_in_kb";
static const char *kDevKFDNodePropNumGWSSName = "num_gws";
static const char *kDevKFDNodePropWaveFrontSizeSName = "wave_front_size";
static const char *kDevKFDNodePropArrCntSName = "array_count";
static const char *kDevKFDNodePropSimdArrPerEngSName = "simd_arrays_per_engine";
static const char *kDevKFDNodePropCuPerSimdArrSName = "cu_per_simd_array";
static const char *kDevKFDNodePropSimdPerCUSName = "simd_per_cu";
static const char *kDevKFDNodePropMaxSlotsScratchCuSName =
"max_slots_scratch_cu";
static const char *kDevKFDNodePropVendorIdSName = "vendor_id";
static const char *kDevKFDNodePropDeviceIdSName = "device_id";
static const char *kDevKFDNodePropLocationIdSName = "location_id";
static const char *kDevKFDNodePropDrmRenderMinorSName = "drm_render_minor";
static const char *kDevKFDNodePropHiveIdSName = "hive_id";
static const char *kDevKFDNodePropNumSdmaEnginesSName = "num_sdma_engines";
static const char *kDevKFDNodePropNumSdmaXgmiEngsSName =
"num_sdma_xgmi_engines";
static const char *kDevKFDNodePropMaxEngClkFCompSName =
"max_engine_clk_fcompute";
static const char *kDevKFDNodePropLocMemSzSName = "local_mem_size";
static const char *kDevKFDNodePropFwVerSName = "fw_version";
static const char *kDevKFDNodePropCapabilitySName = "capability";
static const char *kDevKFDNodePropDbgPropSName = "debug_prop";
static const char *kDevKFDNodePropSdmaFwVerSName = "sdma_fw_version";
static const char *kDevKFDNodePropMaxEngClkCCompSName =
"max_engine_clk_ccompute";
static const char *kDevKFDNodePropDomainSName = "domain";
static const std::map<DevKFDNodePropTypes, const char *> kDevKFDPropNameMap = {
{kDevKFDNodePropCachesCnt, kDevKFDNodePropCachesCntSName},
{kDevKFDNodePropIoLinksCnt, kDevKFDNodePropIoLinksCntSName},
{kDevKFDNodePropCPUCoreIdBase, kDevKFDNodePropCPUCoreIdBaseSName},
{kDevKFDNodePropSimdIdBase, kDevKFDNodePropSimdIdBaseSName},
{kDevKFDNodePropMaxWavePerSimd, kDevKFDNodePropMaxWavePerSimdSName},
{kDevKFDNodePropLdsSz, kDevKFDNodePropLdsSzSName},
{kDevKFDNodePropGdsSz, kDevKFDNodePropGdsSzSName},
{kDevKFDNodePropNumGWS, kDevKFDNodePropNumGWSSName},
{kDevKFDNodePropWaveFrontSize, kDevKFDNodePropWaveFrontSizeSName},
{kDevKFDNodePropArrCnt, kDevKFDNodePropArrCntSName},
{kDevKFDNodePropSimdArrPerEng, kDevKFDNodePropSimdArrPerEngSName},
{kDevKFDNodePropCuPerSimdArr, kDevKFDNodePropCuPerSimdArrSName},
{kDevKFDNodePropSimdPerCU, kDevKFDNodePropSimdPerCUSName},
{kDevKFDNodePropMaxSlotsScratchCu, kDevKFDNodePropMaxSlotsScratchCuSName},
{kDevKFDNodePropVendorId, kDevKFDNodePropVendorIdSName},
{kDevKFDNodePropDeviceId, kDevKFDNodePropDeviceIdSName},
{kDevKFDNodePropLocationId, kDevKFDNodePropLocationIdSName},
{kDevKFDNodePropDrmRenderMinor, kDevKFDNodePropDrmRenderMinorSName},
{kDevKFDNodePropHiveId, kDevKFDNodePropHiveIdSName},
{kDevKFDNodePropNumSdmaEngines, kDevKFDNodePropNumSdmaEnginesSName},
{kDevKFDNodePropNumSdmaXgmiEngs, kDevKFDNodePropNumSdmaXgmiEngsSName},
{kDevKFDNodePropMaxEngClkFComp, kDevKFDNodePropMaxEngClkFCompSName},
{kDevKFDNodePropLocMemSz, kDevKFDNodePropLocMemSzSName},
{kDevKFDNodePropFwVer, kDevKFDNodePropFwVerSName},
{kDevKFDNodePropCapability, kDevKFDNodePropCapabilitySName},
{kDevKFDNodePropDbgProp, kDevKFDNodePropDbgPropSName},
{kDevKFDNodePropSdmaFwVer, kDevKFDNodePropSdmaFwVerSName},
{kDevKFDNodePropMaxEngClkCComp, kDevKFDNodePropMaxEngClkCCompSName},
{kDevKFDNodePropDomain, kDevKFDNodePropDomainSName},
};
static const std::map<DevInfoTypes, const char *> kDevAttribNameMap = {
{kDevPerfLevel, kDevPerfLevelFName},
{kDevOverDriveLevel, kDevOverDriveLevelFName},
@@ -520,6 +585,53 @@ int Device::readDevInfo(DevInfoTypes type, std::string *val) {
return 0;
}
int Device::populateKFDNodeProperties(bool force_update) {
int ret;
std::vector<std::string> propVec;
if (kfdNodePropMap_.size() > 0 && !force_update) {
return 0;
}
ret = ReadKFDDeviceProperties(index_, &propVec);
if (ret) {
return ret;
}
std::string key_str;
// std::string val_str;
uint64_t val_int; // Assume all properties are unsigned integers for now
std::istringstream fs;
for (uint32_t i = 0; i < propVec.size(); ++i) {
fs.str(propVec[i]);
fs >> key_str;
fs >> val_int;
kfdNodePropMap_[key_str] = val_int;
fs.str("");
fs.clear();
}
return 0;
}
int Device::getKFDNodeProperty(DevKFDNodePropTypes prop, uint64_t *val) {
assert(val != nullptr);
assert(kDevKFDPropNameMap.find(prop) != kDevKFDPropNameMap.end());
const char *prop_name = kDevKFDPropNameMap.at(prop);
if (kfdNodePropMap_.find(prop_name) == kfdNodePropMap_.end()) {
return EINVAL;
}
*val = kfdNodePropMap_.at(prop_name);
return 0;
}
#undef RET_IF_NONZERO
} // namespace smi
} // namespace amd
projects/rocm-smi-lib/src/rocm_smi_kfd.cc
+45 -1
View File
@@ -62,7 +62,7 @@ namespace amd {
namespace smi {
static const char *kKFDProcPathRoot = "/sys/class/kfd/kfd/proc";
static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
// Sysfs file names
static const char *kKFDPasidFName = "pasid";
@@ -70,6 +70,50 @@ static bool is_number(const std::string &s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
int ReadKFDDeviceProperties(uint32_t dev_id,
std::vector<std::string> *retVec) {
std::string line;
int ret;
std::ifstream fs;
std::string properties_path = kKFDNodesPathRoot;
bool reg_file;
assert(retVec != nullptr);
properties_path += '/';
properties_path += std::to_string(dev_id);
properties_path += "/properties";
ret = isRegularFile(properties_path, &reg_file);
if (ret != 0) {
return ret;
}
if (!reg_file) {
return ENOENT;
}
fs.open(properties_path);
if (!fs.is_open()) {
return errno;
}
while (std::getline(fs, line)) {
retVec->push_back(line);
}
if (retVec->size() == 0) {
return 0;
}
// Remove any *trailing* empty (whitespace) lines
while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
retVec->pop_back();
}
return 0;
}
int GetProcessInfo(rsmi_process_info_t *procs, uint32_t num_allocated,
uint32_t *num_procs_found) {
assert(num_procs_found != nullptr);
projects/rocm-smi-lib/src/rocm_smi_main.cc
+7 -8
View File
@@ -73,7 +73,6 @@ static const char *kAMDMonitorTypes[] = {"radeon", "amdgpu", ""};
namespace amd {
namespace smi {
static uint32_t GetDeviceIndex(const std::string s) {
std::string t = s;
size_t tmp = t.find_last_not_of("0123456789");
@@ -123,12 +122,13 @@ static int SameDevice(const std::string fileA, const std::string fileB) {
// XXXX:XX:XX.X,
// domain:bus:device.function
//
// where X is a hex integer (lower case is expected)
static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
// where X is a hex integer (lower case is expected). If so, write the value
// to bdfid
static bool bdfid_from_path(const std::string in_name, uint64_t *bdfid) {
char *p = nullptr;
char *name_start;
char name[13] = {'\0'};
uint32_t tmp;
uint64_t tmp;
assert(bdfid != nullptr);
@@ -139,7 +139,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
tmp = in_name.copy(name, 12);
assert(tmp == 12);
// BDFID = ((<DOMAIN> & 0xffff) << 13) | ((<BUS> & 0x1f) << 8) |
// BDFID = ((<DOMAIN> & 0xffff) << 32) | ((<BUS> & 0xff) << 8) |
// ((device& 0x1f) <<3 ) | (function & 0x7)
*bdfid = 0;
name_start = name;
@@ -150,7 +150,7 @@ static bool is_bdfid_path_str(const std::string in_name, uint64_t *bdfid) {
if (*p != ':' || p - name_start != 4) {
return false;
}
*bdfid |= tmp << 13;
*bdfid |= tmp << 32;
// Match this: xxxx:XX:xx.x
p++; // Skip past ':'
@@ -205,7 +205,7 @@ static uint32_t ConstructBDFID(std::string path, uint64_t *bdfid) {
slash_i = tpath_str.find_last_of('/', end_i);
tmp = tpath_str.substr(slash_i + 1, end_i - slash_i);
if (is_bdfid_path_str(tmp, bdfid)) {
if (bdfid_from_path(tmp, bdfid)) {
return 0;
}
end_i = slash_i - 1;
@@ -335,7 +335,6 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
uint32_t d_index = GetDeviceIndex(d_name);
dev->set_drm_render_minor(GetDrmRenderMinor(dev_path));
dev->set_index(d_index);
GetSupportedEventGroups(d_index, dev->supported_event_groups());
devices_.push_back(dev);