Add functions that are used to query Hardware topology.

Change-Id: I0f4cd02b237bde4d6dccfb0e83e65376ecb1cfaa
Signed-off-by: Mike Li <Tianxinmike.Li@amd.com>


[ROCm/amdsmi commit: c7d349183a]
Este commit está contenido en:
Mike Li
2020-05-07 11:37:14 -07:00
cometido por Mike (Tianxin) Li
padre b816c4a68b
commit 84e3d0f0e9
Se han modificado 12 ficheros con 1190 adiciones y 1 borrados
+177
Ver fichero
@@ -70,6 +70,7 @@
#include "rocm_smi/rocm_smi_exception.h"
#include "rocm_smi/rocm_smi_counters.h"
#include "rocm_smi/rocm_smi_kfd.h"
#include "rocm_smi/rocm_smi_io_link.h"
#include "rocm_smi/rocm_smi64Config.h"
@@ -1241,6 +1242,30 @@ static rsmi_status_t set_power_profile(uint32_t dv_ind,
CATCH
}
static rsmi_status_t topo_get_numa_node_number(uint32_t dv_ind,
uint32_t *numa_node_number) {
TRY
GET_DEV_AND_KFDNODE_FROM_INDX
*numa_node_number = kfd_node->numa_node_number();
return RSMI_STATUS_SUCCESS;
CATCH
}
static rsmi_status_t topo_get_numa_node_weight(uint32_t dv_ind,
uint64_t *weight) {
TRY
GET_DEV_AND_KFDNODE_FROM_INDX
*weight = kfd_node->numa_node_weight();
return RSMI_STATUS_SUCCESS;
CATCH
}
rsmi_status_t
rsmi_dev_gpu_clk_freq_get(uint32_t dv_ind, rsmi_clk_type_t clk_type,
rsmi_frequencies_t *f) {
@@ -2964,6 +2989,158 @@ rsmi_dev_xgmi_error_reset(uint32_t dv_ind) {
CATCH
}
rsmi_status_t
rsmi_topo_get_numa_node_number(uint32_t dv_ind, uint32_t *numa_node) {
TRY
return topo_get_numa_node_number(dv_ind, numa_node);
CATCH
}
rsmi_status_t
rsmi_topo_get_link_weight(uint32_t dv_ind_src, uint32_t dv_ind_dst,
uint64_t *weight) {
TRY
uint32_t dv_ind = dv_ind_src;
GET_DEV_AND_KFDNODE_FROM_INDX
DEVICE_MUTEX
if (weight == nullptr) {
return RSMI_STATUS_INVALID_ARGS;
}
rsmi_status_t status;
uint32_t node_ind_dst;
uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);
if (!ret) {
amd::smi::IO_LINK_TYPE type;
ret = kfd_node->get_io_link_type(node_ind_dst, &type);
if (!ret) {
if (type == amd::smi::IOLINK_TYPE_XGMI) {
ret = kfd_node->get_io_link_weight(node_ind_dst, weight);
if (!ret)
status = RSMI_STATUS_SUCCESS;
else
status = RSMI_STATUS_INIT_ERROR;
} else {
assert(!"Unexpected IO Link type read");
status = RSMI_STATUS_NOT_SUPPORTED;
}
} else {
*weight = kfd_node->numa_node_weight(); // from src GPU to it's CPU node
uint64_t numa_weight_dst;
status = topo_get_numa_node_weight(dv_ind_dst, &numa_weight_dst);
// from dst GPU to it's CPU node
if (status == RSMI_STATUS_SUCCESS) {
*weight = *weight + numa_weight_dst;
uint32_t numa_number_src = kfd_node->numa_node_number();
uint32_t numa_number_dst;
status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
if (status == RSMI_STATUS_SUCCESS) {
if (numa_number_src != numa_number_dst) {
uint64_t io_link_weight;
ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
&io_link_weight);
if (!ret) {
*weight = *weight + io_link_weight;
// from src numa CPU node to dst numa CPU node
} else {
*weight = *weight + 10;
// More than one CPU hops, hard coded 10
}
}
status = RSMI_STATUS_SUCCESS;
} else {
assert(!"Error to read numa node number");
status = RSMI_STATUS_INIT_ERROR;
}
} else {
assert(!"Error to read numa node weight");
status = RSMI_STATUS_INIT_ERROR;
}
}
} else {
status = RSMI_STATUS_INVALID_ARGS;
}
return status;
CATCH
}
rsmi_status_t
rsmi_topo_get_link_type(uint32_t dv_ind_src, uint32_t dv_ind_dst,
uint64_t *hops, RSMI_IO_LINK_TYPE *type) {
TRY
uint32_t dv_ind = dv_ind_src;
GET_DEV_AND_KFDNODE_FROM_INDX
if (hops == nullptr) {
return RSMI_STATUS_INVALID_ARGS;
}
if (type == nullptr) {
return RSMI_STATUS_INVALID_ARGS;
}
rsmi_status_t status;
uint32_t node_ind_dst;
uint32_t ret = smi.get_node_index(dv_ind_dst, &node_ind_dst);
if (!ret) {
amd::smi::IO_LINK_TYPE io_link_type;
ret = kfd_node->get_io_link_type(node_ind_dst, &io_link_type);
if (!ret) {
if (io_link_type == amd::smi::IOLINK_TYPE_XGMI) {
*type = RSMI_IOLINK_TYPE_XGMI;
*hops = 1;
status = RSMI_STATUS_SUCCESS;
} else {
assert(!"Unexpected IO Link type read");
status = RSMI_STATUS_NOT_SUPPORTED;
}
} else {
uint32_t numa_number_dst;
status = topo_get_numa_node_number(dv_ind_dst, &numa_number_dst);
if (status == RSMI_STATUS_SUCCESS) {
uint32_t numa_number_src = kfd_node->numa_node_number();
if (numa_number_src == numa_number_dst) {
*hops = 2; // same CPU node
} else {
uint64_t io_link_weight;
ret = smi.get_io_link_weight(numa_number_src, numa_number_dst,
&io_link_weight);
if (!ret)
*hops = 3; // from src CPU node to dst CPU node
else
*hops = 4; // More than one CPU hops, hard coded as 4
}
amd::smi::IO_LINK_TYPE numa_node_type = kfd_node->numa_node_type();
if (numa_node_type == amd::smi::IOLINK_TYPE_PCIEXPRESS) {
*type = RSMI_IOLINK_TYPE_PCIEXPRESS;
status = RSMI_STATUS_SUCCESS;
} else if (numa_node_type == amd::smi::IOLINK_TYPE_XGMI) {
*type = RSMI_IOLINK_TYPE_XGMI;
status = RSMI_STATUS_SUCCESS;
} else {
status = RSMI_STATUS_INIT_ERROR;
}
} else {
assert(!"Error to get numa node number");
status = RSMI_STATUS_INIT_ERROR;
}
}
} else {
status = RSMI_STATUS_INVALID_ARGS;
}
return status;
CATCH
}
enum iterator_handle_type {
FUNC_ITER = 0,
VARIANT_ITER,
+356
Ver fichero
@@ -0,0 +1,356 @@
/*
* =============================================================================
* The University of Illinois/NCSA
* Open Source License (NCSA)
*
* Copyright (c) 2020, Advanced Micro Devices, Inc.
* All rights reserved.
*
* Developed by:
*
* AMD Research and AMD ROC Software Development
*
* Advanced Micro Devices, Inc.
*
* www.amd.com
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to
* deal with the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* - Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimers.
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimers in
* the documentation and/or other materials provided with the distribution.
* - Neither the names of <Name of Development Group, Name of Institution>,
* nor the names of its contributors may be used to endorse or promote
* products derived from this Software without specific prior written
* permission.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS WITH THE SOFTWARE.
*
*/
#include <assert.h>
#include <sys/stat.h>
#include <dirent.h>
#include <algorithm>
#include <string>
#include <unordered_set>
#include <fstream>
#include <cstdint>
#include <iostream>
#include <sstream>
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi_exception.h"
#include "rocm_smi/rocm_smi_utils.h"
#include "rocm_smi/rocm_smi_io_link.h"
namespace amd {
namespace smi {
static const char *kKFDNodesPathRoot = "/sys/class/kfd/kfd/topology/nodes";
// IO Link Property strings
static const char *kIOLinkPropTYPEStr = "type";
// static const char *kIOLinkPropVERSION_MAJORStr = "version_major";
// static const char *kIOLinkPropVERSION_MINORStr = "version_minor";
static const char *kIOLinkPropNODE_FROMStr = "node_from";
static const char *kIOLinkPropNODE_TOStr = "node_to";
static const char *kIOLinkPropWEIGHTStr = "weight";
// static const char *kIOLinkPropMIN_LATENCYStr = "min_latency";
// static const char *kIOLinkPropMAX_LATENCYStr = "max_latency";
// static const char *kIOLinkPropMIN_BANDWIDTHStr = "min_bandwidth";
// static const char *kIOLinkPropMAX_BANDWIDTHStr = "max_bandwidth";
// static const char *kIOLinkPropRECOMMENDED_TRANSFER_SIZEStr =
// "recommended_transfer_size";
// static const char *kIOLinkPropFLAGSStr = "flags";
static bool is_number(const std::string &s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
static std::string IOLinkPathRoot(uint32_t node_indx) {
std::string io_link_path = kKFDNodesPathRoot;
io_link_path += '/';
io_link_path += std::to_string(node_indx);
io_link_path += '/';
io_link_path += "io_links";
return io_link_path;
}
static std::string IOLinkPath(uint32_t node_indx, uint32_t link_indx) {
std::string io_link_path = IOLinkPathRoot(node_indx);
io_link_path += '/';
io_link_path += std::to_string(link_indx);
return io_link_path;
}
static int OpenIOLinkProperties(uint32_t node_indx, uint32_t link_indx,
std::ifstream *fs) {
int ret;
std::string f_path;
bool reg_file;
assert(fs != nullptr);
if (fs == nullptr) {
return EINVAL;
}
f_path = IOLinkPath(node_indx, link_indx);
f_path += "/";
f_path += "properties";
ret = isRegularFile(f_path, &reg_file);
if (ret != 0) {
return ret;
}
if (!reg_file) {
return ENOENT;
}
fs->open(f_path);
if (!fs->is_open()) {
return errno;
}
return 0;
}
static int ReadIOLinkProperties(uint32_t node_indx, uint32_t link_indx,
std::vector<std::string> *retVec) {
std::string line;
int ret;
std::ifstream fs;
assert(retVec != nullptr);
if (retVec == nullptr) {
return EINVAL;
}
ret = OpenIOLinkProperties(node_indx, link_indx, &fs);
if (ret) {
return ret;
}
while (std::getline(fs, line)) {
retVec->push_back(line);
}
if (retVec->size() == 0) {
fs.close();
return 0;
}
// Remove any *trailing* empty (whitespace) lines
while (retVec->back().find_first_not_of(" \t\n\v\f\r") == std::string::npos) {
retVec->pop_back();
}
fs.close();
return 0;
}
int DiscoverIOLinks(std::map<std::pair<uint32_t, uint32_t>,
std::shared_ptr<IOLink>> *links) {
assert(links != nullptr);
if (links == nullptr) {
return EINVAL;
}
assert(links->size() == 0);
links->clear();
auto kfd_node_dir = opendir(kKFDNodesPathRoot);
assert(kfd_node_dir != nullptr);
auto dentry_kfd = readdir(kfd_node_dir);
while (dentry_kfd != nullptr) {
if (dentry_kfd->d_name[0] == '.') {
dentry_kfd = readdir(kfd_node_dir);
continue;
}
if (!is_number(dentry_kfd->d_name)) {
dentry_kfd = readdir(kfd_node_dir);
continue;
}
uint32_t node_indx = std::stoi(dentry_kfd->d_name);
std::shared_ptr<IOLink> link;
uint32_t link_indx;
std::string io_link_path_root = IOLinkPathRoot(node_indx);
auto io_link_dir = opendir(io_link_path_root.c_str());
assert(io_link_dir != nullptr);
auto dentry_io_link = readdir(io_link_dir);
while (dentry_io_link != nullptr) {
if (dentry_io_link->d_name[0] == '.') {
dentry_io_link = readdir(io_link_dir);
continue;
}
if (!is_number(dentry_io_link->d_name)) {
dentry_io_link = readdir(io_link_dir);
continue;
}
link_indx = std::stoi(dentry_io_link->d_name);
link = std::shared_ptr<IOLink>(new IOLink(node_indx, link_indx));
link->Initialize();
(*links)[std::make_pair(link->node_from(), link->node_to())] = link;
dentry_io_link = readdir(io_link_dir);
}
if (closedir(io_link_dir)) {
return 1;
}
dentry_kfd = readdir(kfd_node_dir);
}
if (closedir(kfd_node_dir)) {
return 1;
}
return 0;
}
int DiscoverIOLinksPerNode(uint32_t node_indx, std::map<uint32_t,
std::shared_ptr<IOLink>> *links) {
assert(links != nullptr);
if (links == nullptr) {
return EINVAL;
}
assert(links->size() == 0);
links->clear();
std::shared_ptr<IOLink> link;
uint32_t link_indx;
std::string io_link_path_root = IOLinkPathRoot(node_indx);
auto io_link_dir = opendir(io_link_path_root.c_str());
assert(io_link_dir != nullptr);
auto dentry = readdir(io_link_dir);
while (dentry != nullptr) {
if (dentry->d_name[0] == '.') {
dentry = readdir(io_link_dir);
continue;
}
if (!is_number(dentry->d_name)) {
dentry = readdir(io_link_dir);
continue;
}
link_indx = std::stoi(dentry->d_name);
link = std::shared_ptr<IOLink>(new IOLink(node_indx, link_indx));
link->Initialize();
(*links)[link->node_to()] = link;
dentry = readdir(io_link_dir);
}
if (closedir(io_link_dir)) {
return 1;
}
return 0;
}
IOLink::~IOLink() {
}
int IOLink::ReadProperties(void) {
int ret;
std::vector<std::string> propVec;
assert(properties_.size() == 0);
if (properties_.size() > 0) {
return 0;
}
ret = ReadIOLinkProperties(node_indx_, link_indx_, &propVec);
if (ret) {
return ret;
}
std::string key_str;
uint64_t val_int; // Assume all properties are unsigned integers for now
std::istringstream fs;
for (uint32_t i = 0; i < propVec.size(); ++i) {
fs.str(propVec[i]);
fs >> key_str;
fs >> val_int;
properties_[key_str] = val_int;
fs.str("");
fs.clear();
}
return 0;
}
int
IOLink::Initialize(void) {
int ret = 0;
ret = ReadProperties();
if (ret) {return ret;}
ret = get_property_value(kIOLinkPropTYPEStr,
reinterpret_cast<uint64_t *>(&type_));
if (ret) {return ret;}
ret = get_property_value(kIOLinkPropNODE_FROMStr,
reinterpret_cast<uint64_t *>(&node_from_));
if (ret) {return ret;}
ret = get_property_value(kIOLinkPropNODE_TOStr,
reinterpret_cast<uint64_t *>(&node_to_));
if (ret) {return ret;}
ret = get_property_value(kIOLinkPropWEIGHTStr, &weight_);
return ret;
}
int
IOLink::get_property_value(std::string property, uint64_t *value) {
assert(value != nullptr);
if (value == nullptr) {
return EINVAL;
}
if (properties_.find(property) == properties_.end()) {
return EINVAL;
}
*value = properties_[property];
return 0;
}
} // namespace smi
} // namespace amd
+65 -1
Ver fichero
@@ -53,6 +53,7 @@
#include <iostream>
#include <sstream>
#include "rocm_smi/rocm_smi_io_link.h"
#include "rocm_smi/rocm_smi_kfd.h"
#include "rocm_smi/rocm_smi.h"
#include "rocm_smi/rocm_smi_exception.h"
@@ -531,12 +532,49 @@ KFDNode::Initialize(void) {
if (ret) {return ret;}
ret = ReadKFDGpuId(node_indx_, &gpu_id_);
if (ret) {return ret;}
if (ret || (gpu_id_ == 0)) {return ret;}
ret = ReadKFDGpuName(node_indx_, &name_);
std::map<uint32_t, std::shared_ptr<IOLink>> io_link_map_tmp;
ret = DiscoverIOLinksPerNode(node_indx_, &io_link_map_tmp);
if (ret != 0) {
throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
"Failed to initialize rocm_smi library (IO Links discovery per node).");
}
std::map<uint32_t, std::shared_ptr<IOLink>>::iterator it;
uint32_t node_to;
uint64_t node_to_gpu_id;
std::shared_ptr<IOLink> link;
bool numa_node_found = false;
for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++) {
io_link_map_[it->first] = it->second;
node_to = it->first;
link = it->second;
ret = ReadKFDGpuId(node_to, &node_to_gpu_id);
if (ret) {return ret;}
if (node_to_gpu_id == 0) { // CPU node
if (numa_node_found) {
if (numa_node_weight_ > link->weight()) {
numa_node_number_ = node_to;
numa_node_weight_ = link->weight();
numa_node_type_ = link->type();
}
} else {
numa_node_number_ = node_to;
numa_node_weight_ = link->weight();
numa_node_type_ = link->type();
numa_node_found = true;
}
} else {
io_link_type_[node_to] = link->type();
io_link_weight_[node_to] = link->weight();
}
}
return ret;
}
int
KFDNode::get_property_value(std::string property, uint64_t *value) {
assert(value != nullptr);
@@ -550,5 +588,31 @@ KFDNode::get_property_value(std::string property, uint64_t *value) {
return 0;
}
int
KFDNode::get_io_link_type(uint32_t node_to, IO_LINK_TYPE *type) {
assert(type != nullptr);
if (type == nullptr) {
return EINVAL;
}
if (io_link_type_.find(node_to) == io_link_type_.end()) {
return EINVAL;
}
*type = io_link_type_[node_to];
return 0;
}
int
KFDNode::get_io_link_weight(uint32_t node_to, uint64_t *weight) {
assert(weight != nullptr);
if (weight == nullptr) {
return EINVAL;
}
if (io_link_weight_.find(node_to) == io_link_weight_.end()) {
return EINVAL;
}
*weight = io_link_weight_[node_to];
return 0;
}
} // namespace smi
} // namespace amd
+34
Ver fichero
@@ -280,6 +280,17 @@ RocmSMI::Initialize(uint64_t flags) {
"Failed to initialize rocm_smi library (KFD node discovery).");
}
std::map<std::pair<uint32_t, uint32_t>, std::shared_ptr<IOLink>>
io_link_map_tmp;
ret = DiscoverIOLinks(&io_link_map_tmp);
if (ret != 0) {
throw amd::smi::rsmi_exception(RSMI_INITIALIZATION_ERROR,
"Failed to initialize rocm_smi library (IO Links discovery).");
}
std::map<std::pair<uint32_t, uint32_t>, std::shared_ptr<IOLink>>::iterator it;
for (it = io_link_map_tmp.begin(); it != io_link_map_tmp.end(); it++)
io_link_map_[it->first] = it->second;
std::shared_ptr<amd::smi::Device> dev;
// 1. construct kfd_node_map_ with gpu_id as key and *Device as value
@@ -295,6 +306,7 @@ RocmSMI::Initialize(uint64_t flags) {
}
tmp_map[bdfid]->set_amdgpu_dev_index(dv_ind);
dev_ind_to_node_ind_map_[dv_ind] = tmp_map[bdfid]->node_index();
uint64_t gpu_id = tmp_map[bdfid]->gpu_id();
dev->set_kfd_gpu_id(gpu_id);
kfd_node_map_[gpu_id] = tmp_map[bdfid];
@@ -591,5 +603,27 @@ uint32_t RocmSMI::IterateSMIDevices(
return 0;
}
int RocmSMI::get_node_index(uint32_t dv_ind, uint32_t *node_ind) {
if (dev_ind_to_node_ind_map_.find(dv_ind) == dev_ind_to_node_ind_map_.end()) {
return EINVAL;
}
*node_ind = dev_ind_to_node_ind_map_[dv_ind];
return 0;
}
int RocmSMI::get_io_link_weight(uint32_t node_from, uint32_t node_to,
uint64_t *weight) {
assert(weight != nullptr);
if (weight == nullptr) {
return EINVAL;
}
if (io_link_map_.find(std::make_pair(node_from, node_to)) ==
io_link_map_.end()) {
return EINVAL;
}
*weight = io_link_map_[std::make_pair(node_from, node_to)]->weight();
return 0;
}
} // namespace smi
} // namespace amd