AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[SWDEV-399953] Smart Temperature detection + partitioning display

Browse Source 
* Updates:
    - Fix for devices which do not have edge sensors, but junction
    - Added partitioning (memory and dynamic) displays for
      base rocm-smi CLI calls
    - Added subheading for base rocm-smi call output
    - Added better hwmon and device detection logging

Change-Id: I8219884b2e532d6ed379527cacdc1f2b232a5451
Signed-off-by: Charis Poag <Charis.Poag@amd.com>
This commit is contained in:
Charis Poag
2023-08-10 18:25:02 -05:00
parent cc5ab079df
commit 755e14dbad
7 changed files with 258 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rocm_smi.cc
+68 -3
View File
@@ -78,6 +78,7 @@
#include "rocm_smi/rocm_smi_logger.h"
using namespace ROCmLogging;
using namespace amd::smi;
static const uint32_t kMaxOverdriveLevel = 20;
static const float kEnergyCounterResolution = 15.3f;
@@ -2475,7 +2476,16 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
}
if (temperature == nullptr) {
return RSMI_STATUS_INVALID_ARGS;
ss << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | Fail "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Cause: temperature was a null ptr reference"
<< " | Returning = "
<< getRSMIStatusString(RSMI_STATUS_INVALID_ARGS) << " |";
LOG_ERROR(ss);
return RSMI_STATUS_INVALID_ARGS;
}
// The HBM temperature is retreived from the gpu_metrics
@@ -2484,12 +2494,32 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
|| sensor_type == RSMI_TEMP_TYPE_HBM_2
|| sensor_type == RSMI_TEMP_TYPE_HBM_3) {
if (metric != RSMI_TEMP_CURRENT) { // only support RSMI_TEMP_CURRENT
ss << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | Fail "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Cause: To retreive HBM temp, we only support metric = "
<< "RSMI_TEMP_CURRENT"
<< " | Returning = "
<< getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
LOG_ERROR(ss);
return RSMI_STATUS_NOT_SUPPORTED;
}
rsmi_gpu_metrics_t gpu_metrics;
ret = rsmi_dev_gpu_metrics_info_get(dv_ind, &gpu_metrics);
if (ret != RSMI_STATUS_SUCCESS) {
ss << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | Fail "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Cause: rsmi_dev_gpu_metrics_info_get returned "
<< getRSMIStatusString(ret)
<< " | Returning = "
<< getRSMIStatusString(ret) << " |";
LOG_ERROR(ss);
return ret;
}
@@ -2509,11 +2539,28 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
default:
return RSMI_STATUS_INVALID_ARGS;
}
if (val_ui16 == UINT16_MAX)
if (val_ui16 == UINT16_MAX) {
ss << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | Fail "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Cause: Reached UINT16 max value, overflow"
<< " | Returning = "
<< getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
LOG_ERROR(ss);
return RSMI_STATUS_NOT_SUPPORTED;
else
} else
*temperature = val_ui16 * CENTRIGRADE_TO_MILLI_CENTIGRADE;
ss << __PRETTY_FUNCTION__ << " | ======= end ======= "
<< " | Success "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Data: " << *temperature
<< " | Returning = "
<< getRSMIStatusString(RSMI_STATUS_SUCCESS) << " | ";
LOG_INFO(ss);
return RSMI_STATUS_SUCCESS;
} // end HBM temperature
@@ -2522,6 +2569,15 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
GET_DEV_FROM_INDX
if (dev->monitor() == nullptr) {
ss << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | Fail "
<< " | Device #: " << dv_ind
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Cause: monitor returned nullptr"
<< " | Returning = "
<< getRSMIStatusString(RSMI_STATUS_NOT_SUPPORTED) << " |";
LOG_ERROR(ss);
return RSMI_STATUS_NOT_SUPPORTED;
}
std::shared_ptr<amd::smi::Monitor> m = dev->monitor();
@@ -2535,6 +2591,15 @@ rsmi_dev_temp_metric_get(uint32_t dv_ind, uint32_t sensor_type,
CHK_API_SUPPORT_ONLY(temperature, metric, sensor_index)
ret = get_dev_mon_value(mon_type, dv_ind, sensor_index, temperature);
ss << __PRETTY_FUNCTION__ << " | ======= end ======= "
<< " | Success "
<< " | Device #: " << dv_ind
<< " | Sensor_index: " << sensor_index
<< " | Type: " << monitorTypesToString.at(mon_type)
<< " | Data: " << *temperature
<< " | Returning = "
<< getRSMIStatusString(ret) << " | ";
LOG_INFO(ss);
return ret;
CATCH
src/rocm_smi_main.cc
+37 -4
View File
@@ -170,6 +170,7 @@ static uint32_t GetDeviceIndex(const std::string s) {
// computed for cardX.
// On success, return drm_minor which is >= 128 otherwise return 0
static uint32_t GetDrmRenderMinor(const std::string s) {
std::ostringstream ss;
std::string drm_path = s;
int drm_minor = 0;
const std::string render_file_prefix = "renderD";
@@ -195,6 +196,10 @@ static uint32_t GetDrmRenderMinor(const std::string s) {
if (closedir(drm_dir)) {
return 0;
}
ss << __PRETTY_FUNCTION__ << " | Discovered drmRenderMinor = "
<< std::to_string(drm_minor) << " | For drm_path = " << drm_path << " | ";
LOG_DEBUG(ss);
return static_cast<uint32_t>(drm_minor);
}
@@ -377,11 +382,15 @@ RocmSMI::Initialize(uint64_t flags) {
// Remove any drm nodes that don't have a corresponding readable kfd node.
// kfd nodes will not be added if their properties file is not readable.
std::ostringstream ss;
auto dev_iter = devices_.begin();
while (dev_iter != devices_.end()) {
uint64_t bdfid = (*dev_iter)->bdfid();
if (tmp_map.find(bdfid) == tmp_map.end()) {
ss << __PRETTY_FUNCTION__ << " | removing device = "
<< (*dev_iter)->path();
dev_iter = devices_.erase(dev_iter);
LOG_DEBUG(ss);
continue;
}
dev_iter++;
@@ -411,6 +420,9 @@ RocmSMI::Initialize(uint64_t flags) {
}
// Leaving below to help debug temp file issues
// displayAppTmpFilesContent();
std::string amdGPUDeviceList = displayAllDevicePaths(devices_);
ss << __PRETTY_FUNCTION__ << " | current device paths = " << amdGPUDeviceList;
LOG_DEBUG(ss);
}
void
@@ -646,6 +658,9 @@ RocmSMI::FindMonitor(std::string monitor_path) {
}
void
RocmSMI::AddToDeviceList(std::string dev_name) {
std::ostringstream ss;
ss << __PRETTY_FUNCTION__ << " | ======= start =======";
LOG_TRACE(ss);
auto dev_path = std::string(kPathDRMRoot);
dev_path += "/";
dev_path += dev_name;
@@ -662,6 +677,10 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
GetSupportedEventGroups(card_indx, dev->supported_event_groups());
devices_.push_back(dev);
ss << __PRETTY_FUNCTION__ << " | Adding to device list dev_name = "
<< dev_name << " | path = " << dev_path
<< " | card index = " << std::to_string(card_indx) << " | ";
LOG_DEBUG(ss);
return;
}
@@ -669,16 +688,26 @@ RocmSMI::AddToDeviceList(std::string dev_name) {
static const uint32_t kAmdGpuId = 0x1002;
static bool isAMDGPU(std::string dev_path) {
bool isAmdGpu = false;
std::ostringstream ss;
std::string vend_path = dev_path + "/device/vendor";
if (!FileExists(vend_path.c_str())) {
return false;
ss << __PRETTY_FUNCTION__ << " | device_path = " << dev_path
<< " is " << (isAmdGpu ? "is an amdgpu device - TRUE":
"is an amdgpu device - FALSE");
LOG_DEBUG(ss);
return isAmdGpu;
}
std::ifstream fs;
fs.open(vend_path);
if (!fs.is_open()) {
return false;
ss << __PRETTY_FUNCTION__ << " | device_path = " << dev_path
<< " is " << (isAmdGpu ? "is an amdgpu device - TRUE":
"is an amdgpu device - FALSE");
LOG_DEBUG(ss);
return isAmdGpu;
}
uint32_t vendor_id;
@@ -688,9 +717,13 @@ static bool isAMDGPU(std::string dev_path) {
fs.close();
if (vendor_id == kAmdGpuId) {
return true;
isAmdGpu = true;
}
return false;
ss << __PRETTY_FUNCTION__ << " | device_path = " << dev_path
<< " is " << (isAmdGpu ? "is an amdgpu device - TRUE":
"is an amdgpu device - FALSE");
LOG_DEBUG(ss);
return isAmdGpu;
}
uint32_t RocmSMI::DiscoverAmdgpuDevices(void) {
src/rocm_smi_monitor.cc
+11
View File
@@ -313,6 +313,7 @@ int Monitor::writeMonitor(MonitorTypes type, uint32_t sensor_id,
// This string version should work for all valid monitor types
int Monitor::readMonitor(MonitorTypes type, uint32_t sensor_id,
std::string *val) {
std::ostringstream ss;
assert(val != nullptr);
std::string temp_str;
@@ -320,11 +321,21 @@ int Monitor::readMonitor(MonitorTypes type, uint32_t sensor_id,
DBG_FILE_ERROR(sysfs_path, (std::string *)nullptr)
int ret = ReadSysfsStr(sysfs_path, val);
ss << __PRETTY_FUNCTION__
<< " | Success | Read hwmon file: " << sysfs_path
<< " | Type: " << monitorTypesToString.at(type)
<< " | Sensor id: " << std::to_string(sensor_id)
<< " | Data: " << *val
<< " | Returning: " << std::to_string(ret) << " |";
LOG_INFO(ss);
return ret;
}
int32_t
Monitor::setTempSensorLabelMap(void) {
std::ostringstream ss;
ss << __PRETTY_FUNCTION__ << " | ======= start =======";
LOG_TRACE(ss);
std::string type_str;
int ret;
src/rocm_smi_utils.cc
+30 -9
View File
@@ -204,9 +204,10 @@ int ReadSysfsStr(std::string path, std::string *retStr) {
if (!fs.is_open()) {
ret = errno;
errno = 0;
oss << "Could not read SYSFS file (" << path << ")"
<< ", returning " << std::to_string(ret) << " ("
<< std::strerror(ret) << ")";
oss << __PRETTY_FUNCTION__
<< " | Fail | Cause: file does not exist or permissions issue"
<< " | SYSFS file: " << path
<< " | Returning: " << std::strerror(ret) << " |";
LOG_ERROR(oss);
return ret;
}
@@ -516,19 +517,39 @@ void displayAppTmpFilesContent() {
}
// Used to debug vector string list and their content
void displayVectorContent(std::vector<std::string> v) {
std::cout << "Vector = {";
std::string debugVectorContent(std::vector<std::string> v) {
std::ostringstream ss;
ss << "Vector = {";
if (v.size() > 0) {
for (auto it=v.begin(); it < v.end(); it++) {
std::cout << *it;
ss << *it;
auto temp_it = it;
if(++temp_it != v.end()) {
std::cout << ", ";
ss << ", ";
}
}
} else {
std::cout << "}" << std::endl;
}
ss << "}" << std::endl;
return ss.str();
}
// Used to debug vector string list and their content
std::string displayAllDevicePaths(std::vector<std::shared_ptr<Device>> v) {
std::ostringstream ss;
ss << "Vector = {";
if (v.size() > 0) {
for (auto it=v.begin(); it < v.end(); it++) {
ss << (*it)->path();
auto temp_it = it;
if(++temp_it != v.end()) {
ss << ", ";
}
}
}
ss << "}" << std::endl;
return ss.str();
}
// Attempts to read application specific temporary file