AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rocm_smi_lib: Fix GPU Metrics Max Elements Read Exceeded

Browse Source 
Code changes related to the following:
  * Check smallest copy size for multi-valued metrics
  * Unit tests: gpu_metric_read
  * ROCMSMI examples

Build changes related to the following:
  * CMakeLists.txt

Change-Id: Ieb2363020fa21c93fbacd0edcc1d394eed183051
Signed-off-by: Oliveira, Daniel <daniel.oliveira@amd.com>
This commit is contained in:
Oliveira, Daniel
2023-11-30 15:03:51 -06:00
parent a128867497
commit 8e0d3d5a39
5 changed files with 413 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/rocm_smi.cc
+116 -79
View File
@@ -6375,19 +6375,24 @@ rsmi_dev_metrics_temp_hbm_get(uint32_t dv_ind, GPUMetricTempHbm_t* temp_hbm_valu
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricTempHbm);
amd::smi::GPUMetricTempHbmTbl_t tmp_hbl_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_hbl_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*temp_hbm_value) - std::begin(*temp_hbm_value));
std::copy_n(std::begin(tmp_hbl_tbl), max_num_elems, *temp_hbm_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*temp_hbm_value) - std::begin(*temp_hbm_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_hbl_tbl.size()) ? max_num_elems : tmp_hbl_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_hbl_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_hbl_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(temp_hbm_value, 0, sizeof(temp_hbm_value));
std::copy_n(std::begin(tmp_hbl_tbl), copy_size, *temp_hbm_value);
}
return status_code;
CATCH
@@ -6409,19 +6414,24 @@ rsmi_dev_metrics_vcn_activity_get(uint32_t dv_ind, GPUMetricVcnActivity_t* vcn_a
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricVcnActivity);
amd::smi::GPUMetricVcnActivityTbl_t tmp_vcn_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_vcn_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*vcn_activity_value) - std::begin(*vcn_activity_value));
std::copy_n(std::begin(tmp_vcn_tbl), max_num_elems, *vcn_activity_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*vcn_activity_value) - std::begin(*vcn_activity_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_vcn_tbl.size()) ? max_num_elems : tmp_vcn_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_vcn_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_vcn_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(vcn_activity_value, 0, sizeof(vcn_activity_value));
std::copy_n(std::begin(tmp_vcn_tbl), copy_size, *vcn_activity_value);
}
return status_code;
CATCH
@@ -6443,19 +6453,24 @@ rsmi_dev_metrics_xgmi_read_data_get(uint32_t dv_ind, GPUMetricXgmiReadDataAcc_t*
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiReadDataAccumulator);
amd::smi::GPUMetricXgmiAccTbl_t tmp_xgmi_acc_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_xgmi_acc_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*xgmi_read_data_acc_value) - std::begin(*xgmi_read_data_acc_value));
std::copy_n(std::begin(tmp_xgmi_acc_tbl), max_num_elems, *xgmi_read_data_acc_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*xgmi_read_data_acc_value) - std::begin(*xgmi_read_data_acc_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_xgmi_acc_tbl.size()) ? max_num_elems : tmp_xgmi_acc_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_xgmi_acc_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_xgmi_acc_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(xgmi_read_data_acc_value, 0, sizeof(xgmi_read_data_acc_value));
std::copy_n(std::begin(tmp_xgmi_acc_tbl), copy_size, *xgmi_read_data_acc_value);
}
return status_code;
CATCH
@@ -6477,19 +6492,24 @@ rsmi_dev_metrics_xgmi_write_data_get(uint32_t dv_ind, GPUMetricXgmiWriteDataAcc_
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricXgmiWriteDataAccumulator);
amd::smi::GPUMetricXgmiAccTbl_t tmp_xgmi_acc_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_xgmi_acc_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*xgmi_write_data_acc_value) - std::begin(*xgmi_write_data_acc_value));
std::copy_n(std::begin(tmp_xgmi_acc_tbl), max_num_elems, *xgmi_write_data_acc_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*xgmi_write_data_acc_value) - std::begin(*xgmi_write_data_acc_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_xgmi_acc_tbl.size()) ? max_num_elems : tmp_xgmi_acc_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_xgmi_acc_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_xgmi_acc_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(xgmi_write_data_acc_value, 0, sizeof(xgmi_write_data_acc_value));
std::copy_n(std::begin(tmp_xgmi_acc_tbl), copy_size, *xgmi_write_data_acc_value);
}
return status_code;
CATCH
@@ -6511,19 +6531,24 @@ rsmi_dev_metrics_curr_gfxclk_get(uint32_t dv_ind, GPUMetricCurrGfxClk_t* current
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrGfxClock);
amd::smi::GPUMetricCurrGfxClkTbl_t tmp_curr_gfxclk_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_gfxclk_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_gfxclk_value) - std::begin(*current_gfxclk_value));
std::copy_n(std::begin(tmp_curr_gfxclk_tbl), max_num_elems, *current_gfxclk_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_gfxclk_value) - std::begin(*current_gfxclk_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_curr_gfxclk_tbl.size()) ? max_num_elems : tmp_curr_gfxclk_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_curr_gfxclk_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_curr_gfxclk_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(current_gfxclk_value, 0, sizeof(current_gfxclk_value));
std::copy_n(std::begin(tmp_curr_gfxclk_tbl), copy_size, *current_gfxclk_value);
}
return status_code;
CATCH
@@ -6545,19 +6570,23 @@ rsmi_dev_metrics_curr_socclk_get(uint32_t dv_ind, GPUMetricCurrSocClk_t* current
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrSocClock);
amd::smi::GPUMetricCurrSocClkTbl_t tmp_curr_socclk_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_socclk_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_socclk_value) - std::begin(*current_socclk_value));
std::copy_n(std::begin(tmp_curr_socclk_tbl), max_num_elems, *current_socclk_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_socclk_value) - std::begin(*current_socclk_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_curr_socclk_tbl.size()) ? max_num_elems : tmp_curr_socclk_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_curr_socclk_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_curr_socclk_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(current_socclk_value, 0, sizeof(current_socclk_value));
std::copy_n(std::begin(tmp_curr_socclk_tbl), copy_size, *current_socclk_value);
}
return status_code;
CATCH
@@ -6579,19 +6608,24 @@ rsmi_dev_metrics_curr_vclk0_get(uint32_t dv_ind, GPUMetricCurrVClk0_t* current_v
const auto gpu_metric_unit(AMDGpuMetricsUnitType_t::kMetricCurrVClock0);
amd::smi::GPUMetricCurrVClkTbl_t tmp_curr_vclk0_tbl{};
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_vclk0_tbl);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_vclk_value) - std::begin(*current_vclk_value));
std::copy_n(std::begin(tmp_curr_vclk0_tbl), max_num_elems, *current_vclk_value);
}
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_vclk_value) - std::begin(*current_vclk_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_curr_vclk0_tbl.size()) ? max_num_elems : tmp_curr_vclk0_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< " | ======= end ======= "
<< " | End Result "
<< " | Device #: " << dv_ind
<< " | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< " | Metric Size: " << tmp_curr_vclk0_tbl.size()
<< " | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
<< "\n | ======= end ======= "
<< "\n | End Result "
<< "\n | Device #: " << dv_ind
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_curr_vclk0_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::memset(current_vclk_value, 0, sizeof(current_vclk_value));
std::copy_n(std::begin(tmp_curr_vclk0_tbl), copy_size, *current_vclk_value);
}
return status_code;
CATCH
@@ -6642,6 +6676,8 @@ rsmi_dev_metrics_curr_dclk0_get(uint32_t dv_ind, GPUMetricCurrDClk0_t* current_d
auto status_code = rsmi_dev_gpu_metrics_info_query(dv_ind, gpu_metric_unit, tmp_curr_dclk0_tbl);
const auto max_num_elems =
static_cast<uint16_t>(std::end(*current_dclk_value) - std::begin(*current_dclk_value));
const auto copy_size =
static_cast<uint16_t>((max_num_elems < tmp_curr_dclk0_tbl.size()) ? max_num_elems : tmp_curr_dclk0_tbl.size());
ostrstream << __PRETTY_FUNCTION__
<< "\n | ======= end ======= "
<< "\n | End Result "
@@ -6649,11 +6685,12 @@ rsmi_dev_metrics_curr_dclk0_get(uint32_t dv_ind, GPUMetricCurrDClk0_t* current_d
<< "\n | Metric Type: " << static_cast<AMDGpuMetricTypeId_t>(gpu_metric_unit)
<< "\n | Metric Size: " << tmp_curr_dclk0_tbl.size()
<< "\n | Max num of elements: " << max_num_elems
<< "\n | Copy size: " << copy_size
<< "\n | Returning = " << status_code << " " << getRSMIStatusString(status_code) << " |";
LOG_INFO(ostrstream);
if (status_code == rsmi_status_t::RSMI_STATUS_SUCCESS) {
std::copy_n(std::begin(tmp_curr_dclk0_tbl), max_num_elems, *current_dclk_value);
std::memset(current_dclk_value, 0, sizeof(current_dclk_value));
std::copy_n(std::begin(tmp_curr_dclk0_tbl), copy_size, *current_dclk_value);
}
return status_code;