AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

RVS - Use config files and make GPU aware

Просмотр исходного кода 
Change-Id: I7a5c80ed4e6122d102e494d1ae38b4b7d40c42cd
Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>
Этот коммит содержится в:
Galantsev, Dmitrii
2025-02-28 11:17:40 +00:00
коммит произвёл Galantsev, Dmitrii
родитель 247c8c7d5e
Коммит f5a4402ce5
63 изменённых файлов: 4673 добавлений и 148 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
rdc_libs/rdc/src/RdcDiagnosticModule.cc
+39 -17
Просмотреть файл
@@ -49,9 +49,12 @@ rdc_status_t RdcDiagnosticModule::rdc_diag_test_cases_query(
return RDC_ST_OK;
}
rdc_status_t RdcDiagnosticModule::rdc_test_case_run(
rdc_diag_test_cases_t test_case, uint32_t gpu_index[RDC_MAX_NUM_DEVICES], uint32_t gpu_count,
const char* config, size_t config_size, rdc_diag_test_result_t* result, rdc_diag_callback_t* callback) {
rdc_status_t RdcDiagnosticModule::rdc_test_case_run(rdc_diag_test_cases_t test_case,
uint32_t gpu_index[RDC_MAX_NUM_DEVICES],
uint32_t gpu_count, const char* config,
size_t config_size,
rdc_diag_test_result_t* result,
rdc_diag_callback_t* callback) {
if (result == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
@@ -74,24 +77,41 @@ rdc_status_t RdcDiagnosticModule::rdc_diagnostic_run(const rdc_group_info_t& gpu
size_t config_size,
rdc_diag_response_t* response,
rdc_diag_callback_t* callback) {
const bool is_custom = config != nullptr && config_size != 0;
if (response == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
std::vector<rdc_diag_test_cases_t> rdc_runs;
std::vector<rdc_diag_test_cases_t> tests_to_search_for;
if (level >= RDC_DIAG_LVL_SHORT) { // Short run and above
rdc_runs.push_back(RDC_DIAG_COMPUTE_PROCESS);
rdc_runs.push_back(RDC_DIAG_NODE_TOPOLOGY);
rdc_runs.push_back(RDC_DIAG_GPU_PARAMETERS);
rdc_runs.push_back(RDC_DIAG_COMPUTE_QUEUE);
rdc_runs.push_back(RDC_DIAG_SYS_MEM_CHECK);
tests_to_search_for.push_back(RDC_DIAG_COMPUTE_PROCESS);
tests_to_search_for.push_back(RDC_DIAG_NODE_TOPOLOGY);
tests_to_search_for.push_back(RDC_DIAG_GPU_PARAMETERS);
tests_to_search_for.push_back(RDC_DIAG_COMPUTE_QUEUE);
tests_to_search_for.push_back(RDC_DIAG_SYS_MEM_CHECK);
}
if (level >= RDC_DIAG_LVL_MED) { // Medium run and above
rdc_runs.push_back(RDC_DIAG_RVS_GST_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_MEMBW_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_H2DD2H_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_IET_TEST);
tests_to_search_for.push_back(RDC_DIAG_RVS_GST_TEST);
tests_to_search_for.push_back(RDC_DIAG_RVS_MEMBW_TEST);
tests_to_search_for.push_back(RDC_DIAG_RVS_H2DD2H_TEST);
tests_to_search_for.push_back(RDC_DIAG_RVS_IET_TEST);
}
std::vector<rdc_diag_test_cases_t> tests_to_run;
if (is_custom) {
// respect custom config
tests_to_run.push_back(RDC_DIAG_RVS_CUSTOM);
} else {
// respect level
for (auto& test : tests_to_search_for) {
if (testcases_to_module_.find(test) != testcases_to_module_.end()) {
tests_to_run.push_back(test);
} else {
RDC_LOG(RDC_DEBUG, "test not found: " << test);
}
}
}
if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) {
@@ -99,15 +119,17 @@ rdc_status_t RdcDiagnosticModule::rdc_diagnostic_run(const rdc_group_info_t& gpu
callback->callback(callback->cookie, log.data());
}
unsigned int i = 0;
response->results_count = 0;
for (unsigned int i = 0; i < rdc_runs.size(); i++) {
for (i = 0; i < tests_to_run.size(); i++) {
if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) {
std::string log = "Test " + std::to_string(i) + " / " + std::to_string(rdc_runs.size());
std::string log =
"Test " + std::to_string(i + 1) + " / " + std::to_string(tests_to_run.size());
callback->callback(callback->cookie, log.data());
}
response->diag_info[i].test_case = rdc_runs[i];
response->diag_info[i].test_case = tests_to_run[i];
// NOTE: rdc_test_case_run reuses the diagnostic_run callback
rdc_test_case_run(rdc_runs[i], const_cast<uint32_t*>(gpus.entity_ids), gpus.count, config,
rdc_test_case_run(tests_to_run[i], const_cast<uint32_t*>(gpus.entity_ids), gpus.count, config,
config_size, &(response->diag_info[i]), callback);
response->results_count++;
}
rdc_libs/rdc/src/RdcMetricFetcherImpl.cc
+72 -47
Просмотреть файл
@@ -432,33 +432,33 @@ rdc_status_t RdcMetricFetcherImpl::bulk_fetch_smi_fields(
constexpr double kGig = 1000000000.0;
static uint64_t sum_xgmi_read(const amdsmi_gpu_metrics_t& gpu_metrics) {
uint64_t total = 0;
const auto not_supported_metrics_data = std::numeric_limits<uint64_t>::max();
for (int i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i) {
if (gpu_metrics.xgmi_read_data_acc[i] == not_supported_metrics_data){
continue;
}
total += gpu_metrics.xgmi_read_data_acc[i];
uint64_t total = 0;
const auto not_supported_metrics_data = std::numeric_limits<uint64_t>::max();
for (int i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i) {
if (gpu_metrics.xgmi_read_data_acc[i] == not_supported_metrics_data) {
continue;
}
if (total == 0){
return not_supported_metrics_data;
}
return total;
total += gpu_metrics.xgmi_read_data_acc[i];
}
if (total == 0) {
return not_supported_metrics_data;
}
return total;
}
static uint64_t sum_xgmi_write(const amdsmi_gpu_metrics_t& gpu_metrics) {
uint64_t total = 0;
const auto not_supported_metrics_data = std::numeric_limits<uint64_t>::max();
for (int i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i) {
if (gpu_metrics.xgmi_write_data_acc[i] == not_supported_metrics_data){
continue;
}
total += gpu_metrics.xgmi_write_data_acc[i];
uint64_t total = 0;
const auto not_supported_metrics_data = std::numeric_limits<uint64_t>::max();
for (int i = 0; i < AMDSMI_MAX_NUM_XGMI_LINKS; ++i) {
if (gpu_metrics.xgmi_write_data_acc[i] == not_supported_metrics_data) {
continue;
}
if (total == 0){
return not_supported_metrics_data;
}
return total;
total += gpu_metrics.xgmi_write_data_acc[i];
}
if (total == 0) {
return not_supported_metrics_data;
}
return total;
}
rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id,
@@ -659,6 +659,7 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
break;
}
case RDC_FI_DEV_NAME: {
// source values from asic_info
amdsmi_asic_info_t asic_info;
value->status = amdsmi_get_gpu_asic_info(processor_handle, &asic_info);
value->type = STRING;
@@ -700,17 +701,44 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
value->value.l_int = num_pages;
}
break;
case RDC_FI_OAM_ID: {
case RDC_FI_OAM_ID:
case RDC_FI_DEV_ID:
case RDC_FI_REV_ID:
case RDC_FI_TARGET_GRAPHICS_VERSION:
case RDC_FI_NUM_OF_COMPUTE_UNITS: {
amdsmi_asic_info_t asic_info;
value->status = amdsmi_get_gpu_asic_info(processor_handle, &asic_info);
value->type = INTEGER;
if (value->status == AMDSMI_STATUS_SUCCESS) {
if (value->status != AMDSMI_STATUS_SUCCESS) {
break;
}
if (field_id == RDC_FI_OAM_ID) {
// 0xFFFF means not supported for OAM ID
if (asic_info.oam_id == 0xFFFF) {
value->status = AMDSMI_STATUS_NOT_SUPPORTED;
} else {
value->value.l_int = asic_info.oam_id;
}
} else if (field_id == RDC_FI_DEV_ID) {
value->value.l_int = asic_info.device_id;
} else if (field_id == RDC_FI_REV_ID) {
value->value.l_int = asic_info.rev_id;
} else if (field_id == RDC_FI_TARGET_GRAPHICS_VERSION) {
if (asic_info.target_graphics_version == 0xFFFFFFFFFFFFFFFF) {
value->status = AMDSMI_STATUS_NOT_SUPPORTED;
} else {
value->value.l_int = asic_info.target_graphics_version;
}
} else if (field_id == RDC_FI_NUM_OF_COMPUTE_UNITS) {
if (asic_info.num_of_compute_units == 0xFFFFFFFF) {
value->status = AMDSMI_STATUS_NOT_SUPPORTED;
} else {
value->value.l_int = asic_info.num_of_compute_units;
}
} else {
// this should never happen as all fields are handled above
RDC_LOG(RDC_ERROR, "Unexpected field id: " << field_id);
value->status = AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS;
}
break;
}
@@ -726,7 +754,7 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
uint64_t timestamp;
value->status = amdsmi_get_utilization_count(processor_handle, utilization_counters,
kUTILIZATION_COUNTERS, &timestamp);
kUTILIZATION_COUNTERS, &timestamp);
value->type = INTEGER;
if (value->status == AMDSMI_STATUS_SUCCESS) {
value->value.l_int = static_cast<int64_t>(utilization_counters[0].value);
@@ -858,32 +886,29 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
uint32_t num_pages = 0;
ret = amdsmi_get_gpu_bad_page_info(processor_handle, &num_pages, nullptr);
if (AMDSMI_STATUS_SUCCESS == ret) {
if (RDC_HEALTH_RETIRED_PAGE_NUM == field_id) {
value->status = Smi2RdcError(ret);
value->type = INTEGER;
value->value.l_int = static_cast<int64_t>(num_pages);
break;
}
if (RDC_HEALTH_RETIRED_PAGE_NUM == field_id) {
value->status = Smi2RdcError(ret);
value->type = INTEGER;
value->value.l_int = static_cast<int64_t>(num_pages);
break;
}
if ((0 < num_pages) &&
(RDC_HEALTH_PENDING_PAGE_NUM == field_id)) {
std::vector<amdsmi_retired_page_record_t> bad_page_info(num_pages);
ret = amdsmi_get_gpu_bad_page_info(processor_handle, &num_pages,
bad_page_info.data());
value->status = Smi2RdcError(ret);
value->type = INTEGER;
if (AMDSMI_STATUS_SUCCESS == ret) {
uint64_t pending_page_num = 0;
for (uint32_t i=0; i < num_pages; i++) {
if (AMDSMI_MEM_PAGE_STATUS_PENDING == bad_page_info[i].status)
pending_page_num++;
}
value->value.l_int = static_cast<int64_t>(pending_page_num);
if ((0 < num_pages) && (RDC_HEALTH_PENDING_PAGE_NUM == field_id)) {
std::vector<amdsmi_retired_page_record_t> bad_page_info(num_pages);
ret = amdsmi_get_gpu_bad_page_info(processor_handle, &num_pages, bad_page_info.data());
value->status = Smi2RdcError(ret);
value->type = INTEGER;
if (AMDSMI_STATUS_SUCCESS == ret) {
uint64_t pending_page_num = 0;
for (uint32_t i = 0; i < num_pages; i++) {
if (AMDSMI_MEM_PAGE_STATUS_PENDING == bad_page_info[i].status) pending_page_num++;
}
value->value.l_int = static_cast<int64_t>(pending_page_num);
}
}
} else
value->status = Smi2RdcError(ret);
value->status = Smi2RdcError(ret);
break;
}