From 5861ec7663e126908b141938cdced256578ec58b Mon Sep 17 00:00:00 2001 From: "Galantsev, Dmitrii" Date: Mon, 6 Jan 2025 18:14:44 -0600 Subject: [PATCH] RVS - Add IET and PEBB tests Change-Id: Ia032901d74c882e5cbfa5a3164199cd4d571341f Signed-off-by: Galantsev, Dmitrii --- include/rdc/rdc.h | 36 ++++++++++--------- include/rdc_modules/rdc_rvs/RvsBase.h | 35 ++++++++++++++++++ protos/rdc.proto | 16 +++++---- rdc_libs/rdc/src/RdcDiagnosticModule.cc | 4 ++- .../rdc_modules/rdc_rvs/RdcDiagnosticLib.cc | 33 +++++++++-------- rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc | 14 ++------ rdci/src/RdciDiagSubSystem.cc | 6 ++-- 7 files changed, 92 insertions(+), 52 deletions(-) diff --git a/include/rdc/rdc.h b/include/rdc/rdc.h index aa5e46be91..6ab4e7cd69 100644 --- a/include/rdc/rdc.h +++ b/include/rdc/rdc.h @@ -187,12 +187,12 @@ typedef enum { /** * @brief GPU usage related fields */ - RDC_FI_GPU_UTIL = 500, //!< GPU Utilization - RDC_FI_GPU_MEMORY_USAGE, //!< Memory usage of the GPU instance - RDC_FI_GPU_MEMORY_TOTAL, //!< Total memory of the GPU instance - RDC_FI_GPU_MM_ENC_UTIL, //!< Multimedia encoder busy percentage - RDC_FI_GPU_MM_DEC_UTIL, //!< Multimedia decoder busy percentage - RDC_FI_GPU_MEMORY_ACTIVITY, //!< Memory busy percentage + RDC_FI_GPU_UTIL = 500, //!< GPU Utilization + RDC_FI_GPU_MEMORY_USAGE, //!< Memory usage of the GPU instance + RDC_FI_GPU_MEMORY_TOTAL, //!< Total memory of the GPU instance + RDC_FI_GPU_MM_ENC_UTIL, //!< Multimedia encoder busy percentage + RDC_FI_GPU_MM_DEC_UTIL, //!< Multimedia decoder busy percentage + RDC_FI_GPU_MEMORY_ACTIVITY, //!< Memory busy percentage RDC_FI_GPU_MEMORY_MAX_BANDWIDTH, //Device transfer speed test + RDC_DIAG_RVS_IET_TEST, //!< RVS IET test + RDC_DIAG_TEST_LAST = RDC_DIAG_RVS_IET_TEST } rdc_diag_test_cases_t; /** @@ -661,13 +663,13 @@ typedef enum { typedef struct { uint32_t gpu_index; - uint32_t num_of_links; // The size of the array link_states - rdc_topology_link_type_t link_types; // XGMI, PCIe, and so on + uint32_t num_of_links; // The size of the array link_states + rdc_topology_link_type_t link_types; // XGMI, PCIe, and so on rdc_link_state_t link_states[RDC_MAX_NUM_OF_LINKS]; } rdc_gpu_link_status_t; typedef struct { - int32_t num_of_gpus; // The size of gpus array + int32_t num_of_gpus; // The size of gpus array rdc_gpu_link_status_t gpus[RDC_MAX_NUM_DEVICES]; } rdc_link_status_t; @@ -1522,7 +1524,7 @@ rdc_status_t rdc_health_check(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_i */ rdc_status_t rdc_health_clear(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id); - /** +/** * @brief Get the topology of the device * * @details topology of the device diff --git a/include/rdc_modules/rdc_rvs/RvsBase.h b/include/rdc_modules/rdc_rvs/RvsBase.h index 5fe732bd97..6e5dae6c01 100644 --- a/include/rdc_modules/rdc_rvs/RvsBase.h +++ b/include/rdc_modules/rdc_rvs/RvsBase.h @@ -24,12 +24,47 @@ THE SOFTWARE. #include #include +#include #include #include "rdc/rdc.h" #include "rvs/rvs.h" static constexpr size_t MAX_CONFIG_LENGTH = 1024; +// NOTE: There MUST be a space after : +static const std::map test_to_conf = { + // derived from conf/gst_single.conf + {RDC_DIAG_RVS_GST_TEST, + "{actions: [{name: gpustress-9000-sgemm-false, device: all, " + "device_index: '0', module: gst, parallel: false, count: 1, duration: " + "10000, copy_matrix: false, target_stress: 9000, matrix_size_a: 8640, " + "matrix_size_b: 8640, matrix_size_c: 8640, ops_type: sgemm, lda: 8640, " + "ldb: 8640, ldc: 8640}]}"}, + // derived from conf/MI300X/babel.conf + {RDC_DIAG_RVS_MEMBW_TEST, + "{actions: [{name: babel-float-256MiB," + "device: all, module: babel, " + "parallel: false, count: 1, num_iter: 5000, array_size: 268435456, " + "test_type: 1, mibibytes: true, o/p_csv: false, subtest: 5}]}"}, + // derived from conf/MI300X/pebb_single.conf + {RDC_DIAG_RVS_H2DD2H_TEST, + "{actions: [{name: h2d-d2h-sequential-64MB," + "device: all, module: pebb, duration: 120000, device_to_host: true, " + "host_to_device: true, parallel: false, block_size: 67108864, " + "link_type: 2, warm_calls: 10, hot_calls: 100, b2b: true}]}"}, + // derived from conf/MI300X/iet_single.conf + {RDC_DIAG_RVS_IET_TEST, + "{actions: [{name: iet-400W-1K-rand-dgemm," + "device: all, module: iet, parallel: true, duration: 60000, " + "sample_interval: 3000, target_power: 400, matrix_size: 1024, " + "matrix_init: rand, ops_type: dgemm}]}"}, +}; +static const std::map test_to_name = { + {RDC_DIAG_RVS_GST_TEST, "RVS_GST_TEST"}, + {RDC_DIAG_RVS_MEMBW_TEST, "RVS_MEMBW_TEST"}, + {RDC_DIAG_RVS_H2DD2H_TEST, "RVS_H2DD2H_TEST"}, + {RDC_DIAG_RVS_IET_TEST, "RVS_IET_TEST"}, +}; namespace amd { namespace rdc { diff --git a/protos/rdc.proto b/protos/rdc.proto index 74c76375f9..d255a122aa 100755 --- a/protos/rdc.proto +++ b/protos/rdc.proto @@ -503,9 +503,11 @@ message DiagnosticTestResult { VRAM_CHECK = 3; SYS_MEM_CHECK = 4; NODE_TOPOLOGY = 5; - RVS_TEST = 6; - RVS_MEMBW_TEST = 7; - GPU_PARAMETERS = 8; + GPU_PARAMETERS = 6; + RVS_GST_TEST = 7; + RVS_MEMBW_TEST = 8; + RVS_H2DD2H_TEST = 9; + RVS_IET_TEST = 10; }; DiagnosticTestCase test_case = 3; uint32 per_gpu_result_count = 4; @@ -533,9 +535,11 @@ message DiagnosticTestCaseRunRequest { VRAM_CHECK = 3; SYS_MEM_CHECK = 4; NODE_TOPOLOGY = 5; - RVS_TEST = 6; - RVS_MEMBW_TEST = 7; - GPU_PARAMETERS = 8; + GPU_PARAMETERS = 6; + RVS_GST_TEST = 7; + RVS_MEMBW_TEST = 8; + RVS_H2DD2H_TEST = 9; + RVS_IET_TEST = 10; }; TestCaseType test_case = 2; string config = 3; diff --git a/rdc_libs/rdc/src/RdcDiagnosticModule.cc b/rdc_libs/rdc/src/RdcDiagnosticModule.cc index 3edf512407..1da5a27f57 100644 --- a/rdc_libs/rdc/src/RdcDiagnosticModule.cc +++ b/rdc_libs/rdc/src/RdcDiagnosticModule.cc @@ -88,8 +88,10 @@ rdc_status_t RdcDiagnosticModule::rdc_diagnostic_run(const rdc_group_info_t& gpu } if (level >= RDC_DIAG_LVL_MED) { // Medium run and above - rdc_runs.push_back(RDC_DIAG_RVS_TEST); + rdc_runs.push_back(RDC_DIAG_RVS_GST_TEST); rdc_runs.push_back(RDC_DIAG_RVS_MEMBW_TEST); + rdc_runs.push_back(RDC_DIAG_RVS_H2DD2H_TEST); + rdc_runs.push_back(RDC_DIAG_RVS_IET_TEST); } if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) { diff --git a/rdc_libs/rdc_modules/rdc_rvs/RdcDiagnosticLib.cc b/rdc_libs/rdc_modules/rdc_rvs/RdcDiagnosticLib.cc index 13b1ba78fa..484677505f 100644 --- a/rdc_libs/rdc_modules/rdc_rvs/RdcDiagnosticLib.cc +++ b/rdc_libs/rdc_modules/rdc_rvs/RdcDiagnosticLib.cc @@ -27,11 +27,6 @@ THE SOFTWARE. #include "rdc_lib/rdc_common.h" #include "rdc_modules/rdc_rvs/RvsBase.h" - static const char babel_config[MAX_CONFIG_LENGTH] = - "{actions: [{name: babel-float-256MiB, device: all, module: babel, " - "parallel: false, count: 1, num_iter: 5000, array_size: 268435456, " - "test_type: 1, mibibytes: true, o/p_csv: false, subtest: 5}]}"; - rdc_status_t rdc_diag_init(uint64_t) { return RDC_ST_OK; } rdc_status_t rdc_diag_destroy() { return RDC_ST_OK; } @@ -42,9 +37,11 @@ rdc_status_t rdc_diag_test_cases_query(rdc_diag_test_cases_t test_cases[MAX_TEST return RDC_ST_BAD_PARAMETER; } - *test_case_count = 2; - test_cases[0] = RDC_DIAG_RVS_TEST; + *test_case_count = 4; + test_cases[0] = RDC_DIAG_RVS_GST_TEST; test_cases[1] = RDC_DIAG_RVS_MEMBW_TEST; + test_cases[2] = RDC_DIAG_RVS_H2DD2H_TEST; + test_cases[3] = RDC_DIAG_RVS_IET_TEST; return RDC_ST_OK; } @@ -68,18 +65,26 @@ rdc_status_t rdc_diag_test_case_run(rdc_diag_test_cases_t test_case, result->per_gpu_result_count = 0; if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) { - std::string str = "RVS test"; + std::string str = "RVS test [" + test_to_name.at(test_case) + "]"; callback->callback(callback->cookie, str.data()); } switch (test_case) { - case RDC_DIAG_RVS_TEST: - strncpy_with_null(result->info, "Finished running RDC_DIAG_RVS_TEST", MAX_DIAG_MSG_LENGTH); - rvs_status = rvs_base.run_rvs_app(config, config_size, callback); - break; + case RDC_DIAG_RVS_GST_TEST: case RDC_DIAG_RVS_MEMBW_TEST: - strncpy_with_null(result->info, "Finished running RDC_DIAG_RVS_MEMBW_TEST", MAX_DIAG_MSG_LENGTH); - rvs_status = rvs_base.run_rvs_app(babel_config, MAX_CONFIG_LENGTH, callback); + case RDC_DIAG_RVS_H2DD2H_TEST: + case RDC_DIAG_RVS_IET_TEST: { + const std::string test_name = "Finished running " + test_to_name.at(test_case); + const std::string predefined_config = test_to_conf.at(test_case); + // +1 to copy null + strncpy_with_null(result->info, test_name.c_str(), test_name.length() + 1); + if (config == nullptr || config_size == 0) { + rvs_status = rvs_base.run_rvs_app(predefined_config.c_str(), predefined_config.length() + 1, + callback); + } else { + rvs_status = rvs_base.run_rvs_app(config, config_size, callback); + } break; + } default: result->status = RDC_DIAG_RESULT_SKIP; strncpy_with_null(result->info, "Not supported yet", MAX_DIAG_MSG_LENGTH); diff --git a/rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc b/rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc index 57d45db518..738cdb3866 100644 --- a/rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc +++ b/rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc @@ -42,21 +42,11 @@ rvs_status_t amd::rdc::RdcRVSBase::run_rvs_app(const char* config, const size_t rvs_status_t status; // NOTE: device_index is NOT set by RDC unless a custom config is provided. // Meaning RDC index has no impact on RVS index. - const char mem_config[MAX_CONFIG_LENGTH] = - "{actions: [{name: action_1, device: all, module: mem, parallel: true, " - "count: 1, wait: 100, mapped_memory: false, mem_blocks: 128, " - "num_passes: 500, thrds_per_blk: 64, stress: true, num_iter: 50000, " - "exclude: '5 6 7 8 9 10 11'}]}"; - const char gst_config[MAX_CONFIG_LENGTH] = - "{actions: [{name: gpustress-9000-sgemm-false, device: all, " - "device_index: '0', module: gst, parallel: false, count: 1, duration: " - "10000, copy_matrix: false, target_stress: 9000, matrix_size_a: 8640, " - "matrix_size_b: 8640, matrix_size_c: 8640, ops_type: sgemm, lda: 8640, " - "ldb: 8640, ldc: 8640}]}"; if ((config == nullptr) || (config_size == 0)) { RDC_LOG(RDC_INFO, "given config is NULL! Using predefined gst_config"); - strncpy_with_null(active_config, gst_config, MAX_CONFIG_LENGTH); + strncpy_with_null(active_config, test_to_conf.at(RDC_DIAG_RVS_GST_TEST).c_str(), + test_to_conf.at(RDC_DIAG_RVS_GST_TEST).length()+1); } else if (config_size > MAX_CONFIG_LENGTH) { RDC_LOG(RDC_ERROR, "given config size is too large! Expected at most " << MAX_CONFIG_LENGTH << ", got " << config_size << " instead."); diff --git a/rdci/src/RdciDiagSubSystem.cc b/rdci/src/RdciDiagSubSystem.cc index 1dfc1caa58..95a3b9aca3 100644 --- a/rdci/src/RdciDiagSubSystem.cc +++ b/rdci/src/RdciDiagSubSystem.cc @@ -136,9 +136,11 @@ std::string RdciDiagSubSystem::get_test_name(rdc_diag_test_cases_t test_case) co {RDC_DIAG_COMPUTE_QUEUE, "Compute Queue ready"}, {RDC_DIAG_SYS_MEM_CHECK, "System memory check"}, {RDC_DIAG_NODE_TOPOLOGY, "Node topology check"}, - {RDC_DIAG_RVS_TEST, "Pre-defined config RVS check"}, - {RDC_DIAG_RVS_MEMBW_TEST, "Pre-defined memory bandwidth RVS check"}, {RDC_DIAG_GPU_PARAMETERS, "GPU parameters check"}, + {RDC_DIAG_RVS_GST_TEST, "RVS GST test"}, + {RDC_DIAG_RVS_MEMBW_TEST, "RVS babel memory bandwidth test"}, + {RDC_DIAG_RVS_H2DD2H_TEST, "RVS Host<->Device transfer speed test"}, + {RDC_DIAG_RVS_IET_TEST, "RVS IET test"}, {RDC_DIAG_TEST_LAST, "Unknown"}}; auto test_name = test_desc.find(test_case);