AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

RVS - Add IET and PEBB tests

Browse Source 
Change-Id: Ia032901d74c882e5cbfa5a3164199cd4d571341f
Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>
This commit is contained in:
Galantsev, Dmitrii
2025-01-06 18:14:44 -06:00
committed by Galantsev, Dmitrii
parent b058cbecf1
commit 5861ec7663
7 changed files with 92 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/rdc/rdc.h
+19 -17
View File
@@ -187,12 +187,12 @@ typedef enum {
/**
* @brief GPU usage related fields
*/
RDC_FI_GPU_UTIL = 500, //!< GPU Utilization
RDC_FI_GPU_MEMORY_USAGE, //!< Memory usage of the GPU instance
RDC_FI_GPU_MEMORY_TOTAL, //!< Total memory of the GPU instance
RDC_FI_GPU_MM_ENC_UTIL, //!< Multimedia encoder busy percentage
RDC_FI_GPU_MM_DEC_UTIL, //!< Multimedia decoder busy percentage
RDC_FI_GPU_MEMORY_ACTIVITY, //!< Memory busy percentage
RDC_FI_GPU_UTIL = 500, //!< GPU Utilization
RDC_FI_GPU_MEMORY_USAGE, //!< Memory usage of the GPU instance
RDC_FI_GPU_MEMORY_TOTAL, //!< Total memory of the GPU instance
RDC_FI_GPU_MM_ENC_UTIL, //!< Multimedia encoder busy percentage
RDC_FI_GPU_MM_DEC_UTIL, //!< Multimedia decoder busy percentage
RDC_FI_GPU_MEMORY_ACTIVITY, //!< Memory busy percentage
RDC_FI_GPU_MEMORY_MAX_BANDWIDTH, //<! The Memory max bandwidth at current memory clock in Mb/Second
RDC_FI_GPU_MEMORY_CUR_BANDWIDTH, //<! The Memory current bandwidth in Mb/Second
@@ -515,13 +515,15 @@ typedef enum {
RDC_DIAG_TEST_FIRST = 0,
//!< The diagnostic test pass
RDC_DIAG_COMPUTE_PROCESS = RDC_DIAG_TEST_FIRST,
RDC_DIAG_COMPUTE_QUEUE, //!< The Compute Queue is ready
RDC_DIAG_SYS_MEM_CHECK, //!< Check System memory
RDC_DIAG_NODE_TOPOLOGY, //!< Report node topology
RDC_DIAG_RVS_TEST, //!< TODO: Replace with real RVS tests
RDC_DIAG_RVS_MEMBW_TEST, //!< RVS memory bandwidth test
RDC_DIAG_GPU_PARAMETERS, //!< GPU parameters in range
RDC_DIAG_TEST_LAST = RDC_DIAG_GPU_PARAMETERS
RDC_DIAG_COMPUTE_QUEUE, //!< The Compute Queue is ready
RDC_DIAG_SYS_MEM_CHECK, //!< Check System memory
RDC_DIAG_NODE_TOPOLOGY, //!< Report node topology
RDC_DIAG_GPU_PARAMETERS, //!< GPU parameters in range
RDC_DIAG_RVS_GST_TEST, //!< RVS GST test
RDC_DIAG_RVS_MEMBW_TEST, //!< RVS bandwidth test
RDC_DIAG_RVS_H2DD2H_TEST, //!< RVS Host<->Device transfer speed test
RDC_DIAG_RVS_IET_TEST, //!< RVS IET test
RDC_DIAG_TEST_LAST = RDC_DIAG_RVS_IET_TEST
} rdc_diag_test_cases_t;
/**
@@ -661,13 +663,13 @@ typedef enum {
typedef struct {
uint32_t gpu_index;
uint32_t num_of_links; // The size of the array link_states
rdc_topology_link_type_t link_types; // XGMI, PCIe, and so on
uint32_t num_of_links; // The size of the array link_states
rdc_topology_link_type_t link_types; // XGMI, PCIe, and so on
rdc_link_state_t link_states[RDC_MAX_NUM_OF_LINKS];
} rdc_gpu_link_status_t;
typedef struct {
int32_t num_of_gpus; // The size of gpus array
int32_t num_of_gpus; // The size of gpus array
rdc_gpu_link_status_t gpus[RDC_MAX_NUM_DEVICES];
} rdc_link_status_t;
@@ -1522,7 +1524,7 @@ rdc_status_t rdc_health_check(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_i
*/
rdc_status_t rdc_health_clear(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id);
/**
/**
* @brief Get the topology of the device
*
* @details topology of the device
include/rdc_modules/rdc_rvs/RvsBase.h
+35
View File
@@ -24,12 +24,47 @@ THE SOFTWARE.
#include <cstddef>
#include <cstdio>
#include <map>
#include <string>
#include "rdc/rdc.h"
#include "rvs/rvs.h"
static constexpr size_t MAX_CONFIG_LENGTH = 1024;
// NOTE: There MUST be a space after :
static const std::map<rdc_diag_test_cases_t, std::string> test_to_conf = {
// derived from conf/gst_single.conf
{RDC_DIAG_RVS_GST_TEST,
"{actions: [{name: gpustress-9000-sgemm-false, device: all, "
"device_index: '0', module: gst, parallel: false, count: 1, duration: "
"10000, copy_matrix: false, target_stress: 9000, matrix_size_a: 8640, "
"matrix_size_b: 8640, matrix_size_c: 8640, ops_type: sgemm, lda: 8640, "
"ldb: 8640, ldc: 8640}]}"},
// derived from conf/MI300X/babel.conf
{RDC_DIAG_RVS_MEMBW_TEST,
"{actions: [{name: babel-float-256MiB,"
"device: all, module: babel, "
"parallel: false, count: 1, num_iter: 5000, array_size: 268435456, "
"test_type: 1, mibibytes: true, o/p_csv: false, subtest: 5}]}"},
// derived from conf/MI300X/pebb_single.conf
{RDC_DIAG_RVS_H2DD2H_TEST,
"{actions: [{name: h2d-d2h-sequential-64MB,"
"device: all, module: pebb, duration: 120000, device_to_host: true, "
"host_to_device: true, parallel: false, block_size: 67108864, "
"link_type: 2, warm_calls: 10, hot_calls: 100, b2b: true}]}"},
// derived from conf/MI300X/iet_single.conf
{RDC_DIAG_RVS_IET_TEST,
"{actions: [{name: iet-400W-1K-rand-dgemm,"
"device: all, module: iet, parallel: true, duration: 60000, "
"sample_interval: 3000, target_power: 400, matrix_size: 1024, "
"matrix_init: rand, ops_type: dgemm}]}"},
};
static const std::map<rdc_diag_test_cases_t, std::string> test_to_name = {
{RDC_DIAG_RVS_GST_TEST, "RVS_GST_TEST"},
{RDC_DIAG_RVS_MEMBW_TEST, "RVS_MEMBW_TEST"},
{RDC_DIAG_RVS_H2DD2H_TEST, "RVS_H2DD2H_TEST"},
{RDC_DIAG_RVS_IET_TEST, "RVS_IET_TEST"},
};
namespace amd {
namespace rdc {
protos/rdc.proto
+10 -6
View File
@@ -503,9 +503,11 @@ message DiagnosticTestResult {
VRAM_CHECK = 3;
SYS_MEM_CHECK = 4;
NODE_TOPOLOGY = 5;
RVS_TEST = 6;
RVS_MEMBW_TEST = 7;
GPU_PARAMETERS = 8;
GPU_PARAMETERS = 6;
RVS_GST_TEST = 7;
RVS_MEMBW_TEST = 8;
RVS_H2DD2H_TEST = 9;
RVS_IET_TEST = 10;
};
DiagnosticTestCase test_case = 3;
uint32 per_gpu_result_count = 4;
@@ -533,9 +535,11 @@ message DiagnosticTestCaseRunRequest {
VRAM_CHECK = 3;
SYS_MEM_CHECK = 4;
NODE_TOPOLOGY = 5;
RVS_TEST = 6;
RVS_MEMBW_TEST = 7;
GPU_PARAMETERS = 8;
GPU_PARAMETERS = 6;
RVS_GST_TEST = 7;
RVS_MEMBW_TEST = 8;
RVS_H2DD2H_TEST = 9;
RVS_IET_TEST = 10;
};
TestCaseType test_case = 2;
string config = 3;
rdc_libs/rdc/src/RdcDiagnosticModule.cc
+3 -1
View File
@@ -88,8 +88,10 @@ rdc_status_t RdcDiagnosticModule::rdc_diagnostic_run(const rdc_group_info_t& gpu
}
if (level >= RDC_DIAG_LVL_MED) { // Medium run and above
rdc_runs.push_back(RDC_DIAG_RVS_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_GST_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_MEMBW_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_H2DD2H_TEST);
rdc_runs.push_back(RDC_DIAG_RVS_IET_TEST);
}
if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) {
rdc_libs/rdc_modules/rdc_rvs/RdcDiagnosticLib.cc
+19 -14
View File
@@ -27,11 +27,6 @@ THE SOFTWARE.
#include "rdc_lib/rdc_common.h"
#include "rdc_modules/rdc_rvs/RvsBase.h"
static const char babel_config[MAX_CONFIG_LENGTH] =
"{actions: [{name: babel-float-256MiB, device: all, module: babel, "
"parallel: false, count: 1, num_iter: 5000, array_size: 268435456, "
"test_type: 1, mibibytes: true, o/p_csv: false, subtest: 5}]}";
rdc_status_t rdc_diag_init(uint64_t) { return RDC_ST_OK; }
rdc_status_t rdc_diag_destroy() { return RDC_ST_OK; }
@@ -42,9 +37,11 @@ rdc_status_t rdc_diag_test_cases_query(rdc_diag_test_cases_t test_cases[MAX_TEST
return RDC_ST_BAD_PARAMETER;
}
*test_case_count = 2;
test_cases[0] = RDC_DIAG_RVS_TEST;
*test_case_count = 4;
test_cases[0] = RDC_DIAG_RVS_GST_TEST;
test_cases[1] = RDC_DIAG_RVS_MEMBW_TEST;
test_cases[2] = RDC_DIAG_RVS_H2DD2H_TEST;
test_cases[3] = RDC_DIAG_RVS_IET_TEST;
return RDC_ST_OK;
}
@@ -68,18 +65,26 @@ rdc_status_t rdc_diag_test_case_run(rdc_diag_test_cases_t test_case,
result->per_gpu_result_count = 0;
if (callback != nullptr && callback->callback != nullptr && callback->cookie != nullptr) {
std::string str = "RVS test";
std::string str = "RVS test [" + test_to_name.at(test_case) + "]";
callback->callback(callback->cookie, str.data());
}
switch (test_case) {
case RDC_DIAG_RVS_TEST:
strncpy_with_null(result->info, "Finished running RDC_DIAG_RVS_TEST", MAX_DIAG_MSG_LENGTH);
rvs_status = rvs_base.run_rvs_app(config, config_size, callback);
break;
case RDC_DIAG_RVS_GST_TEST:
case RDC_DIAG_RVS_MEMBW_TEST:
strncpy_with_null(result->info, "Finished running RDC_DIAG_RVS_MEMBW_TEST", MAX_DIAG_MSG_LENGTH);
rvs_status = rvs_base.run_rvs_app(babel_config, MAX_CONFIG_LENGTH, callback);
case RDC_DIAG_RVS_H2DD2H_TEST:
case RDC_DIAG_RVS_IET_TEST: {
const std::string test_name = "Finished running " + test_to_name.at(test_case);
const std::string predefined_config = test_to_conf.at(test_case);
// +1 to copy null
strncpy_with_null(result->info, test_name.c_str(), test_name.length() + 1);
if (config == nullptr || config_size == 0) {
rvs_status = rvs_base.run_rvs_app(predefined_config.c_str(), predefined_config.length() + 1,
callback);
} else {
rvs_status = rvs_base.run_rvs_app(config, config_size, callback);
}
break;
}
default:
result->status = RDC_DIAG_RESULT_SKIP;
strncpy_with_null(result->info, "Not supported yet", MAX_DIAG_MSG_LENGTH);
rdc_libs/rdc_modules/rdc_rvs/RvsBase.cc
+2 -12
View File
@@ -42,21 +42,11 @@ rvs_status_t amd::rdc::RdcRVSBase::run_rvs_app(const char* config, const size_t
rvs_status_t status;
// NOTE: device_index is NOT set by RDC unless a custom config is provided.
// Meaning RDC index has no impact on RVS index.
const char mem_config[MAX_CONFIG_LENGTH] =
"{actions: [{name: action_1, device: all, module: mem, parallel: true, "
"count: 1, wait: 100, mapped_memory: false, mem_blocks: 128, "
"num_passes: 500, thrds_per_blk: 64, stress: true, num_iter: 50000, "
"exclude: '5 6 7 8 9 10 11'}]}";
const char gst_config[MAX_CONFIG_LENGTH] =
"{actions: [{name: gpustress-9000-sgemm-false, device: all, "
"device_index: '0', module: gst, parallel: false, count: 1, duration: "
"10000, copy_matrix: false, target_stress: 9000, matrix_size_a: 8640, "
"matrix_size_b: 8640, matrix_size_c: 8640, ops_type: sgemm, lda: 8640, "
"ldb: 8640, ldc: 8640}]}";
if ((config == nullptr) || (config_size == 0)) {
RDC_LOG(RDC_INFO, "given config is NULL! Using predefined gst_config");
strncpy_with_null(active_config, gst_config, MAX_CONFIG_LENGTH);
strncpy_with_null(active_config, test_to_conf.at(RDC_DIAG_RVS_GST_TEST).c_str(),
test_to_conf.at(RDC_DIAG_RVS_GST_TEST).length()+1);
} else if (config_size > MAX_CONFIG_LENGTH) {
RDC_LOG(RDC_ERROR, "given config size is too large! Expected at most "
<< MAX_CONFIG_LENGTH << ", got " << config_size << " instead.");
rdci/src/RdciDiagSubSystem.cc
+4 -2
View File
@@ -136,9 +136,11 @@ std::string RdciDiagSubSystem::get_test_name(rdc_diag_test_cases_t test_case) co
{RDC_DIAG_COMPUTE_QUEUE, "Compute Queue ready"},
{RDC_DIAG_SYS_MEM_CHECK, "System memory check"},
{RDC_DIAG_NODE_TOPOLOGY, "Node topology check"},
{RDC_DIAG_RVS_TEST, "Pre-defined config RVS check"},
{RDC_DIAG_RVS_MEMBW_TEST, "Pre-defined memory bandwidth RVS check"},
{RDC_DIAG_GPU_PARAMETERS, "GPU parameters check"},
{RDC_DIAG_RVS_GST_TEST, "RVS GST test"},
{RDC_DIAG_RVS_MEMBW_TEST, "RVS babel memory bandwidth test"},
{RDC_DIAG_RVS_H2DD2H_TEST, "RVS Host<->Device transfer speed test"},
{RDC_DIAG_RVS_IET_TEST, "RVS IET test"},
{RDC_DIAG_TEST_LAST, "Unknown"}};
auto test_name = test_desc.find(test_case);