2
0

SWDEV-452795 - Disable RAS plugin, fix XGMI

RAS plugin loaded rocm-smi which is in conflict with amd-smi library

Main source of grief was the map 'devInfoTypesStrings' that is defined
in both rocm-smi and amd-smi

We assume that rocm-smi would get lazy-loaded by RAS library and
overwrite symbols defined in amd-smi. devInfoTypesStrings in rocm-smi
contains different number of elements, the enums are also different.
RDC relies on amd-smi's enums.

One such enum is kDevGpuMetrics:
  rocm-smi: kDevGpuMetrics = 68
  amd-smi:  kDevGpuMetrics = 75

Example of overlapping map definitions:

  $ objdump --dynamic-syms /opt/rocm/lib/libamd_smi.so | grep devInfoTypesStrings
  00000000003c4980 g    DO .data.rel.ro0000000000000008  Base        devInfoTypesStrings
  00000000003db830 g    DO .bss0000000000000030  Base        _ZN3amd3smi6Device19devInfoTypesStringsE
  $ objdump --dynamic-syms /opt/rocm/lib/librocm_smi64.so  | grep devInfoTypesStrings
  00000000003dc590 g    DO .bss0000000000000030  Base        _ZN3amd3smi6Device19devInfoTypesStringsE
  00000000003c9c68 g    DO .data.rel.ro0000000000000008  Base        devInfoTypesStrings

Change-Id: Ib2f2db32b6abd7ebe84e7807c25581461eb86bae
Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>
Este cometimento está contido em:
Galantsev, Dmitrii
2024-06-24 17:55:37 -05:00
ascendente a885944d97
cometimento d85657e5f2
184 ficheiros modificados com 277 adições e 25987 eliminações
+151 -2
Ver ficheiro
@@ -91,7 +91,116 @@ uint64_t RdcMetricFetcherImpl::now() {
return static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
}
void RdcMetricFetcherImpl::get_ecc_error(uint32_t gpu_index, rdc_field_t field_id,
void RdcMetricFetcherImpl::get_ecc(uint32_t gpu_index, rdc_field_t field_id,
rdc_field_value* value) {
amdsmi_status_t err = AMDSMI_STATUS_SUCCESS;
amdsmi_ras_err_state_t err_state;
amdsmi_processor_handle processor_handle;
err = get_processor_handle_from_id(gpu_index, &processor_handle);
// because RDC already had an established order that is different from amd-smi : map blocks to
// fields manually
auto field_to_block_ = [](rdc_field_t field) -> amdsmi_gpu_block_t {
switch (field) {
case RDC_FI_ECC_SDMA_CE:
case RDC_FI_ECC_SDMA_UE:
return AMDSMI_GPU_BLOCK_SDMA;
case RDC_FI_ECC_GFX_CE:
case RDC_FI_ECC_GFX_UE:
return AMDSMI_GPU_BLOCK_GFX;
case RDC_FI_ECC_MMHUB_CE:
case RDC_FI_ECC_MMHUB_UE:
return AMDSMI_GPU_BLOCK_MMHUB;
case RDC_FI_ECC_ATHUB_CE:
case RDC_FI_ECC_ATHUB_UE:
return AMDSMI_GPU_BLOCK_ATHUB;
case RDC_FI_ECC_PCIE_BIF_CE:
case RDC_FI_ECC_PCIE_BIF_UE:
return AMDSMI_GPU_BLOCK_PCIE_BIF;
case RDC_FI_ECC_HDP_CE:
case RDC_FI_ECC_HDP_UE:
return AMDSMI_GPU_BLOCK_HDP;
case RDC_FI_ECC_XGMI_WAFL_CE:
case RDC_FI_ECC_XGMI_WAFL_UE:
return AMDSMI_GPU_BLOCK_XGMI_WAFL;
case RDC_FI_ECC_DF_CE:
case RDC_FI_ECC_DF_UE:
return AMDSMI_GPU_BLOCK_DF;
case RDC_FI_ECC_SMN_CE:
case RDC_FI_ECC_SMN_UE:
return AMDSMI_GPU_BLOCK_SMN;
case RDC_FI_ECC_SEM_CE:
case RDC_FI_ECC_SEM_UE:
return AMDSMI_GPU_BLOCK_SEM;
case RDC_FI_ECC_MP0_CE:
case RDC_FI_ECC_MP0_UE:
return AMDSMI_GPU_BLOCK_MP0;
case RDC_FI_ECC_MP1_CE:
case RDC_FI_ECC_MP1_UE:
return AMDSMI_GPU_BLOCK_MP1;
case RDC_FI_ECC_FUSE_CE:
case RDC_FI_ECC_FUSE_UE:
return AMDSMI_GPU_BLOCK_FUSE;
case RDC_FI_ECC_UMC_CE:
case RDC_FI_ECC_UMC_UE:
return AMDSMI_GPU_BLOCK_UMC;
case RDC_FI_ECC_MCA_CE:
case RDC_FI_ECC_MCA_UE:
return AMDSMI_GPU_BLOCK_MCA;
case RDC_FI_ECC_VCN_CE:
case RDC_FI_ECC_VCN_UE:
return AMDSMI_GPU_BLOCK_VCN;
case RDC_FI_ECC_JPEG_CE:
case RDC_FI_ECC_JPEG_UE:
return AMDSMI_GPU_BLOCK_JPEG;
case RDC_FI_ECC_IH_CE:
case RDC_FI_ECC_IH_UE:
return AMDSMI_GPU_BLOCK_IH;
case RDC_FI_ECC_MPIO_CE:
case RDC_FI_ECC_MPIO_UE:
return AMDSMI_GPU_BLOCK_MPIO;
default:
return AMDSMI_GPU_BLOCK_INVALID;
}
};
const bool is_correctable = (field_id % 2 == 0);
if (!value) {
return;
}
auto gpu_block = field_to_block_(field_id);
if (gpu_block == AMDSMI_GPU_BLOCK_INVALID) {
value->status = AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS;
}
err = amdsmi_get_gpu_ecc_status(processor_handle, gpu_block, &err_state);
if (err != AMDSMI_STATUS_SUCCESS) {
RDC_LOG(RDC_INFO, "Error in ecc status [" << gpu_block << "]:" << err);
value->status = err;
return;
}
amdsmi_error_count_t ec;
err = amdsmi_get_gpu_ecc_count(processor_handle, gpu_block, &ec);
if (err != AMDSMI_STATUS_SUCCESS) {
RDC_LOG(RDC_ERROR, "Error in ecc count [" << gpu_block << "]:" << err);
value->status = err;
return;
}
value->status = AMDSMI_STATUS_SUCCESS;
value->type = INTEGER;
if (is_correctable) {
value->value.l_int = ec.correctable_count;
} else {
value->value.l_int = ec.uncorrectable_count;
}
}
void RdcMetricFetcherImpl::get_ecc_total(uint32_t gpu_index, rdc_field_t field_id,
rdc_field_value* value) {
amdsmi_status_t err = AMDSMI_STATUS_SUCCESS;
uint64_t correctable_count = 0;
@@ -516,7 +625,47 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
}
case RDC_FI_ECC_CORRECT_TOTAL:
case RDC_FI_ECC_UNCORRECT_TOTAL:
get_ecc_error(gpu_index, field_id, value);
get_ecc_total(gpu_index, field_id, value);
break;
case RDC_FI_ECC_SDMA_CE:
case RDC_FI_ECC_SDMA_UE:
case RDC_FI_ECC_GFX_CE:
case RDC_FI_ECC_GFX_UE:
case RDC_FI_ECC_MMHUB_CE:
case RDC_FI_ECC_MMHUB_UE:
case RDC_FI_ECC_ATHUB_CE:
case RDC_FI_ECC_ATHUB_UE:
case RDC_FI_ECC_PCIE_BIF_CE:
case RDC_FI_ECC_PCIE_BIF_UE:
case RDC_FI_ECC_HDP_CE:
case RDC_FI_ECC_HDP_UE:
case RDC_FI_ECC_XGMI_WAFL_CE:
case RDC_FI_ECC_XGMI_WAFL_UE:
case RDC_FI_ECC_DF_CE:
case RDC_FI_ECC_DF_UE:
case RDC_FI_ECC_SMN_CE:
case RDC_FI_ECC_SMN_UE:
case RDC_FI_ECC_SEM_CE:
case RDC_FI_ECC_SEM_UE:
case RDC_FI_ECC_MP0_CE:
case RDC_FI_ECC_MP0_UE:
case RDC_FI_ECC_MP1_CE:
case RDC_FI_ECC_MP1_UE:
case RDC_FI_ECC_FUSE_CE:
case RDC_FI_ECC_FUSE_UE:
case RDC_FI_ECC_UMC_CE:
case RDC_FI_ECC_UMC_UE:
case RDC_FI_ECC_MCA_CE:
case RDC_FI_ECC_MCA_UE:
case RDC_FI_ECC_VCN_CE:
case RDC_FI_ECC_VCN_UE:
case RDC_FI_ECC_JPEG_CE:
case RDC_FI_ECC_JPEG_UE:
case RDC_FI_ECC_IH_CE:
case RDC_FI_ECC_IH_UE:
case RDC_FI_ECC_MPIO_CE:
case RDC_FI_ECC_MPIO_UE:
get_ecc(gpu_index, field_id, value);
break;
case RDC_FI_PCIE_TX:
case RDC_FI_PCIE_RX:
+1 -2
Ver ficheiro
@@ -30,7 +30,6 @@ THE SOFTWARE.
#include "rdc_lib/RdcTelemetry.h"
#include "rdc_lib/impl/RdcDiagnosticModule.h"
#include "rdc_lib/impl/RdcRVSLib.h"
#include "rdc_lib/impl/RdcRasLib.h"
#include "rdc_lib/impl/RdcRocpLib.h"
#include "rdc_lib/impl/RdcRocrLib.h"
#include "rdc_lib/impl/RdcSmiLib.h"
@@ -89,7 +88,7 @@ RdcModuleMgrImpl::RdcModuleMgrImpl(const RdcMetricFetcherPtr& fetcher) : fetcher
}
// all other modules get initialized by insert_modules
insert_modules<RdcRasLib, RdcRVSLib, RdcRocrLib, RdcRocpLib>();
insert_modules<RdcRVSLib, RdcRocrLib, RdcRocpLib>();
}
RdcTelemetryPtr RdcModuleMgrImpl::get_telemetry_module() {
-178
Ver ficheiro
@@ -1,178 +0,0 @@
/*
Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "rdc_lib/impl/RdcRasLib.h"
namespace amd {
namespace rdc {
RdcRasLib::RdcRasLib()
: fields_value_get_(nullptr),
fields_query_(nullptr),
fields_watch_(nullptr),
fields_unwatch_(nullptr),
rdc_module_init_(nullptr),
rdc_module_destroy_(nullptr) {
rdc_status_t status = lib_loader_.load("librdc_ras.so");
if (status != RDC_ST_OK) {
RDC_LOG(RDC_ERROR, "RAS related function will not work.");
return;
}
status = lib_loader_.load_symbol(&rdc_module_init_, "rdc_module_init");
if (status != RDC_ST_OK) {
rdc_module_init_ = nullptr;
return;
}
status = rdc_module_init_(0);
if (status != RDC_ST_OK) {
RDC_LOG(RDC_ERROR, "Fail to init librdc_ras.so:" << rdc_status_string(status)
<< ". RAS related function will not work.");
return;
}
status = lib_loader_.load_symbol(&rdc_module_destroy_, "rdc_module_destroy");
if (status != RDC_ST_OK) {
rdc_module_destroy_ = nullptr;
}
status = lib_loader_.load_symbol(&fields_value_get_, "rdc_telemetry_fields_value_get");
if (status != RDC_ST_OK) {
fields_value_get_ = nullptr;
}
status = lib_loader_.load_symbol(&fields_query_, "rdc_telemetry_fields_query");
if (status != RDC_ST_OK) {
fields_query_ = nullptr;
}
status = lib_loader_.load_symbol(&fields_watch_, "rdc_telemetry_fields_watch");
if (status != RDC_ST_OK) {
fields_watch_ = nullptr;
}
status = lib_loader_.load_symbol(&fields_unwatch_, "rdc_telemetry_fields_unwatch");
if (status != RDC_ST_OK) {
fields_unwatch_ = nullptr;
}
}
RdcRasLib::~RdcRasLib() {
if (rdc_module_destroy_) {
rdc_module_destroy_();
}
}
rdc_status_t RdcRasLib::rdc_telemetry_fields_query(uint32_t field_ids[MAX_NUM_FIELDS],
uint32_t* field_count) {
if (field_count == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
if (!fields_query_) {
*field_count = 0;
return RDC_ST_FAIL_LOAD_MODULE;
}
auto status = fields_query_(field_ids, field_count);
RDC_LOG(RDC_DEBUG, "RAS support " << *field_count << " fields");
return status;
}
rdc_status_t RdcRasLib::rdc_telemetry_fields_value_get(rdc_gpu_field_t* fields,
uint32_t fields_count,
rdc_field_value_f callback,
void* user_data) {
if (fields == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
if (!fields_value_get_) {
return RDC_ST_FAIL_LOAD_MODULE;
}
rdc_status_t status = fields_value_get_(fields, fields_count, callback, user_data);
RDC_LOG(RDC_DEBUG,
"Bulk fetched " << fields_count << " fields from RAS: " << rdc_status_string(status));
return status;
}
rdc_status_t RdcRasLib::rdc_telemetry_fields_watch(rdc_gpu_field_t* fields, uint32_t fields_count) {
if (fields == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
if (!fields_watch_) {
return RDC_ST_FAIL_LOAD_MODULE;
}
rdc_status_t status = fields_watch_(fields, fields_count);
RDC_LOG(RDC_DEBUG, "Watch " << fields_count << " fields from RAS: " << rdc_status_string(status));
return status;
}
rdc_status_t RdcRasLib::rdc_telemetry_fields_unwatch(rdc_gpu_field_t* fields,
uint32_t fields_count) {
if (fields == nullptr) {
return RDC_ST_BAD_PARAMETER;
}
if (!fields_unwatch_) {
return RDC_ST_FAIL_LOAD_MODULE;
}
rdc_status_t status = fields_unwatch_(fields, fields_count);
RDC_LOG(RDC_DEBUG,
"Unwatch " << fields_count << " fields from RAS: " << rdc_status_string(status));
return status;
}
rdc_status_t RdcRasLib::rdc_diag_test_cases_query(rdc_diag_test_cases_t test_cases[MAX_TEST_CASES],
uint32_t* test_case_count) {
(void)test_cases;
(void)test_case_count;
return RDC_ST_NOT_SUPPORTED;
}
// Run a specific test case
rdc_status_t RdcRasLib::rdc_test_case_run(rdc_diag_test_cases_t test_case,
uint32_t gpu_index[RDC_MAX_NUM_DEVICES],
uint32_t gpu_count, const char* config,
size_t config_size, rdc_diag_test_result_t* result) {
(void)test_case;
(void)gpu_index;
(void)result;
(void)gpu_count;
(void)config;
(void)config_size;
return RDC_ST_NOT_SUPPORTED;
}
rdc_status_t RdcRasLib::rdc_diagnostic_run(const rdc_group_info_t& gpus, rdc_diag_level_t level,
const char* config, size_t config_size,
rdc_diag_response_t* response) {
(void)gpus;
(void)level;
(void)config;
(void)config_size;
(void)response;
return RDC_ST_NOT_SUPPORTED;
}
rdc_status_t RdcRasLib::rdc_diag_init(uint64_t flags) {
(void)flags;
return RDC_ST_NOT_SUPPORTED;
}
rdc_status_t RdcRasLib::rdc_diag_destroy() { return RDC_ST_NOT_SUPPORTED; }
} // namespace rdc
} // namespace amd
+28 -22
Ver ficheiro
@@ -148,28 +148,34 @@ rdc_status_t RdcSmiLib::rdc_telemetry_fields_query(uint32_t field_ids[MAX_NUM_FI
// List of fields supported by amd_smi_lib
const std::vector<uint32_t> fields{
RDC_FI_GPU_COUNT, RDC_FI_DEV_NAME,
RDC_FI_GPU_CLOCK, RDC_FI_MEM_CLOCK,
RDC_FI_MEMORY_TEMP, RDC_FI_GPU_TEMP,
RDC_FI_POWER_USAGE, RDC_FI_PCIE_TX,
RDC_FI_PCIE_RX, RDC_FI_PCIE_BANDWIDTH,
RDC_FI_GPU_UTIL,
RDC_FI_GPU_MEMORY_USAGE, RDC_FI_GPU_MEMORY_TOTAL,
RDC_FI_ECC_CORRECT_TOTAL, RDC_FI_ECC_UNCORRECT_TOTAL,
RDC_FI_XGMI_0_READ_KB, RDC_FI_XGMI_1_READ_KB,
RDC_FI_XGMI_2_READ_KB, RDC_FI_XGMI_3_READ_KB,
RDC_FI_XGMI_4_READ_KB, RDC_FI_XGMI_5_READ_KB,
RDC_FI_XGMI_6_READ_KB, RDC_FI_XGMI_7_READ_KB,
RDC_FI_XGMI_0_WRITE_KB, RDC_FI_XGMI_1_WRITE_KB,
RDC_FI_XGMI_2_WRITE_KB, RDC_FI_XGMI_3_WRITE_KB,
RDC_FI_XGMI_4_WRITE_KB, RDC_FI_XGMI_5_WRITE_KB,
RDC_FI_XGMI_6_WRITE_KB, RDC_FI_XGMI_7_WRITE_KB,
RDC_EVNT_XGMI_0_NOP_TX, RDC_EVNT_XGMI_0_REQ_TX,
RDC_EVNT_XGMI_0_RESP_TX, RDC_EVNT_XGMI_0_BEATS_TX,
RDC_EVNT_XGMI_1_NOP_TX, RDC_EVNT_XGMI_1_REQ_TX,
RDC_EVNT_XGMI_1_RESP_TX, RDC_EVNT_XGMI_1_BEATS_TX,
RDC_EVNT_XGMI_0_THRPUT, RDC_EVNT_XGMI_1_THRPUT,
RDC_EVNT_XGMI_2_THRPUT, RDC_EVNT_XGMI_3_THRPUT,
RDC_FI_GPU_COUNT, RDC_FI_DEV_NAME, RDC_FI_GPU_CLOCK,
RDC_FI_MEM_CLOCK, RDC_FI_MEMORY_TEMP, RDC_FI_GPU_TEMP,
RDC_FI_POWER_USAGE, RDC_FI_PCIE_TX, RDC_FI_PCIE_RX,
RDC_FI_PCIE_BANDWIDTH, RDC_FI_GPU_UTIL, RDC_FI_GPU_MEMORY_USAGE,
RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_ECC_CORRECT_TOTAL, RDC_FI_ECC_UNCORRECT_TOTAL,
RDC_FI_ECC_SDMA_CE, RDC_FI_ECC_SDMA_UE, RDC_FI_ECC_GFX_CE,
RDC_FI_ECC_GFX_UE, RDC_FI_ECC_MMHUB_CE, RDC_FI_ECC_MMHUB_UE,
RDC_FI_ECC_ATHUB_CE, RDC_FI_ECC_ATHUB_UE, RDC_FI_ECC_PCIE_BIF_CE,
RDC_FI_ECC_PCIE_BIF_UE, RDC_FI_ECC_HDP_CE, RDC_FI_ECC_HDP_UE,
RDC_FI_ECC_XGMI_WAFL_CE, RDC_FI_ECC_XGMI_WAFL_UE, RDC_FI_ECC_DF_CE,
RDC_FI_ECC_DF_UE, RDC_FI_ECC_SMN_CE, RDC_FI_ECC_SMN_UE,
RDC_FI_ECC_SEM_CE, RDC_FI_ECC_SEM_UE, RDC_FI_ECC_MP0_CE,
RDC_FI_ECC_MP0_UE, RDC_FI_ECC_MP1_CE, RDC_FI_ECC_MP1_UE,
RDC_FI_ECC_FUSE_CE, RDC_FI_ECC_FUSE_UE, RDC_FI_ECC_UMC_CE,
RDC_FI_ECC_UMC_UE, RDC_FI_ECC_MCA_CE, RDC_FI_ECC_MCA_UE,
RDC_FI_ECC_VCN_CE, RDC_FI_ECC_VCN_UE, RDC_FI_ECC_JPEG_CE,
RDC_FI_ECC_JPEG_UE, RDC_FI_ECC_IH_CE, RDC_FI_ECC_IH_UE,
RDC_FI_ECC_MPIO_CE, RDC_FI_ECC_MPIO_UE, RDC_FI_XGMI_0_READ_KB,
RDC_FI_XGMI_1_READ_KB, RDC_FI_XGMI_2_READ_KB, RDC_FI_XGMI_3_READ_KB,
RDC_FI_XGMI_4_READ_KB, RDC_FI_XGMI_5_READ_KB, RDC_FI_XGMI_6_READ_KB,
RDC_FI_XGMI_7_READ_KB, RDC_FI_XGMI_0_WRITE_KB, RDC_FI_XGMI_1_WRITE_KB,
RDC_FI_XGMI_2_WRITE_KB, RDC_FI_XGMI_3_WRITE_KB, RDC_FI_XGMI_4_WRITE_KB,
RDC_FI_XGMI_5_WRITE_KB, RDC_FI_XGMI_6_WRITE_KB, RDC_FI_XGMI_7_WRITE_KB,
RDC_EVNT_XGMI_0_NOP_TX, RDC_EVNT_XGMI_0_REQ_TX, RDC_EVNT_XGMI_0_RESP_TX,
RDC_EVNT_XGMI_0_BEATS_TX, RDC_EVNT_XGMI_1_NOP_TX, RDC_EVNT_XGMI_1_REQ_TX,
RDC_EVNT_XGMI_1_RESP_TX, RDC_EVNT_XGMI_1_BEATS_TX, RDC_EVNT_XGMI_0_THRPUT,
RDC_EVNT_XGMI_1_THRPUT, RDC_EVNT_XGMI_2_THRPUT, RDC_EVNT_XGMI_3_THRPUT,
RDC_EVNT_XGMI_4_THRPUT, RDC_EVNT_XGMI_5_THRPUT,
};
std::copy(fields.begin(), fields.end(), field_ids);
+1 -1
Ver ficheiro
@@ -24,8 +24,8 @@ THE SOFTWARE.
#include <memory>
#include "rdc_lib/RdcException.h"
#include "rdc_lib/RdcLogger.h"
#include "rdc_lib/RdcMetricFetcher.h"
#include "rdc_lib/impl/RdcRasLib.h"
#include "rdc_lib/impl/RdcSmiLib.h"
namespace amd {