AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Backgroud health check

Browse Source 
Add the RdcSmiHealth module, which will call rocm_smi_lib.
It will support following health:
 - XGMI error detected
 - PCIE replay count detected
 - Memory check
 - InfoROM check
 - Power/Thermal check
The grpc client and server side health function is added.
The health module is added to the rdci.

At present, XGMI/PCIE and a part of Memory have been implemented.
Others will be added as soon as possible.

Change-Id: I1bd99290bdc7dea733f21a41a8c4bcefb2138112
This commit is contained in:
limeng12
2024-10-23 16:42:24 +08:00
committed by Meng, Li (Jassmine)
parent f1428a8226
commit 853d3b0cc5
26 changed files with 2260 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/rdc/rdc.h
+153
View File
@@ -335,6 +335,18 @@ typedef enum {
RDC_EVNT_NOTIF_RING_HANG, //!< GPU ring hang just occurred
RDC_EVNT_NOTIF_LAST = RDC_EVNT_NOTIF_RING_HANG,
/**
* @brief RDC health related fields
*/
RDC_HEALTH_XGMI_ERROR = 3000, //!< XGMI one or more errors detected
RDC_HEALTH_PCIE_REPLAY_COUNT, //!< Total PCIE replay count
RDC_HEALTH_RETIRED_PAGE_NUM, //!< Retired page number
RDC_HEALTH_PENDING_PAGE_NUM, //!< Pending page number
RDC_HEALTH_RETIRED_PAGE_LIMIT, //!< The threshold of retired page
RDC_HEALTH_UNCORRECTABLE_PAGE_LIMIT,//!< The threshold of uncorrectable page
RDC_HEALTH_POWER_THROTTLE_TIME, //!< Power throttle status counter
RDC_HEALTH_THERMAL_THROTTLE_TIME, //!< Total time in thermal throttle status (microseconds)
} rdc_field_t;
// even and odd numbers are used for correctable and uncorrectable errors
@@ -589,6 +601,81 @@ typedef struct {
rdc_policy_action_t action; //!< Action to take
} rdc_policy_t;
/**
* @brief type of health watches
*/
typedef enum {
RDC_HEALTH_WATCH_PCIE = 0x1, //!< PCIe system watches
RDC_HEALTH_WATCH_XGMI = 0x2, //!< XGMI system watches
RDC_HEALTH_WATCH_MEM = 0x4, //!< Memory watches
RDC_HEALTH_WATCH_INFOROM = 0x8, //!< Inforom watches
RDC_HEALTH_WATCH_THERMAL = 0x10, //!< Temperature watches
RDC_HEALTH_WATCH_POWER = 0x20, //!< Power watches
} rdc_health_system_t;
/**
* @brief type of health result
*/
typedef enum {
RDC_HEALTH_RESULT_PASS, //!< The health test pass
RDC_HEALTH_RESULT_WARN, //!< The health test has warnings
RDC_HEALTH_RESULT_FAIL //!< The health test fail
} rdc_health_result_t;
/**
* @brief The maximum length of the health messages
*/
#define MAX_HEALTH_MSG_LENGTH 4096
/**
* 8 replays per minute is the maximum recommended
*/
#define PCIE_MAX_REPLAYS_PERMIN 8
// The error code set at rdc_health_incidents_t.error.code
typedef enum {
RDC_FR_PCI_REPLAY_RATE = 1000,
RDC_FR_ECC_UNCORRECTABLE_DETECTED = 1001,
RDC_FR_PENDING_PAGE_RETIREMENTS = 1002,
RDC_FR_RETIRED_PAGES_LIMIT = 1003,
RDC_FR_RETIRED_PAGES_UNCORRECTABLE_LIMIT = 1004,
RDC_FR_CLOCKS_THROTTLE_THERMAL = 1005,
RDC_FR_CLOCKS_THROTTLE_POWER = 1006,
RDC_FR_XGMI_SINGLE_ERROR = 1007,
RDC_FR_XGMI_MULTIPLE_ERROR = 1008,
RDC_FR_CORRUPT_INFOROM = 1009
} rdc_health_error_code_t;
/**
* @brief details of the health errors
*/
typedef struct {
char msg[MAX_HEALTH_MSG_LENGTH]; //!< The test result details
uint32_t code; //!< The low level error code
} rdc_health_detail_t;
/**
* @brief details of the per health incidents
*/
typedef struct {
uint32_t gpu_index; //!< which GPU in this group have the issue
rdc_health_system_t component; //!< which components have the issue
rdc_health_result_t health; //!< health diagnosis of this incident
rdc_health_detail_t error; //!< The details of the error, rdc_health_error_code_t
} rdc_health_incidents_t;
#define HEALTH_MAX_ERROR_ITEMS 64
/**
* @brief The health responses for test cases
*/
typedef struct {
rdc_health_result_t overall_health; //!< The overall health of this entire host
unsigned int incidents_count; //!< The number of health incidents reported in this struct
rdc_health_incidents_t incidents[HEALTH_MAX_ERROR_ITEMS]; //!< Report of the errors detected
} rdc_health_response_t;
/**
* @brief Initialize ROCm RDC.
*
@@ -1274,6 +1361,72 @@ rdc_status_t rdc_policy_register(rdc_handle_t p_rdc_handle, rdc_gpu_group_t grou
*/
rdc_status_t rdc_policy_unregister(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id);
/**
* @brief enable the health check for a group
*
* @details For each group, only one parameter can be set. If you want to
* clear the setting for a group, set component == 0x0
*
* @param[in] p_rdc_handle The RDC handler.
*
* @param[in] group_id The GPU group id.
*
* @param[in] components The list of components that should be enabled for health check
* for example, RDC_HEALTH_WATCH_THERMAL | RDC_HEALTH_WATCH_POWER
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_health_set(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
unsigned int components);
/**
* @brief get the health check settings of a group
*
* @details get the health check settings of a component
*
* @param[in] p_rdc_handle The RDC handler.
*
* @param[in] group_id The GPU group id.
*
* @param[out] components The list of components that should be enabled for health check
* for example, RDC_HEALTH_WATCH_THERMAL | RDC_HEALTH_WATCH_POWER
* if it is 0x0, then the health check not set for the group yet.
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_health_get(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
unsigned int* components);
/**
* @brief Check health watch results
*
* @details If it has incidents.
* For each incident, check the component and error message.
*
* @param[in] p_rdc_handle The RDC handler.
*
* @param[in] group_id The GPU group id.
*
* @param[inout] response The detail results of the health.
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_health_check(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
rdc_health_response_t* response);
/**
* @brief clear the health watch
*
* @details For each group, clear the setting.
*
* @param[in] p_rdc_handle The RDC handler.
*
* @param[in] group_id The GPU group id.
*
* @retval ::RDC_ST_OK is returned upon successful call.
*/
rdc_status_t rdc_health_clear(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id);
#ifdef __cplusplus
}
#endif // __cplusplus
include/rdc_lib/RdcCacheManager.h
+15
View File
@@ -59,6 +59,21 @@ class RdcCacheManager {
virtual rdc_status_t rdc_job_remove(const char job_id[64]) = 0;
virtual rdc_status_t rdc_job_remove_all() = 0;
virtual rdc_status_t rdc_health_set(rdc_gpu_group_t group_id,
uint32_t gpu_index,
const rdc_field_value& value) = 0;
virtual rdc_status_t rdc_health_get_values(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_field_t field_id,
uint64_t start_timestamp,
uint64_t end_timestamp,
rdc_field_value* start_value,
rdc_field_value* end_value) = 0;
virtual rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) = 0;
virtual rdc_status_t rdc_update_health_stats(rdc_gpu_group_t group_id,
uint32_t gpu_index,
const rdc_field_value& value) = 0;
virtual ~RdcCacheManager() {}
};
include/rdc_lib/RdcHandler.h
+6
View File
@@ -106,6 +106,12 @@ class RdcHandler {
virtual rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) = 0;
// Health API
virtual rdc_status_t rdc_health_set(rdc_gpu_group_t group_id, unsigned int components) = 0;
virtual rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) = 0;
virtual rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) = 0;
virtual rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) = 0;
virtual ~RdcHandler() {}
};
include/rdc_lib/RdcWatchTable.h
+8
View File
@@ -50,6 +50,14 @@ class RdcWatchTable {
virtual rdc_status_t rdc_field_unwatch(rdc_gpu_group_t group_id,
rdc_field_grp_t field_group_id) = 0;
virtual rdc_status_t rdc_health_set(rdc_gpu_group_t group_id,
unsigned int components) = 0;
virtual rdc_status_t rdc_health_get(rdc_gpu_group_t group_id,
unsigned int* components) = 0;
virtual rdc_status_t rdc_health_check(rdc_gpu_group_t group_id,
rdc_health_response_t *response) = 0;
virtual rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) = 0;
virtual ~RdcWatchTable() {}
};
include/rdc_lib/impl/RdcCacheManagerImpl.h
+21
View File
@@ -32,6 +32,8 @@ THE SOFTWARE.
#include "rdc_lib/RdcCacheManager.h"
#include "rdc_lib/rdc_common.h"
#define HEALTH_MAX_KEEP_SAMPLES 300
namespace amd {
namespace rdc {
@@ -81,6 +83,9 @@ struct RdcJobStatsCacheEntry {
// <job_id, job_stats>
typedef std::map<std::string, RdcJobStatsCacheEntry> RdcJobStatsCache;
// <group_id, health_samples>
typedef std::map<rdc_gpu_group_t, RdcCacheSamples> RdcHealthStatsCache;
class RdcCacheManagerImpl : public RdcCacheManager {
public:
rdc_status_t rdc_field_get_latest_value(uint32_t gpu_index, rdc_field_t field,
@@ -105,6 +110,21 @@ class RdcCacheManagerImpl : public RdcCacheManager {
rdc_status_t rdc_job_remove(const char job_id[64]) override;
rdc_status_t rdc_job_remove_all() override;
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id,
uint32_t gpu_index,
const rdc_field_value& value) override;
rdc_status_t rdc_health_get_values(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_field_t field_id,
uint64_t start_timestamp,
uint64_t end_timestamp,
rdc_field_value* start_value,
rdc_field_value* end_value) override;
rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
rdc_status_t rdc_update_health_stats(rdc_gpu_group_t group_id,
uint32_t gpu_index,
const rdc_field_value& value) override;
private:
void set_summary(const FieldSummaryStats& stats, rdc_stats_summary_t& gpu,
rdc_stats_summary_t& summary, // NOLINT
@@ -113,6 +133,7 @@ class RdcCacheManagerImpl : public RdcCacheManager {
uint32_t num_gpus); // NOLINT
RdcCacheSamples cache_samples_;
RdcJobStatsCache cache_jobs_;
RdcHealthStatsCache cache_health_;
std::mutex cache_mutex_;
};
include/rdc_lib/impl/RdcEmbeddedHandler.h
+6
View File
@@ -108,6 +108,12 @@ class RdcEmbeddedHandler final : public RdcHandler {
rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) override;
// Health API
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id, unsigned int components) override;
rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) override;
rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) override;
rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
explicit RdcEmbeddedHandler(rdc_operation_mode_t op_mode);
~RdcEmbeddedHandler() final;
include/rdc_lib/impl/RdcStandaloneHandler.h
+6
View File
@@ -105,6 +105,12 @@ class RdcStandaloneHandler : public RdcHandler {
rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) override;
// Health API
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id, unsigned int components) override;
rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) override;
rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) override;
rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
explicit RdcStandaloneHandler(const char* ip_and_port, const char* root_ca,
const char* client_cert, const char* client_key);
include/rdc_lib/impl/RdcWatchTableImpl.h
+42 -1
View File
@@ -55,6 +55,12 @@ struct JobWatchTableEntry {
std::vector<RdcFieldKey> fields; //< store fields for faster query
};
struct HealthWatchTableEntry {
unsigned int components;
rdc_field_grp_t field_group_id;
std::vector<RdcFieldKey> fields; //< store fields for faster query
};
class RdcWatchTableImpl : public RdcWatchTable {
public:
rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id, const char job_id[64],
@@ -74,6 +80,11 @@ class RdcWatchTableImpl : public RdcWatchTable {
//!< is reached, which will be handled in the clean_up() function.
rdc_status_t rdc_field_unwatch(rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id) override;
rdc_status_t rdc_health_set(rdc_gpu_group_t group_id, unsigned int components) override;
rdc_status_t rdc_health_get(rdc_gpu_group_t group_id, unsigned int* components) override;
rdc_status_t rdc_health_check(rdc_gpu_group_t group_id, rdc_health_response_t *response) override;
rdc_status_t rdc_health_clear(rdc_gpu_group_t group_id) override;
//!< When the RDC is running as RDC_OPERATION_MODE_MANUAL, the user will
//!< call this function periodically. Instead of providing other APIs to
//!< cleanup the cache, this function will update and cleanup the cache.
@@ -85,7 +96,8 @@ class RdcWatchTableImpl : public RdcWatchTable {
rdc_status_t rdc_field_listen_notif(uint32_t timeout_ms) override;
RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings, const RdcCacheManagerPtr& cache_mgr,
const RdcModuleMgrPtr& module_mgr, const RdcNotificationPtr& notif);
const RdcMetricFetcherPtr& metric_fetcher, const RdcModuleMgrPtr& module_mgr,
const RdcNotificationPtr& notif);
private:
//!< Helper function to Update the fields_in_table when unwatch tables
@@ -104,13 +116,39 @@ class RdcWatchTableImpl : public RdcWatchTable {
bool is_job_watch_field(uint32_t gpu_index, rdc_field_t field_id,
std::string& job_id) const; // NOLINT
bool is_health_watch_field(uint32_t gpu_index, rdc_field_t field_id,
rdc_gpu_group_t& group_id) const;
rdc_status_t rdc_notif_update_cache(rdc_evnt_notification_t* events, uint32_t num_events);
//!< The function will be pass as the callback for bulk fetch
static rdc_status_t handle_fields(rdc_gpu_field_value_t* values, uint32_t num_values,
void* user_data);
rdc_status_t create_health_field_group(unsigned int components,
rdc_field_grp_t* field_group_id);
//!< output: Whether health incidents are full
bool add_health_incident(uint32_t gpu_index,
rdc_health_system_t component,
rdc_health_result_t health,
uint32_t err_code,
std::string err_msg,
rdc_health_incidents_t* incident,
rdc_health_response_t* response);
rdc_status_t get_start_end_values(rdc_gpu_group_t group_id,
uint32_t gpu_index,
rdc_field_t field,
rdc_field_value *start_value,
rdc_field_value *end_value);
rdc_status_t pcie_check(rdc_gpu_group_t group_id,
uint32_t gpu_index, rdc_health_response_t* response);
rdc_status_t xgmi_check(rdc_gpu_group_t group_id,
uint32_t gpu_index, rdc_health_response_t* response);
rdc_status_t memory_check(rdc_gpu_group_t group_id,
uint32_t gpu_index, rdc_health_response_t* response);
RdcGroupSettingsPtr group_settings_;
RdcCacheManagerPtr cache_mgr_;
RdcMetricFetcherPtr metric_fetcher_;
RdcModuleMgrPtr rdc_module_mgr_;
RdcNotificationPtr notifications_;
@@ -126,6 +164,9 @@ class RdcWatchTableImpl : public RdcWatchTable {
//!< Those settings will only be updated when watching or unwatching.
std::map<RdcFieldKey, FieldSettings> fields_to_watch_;
//!< The health watch table to store the health settings.
std::map<uint32_t, HealthWatchTableEntry> health_watch_table_;
//!< The last clean up time
std::atomic<uint64_t> last_cleanup_time_;
std::mutex watch_mutex_;