Add event counter support
Adds support for RSMI event counters. This also includes "macro" or "pseudo" events, in which an event value is obtained from RSMI, followed by some post processing before being displayed in rdci. Aside from the support of new fields, the main update here is to introduce an initialization and "shutdown" call for new fields that will require this. Also, includes some modifications to the rdci dmon list command: * in rdc_field_data.data, added the ability to specify whether a field should be hidden or not, by default. This will allow us to support many fields, even those that are not typically of interest (but sometimes may be), without confusing the user or unnecessary clutter. * added a --list-all option which lists all available field including the more obscure fields. Change-Id: I01dd0edea963c12f82c6e44f893a390711ef3e83
This commit is contained in:
Chris Freehill
+49
-1
@@ -57,7 +57,16 @@ typedef enum {
|
||||
RDC_ST_CONFLICT, //!< Conflict with current state
|
||||
RDC_ST_CLIENT_ERROR, //!< The RDC client error
|
||||
RDC_ST_ALREADY_EXIST, //!< The item already exists
|
||||
RDC_ST_MAX_LIMIT //!< Max limit recording for the object
|
||||
RDC_ST_MAX_LIMIT, //!< Max limit recording for the object
|
||||
RDC_ST_INSUFF_RESOURCES, //!< Not enough resources to complete
|
||||
//!< operation
|
||||
RDC_ST_FILE_ERROR, //!< Failed to access a file
|
||||
RDC_ST_NO_DATA, //!< Data was requested,
|
||||
//!< but none was found
|
||||
RDC_ST_PERM_ERROR, //!< Insufficient permission to complete
|
||||
//!< operation
|
||||
|
||||
RDC_ST_UNKNOWN_ERROR = 0xFFFFFFFF //!< Unknown error
|
||||
} rdc_status_t;
|
||||
|
||||
/**
|
||||
@@ -167,6 +176,45 @@ typedef enum {
|
||||
*/
|
||||
RDC_FI_ECC_CORRECT_TOTAL = 600, //!< Accumulated correctable ECC errors
|
||||
RDC_FI_ECC_UNCORRECT_TOTAL, //!< Accumulated uncorrectable ECC errors
|
||||
|
||||
/*
|
||||
* @brief Raw XGMI counter events
|
||||
*/
|
||||
RDC_EVNT_XGMI_0_NOP_TX = 1000, //!< NOPs sent to neighbor 0
|
||||
RDC_EVNT_XGMI_0_REQ_TX, //!< Outgoing requests to
|
||||
//!< neighbor 0
|
||||
RDC_EVNT_XGMI_0_RESP_TX, //!< Outgoing responses to
|
||||
//!< neighbor 0
|
||||
/**
|
||||
* @brief
|
||||
*
|
||||
* Data beats sent to neighbor 0; Each beat represents 32 bytes.<br><br>
|
||||
*
|
||||
* XGMI throughput can be calculated by multiplying a BEATs event
|
||||
* such as ::RSMI_EVNT_XGMI_0_BEATS_TX by 32 and dividing by
|
||||
* the time for which event collection occurred,
|
||||
* ::rsmi_counter_value_t.time_running (which is in nanoseconds). To get
|
||||
* bytes per second, multiply this value by 10<sup>9</sup>.<br>
|
||||
* <br>
|
||||
* Throughput = BEATS/time_running * 10<sup>9</sup> (bytes/second)<br>
|
||||
*/
|
||||
// ie, Throughput = BEATS/time_running 10^9 bytes/sec
|
||||
RDC_EVNT_XGMI_0_BEATS_TX,
|
||||
RDC_EVNT_XGMI_1_NOP_TX, //!< NOPs sent to neighbor 1
|
||||
RDC_EVNT_XGMI_1_REQ_TX, //!< Outgoing requests to
|
||||
//!< neighbor 1
|
||||
RDC_EVNT_XGMI_1_RESP_TX, //!< Outgoing responses to
|
||||
//!< neighbor 1
|
||||
RDC_EVNT_XGMI_1_BEATS_TX, //!< Data beats sent to
|
||||
//!< neighbor 1; Each beat
|
||||
//!< represnts 32 bytes
|
||||
|
||||
// "Composite" events. These events have additional processing beyond
|
||||
// the value provided by the rocm_smi library.
|
||||
RDC_EVNT_XGMI_0_THRPUT = 1500, //!< Transmit throughput to XGMI
|
||||
//!< neighbor 0 in byes/sec
|
||||
RDC_EVNT_XGMI_1_THRPUT, //!< Transmit throughput to XGMI
|
||||
//!< neighbor 1 in byes/sec
|
||||
} rdc_field_t;
|
||||
|
||||
/**
|
||||
|
||||
@@ -32,6 +32,9 @@ namespace rdc {
|
||||
|
||||
class RdcMetricFetcher {
|
||||
public:
|
||||
virtual rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) = 0;
|
||||
virtual rdc_status_t delete_rsmi_handle(RdcFieldKey fk) = 0;
|
||||
|
||||
virtual rdc_status_t fetch_smi_field(uint32_t gpu_index,
|
||||
rdc_field_t field_id, rdc_field_value* value) = 0;
|
||||
virtual ~RdcMetricFetcher() {}
|
||||
|
||||
@@ -29,6 +29,7 @@ THE SOFTWARE.
|
||||
#include <queue>
|
||||
#include "rdc_lib/RdcMetricFetcher.h"
|
||||
#include "rdc_lib/rdc_common.h"
|
||||
#include "rocm_smi/rocm_smi.h"
|
||||
|
||||
namespace amd {
|
||||
namespace rdc {
|
||||
@@ -41,6 +42,21 @@ struct MetricValue {
|
||||
rdc_field_value value;
|
||||
};
|
||||
|
||||
// This union represents any RSMI handles require initialization and/or
|
||||
// shut down. There should only be one instance of this for each raw event
|
||||
// used. For example, if a field group includes a pseudo-event and the
|
||||
// underlying raw event, then only one FieldRSMIData should be created,
|
||||
// and it should be used by both events.
|
||||
struct FieldRSMIData {
|
||||
union {
|
||||
rsmi_event_handle_t evt_handle;
|
||||
};
|
||||
union {
|
||||
rsmi_counter_value_t counter_val;
|
||||
};
|
||||
~FieldRSMIData(){}
|
||||
FieldRSMIData() : evt_handle(0), counter_val{0, 0, 0}{}
|
||||
};
|
||||
|
||||
//!< The data structure to store the async fetch task
|
||||
class RdcMetricFetcherImpl;
|
||||
@@ -55,7 +71,13 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher {
|
||||
rdc_field_t field_id, rdc_field_value* value) override;
|
||||
RdcMetricFetcherImpl();
|
||||
~RdcMetricFetcherImpl();
|
||||
|
||||
rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) override;
|
||||
rdc_status_t delete_rsmi_handle(RdcFieldKey fk) override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<FieldRSMIData> get_rsmi_data(RdcFieldKey key);
|
||||
|
||||
uint64_t now();
|
||||
void get_ecc_error(uint32_t gpu_index,
|
||||
rdc_field_t field_id, rdc_field_value* value);
|
||||
@@ -67,6 +89,7 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher {
|
||||
|
||||
//!< Async metric retreive
|
||||
std::map<RdcFieldKey, MetricValue> async_metrics_;
|
||||
std::map<RdcFieldKey, std::shared_ptr<FieldRSMIData>> rsmi_data_;
|
||||
std::queue<MetricTask> updated_tasks_;
|
||||
std::mutex task_mutex_;
|
||||
std::future<void> updater_; // keep the future of updater
|
||||
@@ -74,6 +97,8 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher {
|
||||
std::atomic<bool> task_started_;
|
||||
};
|
||||
|
||||
rdc_status_t Rsmi2RdcError(rsmi_status_t rsmi);
|
||||
|
||||
} // namespace rdc
|
||||
} // namespace amd
|
||||
|
||||
|
||||
@@ -29,10 +29,12 @@ THE SOFTWARE.
|
||||
#include <memory>
|
||||
#include <mutex> // NOLINT
|
||||
#include <atomic>
|
||||
#include <map>
|
||||
#include "rdc_lib/RdcWatchTable.h"
|
||||
#include "rdc_lib/RdcGroupSettings.h"
|
||||
#include "rdc_lib/RdcCacheManager.h"
|
||||
#include "rdc_lib/RdcMetricFetcher.h"
|
||||
#include "rocm_smi/rocm_smi.h"
|
||||
|
||||
namespace amd {
|
||||
namespace rdc {
|
||||
@@ -51,6 +53,7 @@ struct JobWatchTableEntry {
|
||||
std::vector<RdcFieldKey> fields; //< store fields for faster query
|
||||
};
|
||||
|
||||
|
||||
class RdcWatchTableImpl : public RdcWatchTable {
|
||||
public:
|
||||
rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
|
||||
@@ -103,6 +106,8 @@ class RdcWatchTableImpl : public RdcWatchTable {
|
||||
bool is_job_watch_field(uint32_t gpu_index, rdc_field_t field_id,
|
||||
std::string& job_id) const; // NOLINT
|
||||
|
||||
rdc_status_t initialize_rsmi_handles(RdcFieldKey fk);
|
||||
|
||||
RdcGroupSettingsPtr group_settings_;
|
||||
RdcCacheManagerPtr cache_mgr_;
|
||||
RdcMetricFetcherPtr metric_fetcher_;
|
||||
@@ -122,7 +127,6 @@ class RdcWatchTableImpl : public RdcWatchTable {
|
||||
|
||||
//!< The last clean up time
|
||||
std::atomic<uint64_t> last_cleanup_time_;
|
||||
|
||||
std::mutex watch_mutex_;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user