From 17430dde45aa7caba72986c4f2974ec10ab2b32a Mon Sep 17 00:00:00 2001 From: Chris Freehill Date: Sun, 16 Aug 2020 10:38:41 -0500 Subject: [PATCH] Add event counter support Adds support for RSMI event counters. This also includes "macro" or "pseudo" events, in which an event value is obtained from RSMI, followed by some post processing before being displayed in rdci. Aside from the support of new fields, the main update here is to introduce an initialization and "shutdown" call for new fields that will require this. Also, includes some modifications to the rdci dmon list command: * in rdc_field_data.data, added the ability to specify whether a field should be hidden or not, by default. This will allow us to support many fields, even those that are not typically of interest (but sometimes may be), without confusing the user or unnecessary clutter. * added a --list-all option which lists all available field including the more obscure fields. Change-Id: I01dd0edea963c12f82c6e44f893a390711ef3e83 [ROCm/rdc commit: d7c9625fc60a850217dd53328944188a0b357c58] --- projects/rdc/common/rdc_field_data.data | 54 ++-- projects/rdc/common/rdc_fields_supported.cc | 6 +- projects/rdc/common/rdc_fields_supported.h | 4 +- projects/rdc/common/rdc_utils.cc | 2 + projects/rdc/common/rdc_utils.h | 2 - projects/rdc/include/rdc/rdc.h | 50 +++- .../rdc/include/rdc_lib/RdcMetricFetcher.h | 3 + .../rdc_lib/impl/RdcMetricFetcherImpl.h | 25 ++ .../include/rdc_lib/impl/RdcWatchTableImpl.h | 6 +- .../rdc_libs/bootstrap/src/RdcBootStrap.cc | 10 + .../rdc_libs/rdc/src/RdcMetricFetcherImpl.cc | 251 +++++++++++++++++- .../rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc | 22 +- projects/rdc/rdci/include/RdciDmonSubSystem.h | 1 + projects/rdc/rdci/src/RdciDmonSubSystem.cc | 50 +++- 14 files changed, 444 insertions(+), 42 deletions(-) diff --git a/projects/rdc/common/rdc_field_data.data b/projects/rdc/common/rdc_field_data.data index 3b27d56b6d..a6f40d34ff 100644 --- a/projects/rdc/common/rdc_field_data.data +++ b/projects/rdc/common/rdc_field_data.data @@ -19,21 +19,41 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -FLD_DESC_ENT(RDC_FI_INVALID, "Unknown/Invalid field", "INVALID") -FLD_DESC_ENT(RDC_FI_GPU_COUNT, "GPU count in the system", "GPU_COUNT") -FLD_DESC_ENT(RDC_FI_DEV_NAME, "Name of the device", "DEV_NAME") -FLD_DESC_ENT(RDC_FI_GPU_CLOCK, "Current GPU clock frequencies.", "GPU_CLOCK") -FLD_DESC_ENT(RDC_FI_MEM_CLOCK, "Current Memory clock frequencies.", "MEM_CLOCK") -FLD_DESC_ENT(RDC_FI_MEMORY_TEMP, "Memory temperature in millidegrees Celsius.", "MEMORY_TEMP") -FLD_DESC_ENT(RDC_FI_GPU_TEMP, "GPU temperature in millidegrees Celsius.", "GPU_TEMP") -FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts.", "POWER_USAGE") -FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second.", "PCIE_TX") -FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second.", "PCIE_RX") -FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage.", "GPU_UTIL") -FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, - "Memory usage of the GPU instance in bytes.", "GPU_MEMORY_USAGE") -FLD_DESC_ENT(RDC_FI_GPU_MEMORY_TOTAL, "Total memory of the GPU instance", "GPU_MEMORY_TOTAL") -FLD_DESC_ENT(RDC_FI_ECC_CORRECT_TOTAL, "Accumulated correctable ECC errors.", "ECC_CORRECT") -FLD_DESC_ENT(RDC_FI_ECC_UNCORRECT_TOTAL, - "Accumulated uncorrectable ECC errors.", "ECC_UNCORRECT") + +// Description Fields: +// Arg # Type Meaning +// ------------------------------------------------- +// 1 rdc_field_t enum of field +// 2 string description of enum +// 3 string rdci display label +// 4 bool do or do not display in rdci +// rdc_field_t Description rdci label To Display +// =========== =========== ========= ========== +FLD_DESC_ENT(RDC_FI_INVALID, "Unknown/Invalid field", "INVALID", true) +FLD_DESC_ENT(RDC_FI_GPU_COUNT, "GPU count in the system", "GPU_COUNT", true) +FLD_DESC_ENT(RDC_FI_DEV_NAME, "Name of the device", "DEV_NAME", true) +FLD_DESC_ENT(RDC_FI_GPU_CLOCK, "Current GPU clock frequencies.", "GPU_CLOCK", true) +FLD_DESC_ENT(RDC_FI_MEM_CLOCK, "Current Memory clock frequencies.", "MEM_CLOCK", true) +FLD_DESC_ENT(RDC_FI_MEMORY_TEMP, "Memory temperature in millidegrees Celsius.", "MEMORY_TEMP", true) +FLD_DESC_ENT(RDC_FI_GPU_TEMP, "GPU temperature in millidegrees Celsius.", "GPU_TEMP", true) +FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts.", "POWER_USAGE", true) +FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second.", "PCIE_TX", true) +FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second.", "PCIE_RX", true) +FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage.", "GPU_UTIL", true) +FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, "Memory usage of the GPU instance in bytes.", "GPU_MEMORY_USAGE", true) +FLD_DESC_ENT(RDC_FI_GPU_MEMORY_TOTAL, "Total memory of the GPU instance", "GPU_MEMORY_TOTAL", true) +FLD_DESC_ENT(RDC_FI_ECC_CORRECT_TOTAL, "Accumulated correctable ECC errors.", "ECC_CORRECT", true) +FLD_DESC_ENT(RDC_FI_ECC_UNCORRECT_TOTAL, "Accumulated uncorrectable ECC errors.", "ECC_UNCORRECT", true) + +FLD_DESC_ENT(RDC_EVNT_XGMI_0_NOP_TX, "NOPs sent to neighbor 0", "XGMI_NOP_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_REQ_TX, "Outgoing requests to neighbor 0", "XGMI_REQ_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_RESP_TX, "Outgoing responses to neighbor 0", "XGMI_RES_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_BEATS_TX, "Data sent to neighbor 0 (32 byte pkts)", "XGMI_BTS_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_NOP_TX, "NOPs sent to neighbor 1", "XGMI_NOP_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_REQ_TX, "Outgoing requests to neighbor 1", "XGMI_REQ_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_RESP_TX, "Outgoing responses to neighbor 1", "XGMI_RES_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_BEATS_TX, "Data sent to neighbor 1 (32 byte pkts)", "XGMI_BTS_1", false) + +FLD_DESC_ENT(RDC_EVNT_XGMI_0_THRPUT, "Tx throughput to XGMI neighbor 0 in b/s", "XGMI_0_T", true) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_THRPUT, "Tx throughput to XGMI neighbor 1 in b/s", "XGMI_1_T", true) diff --git a/projects/rdc/common/rdc_fields_supported.cc b/projects/rdc/common/rdc_fields_supported.cc index 83c676d852..7d1ca7226d 100644 --- a/projects/rdc/common/rdc_fields_supported.cc +++ b/projects/rdc/common/rdc_fields_supported.cc @@ -28,14 +28,14 @@ THE SOFTWARE. namespace amd { namespace rdc { -#define FLD_DESC_ENT(ID, DESC, LABEL) \ - {static_cast(ID), {#ID, (DESC), (LABEL)}}, +#define FLD_DESC_ENT(ID, DESC, LABEL, DISPLAY) \ + {static_cast(ID), {#ID, (DESC), (LABEL), (DISPLAY)}}, static const fld_id2name_map_t field_id_to_descript = { #include "common/rdc_field_data.data" }; #undef FLD_DESC_ENT -#define FLD_DESC_ENT(ID, DESC, LABEL) {#ID, (ID)}, +#define FLD_DESC_ENT(ID, DESC, LABEL, DISPLAY) {#ID, (ID)}, static fld_name2id_map_t field_name_to_id = { #include "common/rdc_field_data.data" // NOLINT }; diff --git a/projects/rdc/common/rdc_fields_supported.h b/projects/rdc/common/rdc_fields_supported.h index 02c0ebbd09..90e5c31d86 100644 --- a/projects/rdc/common/rdc_fields_supported.h +++ b/projects/rdc/common/rdc_fields_supported.h @@ -23,6 +23,7 @@ THE SOFTWARE. #define COMMON_RDC_FIELDS_SUPPORTED_H_ #include +#include #include #include "rdc/rdc.h" @@ -34,9 +35,10 @@ typedef struct { std::string enum_name; std::string description; std::string label; + bool do_display; } field_id_descript; -typedef const std::unordered_map +typedef const std::map fld_id2name_map_t; typedef std::unordered_map fld_name2id_map_t; diff --git a/projects/rdc/common/rdc_utils.cc b/projects/rdc/common/rdc_utils.cc index c291d40355..a7433a1a9a 100755 --- a/projects/rdc/common/rdc_utils.cc +++ b/projects/rdc/common/rdc_utils.cc @@ -21,10 +21,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include +#include #include "common/rdc_utils.h" diff --git a/projects/rdc/common/rdc_utils.h b/projects/rdc/common/rdc_utils.h index 5c09d44c35..1deb05933b 100755 --- a/projects/rdc/common/rdc_utils.h +++ b/projects/rdc/common/rdc_utils.h @@ -24,8 +24,6 @@ THE SOFTWARE. #ifndef COMMON_RDC_UTILS_H_ #define COMMON_RDC_UTILS_H_ -#include - #include namespace amd { diff --git a/projects/rdc/include/rdc/rdc.h b/projects/rdc/include/rdc/rdc.h index c8d58103c9..debde6473b 100755 --- a/projects/rdc/include/rdc/rdc.h +++ b/projects/rdc/include/rdc/rdc.h @@ -57,7 +57,16 @@ typedef enum { RDC_ST_CONFLICT, //!< Conflict with current state RDC_ST_CLIENT_ERROR, //!< The RDC client error RDC_ST_ALREADY_EXIST, //!< The item already exists - RDC_ST_MAX_LIMIT //!< Max limit recording for the object + RDC_ST_MAX_LIMIT, //!< Max limit recording for the object + RDC_ST_INSUFF_RESOURCES, //!< Not enough resources to complete + //!< operation + RDC_ST_FILE_ERROR, //!< Failed to access a file + RDC_ST_NO_DATA, //!< Data was requested, + //!< but none was found + RDC_ST_PERM_ERROR, //!< Insufficient permission to complete + //!< operation + + RDC_ST_UNKNOWN_ERROR = 0xFFFFFFFF //!< Unknown error } rdc_status_t; /** @@ -167,6 +176,45 @@ typedef enum { */ RDC_FI_ECC_CORRECT_TOTAL = 600, //!< Accumulated correctable ECC errors RDC_FI_ECC_UNCORRECT_TOTAL, //!< Accumulated uncorrectable ECC errors + + /* + * @brief Raw XGMI counter events + */ + RDC_EVNT_XGMI_0_NOP_TX = 1000, //!< NOPs sent to neighbor 0 + RDC_EVNT_XGMI_0_REQ_TX, //!< Outgoing requests to + //!< neighbor 0 + RDC_EVNT_XGMI_0_RESP_TX, //!< Outgoing responses to + //!< neighbor 0 + /** + * @brief + * + * Data beats sent to neighbor 0; Each beat represents 32 bytes.

+ * + * XGMI throughput can be calculated by multiplying a BEATs event + * such as ::RSMI_EVNT_XGMI_0_BEATS_TX by 32 and dividing by + * the time for which event collection occurred, + * ::rsmi_counter_value_t.time_running (which is in nanoseconds). To get + * bytes per second, multiply this value by 109.
+ *
+ * Throughput = BEATS/time_running * 109 (bytes/second)
+ */ + // ie, Throughput = BEATS/time_running 10^9 bytes/sec + RDC_EVNT_XGMI_0_BEATS_TX, + RDC_EVNT_XGMI_1_NOP_TX, //!< NOPs sent to neighbor 1 + RDC_EVNT_XGMI_1_REQ_TX, //!< Outgoing requests to + //!< neighbor 1 + RDC_EVNT_XGMI_1_RESP_TX, //!< Outgoing responses to + //!< neighbor 1 + RDC_EVNT_XGMI_1_BEATS_TX, //!< Data beats sent to + //!< neighbor 1; Each beat + //!< represnts 32 bytes + + // "Composite" events. These events have additional processing beyond + // the value provided by the rocm_smi library. + RDC_EVNT_XGMI_0_THRPUT = 1500, //!< Transmit throughput to XGMI + //!< neighbor 0 in byes/sec + RDC_EVNT_XGMI_1_THRPUT, //!< Transmit throughput to XGMI + //!< neighbor 1 in byes/sec } rdc_field_t; /** diff --git a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h index 81ef756832..b7ba515893 100644 --- a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h +++ b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h @@ -32,6 +32,9 @@ namespace rdc { class RdcMetricFetcher { public: + virtual rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) = 0; + virtual rdc_status_t delete_rsmi_handle(RdcFieldKey fk) = 0; + virtual rdc_status_t fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) = 0; virtual ~RdcMetricFetcher() {} diff --git a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h index 4267acd845..4207f2fe62 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h @@ -29,6 +29,7 @@ THE SOFTWARE. #include #include "rdc_lib/RdcMetricFetcher.h" #include "rdc_lib/rdc_common.h" +#include "rocm_smi/rocm_smi.h" namespace amd { namespace rdc { @@ -41,6 +42,21 @@ struct MetricValue { rdc_field_value value; }; +// This union represents any RSMI handles require initialization and/or +// shut down. There should only be one instance of this for each raw event +// used. For example, if a field group includes a pseudo-event and the +// underlying raw event, then only one FieldRSMIData should be created, +// and it should be used by both events. +struct FieldRSMIData { + union { + rsmi_event_handle_t evt_handle; + }; + union { + rsmi_counter_value_t counter_val; + }; + ~FieldRSMIData(){} + FieldRSMIData() : evt_handle(0), counter_val{0, 0, 0}{} +}; //!< The data structure to store the async fetch task class RdcMetricFetcherImpl; @@ -55,7 +71,13 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { rdc_field_t field_id, rdc_field_value* value) override; RdcMetricFetcherImpl(); ~RdcMetricFetcherImpl(); + + rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) override; + rdc_status_t delete_rsmi_handle(RdcFieldKey fk) override; + private: + std::shared_ptr get_rsmi_data(RdcFieldKey key); + uint64_t now(); void get_ecc_error(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value); @@ -67,6 +89,7 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { //!< Async metric retreive std::map async_metrics_; + std::map> rsmi_data_; std::queue updated_tasks_; std::mutex task_mutex_; std::future updater_; // keep the future of updater @@ -74,6 +97,8 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { std::atomic task_started_; }; +rdc_status_t Rsmi2RdcError(rsmi_status_t rsmi); + } // namespace rdc } // namespace amd diff --git a/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h index 134cec02fd..4216e80968 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h @@ -29,10 +29,12 @@ THE SOFTWARE. #include #include // NOLINT #include +#include #include "rdc_lib/RdcWatchTable.h" #include "rdc_lib/RdcGroupSettings.h" #include "rdc_lib/RdcCacheManager.h" #include "rdc_lib/RdcMetricFetcher.h" +#include "rocm_smi/rocm_smi.h" namespace amd { namespace rdc { @@ -51,6 +53,7 @@ struct JobWatchTableEntry { std::vector fields; //< store fields for faster query }; + class RdcWatchTableImpl : public RdcWatchTable { public: rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id, @@ -103,6 +106,8 @@ class RdcWatchTableImpl : public RdcWatchTable { bool is_job_watch_field(uint32_t gpu_index, rdc_field_t field_id, std::string& job_id) const; // NOLINT + rdc_status_t initialize_rsmi_handles(RdcFieldKey fk); + RdcGroupSettingsPtr group_settings_; RdcCacheManagerPtr cache_mgr_; RdcMetricFetcherPtr metric_fetcher_; @@ -122,7 +127,6 @@ class RdcWatchTableImpl : public RdcWatchTable { //!< The last clean up time std::atomic last_cleanup_time_; - std::mutex watch_mutex_; }; diff --git a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc index d89ae39536..624c1ff9a9 100644 --- a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc +++ b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc @@ -346,6 +346,16 @@ const char* rdc_status_string(rdc_status_t result) { return "The value already exists"; case RDC_ST_CLIENT_ERROR: return "RDC Client error"; + case RDC_ST_INSUFF_RESOURCES: + return "Not enough resources to complete operation"; + case RDC_ST_FILE_ERROR: + return "Failed to access a file"; + case RDC_ST_NO_DATA: + return "Data was requested, but none was found"; + case RDC_ST_PERM_ERROR: + return "Insufficient permission to complete operation"; + case RDC_ST_UNKNOWN_ERROR: + return "Unknown error"; default: return "Unknown"; } diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index 99802fa310..7760a97fcc 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -22,6 +22,8 @@ THE SOFTWARE. #include "rdc_lib/impl/RdcMetricFetcherImpl.h" #include #include +#include + #include //NOLINT #include #include @@ -33,6 +35,25 @@ THE SOFTWARE. namespace amd { namespace rdc { +static const std::unordered_map + rdc_evnt_2_rsmi_field = { + {RDC_EVNT_XGMI_0_NOP_TX, RSMI_EVNT_XGMI_0_NOP_TX}, + {RDC_EVNT_XGMI_0_REQ_TX, RSMI_EVNT_XGMI_0_REQUEST_TX}, + {RDC_EVNT_XGMI_0_RESP_TX, RSMI_EVNT_XGMI_0_RESPONSE_TX}, + {RDC_EVNT_XGMI_0_BEATS_TX, RSMI_EVNT_XGMI_0_BEATS_TX}, + {RDC_EVNT_XGMI_1_NOP_TX, RSMI_EVNT_XGMI_1_NOP_TX}, + {RDC_EVNT_XGMI_1_REQ_TX, RSMI_EVNT_XGMI_1_REQUEST_TX}, + {RDC_EVNT_XGMI_1_RESP_TX, RSMI_EVNT_XGMI_1_RESPONSE_TX}, + {RDC_EVNT_XGMI_1_BEATS_TX, RSMI_EVNT_XGMI_1_BEATS_TX}, +}; + +// This maps pseudo-events to the raw events that they use. +static const std::unordered_map pseudo_evt_map = { + {RDC_EVNT_XGMI_0_THRPUT, RDC_EVNT_XGMI_0_BEATS_TX}, + {RDC_EVNT_XGMI_1_THRPUT, RDC_EVNT_XGMI_1_BEATS_TX}, +}; + + RdcMetricFetcherImpl::RdcMetricFetcherImpl() { task_started_ = true; @@ -205,6 +226,8 @@ void RdcMetricFetcherImpl::get_pcie_throughput(const RdcFieldKey& key) { } while (0); } +static const uint64_t kGig = 1000000000; + rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) { if (!value) { @@ -214,6 +237,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rsmi_temperature_type_t sensor_type; rsmi_clk_type_t clk_type; bool async_fetching = false; + RdcFieldKey f_key(gpu_index, field_id); + std::shared_ptr rsmi_data; + double coll_time_sec; if (!is_field_valid(field_id)) { RDC_LOG(RDC_ERROR, "Fail to fetch field " << field_id @@ -225,6 +251,15 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, value->field_id = field_id; value->status = RSMI_STATUS_NOT_SUPPORTED; + auto read_rsmi_counter = [&](void) { + assert(get_rsmi_data(f_key) != nullptr); + rsmi_data = get_rsmi_data(f_key); + value->status = rsmi_counter_read(rsmi_data->evt_handle, + &rsmi_data->counter_val); + value->value.l_int = rsmi_data->counter_val.value; + value->type = INTEGER; + }; + switch (field_id) { case RDC_FI_GPU_MEMORY_USAGE: value->status = rsmi_dev_memory_usage_get(gpu_index, @@ -309,6 +344,30 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, async_fetching = async_get_pcie_throughput( gpu_index, field_id, value); break; + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + read_rsmi_counter(); + break; + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: + read_rsmi_counter(); + if (value->status == RDC_ST_OK) { + if (rsmi_data->counter_val.time_running > 0) { + coll_time_sec = + static_cast(rsmi_data->counter_val.time_running)/kGig; + value->value.l_int = (value->value.l_int * 32)/coll_time_sec; + } else { + value->value.l_int = 0; + } + } + break; + default: break; } @@ -336,9 +395,199 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, << ", latency " << latency); } - return value->status == RSMI_STATUS_SUCCESS ? RDC_ST_OK : RDC_ST_MSI_ERROR; + return value->status == RSMI_STATUS_SUCCESS ? RDC_ST_OK : RDC_ST_MSI_ERROR; } +std::shared_ptr +RdcMetricFetcherImpl::get_rsmi_data(RdcFieldKey key) { + std::map>::iterator r_info = + rsmi_data_.find(key); + + if (r_info != rsmi_data_.end()) { + return r_info->second; + } + auto pseudo_key_field_id = pseudo_evt_map.find(key.second); + if (pseudo_key_field_id != pseudo_evt_map.end()) { + RdcFieldKey new_key; + + new_key.first = key.first; + new_key.second = pseudo_key_field_id->second; + + r_info = rsmi_data_.find(new_key); + if (r_info != rsmi_data_.end()) { + return r_info->second; + } + } + return nullptr; +} + +static rdc_status_t init_rsmi_counter(RdcFieldKey fk, + rsmi_event_group_t grp, rsmi_event_handle_t *handle) { + rsmi_status_t ret; + uint32_t counters_available; + uint32_t dv_ind = fk.first; + rdc_field_t f = fk.second; + + assert(handle != nullptr); + + ret = rsmi_dev_counter_group_supported(dv_ind, grp); + + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + + ret = rsmi_counter_available_counters_get(dv_ind, grp, &counters_available); + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + if (counters_available == 0) { + return RDC_ST_INSUFF_RESOURCES; + } + + auto raw_evt = pseudo_evt_map.find(f); + if (raw_evt != pseudo_evt_map.end()) { + f = raw_evt->second; + } + rsmi_event_type_t evt = rdc_evnt_2_rsmi_field.at(f); + ret = rsmi_dev_counter_create(dv_ind, evt, handle); + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + + ret = rsmi_counter_control(*handle, RSMI_CNTR_CMD_START, NULL); + return Rsmi2RdcError(ret); +} + +rdc_status_t RdcMetricFetcherImpl::delete_rsmi_handle(RdcFieldKey fk) { + rsmi_status_t ret; + + switch (fk.second) { + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: { + rsmi_event_handle_t h; + if (rsmi_data_.find(fk) == rsmi_data_.end()) { + return RDC_ST_NOT_SUPPORTED; + } + + h = rsmi_data_[fk]->evt_handle; + + // Stop counting. + ret = rsmi_counter_control(h, RSMI_CNTR_CMD_STOP, nullptr); + if (ret != RSMI_STATUS_SUCCESS) { + rsmi_data_.erase(fk); + return Rsmi2RdcError(ret); + } + + // Release all resources (e.g., counter and memory resources) associated + // with evnt_handle. + ret = rsmi_dev_counter_destroy(h); + + rsmi_data_.erase(fk); + return Rsmi2RdcError(ret); + } + default: + return RDC_ST_NOT_SUPPORTED; + } +} + +rdc_status_t RdcMetricFetcherImpl::acquire_rsmi_handle(RdcFieldKey fk) { + rdc_status_t result; + + switch (fk.second) { + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: { + rsmi_event_handle_t handle; + + if (get_rsmi_data(fk) != nullptr) { + // This event has already been initialized. + return RDC_ST_ALREADY_EXIST; + } + + result = init_rsmi_counter(fk, RSMI_EVNT_GRP_XGMI, &handle); + if (result != RDC_ST_OK) { + return result; + } + auto fsh = std::shared_ptr(new FieldRSMIData); + + if (fsh == nullptr) { + return RDC_ST_INSUFF_RESOURCES; + } + + fsh->evt_handle = handle; + + auto pseudo_key = pseudo_evt_map.find(fk.second); + + if (pseudo_key != pseudo_evt_map.end()) { + fk.second = pseudo_key->second; + } + rsmi_data_[fk] = fsh; + } + break; + + default: + break; + } + + return RDC_ST_OK; +} + +rdc_status_t Rsmi2RdcError(rsmi_status_t rsmi) { + switch (rsmi) { + case RSMI_STATUS_SUCCESS: + return RDC_ST_OK; + + case RSMI_STATUS_INVALID_ARGS: + return RDC_ST_BAD_PARAMETER; + + case RSMI_STATUS_NOT_SUPPORTED: + return RDC_ST_NOT_SUPPORTED; + + case RSMI_STATUS_NOT_FOUND: + return RDC_ST_NOT_FOUND; + + case RSMI_STATUS_OUT_OF_RESOURCES: + return RDC_ST_INSUFF_RESOURCES; + + case RSMI_STATUS_FILE_ERROR: + return RDC_ST_FILE_ERROR; + + case RSMI_STATUS_NO_DATA: + return RDC_ST_NO_DATA; + + case RSMI_STATUS_PERMISSION: + return RDC_ST_PERM_ERROR; + + case RSMI_STATUS_BUSY: + case RSMI_STATUS_UNKNOWN_ERROR: + case RSMI_STATUS_INTERNAL_EXCEPTION: + case RSMI_STATUS_INPUT_OUT_OF_BOUNDS: + case RSMI_STATUS_INIT_ERROR: + case RSMI_STATUS_NOT_YET_IMPLEMENTED: + case RSMI_STATUS_INSUFFICIENT_SIZE: + case RSMI_STATUS_INTERRUPT: + case RSMI_STATUS_UNEXPECTED_SIZE: + case RSMI_STATUS_UNEXPECTED_DATA: + case RSMI_STATUS_REFCOUNT_OVERFLOW: + default: + return RDC_ST_UNKNOWN_ERROR; + } +} } // namespace rdc } // namespace amd diff --git a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc index e70fbece9d..f019395101 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc @@ -20,13 +20,18 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "rdc_lib/impl/RdcWatchTableImpl.h" #include #include #include #include +#include +#include +#include "rdc_lib/impl/RdcWatchTableImpl.h" +#include "rdc_lib/impl/RdcMetricFetcherImpl.h" #include "rdc_lib/rdc_common.h" +#include "common/rdc_utils.h" #include "rdc_lib/RdcLogger.h" +#include "rdc/rdc.h" namespace amd { namespace rdc { @@ -171,7 +176,6 @@ rdc_status_t RdcWatchTableImpl::get_fields_from_group(rdc_gpu_group_t group_id, return RDC_ST_OK; } - rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id, uint64_t update_freq, double max_keep_age, uint32_t max_keep_samples) { @@ -207,6 +211,7 @@ rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, // Update the fields_to_watch_ auto f_in_watch_iter = fields_in_watch.begin(); + for (; f_in_watch_iter != fields_in_watch.end(); f_in_watch_iter++) { auto ite = fields_to_watch_.find(*f_in_watch_iter); if (ite == fields_to_watch_.end()) { // A new field @@ -225,6 +230,10 @@ rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, f_in_table.update_freq = update_freq; } } + result = metric_fetcher_->acquire_rsmi_handle(*f_in_watch_iter); + if (result != RDC_ST_OK && result != RDC_ST_ALREADY_EXIST) { + return result; + } } // Add to the watch table @@ -295,6 +304,15 @@ rdc_status_t RdcWatchTableImpl::update_field_in_table_when_unwatch( } } + fite = fields.begin(); + for (; fite != fields.end(); fite++) { + result = metric_fetcher_->delete_rsmi_handle(*fite); + + if (result != RDC_ST_OK && result != RDC_ST_NOT_SUPPORTED) { + return result; + } + } + return RDC_ST_OK; } diff --git a/projects/rdc/rdci/include/RdciDmonSubSystem.h b/projects/rdc/rdci/include/RdciDmonSubSystem.h index 55169cc666..d9d44b00f0 100644 --- a/projects/rdc/rdci/include/RdciDmonSubSystem.h +++ b/projects/rdc/rdci/include/RdciDmonSubSystem.h @@ -49,6 +49,7 @@ class RdciDmonSubSystem: public RdciSubSystem { DMON_UNKNOWN = 0, DMON_HELP, DMON_LIST_FIELDS, + DMON_LIST_ALL_FIELDS, DMON_MONITOR } dmon_ops_; diff --git a/projects/rdc/rdci/src/RdciDmonSubSystem.cc b/projects/rdc/rdci/src/RdciDmonSubSystem.cc index f4fe1fd394..47777e9f74 100644 --- a/projects/rdc/rdci/src/RdciDmonSubSystem.cc +++ b/projects/rdc/rdci/src/RdciDmonSubSystem.cc @@ -60,15 +60,17 @@ void RdciDmonSubSystem::set_terminating(int sig) { void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { const int HOST_OPTIONS = 1000; + const int LIST_ALL_FIELDS_OPT = 1001; const struct option long_options[] = { - {"host", required_argument, nullptr, HOST_OPTIONS }, - {"help", optional_argument, nullptr, 'h' }, - {"unauth", optional_argument, nullptr, 'u' }, - {"list", optional_argument, nullptr, 'l' }, - {"field-group-id", required_argument, nullptr, 'f' }, + {"host", required_argument, nullptr, HOST_OPTIONS}, + {"help", optional_argument, nullptr, 'h'}, + {"unauth", optional_argument, nullptr, 'u'}, + {"list", optional_argument, nullptr, 'l'}, + {"list-all", optional_argument, nullptr, LIST_ALL_FIELDS_OPT}, + {"field-group-id", required_argument, nullptr, 'f'}, {"field-id", required_argument, nullptr, 'e' }, {"gpu_index", required_argument, nullptr, 'i'}, - {"group-id", required_argument, nullptr, 'g' }, + {"group-id", required_argument, nullptr, 'g'}, {"count", required_argument, nullptr, 'c'}, {"delay", required_argument, nullptr, 'd'}, { nullptr, 0 , nullptr, 0 } @@ -132,6 +134,9 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } options_.insert({OPTIONS_DELAY, std::stoi(optarg)}); break; + case LIST_ALL_FIELDS_OPT: + dmon_ops_ = DMON_LIST_ALL_FIELDS; + break; default: show_help(); throw RdcException(RDC_ST_BAD_PARAMETER, @@ -139,7 +144,7 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } } - if (dmon_ops_ == DMON_LIST_FIELDS) { + if (dmon_ops_ == DMON_LIST_FIELDS || dmon_ops_ == DMON_LIST_ALL_FIELDS) { return; } @@ -212,6 +217,11 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } void RdciDmonSubSystem::show_help() const { + // Try to keep total output line length to <= 80 chars for better + // readability. For reference: + // *********************** 60 Chars ************************** + // ************** 40 Chars *************** + // ***** 20 Chars **** std::cout << " dmon -- Used to monitor GPUs and their stats.\n\n"; std::cout << "Usage\n"; std::cout << " rdci dmon [--host :port] [-u] -f " @@ -228,16 +238,26 @@ void RdciDmonSubSystem::show_help() const { std::cout << " -g --group-id The GPU group to query " << "on the specified host.\n"; std::cout << " -c --count count Integer representing How" - << " many times to loop before exiting. [default = runs forever.]\n"; + << " many times to loop\n" + << " before exiting. [default " + << "= runs forever.]\n"; std::cout << " -e --field-id fieldIds Comma-separated list " - << "of the field ids to monitor.\n"; + << "of field ids to monitor.\n"; std::cout << " -i --gpu_index gpuIndexes Comma-separated list " - << "of the GPU index to monitor.\n"; + << "of GPU indexes to monitor.\n"; std::cout << " -d --delay delay How often to query RDC " - << "in milli seconds. [default = 1000 msec, " + << "in milli seconds. \n" + << " [default = 1000 msec, " << "Minimum value = 100 msec.]\n"; std::cout << " -l --list List to look up the long " - << "names and descriptions of the field ids\n"; + << "names and \n" + << " descriptions of the field " + << "ids\n"; + std::cout << " --list-all Same as -l, except this " + << "lists all possible\n" + << " fields, including " + << "those that are less \n" + << " commonly used.\n"; } void RdciDmonSubSystem::create_temp_group() { @@ -295,8 +315,10 @@ void RdciDmonSubSystem::show_field_usage() const { amd::rdc::get_field_id_description_from_id(); for (auto i = field_id_to_descript.begin(); i != field_id_to_descript.end(); i++) { - std::cout << i->first << " " << i->second.enum_name << " : " << + if (i->second.do_display || dmon_ops_ == DMON_LIST_ALL_FIELDS) { + std::cout << i->first << " " << i->second.enum_name << " : " << i->second.description << "." << std::endl; + } } std::cout << std::endl; std::cout << "* Note: The field ID number associated with a field ID can " @@ -312,7 +334,7 @@ void RdciDmonSubSystem::process() { return; } - if (dmon_ops_ == DMON_LIST_FIELDS) { + if (dmon_ops_ == DMON_LIST_FIELDS || dmon_ops_ == DMON_LIST_ALL_FIELDS) { show_field_usage(); return; }