diff --git a/projects/rdc/common/rdc_field_data.data b/projects/rdc/common/rdc_field_data.data index 3b27d56b6d..a6f40d34ff 100644 --- a/projects/rdc/common/rdc_field_data.data +++ b/projects/rdc/common/rdc_field_data.data @@ -19,21 +19,41 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -FLD_DESC_ENT(RDC_FI_INVALID, "Unknown/Invalid field", "INVALID") -FLD_DESC_ENT(RDC_FI_GPU_COUNT, "GPU count in the system", "GPU_COUNT") -FLD_DESC_ENT(RDC_FI_DEV_NAME, "Name of the device", "DEV_NAME") -FLD_DESC_ENT(RDC_FI_GPU_CLOCK, "Current GPU clock frequencies.", "GPU_CLOCK") -FLD_DESC_ENT(RDC_FI_MEM_CLOCK, "Current Memory clock frequencies.", "MEM_CLOCK") -FLD_DESC_ENT(RDC_FI_MEMORY_TEMP, "Memory temperature in millidegrees Celsius.", "MEMORY_TEMP") -FLD_DESC_ENT(RDC_FI_GPU_TEMP, "GPU temperature in millidegrees Celsius.", "GPU_TEMP") -FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts.", "POWER_USAGE") -FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second.", "PCIE_TX") -FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second.", "PCIE_RX") -FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage.", "GPU_UTIL") -FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, - "Memory usage of the GPU instance in bytes.", "GPU_MEMORY_USAGE") -FLD_DESC_ENT(RDC_FI_GPU_MEMORY_TOTAL, "Total memory of the GPU instance", "GPU_MEMORY_TOTAL") -FLD_DESC_ENT(RDC_FI_ECC_CORRECT_TOTAL, "Accumulated correctable ECC errors.", "ECC_CORRECT") -FLD_DESC_ENT(RDC_FI_ECC_UNCORRECT_TOTAL, - "Accumulated uncorrectable ECC errors.", "ECC_UNCORRECT") + +// Description Fields: +// Arg # Type Meaning +// ------------------------------------------------- +// 1 rdc_field_t enum of field +// 2 string description of enum +// 3 string rdci display label +// 4 bool do or do not display in rdci +// rdc_field_t Description rdci label To Display +// =========== =========== ========= ========== +FLD_DESC_ENT(RDC_FI_INVALID, "Unknown/Invalid field", "INVALID", true) +FLD_DESC_ENT(RDC_FI_GPU_COUNT, "GPU count in the system", "GPU_COUNT", true) +FLD_DESC_ENT(RDC_FI_DEV_NAME, "Name of the device", "DEV_NAME", true) +FLD_DESC_ENT(RDC_FI_GPU_CLOCK, "Current GPU clock frequencies.", "GPU_CLOCK", true) +FLD_DESC_ENT(RDC_FI_MEM_CLOCK, "Current Memory clock frequencies.", "MEM_CLOCK", true) +FLD_DESC_ENT(RDC_FI_MEMORY_TEMP, "Memory temperature in millidegrees Celsius.", "MEMORY_TEMP", true) +FLD_DESC_ENT(RDC_FI_GPU_TEMP, "GPU temperature in millidegrees Celsius.", "GPU_TEMP", true) +FLD_DESC_ENT(RDC_FI_POWER_USAGE, "Power usage in microwatts.", "POWER_USAGE", true) +FLD_DESC_ENT(RDC_FI_PCIE_TX, "PCIe Tx utilization in bytes/second.", "PCIE_TX", true) +FLD_DESC_ENT(RDC_FI_PCIE_RX, "PCIe Rx utilization in bytes/second.", "PCIE_RX", true) +FLD_DESC_ENT(RDC_FI_GPU_UTIL, "GPU busy percentage.", "GPU_UTIL", true) +FLD_DESC_ENT(RDC_FI_GPU_MEMORY_USAGE, "Memory usage of the GPU instance in bytes.", "GPU_MEMORY_USAGE", true) +FLD_DESC_ENT(RDC_FI_GPU_MEMORY_TOTAL, "Total memory of the GPU instance", "GPU_MEMORY_TOTAL", true) +FLD_DESC_ENT(RDC_FI_ECC_CORRECT_TOTAL, "Accumulated correctable ECC errors.", "ECC_CORRECT", true) +FLD_DESC_ENT(RDC_FI_ECC_UNCORRECT_TOTAL, "Accumulated uncorrectable ECC errors.", "ECC_UNCORRECT", true) + +FLD_DESC_ENT(RDC_EVNT_XGMI_0_NOP_TX, "NOPs sent to neighbor 0", "XGMI_NOP_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_REQ_TX, "Outgoing requests to neighbor 0", "XGMI_REQ_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_RESP_TX, "Outgoing responses to neighbor 0", "XGMI_RES_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_0_BEATS_TX, "Data sent to neighbor 0 (32 byte pkts)", "XGMI_BTS_0", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_NOP_TX, "NOPs sent to neighbor 1", "XGMI_NOP_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_REQ_TX, "Outgoing requests to neighbor 1", "XGMI_REQ_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_RESP_TX, "Outgoing responses to neighbor 1", "XGMI_RES_1", false) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_BEATS_TX, "Data sent to neighbor 1 (32 byte pkts)", "XGMI_BTS_1", false) + +FLD_DESC_ENT(RDC_EVNT_XGMI_0_THRPUT, "Tx throughput to XGMI neighbor 0 in b/s", "XGMI_0_T", true) +FLD_DESC_ENT(RDC_EVNT_XGMI_1_THRPUT, "Tx throughput to XGMI neighbor 1 in b/s", "XGMI_1_T", true) diff --git a/projects/rdc/common/rdc_fields_supported.cc b/projects/rdc/common/rdc_fields_supported.cc index 83c676d852..7d1ca7226d 100644 --- a/projects/rdc/common/rdc_fields_supported.cc +++ b/projects/rdc/common/rdc_fields_supported.cc @@ -28,14 +28,14 @@ THE SOFTWARE. namespace amd { namespace rdc { -#define FLD_DESC_ENT(ID, DESC, LABEL) \ - {static_cast(ID), {#ID, (DESC), (LABEL)}}, +#define FLD_DESC_ENT(ID, DESC, LABEL, DISPLAY) \ + {static_cast(ID), {#ID, (DESC), (LABEL), (DISPLAY)}}, static const fld_id2name_map_t field_id_to_descript = { #include "common/rdc_field_data.data" }; #undef FLD_DESC_ENT -#define FLD_DESC_ENT(ID, DESC, LABEL) {#ID, (ID)}, +#define FLD_DESC_ENT(ID, DESC, LABEL, DISPLAY) {#ID, (ID)}, static fld_name2id_map_t field_name_to_id = { #include "common/rdc_field_data.data" // NOLINT }; diff --git a/projects/rdc/common/rdc_fields_supported.h b/projects/rdc/common/rdc_fields_supported.h index 02c0ebbd09..90e5c31d86 100644 --- a/projects/rdc/common/rdc_fields_supported.h +++ b/projects/rdc/common/rdc_fields_supported.h @@ -23,6 +23,7 @@ THE SOFTWARE. #define COMMON_RDC_FIELDS_SUPPORTED_H_ #include +#include #include #include "rdc/rdc.h" @@ -34,9 +35,10 @@ typedef struct { std::string enum_name; std::string description; std::string label; + bool do_display; } field_id_descript; -typedef const std::unordered_map +typedef const std::map fld_id2name_map_t; typedef std::unordered_map fld_name2id_map_t; diff --git a/projects/rdc/common/rdc_utils.cc b/projects/rdc/common/rdc_utils.cc index c291d40355..a7433a1a9a 100755 --- a/projects/rdc/common/rdc_utils.cc +++ b/projects/rdc/common/rdc_utils.cc @@ -21,10 +21,12 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include +#include #include #include #include +#include #include "common/rdc_utils.h" diff --git a/projects/rdc/common/rdc_utils.h b/projects/rdc/common/rdc_utils.h index 5c09d44c35..1deb05933b 100755 --- a/projects/rdc/common/rdc_utils.h +++ b/projects/rdc/common/rdc_utils.h @@ -24,8 +24,6 @@ THE SOFTWARE. #ifndef COMMON_RDC_UTILS_H_ #define COMMON_RDC_UTILS_H_ -#include - #include namespace amd { diff --git a/projects/rdc/include/rdc/rdc.h b/projects/rdc/include/rdc/rdc.h index c8d58103c9..debde6473b 100755 --- a/projects/rdc/include/rdc/rdc.h +++ b/projects/rdc/include/rdc/rdc.h @@ -57,7 +57,16 @@ typedef enum { RDC_ST_CONFLICT, //!< Conflict with current state RDC_ST_CLIENT_ERROR, //!< The RDC client error RDC_ST_ALREADY_EXIST, //!< The item already exists - RDC_ST_MAX_LIMIT //!< Max limit recording for the object + RDC_ST_MAX_LIMIT, //!< Max limit recording for the object + RDC_ST_INSUFF_RESOURCES, //!< Not enough resources to complete + //!< operation + RDC_ST_FILE_ERROR, //!< Failed to access a file + RDC_ST_NO_DATA, //!< Data was requested, + //!< but none was found + RDC_ST_PERM_ERROR, //!< Insufficient permission to complete + //!< operation + + RDC_ST_UNKNOWN_ERROR = 0xFFFFFFFF //!< Unknown error } rdc_status_t; /** @@ -167,6 +176,45 @@ typedef enum { */ RDC_FI_ECC_CORRECT_TOTAL = 600, //!< Accumulated correctable ECC errors RDC_FI_ECC_UNCORRECT_TOTAL, //!< Accumulated uncorrectable ECC errors + + /* + * @brief Raw XGMI counter events + */ + RDC_EVNT_XGMI_0_NOP_TX = 1000, //!< NOPs sent to neighbor 0 + RDC_EVNT_XGMI_0_REQ_TX, //!< Outgoing requests to + //!< neighbor 0 + RDC_EVNT_XGMI_0_RESP_TX, //!< Outgoing responses to + //!< neighbor 0 + /** + * @brief + * + * Data beats sent to neighbor 0; Each beat represents 32 bytes.

+ * + * XGMI throughput can be calculated by multiplying a BEATs event + * such as ::RSMI_EVNT_XGMI_0_BEATS_TX by 32 and dividing by + * the time for which event collection occurred, + * ::rsmi_counter_value_t.time_running (which is in nanoseconds). To get + * bytes per second, multiply this value by 109.
+ *
+ * Throughput = BEATS/time_running * 109 (bytes/second)
+ */ + // ie, Throughput = BEATS/time_running 10^9 bytes/sec + RDC_EVNT_XGMI_0_BEATS_TX, + RDC_EVNT_XGMI_1_NOP_TX, //!< NOPs sent to neighbor 1 + RDC_EVNT_XGMI_1_REQ_TX, //!< Outgoing requests to + //!< neighbor 1 + RDC_EVNT_XGMI_1_RESP_TX, //!< Outgoing responses to + //!< neighbor 1 + RDC_EVNT_XGMI_1_BEATS_TX, //!< Data beats sent to + //!< neighbor 1; Each beat + //!< represnts 32 bytes + + // "Composite" events. These events have additional processing beyond + // the value provided by the rocm_smi library. + RDC_EVNT_XGMI_0_THRPUT = 1500, //!< Transmit throughput to XGMI + //!< neighbor 0 in byes/sec + RDC_EVNT_XGMI_1_THRPUT, //!< Transmit throughput to XGMI + //!< neighbor 1 in byes/sec } rdc_field_t; /** diff --git a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h index 81ef756832..b7ba515893 100644 --- a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h +++ b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h @@ -32,6 +32,9 @@ namespace rdc { class RdcMetricFetcher { public: + virtual rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) = 0; + virtual rdc_status_t delete_rsmi_handle(RdcFieldKey fk) = 0; + virtual rdc_status_t fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) = 0; virtual ~RdcMetricFetcher() {} diff --git a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h index 4267acd845..4207f2fe62 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h @@ -29,6 +29,7 @@ THE SOFTWARE. #include #include "rdc_lib/RdcMetricFetcher.h" #include "rdc_lib/rdc_common.h" +#include "rocm_smi/rocm_smi.h" namespace amd { namespace rdc { @@ -41,6 +42,21 @@ struct MetricValue { rdc_field_value value; }; +// This union represents any RSMI handles require initialization and/or +// shut down. There should only be one instance of this for each raw event +// used. For example, if a field group includes a pseudo-event and the +// underlying raw event, then only one FieldRSMIData should be created, +// and it should be used by both events. +struct FieldRSMIData { + union { + rsmi_event_handle_t evt_handle; + }; + union { + rsmi_counter_value_t counter_val; + }; + ~FieldRSMIData(){} + FieldRSMIData() : evt_handle(0), counter_val{0, 0, 0}{} +}; //!< The data structure to store the async fetch task class RdcMetricFetcherImpl; @@ -55,7 +71,13 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { rdc_field_t field_id, rdc_field_value* value) override; RdcMetricFetcherImpl(); ~RdcMetricFetcherImpl(); + + rdc_status_t acquire_rsmi_handle(RdcFieldKey fk) override; + rdc_status_t delete_rsmi_handle(RdcFieldKey fk) override; + private: + std::shared_ptr get_rsmi_data(RdcFieldKey key); + uint64_t now(); void get_ecc_error(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value); @@ -67,6 +89,7 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { //!< Async metric retreive std::map async_metrics_; + std::map> rsmi_data_; std::queue updated_tasks_; std::mutex task_mutex_; std::future updater_; // keep the future of updater @@ -74,6 +97,8 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { std::atomic task_started_; }; +rdc_status_t Rsmi2RdcError(rsmi_status_t rsmi); + } // namespace rdc } // namespace amd diff --git a/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h index 134cec02fd..4216e80968 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h @@ -29,10 +29,12 @@ THE SOFTWARE. #include #include // NOLINT #include +#include #include "rdc_lib/RdcWatchTable.h" #include "rdc_lib/RdcGroupSettings.h" #include "rdc_lib/RdcCacheManager.h" #include "rdc_lib/RdcMetricFetcher.h" +#include "rocm_smi/rocm_smi.h" namespace amd { namespace rdc { @@ -51,6 +53,7 @@ struct JobWatchTableEntry { std::vector fields; //< store fields for faster query }; + class RdcWatchTableImpl : public RdcWatchTable { public: rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id, @@ -103,6 +106,8 @@ class RdcWatchTableImpl : public RdcWatchTable { bool is_job_watch_field(uint32_t gpu_index, rdc_field_t field_id, std::string& job_id) const; // NOLINT + rdc_status_t initialize_rsmi_handles(RdcFieldKey fk); + RdcGroupSettingsPtr group_settings_; RdcCacheManagerPtr cache_mgr_; RdcMetricFetcherPtr metric_fetcher_; @@ -122,7 +127,6 @@ class RdcWatchTableImpl : public RdcWatchTable { //!< The last clean up time std::atomic last_cleanup_time_; - std::mutex watch_mutex_; }; diff --git a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc index d89ae39536..624c1ff9a9 100644 --- a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc +++ b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc @@ -346,6 +346,16 @@ const char* rdc_status_string(rdc_status_t result) { return "The value already exists"; case RDC_ST_CLIENT_ERROR: return "RDC Client error"; + case RDC_ST_INSUFF_RESOURCES: + return "Not enough resources to complete operation"; + case RDC_ST_FILE_ERROR: + return "Failed to access a file"; + case RDC_ST_NO_DATA: + return "Data was requested, but none was found"; + case RDC_ST_PERM_ERROR: + return "Insufficient permission to complete operation"; + case RDC_ST_UNKNOWN_ERROR: + return "Unknown error"; default: return "Unknown"; } diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index 99802fa310..7760a97fcc 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -22,6 +22,8 @@ THE SOFTWARE. #include "rdc_lib/impl/RdcMetricFetcherImpl.h" #include #include +#include + #include //NOLINT #include #include @@ -33,6 +35,25 @@ THE SOFTWARE. namespace amd { namespace rdc { +static const std::unordered_map + rdc_evnt_2_rsmi_field = { + {RDC_EVNT_XGMI_0_NOP_TX, RSMI_EVNT_XGMI_0_NOP_TX}, + {RDC_EVNT_XGMI_0_REQ_TX, RSMI_EVNT_XGMI_0_REQUEST_TX}, + {RDC_EVNT_XGMI_0_RESP_TX, RSMI_EVNT_XGMI_0_RESPONSE_TX}, + {RDC_EVNT_XGMI_0_BEATS_TX, RSMI_EVNT_XGMI_0_BEATS_TX}, + {RDC_EVNT_XGMI_1_NOP_TX, RSMI_EVNT_XGMI_1_NOP_TX}, + {RDC_EVNT_XGMI_1_REQ_TX, RSMI_EVNT_XGMI_1_REQUEST_TX}, + {RDC_EVNT_XGMI_1_RESP_TX, RSMI_EVNT_XGMI_1_RESPONSE_TX}, + {RDC_EVNT_XGMI_1_BEATS_TX, RSMI_EVNT_XGMI_1_BEATS_TX}, +}; + +// This maps pseudo-events to the raw events that they use. +static const std::unordered_map pseudo_evt_map = { + {RDC_EVNT_XGMI_0_THRPUT, RDC_EVNT_XGMI_0_BEATS_TX}, + {RDC_EVNT_XGMI_1_THRPUT, RDC_EVNT_XGMI_1_BEATS_TX}, +}; + + RdcMetricFetcherImpl::RdcMetricFetcherImpl() { task_started_ = true; @@ -205,6 +226,8 @@ void RdcMetricFetcherImpl::get_pcie_throughput(const RdcFieldKey& key) { } while (0); } +static const uint64_t kGig = 1000000000; + rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field_t field_id, rdc_field_value* value) { if (!value) { @@ -214,6 +237,9 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rsmi_temperature_type_t sensor_type; rsmi_clk_type_t clk_type; bool async_fetching = false; + RdcFieldKey f_key(gpu_index, field_id); + std::shared_ptr rsmi_data; + double coll_time_sec; if (!is_field_valid(field_id)) { RDC_LOG(RDC_ERROR, "Fail to fetch field " << field_id @@ -225,6 +251,15 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, value->field_id = field_id; value->status = RSMI_STATUS_NOT_SUPPORTED; + auto read_rsmi_counter = [&](void) { + assert(get_rsmi_data(f_key) != nullptr); + rsmi_data = get_rsmi_data(f_key); + value->status = rsmi_counter_read(rsmi_data->evt_handle, + &rsmi_data->counter_val); + value->value.l_int = rsmi_data->counter_val.value; + value->type = INTEGER; + }; + switch (field_id) { case RDC_FI_GPU_MEMORY_USAGE: value->status = rsmi_dev_memory_usage_get(gpu_index, @@ -309,6 +344,30 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, async_fetching = async_get_pcie_throughput( gpu_index, field_id, value); break; + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + read_rsmi_counter(); + break; + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: + read_rsmi_counter(); + if (value->status == RDC_ST_OK) { + if (rsmi_data->counter_val.time_running > 0) { + coll_time_sec = + static_cast(rsmi_data->counter_val.time_running)/kGig; + value->value.l_int = (value->value.l_int * 32)/coll_time_sec; + } else { + value->value.l_int = 0; + } + } + break; + default: break; } @@ -336,9 +395,199 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, << ", latency " << latency); } - return value->status == RSMI_STATUS_SUCCESS ? RDC_ST_OK : RDC_ST_MSI_ERROR; + return value->status == RSMI_STATUS_SUCCESS ? RDC_ST_OK : RDC_ST_MSI_ERROR; } +std::shared_ptr +RdcMetricFetcherImpl::get_rsmi_data(RdcFieldKey key) { + std::map>::iterator r_info = + rsmi_data_.find(key); + + if (r_info != rsmi_data_.end()) { + return r_info->second; + } + auto pseudo_key_field_id = pseudo_evt_map.find(key.second); + if (pseudo_key_field_id != pseudo_evt_map.end()) { + RdcFieldKey new_key; + + new_key.first = key.first; + new_key.second = pseudo_key_field_id->second; + + r_info = rsmi_data_.find(new_key); + if (r_info != rsmi_data_.end()) { + return r_info->second; + } + } + return nullptr; +} + +static rdc_status_t init_rsmi_counter(RdcFieldKey fk, + rsmi_event_group_t grp, rsmi_event_handle_t *handle) { + rsmi_status_t ret; + uint32_t counters_available; + uint32_t dv_ind = fk.first; + rdc_field_t f = fk.second; + + assert(handle != nullptr); + + ret = rsmi_dev_counter_group_supported(dv_ind, grp); + + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + + ret = rsmi_counter_available_counters_get(dv_ind, grp, &counters_available); + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + if (counters_available == 0) { + return RDC_ST_INSUFF_RESOURCES; + } + + auto raw_evt = pseudo_evt_map.find(f); + if (raw_evt != pseudo_evt_map.end()) { + f = raw_evt->second; + } + rsmi_event_type_t evt = rdc_evnt_2_rsmi_field.at(f); + ret = rsmi_dev_counter_create(dv_ind, evt, handle); + if (ret != RSMI_STATUS_SUCCESS) { + return Rsmi2RdcError(ret); + } + + ret = rsmi_counter_control(*handle, RSMI_CNTR_CMD_START, NULL); + return Rsmi2RdcError(ret); +} + +rdc_status_t RdcMetricFetcherImpl::delete_rsmi_handle(RdcFieldKey fk) { + rsmi_status_t ret; + + switch (fk.second) { + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: { + rsmi_event_handle_t h; + if (rsmi_data_.find(fk) == rsmi_data_.end()) { + return RDC_ST_NOT_SUPPORTED; + } + + h = rsmi_data_[fk]->evt_handle; + + // Stop counting. + ret = rsmi_counter_control(h, RSMI_CNTR_CMD_STOP, nullptr); + if (ret != RSMI_STATUS_SUCCESS) { + rsmi_data_.erase(fk); + return Rsmi2RdcError(ret); + } + + // Release all resources (e.g., counter and memory resources) associated + // with evnt_handle. + ret = rsmi_dev_counter_destroy(h); + + rsmi_data_.erase(fk); + return Rsmi2RdcError(ret); + } + default: + return RDC_ST_NOT_SUPPORTED; + } +} + +rdc_status_t RdcMetricFetcherImpl::acquire_rsmi_handle(RdcFieldKey fk) { + rdc_status_t result; + + switch (fk.second) { + case RDC_EVNT_XGMI_0_NOP_TX: + case RDC_EVNT_XGMI_0_REQ_TX: + case RDC_EVNT_XGMI_0_RESP_TX: + case RDC_EVNT_XGMI_0_BEATS_TX: + case RDC_EVNT_XGMI_1_NOP_TX: + case RDC_EVNT_XGMI_1_REQ_TX: + case RDC_EVNT_XGMI_1_RESP_TX: + case RDC_EVNT_XGMI_1_BEATS_TX: + case RDC_EVNT_XGMI_0_THRPUT: + case RDC_EVNT_XGMI_1_THRPUT: { + rsmi_event_handle_t handle; + + if (get_rsmi_data(fk) != nullptr) { + // This event has already been initialized. + return RDC_ST_ALREADY_EXIST; + } + + result = init_rsmi_counter(fk, RSMI_EVNT_GRP_XGMI, &handle); + if (result != RDC_ST_OK) { + return result; + } + auto fsh = std::shared_ptr(new FieldRSMIData); + + if (fsh == nullptr) { + return RDC_ST_INSUFF_RESOURCES; + } + + fsh->evt_handle = handle; + + auto pseudo_key = pseudo_evt_map.find(fk.second); + + if (pseudo_key != pseudo_evt_map.end()) { + fk.second = pseudo_key->second; + } + rsmi_data_[fk] = fsh; + } + break; + + default: + break; + } + + return RDC_ST_OK; +} + +rdc_status_t Rsmi2RdcError(rsmi_status_t rsmi) { + switch (rsmi) { + case RSMI_STATUS_SUCCESS: + return RDC_ST_OK; + + case RSMI_STATUS_INVALID_ARGS: + return RDC_ST_BAD_PARAMETER; + + case RSMI_STATUS_NOT_SUPPORTED: + return RDC_ST_NOT_SUPPORTED; + + case RSMI_STATUS_NOT_FOUND: + return RDC_ST_NOT_FOUND; + + case RSMI_STATUS_OUT_OF_RESOURCES: + return RDC_ST_INSUFF_RESOURCES; + + case RSMI_STATUS_FILE_ERROR: + return RDC_ST_FILE_ERROR; + + case RSMI_STATUS_NO_DATA: + return RDC_ST_NO_DATA; + + case RSMI_STATUS_PERMISSION: + return RDC_ST_PERM_ERROR; + + case RSMI_STATUS_BUSY: + case RSMI_STATUS_UNKNOWN_ERROR: + case RSMI_STATUS_INTERNAL_EXCEPTION: + case RSMI_STATUS_INPUT_OUT_OF_BOUNDS: + case RSMI_STATUS_INIT_ERROR: + case RSMI_STATUS_NOT_YET_IMPLEMENTED: + case RSMI_STATUS_INSUFFICIENT_SIZE: + case RSMI_STATUS_INTERRUPT: + case RSMI_STATUS_UNEXPECTED_SIZE: + case RSMI_STATUS_UNEXPECTED_DATA: + case RSMI_STATUS_REFCOUNT_OVERFLOW: + default: + return RDC_ST_UNKNOWN_ERROR; + } +} } // namespace rdc } // namespace amd diff --git a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc index e70fbece9d..f019395101 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc @@ -20,13 +20,18 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -#include "rdc_lib/impl/RdcWatchTableImpl.h" #include #include #include #include +#include +#include +#include "rdc_lib/impl/RdcWatchTableImpl.h" +#include "rdc_lib/impl/RdcMetricFetcherImpl.h" #include "rdc_lib/rdc_common.h" +#include "common/rdc_utils.h" #include "rdc_lib/RdcLogger.h" +#include "rdc/rdc.h" namespace amd { namespace rdc { @@ -171,7 +176,6 @@ rdc_status_t RdcWatchTableImpl::get_fields_from_group(rdc_gpu_group_t group_id, return RDC_ST_OK; } - rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id, uint64_t update_freq, double max_keep_age, uint32_t max_keep_samples) { @@ -207,6 +211,7 @@ rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, // Update the fields_to_watch_ auto f_in_watch_iter = fields_in_watch.begin(); + for (; f_in_watch_iter != fields_in_watch.end(); f_in_watch_iter++) { auto ite = fields_to_watch_.find(*f_in_watch_iter); if (ite == fields_to_watch_.end()) { // A new field @@ -225,6 +230,10 @@ rdc_status_t RdcWatchTableImpl::rdc_field_watch(rdc_gpu_group_t group_id, f_in_table.update_freq = update_freq; } } + result = metric_fetcher_->acquire_rsmi_handle(*f_in_watch_iter); + if (result != RDC_ST_OK && result != RDC_ST_ALREADY_EXIST) { + return result; + } } // Add to the watch table @@ -295,6 +304,15 @@ rdc_status_t RdcWatchTableImpl::update_field_in_table_when_unwatch( } } + fite = fields.begin(); + for (; fite != fields.end(); fite++) { + result = metric_fetcher_->delete_rsmi_handle(*fite); + + if (result != RDC_ST_OK && result != RDC_ST_NOT_SUPPORTED) { + return result; + } + } + return RDC_ST_OK; } diff --git a/projects/rdc/rdci/include/RdciDmonSubSystem.h b/projects/rdc/rdci/include/RdciDmonSubSystem.h index 55169cc666..d9d44b00f0 100644 --- a/projects/rdc/rdci/include/RdciDmonSubSystem.h +++ b/projects/rdc/rdci/include/RdciDmonSubSystem.h @@ -49,6 +49,7 @@ class RdciDmonSubSystem: public RdciSubSystem { DMON_UNKNOWN = 0, DMON_HELP, DMON_LIST_FIELDS, + DMON_LIST_ALL_FIELDS, DMON_MONITOR } dmon_ops_; diff --git a/projects/rdc/rdci/src/RdciDmonSubSystem.cc b/projects/rdc/rdci/src/RdciDmonSubSystem.cc index f4fe1fd394..47777e9f74 100644 --- a/projects/rdc/rdci/src/RdciDmonSubSystem.cc +++ b/projects/rdc/rdci/src/RdciDmonSubSystem.cc @@ -60,15 +60,17 @@ void RdciDmonSubSystem::set_terminating(int sig) { void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { const int HOST_OPTIONS = 1000; + const int LIST_ALL_FIELDS_OPT = 1001; const struct option long_options[] = { - {"host", required_argument, nullptr, HOST_OPTIONS }, - {"help", optional_argument, nullptr, 'h' }, - {"unauth", optional_argument, nullptr, 'u' }, - {"list", optional_argument, nullptr, 'l' }, - {"field-group-id", required_argument, nullptr, 'f' }, + {"host", required_argument, nullptr, HOST_OPTIONS}, + {"help", optional_argument, nullptr, 'h'}, + {"unauth", optional_argument, nullptr, 'u'}, + {"list", optional_argument, nullptr, 'l'}, + {"list-all", optional_argument, nullptr, LIST_ALL_FIELDS_OPT}, + {"field-group-id", required_argument, nullptr, 'f'}, {"field-id", required_argument, nullptr, 'e' }, {"gpu_index", required_argument, nullptr, 'i'}, - {"group-id", required_argument, nullptr, 'g' }, + {"group-id", required_argument, nullptr, 'g'}, {"count", required_argument, nullptr, 'c'}, {"delay", required_argument, nullptr, 'd'}, { nullptr, 0 , nullptr, 0 } @@ -132,6 +134,9 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } options_.insert({OPTIONS_DELAY, std::stoi(optarg)}); break; + case LIST_ALL_FIELDS_OPT: + dmon_ops_ = DMON_LIST_ALL_FIELDS; + break; default: show_help(); throw RdcException(RDC_ST_BAD_PARAMETER, @@ -139,7 +144,7 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } } - if (dmon_ops_ == DMON_LIST_FIELDS) { + if (dmon_ops_ == DMON_LIST_FIELDS || dmon_ops_ == DMON_LIST_ALL_FIELDS) { return; } @@ -212,6 +217,11 @@ void RdciDmonSubSystem::parse_cmd_opts(int argc, char ** argv) { } void RdciDmonSubSystem::show_help() const { + // Try to keep total output line length to <= 80 chars for better + // readability. For reference: + // *********************** 60 Chars ************************** + // ************** 40 Chars *************** + // ***** 20 Chars **** std::cout << " dmon -- Used to monitor GPUs and their stats.\n\n"; std::cout << "Usage\n"; std::cout << " rdci dmon [--host :port] [-u] -f " @@ -228,16 +238,26 @@ void RdciDmonSubSystem::show_help() const { std::cout << " -g --group-id The GPU group to query " << "on the specified host.\n"; std::cout << " -c --count count Integer representing How" - << " many times to loop before exiting. [default = runs forever.]\n"; + << " many times to loop\n" + << " before exiting. [default " + << "= runs forever.]\n"; std::cout << " -e --field-id fieldIds Comma-separated list " - << "of the field ids to monitor.\n"; + << "of field ids to monitor.\n"; std::cout << " -i --gpu_index gpuIndexes Comma-separated list " - << "of the GPU index to monitor.\n"; + << "of GPU indexes to monitor.\n"; std::cout << " -d --delay delay How often to query RDC " - << "in milli seconds. [default = 1000 msec, " + << "in milli seconds. \n" + << " [default = 1000 msec, " << "Minimum value = 100 msec.]\n"; std::cout << " -l --list List to look up the long " - << "names and descriptions of the field ids\n"; + << "names and \n" + << " descriptions of the field " + << "ids\n"; + std::cout << " --list-all Same as -l, except this " + << "lists all possible\n" + << " fields, including " + << "those that are less \n" + << " commonly used.\n"; } void RdciDmonSubSystem::create_temp_group() { @@ -295,8 +315,10 @@ void RdciDmonSubSystem::show_field_usage() const { amd::rdc::get_field_id_description_from_id(); for (auto i = field_id_to_descript.begin(); i != field_id_to_descript.end(); i++) { - std::cout << i->first << " " << i->second.enum_name << " : " << + if (i->second.do_display || dmon_ops_ == DMON_LIST_ALL_FIELDS) { + std::cout << i->first << " " << i->second.enum_name << " : " << i->second.description << "." << std::endl; + } } std::cout << std::endl; std::cout << "* Note: The field ID number associated with a field ID can " @@ -312,7 +334,7 @@ void RdciDmonSubSystem::process() { return; } - if (dmon_ops_ == DMON_LIST_FIELDS) { + if (dmon_ops_ == DMON_LIST_FIELDS || dmon_ops_ == DMON_LIST_ALL_FIELDS) { show_field_usage(); return; }