Support extra metrics in the RDC

Remove the * in the rdci stats
When a group is created, the GPUs can be added in the same command.
Add the support to the memory temperature.
Add the support to the memory clock.
Add the support to report the ECC errors.
Add the support to report the PCIe bandwidth throughput.

Since the RX/TX throughput may take 1 second to retreive, an async fetch is implemented
in the RdcMetricFetcherImpl.

Change-Id: If04f602fe1f2d14dbf7c2fb189549fd030523f9a

include/rdc/rdc.h

@@ -144,16 +144,48 @@ typedef enum {
  */
 #define RDC_FI_GPU_SM_CLOCK                 100
 
+/**
+ * Clock for the memory
+ */
+#define RDC_FI_MEM_CLOCK                101
+
+/**
+ * PCIe Tx utilization information
+ */
+#define RDC_FI_PCIE_TX    200
+
+/**
+ * PCIe Rx utilization information
+ */
+#define RDC_FI_PCIE_RX    201
+
+
 /**
  * GPU Utilization
  */
 #define RDC_FI_GPU_UTIL                     203
 
+/**
+ * Accumulated correctable ECC errors
+ */
+#define RDC_FI_ECC_CORRECT_TOTAL        312
+
+/**
+ * Accumulated uncorrectable ECC errors
+ */
+#define RDC_FI_ECC_UNCORRECT_TOTAL      313
+
+/**
+ * Memory temperature for the device
+ */
+#define RDC_FI_MEMORY_TEMP           140
+
 /**
  * Current temperature for the device
  */
 #define RDC_FI_GPU_TEMP                     150
 
+
 /**
  * GPU count in the system
  */
@@ -209,9 +241,15 @@ typedef struct {
     uint64_t end_time;      //!< The time to stop the watching
 
     uint64_t energy_consumed;
+    uint64_t ecc_correct;
+    uint64_t ecc_uncorrect;
+    rdc_stats_summary_t pcie_tx;
+    rdc_stats_summary_t pcie_rx;
     rdc_stats_summary_t power_usage;
     rdc_stats_summary_t gpu_clock;
+    rdc_stats_summary_t memory_clock;
     rdc_stats_summary_t gpu_utilization;
+    rdc_stats_summary_t gpu_temperature;
 
     uint64_t max_gpu_memory_used;
     rdc_stats_summary_t memory_utilization;

include/rdc_lib/RdcCacheManager.h

@@ -32,7 +32,6 @@ THE SOFTWARE.
 
 namespace amd {
 namespace rdc {
-typedef std::map<uint32_t, uint64_t> rdc_gpu_total_memory_t;
 
 class RdcCacheManager {
  public:
@@ -48,12 +47,14 @@ class RdcCacheManager {
     virtual std::string  get_cache_stats() = 0;
 
     virtual rdc_status_t rdc_job_get_stats(char jobId[64],
-        const rdc_gpu_total_memory_t& total_memory,
+        const rdc_gpu_gauges_t& gpu_gauges,
         rdc_job_info_t* p_job_info) = 0;
     virtual rdc_status_t rdc_job_start_stats(char jobId[64],
         const rdc_group_info_t& group,
-        const rdc_field_group_info_t& finfo) = 0;
-    virtual rdc_status_t rdc_job_stop_stats(char job_id[64]) = 0;
+        const rdc_field_group_info_t& finfo,
+        const rdc_gpu_gauges_t& gpu_gauges) = 0;
+    virtual rdc_status_t rdc_job_stop_stats(char job_id[64],
+                const rdc_gpu_gauges_t& gpu_gauge) = 0;
     virtual rdc_status_t rdc_update_job_stats(uint32_t gpu_index,
         const std::string& job_id, const rdc_field_value& value) = 0;
     virtual rdc_status_t rdc_job_remove(char job_id[64]) = 0;
@@ -64,8 +65,6 @@ class RdcCacheManager {
 
 typedef std::shared_ptr<RdcCacheManager> RdcCacheManagerPtr;
 
-//<! The key to identify the field with <gpu_id, field_id>
-typedef std::pair<uint32_t, uint32_t> RdcFieldKey;
 
 }  // namespace rdc
 }  // namespace amd

include/rdc_lib/RdcWatchTable.h

@@ -36,8 +36,10 @@ class RdcWatchTable {
     virtual rdc_status_t rdc_field_update_all() = 0;
 
     virtual rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
-                char job_id[64], uint64_t update_freq) = 0;
-    virtual rdc_status_t rdc_job_stop_stats(char job_id[64]) = 0;
+                char job_id[64], uint64_t update_freq,
+                const rdc_gpu_gauges_t& gpu_gauge) = 0;
+    virtual rdc_status_t rdc_job_stop_stats(char job_id[64],
+                const rdc_gpu_gauges_t& gpu_gauge) = 0;
     virtual rdc_status_t rdc_job_remove(char job_id[64]) = 0;
     virtual rdc_status_t rdc_job_remove_all() = 0;
 

include/rdc_lib/impl/RdcCacheManagerImpl.h

@@ -53,6 +53,8 @@ struct FieldSummaryStats {
 struct GpuSummaryStats {
     uint64_t energy_consumed;
     uint64_t energy_last_time;
+    uint64_t ecc_correct_init;    // Init counter when job starts
+    uint64_t ecc_uncorrect_init;  // Init counter when job starts
     std::map<uint32_t, FieldSummaryStats> field_summaries;
 };
 
@@ -80,12 +82,14 @@ class RdcCacheManagerImpl: public RdcCacheManager {
     std::string  get_cache_stats()  override;
 
     rdc_status_t rdc_job_get_stats(char  job_id[64],
-        const rdc_gpu_total_memory_t& total_memory,
+        const rdc_gpu_gauges_t& gpu_gauges,
         rdc_job_info_t* p_job_info) override;
     rdc_status_t rdc_job_start_stats(char job_id[64],
         const rdc_group_info_t& group,
-        const rdc_field_group_info_t& finfo) override;
-    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+        const rdc_field_group_info_t& finfo,
+        const rdc_gpu_gauges_t& gpu_gauges) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64],
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
     rdc_status_t rdc_update_job_stats(uint32_t gpu_index,
         const std::string& job_id,
         const rdc_field_value& value) override;

include/rdc_lib/impl/RdcEmbeddedHandler.h

@@ -92,6 +92,7 @@ class RdcEmbeddedHandler: public RdcHandler {
     ~RdcEmbeddedHandler();
 
  private:
+    rdc_status_t get_gpu_gauges(rdc_gpu_gauges_t* gpu_gauges);
     RdcGroupSettingsPtr group_settings_;
     RdcCacheManagerPtr cache_mgr_;
     RdcMetricFetcherPtr metric_fetcher_;

include/rdc_lib/impl/RdcMetricFetcherImpl.h

@@ -22,16 +22,55 @@ THE SOFTWARE.
 #ifndef RDC_LIB_IMPL_RDCMETRICFETCHERIMPL_H_
 #define RDC_LIB_IMPL_RDCMETRICFETCHERIMPL_H_
 
+#include <mutex>  // NOLINT(build/c++11)
+#include <future>  // NOLINT(build/c++11)
+#include <condition_variable>  // NOLINT(build/c++11)
+#include <map>
+#include <queue>
 #include "rdc_lib/RdcMetricFetcher.h"
+#include "rdc_lib/rdc_common.h"
 
 namespace amd {
 namespace rdc {
 
+//!< Some metrics, like PCIe throughput may take a second to retreive. The
+//!< MetricValue will cache those metrics for async retreive.
+struct MetricValue {
+    uint64_t cache_ttl;
+    uint64_t last_time;
+    rdc_field_value value;
+};
+
+
+//!< The data structure to store the async fetch task
+class RdcMetricFetcherImpl;
+struct MetricTask {
+    RdcFieldKey field;
+    std::function<void(RdcMetricFetcherImpl&, RdcFieldKey)> task;
+};
+
 class RdcMetricFetcherImpl: public RdcMetricFetcher {
  public:
     rdc_status_t fetch_smi_field(uint32_t gpu_index,
         uint32_t field_id, rdc_field_value* value) override;
     bool is_field_valid(uint32_t field_id) const override;
+    RdcMetricFetcherImpl();
+    ~RdcMetricFetcherImpl();
+ private:
+    uint64_t now();
+    void get_ecc_error(uint32_t gpu_index,
+        uint32_t field_id, rdc_field_value* value);
+    void async_get_pcie_throughput(uint32_t gpu_index,
+        uint32_t field_id, rdc_field_value* value);
+    void get_pcie_throughput(const RdcFieldKey& key);
+
+    //!< Async metric retreive
+    std::map<RdcFieldKey, MetricValue> async_metrics_;
+    std::queue<MetricTask> updated_tasks_;
+    std::mutex task_mutex_;
+    std::future<void> updater_;  // keep the future of updater
+    std::condition_variable cv_;
+    std::atomic<bool> task_started_;
 };
 
 }  // namespace rdc

include/rdc_lib/impl/RdcWatchTableImpl.h

@@ -54,8 +54,10 @@ struct JobWatchTableEntry {
 class RdcWatchTableImpl : public RdcWatchTable {
  public:
     rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
-                    char job_id[64], uint64_t update_freq) override;
-    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+                    char job_id[64], uint64_t update_freq,
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64],
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
     rdc_status_t rdc_job_remove(char job_id[64]) override;
     rdc_status_t rdc_job_remove_all() override;
 

include/rdc_lib/rdc_common.h

@@ -23,6 +23,8 @@ THE SOFTWARE.
 #ifndef RDC_LIB_RDC_COMMON_H_
 #define RDC_LIB_RDC_COMMON_H_
 #include <iostream>
+#include <map>
+#include <utility>
 
 #define RDC_ERROR  0
 #define RDC_INFO   1
@@ -37,6 +39,12 @@ THE SOFTWARE.
     } \
 } while (0)
 
+//<! The key to identify the field with <gpu_id, field_id>
+typedef std::pair<uint32_t, uint32_t> RdcFieldKey;
+
+//!< The gauge metrics do not require aggregations
+typedef std::map<RdcFieldKey, uint64_t> rdc_gpu_gauges_t;
+
 /**
  *  @brief The strncpy but with null terminated
  *