Support extra metrics in the RDC

Remove the * in the rdci stats
When a group is created, the GPUs can be added in the same command.
Add the support to the memory temperature.
Add the support to the memory clock.
Add the support to report the ECC errors.
Add the support to report the PCIe bandwidth throughput.

Since the RX/TX throughput may take 1 second to retreive, an async fetch is implemented
in the RdcMetricFetcherImpl.

Change-Id: If04f602fe1f2d14dbf7c2fb189549fd030523f9a

include/rdc_lib/impl/RdcCacheManagerImpl.h

@@ -53,6 +53,8 @@ struct FieldSummaryStats {
 struct GpuSummaryStats {
     uint64_t energy_consumed;
     uint64_t energy_last_time;
+    uint64_t ecc_correct_init;    // Init counter when job starts
+    uint64_t ecc_uncorrect_init;  // Init counter when job starts
     std::map<uint32_t, FieldSummaryStats> field_summaries;
 };
 
@@ -80,12 +82,14 @@ class RdcCacheManagerImpl: public RdcCacheManager {
     std::string  get_cache_stats()  override;
 
     rdc_status_t rdc_job_get_stats(char  job_id[64],
-        const rdc_gpu_total_memory_t& total_memory,
+        const rdc_gpu_gauges_t& gpu_gauges,
         rdc_job_info_t* p_job_info) override;
     rdc_status_t rdc_job_start_stats(char job_id[64],
         const rdc_group_info_t& group,
-        const rdc_field_group_info_t& finfo) override;
-    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+        const rdc_field_group_info_t& finfo,
+        const rdc_gpu_gauges_t& gpu_gauges) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64],
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
     rdc_status_t rdc_update_job_stats(uint32_t gpu_index,
         const std::string& job_id,
         const rdc_field_value& value) override;

include/rdc_lib/impl/RdcEmbeddedHandler.h

@@ -92,6 +92,7 @@ class RdcEmbeddedHandler: public RdcHandler {
     ~RdcEmbeddedHandler();
 
  private:
+    rdc_status_t get_gpu_gauges(rdc_gpu_gauges_t* gpu_gauges);
     RdcGroupSettingsPtr group_settings_;
     RdcCacheManagerPtr cache_mgr_;
     RdcMetricFetcherPtr metric_fetcher_;

include/rdc_lib/impl/RdcMetricFetcherImpl.h

@@ -22,16 +22,55 @@ THE SOFTWARE.
 #ifndef RDC_LIB_IMPL_RDCMETRICFETCHERIMPL_H_
 #define RDC_LIB_IMPL_RDCMETRICFETCHERIMPL_H_
 
+#include <mutex>  // NOLINT(build/c++11)
+#include <future>  // NOLINT(build/c++11)
+#include <condition_variable>  // NOLINT(build/c++11)
+#include <map>
+#include <queue>
 #include "rdc_lib/RdcMetricFetcher.h"
+#include "rdc_lib/rdc_common.h"
 
 namespace amd {
 namespace rdc {
 
+//!< Some metrics, like PCIe throughput may take a second to retreive. The
+//!< MetricValue will cache those metrics for async retreive.
+struct MetricValue {
+    uint64_t cache_ttl;
+    uint64_t last_time;
+    rdc_field_value value;
+};
+
+
+//!< The data structure to store the async fetch task
+class RdcMetricFetcherImpl;
+struct MetricTask {
+    RdcFieldKey field;
+    std::function<void(RdcMetricFetcherImpl&, RdcFieldKey)> task;
+};
+
 class RdcMetricFetcherImpl: public RdcMetricFetcher {
  public:
     rdc_status_t fetch_smi_field(uint32_t gpu_index,
         uint32_t field_id, rdc_field_value* value) override;
     bool is_field_valid(uint32_t field_id) const override;
+    RdcMetricFetcherImpl();
+    ~RdcMetricFetcherImpl();
+ private:
+    uint64_t now();
+    void get_ecc_error(uint32_t gpu_index,
+        uint32_t field_id, rdc_field_value* value);
+    void async_get_pcie_throughput(uint32_t gpu_index,
+        uint32_t field_id, rdc_field_value* value);
+    void get_pcie_throughput(const RdcFieldKey& key);
+
+    //!< Async metric retreive
+    std::map<RdcFieldKey, MetricValue> async_metrics_;
+    std::queue<MetricTask> updated_tasks_;
+    std::mutex task_mutex_;
+    std::future<void> updater_;  // keep the future of updater
+    std::condition_variable cv_;
+    std::atomic<bool> task_started_;
 };
 
 }  // namespace rdc

include/rdc_lib/impl/RdcWatchTableImpl.h

@@ -54,8 +54,10 @@ struct JobWatchTableEntry {
 class RdcWatchTableImpl : public RdcWatchTable {
  public:
     rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
-                    char job_id[64], uint64_t update_freq) override;
-    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+                    char job_id[64], uint64_t update_freq,
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64],
+                    const rdc_gpu_gauges_t& gpu_gauge) override;
     rdc_status_t rdc_job_remove(char job_id[64]) override;
     rdc_status_t rdc_job_remove_all() override;