Implement the rdc_lib API to support the job stats

Add the function to start and stop the job recording.
Add the function to get the job stats for each GPU and summary of multiple GPUs
Add the function to remove the jobs.

Add a class RdcLogger which can control the log level using the environment variable RDC_LOG.
This is similar to GRPC_VERBOSITY gRPC. When the customer has the issues, he can enable the verbose
log to help us to troubleshoot the issues.

Add the -u support in the rdci group, fieldgroup and dmon for connecting to rdcd without authentication.

Change-Id: I22c591823c1ee6485db106b911bed8271d1b2769

include/rdc_lib/impl/RdcCacheManagerImpl.h

@@ -23,7 +23,8 @@ THE SOFTWARE.
 #define RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_
 
 #include <memory>
-#include <mutex>
+#include <mutex>  // NOLINT(build/c++11)
+#include <string>
 #include <vector>
 #include <map>
 #include "rdc_lib/RdcCacheManager.h"
@@ -41,6 +42,29 @@ struct RdcCacheEntry {
 
 typedef std::map<RdcFieldKey, std::vector<RdcCacheEntry>> RdcCacheSamples;
 
+struct FieldSummaryStats {
+    int64_t max_value;
+    int64_t min_value;
+    int64_t total_value;
+    uint64_t last_time;
+    uint64_t count;
+};
+
+struct GpuSummaryStats {
+    uint64_t energy_consumed;
+    uint64_t energy_last_time;
+    std::map<uint32_t, FieldSummaryStats> field_summaries;
+};
+
+// Per job entry
+struct RdcJobStatsCacheEntry {
+    uint64_t start_time;
+    uint64_t end_time;
+    std::map<uint32_t, GpuSummaryStats> gpu_stats;
+};
+
+// <job_id, job_stats>
+typedef std::map<std::string, RdcJobStatsCacheEntry> RdcJobStatsCache;
 
 class RdcCacheManagerImpl: public RdcCacheManager {
  public:
@@ -53,10 +77,27 @@ class RdcCacheManagerImpl: public RdcCacheManager {
                 const rdc_field_value& value) override;
     rdc_status_t evict_cache(uint32_t gpu_index, uint32_t field_id,
                 uint64_t max_keep_samples, double  max_keep_age) override;
-    uint32_t get_cache_size()  override;
+    std::string  get_cache_stats()  override;
+
+    rdc_status_t rdc_job_get_stats(char  job_id[64],
+        const rdc_gpu_total_memory_t& total_memory,
+        rdc_job_info_t* p_job_info) override;
+    rdc_status_t rdc_job_start_stats(char job_id[64],
+        const rdc_group_info_t& group,
+        const rdc_field_group_info_t& finfo) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+    rdc_status_t rdc_update_job_stats(uint32_t gpu_index,
+        const std::string& job_id,
+        const rdc_field_value& value) override;
+    rdc_status_t rdc_job_remove(char job_id[64]) override;
+    rdc_status_t rdc_job_remove_all() override;
 
  private:
+    void set_summary(const FieldSummaryStats & stats,
+        rdc_stats_summary_t& gpu, rdc_stats_summary_t& summary, // NOLINT
+        unsigned int adjuster);
     RdcCacheSamples cache_samples_;
+    RdcJobStatsCache cache_jobs_;
     std::mutex cache_mutex_;
 };
 

include/rdc_lib/impl/RdcEmbeddedHandler.h

@@ -36,11 +36,12 @@ class RdcEmbeddedHandler: public RdcHandler {
  public:
     // Job API
     rdc_status_t rdc_job_start_stats(rdc_gpu_group_t groupId,
-                char  job_id[64], uint64_t update_freq, double  max_keep_age,
-            uint32_t  max_keep_samples) override;
-    rdc_status_t rdc_job_get_stats(char  jobId[64],
+                char job_id[64], uint64_t update_freq) override;
+    rdc_status_t rdc_job_get_stats(char jobId[64],
                 rdc_job_info_t* p_job_info) override;
-    rdc_status_t rdc_job_stop_stats(char  job_id[64]) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+    rdc_status_t rdc_job_remove(char job_id[64]) override;
+    rdc_status_t rdc_job_remove_all() override;
 
     // Discovery API
     rdc_status_t rdc_device_get_all(

include/rdc_lib/impl/RdcGroupSettingsImpl.h

@@ -61,8 +61,8 @@ class RdcGroupSettingsImpl: public RdcGroupSettings {
  private:
     std::map<rdc_gpu_group_t, rdc_group_info_t> gpu_group_;
     std::map<rdc_field_grp_t, rdc_field_group_info_t> field_group_;
-    uint32_t cur_group_id_ = 0;
-    uint32_t cur_filed_group_id_ = 0;
+    uint32_t cur_group_id_ = 1;
+    uint32_t cur_field_group_id_ = 0;
     std::mutex group_mutex_;
     std::mutex field_group_mutex_;
 };

include/rdc_lib/impl/RdcStandaloneHandler.h

@@ -33,11 +33,12 @@ class RdcStandaloneHandler: public RdcHandler {
  public:
     // Job RdcAPI
     rdc_status_t rdc_job_start_stats(rdc_gpu_group_t groupId,
-                char  job_id[64], uint64_t update_freq, double  max_keep_age,
-            uint32_t  max_keep_samples) override;
-    rdc_status_t rdc_job_get_stats(char  jobId[64],
+                char job_id[64], uint64_t update_freq) override;
+    rdc_status_t rdc_job_get_stats(char jobId[64],
                 rdc_job_info_t* p_job_info) override;
-    rdc_status_t rdc_job_stop_stats(char  job_id[64]) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+    rdc_status_t rdc_job_remove(char job_id[64]) override;
+    rdc_status_t rdc_job_remove_all() override;
 
     // Discovery RdcAPI
     rdc_status_t rdc_device_get_all(

include/rdc_lib/impl/RdcWatchTableImpl.h

@@ -22,11 +22,12 @@ THE SOFTWARE.
 #ifndef RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_
 #define RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_
 
+#include <string>
 #include <map>
 #include <vector>
 #include <utility>
 #include <memory>
-#include <mutex>
+#include <mutex>  // NOLINT
 #include <atomic>
 #include "rdc_lib/RdcWatchTable.h"
 #include "rdc_lib/RdcGroupSettings.h"
@@ -45,14 +46,18 @@ struct FieldSettings {
     uint64_t last_update_time;
 };
 
+struct JobWatchTableEntry {
+    uint32_t group_id;
+    std::vector<RdcFieldKey> fields;  //< store fields for faster query
+};
 
 class RdcWatchTableImpl : public RdcWatchTable {
  public:
     rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
-                    char job_id[64]) override;
-    rdc_status_t rdc_watch_job_fields(rdc_gpu_group_t group_id,
-                    uint64_t update_freq, double  max_keep_age,
-                    uint32_t  max_keep_samples)  override;
+                    char job_id[64], uint64_t update_freq) override;
+    rdc_status_t rdc_job_stop_stats(char job_id[64]) override;
+    rdc_status_t rdc_job_remove(char job_id[64]) override;
+    rdc_status_t rdc_job_remove_all() override;
 
     rdc_status_t rdc_field_watch(rdc_gpu_group_t group_id,
         rdc_field_grp_t field_group_id, uint64_t update_freq,
@@ -84,10 +89,16 @@ class RdcWatchTableImpl : public RdcWatchTable {
     //!< Helper function to clean up the watch table and cache
     void clean_up();
 
+    //!< Helper function for debug information in watch table and cache
+    void debug_status();
+
     //!< Helper function to get the fields using the group and the field group.
     rdc_status_t get_fields_from_group(rdc_gpu_group_t group_id,
-         rdc_field_grp_t field_group_id, std::vector<RdcFieldKey> & fields);
+         rdc_field_grp_t field_group_id,
+         std::vector<RdcFieldKey> & fields); // NOLINT
 
+    bool is_job_watch_field(uint32_t gpu_index, uint32_t field_id,
+                std::string& job_id) const;  // NOLINT
 
     RdcGroupSettingsPtr group_settings_;
     RdcCacheManagerPtr cache_mgr_;
@@ -96,6 +107,10 @@ class RdcWatchTableImpl : public RdcWatchTable {
     //!< The watch table to store the watch settings.
     std::map<RdcFieldKey, FieldSettings> watch_table_;
 
+    //!< <job_id, gpu_group_id> pairs
+    std::map<std::string, JobWatchTableEntry> job_watch_table_;
+
+
     //!< The settings for each field can be deduced from watch_table. But every
     //!< rdc_field_update_all() call needs to deduce them. To improve the
     //!< performance, the fields_to_watch_ is used to track the field settings.