SWDEV-223878 - Add cache manager and watch table to skeleton rdc_lib

Support cache manager and watch table in rdc_lib RdcCacheManagerImpl.cc is added to implement cache of metrics. Currently, only integer mertics are supported. The cache manager provids function to retrieve the latest and history metrics from cache. It also provides interfaces to update and evict the cache. RdcWatchTableImpl.cc is added to implement watch and unwatch fields. It uses the field settings to control how frequently a field needs to be updated. We have a preliminarily performance optimization for this class as it may be called very frequently. RdcMetricsUpdaterImpl.cc is added to run the update at background thread when RDC_OPERATION_MODE_AUTO is set. After this code change, the rdcd/rdci should be able to implement basic discovery, group and dmon function. The job management function is not implemented in the skeleton rdc_lib yet. Change-Id: I26cff8c2ec85d1ad8e7df24c66b02f0060838d37 [ROCm/rdc commit: 1ff1c7b617]
2020-02-26 11:04:42 -05:00
@@ -50,7 +50,7 @@ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}

 ## Compiler flags
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -m64")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -msse2 -std=c++11 ")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -msse2 -std=c++11 -pthread ")
 # Use this instead of above for 32 bit
 # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")

@@ -50,6 +50,7 @@ typedef enum {
     RDC_ST_INVALID_HANDLER,         //!< Invalid handler
     RDC_ST_BAD_PARAMETER,      //!< A parameter is invalid
     RDC_ST_NOT_FOUND,          //!< Cannot find the value
+     RDC_ST_CONFLICT,           //!< Conflict with current state
     RDC_ST_MAX_LIMIT               //!< Max limit recording for the object
 } rdc_status_t;

@@ -0,0 +1,59 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_RDCCACHEMANAGER_H_
+#define RDC_LIB_RDCCACHEMANAGER_H_
+
+#include <memory>
+#include <utility>
+#include <vector>
+#include <map>
+#include "rdc_lib/rdc_common.h"
+#include "rdc/rdc.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcCacheManager {
+ public:
+    virtual rdc_status_t rdc_get_latest_value_for_field(uint32_t gpu_index,
+                uint32_t field, rdc_field_value* value) = 0;
+    virtual rdc_status_t rdc_get_field_value_since(uint32_t gpu_index,
+                uint32_t field, uint64_t since_time_stamp,
+                uint64_t *next_since_time_stamp, rdc_field_value* value) = 0;
+    virtual rdc_status_t rdc_update_cache(uint32_t gpu_index,
+                const rdc_field_value& value) = 0;
+    virtual rdc_status_t evict_cache(uint32_t gpu_index, uint32_t field_id,
+                uint64_t max_keep_samples, double  max_keep_age) = 0;
+    virtual uint32_t  get_cache_size() = 0;
+
+    virtual ~RdcCacheManager() {}
+};
+
+typedef std::shared_ptr<RdcCacheManager> RdcCacheManagerPtr;
+
+//<! The key to identify the field with <gpu_id, field_id>
+typedef std::pair<uint32_t, uint32_t> RdcFieldKey;
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // RDC_LIB_RDCCACHEMANAGER_H_
@@ -34,6 +34,7 @@ class  RdcMetricFetcher {
 public:
    virtual rdc_status_t fetch_smi_field(uint32_t gpu_index,
                 uint32_t field_id, rdc_field_value* value) = 0;
+    virtual bool is_field_valid(uint32_t field_id) const = 0;
    virtual ~RdcMetricFetcher() {}
 };

@@ -0,0 +1,42 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_RDCMETRICSUPDATER_H_
+#define  RDC_LIB_RDCMETRICSUPDATER_H_
+
+#include <memory>
+
+namespace amd {
+namespace rdc {
+
+class RdcMetricsUpdater {
+ public:
+     virtual void start() = 0;
+     virtual void stop() = 0;
+};
+
+typedef std::shared_ptr<RdcMetricsUpdater> RdcMetricsUpdaterPtr;
+
+}  // namespace rdc
+}  // namespace amd
+
+
+#endif  // RDC_LIB_RDCMETRICSUPDATER_H_
@@ -0,0 +1,58 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_RDCWATCHTABLE_H_
+#define RDC_LIB_RDCWATCHTABLE_H_
+
+#include <memory>
+#include <vector>
+#include "rdc_lib/rdc_common.h"
+#include "rdc/rdc.h"
+
+
+namespace amd {
+namespace rdc {
+
+class RdcWatchTable {
+ public:
+    virtual rdc_status_t rdc_update_all_fields() = 0;
+
+    virtual rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
+                char  job_id[64]) = 0;
+    virtual rdc_status_t rdc_watch_job_fields(rdc_gpu_group_t group_id,
+                uint64_t update_freq, double  max_keep_age,
+                uint32_t max_keep_samples) = 0;
+
+    virtual rdc_status_t rdc_watch_fields(rdc_gpu_group_t group_id,
+                rdc_field_grp_t field_group_id, uint64_t update_freq,
+                double  max_keep_age, uint32_t max_keep_samples) = 0;
+    virtual rdc_status_t rdc_unwatch_fields(rdc_gpu_group_t group_id,
+                rdc_field_grp_t field_group_id) = 0;
+
+    virtual ~RdcWatchTable() {}
+};
+
+typedef std::shared_ptr<RdcWatchTable> RdcWatchTablePtr;
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // RDC_LIB_RDCWATCHTABLE_H_
@@ -0,0 +1,67 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_
+#define RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_
+
+#include <memory>
+#include <mutex>
+#include <vector>
+#include <map>
+#include "rdc_lib/RdcCacheManager.h"
+#include "rdc_lib/rdc_common.h"
+#include "rdc/rdc.h"
+
+
+namespace amd {
+namespace rdc {
+
+struct RdcCacheEntry {
+    uint64_t last_time;
+    int64_t value;
+};
+
+typedef std::map<RdcFieldKey, std::vector<RdcCacheEntry>> RdcCacheSamples;
+
+
+class RdcCacheManagerImpl: public RdcCacheManager {
+ public:
+    rdc_status_t rdc_get_latest_value_for_field(uint32_t gpu_index,
+                uint32_t field, rdc_field_value* value) override;
+    rdc_status_t rdc_get_field_value_since(uint32_t gpu_index, uint32_t field,
+                uint64_t since_time_stamp, uint64_t *next_since_time_stamp,
+                rdc_field_value* value) override;
+    rdc_status_t rdc_update_cache(uint32_t gpu_index,
+                const rdc_field_value& value) override;
+    rdc_status_t evict_cache(uint32_t gpu_index, uint32_t field_id,
+                uint64_t max_keep_samples, double  max_keep_age) override;
+    uint32_t get_cache_size()  override;
+
+ private:
+    RdcCacheSamples cache_samples_;
+    std::mutex cache_mutex_;
+};
+
+}  // namespace rdc
+}  // namespace amd
+
+
+#endif  // RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_
@@ -25,6 +25,9 @@ THE SOFTWARE.
 #include "rdc_lib/RdcHandler.h"
 #include "rdc_lib/RdcGroupSettings.h"
 #include "rdc_lib/RdcMetricFetcher.h"
+#include "rdc_lib/RdcCacheManager.h"
+#include "rdc_lib/RdcMetricsUpdater.h"
+#include "rdc_lib/RdcWatchTable.h"

 namespace amd {
 namespace rdc {
@@ -80,10 +83,14 @@ class RdcEmbeddedHandler: public RdcHandler {
    rdc_status_t rdc_update_all_fields(uint32_t wait_for_update) override;

    explicit RdcEmbeddedHandler(rdc_operation_mode_t op_mode);
+    ~RdcEmbeddedHandler();

 private:
    RdcGroupSettingsPtr group_settings_;
+    RdcCacheManagerPtr cache_mgr_;
    RdcMetricFetcherPtr metric_fetcher_;
+    RdcWatchTablePtr watch_table_;
+    RdcMetricsUpdaterPtr metrics_updater_;
 };

 }  // namespace rdc
@@ -61,7 +61,6 @@ class RdcGroupSettingsImpl: public RdcGroupSettings {
    uint32_t cur_filed_group_id_ = 0;
    std::mutex group_mutex_;
    std::mutex field_group_mutex_;
-
 };

 }  // namespace rdc
@@ -31,6 +31,7 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher {
 public:
    rdc_status_t fetch_smi_field(uint32_t gpu_index,
        uint32_t field_id, rdc_field_value* value) override;
+    bool is_field_valid(uint32_t field_id) const override;
 };

 }  // namespace rdc
@@ -0,0 +1,49 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_
+#define RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_
+
+#include <future>
+#include <memory>
+#include "rdc_lib/RdcMetricsUpdater.h"
+#include "rdc_lib/RdcWatchTable.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcMetricsUpdaterImpl: public RdcMetricsUpdater {
+ public:
+     void start() override;
+     void stop() override;
+     explicit RdcMetricsUpdaterImpl(const RdcWatchTablePtr& watch_table,
+                const uint32_t check_frequency);
+ private:
+     RdcWatchTablePtr watch_table_;
+     std::atomic<bool> started_;
+     std::future<void> updater_;  // keep the future of updater
+     const uint32_t _check_frequency;  // Check frequency in milliseconds
+};
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_
@@ -0,0 +1,115 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_
+#define RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_
+
+#include <map>
+#include <vector>
+#include <utility>
+#include <memory>
+#include <mutex>
+#include <atomic>
+#include "rdc_lib/RdcWatchTable.h"
+#include "rdc_lib/RdcGroupSettings.h"
+#include "rdc_lib/RdcCacheManager.h"
+#include "rdc_lib/RdcMetricFetcher.h"
+
+namespace amd {
+namespace rdc {
+
+//!< The settings for a field or a group of field in the watch table.
+struct FieldSettings {
+    uint64_t  update_freq;
+    uint32_t  max_keep_samples;
+    double  max_keep_age;
+    bool is_watching;
+    uint64_t last_update_time;
+};
+
+
+class RdcWatchTableImpl : public RdcWatchTable {
+ public:
+    rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id,
+                    char job_id[64]) override;
+    rdc_status_t rdc_watch_job_fields(rdc_gpu_group_t group_id,
+                    uint64_t update_freq, double  max_keep_age,
+                    uint32_t  max_keep_samples)  override;
+
+    rdc_status_t rdc_watch_fields(rdc_gpu_group_t group_id,
+        rdc_field_grp_t field_group_id, uint64_t update_freq,
+        double  max_keep_age, uint32_t max_keep_samples) override;
+
+    //!< rdc_unwatch_fields() will not remove the entry from watch_table.
+    //!< The unwatched entry is still kept until the max_keep_age of the entry
+    //!< is reached, which will be handled in the clean_up() function.
+    rdc_status_t rdc_unwatch_fields(rdc_gpu_group_t group_id,
+                rdc_field_grp_t field_group_id) override;
+
+    //!< When the RDC is running as RDC_OPERATION_MODE_MANUAL, the user will
+    //!< call this function periodically. Instead of providing other APIs to
+    //!< cleanup the cache, this function will update and cleanup the cache.
+    //!<
+    //!< This function may be called very frequently, and the cache cleanup
+    //!< is expensive. Internally, this function will throttle the cleanup to
+    //!< once per second.
+    rdc_status_t rdc_update_all_fields() override;
+
+    RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings,
+        const RdcCacheManagerPtr& cache_mgr,
+        const RdcMetricFetcherPtr& metric_fetcher);
+
+ private:
+    //!< Helper function to Update the fields_in_table when unwatch tables
+    rdc_status_t update_field_in_table_when_unwatch(const RdcFieldKey& entry);
+
+    //!< Helper function to clean up the watch table and cache
+    void clean_up();
+
+    //!< Helper function to get the fields using the group and the field group.
+    rdc_status_t get_fields_from_group(rdc_gpu_group_t group_id,
+         rdc_field_grp_t field_group_id, std::vector<RdcFieldKey> & fields);
+
+
+    RdcGroupSettingsPtr group_settings_;
+    RdcCacheManagerPtr cache_mgr_;
+    RdcMetricFetcherPtr metric_fetcher_;
+
+    //!< The watch table to store the watch settings.
+    std::map<RdcFieldKey, FieldSettings> watch_table_;
+
+    //!< The settings for each field can be deduced from watch_table. But every
+    //!< rdc_update_all_fields() call needs to deduce them. To improve the
+    //!< performance, the fields_to_watch_ is used to track the field settings.
+    //!< Those settings will only be updated when watching or unwatching.
+    std::map<RdcFieldKey, FieldSettings> fields_to_watch_;
+
+    //!< The last clean up time
+    std::atomic<uint64_t> last_cleanup_time_;
+
+    std::mutex watch_mutex_;
+};
+
+}  // namespace rdc
+}  // namespace amd
+
+
+#endif  // RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_
@@ -140,12 +140,22 @@ set(RDC_LIB_COMPONENT "lib${RDC_LIB}")
 set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcEmbeddedHandler.cc")
 set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricFetcherImpl.cc")
 set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcGroupSettingsImpl.cc")
+set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcCacheManagerImpl.cc")
+set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricsUpdaterImpl.cc")
+set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcWatchTableImpl.cc")

 set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcEmbeddedHandler.h")
 set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricFetcher.h")
 set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricFetcherImpl.h")
 set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcGroupSettings.h")
 set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcGroupSettingsImpl.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcCacheManager.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcCacheManagerImpl.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricsUpdater.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricsUpdaterImpl.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcWatchTable.h")
+set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcWatchTableImpl.h")
+
 message("RDC_LIB_INC_LIST=${RDC_LIB_INC_LIST}")

 link_directories(${RSMI_LIB_DIR})
@@ -318,6 +318,8 @@ const char* rdc_status_string(rdc_status_t result) {
                return "SMI error";
        case RDC_ST_MAX_LIMIT:
                return "The max limit reached";
+        case RDC_ST_CONFLICT:
+                return "Conflict with current state";
        default:
                return "Unknown";
    }
@@ -0,0 +1,166 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#include "rdc_lib/impl/RdcCacheManagerImpl.h"
+#include <sys/time.h>
+#include <ctime>
+#include "rdc_lib/rdc_common.h"
+
+
+namespace amd {
+namespace rdc {
+
+rdc_status_t RdcCacheManagerImpl::rdc_get_field_value_since(
+    uint32_t gpu_index, uint32_t field_id, uint64_t since_time_stamp,
+    uint64_t *next_since_time_stamp, rdc_field_value* value) {
+    if (!next_since_time_stamp || !value) {
+        return RDC_ST_BAD_PARAMETER;
+    }
+
+    std::lock_guard<std::mutex> guard(cache_mutex_);
+    RdcFieldKey field{gpu_index, field_id};
+    auto cache_samples_ite = cache_samples_.find(field);
+    if (cache_samples_ite == cache_samples_.end() ||
+             cache_samples_ite->second.size() == 0) {
+        return RDC_ST_NOT_FOUND;
+    }
+
+    // TODO(bill_liu): Optimize it using the binary search
+    auto cache_values = cache_samples_ite->second;
+    for (auto cache_value=cache_values.begin();
+                cache_value != cache_values.end(); cache_value++) {
+        if ( cache_value->last_time >= since_time_stamp ) {
+            // move to next potential timestamp
+            auto next_iter = std::next(cache_value);
+            if (next_iter != cache_values.end()) {
+                *next_since_time_stamp = next_iter->last_time;
+            } else {  // Last item, set it to the future by adding 1us
+                *next_since_time_stamp = cache_value->last_time + 1;
+            }
+            value->ts = cache_value->last_time;
+            value->type = INTEGER;
+            value->value.l_int = cache_value->value;
+            value->field_id = field_id;
+            return RDC_ST_OK;
+        }
+    }
+
+    *next_since_time_stamp = since_time_stamp;
+    return RDC_ST_NOT_FOUND;
+}
+
+
+rdc_status_t RdcCacheManagerImpl::evict_cache(uint32_t gpu_index,
+        uint32_t field_id, uint64_t max_keep_samples, double  max_keep_age) {
+    std::lock_guard<std::mutex> guard(cache_mutex_);
+
+    RdcFieldKey field{gpu_index, field_id};
+    auto cache_samples_ite = cache_samples_.find(field);
+    if (cache_samples_ite == cache_samples_.end() ||
+             cache_samples_ite->second.size() == 0) {
+        return RDC_ST_NOT_FOUND;
+    }
+
+    // Check max_keep_samples
+    auto& cache_values = cache_samples_ite->second;
+    int item_remove = cache_values.size() - max_keep_samples;
+    if (item_remove > 0) {
+       cache_values.erase(cache_values.begin(),
+                cache_values.begin()+item_remove);
+    }
+
+    // Check max_keep_age
+    struct timeval  tv;
+    gettimeofday(&tv, NULL);
+    uint64_t now = static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
+
+    auto ite = cache_values.begin();
+    while (ite != cache_values.end()) {
+        if (ite->last_time + max_keep_age*1000 >= now) {
+            break;
+        } else {
+            ite = cache_values.erase(ite);
+        }
+    }
+
+    return RDC_ST_OK;
+}
+
+rdc_status_t RdcCacheManagerImpl::rdc_get_latest_value_for_field(
+    uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) {
+    if (!value) {
+        return RDC_ST_BAD_PARAMETER;
+    }
+
+    std::lock_guard<std::mutex> guard(cache_mutex_);
+    RdcFieldKey field{gpu_index, field_id};
+    auto cache_samples_ite = cache_samples_.find(field);
+    if (cache_samples_ite == cache_samples_.end() ||
+             cache_samples_ite->second.size() == 0) {
+        return RDC_ST_NOT_FOUND;
+    }
+
+    auto& cache_value = cache_samples_ite->second.back();
+    value->ts = cache_value.last_time;
+    value->type = INTEGER;
+    value->value.l_int = cache_value.value;
+    value->field_id = field_id;
+
+    return RDC_ST_OK;
+}
+
+uint32_t RdcCacheManagerImpl::get_cache_size() {
+    uint32_t cache_size = 0;
+    std::lock_guard<std::mutex> guard(cache_mutex_);
+
+    auto cache_samples_ite = cache_samples_.begin();
+    for (; cache_samples_ite != cache_samples_.end(); cache_samples_ite++) {
+        cache_size+=cache_samples_ite->second.size();
+    }
+    return cache_size;
+}
+
+rdc_status_t RdcCacheManagerImpl::rdc_update_cache(uint32_t gpu_index,
+        const rdc_field_value& value) {
+    RdcCacheEntry entry;
+    entry.last_time = value.ts;
+    if (value.type == INTEGER) {
+        entry.value = value.value.l_int;
+    } else {
+        return RDC_ST_NOT_SUPPORTED;
+    }
+
+    std::lock_guard<std::mutex> guard(cache_mutex_);
+    RdcFieldKey field{gpu_index, value.field_id};
+    auto cache_samples_ite = cache_samples_.find(field);
+    if (cache_samples_ite == cache_samples_.end()) {
+        std::vector<RdcCacheEntry> ve;
+        ve.push_back(entry);
+        cache_samples_.insert({field, ve});
+    } else {
+        cache_samples_ite->second.push_back(entry);
+    }
+
+    return RDC_ST_OK;
+}
+
+}  // namespace rdc
+}  // namespace amd
@@ -23,6 +23,9 @@ THE SOFTWARE.
 #include <string.h>
 #include "rdc_lib/impl/RdcMetricFetcherImpl.h"
 #include "rdc_lib/impl/RdcGroupSettingsImpl.h"
+#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
+#include "rdc_lib/impl/RdcCacheManagerImpl.h"
+#include "rdc_lib/impl/RdcWatchTableImpl.h"
 #include "rdc_lib/rdc_common.h"
 #include "rocm_smi/rocm_smi.h"

@@ -49,12 +52,24 @@ amd::rdc::RdcHandler *make_handler(rdc_operation_mode_t op_mode) {
 namespace amd {
 namespace rdc {

+// TODO(bill_liu): make it configurable
+const uint32_t METIC_UPDATE_FREQUENCY = 100;  // 100ms by default

-RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode) :
+RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode):
    group_settings_(new RdcGroupSettingsImpl())
-    , metric_fetcher_(new RdcMetricFetcherImpl()) {
-    // TODO(bill_liu): implement the operation mode
-    (void)(mode);
+    , cache_mgr_(new RdcCacheManagerImpl())
+    , metric_fetcher_(new RdcMetricFetcherImpl())
+    , watch_table_(new RdcWatchTableImpl(group_settings_,
+                cache_mgr_, metric_fetcher_))
+    , metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_,
+                        METIC_UPDATE_FREQUENCY)) {
+    if (mode == RDC_OPERATION_MODE_AUTO) {
+        metrics_updater_->start();
+    }
+}
+
+RdcEmbeddedHandler::~RdcEmbeddedHandler() {
+     metrics_updater_->stop();
 }

 // JOB API
@@ -122,7 +137,7 @@ rdc_status_t RdcEmbeddedHandler::rdc_get_device_attributes(uint32_t gpu_index,
 }


-    // Group API
+// Group API
 rdc_status_t RdcEmbeddedHandler::rdc_group_gpu_create(rdc_group_type_t type,
                const char* group_name,
                rdc_gpu_group_t* p_rdc_group_id) {
@@ -144,6 +159,18 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_create(uint32_t num_field_ids,
    if (!field_group_name || !rdc_field_group_id || !field_ids) {
        return RDC_ST_BAD_PARAMETER;
    }
+
+    // Check the field is valid or not
+    if (num_field_ids <= RDC_MAX_FIELD_IDS_PER_FIELD_GROUP) {
+        for (uint32_t i = 0; i < num_field_ids; i++) {
+            if (!metric_fetcher_->is_field_valid(field_ids[i])) {
+                return RDC_ST_NOT_SUPPORTED;
+            }
+        }
+    } else {
+        return RDC_ST_MAX_LIMIT;
+    }
+
    return group_settings_->rdc_group_field_create(
        num_field_ids, field_ids, field_group_name, rdc_field_group_id);
 }
@@ -183,56 +210,45 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_destroy(
 rdc_status_t RdcEmbeddedHandler::rdc_watch_fields(rdc_gpu_group_t group_id,
        rdc_field_grp_t field_group_id, uint64_t update_freq,
        double max_keep_age, uint32_t max_keep_samples) {
-    // TODO(bill_liu): implement
-    (void)(group_id);
-    (void)(field_group_id);
-    (void)(update_freq);
-    (void)(max_keep_age);
-    (void)(max_keep_samples);
-    return RDC_ST_OK;
+    return watch_table_->rdc_watch_fields(group_id, field_group_id,
+                update_freq, max_keep_age, max_keep_samples);
 }

 rdc_status_t RdcEmbeddedHandler::rdc_get_latest_value_for_field(
    uint32_t gpu_index, uint32_t field, rdc_field_value* value) {
-    // TODO(bill_liu): implement
    if (!value) {
        return RDC_ST_BAD_PARAMETER;
    }
-    (void)(gpu_index);
-    (void)(field);
-    return RDC_ST_NOT_FOUND;
+    if (!metric_fetcher_->is_field_valid(field)) {
+        return RDC_ST_NOT_SUPPORTED;
+    }
+    return cache_mgr_->rdc_get_latest_value_for_field(gpu_index, field, value);
 }

 rdc_status_t RdcEmbeddedHandler::rdc_get_field_value_since(uint32_t gpu_index,
        uint32_t field, uint64_t since_time_stamp,
        uint64_t *next_since_time_stamp, rdc_field_value* value) {
-    // TODO(bill_liu): implement
    if (!next_since_time_stamp || !value) {
        return RDC_ST_BAD_PARAMETER;
    }
-    (void)(since_time_stamp);
-    (void)(gpu_index);
-    (void)(field);
-    (void)(value);
-
-    return RDC_ST_NOT_FOUND;
+    if (!metric_fetcher_->is_field_valid(field)) {
+        return RDC_ST_NOT_SUPPORTED;
+    }
+    return cache_mgr_->rdc_get_field_value_since(gpu_index, field,
+                since_time_stamp, next_since_time_stamp, value);
 }

 rdc_status_t RdcEmbeddedHandler::rdc_unwatch_fields(rdc_gpu_group_t group_id,
        rdc_field_grp_t field_group_id) {
-    // TODO(bill_liu): implement
-    (void)(group_id);
-    (void)(field_group_id);
-    return RDC_ST_OK;
+    return watch_table_->rdc_unwatch_fields(group_id, field_group_id);
 }

-
 // Control API
 rdc_status_t RdcEmbeddedHandler::rdc_update_all_fields(
    uint32_t wait_for_update) {
-    // TODO(bill_liu): implement
+    // TODO(bill_liu): implement the case wait_for_update==0
    (void)(wait_for_update);
-    return RDC_ST_OK;
+    return watch_table_->rdc_update_all_fields();
 }

 }  // namespace rdc
@@ -30,8 +30,7 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl() {
 }

 rdc_status_t RdcGroupSettingsImpl::rdc_group_gpu_create(rdc_group_type_t type,
-                  const char* group_name, rdc_gpu_group_t* p_rdc_group_id)
-{
+                  const char* group_name, rdc_gpu_group_t* p_rdc_group_id) {
    // TODO(bill_liu): handle type to create default group for all GPUs
    if (type == RDC_GROUP_DEFAULT) {
        return RDC_ST_NOT_SUPPORTED;
@@ -23,6 +23,7 @@ THE SOFTWARE.
 #include <sys/time.h>
 #include <string.h>
 #include <chrono>
+#include <algorithm>
 #include <vector>
 #include "rdc_lib/rdc_common.h"
 #include "rocm_smi/rocm_smi.h"
@@ -30,6 +31,16 @@ THE SOFTWARE.
 namespace amd {
 namespace rdc {

+bool RdcMetricFetcherImpl::is_field_valid(uint32_t field_id) const {
+     const std::vector<uint32_t> all_fields = {RDC_FI_GPU_MEMORY_USAGE,
+     RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_GPU_COUNT, RDC_FI_POWER_USAGE,
+     RDC_FI_GPU_SM_CLOCK, RDC_FI_GPU_UTIL, RDC_FI_DEV_NAME};
+
+     return std::find(all_fields.begin(), all_fields.end(), field_id)
+               != all_fields.end();
+
+}
+
 rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index,
         uint32_t field_id, rdc_field_value* value) {
    if (!value) {
@@ -37,6 +48,10 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index,
    }
    uint64_t i64 = 0;

+    if (!is_field_valid(field_id)) {
+         return RDC_ST_NOT_SUPPORTED;
+    }
+
    struct timeval  tv;
    gettimeofday(&tv, NULL);
    value->ts = static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
@@ -0,0 +1,57 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
+#include <sys/time.h>
+#include <ctime>
+#include <chrono>
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcMetricsUpdaterImpl::RdcMetricsUpdaterImpl(
+            const RdcWatchTablePtr& watch_table,
+            const uint32_t check_frequency):
+            watch_table_(watch_table)
+            , started_(false)
+            , _check_frequency(check_frequency) {
+}
+
+void RdcMetricsUpdaterImpl::start() {
+    if (started_) {
+        return;
+    }
+    started_ = true;
+    updater_ = std::async(std::launch::async, [this](){
+        while (started_) {
+            watch_table_->rdc_update_all_fields();
+            std::this_thread::sleep_for(std::chrono::microseconds(100));
+        }
+    });
+}
+
+void RdcMetricsUpdaterImpl::stop() {
+    started_ = false;
+}
+
+}  // namespace rdc
+}  // namespace amd
@@ -0,0 +1,309 @@
+/*
+Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "rdc_lib/impl/RdcWatchTableImpl.h"
+#include <sys/time.h>
+#include <ctime>
+#include <algorithm>
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcWatchTableImpl::RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings,
+        const RdcCacheManagerPtr& cache_mgr,
+        const RdcMetricFetcherPtr& metric_fetcher):
+     group_settings_(group_settings)
+    , cache_mgr_(cache_mgr)
+    , metric_fetcher_(metric_fetcher)
+    , last_cleanup_time_(0) {
+}
+
+rdc_status_t  RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id,
+                char  job_id[64]) {
+    // TODO(bill_liu): implement
+    (void)(group_id);
+    (void)(job_id);
+    return RDC_ST_OK;
+}
+
+rdc_status_t RdcWatchTableImpl::rdc_watch_job_fields(rdc_gpu_group_t group_id,
+                uint64_t update_freq, double  max_keep_age,
+                uint32_t max_keep_samples) {
+    // TODO(bill_liu): implement
+    (void)(group_id);
+    (void)(update_freq);
+    (void)(max_keep_age);
+    (void)(max_keep_samples);
+    return RDC_ST_OK;
+}
+
+rdc_status_t RdcWatchTableImpl::get_fields_from_group(rdc_gpu_group_t group_id,
+    rdc_field_grp_t field_group_id, std::vector<RdcFieldKey> & fields) {
+    rdc_field_group_info_t finfo;
+    rdc_group_info_t ginfo;
+    rdc_status_t result = group_settings_->
+                    rdc_group_gpu_get_info(group_id, &ginfo);
+    if (result != RDC_ST_OK) {
+        return result;
+    }
+
+    result = group_settings_->rdc_group_field_get_info(field_group_id, &finfo);
+    if (result != RDC_ST_OK) {
+        return result;
+    }
+
+    for (uint32_t i = 0 ; i < ginfo.count; i++) {  // GPUs
+        for (uint32_t j = 0; j < finfo.count; j++) {  // Fields
+            fields.push_back(RdcFieldKey({ginfo.entity_ids[i],
+                        finfo.field_ids[j]}));
+        }
+    }
+
+    return RDC_ST_OK;
+}
+
+
+rdc_status_t RdcWatchTableImpl::rdc_watch_fields(rdc_gpu_group_t group_id,
+        rdc_field_grp_t field_group_id, uint64_t update_freq,
+        double  max_keep_age, uint32_t max_keep_samples) {
+    std::lock_guard<std::mutex> guard(watch_mutex_);
+    RdcFieldKey gkey({group_id, field_group_id});
+    auto table_iter = watch_table_.find(gkey);
+
+    // Already in the watch table
+    if (table_iter != watch_table_.end()) {
+        if (table_iter->second.is_watching) {
+            return RDC_ST_CONFLICT;
+        } else {  // delete to overwrite
+            watch_table_.erase(table_iter);
+        }
+    }
+
+    // The field settings for this watch
+    FieldSettings f;
+    f.update_freq = update_freq;
+    f.max_keep_age = max_keep_age;
+    f.max_keep_samples = max_keep_samples;
+    f.last_update_time = 0;
+    f.is_watching = true;
+
+
+    // Get individual fields for the watch
+    std::vector<RdcFieldKey> fields_in_watch;
+    rdc_status_t result = get_fields_from_group(group_id,
+                            field_group_id, fields_in_watch);
+    if (result != RDC_ST_OK) {
+        return result;
+    }
+
+    // Update the fields_to_watch_
+    auto f_in_watch_iter = fields_in_watch.begin();
+    for (; f_in_watch_iter != fields_in_watch.end(); f_in_watch_iter++) {
+       auto ite = fields_to_watch_.find(*f_in_watch_iter);
+       if (ite == fields_to_watch_.end()) {  // A new field
+          fields_to_watch_.insert({*f_in_watch_iter, f});
+       } else {  // Merge the settings
+          auto& f_in_table = ite->second;
+          f_in_table.max_keep_age =
+                std::max(f_in_table.max_keep_age, max_keep_age);
+          f_in_table.max_keep_samples =
+                std::max(f_in_table.max_keep_samples, max_keep_samples);
+          if (f_in_table.is_watching) {  // Already watching
+              f_in_table.update_freq =
+                std::min(f_in_table.update_freq, update_freq);
+          } else {  // Not watching before
+              f_in_table.is_watching = true;
+              f_in_table.update_freq = update_freq;
+          }
+       }
+    }
+
+    // Add to the watch table
+    watch_table_.insert({gkey, f});
+
+    return RDC_ST_OK;
+}
+
+rdc_status_t RdcWatchTableImpl::update_field_in_table_when_unwatch(
+                    const RdcFieldKey& entry) {
+    // Get individual fields for this unwatch
+    std::vector<RdcFieldKey> fields;
+    rdc_status_t result = get_fields_from_group(
+        entry.first, entry.second, fields);
+    if (result != RDC_ST_OK) {
+        return result;
+    }
+
+    // Unwatch will only impact the update_freq, but not the max_keep_age
+    // and max_keep_samples. Walk through  watch_table_ to get new update
+    // frequency for all fields and store it in update_frequencies
+    std::map<RdcFieldKey, uint64_t> update_frequencies;
+    auto w_iter = watch_table_.begin();
+    for (; w_iter != watch_table_.end(); w_iter++) {
+        // Skip the table is not in watching status
+        if (w_iter->second.is_watching == false) {
+            continue;
+        }
+
+        // Get all fields in current table
+        std::vector<RdcFieldKey> watch_fields;
+        result = get_fields_from_group(w_iter->first.first,
+                    w_iter->first.second, watch_fields);
+        if (result != RDC_ST_OK) {
+            return result;
+        }
+
+        // Get the update_freq
+        auto fields_in_table_iter = watch_fields.begin();
+        for (; fields_in_table_iter != watch_fields.end();
+                        fields_in_table_iter++) {
+            auto f_in_freq_iter = update_frequencies.find(
+                    *fields_in_table_iter);
+            if (f_in_freq_iter == update_frequencies.end()) {
+                update_frequencies.insert(
+                    {*fields_in_table_iter, w_iter->second.update_freq});
+            } else {
+                f_in_freq_iter->second =
+                    std::min(f_in_freq_iter->second,
+                    w_iter->second.update_freq);
+            }
+        }
+    }
+
+    // Update the fields that impacted by this unwatch
+    auto fite = fields.begin();
+    for (; fite != fields.end(); fite++) {
+         auto f_in_table = fields_to_watch_.find((*fite));
+         if (f_in_table == fields_to_watch_.end()) {  // Not in fields_to_watch_
+            continue;
+         }
+
+         auto freq_iter = update_frequencies.find(*fite);
+         if (freq_iter == update_frequencies.end()) {
+             f_in_table->second.is_watching = false;
+         } else {
+             f_in_table->second.update_freq = freq_iter->second;
+         }
+    }
+
+    return RDC_ST_OK;
+}
+
+rdc_status_t RdcWatchTableImpl::rdc_unwatch_fields(
+        rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id) {
+    struct timeval  tv;
+    gettimeofday(&tv, NULL);
+    uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
+
+    std::lock_guard<std::mutex> guard(watch_mutex_);
+    // Set is_watching = false
+    auto ite = watch_table_.find(RdcFieldKey({group_id, field_group_id}));
+    if (ite == watch_table_.end()) {
+        return RDC_ST_NOT_FOUND;
+    }
+    ite->second.is_watching = false;
+    ite->second.last_update_time = now;
+
+    // Update the fields_to_watch_
+    return update_field_in_table_when_unwatch(ite->first);
+}
+
+
+rdc_status_t RdcWatchTableImpl::rdc_update_all_fields() {
+    uint32_t items_fetched = 0;
+    rdc_status_t result;
+
+    struct timeval  tv;
+    gettimeofday(&tv, NULL);
+    uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
+
+    std::lock_guard<std::mutex> guard(watch_mutex_);
+    auto fite = fields_to_watch_.begin();
+    for (; fite != fields_to_watch_.end(); fite++) {
+        // Is this field need to be updated?
+        if (!fite->second.is_watching ||
+            fite->second.last_update_time+fite->second.update_freq/1000 > now) {
+           continue;
+        }
+
+        // Fetch the metric from rocm_smi_lib
+        rdc_field_value value;
+        result = metric_fetcher_->fetch_smi_field(
+                    fite->first.first, fite->first.second, &value);
+        if (result != RDC_ST_OK) {
+           LOG_DEBUG("Fail to fetch the field: " << rdc_status_string(result));
+           continue;
+        }
+
+        // Update the cache
+        cache_mgr_->rdc_update_cache(fite->first.first, value);
+
+        // Update the last_upate_time
+        gettimeofday(&tv, NULL);
+        now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
+        fite->second.last_update_time = now;
+
+        items_fetched++;
+    }
+
+    // Clean up is expensive, only do it once per second
+    if (now - last_cleanup_time_ >1000) {
+        clean_up();
+        last_cleanup_time_ = now;
+    }
+
+    return RDC_ST_OK;
+}
+
+void RdcWatchTableImpl::clean_up() {
+    struct timeval  tv;
+    gettimeofday(&tv, NULL);
+    uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
+
+    // Clean the cache and the fields_to_watch_ table
+    auto fite = fields_to_watch_.begin();
+    while (fite != fields_to_watch_.end()) {
+        cache_mgr_->evict_cache(fite->first.first, fite->first.second,
+                fite->second.max_keep_samples, fite->second.max_keep_age);
+        if (!fite->second.is_watching && fite->second.last_update_time +
+                        fite->second.max_keep_age*1000 < now ) {
+            fite = fields_to_watch_.erase(fite);
+        } else {
+            ++fite;
+        }
+    }
+
+    // Clean the watch table
+    auto wite = watch_table_.begin();
+    while (wite != watch_table_.end()) {
+        if (!wite->second.is_watching && wite->second.last_update_time +
+                    wite->second.max_keep_age*1000 < now ) {
+            wite = watch_table_.erase(wite);
+        } else {
+            ++wite;
+        }
+    }
+}
+
+}  // namespace rdc
+}  // namespace amd