diff --git a/projects/rdc/example/CMakeLists.txt b/projects/rdc/example/CMakeLists.txt index 1ffda1b83f..fa8dc2c740 100755 --- a/projects/rdc/example/CMakeLists.txt +++ b/projects/rdc/example/CMakeLists.txt @@ -50,7 +50,7 @@ set(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR} ## Compiler flags set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -m64") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -msse2 -std=c++11 ") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse -msse2 -std=c++11 -pthread ") # Use this instead of above for 32 bit # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32") diff --git a/projects/rdc/include/rdc/rdc.h b/projects/rdc/include/rdc/rdc.h index 3b54e7abe8..3a43e862fc 100644 --- a/projects/rdc/include/rdc/rdc.h +++ b/projects/rdc/include/rdc/rdc.h @@ -50,6 +50,7 @@ typedef enum { RDC_ST_INVALID_HANDLER, //!< Invalid handler RDC_ST_BAD_PARAMETER, //!< A parameter is invalid RDC_ST_NOT_FOUND, //!< Cannot find the value + RDC_ST_CONFLICT, //!< Conflict with current state RDC_ST_MAX_LIMIT //!< Max limit recording for the object } rdc_status_t; diff --git a/projects/rdc/include/rdc_lib/RdcCacheManager.h b/projects/rdc/include/rdc_lib/RdcCacheManager.h new file mode 100644 index 0000000000..34f41461de --- /dev/null +++ b/projects/rdc/include/rdc_lib/RdcCacheManager.h @@ -0,0 +1,59 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_RDCCACHEMANAGER_H_ +#define RDC_LIB_RDCCACHEMANAGER_H_ + +#include +#include +#include +#include +#include "rdc_lib/rdc_common.h" +#include "rdc/rdc.h" + +namespace amd { +namespace rdc { + +class RdcCacheManager { + public: + virtual rdc_status_t rdc_get_latest_value_for_field(uint32_t gpu_index, + uint32_t field, rdc_field_value* value) = 0; + virtual rdc_status_t rdc_get_field_value_since(uint32_t gpu_index, + uint32_t field, uint64_t since_time_stamp, + uint64_t *next_since_time_stamp, rdc_field_value* value) = 0; + virtual rdc_status_t rdc_update_cache(uint32_t gpu_index, + const rdc_field_value& value) = 0; + virtual rdc_status_t evict_cache(uint32_t gpu_index, uint32_t field_id, + uint64_t max_keep_samples, double max_keep_age) = 0; + virtual uint32_t get_cache_size() = 0; + + virtual ~RdcCacheManager() {} +}; + +typedef std::shared_ptr RdcCacheManagerPtr; + +// +typedef std::pair RdcFieldKey; + +} // namespace rdc +} // namespace amd + +#endif // RDC_LIB_RDCCACHEMANAGER_H_ diff --git a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h index 3f8ad0b493..c4c0cd6122 100644 --- a/projects/rdc/include/rdc_lib/RdcMetricFetcher.h +++ b/projects/rdc/include/rdc_lib/RdcMetricFetcher.h @@ -34,6 +34,7 @@ class RdcMetricFetcher { public: virtual rdc_status_t fetch_smi_field(uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) = 0; + virtual bool is_field_valid(uint32_t field_id) const = 0; virtual ~RdcMetricFetcher() {} }; diff --git a/projects/rdc/include/rdc_lib/RdcMetricsUpdater.h b/projects/rdc/include/rdc_lib/RdcMetricsUpdater.h new file mode 100644 index 0000000000..d724c483e5 --- /dev/null +++ b/projects/rdc/include/rdc_lib/RdcMetricsUpdater.h @@ -0,0 +1,42 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_RDCMETRICSUPDATER_H_ +#define RDC_LIB_RDCMETRICSUPDATER_H_ + +#include + +namespace amd { +namespace rdc { + +class RdcMetricsUpdater { + public: + virtual void start() = 0; + virtual void stop() = 0; +}; + +typedef std::shared_ptr RdcMetricsUpdaterPtr; + +} // namespace rdc +} // namespace amd + + +#endif // RDC_LIB_RDCMETRICSUPDATER_H_ diff --git a/projects/rdc/include/rdc_lib/RdcWatchTable.h b/projects/rdc/include/rdc_lib/RdcWatchTable.h new file mode 100644 index 0000000000..595eb07672 --- /dev/null +++ b/projects/rdc/include/rdc_lib/RdcWatchTable.h @@ -0,0 +1,58 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_RDCWATCHTABLE_H_ +#define RDC_LIB_RDCWATCHTABLE_H_ + +#include +#include +#include "rdc_lib/rdc_common.h" +#include "rdc/rdc.h" + + +namespace amd { +namespace rdc { + +class RdcWatchTable { + public: + virtual rdc_status_t rdc_update_all_fields() = 0; + + virtual rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id, + char job_id[64]) = 0; + virtual rdc_status_t rdc_watch_job_fields(rdc_gpu_group_t group_id, + uint64_t update_freq, double max_keep_age, + uint32_t max_keep_samples) = 0; + + virtual rdc_status_t rdc_watch_fields(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id, uint64_t update_freq, + double max_keep_age, uint32_t max_keep_samples) = 0; + virtual rdc_status_t rdc_unwatch_fields(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id) = 0; + + virtual ~RdcWatchTable() {} +}; + +typedef std::shared_ptr RdcWatchTablePtr; + +} // namespace rdc +} // namespace amd + +#endif // RDC_LIB_RDCWATCHTABLE_H_ diff --git a/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h b/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h new file mode 100644 index 0000000000..c4445076c9 --- /dev/null +++ b/projects/rdc/include/rdc_lib/impl/RdcCacheManagerImpl.h @@ -0,0 +1,67 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_ +#define RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_ + +#include +#include +#include +#include +#include "rdc_lib/RdcCacheManager.h" +#include "rdc_lib/rdc_common.h" +#include "rdc/rdc.h" + + +namespace amd { +namespace rdc { + +struct RdcCacheEntry { + uint64_t last_time; + int64_t value; +}; + +typedef std::map> RdcCacheSamples; + + +class RdcCacheManagerImpl: public RdcCacheManager { + public: + rdc_status_t rdc_get_latest_value_for_field(uint32_t gpu_index, + uint32_t field, rdc_field_value* value) override; + rdc_status_t rdc_get_field_value_since(uint32_t gpu_index, uint32_t field, + uint64_t since_time_stamp, uint64_t *next_since_time_stamp, + rdc_field_value* value) override; + rdc_status_t rdc_update_cache(uint32_t gpu_index, + const rdc_field_value& value) override; + rdc_status_t evict_cache(uint32_t gpu_index, uint32_t field_id, + uint64_t max_keep_samples, double max_keep_age) override; + uint32_t get_cache_size() override; + + private: + RdcCacheSamples cache_samples_; + std::mutex cache_mutex_; +}; + +} // namespace rdc +} // namespace amd + + +#endif // RDC_LIB_IMPL_RDCCACHEMANAGERIMPL_H_ diff --git a/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h b/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h index 72638f855c..28ca4ea54b 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h +++ b/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h @@ -25,6 +25,9 @@ THE SOFTWARE. #include "rdc_lib/RdcHandler.h" #include "rdc_lib/RdcGroupSettings.h" #include "rdc_lib/RdcMetricFetcher.h" +#include "rdc_lib/RdcCacheManager.h" +#include "rdc_lib/RdcMetricsUpdater.h" +#include "rdc_lib/RdcWatchTable.h" namespace amd { namespace rdc { @@ -80,10 +83,14 @@ class RdcEmbeddedHandler: public RdcHandler { rdc_status_t rdc_update_all_fields(uint32_t wait_for_update) override; explicit RdcEmbeddedHandler(rdc_operation_mode_t op_mode); + ~RdcEmbeddedHandler(); private: RdcGroupSettingsPtr group_settings_; + RdcCacheManagerPtr cache_mgr_; RdcMetricFetcherPtr metric_fetcher_; + RdcWatchTablePtr watch_table_; + RdcMetricsUpdaterPtr metrics_updater_; }; } // namespace rdc diff --git a/projects/rdc/include/rdc_lib/impl/RdcGroupSettingsImpl.h b/projects/rdc/include/rdc_lib/impl/RdcGroupSettingsImpl.h index 5470bb38f8..4d5e537383 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcGroupSettingsImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcGroupSettingsImpl.h @@ -61,7 +61,6 @@ class RdcGroupSettingsImpl: public RdcGroupSettings { uint32_t cur_filed_group_id_ = 0; std::mutex group_mutex_; std::mutex field_group_mutex_; - }; } // namespace rdc diff --git a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h index d82c06f9b9..cac7ffc0e4 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h +++ b/projects/rdc/include/rdc_lib/impl/RdcMetricFetcherImpl.h @@ -31,6 +31,7 @@ class RdcMetricFetcherImpl: public RdcMetricFetcher { public: rdc_status_t fetch_smi_field(uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) override; + bool is_field_valid(uint32_t field_id) const override; }; } // namespace rdc diff --git a/projects/rdc/include/rdc_lib/impl/RdcMetricsUpdaterImpl.h b/projects/rdc/include/rdc_lib/impl/RdcMetricsUpdaterImpl.h new file mode 100644 index 0000000000..8612300db6 --- /dev/null +++ b/projects/rdc/include/rdc_lib/impl/RdcMetricsUpdaterImpl.h @@ -0,0 +1,49 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_ +#define RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_ + +#include +#include +#include "rdc_lib/RdcMetricsUpdater.h" +#include "rdc_lib/RdcWatchTable.h" + +namespace amd { +namespace rdc { + +class RdcMetricsUpdaterImpl: public RdcMetricsUpdater { + public: + void start() override; + void stop() override; + explicit RdcMetricsUpdaterImpl(const RdcWatchTablePtr& watch_table, + const uint32_t check_frequency); + private: + RdcWatchTablePtr watch_table_; + std::atomic started_; + std::future updater_; // keep the future of updater + const uint32_t _check_frequency; // Check frequency in milliseconds +}; + +} // namespace rdc +} // namespace amd + +#endif // RDC_LIB_IMPL_RDCMETRICSUPDATERIMPL_H_ diff --git a/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h new file mode 100644 index 0000000000..7a4fb39c64 --- /dev/null +++ b/projects/rdc/include/rdc_lib/impl/RdcWatchTableImpl.h @@ -0,0 +1,115 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_ +#define RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_ + +#include +#include +#include +#include +#include +#include +#include "rdc_lib/RdcWatchTable.h" +#include "rdc_lib/RdcGroupSettings.h" +#include "rdc_lib/RdcCacheManager.h" +#include "rdc_lib/RdcMetricFetcher.h" + +namespace amd { +namespace rdc { + +//!< The settings for a field or a group of field in the watch table. +struct FieldSettings { + uint64_t update_freq; + uint32_t max_keep_samples; + double max_keep_age; + bool is_watching; + uint64_t last_update_time; +}; + + +class RdcWatchTableImpl : public RdcWatchTable { + public: + rdc_status_t rdc_job_start_stats(rdc_gpu_group_t group_id, + char job_id[64]) override; + rdc_status_t rdc_watch_job_fields(rdc_gpu_group_t group_id, + uint64_t update_freq, double max_keep_age, + uint32_t max_keep_samples) override; + + rdc_status_t rdc_watch_fields(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id, uint64_t update_freq, + double max_keep_age, uint32_t max_keep_samples) override; + + //!< rdc_unwatch_fields() will not remove the entry from watch_table. + //!< The unwatched entry is still kept until the max_keep_age of the entry + //!< is reached, which will be handled in the clean_up() function. + rdc_status_t rdc_unwatch_fields(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id) override; + + //!< When the RDC is running as RDC_OPERATION_MODE_MANUAL, the user will + //!< call this function periodically. Instead of providing other APIs to + //!< cleanup the cache, this function will update and cleanup the cache. + //!< + //!< This function may be called very frequently, and the cache cleanup + //!< is expensive. Internally, this function will throttle the cleanup to + //!< once per second. + rdc_status_t rdc_update_all_fields() override; + + RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings, + const RdcCacheManagerPtr& cache_mgr, + const RdcMetricFetcherPtr& metric_fetcher); + + private: + //!< Helper function to Update the fields_in_table when unwatch tables + rdc_status_t update_field_in_table_when_unwatch(const RdcFieldKey& entry); + + //!< Helper function to clean up the watch table and cache + void clean_up(); + + //!< Helper function to get the fields using the group and the field group. + rdc_status_t get_fields_from_group(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id, std::vector & fields); + + + RdcGroupSettingsPtr group_settings_; + RdcCacheManagerPtr cache_mgr_; + RdcMetricFetcherPtr metric_fetcher_; + + //!< The watch table to store the watch settings. + std::map watch_table_; + + //!< The settings for each field can be deduced from watch_table. But every + //!< rdc_update_all_fields() call needs to deduce them. To improve the + //!< performance, the fields_to_watch_ is used to track the field settings. + //!< Those settings will only be updated when watching or unwatching. + std::map fields_to_watch_; + + //!< The last clean up time + std::atomic last_cleanup_time_; + + std::mutex watch_mutex_; +}; + +} // namespace rdc +} // namespace amd + + +#endif // RDC_LIB_IMPL_RDCWATCHTABLEIMPL_H_ diff --git a/projects/rdc/rdc_libs/CMakeLists.txt b/projects/rdc/rdc_libs/CMakeLists.txt index ce16dcc9fa..5b3103bb32 100755 --- a/projects/rdc/rdc_libs/CMakeLists.txt +++ b/projects/rdc/rdc_libs/CMakeLists.txt @@ -140,12 +140,22 @@ set(RDC_LIB_COMPONENT "lib${RDC_LIB}") set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcEmbeddedHandler.cc") set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricFetcherImpl.cc") set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcGroupSettingsImpl.cc") +set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcCacheManagerImpl.cc") +set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricsUpdaterImpl.cc") +set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcWatchTableImpl.cc") set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcEmbeddedHandler.h") set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricFetcher.h") set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricFetcherImpl.h") set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcGroupSettings.h") set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcGroupSettingsImpl.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcCacheManager.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcCacheManagerImpl.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricsUpdater.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricsUpdaterImpl.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcWatchTable.h") +set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcWatchTableImpl.h") + message("RDC_LIB_INC_LIST=${RDC_LIB_INC_LIST}") link_directories(${RSMI_LIB_DIR}) diff --git a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc index 1cbb97d21a..cac3805a35 100644 --- a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc +++ b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc @@ -318,6 +318,8 @@ const char* rdc_status_string(rdc_status_t result) { return "SMI error"; case RDC_ST_MAX_LIMIT: return "The max limit reached"; + case RDC_ST_CONFLICT: + return "Conflict with current state"; default: return "Unknown"; } diff --git a/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc new file mode 100644 index 0000000000..f9607fcabd --- /dev/null +++ b/projects/rdc/rdc_libs/rdc/src/RdcCacheManagerImpl.cc @@ -0,0 +1,166 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include "rdc_lib/impl/RdcCacheManagerImpl.h" +#include +#include +#include "rdc_lib/rdc_common.h" + + +namespace amd { +namespace rdc { + +rdc_status_t RdcCacheManagerImpl::rdc_get_field_value_since( + uint32_t gpu_index, uint32_t field_id, uint64_t since_time_stamp, + uint64_t *next_since_time_stamp, rdc_field_value* value) { + if (!next_since_time_stamp || !value) { + return RDC_ST_BAD_PARAMETER; + } + + std::lock_guard guard(cache_mutex_); + RdcFieldKey field{gpu_index, field_id}; + auto cache_samples_ite = cache_samples_.find(field); + if (cache_samples_ite == cache_samples_.end() || + cache_samples_ite->second.size() == 0) { + return RDC_ST_NOT_FOUND; + } + + // TODO(bill_liu): Optimize it using the binary search + auto cache_values = cache_samples_ite->second; + for (auto cache_value=cache_values.begin(); + cache_value != cache_values.end(); cache_value++) { + if ( cache_value->last_time >= since_time_stamp ) { + // move to next potential timestamp + auto next_iter = std::next(cache_value); + if (next_iter != cache_values.end()) { + *next_since_time_stamp = next_iter->last_time; + } else { // Last item, set it to the future by adding 1us + *next_since_time_stamp = cache_value->last_time + 1; + } + value->ts = cache_value->last_time; + value->type = INTEGER; + value->value.l_int = cache_value->value; + value->field_id = field_id; + return RDC_ST_OK; + } + } + + *next_since_time_stamp = since_time_stamp; + return RDC_ST_NOT_FOUND; +} + + +rdc_status_t RdcCacheManagerImpl::evict_cache(uint32_t gpu_index, + uint32_t field_id, uint64_t max_keep_samples, double max_keep_age) { + std::lock_guard guard(cache_mutex_); + + RdcFieldKey field{gpu_index, field_id}; + auto cache_samples_ite = cache_samples_.find(field); + if (cache_samples_ite == cache_samples_.end() || + cache_samples_ite->second.size() == 0) { + return RDC_ST_NOT_FOUND; + } + + // Check max_keep_samples + auto& cache_values = cache_samples_ite->second; + int item_remove = cache_values.size() - max_keep_samples; + if (item_remove > 0) { + cache_values.erase(cache_values.begin(), + cache_values.begin()+item_remove); + } + + // Check max_keep_age + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t now = static_cast(tv.tv_sec) * 1000 + tv.tv_usec / 1000; + + auto ite = cache_values.begin(); + while (ite != cache_values.end()) { + if (ite->last_time + max_keep_age*1000 >= now) { + break; + } else { + ite = cache_values.erase(ite); + } + } + + return RDC_ST_OK; +} + +rdc_status_t RdcCacheManagerImpl::rdc_get_latest_value_for_field( + uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) { + if (!value) { + return RDC_ST_BAD_PARAMETER; + } + + std::lock_guard guard(cache_mutex_); + RdcFieldKey field{gpu_index, field_id}; + auto cache_samples_ite = cache_samples_.find(field); + if (cache_samples_ite == cache_samples_.end() || + cache_samples_ite->second.size() == 0) { + return RDC_ST_NOT_FOUND; + } + + auto& cache_value = cache_samples_ite->second.back(); + value->ts = cache_value.last_time; + value->type = INTEGER; + value->value.l_int = cache_value.value; + value->field_id = field_id; + + return RDC_ST_OK; +} + +uint32_t RdcCacheManagerImpl::get_cache_size() { + uint32_t cache_size = 0; + std::lock_guard guard(cache_mutex_); + + auto cache_samples_ite = cache_samples_.begin(); + for (; cache_samples_ite != cache_samples_.end(); cache_samples_ite++) { + cache_size+=cache_samples_ite->second.size(); + } + return cache_size; +} + +rdc_status_t RdcCacheManagerImpl::rdc_update_cache(uint32_t gpu_index, + const rdc_field_value& value) { + RdcCacheEntry entry; + entry.last_time = value.ts; + if (value.type == INTEGER) { + entry.value = value.value.l_int; + } else { + return RDC_ST_NOT_SUPPORTED; + } + + std::lock_guard guard(cache_mutex_); + RdcFieldKey field{gpu_index, value.field_id}; + auto cache_samples_ite = cache_samples_.find(field); + if (cache_samples_ite == cache_samples_.end()) { + std::vector ve; + ve.push_back(entry); + cache_samples_.insert({field, ve}); + } else { + cache_samples_ite->second.push_back(entry); + } + + return RDC_ST_OK; +} + +} // namespace rdc +} // namespace amd diff --git a/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc b/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc index 7e6cff9248..089a77cbe6 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc @@ -23,6 +23,9 @@ THE SOFTWARE. #include #include "rdc_lib/impl/RdcMetricFetcherImpl.h" #include "rdc_lib/impl/RdcGroupSettingsImpl.h" +#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h" +#include "rdc_lib/impl/RdcCacheManagerImpl.h" +#include "rdc_lib/impl/RdcWatchTableImpl.h" #include "rdc_lib/rdc_common.h" #include "rocm_smi/rocm_smi.h" @@ -49,12 +52,24 @@ amd::rdc::RdcHandler *make_handler(rdc_operation_mode_t op_mode) { namespace amd { namespace rdc { +// TODO(bill_liu): make it configurable +const uint32_t METIC_UPDATE_FREQUENCY = 100; // 100ms by default -RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode) : +RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode): group_settings_(new RdcGroupSettingsImpl()) - , metric_fetcher_(new RdcMetricFetcherImpl()) { - // TODO(bill_liu): implement the operation mode - (void)(mode); + , cache_mgr_(new RdcCacheManagerImpl()) + , metric_fetcher_(new RdcMetricFetcherImpl()) + , watch_table_(new RdcWatchTableImpl(group_settings_, + cache_mgr_, metric_fetcher_)) + , metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, + METIC_UPDATE_FREQUENCY)) { + if (mode == RDC_OPERATION_MODE_AUTO) { + metrics_updater_->start(); + } +} + +RdcEmbeddedHandler::~RdcEmbeddedHandler() { + metrics_updater_->stop(); } // JOB API @@ -122,7 +137,7 @@ rdc_status_t RdcEmbeddedHandler::rdc_get_device_attributes(uint32_t gpu_index, } - // Group API +// Group API rdc_status_t RdcEmbeddedHandler::rdc_group_gpu_create(rdc_group_type_t type, const char* group_name, rdc_gpu_group_t* p_rdc_group_id) { @@ -144,6 +159,18 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_create(uint32_t num_field_ids, if (!field_group_name || !rdc_field_group_id || !field_ids) { return RDC_ST_BAD_PARAMETER; } + + // Check the field is valid or not + if (num_field_ids <= RDC_MAX_FIELD_IDS_PER_FIELD_GROUP) { + for (uint32_t i = 0; i < num_field_ids; i++) { + if (!metric_fetcher_->is_field_valid(field_ids[i])) { + return RDC_ST_NOT_SUPPORTED; + } + } + } else { + return RDC_ST_MAX_LIMIT; + } + return group_settings_->rdc_group_field_create( num_field_ids, field_ids, field_group_name, rdc_field_group_id); } @@ -183,56 +210,45 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_destroy( rdc_status_t RdcEmbeddedHandler::rdc_watch_fields(rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id, uint64_t update_freq, double max_keep_age, uint32_t max_keep_samples) { - // TODO(bill_liu): implement - (void)(group_id); - (void)(field_group_id); - (void)(update_freq); - (void)(max_keep_age); - (void)(max_keep_samples); - return RDC_ST_OK; + return watch_table_->rdc_watch_fields(group_id, field_group_id, + update_freq, max_keep_age, max_keep_samples); } rdc_status_t RdcEmbeddedHandler::rdc_get_latest_value_for_field( uint32_t gpu_index, uint32_t field, rdc_field_value* value) { - // TODO(bill_liu): implement if (!value) { return RDC_ST_BAD_PARAMETER; } - (void)(gpu_index); - (void)(field); - return RDC_ST_NOT_FOUND; + if (!metric_fetcher_->is_field_valid(field)) { + return RDC_ST_NOT_SUPPORTED; + } + return cache_mgr_->rdc_get_latest_value_for_field(gpu_index, field, value); } rdc_status_t RdcEmbeddedHandler::rdc_get_field_value_since(uint32_t gpu_index, uint32_t field, uint64_t since_time_stamp, uint64_t *next_since_time_stamp, rdc_field_value* value) { - // TODO(bill_liu): implement if (!next_since_time_stamp || !value) { return RDC_ST_BAD_PARAMETER; } - (void)(since_time_stamp); - (void)(gpu_index); - (void)(field); - (void)(value); - - return RDC_ST_NOT_FOUND; + if (!metric_fetcher_->is_field_valid(field)) { + return RDC_ST_NOT_SUPPORTED; + } + return cache_mgr_->rdc_get_field_value_since(gpu_index, field, + since_time_stamp, next_since_time_stamp, value); } rdc_status_t RdcEmbeddedHandler::rdc_unwatch_fields(rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id) { - // TODO(bill_liu): implement - (void)(group_id); - (void)(field_group_id); - return RDC_ST_OK; + return watch_table_->rdc_unwatch_fields(group_id, field_group_id); } - // Control API rdc_status_t RdcEmbeddedHandler::rdc_update_all_fields( uint32_t wait_for_update) { - // TODO(bill_liu): implement + // TODO(bill_liu): implement the case wait_for_update==0 (void)(wait_for_update); - return RDC_ST_OK; + return watch_table_->rdc_update_all_fields(); } } // namespace rdc diff --git a/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc index f95960a528..2631b2f374 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcGroupSettingsImpl.cc @@ -30,8 +30,7 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl() { } rdc_status_t RdcGroupSettingsImpl::rdc_group_gpu_create(rdc_group_type_t type, - const char* group_name, rdc_gpu_group_t* p_rdc_group_id) -{ + const char* group_name, rdc_gpu_group_t* p_rdc_group_id) { // TODO(bill_liu): handle type to create default group for all GPUs if (type == RDC_GROUP_DEFAULT) { return RDC_ST_NOT_SUPPORTED; diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc index aa24bf43b1..fa4b685634 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricFetcherImpl.cc @@ -23,6 +23,7 @@ THE SOFTWARE. #include #include #include +#include #include #include "rdc_lib/rdc_common.h" #include "rocm_smi/rocm_smi.h" @@ -30,6 +31,16 @@ THE SOFTWARE. namespace amd { namespace rdc { +bool RdcMetricFetcherImpl::is_field_valid(uint32_t field_id) const { + const std::vector all_fields = {RDC_FI_GPU_MEMORY_USAGE, + RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_GPU_COUNT, RDC_FI_POWER_USAGE, + RDC_FI_GPU_SM_CLOCK, RDC_FI_GPU_UTIL, RDC_FI_DEV_NAME}; + + return std::find(all_fields.begin(), all_fields.end(), field_id) + != all_fields.end(); + +} + rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) { if (!value) { @@ -37,6 +48,10 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, } uint64_t i64 = 0; + if (!is_field_valid(field_id)) { + return RDC_ST_NOT_SUPPORTED; + } + struct timeval tv; gettimeofday(&tv, NULL); value->ts = static_cast(tv.tv_sec) * 1000 + tv.tv_usec / 1000; diff --git a/projects/rdc/rdc_libs/rdc/src/RdcMetricsUpdaterImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcMetricsUpdaterImpl.cc new file mode 100644 index 0000000000..be5b40a397 --- /dev/null +++ b/projects/rdc/rdc_libs/rdc/src/RdcMetricsUpdaterImpl.cc @@ -0,0 +1,57 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h" +#include +#include +#include +#include "rdc_lib/rdc_common.h" + +namespace amd { +namespace rdc { + +RdcMetricsUpdaterImpl::RdcMetricsUpdaterImpl( + const RdcWatchTablePtr& watch_table, + const uint32_t check_frequency): + watch_table_(watch_table) + , started_(false) + , _check_frequency(check_frequency) { +} + +void RdcMetricsUpdaterImpl::start() { + if (started_) { + return; + } + started_ = true; + updater_ = std::async(std::launch::async, [this](){ + while (started_) { + watch_table_->rdc_update_all_fields(); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + }); +} + +void RdcMetricsUpdaterImpl::stop() { + started_ = false; +} + +} // namespace rdc +} // namespace amd diff --git a/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc new file mode 100644 index 0000000000..ece156a9f6 --- /dev/null +++ b/projects/rdc/rdc_libs/rdc/src/RdcWatchTableImpl.cc @@ -0,0 +1,309 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "rdc_lib/impl/RdcWatchTableImpl.h" +#include +#include +#include +#include "rdc_lib/rdc_common.h" + +namespace amd { +namespace rdc { + +RdcWatchTableImpl::RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings, + const RdcCacheManagerPtr& cache_mgr, + const RdcMetricFetcherPtr& metric_fetcher): + group_settings_(group_settings) + , cache_mgr_(cache_mgr) + , metric_fetcher_(metric_fetcher) + , last_cleanup_time_(0) { +} + +rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id, + char job_id[64]) { + // TODO(bill_liu): implement + (void)(group_id); + (void)(job_id); + return RDC_ST_OK; +} + +rdc_status_t RdcWatchTableImpl::rdc_watch_job_fields(rdc_gpu_group_t group_id, + uint64_t update_freq, double max_keep_age, + uint32_t max_keep_samples) { + // TODO(bill_liu): implement + (void)(group_id); + (void)(update_freq); + (void)(max_keep_age); + (void)(max_keep_samples); + return RDC_ST_OK; +} + +rdc_status_t RdcWatchTableImpl::get_fields_from_group(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id, std::vector & fields) { + rdc_field_group_info_t finfo; + rdc_group_info_t ginfo; + rdc_status_t result = group_settings_-> + rdc_group_gpu_get_info(group_id, &ginfo); + if (result != RDC_ST_OK) { + return result; + } + + result = group_settings_->rdc_group_field_get_info(field_group_id, &finfo); + if (result != RDC_ST_OK) { + return result; + } + + for (uint32_t i = 0 ; i < ginfo.count; i++) { // GPUs + for (uint32_t j = 0; j < finfo.count; j++) { // Fields + fields.push_back(RdcFieldKey({ginfo.entity_ids[i], + finfo.field_ids[j]})); + } + } + + return RDC_ST_OK; +} + + +rdc_status_t RdcWatchTableImpl::rdc_watch_fields(rdc_gpu_group_t group_id, + rdc_field_grp_t field_group_id, uint64_t update_freq, + double max_keep_age, uint32_t max_keep_samples) { + std::lock_guard guard(watch_mutex_); + RdcFieldKey gkey({group_id, field_group_id}); + auto table_iter = watch_table_.find(gkey); + + // Already in the watch table + if (table_iter != watch_table_.end()) { + if (table_iter->second.is_watching) { + return RDC_ST_CONFLICT; + } else { // delete to overwrite + watch_table_.erase(table_iter); + } + } + + // The field settings for this watch + FieldSettings f; + f.update_freq = update_freq; + f.max_keep_age = max_keep_age; + f.max_keep_samples = max_keep_samples; + f.last_update_time = 0; + f.is_watching = true; + + + // Get individual fields for the watch + std::vector fields_in_watch; + rdc_status_t result = get_fields_from_group(group_id, + field_group_id, fields_in_watch); + if (result != RDC_ST_OK) { + return result; + } + + // Update the fields_to_watch_ + auto f_in_watch_iter = fields_in_watch.begin(); + for (; f_in_watch_iter != fields_in_watch.end(); f_in_watch_iter++) { + auto ite = fields_to_watch_.find(*f_in_watch_iter); + if (ite == fields_to_watch_.end()) { // A new field + fields_to_watch_.insert({*f_in_watch_iter, f}); + } else { // Merge the settings + auto& f_in_table = ite->second; + f_in_table.max_keep_age = + std::max(f_in_table.max_keep_age, max_keep_age); + f_in_table.max_keep_samples = + std::max(f_in_table.max_keep_samples, max_keep_samples); + if (f_in_table.is_watching) { // Already watching + f_in_table.update_freq = + std::min(f_in_table.update_freq, update_freq); + } else { // Not watching before + f_in_table.is_watching = true; + f_in_table.update_freq = update_freq; + } + } + } + + // Add to the watch table + watch_table_.insert({gkey, f}); + + return RDC_ST_OK; +} + +rdc_status_t RdcWatchTableImpl::update_field_in_table_when_unwatch( + const RdcFieldKey& entry) { + // Get individual fields for this unwatch + std::vector fields; + rdc_status_t result = get_fields_from_group( + entry.first, entry.second, fields); + if (result != RDC_ST_OK) { + return result; + } + + // Unwatch will only impact the update_freq, but not the max_keep_age + // and max_keep_samples. Walk through watch_table_ to get new update + // frequency for all fields and store it in update_frequencies + std::map update_frequencies; + auto w_iter = watch_table_.begin(); + for (; w_iter != watch_table_.end(); w_iter++) { + // Skip the table is not in watching status + if (w_iter->second.is_watching == false) { + continue; + } + + // Get all fields in current table + std::vector watch_fields; + result = get_fields_from_group(w_iter->first.first, + w_iter->first.second, watch_fields); + if (result != RDC_ST_OK) { + return result; + } + + // Get the update_freq + auto fields_in_table_iter = watch_fields.begin(); + for (; fields_in_table_iter != watch_fields.end(); + fields_in_table_iter++) { + auto f_in_freq_iter = update_frequencies.find( + *fields_in_table_iter); + if (f_in_freq_iter == update_frequencies.end()) { + update_frequencies.insert( + {*fields_in_table_iter, w_iter->second.update_freq}); + } else { + f_in_freq_iter->second = + std::min(f_in_freq_iter->second, + w_iter->second.update_freq); + } + } + } + + // Update the fields that impacted by this unwatch + auto fite = fields.begin(); + for (; fite != fields.end(); fite++) { + auto f_in_table = fields_to_watch_.find((*fite)); + if (f_in_table == fields_to_watch_.end()) { // Not in fields_to_watch_ + continue; + } + + auto freq_iter = update_frequencies.find(*fite); + if (freq_iter == update_frequencies.end()) { + f_in_table->second.is_watching = false; + } else { + f_in_table->second.update_freq = freq_iter->second; + } + } + + return RDC_ST_OK; +} + +rdc_status_t RdcWatchTableImpl::rdc_unwatch_fields( + rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id) { + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t now = static_cast(tv.tv_sec)*1000+tv.tv_usec/1000; + + std::lock_guard guard(watch_mutex_); + // Set is_watching = false + auto ite = watch_table_.find(RdcFieldKey({group_id, field_group_id})); + if (ite == watch_table_.end()) { + return RDC_ST_NOT_FOUND; + } + ite->second.is_watching = false; + ite->second.last_update_time = now; + + // Update the fields_to_watch_ + return update_field_in_table_when_unwatch(ite->first); +} + + +rdc_status_t RdcWatchTableImpl::rdc_update_all_fields() { + uint32_t items_fetched = 0; + rdc_status_t result; + + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t now = static_cast(tv.tv_sec)*1000+tv.tv_usec/1000; + + std::lock_guard guard(watch_mutex_); + auto fite = fields_to_watch_.begin(); + for (; fite != fields_to_watch_.end(); fite++) { + // Is this field need to be updated? + if (!fite->second.is_watching || + fite->second.last_update_time+fite->second.update_freq/1000 > now) { + continue; + } + + // Fetch the metric from rocm_smi_lib + rdc_field_value value; + result = metric_fetcher_->fetch_smi_field( + fite->first.first, fite->first.second, &value); + if (result != RDC_ST_OK) { + LOG_DEBUG("Fail to fetch the field: " << rdc_status_string(result)); + continue; + } + + // Update the cache + cache_mgr_->rdc_update_cache(fite->first.first, value); + + // Update the last_upate_time + gettimeofday(&tv, NULL); + now = static_cast(tv.tv_sec)*1000+tv.tv_usec/1000; + fite->second.last_update_time = now; + + items_fetched++; + } + + // Clean up is expensive, only do it once per second + if (now - last_cleanup_time_ >1000) { + clean_up(); + last_cleanup_time_ = now; + } + + return RDC_ST_OK; +} + +void RdcWatchTableImpl::clean_up() { + struct timeval tv; + gettimeofday(&tv, NULL); + uint64_t now = static_cast(tv.tv_sec)*1000+tv.tv_usec/1000; + + // Clean the cache and the fields_to_watch_ table + auto fite = fields_to_watch_.begin(); + while (fite != fields_to_watch_.end()) { + cache_mgr_->evict_cache(fite->first.first, fite->first.second, + fite->second.max_keep_samples, fite->second.max_keep_age); + if (!fite->second.is_watching && fite->second.last_update_time + + fite->second.max_keep_age*1000 < now ) { + fite = fields_to_watch_.erase(fite); + } else { + ++fite; + } + } + + // Clean the watch table + auto wite = watch_table_.begin(); + while (wite != watch_table_.end()) { + if (!wite->second.is_watching && wite->second.last_update_time + + wite->second.max_keep_age*1000 < now ) { + wite = watch_table_.erase(wite); + } else { + ++wite; + } + } +} + +} // namespace rdc +} // namespace amd