AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SWDEV-223878 - Add cache manager and watch table to skeleton rdc_lib

Browse Source 
Support cache manager and watch table in rdc_lib

RdcCacheManagerImpl.cc is added to implement cache of metrics. Currently, only
integer mertics are supported. The cache manager provids function to retrieve the
latest and history metrics from cache. It also provides interfaces to update and evict the cache.

RdcWatchTableImpl.cc is added to implement watch and unwatch fields. It uses the
field settings to control how frequently a field needs to be updated. We have a preliminarily
performance optimization for this class as it may be called very frequently.

RdcMetricsUpdaterImpl.cc is added to run the update at background thread when
RDC_OPERATION_MODE_AUTO is set.

After this code change, the rdcd/rdci should be able to implement basic discovery, group and dmon
function. The job management function is not implemented in the skeleton rdc_lib yet.

Change-Id: I26cff8c2ec85d1ad8e7df24c66b02f0060838d37
This commit is contained in:
Bill(Shuzhou) Liu
2020-02-26 11:04:42 -05:00
committed by Chris Freehill
parent a5f063f8b3
commit 1ff1c7b617
20 changed files with 1007 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdc_libs/CMakeLists.txt
+10
View File
@@ -140,12 +140,22 @@ set(RDC_LIB_COMPONENT "lib${RDC_LIB}")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcEmbeddedHandler.cc")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricFetcherImpl.cc")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcGroupSettingsImpl.cc")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcCacheManagerImpl.cc")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcMetricsUpdaterImpl.cc")
set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST} "${SRC_DIR}/rdc/src/RdcWatchTableImpl.cc")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcEmbeddedHandler.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricFetcher.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricFetcherImpl.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcGroupSettings.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcGroupSettingsImpl.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcCacheManager.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcCacheManagerImpl.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcMetricsUpdater.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcMetricsUpdaterImpl.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcWatchTable.h")
set(RDC_LIB_INC_LIST ${RDC_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/impl/RdcWatchTableImpl.h")
message("RDC_LIB_INC_LIST=${RDC_LIB_INC_LIST}")
link_directories(${RSMI_LIB_DIR})
rdc_libs/bootstrap/src/RdcBootStrap.cc
+2
View File
@@ -318,6 +318,8 @@ const char* rdc_status_string(rdc_status_t result) {
return "SMI error";
case RDC_ST_MAX_LIMIT:
return "The max limit reached";
case RDC_ST_CONFLICT:
return "Conflict with current state";
default:
return "Unknown";
}
rdc_libs/rdc/src/RdcCacheManagerImpl.cc
+166
View File
@@ -0,0 +1,166 @@
/*
Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "rdc_lib/impl/RdcCacheManagerImpl.h"
#include <sys/time.h>
#include <ctime>
#include "rdc_lib/rdc_common.h"
namespace amd {
namespace rdc {
rdc_status_t RdcCacheManagerImpl::rdc_get_field_value_since(
uint32_t gpu_index, uint32_t field_id, uint64_t since_time_stamp,
uint64_t *next_since_time_stamp, rdc_field_value* value) {
if (!next_since_time_stamp || !value) {
return RDC_ST_BAD_PARAMETER;
}
std::lock_guard<std::mutex> guard(cache_mutex_);
RdcFieldKey field{gpu_index, field_id};
auto cache_samples_ite = cache_samples_.find(field);
if (cache_samples_ite == cache_samples_.end() ||
cache_samples_ite->second.size() == 0) {
return RDC_ST_NOT_FOUND;
}
// TODO(bill_liu): Optimize it using the binary search
auto cache_values = cache_samples_ite->second;
for (auto cache_value=cache_values.begin();
cache_value != cache_values.end(); cache_value++) {
if ( cache_value->last_time >= since_time_stamp ) {
// move to next potential timestamp
auto next_iter = std::next(cache_value);
if (next_iter != cache_values.end()) {
*next_since_time_stamp = next_iter->last_time;
} else { // Last item, set it to the future by adding 1us
*next_since_time_stamp = cache_value->last_time + 1;
}
value->ts = cache_value->last_time;
value->type = INTEGER;
value->value.l_int = cache_value->value;
value->field_id = field_id;
return RDC_ST_OK;
}
}
*next_since_time_stamp = since_time_stamp;
return RDC_ST_NOT_FOUND;
}
rdc_status_t RdcCacheManagerImpl::evict_cache(uint32_t gpu_index,
uint32_t field_id, uint64_t max_keep_samples, double max_keep_age) {
std::lock_guard<std::mutex> guard(cache_mutex_);
RdcFieldKey field{gpu_index, field_id};
auto cache_samples_ite = cache_samples_.find(field);
if (cache_samples_ite == cache_samples_.end() ||
cache_samples_ite->second.size() == 0) {
return RDC_ST_NOT_FOUND;
}
// Check max_keep_samples
auto& cache_values = cache_samples_ite->second;
int item_remove = cache_values.size() - max_keep_samples;
if (item_remove > 0) {
cache_values.erase(cache_values.begin(),
cache_values.begin()+item_remove);
}
// Check max_keep_age
struct timeval tv;
gettimeofday(&tv, NULL);
uint64_t now = static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
auto ite = cache_values.begin();
while (ite != cache_values.end()) {
if (ite->last_time + max_keep_age*1000 >= now) {
break;
} else {
ite = cache_values.erase(ite);
}
}
return RDC_ST_OK;
}
rdc_status_t RdcCacheManagerImpl::rdc_get_latest_value_for_field(
uint32_t gpu_index, uint32_t field_id, rdc_field_value* value) {
if (!value) {
return RDC_ST_BAD_PARAMETER;
}
std::lock_guard<std::mutex> guard(cache_mutex_);
RdcFieldKey field{gpu_index, field_id};
auto cache_samples_ite = cache_samples_.find(field);
if (cache_samples_ite == cache_samples_.end() ||
cache_samples_ite->second.size() == 0) {
return RDC_ST_NOT_FOUND;
}
auto& cache_value = cache_samples_ite->second.back();
value->ts = cache_value.last_time;
value->type = INTEGER;
value->value.l_int = cache_value.value;
value->field_id = field_id;
return RDC_ST_OK;
}
uint32_t RdcCacheManagerImpl::get_cache_size() {
uint32_t cache_size = 0;
std::lock_guard<std::mutex> guard(cache_mutex_);
auto cache_samples_ite = cache_samples_.begin();
for (; cache_samples_ite != cache_samples_.end(); cache_samples_ite++) {
cache_size+=cache_samples_ite->second.size();
}
return cache_size;
}
rdc_status_t RdcCacheManagerImpl::rdc_update_cache(uint32_t gpu_index,
const rdc_field_value& value) {
RdcCacheEntry entry;
entry.last_time = value.ts;
if (value.type == INTEGER) {
entry.value = value.value.l_int;
} else {
return RDC_ST_NOT_SUPPORTED;
}
std::lock_guard<std::mutex> guard(cache_mutex_);
RdcFieldKey field{gpu_index, value.field_id};
auto cache_samples_ite = cache_samples_.find(field);
if (cache_samples_ite == cache_samples_.end()) {
std::vector<RdcCacheEntry> ve;
ve.push_back(entry);
cache_samples_.insert({field, ve});
} else {
cache_samples_ite->second.push_back(entry);
}
return RDC_ST_OK;
}
} // namespace rdc
} // namespace amd
rdc_libs/rdc/src/RdcEmbeddedHandler.cc
+46 -30
View File
@@ -23,6 +23,9 @@ THE SOFTWARE.
#include <string.h>
#include "rdc_lib/impl/RdcMetricFetcherImpl.h"
#include "rdc_lib/impl/RdcGroupSettingsImpl.h"
#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
#include "rdc_lib/impl/RdcCacheManagerImpl.h"
#include "rdc_lib/impl/RdcWatchTableImpl.h"
#include "rdc_lib/rdc_common.h"
#include "rocm_smi/rocm_smi.h"
@@ -49,12 +52,24 @@ amd::rdc::RdcHandler *make_handler(rdc_operation_mode_t op_mode) {
namespace amd {
namespace rdc {
// TODO(bill_liu): make it configurable
const uint32_t METIC_UPDATE_FREQUENCY = 100; // 100ms by default
RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode) :
RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode):
group_settings_(new RdcGroupSettingsImpl())
, metric_fetcher_(new RdcMetricFetcherImpl()) {
// TODO(bill_liu): implement the operation mode
(void)(mode);
, cache_mgr_(new RdcCacheManagerImpl())
, metric_fetcher_(new RdcMetricFetcherImpl())
, watch_table_(new RdcWatchTableImpl(group_settings_,
cache_mgr_, metric_fetcher_))
, metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_,
METIC_UPDATE_FREQUENCY)) {
if (mode == RDC_OPERATION_MODE_AUTO) {
metrics_updater_->start();
}
}
RdcEmbeddedHandler::~RdcEmbeddedHandler() {
metrics_updater_->stop();
}
// JOB API
@@ -122,7 +137,7 @@ rdc_status_t RdcEmbeddedHandler::rdc_get_device_attributes(uint32_t gpu_index,
}
// Group API
// Group API
rdc_status_t RdcEmbeddedHandler::rdc_group_gpu_create(rdc_group_type_t type,
const char* group_name,
rdc_gpu_group_t* p_rdc_group_id) {
@@ -144,6 +159,18 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_create(uint32_t num_field_ids,
if (!field_group_name || !rdc_field_group_id || !field_ids) {
return RDC_ST_BAD_PARAMETER;
}
// Check the field is valid or not
if (num_field_ids <= RDC_MAX_FIELD_IDS_PER_FIELD_GROUP) {
for (uint32_t i = 0; i < num_field_ids; i++) {
if (!metric_fetcher_->is_field_valid(field_ids[i])) {
return RDC_ST_NOT_SUPPORTED;
}
}
} else {
return RDC_ST_MAX_LIMIT;
}
return group_settings_->rdc_group_field_create(
num_field_ids, field_ids, field_group_name, rdc_field_group_id);
}
@@ -183,56 +210,45 @@ rdc_status_t RdcEmbeddedHandler::rdc_group_field_destroy(
rdc_status_t RdcEmbeddedHandler::rdc_watch_fields(rdc_gpu_group_t group_id,
rdc_field_grp_t field_group_id, uint64_t update_freq,
double max_keep_age, uint32_t max_keep_samples) {
// TODO(bill_liu): implement
(void)(group_id);
(void)(field_group_id);
(void)(update_freq);
(void)(max_keep_age);
(void)(max_keep_samples);
return RDC_ST_OK;
return watch_table_->rdc_watch_fields(group_id, field_group_id,
update_freq, max_keep_age, max_keep_samples);
}
rdc_status_t RdcEmbeddedHandler::rdc_get_latest_value_for_field(
uint32_t gpu_index, uint32_t field, rdc_field_value* value) {
// TODO(bill_liu): implement
if (!value) {
return RDC_ST_BAD_PARAMETER;
}
(void)(gpu_index);
(void)(field);
return RDC_ST_NOT_FOUND;
if (!metric_fetcher_->is_field_valid(field)) {
return RDC_ST_NOT_SUPPORTED;
}
return cache_mgr_->rdc_get_latest_value_for_field(gpu_index, field, value);
}
rdc_status_t RdcEmbeddedHandler::rdc_get_field_value_since(uint32_t gpu_index,
uint32_t field, uint64_t since_time_stamp,
uint64_t *next_since_time_stamp, rdc_field_value* value) {
// TODO(bill_liu): implement
if (!next_since_time_stamp || !value) {
return RDC_ST_BAD_PARAMETER;
}
(void)(since_time_stamp);
(void)(gpu_index);
(void)(field);
(void)(value);
return RDC_ST_NOT_FOUND;
if (!metric_fetcher_->is_field_valid(field)) {
return RDC_ST_NOT_SUPPORTED;
}
return cache_mgr_->rdc_get_field_value_since(gpu_index, field,
since_time_stamp, next_since_time_stamp, value);
}
rdc_status_t RdcEmbeddedHandler::rdc_unwatch_fields(rdc_gpu_group_t group_id,
rdc_field_grp_t field_group_id) {
// TODO(bill_liu): implement
(void)(group_id);
(void)(field_group_id);
return RDC_ST_OK;
return watch_table_->rdc_unwatch_fields(group_id, field_group_id);
}
// Control API
rdc_status_t RdcEmbeddedHandler::rdc_update_all_fields(
uint32_t wait_for_update) {
// TODO(bill_liu): implement
// TODO(bill_liu): implement the case wait_for_update==0
(void)(wait_for_update);
return RDC_ST_OK;
return watch_table_->rdc_update_all_fields();
}
} // namespace rdc
rdc_libs/rdc/src/RdcGroupSettingsImpl.cc
+1 -2
View File
@@ -30,8 +30,7 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl() {
}
rdc_status_t RdcGroupSettingsImpl::rdc_group_gpu_create(rdc_group_type_t type,
const char* group_name, rdc_gpu_group_t* p_rdc_group_id)
{
const char* group_name, rdc_gpu_group_t* p_rdc_group_id) {
// TODO(bill_liu): handle type to create default group for all GPUs
if (type == RDC_GROUP_DEFAULT) {
return RDC_ST_NOT_SUPPORTED;
rdc_libs/rdc/src/RdcMetricFetcherImpl.cc
+15
View File
@@ -23,6 +23,7 @@ THE SOFTWARE.
#include <sys/time.h>
#include <string.h>
#include <chrono>
#include <algorithm>
#include <vector>
#include "rdc_lib/rdc_common.h"
#include "rocm_smi/rocm_smi.h"
@@ -30,6 +31,16 @@ THE SOFTWARE.
namespace amd {
namespace rdc {
bool RdcMetricFetcherImpl::is_field_valid(uint32_t field_id) const {
const std::vector<uint32_t> all_fields = {RDC_FI_GPU_MEMORY_USAGE,
RDC_FI_GPU_MEMORY_TOTAL, RDC_FI_GPU_COUNT, RDC_FI_POWER_USAGE,
RDC_FI_GPU_SM_CLOCK, RDC_FI_GPU_UTIL, RDC_FI_DEV_NAME};
return std::find(all_fields.begin(), all_fields.end(), field_id)
!= all_fields.end();
}
rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index,
uint32_t field_id, rdc_field_value* value) {
if (!value) {
@@ -37,6 +48,10 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index,
}
uint64_t i64 = 0;
if (!is_field_valid(field_id)) {
return RDC_ST_NOT_SUPPORTED;
}
struct timeval tv;
gettimeofday(&tv, NULL);
value->ts = static_cast<uint64_t>(tv.tv_sec) * 1000 + tv.tv_usec / 1000;
rdc_libs/rdc/src/RdcMetricsUpdaterImpl.cc
+57
View File
@@ -0,0 +1,57 @@
/*
Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
#include <sys/time.h>
#include <ctime>
#include <chrono>
#include "rdc_lib/rdc_common.h"
namespace amd {
namespace rdc {
RdcMetricsUpdaterImpl::RdcMetricsUpdaterImpl(
const RdcWatchTablePtr& watch_table,
const uint32_t check_frequency):
watch_table_(watch_table)
, started_(false)
, _check_frequency(check_frequency) {
}
void RdcMetricsUpdaterImpl::start() {
if (started_) {
return;
}
started_ = true;
updater_ = std::async(std::launch::async, [this](){
while (started_) {
watch_table_->rdc_update_all_fields();
std::this_thread::sleep_for(std::chrono::microseconds(100));
}
});
}
void RdcMetricsUpdaterImpl::stop() {
started_ = false;
}
} // namespace rdc
} // namespace amd
rdc_libs/rdc/src/RdcWatchTableImpl.cc
+309
View File
@@ -0,0 +1,309 @@
/*
Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "rdc_lib/impl/RdcWatchTableImpl.h"
#include <sys/time.h>
#include <ctime>
#include <algorithm>
#include "rdc_lib/rdc_common.h"
namespace amd {
namespace rdc {
RdcWatchTableImpl::RdcWatchTableImpl(const RdcGroupSettingsPtr& group_settings,
const RdcCacheManagerPtr& cache_mgr,
const RdcMetricFetcherPtr& metric_fetcher):
group_settings_(group_settings)
, cache_mgr_(cache_mgr)
, metric_fetcher_(metric_fetcher)
, last_cleanup_time_(0) {
}
rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id,
char job_id[64]) {
// TODO(bill_liu): implement
(void)(group_id);
(void)(job_id);
return RDC_ST_OK;
}
rdc_status_t RdcWatchTableImpl::rdc_watch_job_fields(rdc_gpu_group_t group_id,
uint64_t update_freq, double max_keep_age,
uint32_t max_keep_samples) {
// TODO(bill_liu): implement
(void)(group_id);
(void)(update_freq);
(void)(max_keep_age);
(void)(max_keep_samples);
return RDC_ST_OK;
}
rdc_status_t RdcWatchTableImpl::get_fields_from_group(rdc_gpu_group_t group_id,
rdc_field_grp_t field_group_id, std::vector<RdcFieldKey> & fields) {
rdc_field_group_info_t finfo;
rdc_group_info_t ginfo;
rdc_status_t result = group_settings_->
rdc_group_gpu_get_info(group_id, &ginfo);
if (result != RDC_ST_OK) {
return result;
}
result = group_settings_->rdc_group_field_get_info(field_group_id, &finfo);
if (result != RDC_ST_OK) {
return result;
}
for (uint32_t i = 0 ; i < ginfo.count; i++) { // GPUs
for (uint32_t j = 0; j < finfo.count; j++) { // Fields
fields.push_back(RdcFieldKey({ginfo.entity_ids[i],
finfo.field_ids[j]}));
}
}
return RDC_ST_OK;
}
rdc_status_t RdcWatchTableImpl::rdc_watch_fields(rdc_gpu_group_t group_id,
rdc_field_grp_t field_group_id, uint64_t update_freq,
double max_keep_age, uint32_t max_keep_samples) {
std::lock_guard<std::mutex> guard(watch_mutex_);
RdcFieldKey gkey({group_id, field_group_id});
auto table_iter = watch_table_.find(gkey);
// Already in the watch table
if (table_iter != watch_table_.end()) {
if (table_iter->second.is_watching) {
return RDC_ST_CONFLICT;
} else { // delete to overwrite
watch_table_.erase(table_iter);
}
}
// The field settings for this watch
FieldSettings f;
f.update_freq = update_freq;
f.max_keep_age = max_keep_age;
f.max_keep_samples = max_keep_samples;
f.last_update_time = 0;
f.is_watching = true;
// Get individual fields for the watch
std::vector<RdcFieldKey> fields_in_watch;
rdc_status_t result = get_fields_from_group(group_id,
field_group_id, fields_in_watch);
if (result != RDC_ST_OK) {
return result;
}
// Update the fields_to_watch_
auto f_in_watch_iter = fields_in_watch.begin();
for (; f_in_watch_iter != fields_in_watch.end(); f_in_watch_iter++) {
auto ite = fields_to_watch_.find(*f_in_watch_iter);
if (ite == fields_to_watch_.end()) { // A new field
fields_to_watch_.insert({*f_in_watch_iter, f});
} else { // Merge the settings
auto& f_in_table = ite->second;
f_in_table.max_keep_age =
std::max(f_in_table.max_keep_age, max_keep_age);
f_in_table.max_keep_samples =
std::max(f_in_table.max_keep_samples, max_keep_samples);
if (f_in_table.is_watching) { // Already watching
f_in_table.update_freq =
std::min(f_in_table.update_freq, update_freq);
} else { // Not watching before
f_in_table.is_watching = true;
f_in_table.update_freq = update_freq;
}
}
}
// Add to the watch table
watch_table_.insert({gkey, f});
return RDC_ST_OK;
}
rdc_status_t RdcWatchTableImpl::update_field_in_table_when_unwatch(
const RdcFieldKey& entry) {
// Get individual fields for this unwatch
std::vector<RdcFieldKey> fields;
rdc_status_t result = get_fields_from_group(
entry.first, entry.second, fields);
if (result != RDC_ST_OK) {
return result;
}
// Unwatch will only impact the update_freq, but not the max_keep_age
// and max_keep_samples. Walk through watch_table_ to get new update
// frequency for all fields and store it in update_frequencies
std::map<RdcFieldKey, uint64_t> update_frequencies;
auto w_iter = watch_table_.begin();
for (; w_iter != watch_table_.end(); w_iter++) {
// Skip the table is not in watching status
if (w_iter->second.is_watching == false) {
continue;
}
// Get all fields in current table
std::vector<RdcFieldKey> watch_fields;
result = get_fields_from_group(w_iter->first.first,
w_iter->first.second, watch_fields);
if (result != RDC_ST_OK) {
return result;
}
// Get the update_freq
auto fields_in_table_iter = watch_fields.begin();
for (; fields_in_table_iter != watch_fields.end();
fields_in_table_iter++) {
auto f_in_freq_iter = update_frequencies.find(
*fields_in_table_iter);
if (f_in_freq_iter == update_frequencies.end()) {
update_frequencies.insert(
{*fields_in_table_iter, w_iter->second.update_freq});
} else {
f_in_freq_iter->second =
std::min(f_in_freq_iter->second,
w_iter->second.update_freq);
}
}
}
// Update the fields that impacted by this unwatch
auto fite = fields.begin();
for (; fite != fields.end(); fite++) {
auto f_in_table = fields_to_watch_.find((*fite));
if (f_in_table == fields_to_watch_.end()) { // Not in fields_to_watch_
continue;
}
auto freq_iter = update_frequencies.find(*fite);
if (freq_iter == update_frequencies.end()) {
f_in_table->second.is_watching = false;
} else {
f_in_table->second.update_freq = freq_iter->second;
}
}
return RDC_ST_OK;
}
rdc_status_t RdcWatchTableImpl::rdc_unwatch_fields(
rdc_gpu_group_t group_id, rdc_field_grp_t field_group_id) {
struct timeval tv;
gettimeofday(&tv, NULL);
uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
std::lock_guard<std::mutex> guard(watch_mutex_);
// Set is_watching = false
auto ite = watch_table_.find(RdcFieldKey({group_id, field_group_id}));
if (ite == watch_table_.end()) {
return RDC_ST_NOT_FOUND;
}
ite->second.is_watching = false;
ite->second.last_update_time = now;
// Update the fields_to_watch_
return update_field_in_table_when_unwatch(ite->first);
}
rdc_status_t RdcWatchTableImpl::rdc_update_all_fields() {
uint32_t items_fetched = 0;
rdc_status_t result;
struct timeval tv;
gettimeofday(&tv, NULL);
uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
std::lock_guard<std::mutex> guard(watch_mutex_);
auto fite = fields_to_watch_.begin();
for (; fite != fields_to_watch_.end(); fite++) {
// Is this field need to be updated?
if (!fite->second.is_watching ||
fite->second.last_update_time+fite->second.update_freq/1000 > now) {
continue;
}
// Fetch the metric from rocm_smi_lib
rdc_field_value value;
result = metric_fetcher_->fetch_smi_field(
fite->first.first, fite->first.second, &value);
if (result != RDC_ST_OK) {
LOG_DEBUG("Fail to fetch the field: " << rdc_status_string(result));
continue;
}
// Update the cache
cache_mgr_->rdc_update_cache(fite->first.first, value);
// Update the last_upate_time
gettimeofday(&tv, NULL);
now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
fite->second.last_update_time = now;
items_fetched++;
}
// Clean up is expensive, only do it once per second
if (now - last_cleanup_time_ >1000) {
clean_up();
last_cleanup_time_ = now;
}
return RDC_ST_OK;
}
void RdcWatchTableImpl::clean_up() {
struct timeval tv;
gettimeofday(&tv, NULL);
uint64_t now = static_cast<uint64_t>(tv.tv_sec)*1000+tv.tv_usec/1000;
// Clean the cache and the fields_to_watch_ table
auto fite = fields_to_watch_.begin();
while (fite != fields_to_watch_.end()) {
cache_mgr_->evict_cache(fite->first.first, fite->first.second,
fite->second.max_keep_samples, fite->second.max_keep_age);
if (!fite->second.is_watching && fite->second.last_update_time +
fite->second.max_keep_age*1000 < now ) {
fite = fields_to_watch_.erase(fite);
} else {
++fite;
}
}
// Clean the watch table
auto wite = watch_table_.begin();
while (wite != watch_table_.end()) {
if (!wite->second.is_watching && wite->second.last_update_time +
wite->second.max_keep_age*1000 < now ) {
wite = watch_table_.erase(wite);
} else {
++wite;
}
}
}
} // namespace rdc
} // namespace amd