AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement the stats subsystem in rdci

Browse Source 
Add support for the stats subsystem in rdci
Modify the dmon system to handle the case when no GPUs in a group

Change-Id: I5a18e1201d24b5318b8e324a77551a757b108f25
This commit is contained in:
Bill(Shuzhou) Liu
2020-04-21 09:48:14 -04:00
committed by Chris Freehill
parent 1b58033183
commit 096dc2dadb
7 changed files with 369 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdc_libs/rdc/src/RdcCacheManagerImpl.cc
+3 -1
View File
@@ -257,7 +257,7 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(char jobId[64],
}
//< Init the summary info
RDC_LOG(RDC_DEBUG, "rdc_job_get_stats for job " <<jobId);
RDC_LOG(RDC_DEBUG, "rdc_job_get_stats for job " << jobId);
auto& summary_info = p_job_info->summary;
summary_info.start_time = job_stats->second.start_time;
if (job_stats->second.end_time == 0) {
@@ -340,6 +340,8 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_start_stats(char job_id[64],
}
std::lock_guard<std::mutex> guard(cache_mutex_);
// Remove the old stats if it exists
cache_jobs_.erase(job_id);
cache_jobs_.insert({job_id, cacheEntry});
return RDC_ST_OK;
}
rdc_libs/rdc/src/RdcWatchTableImpl.cc
+16 -2
View File
@@ -52,8 +52,20 @@ rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id,
std::vector<RdcFieldKey> fields_in_watch;
rdc_status_t result = get_fields_from_group(group_id,
JOB_FIELD_ID, fields_in_watch);
if (result != RDC_ST_OK) {
return result;
}
if (fields_in_watch.size() == 0) {
RDC_LOG(RDC_ERROR, "Fail to start job " << job_id <<". The group "
<< group_id << " must contain at least one GPU.");
return RDC_ST_NOT_FOUND;
}
JobWatchTableEntry jentry {group_id, fields_in_watch};
job_watch_table_.insert({job_id, jentry});
do { //< lock guard for thread safe
std::lock_guard<std::mutex> guard(watch_mutex_);
job_watch_table_.insert({job_id, jentry});
} while (0);
result = rdc_field_watch(group_id, JOB_FIELD_ID, update_freq, 0, 0);
if (result != RDC_ST_OK) {
@@ -62,10 +74,12 @@ rdc_status_t RdcWatchTableImpl::rdc_job_start_stats(rdc_gpu_group_t group_id,
rdc_field_group_info_t finfo;
rdc_group_info_t ginfo;
result = group_settings_->rdc_group_gpu_get_info(group_id, &ginfo);
result = group_settings_->rdc_group_gpu_get_info(
group_id, &ginfo);
if (result != RDC_ST_OK) {
return result;
}
result = group_settings_->rdc_group_field_get_info(JOB_FIELD_ID, &finfo);
if (result != RDC_ST_OK) {
return result;