AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support standard deviation and json output for job stats

Browse Source 
In the job stats, in addition to the max, min and average,
it will also display the standard deviation.

A new option --json is added to the rdci to output the results
in json format.

In the job stats, using the GMT time instead of timestamp
for start and end time.

Change-Id: If245c4fc4854a1dc867f97ff5aa9112af7962eca
This commit is contained in:
Bill(Shuzhou) Liu
2020-07-14 15:20:08 -04:00
committed by Chris Freehill
parent 9efb55b06f
commit e6d910f67a
17 changed files with 515 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdc_libs/CMakeLists.txt
+2 -2
View File
@@ -160,7 +160,7 @@ message("RDC_LIB_INC_LIST=${RDC_LIB_INC_LIST}")
link_directories(${RSMI_LIB_DIR} "${GRPC_ROOT}/lib" "${GRPC_ROOT}/lib64")
add_library(${RDC_LIB} SHARED ${RDC_LIB_SRC_LIST} ${RDC_LIB_INC_LIST})
target_link_libraries(${RDC_LIB} pthread rocm_smi64)
target_link_libraries(${RDC_LIB} ${BOOTSTRAP_LIB} pthread rocm_smi64)
target_include_directories(${RDC_LIB} PRIVATE
"${PROJECT_SOURCE_DIR}"
"${PROJECT_SOURCE_DIR}/include"
@@ -191,7 +191,7 @@ set(RDCCLIENT_LIB_INC_LIST ${RDCCLIENT_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib
message("RDCCLIENT_LIB_INC_LIST=${RDCCLIENT_LIB_INC_LIST}")
add_library(${RDCCLIENT_LIB} SHARED ${RDCCLIENT_LIB_SRC_LIST} ${RDCCLIENT_LIB_INC_LIST})
target_link_libraries(${RDCCLIENT_LIB} pthread rt grpc grpc++ grpc++_reflection
target_link_libraries(${RDCCLIENT_LIB} ${BOOTSTRAP_LIB} pthread rt grpc grpc++ grpc++_reflection
dl protobuf)
target_include_directories(${RDCCLIENT_LIB} PRIVATE
"${GRPC_ROOT}/include"
rdc_libs/rdc/src/RdcCacheManagerImpl.cc
+44 -25
View File
@@ -21,6 +21,7 @@ THE SOFTWARE.
*/
#include "rdc_lib/impl/RdcCacheManagerImpl.h"
#include <sys/time.h>
#include <cmath>
#include <ctime>
#include <sstream>
#include "rdc_lib/RdcLogger.h"
@@ -213,6 +214,10 @@ rdc_status_t RdcCacheManagerImpl::rdc_update_job_stats(uint32_t gpu_index,
if (value.field_id == RDC_FI_POWER_USAGE) {
gpu_iter->second.energy_last_time = value.ts;
}
// https://www.johndcook.com/blog/standard_deviation/
fsummary->second.old_s = 0;
fsummary->second.old_m = fsummary->second.new_m = value.value.l_int;
return RDC_ST_OK;
}
if (value.field_id == RDC_FI_POWER_USAGE) {
@@ -229,6 +234,15 @@ rdc_status_t RdcCacheManagerImpl::rdc_update_job_stats(uint32_t gpu_index,
fsummary->second.last_time = value.ts;
fsummary->second.count++;
// https://www.johndcook.com/blog/standard_deviation/
fsummary->second.new_m = fsummary->second.old_m +
(value.value.l_int - fsummary->second.old_m)/fsummary->second.count;
fsummary->second.new_s = fsummary->second.old_s +
(value.value.l_int - fsummary->second.old_m)*
(value.value.l_int - fsummary->second.new_m);
fsummary->second.old_m = fsummary->second.new_m;
fsummary->second.old_s = fsummary->second.new_s;
return RDC_ST_OK;
}
@@ -248,6 +262,11 @@ void RdcCacheManagerImpl::set_summary(const FieldSummaryStats & stats,
summary.min_value = std::min(summary.min_value, gpu.min_value);
//< save total for future average calculation.
summary.average += gpu.average;
//< calculate the sample variance
gpu.standard_deviation = std::sqrt((stats.count > 1)
? stats.new_s/(stats.count - 1) : 0.0)/adjuster;
summary.standard_deviation += gpu.standard_deviation;
}
rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
@@ -274,15 +293,17 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
summary_info.max_gpu_memory_used = 0;
summary_info.ecc_correct = 0;
summary_info.ecc_uncorrect = 0;
summary_info.power_usage = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.pcie_tx = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.pcie_rx = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.gpu_temperature = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.memory_clock = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.gpu_clock = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.gpu_utilization = {0, std::numeric_limits<uint64_t>::max(), 0};
summary_info.power_usage = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.pcie_tx = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.pcie_rx = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.gpu_temperature =
{0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.memory_clock = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.gpu_clock = {0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.gpu_utilization =
{0, std::numeric_limits<uint64_t>::max(), 0, 0};
summary_info.memory_utilization = {0,
std::numeric_limits<uint64_t>::max(), 0};
std::numeric_limits<uint64_t>::max(), 0, 0};
p_job_info->num_gpus = job_stats->second.gpu_stats.size();
@@ -363,27 +384,25 @@ rdc_status_t RdcCacheManagerImpl::rdc_job_get_stats(const char jobId[64],
}
}
}
// Get the average of the summary
summary_info.power_usage.average = summary_info.power_usage.average/
p_job_info->num_gpus;
summary_info.gpu_clock.average = summary_info.gpu_clock.average/
p_job_info->num_gpus;
summary_info.gpu_utilization.average = summary_info.gpu_utilization.average/
p_job_info->num_gpus;
summary_info.memory_utilization.average =
summary_info.memory_utilization.average/p_job_info->num_gpus;
summary_info.pcie_tx.average = summary_info.pcie_tx.average/
p_job_info->num_gpus;
summary_info.pcie_rx.average = summary_info.pcie_rx.average/
p_job_info->num_gpus;
summary_info.gpu_temperature.average = summary_info.gpu_temperature.average/
p_job_info->num_gpus;
summary_info.memory_clock.average = summary_info.memory_clock.average/
p_job_info->num_gpus;
// Set the average of the summary
set_average_summary(summary_info.power_usage, p_job_info->num_gpus);
set_average_summary(summary_info.gpu_clock, p_job_info->num_gpus);
set_average_summary(summary_info.gpu_utilization, p_job_info->num_gpus);
set_average_summary(summary_info.memory_utilization, p_job_info->num_gpus);
set_average_summary(summary_info.pcie_tx, p_job_info->num_gpus);
set_average_summary(summary_info.pcie_rx, p_job_info->num_gpus);
set_average_summary(summary_info.gpu_temperature, p_job_info->num_gpus);
set_average_summary(summary_info.memory_clock, p_job_info->num_gpus);
return RDC_ST_OK;
}
void RdcCacheManagerImpl::set_average_summary(
rdc_stats_summary_t& summary, uint32_t num_gpus) {
summary.average = summary.average/num_gpus;
summary.standard_deviation = summary.standard_deviation/num_gpus;
}
rdc_status_t RdcCacheManagerImpl::rdc_job_start_stats(const char job_id[64],
const rdc_group_info_t& ginfo, const rdc_field_group_info_t& finfo,
const rdc_gpu_gauges_t& gpu_gauges) {
rdc_libs/rdc/src/RdcGroupSettingsImpl.cc
+2
View File
@@ -42,6 +42,7 @@ RdcGroupSettingsImpl::RdcGroupSettingsImpl() {
rdc_status_t RdcGroupSettingsImpl::rdc_group_gpu_create(
const char* group_name, rdc_gpu_group_t* p_rdc_group_id) {
RDC_LOG(RDC_DEBUG, "Create group " << group_name);
rdc_group_info_t ginfo;
strncpy_with_null(ginfo.group_name, group_name, RDC_MAX_STR_LENGTH);
ginfo.count = 0;
@@ -135,6 +136,7 @@ rdc_status_t RdcGroupSettingsImpl::rdc_group_field_create(
uint32_t num_field_ids, uint32_t* field_ids,
const char* field_group_name, rdc_field_grp_t* rdc_field_group_id) {
RDC_LOG(RDC_DEBUG, "Create field group " << field_group_name);
rdc_field_group_info_t finfo;
finfo.count = num_field_ids;
strncpy_with_null(finfo.group_name, field_group_name, RDC_MAX_STR_LENGTH);
rdc_libs/rdc_client/src/RdcStandaloneHandler.cc
+8
View File
@@ -95,41 +95,49 @@ bool RdcStandaloneHandler::copy_gpu_usage_info(
target->power_usage.max_value = pstats.max_value();
target->power_usage.min_value = pstats.min_value();
target->power_usage.average = pstats.average();
target->power_usage.standard_deviation = pstats.standard_deviation();
const ::rdc::JobStatsSummary& cstats = src.gpu_clock();
target->gpu_clock.max_value = cstats.max_value();
target->gpu_clock.min_value = cstats.min_value();
target->gpu_clock.average = cstats.average();
target->gpu_clock.standard_deviation = cstats.standard_deviation();
const ::rdc::JobStatsSummary& ustats = src.gpu_utilization();
target->gpu_utilization.max_value = ustats.max_value();
target->gpu_utilization.min_value = ustats.min_value();
target->gpu_utilization.average = ustats.average();
target->gpu_utilization.standard_deviation = ustats.standard_deviation();
const ::rdc::JobStatsSummary& mstats = src.memory_utilization();
target->memory_utilization.max_value = mstats.max_value();
target->memory_utilization.min_value = mstats.min_value();
target->memory_utilization.average = mstats.average();
target->memory_utilization.standard_deviation = mstats.standard_deviation();
const ::rdc::JobStatsSummary& txstats = src.pcie_tx();
target->pcie_tx.max_value = txstats.max_value();
target->pcie_tx.min_value = txstats.min_value();
target->pcie_tx.average = txstats.average();
target->pcie_tx.standard_deviation = txstats.standard_deviation();
const ::rdc::JobStatsSummary& rxstats = src.pcie_rx();
target->pcie_rx.max_value = rxstats.max_value();
target->pcie_rx.min_value = rxstats.min_value();
target->pcie_rx.average = rxstats.average();
target->pcie_rx.standard_deviation = rxstats.standard_deviation();
const ::rdc::JobStatsSummary& mcstats = src.memory_clock();
target->memory_clock.max_value = mcstats.max_value();
target->memory_clock.min_value = mcstats.min_value();
target->memory_clock.average = mcstats.average();
target->memory_clock.standard_deviation = mcstats.standard_deviation();
const ::rdc::JobStatsSummary& gtstats = src.gpu_temperature();
target->gpu_temperature.max_value = gtstats.max_value();
target->gpu_temperature.min_value = gtstats.min_value();
target->gpu_temperature.average = gtstats.average();
target->gpu_temperature.standard_deviation = gtstats.standard_deviation();
return true;
}