Support extra metrics in the RDC
Remove the * in the rdci stats When a group is created, the GPUs can be added in the same command. Add the support to the memory temperature. Add the support to the memory clock. Add the support to report the ECC errors. Add the support to report the PCIe bandwidth throughput. Since the RX/TX throughput may take 1 second to retreive, an async fetch is implemented in the RdcMetricFetcherImpl. Change-Id: If04f602fe1f2d14dbf7c2fb189549fd030523f9a
Este cometimento está contido em:
Bill(Shuzhou) Liu
cometido por
Chris Freehill
Chris Freehill
ascendente
096dc2dadb
cometimento
f4a3fd4dda
@@ -278,10 +278,19 @@ void RdciDmonSubSystem::create_temp_field_group() {
|
||||
void RdciDmonSubSystem::show_field_usage() const {
|
||||
std::cout << "Supported fields Ids:\n";
|
||||
std::cout << "100 RDC_FI_GPU_SM_CLOCK: Current GPU clock frequencies.\n";
|
||||
std::cout << "101 RDC_FI_MEM_CLOCK: Current Memory clock frequencies.\n";
|
||||
std::cout << "140 RDC_FI_MEMORY_TEMP: Memory "
|
||||
<< "temperature in millidegrees Celsius.\n";
|
||||
std::cout << "150 RDC_FI_GPU_TEMP: GPU "
|
||||
<< "temperature in millidegrees Celcius.\n";
|
||||
<< "temperature in millidegrees Celsius.\n";
|
||||
std::cout << "155 RDC_FI_POWER_USAGE: Power usage in microwatts.\n";
|
||||
std::cout << "200 RDC_FI_PCIE_TX: PCIe Tx utilization in bytes/second.\n";
|
||||
std::cout << "201 RDC_FI_PCIE_RX: PCIe Rx utilization in bytes/second.\n";
|
||||
std::cout << "203 RDC_FI_GPU_UTIL: GPU busy percentage.\n";
|
||||
std::cout << "312 RDC_FI_ECC_CORRECT_TOTAL: Accumulated "
|
||||
<< "correctable ECC errors.\n";
|
||||
std::cout << "313 RDC_FI_ECC_UNCORRECT_TOTAL: Accumulated "
|
||||
<< "uncorrectable ECC errors.\n";
|
||||
std::cout << "525 RDC_FI_GPU_MEMORY_USAGE: Memory usage of the GPU "
|
||||
<< "instance in bytes.\n";
|
||||
}
|
||||
@@ -361,7 +370,7 @@ void RdciDmonSubSystem::process() {
|
||||
group_info.entity_ids[gindex],
|
||||
field_info.field_ids[findex], &value);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << std::left << std::setw(20) << "error";
|
||||
std::cout << std::left << std::setw(20) << "N/A";
|
||||
} else {
|
||||
if (value.type == INTEGER) {
|
||||
std::cout << std::left << std::setw(20)
|
||||
|
||||
@@ -82,7 +82,10 @@ void RdciGroupSubSystem::parse_cmd_opts(int argc, char ** argv) {
|
||||
group_name_ = optarg;
|
||||
break;
|
||||
case 'a':
|
||||
group_ops_ = GROUP_ADD_GPUS;
|
||||
// Create may add GPUs as well.
|
||||
if (group_ops_ != GROUP_CREATE) {
|
||||
group_ops_ = GROUP_ADD_GPUS;
|
||||
}
|
||||
gpu_ids_ = optarg;
|
||||
break;
|
||||
case 'i':
|
||||
@@ -116,7 +119,8 @@ void RdciGroupSubSystem::show_help() const {
|
||||
std::cout << " group -- Used to create and maintain groups of GPUs.\n\n";
|
||||
std::cout << "Usage\n";
|
||||
std::cout << " rdci group [--host <IP/FQDN>:port] [-u] -l\n";
|
||||
std::cout << " rdci group [--host <IP/FQDN>:port] [-u] -c <groupName>\n";
|
||||
std::cout << " rdci group [--host <IP/FQDN>:port] [-u] -c <groupName> "
|
||||
<< "[-a <entityId>]\n";
|
||||
std::cout << " rdci group [--host <IP/FQDN>:port] [-u] -g <groupId> "
|
||||
<< "[-a <entityId>]\n";
|
||||
std::cout << " rdci group [--host <IP/FQDN>:port] [-u] "
|
||||
@@ -157,6 +161,25 @@ void RdciGroupSubSystem::process() {
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle_, RDC_GROUP_EMPTY,
|
||||
group_name_.c_str(), &group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
throw RdcException(result, "Fail to create group "
|
||||
+ group_name_);
|
||||
}
|
||||
|
||||
gpu_ids = split_string(gpu_ids_, ',');
|
||||
for (uint32_t i = 0; i < gpu_ids.size(); i++) {
|
||||
if (!IsNumber(gpu_ids[i])) {
|
||||
throw RdcException(RDC_ST_BAD_PARAMETER,
|
||||
"The GPU Id "+gpu_ids[i]+" needs to be a number");
|
||||
}
|
||||
result = rdc_group_gpu_add(rdc_handle_,
|
||||
group_id, std::stoi(gpu_ids[i]));
|
||||
if (result != RDC_ST_OK) {
|
||||
throw RdcException(result, "Fail to add GPU "
|
||||
+ gpu_ids[i] + " to the group");
|
||||
}
|
||||
}
|
||||
|
||||
if (result == RDC_ST_OK) {
|
||||
std::cout << "Successfully created group with a group ID "
|
||||
<< group_id << std::endl;
|
||||
@@ -214,7 +237,7 @@ void RdciGroupSubSystem::process() {
|
||||
for (uint32_t i = 0; i < gpu_ids.size(); i++) {
|
||||
if (!IsNumber(gpu_ids[i])) {
|
||||
throw RdcException(RDC_ST_BAD_PARAMETER,
|
||||
"The GUP Id "+gpu_ids[i]+" needs to be a number");
|
||||
"The GPU Id "+gpu_ids[i]+" needs to be a number");
|
||||
}
|
||||
result = rdc_group_gpu_add(rdc_handle_,
|
||||
group_id_, std::stoi(gpu_ids[i]));
|
||||
|
||||
@@ -150,11 +150,11 @@ void RdciStatsSubSystem::show_job_stats(
|
||||
const rdc_gpu_usage_info_t& gpu_info) const {
|
||||
std::cout << "|------- Execution Stats ----------"
|
||||
<< "+------------------------------------\n";
|
||||
std::cout << "| Start Time * | "
|
||||
std::cout << "| Start Time | "
|
||||
<< gpu_info.start_time << "\n";
|
||||
std::cout << "| End Time * | "
|
||||
std::cout << "| End Time | "
|
||||
<< gpu_info.end_time << "\n";
|
||||
std::cout << "| Total Execution Time (sec) * | "
|
||||
std::cout << "| Total Execution Time (sec) | "
|
||||
<< (gpu_info.end_time-gpu_info.start_time) << "\n";
|
||||
std::cout << "+------- Performance Stats --------"
|
||||
<< "+------------------------------------\n";
|
||||
@@ -168,16 +168,36 @@ void RdciStatsSubSystem::show_job_stats(
|
||||
<< gpu_info.gpu_clock.max_value << " Min: " <<
|
||||
gpu_info.gpu_clock.min_value << " Avg: "
|
||||
<< gpu_info.gpu_clock.average << "\n";
|
||||
std::cout << "| Memory Clock (MHz) | " << "Max: "
|
||||
<< gpu_info.memory_clock.max_value << " Min: " <<
|
||||
gpu_info.memory_clock.min_value << " Avg: "
|
||||
<< gpu_info.memory_clock.average << "\n";
|
||||
std::cout << "| SM Utilization (%) | " << "Max: "
|
||||
<< gpu_info.gpu_utilization.max_value <<" Min: " <<
|
||||
gpu_info.gpu_utilization.min_value << " Avg: " <<
|
||||
gpu_info.gpu_utilization.average << "\n";
|
||||
std::cout << "| Max GPU Memory Used (bytes) * | " <<
|
||||
std::cout << "| Max GPU Memory Used (bytes) | " <<
|
||||
gpu_info.max_gpu_memory_used << "\n";
|
||||
std::cout << "| Memory Utilization (%) | "
|
||||
<< "Max: " << gpu_info.memory_utilization.max_value
|
||||
<<" Min: "<< gpu_info.memory_utilization.min_value
|
||||
<< " Avg: " << gpu_info.memory_utilization.average << "\n";
|
||||
<<" Min: "<< gpu_info.memory_utilization.min_value
|
||||
<< " Avg: " << gpu_info.memory_utilization.average << "\n";
|
||||
std::cout << "| GPU Temperature (Celsius) | "
|
||||
<< "Max: " << gpu_info.gpu_temperature.max_value
|
||||
<<" Min: "<< gpu_info.gpu_temperature.min_value
|
||||
<< " Avg: " << gpu_info.gpu_temperature.average << "\n";
|
||||
std::cout << "| PCIe Rx Bandwidth (megabytes) | "
|
||||
<< "Max: " << gpu_info.pcie_rx.max_value
|
||||
<<" Min: "<< gpu_info.pcie_rx.min_value
|
||||
<< " Avg: " << gpu_info.pcie_rx.average << "\n";
|
||||
std::cout << "| PCIe Tx Bandwidth (megabytes) | "
|
||||
<< "Max: " << gpu_info.pcie_tx.max_value
|
||||
<<" Min: "<< gpu_info.pcie_tx.min_value
|
||||
<< " Avg: " << gpu_info.pcie_tx.average << "\n";
|
||||
std::cout << "| Correctable ECC Errors | "
|
||||
<< gpu_info.ecc_correct << "\n";
|
||||
std::cout << "| Uncorrectable ECC Errors | "
|
||||
<< gpu_info.ecc_uncorrect << "\n";
|
||||
std::cout << "+----------------------------------"
|
||||
<< "+------------------------------------\n";
|
||||
}
|
||||
|
||||
Criar uma nova questão referindo esta
Bloquear um utilizador