LINT: Add cpplint, clang-format and pre-commit support
Change-Id: I3cbb787ef27d90486b212dfb1a8c77c460acc2ac Signed-off-by: Galantsev, Dmitrii <dmitrii.galantsev@amd.com>
Этот коммит содержится в:
Galantsev, Dmitrii
@@ -20,154 +20,139 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#include <iostream>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
#include "rdc/rdc.h"
|
||||
|
||||
static std::string get_test_name(rdc_diag_test_cases_t test_case) {
|
||||
const std::map<rdc_diag_test_cases_t, std::string> test_desc = {
|
||||
{RDC_DIAG_COMPUTE_PROCESS, "No compute process"},
|
||||
{RDC_DIAG_COMPUTE_QUEUE, "Compute Queue ready"},
|
||||
{RDC_DIAG_SYS_MEM_CHECK, "System memory check"},
|
||||
{RDC_DIAG_NODE_TOPOLOGY, "Node topology check"},
|
||||
{RDC_DIAG_GPU_PARAMETERS, "GPU parameters check"},
|
||||
{RDC_DIAG_TEST_LAST, "Unknown"}
|
||||
};
|
||||
const std::map<rdc_diag_test_cases_t, std::string> test_desc = {
|
||||
{RDC_DIAG_COMPUTE_PROCESS, "No compute process"},
|
||||
{RDC_DIAG_COMPUTE_QUEUE, "Compute Queue ready"},
|
||||
{RDC_DIAG_SYS_MEM_CHECK, "System memory check"},
|
||||
{RDC_DIAG_NODE_TOPOLOGY, "Node topology check"},
|
||||
{RDC_DIAG_GPU_PARAMETERS, "GPU parameters check"},
|
||||
{RDC_DIAG_TEST_LAST, "Unknown"}};
|
||||
|
||||
auto test_name = test_desc.find(test_case);
|
||||
if (test_name == test_desc.end()) {
|
||||
return "Unknown Test";
|
||||
}
|
||||
return test_name->second;
|
||||
auto test_name = test_desc.find(test_case);
|
||||
if (test_name == test_desc.end()) {
|
||||
return "Unknown Test";
|
||||
}
|
||||
return test_name->second;
|
||||
}
|
||||
|
||||
int main(int, char **) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"diag_group"};
|
||||
int main(int, char**) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"diag_group"};
|
||||
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone?
|
||||
"Standalone mode selected.\n":"Embedded mode selected.\n");
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone ? "Standalone mode selected.\n" : "Embedded mode selected.\n");
|
||||
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " << rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle, nullptr, nullptr, nullptr);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " <<
|
||||
rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
std::cout << "Error connecting to remote rdcd. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle,
|
||||
nullptr, nullptr, nullptr);
|
||||
if ( result != RDC_ST_OK ) {
|
||||
std::cout << "Error connecting to remote rdcd. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error starting embedded RDC engine. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// (1) create group for all GPUs
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_DEFAULT,
|
||||
group_name, &group_id);
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
std::cout << "Error starting embedded RDC engine. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// (2) start to run short diagnostic.
|
||||
rdc_diag_response_t response;
|
||||
result = rdc_diagnostic_run(rdc_handle, group_id,
|
||||
RDC_DIAG_LVL_SHORT, &response);
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// (1) create group for all GPUs
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_DEFAULT, group_name, &group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error run RDC_DIAG_LVL_SHORT diagnostic. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
// (2) start to run short diagnostic.
|
||||
rdc_diag_response_t response;
|
||||
result = rdc_diagnostic_run(rdc_handle, group_id, RDC_DIAG_LVL_SHORT, &response);
|
||||
|
||||
// (3) Check diagnostic results
|
||||
for (uint32_t i=0 ; i < response.results_count; i++) {
|
||||
const rdc_diag_test_result_t& test_result =
|
||||
response.diag_info[i];
|
||||
std::cout << std::setw(22) << std::left
|
||||
<< get_test_name(test_result.test_case) + ":"
|
||||
<< rdc_diagnostic_result_string(test_result.status) << "\n";
|
||||
}
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error run RDC_DIAG_LVL_SHORT diagnostic. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// (4) diagnostic detail information
|
||||
std::cout <<" =============== Diagnostic Details ==================\n";
|
||||
for (uint32_t i=0 ; i < response.results_count; i++) {
|
||||
const rdc_diag_test_result_t& test_result =
|
||||
response.diag_info[i];
|
||||
if (test_result.info[0] != '\0') {
|
||||
std::cout << std::setw(22) << std::left
|
||||
<< get_test_name(test_result.test_case) + ":"
|
||||
// (3) Check diagnostic results
|
||||
for (uint32_t i = 0; i < response.results_count; i++) {
|
||||
const rdc_diag_test_result_t& test_result = response.diag_info[i];
|
||||
std::cout << std::setw(22) << std::left << get_test_name(test_result.test_case) + ":"
|
||||
<< rdc_diagnostic_result_string(test_result.status) << "\n";
|
||||
}
|
||||
|
||||
// (4) diagnostic detail information
|
||||
std::cout << " =============== Diagnostic Details ==================\n";
|
||||
for (uint32_t i = 0; i < response.results_count; i++) {
|
||||
const rdc_diag_test_result_t& test_result = response.diag_info[i];
|
||||
if (test_result.info[0] != '\0') {
|
||||
std::cout << std::setw(22) << std::left << get_test_name(test_result.test_case) + ":"
|
||||
<< test_result.info << "\n";
|
||||
}
|
||||
for (uint32_t j=0; j < test_result.per_gpu_result_count; j++) {
|
||||
const rdc_diag_per_gpu_result_t& gpu_result
|
||||
= test_result.gpu_results[j];
|
||||
if (strlen(gpu_result.gpu_result.msg) > 0) {
|
||||
std::cout << " GPU " << gpu_result.gpu_index
|
||||
<< " " << gpu_result.gpu_result.msg << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// (5) run one test case
|
||||
std::cout <<" ============== Run individual diagnostic test ===========\n";
|
||||
rdc_diag_test_result_t test_result;
|
||||
result = rdc_test_case_run(rdc_handle, group_id,
|
||||
RDC_DIAG_COMPUTE_PROCESS, &test_result);
|
||||
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error run RDC_DIAG_COMPUTE_PROCESS diagnostic. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
for (uint32_t j = 0; j < test_result.per_gpu_result_count; j++) {
|
||||
const rdc_diag_per_gpu_result_t& gpu_result = test_result.gpu_results[j];
|
||||
if (strlen(gpu_result.gpu_result.msg) > 0) {
|
||||
std::cout << " GPU " << gpu_result.gpu_index << " " << gpu_result.gpu_result.msg << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << std::setw(22) << std::left
|
||||
<< get_test_name(RDC_DIAG_COMPUTE_PROCESS) + ":"
|
||||
<< test_result.info << "\n";
|
||||
// (5) run one test case
|
||||
std::cout << " ============== Run individual diagnostic test ===========\n";
|
||||
rdc_diag_test_result_t test_result;
|
||||
result = rdc_test_case_run(rdc_handle, group_id, RDC_DIAG_COMPUTE_PROCESS, &test_result);
|
||||
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error run RDC_DIAG_COMPUTE_PROCESS diagnostic. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
std::cout << std::setw(22) << std::left << get_test_name(RDC_DIAG_COMPUTE_PROCESS) + ":"
|
||||
<< test_result.info << "\n";
|
||||
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -21,257 +21,235 @@ THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <iostream>
|
||||
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
|
||||
#include "rdc/rdc.h"
|
||||
|
||||
int main(int, char **) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"group1"};
|
||||
char field_group_name[] = {"fieldgroup1"};
|
||||
uint64_t since_timestamp = 0;
|
||||
uint64_t next_timestamp = 0;
|
||||
uint64_t start_timestamp = 0;
|
||||
uint32_t count = 0;
|
||||
int main(int, char**) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"group1"};
|
||||
char field_group_name[] = {"fieldgroup1"};
|
||||
uint64_t since_timestamp = 0;
|
||||
uint64_t next_timestamp = 0;
|
||||
uint64_t start_timestamp = 0;
|
||||
uint32_t count = 0;
|
||||
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone ? "Standalone mode selected.\n" : "Embedded mode selected.\n");
|
||||
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone?
|
||||
"Standalone mode selected.\n":"Embedded mode selected.\n");
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " << rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle, nullptr, nullptr, nullptr);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " <<
|
||||
rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
std::cout << "Error connecting to remote rdcd. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error starting embedded RDC engine. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle,
|
||||
nullptr, nullptr, nullptr);
|
||||
if ( result != RDC_ST_OK ) {
|
||||
std::cout << "Error connecting to remote rdcd. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// Get the list of devices in the system
|
||||
uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
|
||||
result = rdc_device_get_all(rdc_handle, gpu_index_list, &count);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error to find devices on the system. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
if (count == 0) {
|
||||
std::cout << "No GPUs find on the sytem ";
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << count << " GPUs found in the system.\n";
|
||||
}
|
||||
|
||||
// Create the group
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_EMPTY, group_name, &group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Created the GPU group " << group_id << std::endl;
|
||||
|
||||
// Add all GPUs to the group
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
result = rdc_group_gpu_add(rdc_handle, group_id, gpu_index_list[i]); // Add GPU 0
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error adding group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
rdc_device_attributes_t attribute;
|
||||
result = rdc_device_get_attributes(rdc_handle, gpu_index_list[i], &attribute);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get GPU attribute. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Add GPU " << gpu_index_list[i] << ":" << attribute.device_name << " to group "
|
||||
<< group_id << std::endl;
|
||||
}
|
||||
|
||||
// Create the field groups to monitor POWER and TEMP
|
||||
rdc_field_grp_t field_group_id;
|
||||
rdc_field_t field_ids[2];
|
||||
field_ids[0] = RDC_FI_GPU_MEMORY_USAGE;
|
||||
field_ids[1] = RDC_FI_POWER_USAGE;
|
||||
result = rdc_group_field_create(rdc_handle, 2, &field_ids[0], field_group_name, &field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error create field group, Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Created the field group " << field_group_id << ": "
|
||||
<< field_id_string(RDC_FI_GPU_MEMORY_USAGE) << ", "
|
||||
<< field_id_string(RDC_FI_POWER_USAGE) << std::endl;
|
||||
|
||||
// Let the RDC to watch the fields and groups. The fields will be updated
|
||||
// once per second, the max keep age is 1 minutes and only keep 10 samples.
|
||||
result = rdc_field_watch(rdc_handle, group_id, field_group_id, 1000000, 60, 10);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error watch group fields. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Start to watch group:" << group_id << ", field_group:" << field_group_id
|
||||
<< std::endl;
|
||||
std::cout << "Sleep a few seconds before retreive the data ...\n";
|
||||
|
||||
// Since we are running the RDC_OPERATION_MODE_AUTO mode, the rdc_update_
|
||||
// all_fields() will be called periodically at background. If running as
|
||||
// RDC_OPERATION_MODE_MANUAL mode, we must call rdc_field_update_all()
|
||||
// periodically to take samples.
|
||||
usleep(5000000); // sleep 5 seconds before fetch the stats
|
||||
|
||||
// Retreive the field and group information from RDC
|
||||
rdc_group_info_t group_info;
|
||||
rdc_field_group_info_t field_info;
|
||||
result = rdc_group_gpu_get_info(rdc_handle, group_id, &group_info);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get gpu group info. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
result = rdc_group_field_get_info(rdc_handle, field_group_id, &field_info);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get field group info. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Get the latest metrics
|
||||
std::cout << "Get the latest metrics for group:" << group_id << " field_group:" << field_group_id
|
||||
<< std::endl;
|
||||
std::cout << "time_stamp\t"
|
||||
<< "GPU_index\t"
|
||||
<< "field_name\t\t"
|
||||
<< "field_value\n";
|
||||
for (uint32_t gindex = 0; gindex < group_info.count; gindex++) {
|
||||
for (uint32_t findex = 0; findex < field_info.count; findex++) {
|
||||
rdc_field_value value;
|
||||
result = rdc_field_get_latest_value(rdc_handle, group_info.entity_ids[gindex],
|
||||
field_info.field_ids[findex], &value);
|
||||
if (result == RDC_ST_NOT_FOUND) {
|
||||
continue;
|
||||
}
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get least value. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
// We only support the integer metrics so far
|
||||
std::cout << value.ts << "\t" << group_info.entity_ids[gindex] << "\t\t" << std::left
|
||||
<< std::setw(16) << field_id_string(value.field_id) << "\t" << value.value.l_int
|
||||
<< std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
// Stop watching the field group
|
||||
result = rdc_field_unwatch(rdc_handle, group_id, field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error stop watch fields. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Stop watch group:" << group_id << ", field_group:" << field_group_id << std::endl;
|
||||
|
||||
// Get the history data last 10 seconds
|
||||
std::cout << "Get last 10 seconds metrics for group:" << group_id
|
||||
<< " field_group:" << field_group_id << std::endl;
|
||||
std::cout << "time_stamp\t"
|
||||
<< "GPU_index\t"
|
||||
<< "field_name\t\t"
|
||||
<< "field_value\n";
|
||||
start_timestamp = static_cast<uint64_t>(time(nullptr) - 10) * 1000;
|
||||
for (uint32_t gindex = 0; gindex < group_info.count; gindex++) {
|
||||
for (uint32_t findex = 0; findex < field_info.count; findex++) {
|
||||
since_timestamp = start_timestamp;
|
||||
while (true) {
|
||||
rdc_field_value value;
|
||||
result = rdc_field_get_value_since(rdc_handle, group_info.entity_ids[gindex],
|
||||
field_info.field_ids[findex], since_timestamp,
|
||||
&next_timestamp, &value);
|
||||
if (result == RDC_ST_NOT_FOUND) {
|
||||
break;
|
||||
}
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error starting embedded RDC engine. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
std::cout << "Error get history data. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
std::cout << value.ts << "\t" << group_info.entity_ids[gindex] << "\t\t" << std::left
|
||||
<< std::setw(16) << field_id_string(value.field_id) << "\t" << value.value.l_int
|
||||
<< std::endl;
|
||||
since_timestamp = next_timestamp;
|
||||
} // while
|
||||
} // for findex
|
||||
} // for gindex
|
||||
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// Get the list of devices in the system
|
||||
uint32_t gpu_index_list[RDC_MAX_NUM_DEVICES];
|
||||
result = rdc_device_get_all(rdc_handle, gpu_index_list, &count);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error to find devices on the system. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
if (count == 0) {
|
||||
std::cout << "No GPUs find on the sytem ";
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << count << " GPUs found in the system.\n";
|
||||
}
|
||||
// Delete the field group and GPU group
|
||||
result = rdc_group_field_destroy(rdc_handle, field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error delete field group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Deleted the field group " << field_group_id << std::endl;
|
||||
|
||||
// Create the group
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_EMPTY,
|
||||
group_name, &group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Created the GPU group " << group_id << std::endl;
|
||||
result = rdc_group_gpu_destroy(rdc_handle, group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error delete GPU group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Deleted the GPU group " << group_id << std::endl;
|
||||
|
||||
// Add all GPUs to the group
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
result = rdc_group_gpu_add(rdc_handle,
|
||||
group_id, gpu_index_list[i]); // Add GPU 0
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error adding group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
rdc_device_attributes_t attribute;
|
||||
result = rdc_device_get_attributes(rdc_handle,
|
||||
gpu_index_list[i], &attribute);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get GPU attribute. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Add GPU " <<gpu_index_list[i] << ":"
|
||||
<< attribute.device_name <<" to group "<< group_id << std::endl;
|
||||
}
|
||||
|
||||
// Create the field groups to monitor POWER and TEMP
|
||||
rdc_field_grp_t field_group_id;
|
||||
rdc_field_t field_ids[2];
|
||||
field_ids[0] = RDC_FI_GPU_MEMORY_USAGE;
|
||||
field_ids[1] = RDC_FI_POWER_USAGE;
|
||||
result = rdc_group_field_create(rdc_handle, 2,
|
||||
&field_ids[0], field_group_name, &field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error create field group, Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout<< "Created the field group " << field_group_id
|
||||
<< ": " << field_id_string(RDC_FI_GPU_MEMORY_USAGE) << ", "
|
||||
<< field_id_string(RDC_FI_POWER_USAGE) <<std::endl;
|
||||
|
||||
|
||||
// Let the RDC to watch the fields and groups. The fields will be updated
|
||||
// once per second, the max keep age is 1 minutes and only keep 10 samples.
|
||||
result = rdc_field_watch(rdc_handle, group_id,
|
||||
field_group_id, 1000000, 60, 10);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error watch group fields. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Start to watch group:"
|
||||
<< group_id << ", field_group:" << field_group_id << std::endl;
|
||||
std::cout << "Sleep a few seconds before retreive the data ...\n";
|
||||
|
||||
// Since we are running the RDC_OPERATION_MODE_AUTO mode, the rdc_update_
|
||||
// all_fields() will be called periodically at background. If running as
|
||||
// RDC_OPERATION_MODE_MANUAL mode, we must call rdc_field_update_all()
|
||||
// periodically to take samples.
|
||||
usleep(5000000); // sleep 5 seconds before fetch the stats
|
||||
|
||||
|
||||
// Retreive the field and group information from RDC
|
||||
rdc_group_info_t group_info;
|
||||
rdc_field_group_info_t field_info;
|
||||
result = rdc_group_gpu_get_info(rdc_handle, group_id, &group_info);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get gpu group info. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
result = rdc_group_field_get_info(rdc_handle, field_group_id, &field_info);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get field group info. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// Get the latest metrics
|
||||
std::cout << "Get the latest metrics for group:" << group_id
|
||||
<< " field_group:" << field_group_id << std::endl;
|
||||
std::cout << "time_stamp\t" << "GPU_index\t"
|
||||
<< "field_name\t\t" << "field_value\n";
|
||||
for (uint32_t gindex = 0; gindex < group_info.count; gindex++) {
|
||||
for (uint32_t findex = 0; findex < field_info.count; findex++) {
|
||||
rdc_field_value value;
|
||||
result = rdc_field_get_latest_value(rdc_handle,
|
||||
group_info.entity_ids[gindex], field_info.field_ids[findex], &value);
|
||||
if (result == RDC_ST_NOT_FOUND) {
|
||||
continue;
|
||||
}
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get least value. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
// We only support the integer metrics so far
|
||||
std::cout << value.ts <<"\t" << group_info.entity_ids[gindex]
|
||||
<< "\t\t" << std::left << std::setw(16)
|
||||
<< field_id_string(value.field_id) << "\t"
|
||||
<< value.value.l_int << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
// Stop watching the field group
|
||||
result = rdc_field_unwatch(rdc_handle, group_id, field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error stop watch fields. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Stop watch group:" << group_id << ", field_group:"
|
||||
<< field_group_id << std::endl;
|
||||
|
||||
// Get the history data last 10 seconds
|
||||
std::cout << "Get last 10 seconds metrics for group:" << group_id
|
||||
<< " field_group:" << field_group_id << std::endl;
|
||||
std::cout << "time_stamp\t" << "GPU_index\t"
|
||||
<< "field_name\t\t" << "field_value\n";
|
||||
start_timestamp = static_cast<uint64_t>(time(nullptr)-10)*1000;
|
||||
for (uint32_t gindex = 0; gindex < group_info.count; gindex++) {
|
||||
for (uint32_t findex = 0; findex < field_info.count; findex++) {
|
||||
since_timestamp = start_timestamp;
|
||||
while (true) {
|
||||
rdc_field_value value;
|
||||
result = rdc_field_get_value_since(rdc_handle,
|
||||
group_info.entity_ids[gindex] , field_info.field_ids[findex],
|
||||
since_timestamp, &next_timestamp, &value);
|
||||
if (result == RDC_ST_NOT_FOUND) {
|
||||
break;
|
||||
}
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error get history data. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << value.ts <<"\t" << group_info.entity_ids[gindex]
|
||||
<< "\t\t" << std::left << std::setw(16)
|
||||
<< field_id_string(value.field_id) << "\t"
|
||||
<< value.value.l_int << std::endl;
|
||||
since_timestamp = next_timestamp;
|
||||
} // while
|
||||
} // for findex
|
||||
} // for gindex
|
||||
|
||||
// Delete the field group and GPU group
|
||||
result = rdc_group_field_destroy(rdc_handle, field_group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error delete field group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Deleted the field group " << field_group_id << std::endl;
|
||||
|
||||
result = rdc_group_gpu_destroy(rdc_handle, group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error delete GPU group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
std::cout << "Deleted the GPU group " << group_id << std::endl;
|
||||
|
||||
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
}
|
||||
|
||||
+130
-142
@@ -21,162 +21,150 @@ THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "rdc/rdc.h"
|
||||
|
||||
int main(int, char **) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"group1"};
|
||||
char job_id[] = {"123"};
|
||||
int main(int, char**) {
|
||||
rdc_status_t result;
|
||||
rdc_handle_t rdc_handle;
|
||||
bool standalone = false;
|
||||
char hostIpAddress[] = {"127.0.0.1:50051"};
|
||||
char group_name[] = {"group1"};
|
||||
char job_id[] = {"123"};
|
||||
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone ? "Standalone mode selected.\n" : "Embedded mode selected.\n");
|
||||
|
||||
// Select the embedded mode and standalone mode dynamically.
|
||||
std::cout << "Start rdci in: \n";
|
||||
std::cout << "0 - Embedded mode \n";
|
||||
std::cout << "1 - Standalone mode \n";
|
||||
while (!(std::cin >> standalone)) {
|
||||
std::cout << "Invalid input.\n";
|
||||
std::cin.clear();
|
||||
std::cin.ignore();
|
||||
}
|
||||
std::cout << std::endl;
|
||||
std::cout << (standalone?
|
||||
"Standalone mode selected.\n":"Embedded mode selected.\n");
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
|
||||
// Init the rdc
|
||||
result = rdc_init(0);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " << rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle, nullptr, nullptr, nullptr);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error initializing RDC. Return: " <<
|
||||
rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
} else {
|
||||
std::cout << "RDC Initialized.\n";
|
||||
std::cout << "Error connecting to remote rdcd. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
if (standalone) { // standalone
|
||||
result = rdc_connect(hostIpAddress, &rdc_handle,
|
||||
nullptr, nullptr, nullptr);
|
||||
if ( result != RDC_ST_OK ) {
|
||||
std::cout << "Error connecting to remote rdcd. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_MANUAL, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error starting embedded RDC engine. Return: "
|
||||
<< rdc_status_string(result) << std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// (1) create group and add GPUs
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_EMPTY,
|
||||
group_name, &group_id);
|
||||
} else { // embedded
|
||||
result = rdc_start_embedded(RDC_OPERATION_MODE_MANUAL, &rdc_handle);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
std::cout << "Error starting embedded RDC engine. Return: " << rdc_status_string(result)
|
||||
<< std::endl;
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
// Now we can use the same API for both standalone and embedded
|
||||
// (1) create group and add GPUs
|
||||
rdc_gpu_group_t group_id;
|
||||
result = rdc_group_gpu_create(rdc_handle, RDC_GROUP_EMPTY, group_name, &group_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error creating group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
result = rdc_group_gpu_add(rdc_handle, group_id, 0); // Add GPU 0
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error adding group. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// (2) start the recording. Set the sample frequency to once per second.
|
||||
result = rdc_job_start_stats(rdc_handle, group_id, job_id, 1000000);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error start job stats. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// For standalone mode, the daemon will update and cache the samples
|
||||
// In manual mode, we must call rdc_field_update_all periodically to
|
||||
// take samples.
|
||||
if (!standalone) { // embedded manual mode
|
||||
for (int i = 5; i > 0; i--) { // As an example, we will take 5 samples
|
||||
result = rdc_field_update_all(rdc_handle, 0);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error update all fields. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
usleep(1000000);
|
||||
}
|
||||
} else { // standalone mode, do nothing
|
||||
usleep(5000000); // sleep 5 seconds before fetch the stats
|
||||
}
|
||||
|
||||
result = rdc_group_gpu_add(rdc_handle, group_id, 0); // Add GPU 0
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error adding group. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
// (3) stop the Slurm job, which will stop the watch
|
||||
// We do not have to stop the job to get stats. The rdc_job_get_stats can be
|
||||
// called at any time before stop
|
||||
result = rdc_job_stop_stats(rdc_handle, job_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error stop job stats. Return: " << rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// (2) start the recording. Set the sample frequency to once per second.
|
||||
result = rdc_job_start_stats(rdc_handle, group_id,
|
||||
job_id, 1000000);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error start job stats. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
// (4) Get the stats
|
||||
rdc_job_info_t job_info;
|
||||
result = rdc_job_get_stats(rdc_handle, job_id, &job_info);
|
||||
|
||||
// For standalone mode, the daemon will update and cache the samples
|
||||
// In manual mode, we must call rdc_field_update_all periodically to
|
||||
// take samples.
|
||||
if (!standalone) { // embedded manual mode
|
||||
for (int i=5; i > 0 ; i--) { // As an example, we will take 5 samples
|
||||
result = rdc_field_update_all(rdc_handle, 0);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error update all fields. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
usleep(1000000);
|
||||
}
|
||||
} else { // standalone mode, do nothing
|
||||
usleep(5000000); // sleep 5 seconds before fetch the stats
|
||||
}
|
||||
if (result == RDC_ST_OK) {
|
||||
std::cout << "|------- Execution Stats ----------+"
|
||||
<< "------------------------------------\n";
|
||||
std::cout << "| Start Time * | " << job_info.summary.start_time << "\n";
|
||||
std::cout << "| End Time * | " << job_info.summary.end_time << "\n";
|
||||
std::cout << "| Total Execution Time (sec) * | "
|
||||
<< (job_info.summary.end_time - job_info.summary.start_time) << "\n";
|
||||
std::cout << "+------- Performance Stats --------+"
|
||||
<< "------------------------------------\n";
|
||||
std::cout << "| Energy Consumed (Joules) | " << job_info.summary.energy_consumed
|
||||
<< "\n";
|
||||
std::cout << "| Power Usage (Watts) | "
|
||||
<< "Max: " << job_info.summary.power_usage.max_value
|
||||
<< " Min: " << job_info.summary.power_usage.min_value
|
||||
<< " Avg: " << job_info.summary.power_usage.average << "\n";
|
||||
std::cout << "| GPU Clock (MHz) | "
|
||||
<< "Max: " << job_info.summary.gpu_clock.max_value
|
||||
<< " Min: " << job_info.summary.gpu_clock.min_value
|
||||
<< " Avg: " << job_info.summary.gpu_clock.average << "\n";
|
||||
std::cout << "| GPU Utilization (%) | "
|
||||
<< "Max: " << job_info.summary.gpu_utilization.max_value
|
||||
<< " Min: " << job_info.summary.gpu_utilization.min_value
|
||||
<< " Avg: " << job_info.summary.gpu_utilization.average << "\n";
|
||||
std::cout << "| Max GPU Memory Used (bytes) * | " << job_info.summary.max_gpu_memory_used
|
||||
<< "\n";
|
||||
std::cout << "| Memory Utilization (%) | "
|
||||
<< "Max: " << job_info.summary.memory_utilization.max_value
|
||||
<< " Min: " << job_info.summary.memory_utilization.min_value
|
||||
<< " Avg: " << job_info.summary.memory_utilization.average << "\n";
|
||||
std::cout << "+----------------------------------+"
|
||||
<< "------------------------------------\n";
|
||||
} else {
|
||||
std::cout << "No data for job stats found." << std::endl;
|
||||
}
|
||||
|
||||
// (3) stop the Slurm job, which will stop the watch
|
||||
// We do not have to stop the job to get stats. The rdc_job_get_stats can be
|
||||
// called at any time before stop
|
||||
result = rdc_job_stop_stats(rdc_handle, job_id);
|
||||
if (result != RDC_ST_OK) {
|
||||
std::cout << "Error stop job stats. Return: "
|
||||
<< rdc_status_string(result);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
// (4) Get the stats
|
||||
rdc_job_info_t job_info;
|
||||
result = rdc_job_get_stats(rdc_handle, job_id, &job_info);
|
||||
|
||||
if (result == RDC_ST_OK) {
|
||||
std::cout << "|------- Execution Stats ----------+"
|
||||
<<"------------------------------------\n";
|
||||
std::cout << "| Start Time * | "
|
||||
<< job_info.summary.start_time<< "\n";
|
||||
std::cout << "| End Time * | "
|
||||
<< job_info.summary.end_time <<"\n";
|
||||
std::cout << "| Total Execution Time (sec) * | "
|
||||
<< (job_info.summary.end_time-job_info.summary.start_time)
|
||||
<< "\n";
|
||||
std::cout << "+------- Performance Stats --------+"
|
||||
<< "------------------------------------\n";
|
||||
std::cout << "| Energy Consumed (Joules) | "
|
||||
<< job_info.summary.energy_consumed << "\n";
|
||||
std::cout << "| Power Usage (Watts) | "
|
||||
<< "Max: " << job_info.summary.power_usage.max_value
|
||||
<< " Min: "<< job_info.summary.power_usage.min_value
|
||||
<< " Avg: "<< job_info.summary.power_usage.average << "\n";
|
||||
std::cout << "| GPU Clock (MHz) | "
|
||||
<< "Max: " <<job_info.summary.gpu_clock.max_value
|
||||
<<" Min: " << job_info.summary.gpu_clock.min_value
|
||||
<<" Avg: "<< job_info.summary.gpu_clock.average << "\n";
|
||||
std::cout << "| GPU Utilization (%) | "
|
||||
<< "Max: " <<job_info.summary.gpu_utilization.max_value
|
||||
<<" Min: "<< job_info.summary.gpu_utilization.min_value
|
||||
<<" Avg: "<< job_info.summary.gpu_utilization.average << "\n";
|
||||
std::cout << "| Max GPU Memory Used (bytes) * | "
|
||||
<< job_info.summary.max_gpu_memory_used << "\n";
|
||||
std::cout << "| Memory Utilization (%) | "
|
||||
<< "Max: " << job_info.summary.memory_utilization.max_value
|
||||
<<" Min: "<< job_info.summary.memory_utilization.min_value
|
||||
<<" Avg: "<< job_info.summary.memory_utilization.average << "\n";
|
||||
std::cout << "+----------------------------------+"
|
||||
<< "------------------------------------\n";
|
||||
} else {
|
||||
std::cout << "No data for job stats found." << std::endl;
|
||||
}
|
||||
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
// Cleanup consists of shutting down RDC.
|
||||
cleanup:
|
||||
std::cout << "Cleaning up.\n";
|
||||
if (standalone)
|
||||
rdc_disconnect(rdc_handle);
|
||||
else
|
||||
rdc_stop_embedded(rdc_handle);
|
||||
rdc_shutdown();
|
||||
return result;
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user