diff --git a/projects/rdc/example/job_stats_example.cc b/projects/rdc/example/job_stats_example.cc index 2d9493f9d7..e4de12df6d 100644 --- a/projects/rdc/example/job_stats_example.cc +++ b/projects/rdc/example/job_stats_example.cc @@ -66,7 +66,7 @@ int main(int, char **) { goto cleanup; } } else { // embedded - result = rdc_start_embedded(RDC_OPERATION_MODE_AUTO, &rdc_handle); + result = rdc_start_embedded(RDC_OPERATION_MODE_MANUAL, &rdc_handle); if (result != RDC_ST_OK) { std::cout << "Error starting embedded RDC engine. Return: " << rdc_status_string(result) << std::endl; diff --git a/projects/rdc/include/rdc/rdc.h b/projects/rdc/include/rdc/rdc.h index a1066a2666..f429342727 100755 --- a/projects/rdc/include/rdc/rdc.h +++ b/projects/rdc/include/rdc/rdc.h @@ -290,7 +290,7 @@ rdc_status_t rdc_shutdown(); * @details The RDC is loaded as library so that it does not require rdcd * daemon. In this mode, the user has to periodically call * rdc_field_update_all() when op_mode is RDC_OPERATION_MODE_MANUAL, which - * tells RDC to collect the stats. This function is not thread safe. + * tells RDC to collect the stats. * * @param[in] op_mode Operation modes. When RDC_OPERATION_MODE_AUTO, RDC schedules * background task to collect the stats. When RDC_OPERATION_MODE_MANUAL, the user @@ -308,7 +308,7 @@ rdc_status_t rdc_start_embedded(rdc_operation_mode_t op_mode, * @brief Stop embedded RDC agent. * * @details Stop the embedded RDC agent, and p_rdc_handle becomes - * invalid after this call. This function is not thread safe. + * invalid after this call. * * @param[in] p_rdc_handle The RDC handler that come from * rdc_start_embedded(). @@ -320,7 +320,7 @@ rdc_status_t rdc_stop_embedded(rdc_handle_t p_rdc_handle); * @brief Connect to rdcd daemon * * @details This method is used to connect to a remote stand-alone - * rdcd daemon. This function is not thread safe. + * rdcd daemon. * * @param[in] ipAndPort The IP and port of the remote rdcd. The ipAndPort * can be specified in this x.x.x.x:yyyy format, where x.x.x.x is the @@ -348,7 +348,7 @@ rdc_status_t rdc_connect(const char *ipAndPort, rdc_handle_t* p_rdc_handle, * @brief Disconnect from rdcd daemon. * * @details Disconnect from rdcd daemon, and p_rdc_handle becomes invalid - * after this call. This function is not thread safe. + * after this call. * * @param[in] p_rdc_handle The RDC handler that come from rdc_connect(). * diff --git a/projects/rdc/include/rdc_lib/RdcLibraryLoader.h b/projects/rdc/include/rdc_lib/RdcLibraryLoader.h new file mode 100644 index 0000000000..7a3d7fe2be --- /dev/null +++ b/projects/rdc/include/rdc_lib/RdcLibraryLoader.h @@ -0,0 +1,86 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +#ifndef RDC_LIB_RDCLIBRARYLOADER_H_ +#define RDC_LIB_RDCLIBRARYLOADER_H_ +#include +#include +#include +#include // NOLINT(build/c++11) +#include "rdc/rdc.h" +#include "rdc_lib/RdcLogger.h" +#include "rdc_lib/rdc_common.h" + + +namespace amd { +namespace rdc { +class RdcLibraryLoader { + public: + RdcLibraryLoader(); + + template rdc_status_t load(const char* filename, + T* func_make_handler); + + rdc_status_t unload(); + + ~RdcLibraryLoader(); + + private: + void* libHandler_; + std::mutex library_mutex_; +}; + +template rdc_status_t RdcLibraryLoader::load(const char* filename, + T* func_make_handler) { + if (filename == nullptr || func_make_handler == nullptr) { + return RDC_ST_FAIL_LOAD_MODULE; + } + + if (libHandler_) { + unload(); + } + + std::lock_guard guard(library_mutex_); + libHandler_ = dlopen(filename, RTLD_LAZY); + if (!libHandler_) { + char* error = dlerror(); + RDC_LOG(RDC_ERROR, "Fail to open " << filename <<": " << error); + return RDC_ST_FAIL_LOAD_MODULE; + } + + *reinterpret_cast(func_make_handler) = + dlsym(libHandler_, "make_handler"); + if (*func_make_handler == nullptr) { + char* error = dlerror(); + RDC_LOG(RDC_ERROR, + "Fail to find function make_handler from file " + << filename <<": " << error); + return RDC_ST_FAIL_LOAD_MODULE; + } + + return RDC_ST_OK; +} + +} // namespace rdc +} // namespace amd + + +#endif // RDC_LIB_RDCLIBRARYLOADER_H_ diff --git a/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h b/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h index 59e642993a..57d4f73d96 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h +++ b/projects/rdc/include/rdc_lib/impl/RdcEmbeddedHandler.h @@ -22,6 +22,7 @@ THE SOFTWARE. #ifndef RDC_LIB_IMPL_RDCEMBEDDEDHANDLER_H_ #define RDC_LIB_IMPL_RDCEMBEDDEDHANDLER_H_ +#include // NOLINT(build/c++11) #include "rdc_lib/RdcHandler.h" #include "rdc_lib/RdcGroupSettings.h" #include "rdc_lib/RdcMetricFetcher.h" @@ -96,6 +97,7 @@ class RdcEmbeddedHandler: public RdcHandler { RdcMetricFetcherPtr metric_fetcher_; RdcWatchTablePtr watch_table_; RdcMetricsUpdaterPtr metrics_updater_; + std::future updater_; }; } // namespace rdc diff --git a/projects/rdc/include/rdc_lib/impl/RdcStandaloneHandler.h b/projects/rdc/include/rdc_lib/impl/RdcStandaloneHandler.h index ea216993ae..8573828213 100644 --- a/projects/rdc/include/rdc_lib/impl/RdcStandaloneHandler.h +++ b/projects/rdc/include/rdc_lib/impl/RdcStandaloneHandler.h @@ -90,6 +90,11 @@ class RdcStandaloneHandler: public RdcHandler { private: // Helper function to handle the error rdc_status_t error_handle(::grpc::Status status, uint32_t rdc_status); + + bool copy_gpu_usage_info( + const ::rdc::GpuUsageInfo& src, + rdc_gpu_usage_info_t* target); + std::unique_ptr<::rdc::RdcAPI::Stub> stub_; }; diff --git a/projects/rdc/protos/rdc.proto b/projects/rdc/protos/rdc.proto index 16399a3793..00ab6f4cc2 100755 --- a/projects/rdc/protos/rdc.proto +++ b/projects/rdc/protos/rdc.proto @@ -196,6 +196,24 @@ service RdcAPI { // rdc_status_t rdc_group_field_all_ids(rdc_field_grp_t field_group_id_list[], uint32_t* count) rpc GetFieldGroupAllIds(Empty) returns (GetFieldGroupAllIdsResponse) {} + + // JOB API + // rdc_status_t rdc_job_start_stats(rdc_gpu_group_t groupId, + // char job_id[64], uint64_t update_freq) + rpc StartJobStats(StartJobStatsRequest) returns (StartJobStatsResponse) {} + + // rdc_status_t rdc_job_get_stats(char jobId[64], + // rdc_job_info_t* p_job_info) + rpc GetJobStats(GetJobStatsRequest) returns (GetJobStatsResponse) {} + + // rdc_status_t rdc_job_stop_stats(char job_id[64]) + rpc StopJobStats(StopJobStatsRequest) returns (StopJobStatsResponse) {} + + // rdc_status_t rdc_job_remove(char job_id[64]) + rpc RemoveJob(RemoveJobRequest) returns (RemoveJobResponse) {} + + // rdc_status_t rdc_job_remove_all() + rpc RemoveAllJob(Empty) returns (RemoveAllJobResponse) {} } message Empty { @@ -377,3 +395,62 @@ message GetFieldGroupAllIdsResponse { uint32 status = 1; repeated uint32 field_group_ids = 2; } + +message StartJobStatsRequest { + uint32 group_id = 1; + string job_id = 2; + uint64 update_freq = 3; +} + +message StartJobStatsResponse { + uint32 status = 1; +} + +message GetJobStatsRequest { + string job_id = 1; +} + +message JobStatsSummary { + uint64 max_value = 1; + uint64 min_value = 2; + uint64 average = 3; + +} + +message GpuUsageInfo { + uint32 gpu_id = 1; + uint64 start_time = 2; + uint64 end_time = 3; + uint64 energy_consumed = 4; + JobStatsSummary power_usage = 5; + JobStatsSummary gpu_clock = 6; + JobStatsSummary gpu_utilization = 7; + uint64 max_gpu_memory_used = 8; + JobStatsSummary memory_utilization = 9; +} +message GetJobStatsResponse { + uint32 status = 1; + uint32 num_gpus = 2; + GpuUsageInfo summary = 3; + repeated GpuUsageInfo gpus = 4; +} + +message StopJobStatsRequest { + string job_id = 1; +} + +message StopJobStatsResponse { + uint32 status = 1; +} + +message RemoveJobRequest { + string job_id = 1; +} + +message RemoveJobResponse { + uint32 status = 1; +} + +message RemoveAllJobResponse { + uint32 status = 1; +} \ No newline at end of file diff --git a/projects/rdc/rdc_libs/CMakeLists.txt b/projects/rdc/rdc_libs/CMakeLists.txt index 7d1b39cb3c..5565fba4d2 100755 --- a/projects/rdc/rdc_libs/CMakeLists.txt +++ b/projects/rdc/rdc_libs/CMakeLists.txt @@ -116,10 +116,12 @@ set(BOOTSTRAP_LIB "rdc_bootstrap") set(BOOTSTRAP_LIB_COMPONENT "lib${BOOTSTRAP_LIB}") set(BOOTSTRAP_LIB_SRC_LIST "${SRC_DIR}/bootstrap/src/RdcBootStrap.cc") set(BOOTSTRAP_LIB_SRC_LIST ${BOOTSTRAP_LIB_SRC_LIST} "${SRC_DIR}/bootstrap/src/RdcLogger.cc") +set(BOOTSTRAP_LIB_SRC_LIST ${BOOTSTRAP_LIB_SRC_LIST} "${SRC_DIR}/bootstrap/src/RdcLibraryLoader.cc") set(BOOTSTRAP_LIB_INC_LIST "${RDC_LIB_INC_DIR}/rdc/rdc.h") set(BOOTSTRAP_LIB_INC_LIST ${BOOTSTRAP_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/rdc_common.h") set(BOOTSTRAP_LIB_INC_LIST ${BOOTSTRAP_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcLogger.h") set(BOOTSTRAP_LIB_INC_LIST ${BOOTSTRAP_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcHandler.h") +set(BOOTSTRAP_LIB_INC_LIST ${BOOTSTRAP_LIB_INC_LIST} "${RDC_LIB_INC_DIR}/rdc_lib/RdcLibraryLoader.h") message("BOOTSTRAP_LIB_INC_LIST=${BOOTSTRAP_LIB_INC_LIST}") diff --git a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc index 7d066fd558..0326bd0f26 100644 --- a/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc +++ b/projects/rdc/rdc_libs/bootstrap/src/RdcBootStrap.cc @@ -26,19 +26,17 @@ THE SOFTWARE. #include "rdc_lib/RdcHandler.h" #include "rdc_lib/RdcLogger.h" #include "rdc_lib/rdc_common.h" +#include "rdc_lib/RdcLibraryLoader.h" -static void* libHandler = nullptr; + +static amd::rdc::RdcLibraryLoader rdc_lib_loader; rdc_status_t rdc_init(uint64_t) { return RDC_ST_OK; } rdc_status_t rdc_shutdown() { - if (libHandler) { - dlclose(libHandler); - libHandler = nullptr; - } - return RDC_ST_OK; + return rdc_lib_loader.unload(); } rdc_status_t rdc_connect(const char* ipAddress, @@ -46,24 +44,17 @@ rdc_status_t rdc_connect(const char* ipAddress, const char* root_ca, const char* client_cert, const char* client_key ) { amd::rdc::RdcHandler* (*func_make_handler)(const char*, - const char*, const char*, const char*); + const char*, const char*, const char*) = nullptr; if (!ipAddress || !p_rdc_handle) { return RDC_ST_FAIL_LOAD_MODULE; } - if (!libHandler) { - libHandler = dlopen("librdc_client.so", RTLD_LAZY); - } - - if (!libHandler) { - return RDC_ST_FAIL_LOAD_MODULE; - } - - *reinterpret_cast(&func_make_handler) = - dlsym(libHandler, "make_handler"); - if (!func_make_handler) { - return RDC_ST_FAIL_LOAD_MODULE; + rdc_status_t status = rdc_lib_loader.load("librdc_client.so", + &func_make_handler); + if (status != RDC_ST_OK) { + *p_rdc_handle = nullptr; + return status; } *p_rdc_handle = static_cast @@ -83,31 +74,17 @@ rdc_status_t rdc_disconnect(rdc_handle_t p_rdc_handle) { rdc_status_t rdc_start_embedded(rdc_operation_mode_t op_mode, rdc_handle_t* p_rdc_handle ) { - amd::rdc::RdcHandler* (*func_make_handler)(rdc_operation_mode_t); - char *error; + amd::rdc::RdcHandler* (*func_make_handler)(rdc_operation_mode_t) + = nullptr; if (!p_rdc_handle) { return RDC_ST_FAIL_LOAD_MODULE; } - dlerror(); - - if (!libHandler) { - libHandler = dlopen("librdc.so", RTLD_LAZY); - } - - if (!libHandler) { - error = dlerror(); - RDC_LOG(RDC_ERROR, "Fail to open librdc.so: " << error); - return RDC_ST_FAIL_LOAD_MODULE; - } - - *reinterpret_cast(&func_make_handler) = - dlsym(libHandler, "make_handler"); - if (!func_make_handler) { - error = dlerror(); - RDC_LOG(RDC_ERROR, - "Fail to find function make_handler:" << error); - return RDC_ST_FAIL_LOAD_MODULE; + rdc_status_t status = rdc_lib_loader.load("librdc.so", + &func_make_handler); + if (status != RDC_ST_OK) { + *p_rdc_handle = nullptr; + return status; } *p_rdc_handle = static_cast diff --git a/projects/rdc/rdc_libs/bootstrap/src/RdcLibraryLoader.cc b/projects/rdc/rdc_libs/bootstrap/src/RdcLibraryLoader.cc new file mode 100644 index 0000000000..014a661a94 --- /dev/null +++ b/projects/rdc/rdc_libs/bootstrap/src/RdcLibraryLoader.cc @@ -0,0 +1,45 @@ +/* +Copyright (c) 2020 - present Advanced Micro Devices, Inc. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +#include "rdc_lib/RdcLibraryLoader.h" + +namespace amd { +namespace rdc { + +RdcLibraryLoader::RdcLibraryLoader(): libHandler_(nullptr) { +} + +rdc_status_t RdcLibraryLoader::unload() { + std::lock_guard guard(library_mutex_); + if (libHandler_) { + dlclose(libHandler_); + libHandler_ = nullptr; + } + return RDC_ST_OK; +} + +RdcLibraryLoader::~RdcLibraryLoader() { + unload(); +} + +} // namespace rdc +} // namespace amd diff --git a/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc b/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc index 94313d3610..fa5e3d3dc1 100644 --- a/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc +++ b/projects/rdc/rdc_libs/rdc/src/RdcEmbeddedHandler.cc @@ -341,9 +341,16 @@ rdc_status_t RdcEmbeddedHandler::rdc_field_unwatch(rdc_gpu_group_t group_id, // Control API rdc_status_t RdcEmbeddedHandler::rdc_field_update_all( uint32_t wait_for_update) { - // TODO(bill_liu): implement the case wait_for_update==0 - (void)(wait_for_update); - return watch_table_->rdc_field_update_all(); + if (wait_for_update == 1) { + return watch_table_->rdc_field_update_all(); + } + + // Async update the field and return immediately. + updater_ = std::async(std::launch::async, [this](){ + watch_table_->rdc_field_update_all(); + }); + + return RDC_ST_OK; } } // namespace rdc diff --git a/projects/rdc/rdc_libs/rdc_client/src/RdcStandaloneHandler.cc b/projects/rdc/rdc_libs/rdc_client/src/RdcStandaloneHandler.cc index 3c7088e3a8..d6a845a708 100644 --- a/projects/rdc/rdc_libs/rdc_client/src/RdcStandaloneHandler.cc +++ b/projects/rdc/rdc_libs/rdc_client/src/RdcStandaloneHandler.cc @@ -63,40 +63,112 @@ rdc_status_t RdcStandaloneHandler::error_handle(::grpc::Status status, // JOB RdcAPI rdc_status_t RdcStandaloneHandler::rdc_job_start_stats(rdc_gpu_group_t groupId, char job_id[64], uint64_t update_freq) { - // TODO(bill_liu): implement - (void)(groupId); - (void)(job_id); - (void)(update_freq); + ::rdc::StartJobStatsRequest request; + ::rdc::StartJobStatsResponse reply; + ::grpc::ClientContext context; - return RDC_ST_OK; + request.set_group_id(groupId); + request.set_job_id(job_id); + request.set_update_freq(update_freq); + ::grpc::Status status = stub_->StartJobStats(&context, request, &reply); + rdc_status_t err_status = error_handle(status, reply.status()); + + return err_status; } +bool RdcStandaloneHandler::copy_gpu_usage_info( + const ::rdc::GpuUsageInfo& src, + rdc_gpu_usage_info_t* target) { + if (target == nullptr) { + return false; + } + + target->gpu_id = src.gpu_id(); + target->start_time = src.start_time(); + target->end_time = src.end_time(); + target->energy_consumed = src.energy_consumed(); + target->max_gpu_memory_used = src.max_gpu_memory_used(); + + const ::rdc::JobStatsSummary& pstats = src.power_usage(); + target->power_usage.max_value = pstats.max_value(); + target->power_usage.min_value = pstats.min_value(); + target->power_usage.average = pstats.average(); + + const ::rdc::JobStatsSummary& cstats = src.gpu_clock(); + target->gpu_clock.max_value = cstats.max_value(); + target->gpu_clock.min_value = cstats.min_value(); + target->gpu_clock.average = cstats.average(); + + const ::rdc::JobStatsSummary& ustats = src.gpu_utilization(); + target->gpu_utilization.max_value = ustats.max_value(); + target->gpu_utilization.min_value = ustats.min_value(); + target->gpu_utilization.average = ustats.average(); + + const ::rdc::JobStatsSummary& mstats = src.memory_utilization(); + target->memory_utilization.max_value = mstats.max_value(); + target->memory_utilization.min_value = mstats.min_value(); + target->memory_utilization.average = mstats.average(); + + return true; +} rdc_status_t RdcStandaloneHandler::rdc_job_get_stats(char job_id[64], rdc_job_info_t* p_job_info) { - // TODO(bill_liu): implement - (void)(job_id); - (void)(p_job_info); + if (!p_job_info) { + return RDC_ST_BAD_PARAMETER; + } + + ::rdc::GetJobStatsRequest request; + ::rdc::GetJobStatsResponse reply; + ::grpc::ClientContext context; + + request.set_job_id(job_id); + ::grpc::Status status = stub_->GetJobStats(&context, request, &reply); + rdc_status_t err_status = error_handle(status, reply.status()); + if (err_status != RDC_ST_OK) return err_status; + + p_job_info->num_gpus = reply.num_gpus(); + copy_gpu_usage_info(reply.summary(), &(p_job_info->summary)); + for (int i = 0; i < reply.gpus_size(); i++) { + copy_gpu_usage_info(reply.gpus(i), &(p_job_info->gpus[i])); + } + return RDC_ST_OK; } rdc_status_t RdcStandaloneHandler::rdc_job_stop_stats(char job_id[64]) { - // TODO(bill_liu): implement - (void)(job_id); - return RDC_ST_OK; + ::rdc::StopJobStatsRequest request; + ::rdc::StopJobStatsResponse reply; + ::grpc::ClientContext context; + + request.set_job_id(job_id); + ::grpc::Status status = stub_->StopJobStats(&context, request, &reply); + rdc_status_t err_status = error_handle(status, reply.status()); + + return err_status; } - rdc_status_t RdcStandaloneHandler::rdc_job_remove(char job_id[64]) { - // TODO(bill_liu): implement - (void)(job_id); - return RDC_ST_OK; + ::rdc::RemoveJobRequest request; + ::rdc::RemoveJobResponse reply; + ::grpc::ClientContext context; + + request.set_job_id(job_id); + ::grpc::Status status = stub_->RemoveJob(&context, request, &reply); + rdc_status_t err_status = error_handle(status, reply.status()); + + return err_status; } rdc_status_t RdcStandaloneHandler::rdc_job_remove_all() { - // TODO(bill_liu): implement - return RDC_ST_OK; -} + ::rdc::Empty request; + ::rdc::RemoveAllJobResponse reply; + ::grpc::ClientContext context; + ::grpc::Status status = stub_->RemoveAllJob(&context, request, &reply); + rdc_status_t err_status = error_handle(status, reply.status()); + + return err_status; +} // Discovery RdcAPI rdc_status_t RdcStandaloneHandler::rdc_device_get_all( diff --git a/projects/rdc/server/include/rdc/rdc_api_service.h b/projects/rdc/server/include/rdc/rdc_api_service.h index 50bbe7299c..7991d87e38 100755 --- a/projects/rdc/server/include/rdc/rdc_api_service.h +++ b/projects/rdc/server/include/rdc/rdc_api_service.h @@ -100,7 +100,29 @@ class RdcAPIServiceImpl final : public ::rdc::RdcAPI::Service { ::rdc::UpdateAllFieldsResponse* reply) override; + ::grpc::Status StartJobStats(::grpc::ServerContext* context, + const ::rdc::StartJobStatsRequest* request, + ::rdc::StartJobStatsResponse* reply) override; + + ::grpc::Status GetJobStats(::grpc::ServerContext* context, + const ::rdc::GetJobStatsRequest* request, + ::rdc::GetJobStatsResponse* reply) override; + + ::grpc::Status StopJobStats(::grpc::ServerContext* context, + const ::rdc::StopJobStatsRequest* request, + ::rdc::StopJobStatsResponse* reply) override; + + ::grpc::Status RemoveJob(::grpc::ServerContext* context, + const ::rdc::RemoveJobRequest* request, + ::rdc::RemoveJobResponse* reply) override; + + ::grpc::Status RemoveAllJob(::grpc::ServerContext* context, + const ::rdc::Empty* request, + ::rdc::RemoveAllJobResponse* reply) override; + private: + bool copy_gpu_usage_info(const rdc_gpu_usage_info_t& src, + ::rdc::GpuUsageInfo* target); rdc_handle_t rdc_handle_; }; diff --git a/projects/rdc/server/src/rdc_api_service.cc b/projects/rdc/server/src/rdc_api_service.cc index 3a184118e5..8cd10dc046 100755 --- a/projects/rdc/server/src/rdc_api_service.cc +++ b/projects/rdc/server/src/rdc_api_service.cc @@ -421,6 +421,140 @@ RdcAPIServiceImpl::~RdcAPIServiceImpl() { } +::grpc::Status RdcAPIServiceImpl::StartJobStats( + ::grpc::ServerContext* context, + const ::rdc::StartJobStatsRequest* request, + ::rdc::StartJobStatsResponse* reply) { + (void)(context); + if (!reply || !request) { + return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents"); + } + rdc_status_t result = rdc_job_start_stats( + rdc_handle_, request->group_id(), + const_cast(request->job_id().c_str()), + request->update_freq()); + reply->set_status(result); + + return ::grpc::Status::OK; +} + +::grpc::Status RdcAPIServiceImpl::GetJobStats( + ::grpc::ServerContext* context, + const ::rdc::GetJobStatsRequest* request, + ::rdc::GetJobStatsResponse* reply) { + (void)(context); + if (!reply || !request) { + return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents"); + } + + rdc_job_info_t job_info; + rdc_status_t result = rdc_job_get_stats( + rdc_handle_, + const_cast(request->job_id().c_str()), + &job_info); + + reply->set_num_gpus(job_info.num_gpus); + ::rdc::GpuUsageInfo* sinfo = reply->mutable_summary(); + copy_gpu_usage_info(job_info.summary, sinfo); + + for (uint32_t i = 0; i < job_info.num_gpus; i++) { + ::rdc::GpuUsageInfo* ginfo = reply->add_gpus(); + copy_gpu_usage_info(job_info.gpus[i], ginfo); + } + + reply->set_status(result); + + return ::grpc::Status::OK; +} + +bool RdcAPIServiceImpl::copy_gpu_usage_info(const rdc_gpu_usage_info_t& src, + ::rdc::GpuUsageInfo* target) { + if (target == nullptr) { + return false; + } + + target->set_gpu_id(src.gpu_id); + target->set_start_time(src.start_time); + target->set_end_time(src.end_time); + target->set_energy_consumed(src.energy_consumed); + target->set_max_gpu_memory_used(src.max_gpu_memory_used); + + ::rdc::JobStatsSummary* stats = target->mutable_power_usage(); + stats->set_max_value(src.power_usage.max_value); + stats->set_min_value(src.power_usage.min_value); + stats->set_average(src.power_usage.average); + + stats = target->mutable_gpu_clock(); + stats->set_max_value(src.gpu_clock.max_value); + stats->set_min_value(src.gpu_clock.min_value); + stats->set_average(src.gpu_clock.average); + + stats = target->mutable_gpu_utilization(); + stats->set_max_value(src.gpu_utilization.max_value); + stats->set_min_value(src.gpu_utilization.min_value); + stats->set_average(src.gpu_utilization.average); + + stats = target->mutable_memory_utilization(); + stats->set_max_value(src.memory_utilization.max_value); + stats->set_min_value(src.memory_utilization.min_value); + stats->set_average(src.memory_utilization.average); + + + return true; +} + + +::grpc::Status RdcAPIServiceImpl::StopJobStats( + ::grpc::ServerContext* context, + const ::rdc::StopJobStatsRequest* request, + ::rdc::StopJobStatsResponse* reply) { + (void)(context); + if (!reply || !request) { + return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents"); + } + + rdc_status_t result = rdc_job_stop_stats( + rdc_handle_, + const_cast(request->job_id().c_str())); + reply->set_status(result); + + return ::grpc::Status::OK; +} + + +::grpc::Status RdcAPIServiceImpl::RemoveJob( + ::grpc::ServerContext* context, + const ::rdc::RemoveJobRequest* request, + ::rdc::RemoveJobResponse* reply) { + (void)(context); + if (!reply || !request) { + return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents"); + } + rdc_status_t result = rdc_job_remove( + rdc_handle_, const_cast(request->job_id().c_str())); + reply->set_status(result); + + return ::grpc::Status::OK; +} + + +::grpc::Status RdcAPIServiceImpl::RemoveAllJob( + ::grpc::ServerContext* context, + const ::rdc::Empty* request, + ::rdc::RemoveAllJobResponse* reply) { + (void)(context); + if (!reply || !request) { + return ::grpc::Status(::grpc::StatusCode::INTERNAL, "Empty contents"); + } + rdc_status_t result = rdc_job_remove_all(rdc_handle_); + reply->set_status(result); + + + return ::grpc::Status::OK; +} + + + } // namespace rdc } // namespace amd