Enable RDC policy feature

1. Add policy APIs
2. Add policy example for policy API usage

Change-Id: I14deb7c809d0b865b7bb083842092fc37868025e
Signed-off-by: Chao Fei <Chao.Fei@amd.com>

rdc_libs/bootstrap/src/RdcBootStrap.cc

@@ -409,3 +409,49 @@ char* strncpy_with_null(char* dest, const char* src, size_t n) {
   dest[n - 1] = '\0';
   return dest;
 }
+
+rdc_status_t rdc_policy_set(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                            rdc_policy_t policy) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)->rdc_policy_set(group_id, policy);
+}
+
+rdc_status_t rdc_policy_get(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id, uint32_t* count,
+                            rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)->rdc_policy_get(group_id, count, policies);
+}
+
+
+rdc_status_t rdc_policy_delete(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                                         rdc_policy_condition_type_t condition_type){
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)->rdc_policy_delete(group_id, condition_type);
+}
+
+
+
+rdc_status_t rdc_policy_register(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                                 rdc_policy_register_callback callback) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_policy_register(group_id, callback);
+}
+rdc_status_t rdc_policy_unregister(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id) {
+  if (!p_rdc_handle) {
+    return RDC_ST_INVALID_HANDLER;
+  }
+  return static_cast<amd::rdc::RdcHandler*>(p_rdc_handle)
+      ->rdc_policy_unregister(group_id);
+}

rdc_libs/rdc/CMakeLists.txt

@@ -18,6 +18,7 @@ set(RDC_LIB_SRC_LIST ${RDC_LIB_SRC_LIST}
     "${SRC_DIR}/RdcModuleMgrImpl.cc"
     "${SRC_DIR}/RdcNotificationImpl.cc"
     "${SRC_DIR}/RdcPerfTimer.cc"
+    "${SRC_DIR}/RdcPolicyImpl.cc"
     "${SRC_DIR}/RdcRocpLib.cc"
     "${SRC_DIR}/RdcRocrLib.cc"
     "${SRC_DIR}/RdcRVSLib.cc"

rdc_libs/rdc/src/RdcEmbeddedHandler.cc

@@ -34,6 +34,7 @@ THE SOFTWARE.
 #include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
 #include "rdc_lib/impl/RdcModuleMgrImpl.h"
 #include "rdc_lib/impl/RdcNotificationImpl.h"
+#include "rdc_lib/impl/RdcPolicyImpl.h"
 #include "rdc_lib/impl/RdcWatchTableImpl.h"
 #include "rdc_lib/rdc_common.h"
 
@@ -78,7 +79,8 @@ RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode)
       rdc_module_mgr_(new RdcModuleMgrImpl(metric_fetcher_)),
       rdc_notif_(new RdcNotificationImpl()),
       watch_table_(new RdcWatchTableImpl(group_settings_, cache_mgr_, rdc_module_mgr_, rdc_notif_)),
-      metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)) {
+      metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)),
+      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)) {
   if (mode == RDC_OPERATION_MODE_AUTO) {
     RDC_LOG(RDC_DEBUG, "Run RDC with RDC_OPERATION_MODE_AUTO");
     metrics_updater_->start();
@@ -429,5 +431,33 @@ rdc_status_t RdcEmbeddedHandler::get_mixed_component_version(mixed_component_t c
   return RDC_ST_OK;
 }
 
+// Policy API
+rdc_status_t RdcEmbeddedHandler::rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) {
+  return policy_->rdc_policy_set(group_id, policy);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                                rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  if (count == nullptr) {
+    return RDC_ST_BAD_PARAMETER;
+  }
+
+  return policy_->rdc_policy_get(group_id, count, policies);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_delete(rdc_gpu_group_t group_id,
+                                                   rdc_policy_condition_type_t condition_type) {
+  return policy_->rdc_policy_delete(group_id, condition_type);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_register(rdc_gpu_group_t group_id,
+                                                     rdc_policy_register_callback callback) {
+  return policy_->rdc_policy_register(group_id, callback);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_unregister(rdc_gpu_group_t group_id) {
+  return policy_->rdc_policy_unregister(group_id);
+}
+
 }  // namespace rdc
 }  // namespace amd

rdc_libs/rdc/src/RdcMetricFetcherImpl.cc

@@ -664,6 +664,15 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
       }
       break;
     }
+    case RDC_FI_GPU_PAGE_RETRIED:
+      uint32_t num_pages;
+      amdsmi_retired_page_record_t info;
+      value->status = amdsmi_get_gpu_bad_page_info(processor_handle, &num_pages, &info);
+      value->type = INTEGER;
+      if (value->status == AMDSMI_STATUS_SUCCESS) {
+        value->value.l_int = num_pages;
+      }
+      break;
     case RDC_FI_OAM_ID: {
       amdsmi_asic_info_t asic_info;
       value->status = amdsmi_get_gpu_asic_info(processor_handle, &asic_info);

rdc_libs/rdc/src/RdcPolicyImpl.cc

@@ -0,0 +1,258 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "rdc_lib/impl/RdcPolicyImpl.h"
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <ctime>
+#include <map>
+#include <sstream>
+#include <unordered_map>
+
+#include "common/rdc_utils.h"
+#include "rdc/rdc.h"
+#include "rdc_lib/RdcLogger.h"
+#include "rdc_lib/impl/SmiUtils.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcPolicyImpl::RdcPolicyImpl(const RdcGroupSettingsPtr& group_settings,
+                             const RdcMetricFetcherPtr& metric_fetcher)
+    : group_settings_(group_settings), metric_fetcher_(metric_fetcher), start_(true) {
+  thread_ = std::thread([this]() {
+    while (start_) {
+      rdc_policy_check_condition();
+      usleep(500);  // 500ms
+    }
+  });
+}
+
+RdcPolicyImpl::~RdcPolicyImpl() {
+  start_ = false;
+  thread_.join();
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) {
+  rdc_status_t status = RDC_ST_NOT_SUPPORTED;
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  // parameters check
+  if (policy.condition.type >= RDC_POLICY_COND_MAX) {
+    status = RDC_ST_BAD_PARAMETER;
+    return status;
+  }
+
+  if (policy.action > RDC_POLICY_ACTION_GPU_RESET) {
+    status = RDC_ST_BAD_PARAMETER;
+    return status;
+  }
+
+  // check if support RDC_POLICY_COND_MAX_PAGE_RETRIED
+  if (RDC_POLICY_COND_MAX_PAGE_RETRIED == policy.condition.type) {
+    uint32_t gpu_index;
+    rdc_group_info_t group_info;
+    rdc_field_value value;
+
+    status = group_settings_->rdc_group_gpu_get_info(group_id, &group_info);
+    for (unsigned int i = 0; i < group_info.count; i++) {
+      gpu_index = group_info.entity_ids[i];
+
+      status = metric_fetcher_->fetch_smi_field(gpu_index, RDC_FI_GPU_PAGE_RETRIED, &value);
+      if (status == RDC_ST_MSI_ERROR)
+        return RDC_ST_NOT_SUPPORTED;
+    }
+  }
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies = it->second;
+
+    bool exist = false;
+    for (auto& itpolicy : policies) {
+      // if exist, overwrite the value and action
+      if (itpolicy.condition.type == policy.condition.type) {
+        itpolicy.condition.value = policy.condition.value;
+        itpolicy.action = policy.action;
+        exist = true;
+      }
+    }
+    if (!exist) {
+      policies.push_back(policy);
+    }
+
+    status = RDC_ST_OK;
+  } else {
+    std::vector<rdc_policy_t> policies;
+    policies.push_back(policy);
+
+    settings_.insert(std::make_pair(group_id, policies));
+    status = RDC_ST_OK;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                           rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  rdc_status_t status = RDC_ST_NOT_SUPPORTED;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies_ref = it->second;
+    uint32_t i = 0;
+    for (auto itpolicy : policies_ref) {
+      // if exist
+      policies[i].condition.type = itpolicy.condition.type;
+      policies[i].condition.value = itpolicy.condition.value;
+      policies[i].action = itpolicy.action;
+      ++i;
+    }
+    *count = i;
+
+    status = RDC_ST_OK;
+  } else {
+    status = RDC_ST_NOT_FOUND;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_delete(rdc_gpu_group_t group_id,
+                                              rdc_policy_condition_type_t condition_type) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies_ref = it->second;
+
+    auto itpolicy = policies_ref.begin();
+    while (itpolicy != policies_ref.end()) {
+      if (itpolicy->condition.type == condition_type) {
+        status = RDC_ST_OK;
+        itpolicy = policies_ref.erase(itpolicy);
+      } else {
+        ++itpolicy;
+      }
+    }
+  } else {
+    status = RDC_ST_NOT_FOUND;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_register(rdc_gpu_group_t group_id,
+                                                rdc_policy_register_callback callback)
+
+{
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto run = callbacks_.find(group_id);
+  if (run != callbacks_.end()) {
+    run->second = callback;
+    status = RDC_ST_OK;
+  } else {
+    callbacks_.insert(std::make_pair(group_id, callback));
+    status = RDC_ST_OK;
+  }
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_unregister(rdc_gpu_group_t group_id) {
+  rdc_status_t status = RDC_ST_OK;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  callbacks_.erase(group_id);
+  return status;
+}
+
+void RdcPolicyImpl::rdc_policy_check_condition() {
+  // go through the settings
+  for (auto it : settings_) {
+    rdc_gpu_group_t group_id = it.first;
+    std::vector<rdc_policy_t> policies = it.second;
+    rdc_policy_register_callback callback = rdc_policy_get_callback(group_id);
+
+    for (auto policy : policies) {
+      rdc_status_t status;
+      rdc_field_value value;
+      rdc_group_info_t group_info;
+      uint32_t gpu_index;
+
+      status = group_settings_->rdc_group_gpu_get_info(group_id, &group_info);
+      for (unsigned int i = 0; i < group_info.count; i++) {
+        rdc_field_t map[RDC_POLICY_COND_MAX] = {RDC_FI_GPU_PAGE_RETRIED, RDC_FI_GPU_TEMP,
+                                                RDC_FI_POWER_USAGE};
+
+        gpu_index = group_info.entity_ids[i];
+        status = metric_fetcher_->fetch_smi_field(gpu_index, map[policy.condition.type], &value);
+        if (status == RDC_ST_OK) {
+          if (value.value.l_int > policy.condition.value) {
+            if (RDC_POLICY_ACTION_GPU_RESET == policy.action) {
+              rdc_policy_gpu_reset(gpu_index);
+            }
+
+            // callback if needed
+            if (callback) {
+              rdc_policy_callback_response_t response = {1, policy.condition, group_id,
+                                                         value.value.l_int};
+              callback(&response);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+rdc_policy_register_callback RdcPolicyImpl::rdc_policy_get_callback(rdc_gpu_group_t group_id) {
+  rdc_policy_register_callback cb = nullptr;
+  auto it = callbacks_.find(group_id);
+  if (it != callbacks_.end()) {
+    cb = it->second;
+  }
+  return cb;
+}
+
+void RdcPolicyImpl::rdc_policy_gpu_reset(uint32_t gpu_index) {
+  amdsmi_processor_handle processor_handle = {};
+
+  amdsmi_status_t err = get_processor_handle_from_id(gpu_index, &processor_handle);
+  if (err == AMDSMI_STATUS_SUCCESS) {
+    amdsmi_reset_gpu(processor_handle);
+  }
+}
+
+}  // namespace rdc
+}  // namespace amd

rdc_libs/rdc_client/src/RdcStandaloneHandler.cc

@@ -23,6 +23,8 @@ THE SOFTWARE.
 
 #include <grpcpp/grpcpp.h>
 
+#include <future>
+
 #include "rdc.grpc.pb.h"  // NOLINT
 
 amd::rdc::RdcHandler* make_handler(const char* ip_and_port, const char* root_ca,
@@ -243,8 +245,8 @@ rdc_status_t RdcStandaloneHandler::rdc_device_get_attributes(uint32_t gpu_index,
   return RDC_ST_OK;
 }
 
-rdc_status_t RdcStandaloneHandler::rdc_device_get_component_version(rdc_component_t component, rdc_component_version_t* p_rdc_compv) {
-
+rdc_status_t RdcStandaloneHandler::rdc_device_get_component_version(
+    rdc_component_t component, rdc_component_version_t* p_rdc_compv) {
   if (!p_rdc_compv) {
     return RDC_ST_BAD_PARAMETER;
   }
@@ -663,8 +665,8 @@ rdc_status_t RdcStandaloneHandler::rdc_field_update_all(uint32_t wait_for_update
 }
 
 // It is only an interface for the client under the GRPC framework and is not used as an RDC API.
-rdc_status_t RdcStandaloneHandler::get_mixed_component_version(mixed_component_t component, mixed_component_version_t* p_mixed_compv) {
-
+rdc_status_t RdcStandaloneHandler::get_mixed_component_version(
+    mixed_component_t component, mixed_component_version_t* p_mixed_compv) {
   if (!p_mixed_compv) {
     return RDC_ST_BAD_PARAMETER;
   }
@@ -681,7 +683,153 @@ rdc_status_t RdcStandaloneHandler::get_mixed_component_version(mixed_component_t
 
   strncpy_with_null(p_mixed_compv->version, reply.version().c_str(), USR_MAX_VERSION_STR_LENGTH);
   return RDC_ST_OK;
+}
 
+// Policy RdcAPI
+rdc_status_t RdcStandaloneHandler::rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) {
+  ::rdc::SetPolicyRequest request;
+  ::rdc::SetPolicyResponse reply;
+  ::grpc::ClientContext context;
+
+  request.set_group_id(group_id);
+  auto to_policy = request.mutable_policy();
+  to_policy->set_action(static_cast<::rdc::Policy_Action>(policy.action));
+
+  auto to_condition = to_policy->mutable_condition();
+
+  to_condition->set_type(static_cast<::rdc::PolicyCondition_Type>(policy.condition.type));
+  to_condition->set_value(policy.condition.value);
+
+  // call gRPC
+  ::grpc::Status status = stub_->SetPolicy(&context, request, &reply);
+
+  return error_handle(status, reply.status());
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                                  rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  ::rdc::GetPolicyRequest request;
+  ::rdc::GetPolicyResponse reply;
+  ::grpc::ClientContext context;
+
+  if (count == nullptr) {
+    return RDC_ST_BAD_PARAMETER;
+  }
+
+  request.set_group_id(group_id);
+
+  // call gRPC
+  ::grpc::Status status = stub_->GetPolicy(&context, request, &reply);
+  rdc_status_t err_status = error_handle(status, reply.status());
+  if (err_status != RDC_ST_OK) return err_status;
+
+  auto response = reply.response();
+  uint32_t policy_count = response.count();
+
+  for (uint32_t i = 0; i < policy_count; ++i) {
+    const ::rdc::Policy& policy = response.policies(i);
+
+    ::rdc::PolicyCondition cond = policy.condition();
+    policies[i].condition.type = static_cast<rdc_policy_condition_type_t>(cond.type());
+    policies[i].condition.value = cond.value();
+    policies[i].action = static_cast<rdc_policy_action_t>(policy.action());
+  }
+
+  *count = policy_count;
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_policy_delete(rdc_gpu_group_t group_id,
+                                                     rdc_policy_condition_type_t condition_type) {
+  ::rdc::DeletePolicyRequest request;
+  ::rdc::DeletePolicyResponse reply;
+  ::grpc::ClientContext context;
+
+  request.set_group_id(group_id);
+
+  request.set_condition_type(
+      static_cast<::rdc::DeletePolicyRequest_PolicyConditionType>(condition_type));
+
+  // call gRPC
+  ::grpc::Status status = stub_->DeletePolicy(&context, request, &reply);
+
+  return error_handle(status, reply.status());
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_policy_register(rdc_gpu_group_t group_id,
+                                                       rdc_policy_register_callback callback) {
+  // check if a thread for a group is already registered
+  auto it = policy_threads_.find(group_id);
+  if (it != policy_threads_.end()) {
+    return RDC_ST_CONFLICT;
+  }
+
+  // no registered callback, start the thread to read the stream from rdcd
+  struct policy_thread_context ctx = {true,nullptr};
+
+  ctx.t = new std::thread([this, group_id, callback]() {
+    // call rdcd
+    ::rdc::RegisterPolicyRequest request;
+    ::rdc::RegisterPolicyResponse reply;
+    ::grpc::ClientContext context;
+
+    request.set_group_id(group_id);
+
+    // call to gRPC
+    std::unique_ptr<grpc::ClientReader<::rdc::RegisterPolicyResponse>> reader(
+        stub_->RegisterPolicy(&context, request));
+
+    bool start = true;
+    while (start) {
+      auto it = policy_threads_.find(group_id);
+      if (it != policy_threads_.end()) {
+        if (it->second.start == false) start = false;
+      } else {
+        start = false;
+      }
+
+      if (reader->Read(&reply)) {
+        reply.status();
+        ::rdc::PolicyCondition cond = reply.condition();
+
+        rdc_policy_callback_response_t response;
+        response.version = reply.version();
+        response.condition.type = static_cast<rdc_policy_condition_type_t>(cond.type());
+        response.condition.value = cond.value();
+        response.group_id = reply.group_id();
+        response.value = reply.value();
+
+        callback(&response);
+      }
+    }
+
+    reader->Finish();
+  });
+
+  policy_threads_.insert(std::make_pair(group_id, ctx));
+
+  return RDC_ST_OK;
+}
+
+rdc_status_t RdcStandaloneHandler::rdc_policy_unregister(rdc_gpu_group_t group_id) {
+  ::rdc::UnRegisterPolicyRequest request;
+  ::rdc::UnRegisterPolicyResponse reply;
+  ::grpc::ClientContext context;
+
+  // stop the assocaticted thread of a group
+  auto it = policy_threads_.find(group_id);
+  if (it != policy_threads_.end()) {
+    struct policy_thread_context& ctx = it->second;
+    ctx.start = false;
+  }
+
+  // construcut the request
+  request.set_group_id(group_id);
+
+  // call gRPC
+  ::grpc::Status status = stub_->UnRegisterPolicy(&context, request, &reply);
+  return error_handle(status, reply.status());
 }
 
 }  // namespace rdc