Enable RDC policy feature

1. Add policy APIs
2. Add policy example for policy API usage

Change-Id: I14deb7c809d0b865b7bb083842092fc37868025e
Signed-off-by: Chao Fei <Chao.Fei@amd.com>

rdc_libs/rdc/src/RdcEmbeddedHandler.cc

@@ -34,6 +34,7 @@ THE SOFTWARE.
 #include "rdc_lib/impl/RdcMetricsUpdaterImpl.h"
 #include "rdc_lib/impl/RdcModuleMgrImpl.h"
 #include "rdc_lib/impl/RdcNotificationImpl.h"
+#include "rdc_lib/impl/RdcPolicyImpl.h"
 #include "rdc_lib/impl/RdcWatchTableImpl.h"
 #include "rdc_lib/rdc_common.h"
 
@@ -78,7 +79,8 @@ RdcEmbeddedHandler::RdcEmbeddedHandler(rdc_operation_mode_t mode)
       rdc_module_mgr_(new RdcModuleMgrImpl(metric_fetcher_)),
       rdc_notif_(new RdcNotificationImpl()),
       watch_table_(new RdcWatchTableImpl(group_settings_, cache_mgr_, rdc_module_mgr_, rdc_notif_)),
-      metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)) {
+      metrics_updater_(new RdcMetricsUpdaterImpl(watch_table_, METIC_UPDATE_FREQUENCY)),
+      policy_(new RdcPolicyImpl(group_settings_,metric_fetcher_)) {
   if (mode == RDC_OPERATION_MODE_AUTO) {
     RDC_LOG(RDC_DEBUG, "Run RDC with RDC_OPERATION_MODE_AUTO");
     metrics_updater_->start();
@@ -429,5 +431,33 @@ rdc_status_t RdcEmbeddedHandler::get_mixed_component_version(mixed_component_t c
   return RDC_ST_OK;
 }
 
+// Policy API
+rdc_status_t RdcEmbeddedHandler::rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) {
+  return policy_->rdc_policy_set(group_id, policy);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                                rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  if (count == nullptr) {
+    return RDC_ST_BAD_PARAMETER;
+  }
+
+  return policy_->rdc_policy_get(group_id, count, policies);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_delete(rdc_gpu_group_t group_id,
+                                                   rdc_policy_condition_type_t condition_type) {
+  return policy_->rdc_policy_delete(group_id, condition_type);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_register(rdc_gpu_group_t group_id,
+                                                     rdc_policy_register_callback callback) {
+  return policy_->rdc_policy_register(group_id, callback);
+}
+
+rdc_status_t RdcEmbeddedHandler::rdc_policy_unregister(rdc_gpu_group_t group_id) {
+  return policy_->rdc_policy_unregister(group_id);
+}
+
 }  // namespace rdc
 }  // namespace amd

rdc_libs/rdc/src/RdcMetricFetcherImpl.cc

@@ -664,6 +664,15 @@ rdc_status_t RdcMetricFetcherImpl::fetch_smi_field(uint32_t gpu_index, rdc_field
       }
       break;
     }
+    case RDC_FI_GPU_PAGE_RETRIED:
+      uint32_t num_pages;
+      amdsmi_retired_page_record_t info;
+      value->status = amdsmi_get_gpu_bad_page_info(processor_handle, &num_pages, &info);
+      value->type = INTEGER;
+      if (value->status == AMDSMI_STATUS_SUCCESS) {
+        value->value.l_int = num_pages;
+      }
+      break;
     case RDC_FI_OAM_ID: {
       amdsmi_asic_info_t asic_info;
       value->status = amdsmi_get_gpu_asic_info(processor_handle, &asic_info);

rdc_libs/rdc/src/RdcPolicyImpl.cc

@@ -0,0 +1,258 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "rdc_lib/impl/RdcPolicyImpl.h"
+
+#include <sys/time.h>
+#include <unistd.h>
+
+#include <algorithm>
+#include <ctime>
+#include <map>
+#include <sstream>
+#include <unordered_map>
+
+#include "common/rdc_utils.h"
+#include "rdc/rdc.h"
+#include "rdc_lib/RdcLogger.h"
+#include "rdc_lib/impl/SmiUtils.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+RdcPolicyImpl::RdcPolicyImpl(const RdcGroupSettingsPtr& group_settings,
+                             const RdcMetricFetcherPtr& metric_fetcher)
+    : group_settings_(group_settings), metric_fetcher_(metric_fetcher), start_(true) {
+  thread_ = std::thread([this]() {
+    while (start_) {
+      rdc_policy_check_condition();
+      usleep(500);  // 500ms
+    }
+  });
+}
+
+RdcPolicyImpl::~RdcPolicyImpl() {
+  start_ = false;
+  thread_.join();
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) {
+  rdc_status_t status = RDC_ST_NOT_SUPPORTED;
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  // parameters check
+  if (policy.condition.type >= RDC_POLICY_COND_MAX) {
+    status = RDC_ST_BAD_PARAMETER;
+    return status;
+  }
+
+  if (policy.action > RDC_POLICY_ACTION_GPU_RESET) {
+    status = RDC_ST_BAD_PARAMETER;
+    return status;
+  }
+
+  // check if support RDC_POLICY_COND_MAX_PAGE_RETRIED
+  if (RDC_POLICY_COND_MAX_PAGE_RETRIED == policy.condition.type) {
+    uint32_t gpu_index;
+    rdc_group_info_t group_info;
+    rdc_field_value value;
+
+    status = group_settings_->rdc_group_gpu_get_info(group_id, &group_info);
+    for (unsigned int i = 0; i < group_info.count; i++) {
+      gpu_index = group_info.entity_ids[i];
+
+      status = metric_fetcher_->fetch_smi_field(gpu_index, RDC_FI_GPU_PAGE_RETRIED, &value);
+      if (status == RDC_ST_MSI_ERROR)
+        return RDC_ST_NOT_SUPPORTED;
+    }
+  }
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies = it->second;
+
+    bool exist = false;
+    for (auto& itpolicy : policies) {
+      // if exist, overwrite the value and action
+      if (itpolicy.condition.type == policy.condition.type) {
+        itpolicy.condition.value = policy.condition.value;
+        itpolicy.action = policy.action;
+        exist = true;
+      }
+    }
+    if (!exist) {
+      policies.push_back(policy);
+    }
+
+    status = RDC_ST_OK;
+  } else {
+    std::vector<rdc_policy_t> policies;
+    policies.push_back(policy);
+
+    settings_.insert(std::make_pair(group_id, policies));
+    status = RDC_ST_OK;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                           rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) {
+  rdc_status_t status = RDC_ST_NOT_SUPPORTED;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies_ref = it->second;
+    uint32_t i = 0;
+    for (auto itpolicy : policies_ref) {
+      // if exist
+      policies[i].condition.type = itpolicy.condition.type;
+      policies[i].condition.value = itpolicy.condition.value;
+      policies[i].action = itpolicy.action;
+      ++i;
+    }
+    *count = i;
+
+    status = RDC_ST_OK;
+  } else {
+    status = RDC_ST_NOT_FOUND;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_delete(rdc_gpu_group_t group_id,
+                                              rdc_policy_condition_type_t condition_type) {
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto it = settings_.find(group_id);
+  if (it != settings_.end()) {
+    std::vector<rdc_policy_t>& policies_ref = it->second;
+
+    auto itpolicy = policies_ref.begin();
+    while (itpolicy != policies_ref.end()) {
+      if (itpolicy->condition.type == condition_type) {
+        status = RDC_ST_OK;
+        itpolicy = policies_ref.erase(itpolicy);
+      } else {
+        ++itpolicy;
+      }
+    }
+  } else {
+    status = RDC_ST_NOT_FOUND;
+  }
+
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_register(rdc_gpu_group_t group_id,
+                                                rdc_policy_register_callback callback)
+
+{
+  rdc_status_t status = RDC_ST_NOT_FOUND;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  auto run = callbacks_.find(group_id);
+  if (run != callbacks_.end()) {
+    run->second = callback;
+    status = RDC_ST_OK;
+  } else {
+    callbacks_.insert(std::make_pair(group_id, callback));
+    status = RDC_ST_OK;
+  }
+  return status;
+}
+
+rdc_status_t RdcPolicyImpl::rdc_policy_unregister(rdc_gpu_group_t group_id) {
+  rdc_status_t status = RDC_ST_OK;
+
+  std::lock_guard<std::mutex> guard(policy_mutex_);
+
+  callbacks_.erase(group_id);
+  return status;
+}
+
+void RdcPolicyImpl::rdc_policy_check_condition() {
+  // go through the settings
+  for (auto it : settings_) {
+    rdc_gpu_group_t group_id = it.first;
+    std::vector<rdc_policy_t> policies = it.second;
+    rdc_policy_register_callback callback = rdc_policy_get_callback(group_id);
+
+    for (auto policy : policies) {
+      rdc_status_t status;
+      rdc_field_value value;
+      rdc_group_info_t group_info;
+      uint32_t gpu_index;
+
+      status = group_settings_->rdc_group_gpu_get_info(group_id, &group_info);
+      for (unsigned int i = 0; i < group_info.count; i++) {
+        rdc_field_t map[RDC_POLICY_COND_MAX] = {RDC_FI_GPU_PAGE_RETRIED, RDC_FI_GPU_TEMP,
+                                                RDC_FI_POWER_USAGE};
+
+        gpu_index = group_info.entity_ids[i];
+        status = metric_fetcher_->fetch_smi_field(gpu_index, map[policy.condition.type], &value);
+        if (status == RDC_ST_OK) {
+          if (value.value.l_int > policy.condition.value) {
+            if (RDC_POLICY_ACTION_GPU_RESET == policy.action) {
+              rdc_policy_gpu_reset(gpu_index);
+            }
+
+            // callback if needed
+            if (callback) {
+              rdc_policy_callback_response_t response = {1, policy.condition, group_id,
+                                                         value.value.l_int};
+              callback(&response);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+rdc_policy_register_callback RdcPolicyImpl::rdc_policy_get_callback(rdc_gpu_group_t group_id) {
+  rdc_policy_register_callback cb = nullptr;
+  auto it = callbacks_.find(group_id);
+  if (it != callbacks_.end()) {
+    cb = it->second;
+  }
+  return cb;
+}
+
+void RdcPolicyImpl::rdc_policy_gpu_reset(uint32_t gpu_index) {
+  amdsmi_processor_handle processor_handle = {};
+
+  amdsmi_status_t err = get_processor_handle_from_id(gpu_index, &processor_handle);
+  if (err == AMDSMI_STATUS_SUCCESS) {
+    amdsmi_reset_gpu(processor_handle);
+  }
+}
+
+}  // namespace rdc
+}  // namespace amd