Enable RDC policy feature

1. Add policy APIs 2. Add policy example for policy API usage Change-Id: I14deb7c809d0b865b7bb083842092fc37868025e Signed-off-by: Chao Fei <Chao.Fei@amd.com>
2024-08-27 11:46:40 +08:00
parent 4bd31b605a
commit 345ac64a43
19 changed files with 1382 additions and 18 deletions
@@ -189,6 +189,10 @@ typedef enum {
  RDC_FI_GPU_MM_DEC_UTIL,  //!< Multimedia decoder busy percentage
  RDC_FI_GPU_MEMORY_ACTIVITY,  //!< Memory busy percentage

+  /**
+   * @brief GPU page related fields
+   */
+  RDC_FI_GPU_PAGE_RETRIED = 550,  //!< Retried page of the GPU instance
  /**
   * @brief ECC related fields
   */
@@ -552,6 +556,31 @@ typedef struct {
  rdc_diag_test_result_t diag_info[MAX_TEST_CASES];
 } rdc_diag_response_t;

+/**
+ * @brief The policy type to support
+ */
+typedef enum {
+  RDC_POLICY_COND_MAX_PAGE_RETRIED,  //!< Max number of page retired
+  RDC_POLICY_COND_THERMAL,           //!< Temperature threshold, millidegree Celsius
+  RDC_POLICY_COND_POWER,             //!< Power threshold, unit milliwatt
+  RDC_POLICY_COND_MAX
+} rdc_policy_condition_type_t;
+
+typedef struct {
+  rdc_policy_condition_type_t type;
+  int64_t value;
+} rdc_policy_condition_t;
+
+typedef enum { RDC_POLICY_ACTION_NONE, RDC_POLICY_ACTION_GPU_RESET } rdc_policy_action_t;
+
+/**
+ * @brief The structure to define policy to enforce on GPU.
+ */
+typedef struct {
+  rdc_policy_condition_t condition;  //!< condition to meet
+  rdc_policy_action_t action; //!< Action to take
+} rdc_policy_t;
+
 /**
 *  @brief Initialize ROCm RDC.
 *
@@ -1131,6 +1160,107 @@ rdc_field_t get_field_id_from_name(const char* name);
 */
 const char* rdc_diagnostic_result_string(rdc_diag_result_t result);

+/**
+ *  @brief Set the RDC policy. Each group has multiple policies, these policies can be set by this
+ * API one by one. Multiple calls of this API will override the existing policy.
+ *
+ *  @details Set the RDC policy
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] group_id The GPU group id.
+ *
+ *  @param[in] policy  The policy to set
+ *
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_policy_set(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                            rdc_policy_t policy);
+
+#define RDC_MAX_POLICY_SETTINGS 32
+
+/**
+ *  @brief Get the RDC policy
+ *
+ *  @details Get the RDC policy
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] group_id The GPU group id.
+ *
+ *  @param[out] count The size of policies array
+ *
+ *  @param[out] policies  The policies to get
+ *
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_policy_get(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id, uint32_t* count,
+                            rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]);
+
+/**
+ *  @brief delete the RDC policy for this group based on condition type
+ *
+ *  @details clear the RDC policy for this group based on condition type. In a GPU group, only one
+ * policy can be set for a specific rdc_policy_condition_type_t
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] group_id The GPU group id
+ *
+ *  @param[in] condition_type The condition type to delete
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+
+rdc_status_t rdc_policy_delete(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                               rdc_policy_condition_type_t condition_type);
+
+/**
+ * Define the structure is used in RDC policy callback
+ */
+typedef struct {
+  unsigned int version;
+  rdc_policy_condition_t condition;  //!< the condition that is meet
+  rdc_gpu_group_t group_id;          //!< The group id trigger this callback
+  int64_t value;          //!< The current value that meet the condition
+} rdc_policy_callback_response_t;
+
+/**
+ * The user data is the rdc_policy_callback_response_t
+ */
+typedef int (*rdc_policy_register_callback)(rdc_policy_callback_response_t* userData);
+
+/**
+ *  @brief Register a function to be called when policy condition is meet.
+ *
+ *  @details Register the RDC policy callback
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] group_id The GPU group id.
+ *
+ *  @param[in] callback  The callback function to be called when condition meet.
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_policy_register(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id,
+                                 rdc_policy_register_callback callback);
+
+/**
+ *  @brief un-register a policy callback function for a conditioin.
+ *
+ *  @details Un-register the policy callback for a condition.
+ *
+ *  @param[in] p_rdc_handle The RDC handler.
+ *
+ *  @param[in] group_id The GPU group id.
+ *
+ *  @retval ::RDC_ST_OK is returned upon successful call.
+ */
+rdc_status_t rdc_policy_unregister(rdc_handle_t p_rdc_handle, rdc_gpu_group_t group_id);
+
 #ifdef __cplusplus
 }
 #endif  // __cplusplus
@@ -92,6 +92,19 @@ class RdcHandler {
  // It is just a client interface under the GRPC framework and is not used as an RDC API.
  // The reason is that RdcEmbeddedHandler::get_mixed_component_version does not need to be called.
  virtual rdc_status_t get_mixed_component_version(mixed_component_t component, mixed_component_version_t* p_mixed_compv) = 0;
+  // Policy API
+  virtual rdc_status_t rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) = 0;
+
+  virtual rdc_status_t rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                      rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) = 0;
+
+  virtual rdc_status_t rdc_policy_delete(rdc_gpu_group_t group_id,
+                                         rdc_policy_condition_type_t condition_type) = 0;
+
+  virtual rdc_status_t rdc_policy_register(rdc_gpu_group_t group_id,
+                                           rdc_policy_register_callback callback) = 0;
+
+  virtual rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) = 0;

  virtual ~RdcHandler() {}
 };
@@ -0,0 +1,57 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef INCLUDE_RDC_LIB_RDCPOLICY_H_
+#define INCLUDE_RDC_LIB_RDCPOLICY_H_
+
+#include <memory>
+#include <vector>
+
+#include "rdc/rdc.h"
+#include "rdc_lib/rdc_common.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcPolicy {
+ public:
+  virtual rdc_status_t rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) = 0;
+
+  virtual rdc_status_t rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                                      rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) = 0;
+
+  virtual rdc_status_t rdc_policy_delete(rdc_gpu_group_t group_id,
+                                         rdc_policy_condition_type_t condition_type) = 0;
+
+  virtual rdc_status_t rdc_policy_register(rdc_gpu_group_t group_id,
+                                           rdc_policy_register_callback callback) = 0;
+
+  virtual rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) = 0;
+
+  virtual ~RdcPolicy() {}
+};
+
+typedef std::shared_ptr<RdcPolicy> RdcPolicyPtr;
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // INCLUDE_RDC_LIB_RDCPOLICY_H_
@@ -31,6 +31,7 @@ THE SOFTWARE.
 #include "rdc_lib/RdcMetricsUpdater.h"
 #include "rdc_lib/RdcModuleMgr.h"
 #include "rdc_lib/RdcNotification.h"
+#include "rdc_lib/RdcPolicy.h"
 #include "rdc_lib/RdcWatchTable.h"

 namespace amd {
@@ -94,6 +95,18 @@ class RdcEmbeddedHandler final : public RdcHandler {
  // It is just a client interface under the GRPC framework and is not used as an RDC API.
  // Pure virtual functions need to be overridden.
  rdc_status_t get_mixed_component_version(mixed_component_t component, mixed_component_version_t* p_mixed_compv) override;
+  // Policy API
+  rdc_status_t rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) override;
+
+  rdc_status_t rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                              rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) override;
+
+  rdc_status_t rdc_policy_delete(rdc_gpu_group_t group_id,
+                                 rdc_policy_condition_type_t condition_type) override;
+
+  rdc_status_t rdc_policy_register(rdc_gpu_group_t group_id, rdc_policy_register_callback callback) override;
+
+  rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) override;

  explicit RdcEmbeddedHandler(rdc_operation_mode_t op_mode);
  ~RdcEmbeddedHandler() final;
@@ -107,6 +120,7 @@ class RdcEmbeddedHandler final : public RdcHandler {
  RdcNotificationPtr rdc_notif_;
  RdcWatchTablePtr watch_table_;
  RdcMetricsUpdaterPtr metrics_updater_;
+  RdcPolicyPtr policy_;
  std::future<void> updater_;
 };

@@ -0,0 +1,77 @@
+/*
+Copyright (c) 2024 - present Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+#ifndef INCLUDE_RDC_LIB_IMPL_RDCPOLICYIMPL_H_
+#define INCLUDE_RDC_LIB_IMPL_RDCPOLICYIMPL_H_
+
+#include <atomic>
+#include <map>
+#include <memory>
+#include <mutex>  // NOLINT
+#include <string>
+#include <utility>
+#include <vector>
+#include <future>
+
+#include "amd_smi/amdsmi.h"
+#include "rdc_lib/RdcPolicy.h"
+#include "rdc_lib/RdcMetricFetcher.h"
+#include "rdc_lib/RdcGroupSettings.h"
+
+namespace amd {
+namespace rdc {
+
+class RdcPolicyImpl : public RdcPolicy {
+ public:
+  RdcPolicyImpl(const RdcGroupSettingsPtr& group_settings, const RdcMetricFetcherPtr& metric_fetcher);
+  ~RdcPolicyImpl();
+
+  rdc_status_t rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) override;
+
+  rdc_status_t rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count,
+                              rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) override;
+
+  rdc_status_t rdc_policy_delete(rdc_gpu_group_t group_id,
+                                 rdc_policy_condition_type_t condition_type) override;
+
+  rdc_status_t rdc_policy_register(rdc_gpu_group_t group_id,rdc_policy_register_callback callback) override;
+
+  rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) override;
+
+ private:
+  RdcGroupSettingsPtr group_settings_;
+  RdcMetricFetcherPtr metric_fetcher_;
+  std::mutex policy_mutex_;
+  std::thread thread_;
+  bool start_;
+
+  std::map<rdc_gpu_group_t, std::vector<rdc_policy_t> > settings_;
+  std::map<rdc_gpu_group_t, rdc_policy_register_callback> callbacks_;
+
+  void rdc_policy_check_condition();
+  void rdc_policy_gpu_reset(uint32_t gpu_index);
+  rdc_policy_register_callback rdc_policy_get_callback(rdc_gpu_group_t group_id);
+};
+
+}  // namespace rdc
+}  // namespace amd
+
+#endif  // INCLUDE_RDC_LIB_IMPL_RDCPOLICYIMPL_H_
@@ -24,6 +24,8 @@ THE SOFTWARE.
 #include <grpcpp/grpcpp.h>

 #include <memory>
+#include <future>
+#include <thread>

 #include "rdc.grpc.pb.h"  // NOLINT
 #include "rdc_lib/RdcHandler.h"
@@ -89,6 +91,18 @@ class RdcStandaloneHandler : public RdcHandler {
  // It is just a client interface under the GRPC framework and is not used as an RDC API.
  // Pure virtual functions need to be overridden
  rdc_status_t get_mixed_component_version(mixed_component_t component, mixed_component_version_t* p_mixed_compv) override;
+  // Policy API
+  rdc_status_t rdc_policy_set(rdc_gpu_group_t group_id, rdc_policy_t policy) override;
+
+  rdc_status_t rdc_policy_get(rdc_gpu_group_t group_id, uint32_t* count, rdc_policy_t policies[RDC_MAX_POLICY_SETTINGS]) override;
+
+  rdc_status_t rdc_policy_delete(rdc_gpu_group_t group_id,
+                                 rdc_policy_condition_type_t condition_type) override;
+
+  rdc_status_t rdc_policy_register(rdc_gpu_group_t group_id,
+                                   rdc_policy_register_callback callback) override;
+
+  rdc_status_t rdc_policy_unregister(rdc_gpu_group_t group_id) override;

  explicit RdcStandaloneHandler(const char* ip_and_port, const char* root_ca,
                                const char* client_cert, const char* client_key);
@@ -100,6 +114,15 @@ class RdcStandaloneHandler : public RdcHandler {
  bool copy_gpu_usage_info(const ::rdc::GpuUsageInfo& src, rdc_gpu_usage_info_t* target);

  std::unique_ptr<::rdc::RdcAPI::Stub> stub_;
+  // thread for policy callback
+
+  struct policy_thread_context {
+    bool start;
+    std::thread *t;
+  };
+
+  std::map<uint32_t, struct policy_thread_context> policy_threads_;
+
 };

 }  // namespace rdc