Handle HW_EXCEPTION events

Add handler to handle HW exception events reported by underlying
drivers. These events are generally caused by GPU resets and need the
application to abort.
As an improvement, in the future, we can provide additional information
about the exception (e.g mode-reset level)

Change-Id: If3fb5f19f9fce181a9d3b5e34a5506725856e7b0

runtime/hsa-runtime/core/inc/runtime.h

@@ -136,6 +136,9 @@ class Runtime {
   /// @retval True if the connection to kernel driver is opened.
   static bool IsOpen();
 
+  // @brief Callback handler for HW Exceptions.
+  static bool HwExceptionHandler(hsa_signal_value_t val, void* arg);
+
   // @brief Callback handler for VM fault access.
   static bool VMFaultHandler(hsa_signal_value_t val, void* arg);
 
@@ -577,8 +580,8 @@ class Runtime {
   /// @brief Close tool libraries.
   void CloseTools();
 
-  // @brief Binds virtual memory access fault handler to this node.
-  void BindVmFaultHandler();
+  // @brief Binds Error handlers to this node.
+  void BindErrorHandlers();
 
   // @brief Acquire snapshot of system event handlers.
   // Returns a copy to avoid holding a lock during callbacks.
@@ -676,6 +679,12 @@ class Runtime {
   // @brief HSA signal to contain the VM fault event.
   Signal* vm_fault_signal_;
 
+  // @brief AMD HSA event to monitor for HW exceptions.
+  HsaEvent* hw_exception_event_;
+
+  // @brief HSA signal to contain the HW exceptionevent.
+  Signal* hw_exception_signal_;
+
   // Custom system event handlers.
   std::vector<std::pair<AMD::callback_t<hsa_amd_system_event_callback_t>, void*>>
       system_event_handlers_;

runtime/hsa-runtime/core/runtime/interrupt_signal.cpp

@@ -89,6 +89,8 @@ HsaEvent* InterruptSignal::CreateEvent(HSA_EVENTTYPE type, bool manual_reset) {
     if (type == HSA_EVENTTYPE_MEMORY) {
       memset(&ret->EventData.EventData.MemoryAccessFault.Failure, 0,
              sizeof(HsaAccessAttributeFailure));
+    } else if (type == HSA_EVENTTYPE_HW_EXCEPTION) {
+      memset(&ret->EventData.EventData.HwException, 0, sizeof(HsaHwException));
     }
   }
 

runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -1238,27 +1238,90 @@ void Runtime::AsyncEventsLoop(void*) {
   new_async_events_.Clear();
 }
 
-void Runtime::BindVmFaultHandler() {
-  if (core::g_use_interrupt_wait && !gpu_agents_.empty()) {
-    // Create memory event with manual reset to avoid racing condition
-    // with driver in case of multiple concurrent VM faults.
-    vm_fault_event_ =
-        core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true);
+void Runtime::BindErrorHandlers() {
+  if (!core::g_use_interrupt_wait || gpu_agents_.empty()) return;
 
-    // Create an interrupt signal object to contain the memory event.
-    // This signal object will be registered with the async handler global
-    // thread.
-    vm_fault_signal_ = new core::InterruptSignal(0, vm_fault_event_);
+  // Create memory event with manual reset to avoid racing condition
+  // with driver in case of multiple concurrent VM faults.
+  vm_fault_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true);
 
-    if (!vm_fault_signal_->IsValid() || vm_fault_signal_->EopEvent() == NULL) {
-      assert(false && "Failed on creating VM fault signal");
-      return;
-    }
+  // Create an interrupt signal object to contain the memory event.
+  // This signal object will be registered with the async handler global
+  // thread.
+  vm_fault_signal_ = new core::InterruptSignal(0, vm_fault_event_);
 
-    SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_),
-                          HSA_SIGNAL_CONDITION_NE, 0, VMFaultHandler,
-                          reinterpret_cast<void*>(vm_fault_signal_));
+  if (!vm_fault_signal_->IsValid() || vm_fault_signal_->EopEvent() == NULL) {
+    assert(false && "Failed on creating VM fault signal");
+    return;
   }
+
+  SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_), HSA_SIGNAL_CONDITION_NE, 0,
+                        VMFaultHandler, reinterpret_cast<void*>(vm_fault_signal_));
+
+  // Create HW exception event which is for Non-RAS events
+  hw_exception_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_HW_EXCEPTION, true);
+
+  hw_exception_signal_ = new core::InterruptSignal(0, hw_exception_event_);
+
+  if (!hw_exception_signal_->IsValid() || hw_exception_signal_->EopEvent() == NULL) {
+    assert(false && "Failed on creating HW Exception signal");
+    return;
+  }
+
+  SetAsyncSignalHandler(core::Signal::Convert(hw_exception_signal_), HSA_SIGNAL_CONDITION_NE, 0,
+                        HwExceptionHandler, reinterpret_cast<void*>(hw_exception_signal_));
+}
+
+bool Runtime::HwExceptionHandler(hsa_signal_value_t val, void* arg) {
+  core::InterruptSignal* hw_exception_signal = reinterpret_cast<core::InterruptSignal*>(arg);
+
+  assert(hw_exception_signal != NULL);
+
+  if (hw_exception_signal == NULL) return false;
+
+  HsaEvent* exception_event = hw_exception_signal->EopEvent();
+
+  HsaHwException& exception = exception_event->EventData.EventData.HwException;
+
+  hsa_status_t custom_handler_status = HSA_STATUS_ERROR;
+  auto system_event_handlers = runtime_singleton_->GetSystemEventHandlers();
+  // If custom handler is registered, pack the fault info and call the handler
+
+  if (!system_event_handlers.empty()) {
+    hsa_amd_event_t hw_exception_event;
+    hw_exception_event.event_type = HSA_AMD_GPU_HW_EXCEPTION_EVENT;
+    hsa_amd_gpu_hw_exception_info_t& exception_info = hw_exception_event.hw_exception;
+
+    // Find the faulty agent
+    auto it = runtime_singleton_->agents_by_node_.find(exception.NodeId);
+    assert(it != runtime_singleton_->agents_by_node_.end() && "Can't find faulty agent.");
+    Agent* faulty_agent = it->second.front();
+    exception_info.agent = Agent::Convert(faulty_agent);
+
+    // This field is not set by KFD at the moment
+    exception_info.reset_type = HSA_AMD_HW_EXCEPTION_RESET_TYPE_OTHER;
+
+    exception_info.reset_cause = (exception.ResetCause == HSA_EVENTID_HW_EXCEPTION_ECC)
+        ? HSA_AMD_HW_EXCEPTION_CAUSE_ECC
+        : HSA_AMD_HW_EXCEPTION_CAUSE_GPU_HANG;
+
+    for (auto& callback : system_event_handlers) {
+      hsa_status_t err = callback.first(&hw_exception_event, callback.second);
+      if (err == HSA_STATUS_SUCCESS) custom_handler_status = HSA_STATUS_SUCCESS;
+    }
+  }
+
+  if (custom_handler_status != HSA_STATUS_SUCCESS) {
+    core::Agent* faultingAgent = runtime_singleton_->agents_by_node_[exception.NodeId][0];
+    fprintf(stderr, "HW Exception by GPU node-%u (Agent handle: %p) reason :%s\n", exception.NodeId,
+            reinterpret_cast<void*>(faultingAgent->public_handle().handle),
+            (exception.ResetCause == HSA_EVENTID_HW_EXCEPTION_ECC) ? "ECC" : "GPU Hang");
+
+    assert(false && "GPU HW Exception");
+    std::abort();
+  }
+  // No need to keep the signal because we are done.
+  return false;
 }
 
 bool Runtime::VMFaultHandler(hsa_signal_value_t val, void* arg) {
@@ -1424,6 +1487,8 @@ Runtime::Runtime()
       sys_clock_freq_(0),
       vm_fault_event_(nullptr),
       vm_fault_signal_(nullptr),
+      hw_exception_event_(nullptr),
+      hw_exception_signal_(nullptr),
       ref_count_(0),
       kfd_version{} {}
 
@@ -1453,7 +1518,7 @@ hsa_status_t Runtime::Load() {
     if (sys_clock_freq_ < 100000) debug_warning("System clock resolution is low.");
   }
 
-  BindVmFaultHandler();
+  BindErrorHandlers();
 
   loader_ = amd::hsa::loader::Loader::Create(&loader_context_);
 
@@ -1506,6 +1571,13 @@ void Runtime::Unload() {
   core::InterruptSignal::DestroyEvent(vm_fault_event_);
   vm_fault_event_ = nullptr;
 
+  if (hw_exception_signal_ != nullptr) {
+    hw_exception_signal_->DestroySignal();
+    hw_exception_signal_ = nullptr;
+  }
+  core::InterruptSignal::DestroyEvent(hw_exception_event_);
+  hw_exception_event_ = nullptr;
+
   SharedSignalPool.clear();
 
   EventPool.clear();

runtime/hsa-runtime/core/runtime/signal.cpp

@@ -267,6 +267,9 @@ uint32_t Signal::WaitAny(uint32_t signal_count, const hsa_signal_t* hsa_signals,
           if (fault.Flags == HSA_EVENTID_MEMORY_FATAL_PROCESS) {
             return i;
           }
+        } else if (event_type == HSA_EVENTTYPE_HW_EXCEPTION) {
+          const HsaHwException& exception = signals[i]->EopEvent()->EventData.EventData.HwException;
+          if (exception.MemoryLost) return i;
         }
       }
 

runtime/hsa-runtime/inc/hsa_ext_amd.h

@@ -2241,6 +2241,10 @@ typedef enum hsa_amd_event_type_s {
    AMD GPU memory fault.
    */
   HSA_AMD_GPU_MEMORY_FAULT_EVENT = 0,
+  /*
+   AMD GPU HW Exception.
+   */
+  HSA_AMD_GPU_HW_EXCEPTION_EVENT,
 } hsa_amd_event_type_t;
 
 /**
@@ -2284,6 +2288,36 @@ typedef struct hsa_amd_gpu_memory_fault_info_s {
   uint32_t fault_reason_mask;
 } hsa_amd_gpu_memory_fault_info_t;
 
+/**
+ * @brief Flags denoting the type of a HW exception
+ */
+typedef enum {
+  // Unused for now
+  HSA_AMD_HW_EXCEPTION_RESET_TYPE_OTHER = 1 << 0,
+} hsa_amd_hw_exception_reset_type_t;
+
+/**
+ * @brief Flags denoting the cause of a HW exception
+ */
+typedef enum {
+  // GPU Hang
+  HSA_AMD_HW_EXCEPTION_CAUSE_GPU_HANG = 1 << 0,
+  // SRAM ECC
+  HSA_AMD_HW_EXCEPTION_CAUSE_ECC = 1 << 1,
+} hsa_amd_hw_exception_reset_cause_t;
+
+/**
+ * @brief AMD GPU HW Exception event data.
+ */
+typedef struct hsa_amd_gpu_hw_exception_info_s {
+  /*
+  The agent where the HW exception occurred.
+  */
+  hsa_agent_t agent;
+  hsa_amd_hw_exception_reset_type_t reset_type;
+  hsa_amd_hw_exception_reset_cause_t reset_cause;
+} hsa_amd_gpu_hw_exception_info_t;
+
 /**
  * @brief AMD GPU event data passed to event handler.
  */
@@ -2297,6 +2331,10 @@ typedef struct hsa_amd_event_s {
     The memory fault info, only valid when @p event_type is HSA_AMD_GPU_MEMORY_FAULT_EVENT.
     */
     hsa_amd_gpu_memory_fault_info_t memory_fault;
+    /*
+    The memory fault info, only valid when @p event_type is HSA_AMD_GPU_HW_EXCEPTION_EVENT.
+    */
+    hsa_amd_gpu_hw_exception_info_t hw_exception;
   };
 } hsa_amd_event_t;