diff --git a/runtime/hsa-runtime/core/inc/runtime.h b/runtime/hsa-runtime/core/inc/runtime.h index ee766bfc9c..e7f856b067 100644 --- a/runtime/hsa-runtime/core/inc/runtime.h +++ b/runtime/hsa-runtime/core/inc/runtime.h @@ -136,6 +136,9 @@ class Runtime { /// @retval True if the connection to kernel driver is opened. static bool IsOpen(); + // @brief Callback handler for HW Exceptions. + static bool HwExceptionHandler(hsa_signal_value_t val, void* arg); + // @brief Callback handler for VM fault access. static bool VMFaultHandler(hsa_signal_value_t val, void* arg); @@ -577,8 +580,8 @@ class Runtime { /// @brief Close tool libraries. void CloseTools(); - // @brief Binds virtual memory access fault handler to this node. - void BindVmFaultHandler(); + // @brief Binds Error handlers to this node. + void BindErrorHandlers(); // @brief Acquire snapshot of system event handlers. // Returns a copy to avoid holding a lock during callbacks. @@ -676,6 +679,12 @@ class Runtime { // @brief HSA signal to contain the VM fault event. Signal* vm_fault_signal_; + // @brief AMD HSA event to monitor for HW exceptions. + HsaEvent* hw_exception_event_; + + // @brief HSA signal to contain the HW exceptionevent. + Signal* hw_exception_signal_; + // Custom system event handlers. std::vector, void*>> system_event_handlers_; diff --git a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp index 9d7691aa25..4fd6a1c0d2 100644 --- a/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp +++ b/runtime/hsa-runtime/core/runtime/interrupt_signal.cpp @@ -89,6 +89,8 @@ HsaEvent* InterruptSignal::CreateEvent(HSA_EVENTTYPE type, bool manual_reset) { if (type == HSA_EVENTTYPE_MEMORY) { memset(&ret->EventData.EventData.MemoryAccessFault.Failure, 0, sizeof(HsaAccessAttributeFailure)); + } else if (type == HSA_EVENTTYPE_HW_EXCEPTION) { + memset(&ret->EventData.EventData.HwException, 0, sizeof(HsaHwException)); } } diff --git a/runtime/hsa-runtime/core/runtime/runtime.cpp b/runtime/hsa-runtime/core/runtime/runtime.cpp index 3c9967eab0..0def3224cf 100644 --- a/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -1238,27 +1238,90 @@ void Runtime::AsyncEventsLoop(void*) { new_async_events_.Clear(); } -void Runtime::BindVmFaultHandler() { - if (core::g_use_interrupt_wait && !gpu_agents_.empty()) { - // Create memory event with manual reset to avoid racing condition - // with driver in case of multiple concurrent VM faults. - vm_fault_event_ = - core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true); +void Runtime::BindErrorHandlers() { + if (!core::g_use_interrupt_wait || gpu_agents_.empty()) return; - // Create an interrupt signal object to contain the memory event. - // This signal object will be registered with the async handler global - // thread. - vm_fault_signal_ = new core::InterruptSignal(0, vm_fault_event_); + // Create memory event with manual reset to avoid racing condition + // with driver in case of multiple concurrent VM faults. + vm_fault_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_MEMORY, true); - if (!vm_fault_signal_->IsValid() || vm_fault_signal_->EopEvent() == NULL) { - assert(false && "Failed on creating VM fault signal"); - return; - } + // Create an interrupt signal object to contain the memory event. + // This signal object will be registered with the async handler global + // thread. + vm_fault_signal_ = new core::InterruptSignal(0, vm_fault_event_); - SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_), - HSA_SIGNAL_CONDITION_NE, 0, VMFaultHandler, - reinterpret_cast(vm_fault_signal_)); + if (!vm_fault_signal_->IsValid() || vm_fault_signal_->EopEvent() == NULL) { + assert(false && "Failed on creating VM fault signal"); + return; } + + SetAsyncSignalHandler(core::Signal::Convert(vm_fault_signal_), HSA_SIGNAL_CONDITION_NE, 0, + VMFaultHandler, reinterpret_cast(vm_fault_signal_)); + + // Create HW exception event which is for Non-RAS events + hw_exception_event_ = core::InterruptSignal::CreateEvent(HSA_EVENTTYPE_HW_EXCEPTION, true); + + hw_exception_signal_ = new core::InterruptSignal(0, hw_exception_event_); + + if (!hw_exception_signal_->IsValid() || hw_exception_signal_->EopEvent() == NULL) { + assert(false && "Failed on creating HW Exception signal"); + return; + } + + SetAsyncSignalHandler(core::Signal::Convert(hw_exception_signal_), HSA_SIGNAL_CONDITION_NE, 0, + HwExceptionHandler, reinterpret_cast(hw_exception_signal_)); +} + +bool Runtime::HwExceptionHandler(hsa_signal_value_t val, void* arg) { + core::InterruptSignal* hw_exception_signal = reinterpret_cast(arg); + + assert(hw_exception_signal != NULL); + + if (hw_exception_signal == NULL) return false; + + HsaEvent* exception_event = hw_exception_signal->EopEvent(); + + HsaHwException& exception = exception_event->EventData.EventData.HwException; + + hsa_status_t custom_handler_status = HSA_STATUS_ERROR; + auto system_event_handlers = runtime_singleton_->GetSystemEventHandlers(); + // If custom handler is registered, pack the fault info and call the handler + + if (!system_event_handlers.empty()) { + hsa_amd_event_t hw_exception_event; + hw_exception_event.event_type = HSA_AMD_GPU_HW_EXCEPTION_EVENT; + hsa_amd_gpu_hw_exception_info_t& exception_info = hw_exception_event.hw_exception; + + // Find the faulty agent + auto it = runtime_singleton_->agents_by_node_.find(exception.NodeId); + assert(it != runtime_singleton_->agents_by_node_.end() && "Can't find faulty agent."); + Agent* faulty_agent = it->second.front(); + exception_info.agent = Agent::Convert(faulty_agent); + + // This field is not set by KFD at the moment + exception_info.reset_type = HSA_AMD_HW_EXCEPTION_RESET_TYPE_OTHER; + + exception_info.reset_cause = (exception.ResetCause == HSA_EVENTID_HW_EXCEPTION_ECC) + ? HSA_AMD_HW_EXCEPTION_CAUSE_ECC + : HSA_AMD_HW_EXCEPTION_CAUSE_GPU_HANG; + + for (auto& callback : system_event_handlers) { + hsa_status_t err = callback.first(&hw_exception_event, callback.second); + if (err == HSA_STATUS_SUCCESS) custom_handler_status = HSA_STATUS_SUCCESS; + } + } + + if (custom_handler_status != HSA_STATUS_SUCCESS) { + core::Agent* faultingAgent = runtime_singleton_->agents_by_node_[exception.NodeId][0]; + fprintf(stderr, "HW Exception by GPU node-%u (Agent handle: %p) reason :%s\n", exception.NodeId, + reinterpret_cast(faultingAgent->public_handle().handle), + (exception.ResetCause == HSA_EVENTID_HW_EXCEPTION_ECC) ? "ECC" : "GPU Hang"); + + assert(false && "GPU HW Exception"); + std::abort(); + } + // No need to keep the signal because we are done. + return false; } bool Runtime::VMFaultHandler(hsa_signal_value_t val, void* arg) { @@ -1424,6 +1487,8 @@ Runtime::Runtime() sys_clock_freq_(0), vm_fault_event_(nullptr), vm_fault_signal_(nullptr), + hw_exception_event_(nullptr), + hw_exception_signal_(nullptr), ref_count_(0), kfd_version{} {} @@ -1453,7 +1518,7 @@ hsa_status_t Runtime::Load() { if (sys_clock_freq_ < 100000) debug_warning("System clock resolution is low."); } - BindVmFaultHandler(); + BindErrorHandlers(); loader_ = amd::hsa::loader::Loader::Create(&loader_context_); @@ -1506,6 +1571,13 @@ void Runtime::Unload() { core::InterruptSignal::DestroyEvent(vm_fault_event_); vm_fault_event_ = nullptr; + if (hw_exception_signal_ != nullptr) { + hw_exception_signal_->DestroySignal(); + hw_exception_signal_ = nullptr; + } + core::InterruptSignal::DestroyEvent(hw_exception_event_); + hw_exception_event_ = nullptr; + SharedSignalPool.clear(); EventPool.clear(); diff --git a/runtime/hsa-runtime/core/runtime/signal.cpp b/runtime/hsa-runtime/core/runtime/signal.cpp index 3d554a03b7..06fd9eba90 100644 --- a/runtime/hsa-runtime/core/runtime/signal.cpp +++ b/runtime/hsa-runtime/core/runtime/signal.cpp @@ -267,6 +267,9 @@ uint32_t Signal::WaitAny(uint32_t signal_count, const hsa_signal_t* hsa_signals, if (fault.Flags == HSA_EVENTID_MEMORY_FATAL_PROCESS) { return i; } + } else if (event_type == HSA_EVENTTYPE_HW_EXCEPTION) { + const HsaHwException& exception = signals[i]->EopEvent()->EventData.EventData.HwException; + if (exception.MemoryLost) return i; } } diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index cbaf110c19..10bb5e3e11 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -2241,6 +2241,10 @@ typedef enum hsa_amd_event_type_s { AMD GPU memory fault. */ HSA_AMD_GPU_MEMORY_FAULT_EVENT = 0, + /* + AMD GPU HW Exception. + */ + HSA_AMD_GPU_HW_EXCEPTION_EVENT, } hsa_amd_event_type_t; /** @@ -2284,6 +2288,36 @@ typedef struct hsa_amd_gpu_memory_fault_info_s { uint32_t fault_reason_mask; } hsa_amd_gpu_memory_fault_info_t; +/** + * @brief Flags denoting the type of a HW exception + */ +typedef enum { + // Unused for now + HSA_AMD_HW_EXCEPTION_RESET_TYPE_OTHER = 1 << 0, +} hsa_amd_hw_exception_reset_type_t; + +/** + * @brief Flags denoting the cause of a HW exception + */ +typedef enum { + // GPU Hang + HSA_AMD_HW_EXCEPTION_CAUSE_GPU_HANG = 1 << 0, + // SRAM ECC + HSA_AMD_HW_EXCEPTION_CAUSE_ECC = 1 << 1, +} hsa_amd_hw_exception_reset_cause_t; + +/** + * @brief AMD GPU HW Exception event data. + */ +typedef struct hsa_amd_gpu_hw_exception_info_s { + /* + The agent where the HW exception occurred. + */ + hsa_agent_t agent; + hsa_amd_hw_exception_reset_type_t reset_type; + hsa_amd_hw_exception_reset_cause_t reset_cause; +} hsa_amd_gpu_hw_exception_info_t; + /** * @brief AMD GPU event data passed to event handler. */ @@ -2297,6 +2331,10 @@ typedef struct hsa_amd_event_s { The memory fault info, only valid when @p event_type is HSA_AMD_GPU_MEMORY_FAULT_EVENT. */ hsa_amd_gpu_memory_fault_info_t memory_fault; + /* + The memory fault info, only valid when @p event_type is HSA_AMD_GPU_HW_EXCEPTION_EVENT. + */ + hsa_amd_gpu_hw_exception_info_t hw_exception; }; } hsa_amd_event_t;