debug suport for queue error.
1/ Revised debug event handler to handle different events.
2/ Added queue error handler using the callback in queue create, which will print out wave info when queue in error state.
3/ Preempt queue instead of destory queue when queue error state.
Change-Id: Ib727d208de9caf1c72c76d42268483b24aaebde8
[ROCm/ROCR-Runtime commit: 49d2175c74]
Этот коммит содержится в:
Qingchuan Shi
коммит произвёл
Qingchuan Shi
Qingchuan Shi
родитель
7c4cf726f7
Коммит
286ca924f3
+2
-4
@@ -1097,10 +1097,8 @@ hsa_status_t HSA_API hsa_amd_ipc_signal_attach(const hsa_amd_ipc_signal_t* handl
|
||||
|
||||
// Mirrors Amd Extension Apis
|
||||
hsa_status_t HSA_API hsa_amd_register_system_event_handler(
|
||||
hsa_amd_event_t type,
|
||||
hsa_status_t (*callback)(const void* event_specific_data, void* data),
|
||||
void* data) {
|
||||
return amdExtTable->hsa_amd_register_system_event_handler_fn(type, callback, data);
|
||||
hsa_amd_system_event_callback_t callback, void* data) {
|
||||
return amdExtTable->hsa_amd_register_system_event_handler_fn(callback, data);
|
||||
}
|
||||
|
||||
// Mirrors Amd Extension Apis
|
||||
|
||||
@@ -73,7 +73,7 @@ template <class R, class... Args> class callback_t<R (*)(Args...)> {
|
||||
|
||||
// Should not be marked explicit.
|
||||
callback_t(func_t function_ptr) : function(function_ptr) {}
|
||||
callback_t& operator=(func_t function_ptr) { function = function_ptr; }
|
||||
callback_t& operator=(func_t function_ptr) { function = function_ptr; return *this; }
|
||||
|
||||
// Allows common function pointer idioms, such as if( func != nullptr )...
|
||||
// without allowing silent reversion to the original function pointer type.
|
||||
|
||||
@@ -212,9 +212,7 @@ hsa_status_t hsa_amd_ipc_signal_attach(const hsa_amd_ipc_signal_t* handle, hsa_s
|
||||
|
||||
// Mirrors Amd Extension Apis
|
||||
hsa_status_t hsa_amd_register_system_event_handler(
|
||||
hsa_amd_event_t type,
|
||||
hsa_status_t (*callback)(const void* event_specific_data, void* data),
|
||||
void* data);
|
||||
hsa_amd_system_event_callback_t callback, void* data);
|
||||
|
||||
} // end of AMD namespace
|
||||
|
||||
|
||||
@@ -53,6 +53,7 @@
|
||||
#include "core/inc/hsa_ext_amd_impl.h"
|
||||
|
||||
#include "core/inc/agent.h"
|
||||
#include "core/inc/exceptions.h"
|
||||
#include "core/inc/memory_region.h"
|
||||
#include "core/inc/signal.h"
|
||||
#include "core/util/flag.h"
|
||||
@@ -318,9 +319,14 @@ class Runtime {
|
||||
|
||||
ExtensionEntryPoints extensions_;
|
||||
|
||||
hsa_status_t SetCustomVMFaultHandler(hsa_status_t (*callback)(const void* event_specific_data,
|
||||
void* data),
|
||||
void* data);
|
||||
hsa_status_t SetCustomSystemEventHandler(hsa_amd_system_event_callback_t callback,
|
||||
void* data);
|
||||
|
||||
void* GetCustomSystemEventData() { return system_event_handler_user_data_; }
|
||||
|
||||
AMD::callback_t<hsa_amd_system_event_callback_t> GetCustomSystemEventHandler() {
|
||||
return system_event_handler_;
|
||||
}
|
||||
|
||||
protected:
|
||||
static void AsyncEventsLoop(void*);
|
||||
@@ -504,10 +510,10 @@ class Runtime {
|
||||
// @brief HSA signal to contain the VM fault event.
|
||||
Signal* vm_fault_signal_;
|
||||
|
||||
// custom VM fault handler.
|
||||
hsa_status_t (*vm_fault_handler_custom_)(const void* event_specific_data,
|
||||
void* data);
|
||||
void* vm_fault_handler_user_data_;
|
||||
// Custom system event handler.
|
||||
AMD::callback_t<hsa_amd_system_event_callback_t> system_event_handler_;
|
||||
|
||||
void* system_event_handler_user_data_;
|
||||
|
||||
// Holds reference count to runtime object.
|
||||
std::atomic<uint32_t> ref_count_;
|
||||
|
||||
@@ -844,18 +844,11 @@ hsa_status_t hsa_amd_queue_intercept_register(hsa_queue_t* queue,
|
||||
CATCH;
|
||||
}
|
||||
|
||||
hsa_status_t hsa_amd_register_system_event_handler(
|
||||
hsa_amd_event_t type,
|
||||
hsa_status_t (*callback)(const void* event_specific_data, void* data),
|
||||
void* data) {
|
||||
hsa_status_t hsa_amd_register_system_event_handler(hsa_amd_system_event_callback_t callback,
|
||||
void* data) {
|
||||
TRY;
|
||||
IS_OPEN();
|
||||
switch (type) {
|
||||
case GPU_MEMORY_FAULT_EVENT:
|
||||
return core::Runtime::runtime_singleton_->SetCustomVMFaultHandler(callback, data);
|
||||
default:
|
||||
return HSA_STATUS_ERROR_INVALID_ARGUMENT;
|
||||
}
|
||||
return core::Runtime::runtime_singleton_->SetCustomSystemEventHandler(callback, data);
|
||||
CATCH;
|
||||
}
|
||||
|
||||
|
||||
@@ -1050,38 +1050,40 @@ bool Runtime::VMFaultHandler(hsa_signal_value_t val, void* arg) {
|
||||
|
||||
hsa_status_t custom_handler_status = HSA_STATUS_ERROR;
|
||||
// If custom handler is registered, pack the fault info and call the handler
|
||||
if (runtime_singleton_->vm_fault_handler_custom_ != nullptr) {
|
||||
hsa_amd_gpu_memory_fault_info_t* fault_info = new hsa_amd_gpu_memory_fault_info_t;
|
||||
if (runtime_singleton_->GetCustomSystemEventHandler()) {
|
||||
hsa_amd_gpu_memory_fault_info_t fault_info;
|
||||
|
||||
// Find the faulty agent
|
||||
auto it = runtime_singleton_->agents_by_node_.find(fault.NodeId);
|
||||
assert(it != runtime_singleton_->agents_by_node_.end() && "Can't find faulty agent.");
|
||||
Agent* faulty_agent = it->second.front();
|
||||
fault_info->agent = Agent::Convert(faulty_agent);
|
||||
fault_info.agent = Agent::Convert(faulty_agent);
|
||||
|
||||
fault_info->virtual_address = fault.VirtualAddress;
|
||||
fault_info->fault_reason_mask = 0x00000000;
|
||||
fault_info.virtual_address = fault.VirtualAddress;
|
||||
fault_info.fault_reason_mask = 0x00000000;
|
||||
if (fault.Failure.NotPresent == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000001;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00000001;
|
||||
}
|
||||
if (fault.Failure.ReadOnly == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000010;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00000010;
|
||||
}
|
||||
if (fault.Failure.NoExecute == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00000100;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00000100;
|
||||
}
|
||||
if (fault.Failure.GpuAccess == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00001000;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00001000;
|
||||
}
|
||||
if (fault.Failure.ECC == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00010000;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00010000;
|
||||
}
|
||||
if (fault.Failure.Imprecise == 1) {
|
||||
fault_info->fault_reason_mask = fault_info->fault_reason_mask | 0x00100000;
|
||||
fault_info.fault_reason_mask = fault_info.fault_reason_mask | 0x00100000;
|
||||
}
|
||||
|
||||
custom_handler_status = runtime_singleton_->vm_fault_handler_custom_(fault_info,
|
||||
runtime_singleton_->vm_fault_handler_user_data_);
|
||||
hsa_amd_event_t memory_fault_event;
|
||||
memory_fault_event.event_type = GPU_MEMORY_FAULT_EVENT;
|
||||
memory_fault_event.memory_fault = fault_info;
|
||||
custom_handler_status = runtime_singleton_->GetCustomSystemEventHandler()(
|
||||
&memory_fault_event, runtime_singleton_->GetCustomSystemEventData());
|
||||
}
|
||||
|
||||
// No custom VM fault handler registered or it failed.
|
||||
@@ -1176,7 +1178,7 @@ Runtime::Runtime()
|
||||
sys_clock_freq_(0),
|
||||
vm_fault_event_(nullptr),
|
||||
vm_fault_signal_(nullptr),
|
||||
vm_fault_handler_custom_(nullptr),
|
||||
system_event_handler_user_data_(nullptr),
|
||||
ref_count_(0) {
|
||||
start_svm_address_ = 0;
|
||||
#if defined(HSA_LARGE_MODEL)
|
||||
@@ -1446,15 +1448,15 @@ void Runtime::AsyncEvents::Clear() {
|
||||
arg_.clear();
|
||||
}
|
||||
|
||||
hsa_status_t Runtime::SetCustomVMFaultHandler(
|
||||
hsa_status_t (*callback)(const void* event_specific_data, void* data),
|
||||
void* data) {
|
||||
if (vm_fault_handler_custom_ != nullptr) {
|
||||
hsa_status_t Runtime::SetCustomSystemEventHandler(hsa_amd_system_event_callback_t callback,
|
||||
void* data) {
|
||||
if (system_event_handler_) {
|
||||
return HSA_STATUS_ERROR;
|
||||
} else {
|
||||
vm_fault_handler_custom_ = callback;
|
||||
vm_fault_handler_user_data_ = data;
|
||||
system_event_handler_ = callback;
|
||||
system_event_handler_user_data_ = data;
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace core
|
||||
|
||||
Ссылка в новой задаче
Block a user