diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index 82658e9f38..4f81c1e337 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -456,6 +456,10 @@ class GpuAgent : public GpuAgentInt { size_t trap_code_buf_size_; + // @brief Mappings from doorbell index to queue, for trap handler. + // Correlates with output of s_sendmsg(MSG_GET_DOORBELL) for queue identification. + amd_queue_t** doorbell_queue_map_; + // @brief The GPU memory bus width in bit. uint32_t memory_bus_width_; diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_shaders.h b/runtime/hsa-runtime/core/inc/amd_gpu_shaders.h index 168fb6ceee..76b975c773 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_shaders.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_shaders.h @@ -113,7 +113,15 @@ static const unsigned int kCodeTrapHandler9[] = { .set SQ_WAVE_PC_HI_TRAP_ID_SHIFT , 16 .set SQ_WAVE_PC_HI_TRAP_ID_SIZE , 8 .set SQ_WAVE_PC_HI_TRAP_ID_BFE , (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16)) + .set SQ_WAVE_PC_HI_HT_MASK , 0x1000000 .set SQ_WAVE_STATUS_HALT_MASK , 0x2000 + .set SQ_WAVE_TRAPSTS_MEM_VIOL_MASK , 0x100 + .set SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK , 0x800 + .set SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK , 0x10000000 + .set SIGNAL_CODE_MEM_VIOL , (1 << 29) + .set SIGNAL_CODE_ILLEGAL_INST , (1 << 30) + .set SIGNAL_CODE_LLVM_TRAP , (1 << 31) + .set MAX_NUM_DOORBELLS_MASK , ((1 << 10) - 1) .if .amdgcn.gfx_generation_number == 9 .set TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT , 26 @@ -124,12 +132,26 @@ static const unsigned int kCodeTrapHandler9[] = { .endif trap_entry: - // If this is not a trap then return to the shader. + // If memory violation without XNACK error then signal queue error. + // XNACK error will be handled by VM interrupt, since it has more information. + s_getreg_b32 ttmp2, hwreg(HW_REG_TRAPSTS) + s_and_b32 ttmp4, ttmp2, (SQ_WAVE_TRAPSTS_MEM_VIOL_MASK | SQ_WAVE_TRAPSTS_XNACK_ERROR_MASK) + s_cmp_eq_u32 ttmp4, SQ_WAVE_TRAPSTS_MEM_VIOL_MASK + s_mov_b32 ttmp4, SIGNAL_CODE_MEM_VIOL + s_cbranch_scc1 .signal_error + + // If illegal instruction then signal queue error. + s_and_b32 ttmp4, ttmp2, SQ_WAVE_TRAPSTS_ILLEGAL_INST_MASK + s_mov_b32 ttmp4, SIGNAL_CODE_ILLEGAL_INST + s_cbranch_scc1 .signal_error + + // If any other exception then return to shader. s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE s_cbranch_scc0 .exit_trap // If llvm.trap then signal queue error. s_cmp_eq_u32 ttmp2, 0x2 + s_mov_b32 ttmp4, SIGNAL_CODE_LLVM_TRAP s_cbranch_scc1 .signal_error // For other traps advance PC and return to shader. @@ -138,12 +160,25 @@ static const unsigned int kCodeTrapHandler9[] = { s_branch .exit_trap .signal_error: - // Retrieve queue_inactive_signal from amd_queue_t* passed in s[0:1]. - s_load_dwordx2 [ttmp2, ttmp3], s[0:1], 0xC0 glc + // Fetch doorbell index for our queue. + s_mov_b32 exec_lo, 0x80000000 + s_sendmsg sendmsg(MSG_GET_DOORBELL) + .wait_sendmsg: + s_nop 7 + s_bitcmp0_b32 exec_lo, 0x1F + s_cbranch_scc0 .wait_sendmsg + + // Map doorbell index to amd_queue_t* through TMA (doorbell_queue_map). + s_and_b32 ttmp2, exec_lo, MAX_NUM_DOORBELLS_MASK + s_lshl_b32 ttmp2, ttmp2, 0x3 + s_load_dwordx2 [ttmp2, ttmp3], [ttmp14, ttmp15], ttmp2 glc s_waitcnt lgkmcnt(0) - // Set queue signal value to unhandled exception error. - s_mov_b32 ttmp4, 0x80000000 + // Retrieve queue_inactive_signal from amd_queue_t*. + s_load_dwordx2 [ttmp2, ttmp3], [ttmp2, ttmp3], 0xC0 glc + s_waitcnt lgkmcnt(0) + + // Set queue signal value to error code. s_mov_b32 ttmp5, 0x0 s_atomic_swap_x2 [ttmp4, ttmp5], [ttmp2, ttmp3], 0x8 glc s_waitcnt lgkmcnt(0) @@ -189,9 +224,13 @@ static const unsigned int kCodeTrapHandler9[] = { // Return to shader at unmodified PC. s_rfe_b64 [ttmp0, ttmp1] */ - 0x92eeff6d, 0x00080010, 0xbf84001e, 0xbf06826e, 0xbf850003, 0x806c846c, - 0x826d806d, 0xbf820019, 0xc0071b80, 0x000000c0, 0xbf8cc07f, 0xbef000ff, - 0x80000000, 0xbef10080, 0xc2831c37, 0x00000008, 0xbf8cc07f, 0x87707170, + 0xb8eef803, 0x8670ff6e, 0x10000100, 0xbf06ff70, 0x00000100, 0xbef000ff, + 0x20000000, 0xbf85000e, 0x8670ff6e, 0x00000800, 0xbef000f4, 0xbf85000a, + 0x92eeff6d, 0x00080010, 0xbf84002a, 0xbf06826e, 0xbef000ff, 0x80000000, + 0xbf850003, 0x806c846c, 0x826d806d, 0xbf820023, 0xbefe00ff, 0x80000000, + 0xbf90000a, 0xbf800007, 0xbf0c9f7e, 0xbf84fffd, 0x866eff7e, 0x000003ff, + 0x8e6e836e, 0xc0051bbd, 0x0000006e, 0xbf8cc07f, 0xc0071bb7, 0x000000c0, + 0xbf8cc07f, 0xbef10080, 0xc2831c37, 0x00000008, 0xbf8cc07f, 0x87707170, 0xbf85000c, 0xc0071c37, 0x00000010, 0xbf8cc07f, 0x86f07070, 0xbf840007, 0xc0031bb7, 0x00000018, 0xbf8cc07f, 0xc0431bb8, 0x00000000, 0xbf8cc07f, 0xbf900001, 0x8778ff78, 0x00002000, 0x8f6e8b77, 0x866eff6e, 0x001f8000, diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 7c9ac234ae..3a8e68641f 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -820,6 +820,12 @@ bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) { } else if ((error_code & 128) == 128) { // Out of VGPRs errorCode = HSA_STATUS_ERROR_INVALID_ISA; + } else if ((error_code & 0x20000000) == 0x20000000) { // Memory violation (>48-bit) + errorCode = hsa_status_t(HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION); + + } else if ((error_code & 0x40000000) == 0x40000000) { // Illegal instruction + errorCode = hsa_status_t(HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION); + } else if ((error_code & 0x80000000) == 0x80000000) { // Debug trap errorCode = HSA_STATUS_ERROR_EXCEPTION; fatal = true; diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 32d5b9d4b5..f2fc3c626f 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -68,6 +68,7 @@ // Size of scratch (private) segment pre-allocated per thread, in bytes. #define DEFAULT_SCRATCH_BYTES_PER_THREAD 2048 #define MAX_WAVE_SCRATCH 8387584 // See COMPUTE_TMPRING_SIZE.WAVESIZE +#define MAX_NUM_DOORBELLS 0x400 extern core::HsaApiTable hsa_internal_api_table_; @@ -82,6 +83,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) is_kv_device_(false), trap_code_buf_(NULL), trap_code_buf_size_(0), + doorbell_queue_map_(NULL), memory_bus_width_(0), memory_max_frequency_(0), ape1_base_(0), @@ -155,6 +157,8 @@ GpuAgent::~GpuAgent() { hsaKmtFreeMemory(scratch_pool_.base(), scratch_pool_.size()); } + core::Runtime::runtime_singleton_->system_deallocator()(doorbell_queue_map_); + if (trap_code_buf_ != NULL) { ReleaseShader(trap_code_buf_, trap_code_buf_size_); } @@ -950,7 +954,15 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type, queues_[QueueUtility].touch(); // Create an HW AQL queue - *queue = new AqlQueue(this, size, node_id(), scratch, event_callback, data, is_kv_device_); + auto aql_queue = + new AqlQueue(this, size, node_id(), scratch, event_callback, data, is_kv_device_); + *queue = aql_queue; + + // Calculate index of the queue doorbell within the doorbell aperture. + auto doorbell_addr = uintptr_t(aql_queue->signal_.hardware_doorbell_ptr); + auto doorbell_idx = (doorbell_addr >> 3) & (MAX_NUM_DOORBELLS - 1); + doorbell_queue_map_[doorbell_idx] = &aql_queue->amd_queue_; + scratchGuard.Dismiss(); return HSA_STATUS_SUCCESS; } @@ -1181,9 +1193,19 @@ void GpuAgent::BindTrapHandler() { // Assemble the trap handler source code. AssembleShader("TrapHandler", AssembleTarget::ISA, trap_code_buf_, trap_code_buf_size_); + // Make an empty map from doorbell index to queue. + // The trap handler uses this to retrieve a wave's amd_queue_t*. + auto doorbell_queue_map_size = MAX_NUM_DOORBELLS * sizeof(amd_queue_t*); + + doorbell_queue_map_ = (amd_queue_t**)core::Runtime::runtime_singleton_->system_allocator()( + doorbell_queue_map_size, 0x1000, 0); + assert(doorbell_queue_map_ != NULL && "Doorbell queue map allocation failed"); + + memset(doorbell_queue_map_, 0, doorbell_queue_map_size); + // Bind the trap handler to this node. - HSAKMT_STATUS err = hsaKmtSetTrapHandler(node_id(), trap_code_buf_, - trap_code_buf_size_, NULL, 0); + HSAKMT_STATUS err = hsaKmtSetTrapHandler(node_id(), trap_code_buf_, trap_code_buf_size_, + doorbell_queue_map_, doorbell_queue_map_size); assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtSetTrapHandler() failed"); } diff --git a/runtime/hsa-runtime/core/runtime/hsa.cpp b/runtime/hsa-runtime/core/runtime/hsa.cpp index c98b2e5944..5c71f76f35 100644 --- a/runtime/hsa-runtime/core/runtime/hsa.cpp +++ b/runtime/hsa-runtime/core/runtime/hsa.cpp @@ -2719,6 +2719,17 @@ hsa_status_t hsa_status_string( *status_string = "HSA_STATUS_ERROR_FATAL: The queue received an error that may require process " "termination."; + break; + case HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION: + *status_string = + "HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION: The agent attempted to access " + "memory beyond the largest legal address."; + break; + case HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION: + *status_string = + "HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION: The agent attempted to execute an " + "illegal shader instruction."; + break; case HSA_EXT_STATUS_ERROR_IMAGE_FORMAT_UNSUPPORTED: *status_string = "HSA_EXT_STATUS_ERROR_IMAGE_FORMAT_UNSUPPORTED: Image " diff --git a/runtime/hsa-runtime/inc/hsa_ext_amd.h b/runtime/hsa-runtime/inc/hsa_ext_amd.h index f776795b07..de4d0890b3 100644 --- a/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -64,7 +64,17 @@ enum { /** * The memory pool is invalid. */ - HSA_STATUS_ERROR_INVALID_MEMORY_POOL = 40 + HSA_STATUS_ERROR_INVALID_MEMORY_POOL = 40, + + /** + * Agent accessed memory beyond the maximum legal address. + */ + HSA_STATUS_ERROR_MEMORY_APERTURE_VIOLATION = 41, + + /** + * Agent executed an invalid shader instruction. + */ + HSA_STATUS_ERROR_ILLEGAL_INSTRUCTION = 42, }; /**