From 3603303bc7eeb7f57d840f5a15b5bbaec6b50d87 Mon Sep 17 00:00:00 2001 From: Laurent Morichetti Date: Mon, 20 Sep 2021 21:28:28 -0700 Subject: [PATCH] Update the trap handler for gfx940 gfx940 uses ttmp11 to hold the queue packet index so the first level trap handler uses ttmp13 instead to save ib_sts. Repurpose ttmp11[31] to mean that the ttmps are initialized. The issue was that the debugger could not tell whether ttmp6 was written by the trap handler when determining the stop reason. If ttmp11[31]=0, then the trap handler has not been executed and ttmp6 should be assumed to be 0. If ttmp11[31]=1, then ttmp6 holds the trap_id, if an s_trap instruction caused the exception. Signed-off-by: Laurent Morichetti Signed-off-by: Lancelot Six Change-Id: I9af903abae044b9ec530306229caf3b883f3ee46 [ROCm/ROCR-Runtime commit: f31b312611586899d00e7995426bd282e3252cfc] --- .../core/runtime/amd_gpu_agent.cpp | 11 +++- .../core/runtime/trap_handler/CMakeLists.txt | 5 +- .../core/runtime/trap_handler/trap_handler.s | 64 ++++++++++++------- .../runtime/hsa-runtime/loader/executable.cpp | 3 +- 4 files changed, 56 insertions(+), 27 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 6664d60ce9..fdf033081c 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -243,6 +243,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar ASICShader compute_8; ASICShader compute_9; ASICShader compute_90a; + ASICShader compute_940; ASICShader compute_1010; ASICShader compute_10; ASICShader compute_11; @@ -255,6 +256,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, {kCodeTrapHandler9, sizeof(kCodeTrapHandler9), 2, 4}, {kCodeTrapHandler90a, sizeof(kCodeTrapHandler90a), 2, 4}, + {NULL, 0, 0, 0}, {kCodeTrapHandler1010, sizeof(kCodeTrapHandler1010), 2, 4}, {kCodeTrapHandler10, sizeof(kCodeTrapHandler10), 2, 4}, {NULL, 0, 0, 0}, @@ -265,6 +267,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, + {kCodeTrapHandlerV2_940, sizeof(kCodeTrapHandlerV2_940), 2, 4}, {kCodeTrapHandlerV2_1010, sizeof(kCodeTrapHandlerV2_1010), 2, 4}, {kCodeTrapHandlerV2_10, sizeof(kCodeTrapHandlerV2_10), 2, 4}, {kCodeTrapHandlerV2_11, sizeof(kCodeTrapHandlerV2_11), 2, 4}, @@ -275,6 +278,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, + {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, {kCodeCopyAligned11, sizeof(kCodeCopyAligned11), 32, 12}, @@ -285,6 +289,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, + {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, {kCodeCopyMisaligned11, sizeof(kCodeCopyMisaligned11), 23, 10}, @@ -295,6 +300,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar {kCodeFill8, sizeof(kCodeFill8), 19, 8}, {kCodeFill8, sizeof(kCodeFill8), 19, 8}, {kCodeFill8, sizeof(kCodeFill8), 19, 8}, + {kCodeFill8, sizeof(kCodeFill8), 19, 8}, {kCodeFill10, sizeof(kCodeFill10), 19, 8}, {kCodeFill10, sizeof(kCodeFill10), 19, 8}, {kCodeFill11, sizeof(kCodeFill11), 19, 8}, @@ -314,9 +320,10 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar asic_shader = &compiled_shader_it->second.compute_8; break; case 9: - if(((isa_->GetMinorVersion() == 0) && (isa_->GetStepping() == 10)) || - ((isa_->GetMinorVersion() == 4) && (isa_->GetStepping() == 0))) + if((isa_->GetMinorVersion() == 0) && (isa_->GetStepping() == 10)) asic_shader = &compiled_shader_it->second.compute_90a; + else if((isa_->GetMinorVersion() == 4) && (isa_->GetStepping() == 0)) + asic_shader = &compiled_shader_it->second.compute_940; else asic_shader = &compiled_shader_it->second.compute_9; break; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt index c98e10486a..929b896bc2 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt @@ -46,8 +46,9 @@ cmake_minimum_required ( VERSION 3.7 ) find_package(Clang REQUIRED HINTS ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm ) find_package(LLVM REQUIRED HINTS ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm ) -set (TARGET_DEVS "gfx900;gfx1010;gfx1030;gfx1100") -set (POSTFIX "9;1010;10;11") +set (TARGET_DEVS "gfx900;gfx940;gfx1010;gfx1030;gfx1100") +set (POSTFIX "9;940;1010;10;11") + if(${CMAKE_VERBOSE_MAKEFILE}) get_property(clang_path TARGET clang PROPERTY LOCATION) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s index 9e1d56bd4b..a753247a4d 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s @@ -71,22 +71,27 @@ .set TTMP6_SAVED_TRAP_ID_SIZE , 4 .set TTMP6_SAVED_TRAP_ID_MASK , (((1 << TTMP6_SAVED_TRAP_ID_SIZE) - 1) << TTMP6_SAVED_TRAP_ID_SHIFT) .set TTMP6_SAVED_TRAP_ID_BFE , (TTMP6_SAVED_TRAP_ID_SHIFT | (TTMP6_SAVED_TRAP_ID_SIZE << 16)) -.set TTMP11_PC_HI_SHIFT , 7 -.set TTMP11_DEBUG_ENABLED_SHIFT , 23 + +.set TTMP_PC_HI_SHIFT , 7 +.set TTMP_DEBUG_ENABLED_SHIFT , 23 .if .amdgcn.gfx_generation_number == 9 - .set TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT , 26 + .set TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT , 26 .set SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT , 15 .set SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK , 0x1F8000 .elseif .amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3 - .set TTMP11_SAVE_REPLAY_W64H_SHIFT , 31 - .set TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT , 24 + .set TTMP_SAVE_REPLAY_W64H_SHIFT , 31 + .set TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT , 24 .set SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT , 25 .set SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT , 15 .set SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK , 0x3F8000 .set SQ_WAVE_IB_STS_REPLAY_W64H_MASK , 0x2000000 .endif +.if .amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor >= 4 + .set TTMP11_TTMPS_SETUP_SHIFT , 31 +.endif + // ABI between first and second level trap handler: // ttmp0 = PC[31:0] // ttmp12 = SQ_WAVE_STATUS @@ -94,7 +99,10 @@ // ttmp15 = TMA[63:32] // gfx9: // ttmp1 = 0[2:0], PCRewind[3:0], HostTrap[0], TrapId[7:0], PC[47:32] +// gfx906/gfx908/gfx90a: // ttmp11 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0] +// gfx940: +// ttmp13 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[22:0] // gfx10: // ttmp1 = 0[0], PCRewind[5:0], HostTrap[0], TrapId[7:0], PC[47:32] // gfx1010: @@ -117,7 +125,11 @@ trap_entry: // If llvm.debugtrap and debugger is not attached. s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP s_cbranch_scc0 .no_skip_debugtrap - s_bitcmp0_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT +.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || .amdgcn.gfx_generation_number == 10 + s_bitcmp0_b32 ttmp11, TTMP_DEBUG_ENABLED_SHIFT +.else + s_bitcmp0_b32 ttmp13, TTMP_DEBUG_ENABLED_SHIFT +.endif s_cbranch_scc0 .no_skip_debugtrap // Ignore llvm.debugtrap. @@ -212,26 +224,18 @@ trap_entry: // // ttmp7: pc_lo[31:0] // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0] - -.if ((.amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor >= 3) || .amdgcn.gfx_generation_number > 10) - s_branch .halt_wave -.else +.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || (.amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3) // Save the PC s_mov_b32 ttmp7, ttmp0 s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK - s_lshl_b32 ttmp1, ttmp1, TTMP11_PC_HI_SHIFT - s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP11_PC_HI_SHIFT) + s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT + s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT) s_or_b32 ttmp11, ttmp11, ttmp1 // Park the wave s_getpc_b64 [ttmp0, ttmp1] s_add_u32 ttmp0, ttmp0, .parked - . s_addc_u32 ttmp1, ttmp1, 0x0 - s_branch .halt_wave - -.parked: - s_trap 0x2 - s_branch .parked .endif .halt_wave: @@ -239,17 +243,29 @@ trap_entry: s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_HALT_SHIFT +.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor >= 4) + s_bitcmp1_b32 ttmp11, TTMP11_TTMPS_SETUP_SHIFT + s_cbranch_scc1 .ttmps_initialized + s_mov_b32 ttmp4, 0 + s_mov_b32 ttmp5, 0 + s_bitset1_b32 ttmp11, TTMP11_TTMPS_SETUP_SHIFT +.ttmps_initialized: +.endif + .exit_trap: // Restore SQ_WAVE_IB_STS. .if .amdgcn.gfx_generation_number == 9 - s_lshr_b32 ttmp2, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) +.if .amdgcn.gfx_generation_minor < 4 + s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) +.else + s_lshr_b32 ttmp2, ttmp13, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) +.endif s_and_b32 ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2 -.endif -.if .amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3 - s_lshr_b32 ttmp2, ttmp11, (TTMP11_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) +.elseif .amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3 + s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT) s_and_b32 ttmp3, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK - s_lshr_b32 ttmp2, ttmp11, (TTMP11_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT) + s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT) s_and_b32 ttmp2, ttmp2, SQ_WAVE_IB_STS_REPLAY_W64H_MASK s_or_b32 ttmp2, ttmp2, ttmp3 s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2 @@ -262,3 +278,7 @@ trap_entry: // Return to original (possibly modified) PC. s_rfe_b64 [ttmp0, ttmp1] + +.parked: + s_trap 0x2 + s_branch .parked diff --git a/projects/rocr-runtime/runtime/hsa-runtime/loader/executable.cpp b/projects/rocr-runtime/runtime/hsa-runtime/loader/executable.cpp index dbad4b93b1..b659d757e5 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/loader/executable.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/loader/executable.cpp @@ -80,7 +80,8 @@ __attribute__((noinline)) static void _loader_debug_state() { // 5: New trap handler ABI. Save the PC in ttmp11[22:7] ttmp6[31:0], and park the wave if stopped // 6: New trap handler ABI. ttmp6[25:0] contains dispatch index modulo queue size // 7: New trap handler ABI. Send interrupts as a bitmask, coalescing concurrent exceptions. -HSA_API r_debug _amdgpu_r_debug = {7, +// 8: New trap handler ABI for gfx940: Initialize ttmp[4:5] if ttmp11[31] == 0. +HSA_API r_debug _amdgpu_r_debug = {8, nullptr, reinterpret_cast(&_loader_debug_state), r_debug::RT_CONSISTENT,