Park waves for gfx11 and bump abi version to 9

On gfx11, with a sequence such as

  s_trap 2
  s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
  s_endpgm

the s_sendmsg does deallocate registers while the wave is supposed to be
stopped.  As a result, the wave cannot do the expected context save
operations, and cannot context save.

To avoid this problem, park the wave in the trap handler for gfx11.

Note that gfx11 has implemented an instruction cache prefetch.  When
parked, the prefetch tries to access memory past the end of trap handler
which causes memory violation exceptions to be reported.  To avoid this,
we need to add padding at the end of the trap handler.  The padding
consists of `s_code_end` instructions  Given that the trap handler is
loaded at a 0x1000 aligned address the maximum prefetch amount (in
bytes) is given by `256 - (trap_handler_size % 64)`.

Change-Id: I5446da54a965a64f21cb0fd3ce3caa4b6137a933

runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s

@@ -225,7 +225,7 @@ trap_entry:
   //
   // ttmp7:  pc_lo[31:0]
   // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0]
-.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || (.amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3)
+.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || (.amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3) || (.amdgcn.gfx_generation_number == 11)
   // Save the PC
   s_mov_b32            ttmp7, ttmp0
   s_and_b32            ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK
@@ -283,3 +283,11 @@ trap_entry:
 .parked:
   s_trap               0x2
   s_branch             .parked
+
+// For gfx11, add padding instructions so we can ensure instruction cache
+// prefetch always has something to load.
+.if .amdgcn.gfx_generation_number == 11
+.rept (256 - ((. - trap_entry) % 64)) / 4
+  s_code_end
+.endr
+.endif

runtime/hsa-runtime/loader/executable.cpp

@@ -80,8 +80,9 @@ __attribute__((noinline)) static void _loader_debug_state() {
 // 5: New trap handler ABI. Save the PC in ttmp11[22:7] ttmp6[31:0], and park the wave if stopped
 // 6: New trap handler ABI. ttmp6[25:0] contains dispatch index modulo queue size
 // 7: New trap handler ABI. Send interrupts as a bitmask, coalescing concurrent exceptions.
-// 8: New trap handler ABI for gfx940: Initialize ttmp[4:5] if ttmp11[31] == 0.
-HSA_API r_debug _amdgpu_r_debug = {8,
+// 8: New trap handler ABI. for gfx940: Initialize ttmp[4:5] if ttmp11[31] == 0.
+// 9: New trap handler API. For gfx11: Save PC in ttmp11[22:7] ttmp6[31:0], and park the wave if stopped.
+HSA_API r_debug _amdgpu_r_debug = {9,
                            nullptr,
                            reinterpret_cast<uintptr_t>(&_loader_debug_state),
                            r_debug::RT_CONSISTENT,