AI/rocm-systems
2
0
Çatalla 0
Kod Konular Değişiklik İstekleri İşlemler Paketler Projeler Sürüm Viki Aktivite

SWDEV-517078 - Update 2nd level trap handlers (#148)

Kaynağa Gözat 
* SWDEV-517078 - Maintain the trap handler ABI version in CLR

The trap handler ABI version is communicated to the debugger using
the r_version field in the r_debug structure.  This structure is
an external dependency, which makes it complicated to keep the trap
handler source (in CRL) and the ABI version number (external dependency)
in sync.

This patch proposes to patch the trap handler ABI version number in
_amdgpu_r_debug before communicating it to the debugger.

We can't directly include sc's executable.hpp file in CRL as it relies
on conflicting definition of ELF related types, so instead we need to
rely on a-priori knowledge on the r_debug structure.  Fortunately, this
structure is part of a stable ABI, so its layout is guaranteed to be
kept stable.

Update the 2nd level trap handler to follow updates from the
ROCr-runtime.  The trap handlers are stripped from parts dedicated to
architectures unsupported by CLR.

Bump the r_debug.r_version to track the ABI changes in the trap handler.
Bu işleme şunda yer alıyor:
Six, Lancelot
2025-04-11 18:59:54 +01:00
işlemeyi yapan: GitHub
ebeveyn c4fa3ef927
işleme 7b72c1b786
2 değiştirilmiş dosya ile 297 ekleme ve 234 silme
Yama Dosyasını İndir Diff Dosyasını İndir Tüm dosyaları genişlet Tüm dosyaları daralt
rocclr/device/pal/palblitcl.cpp
+287 -234
Dosyayı Görüntüle
@@ -41,20 +41,44 @@ extern void __amd_scheduler_pal(__global void*, __global void*, uint);
}
\n);
/* Trap handler for compute.
*
* The reference version for this are maintained at
* https://github.com/ROCm/ROCR-Runtime/tree/amd-staging/runtime/hsa-runtime/core/runtime/trap_handler
*
* The trap handler source is copied from the above URL, with the following
* modifications:
* - Add the following directive to declare the trap_entry symbol (this is
* later used by LightningProgram::GetTrapHandlerAddress to locate the load
* address of the trap handler):
*
* .globl trap_entry
* .type trap_entry,@function
* .align 256
* - Remove code related to architectures or functionalities not supported
* by CLR.
*
* Make sure to update the TrapHandlerABIVersion definition when appropriate.
*/
const char* TrapHandlerCode = RUNTIME_KERNEL(
\n.if .amdgcn.gfx_generation_number < 12
\n.if .amdgcn.gfx_generation_number == 11
\n.set SQ_WAVE_PC_HI_ADDRESS_MASK , 0xFFFF
\n.set SQ_WAVE_PC_HI_HT_SHIFT , 24
\n.set SQ_WAVE_PC_HI_TRAP_ID_SHIFT , 16
\n.set SQ_WAVE_PC_HI_TRAP_ID_SIZE , 8
\n.set SQ_WAVE_PC_HI_TRAP_ID_BFE, (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16))
\n.set SQ_WAVE_STATUS_HALT_SHIFT, 13
\n.set SQ_WAVE_STATUS_HALT_BFE, (SQ_WAVE_STATUS_HALT_SHIFT | (1 << 16))
\n.set SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT, 8
\n.set SQ_WAVE_PC_HI_TRAP_ID_BFE , (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16))
\n.set SQ_WAVE_STATUS_HALT_SHIFT , 13
\n.set SQ_WAVE_STATUS_TRAP_SKIP_EXPORT_SHIFT , 18
\n.set SQ_WAVE_STATUS_HALT_BFE , (SQ_WAVE_STATUS_HALT_SHIFT | (1 << 16))
\n.set SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT , 8
\n.set SQ_WAVE_TRAPSTS_ILLEGAL_INST_SHIFT , 11
\n.set SQ_WAVE_TRAPSTS_XNACK_ERROR_SHIFT , 28
\n.set SQ_WAVE_TRAPSTS_MATH_EXCP , 0x7F
\n.set SQ_WAVE_TRAPSTS_PERF_SNAPSHOT_SHIFT , 26
\n.set SQ_WAVE_TRAPSTS_HOST_TRAP_SHIFT , 22
\n.set SQ_WAVE_MODE_EXCP_EN_SHIFT , 12
\n.set SQ_WAVE_MODE_EXCP_EN_SIZE , 8
\n.set TRAP_ID_ABORT , 2
\n.set TRAP_ID_DEBUGTRAP , 3
\n.set DOORBELL_ID_SIZE , 10
@@ -65,202 +89,172 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n.set EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 , (1 << (DOORBELL_ID_SIZE + 3))
\n.set EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 4))
\n.set EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 5))
\n
\n.set TTMP6_SPI_TTMPS_SETUP_DISABLED_SHIFT , 31
\n.set TTMP6_WAVE_STOPPED_SHIFT , 30
\n.set TTMP6_SAVED_STATUS_HALT_SHIFT , 29
\n.set TTMP6_SAVED_STATUS_HALT_MASK, (1 << TTMP6_SAVED_STATUS_HALT_SHIFT)
\n.set TTMP6_SAVED_TRAP_ID_SHIFT, 25
\n.set TTMP6_SAVED_TRAP_ID_SIZE, 4
\n.set TTMP6_SAVED_TRAP_ID_MASK, (((1 << TTMP6_SAVED_TRAP_ID_SIZE) - 1) << TTMP6_SAVED_TRAP_ID_SHIFT)
\n.set TTMP6_SAVED_TRAP_ID_BFE, (TTMP6_SAVED_TRAP_ID_SHIFT | (TTMP6_SAVED_TRAP_ID_SIZE << 16))
\n.set TTMP_PC_HI_SHIFT, 7
\n.set TTMP_DEBUG_ENABLED_SHIFT, 23
\n.if .amdgcn.gfx_generation_number == 9
\n.set TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT, 26
\n.set SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, 15
\n.set SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK, 0x1F8000
\n.elseif .amdgcn.gfx_generation_number == 10 &&.amdgcn.gfx_generation_minor < 3
\n.set TTMP_SAVE_REPLAY_W64H_SHIFT, 31
\n.set TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT, 24
\n.set SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT, 25
\n.set SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT, 15
\n.set SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK, 0x3F8000
\n.set SQ_WAVE_IB_STS_REPLAY_W64H_MASK, 0x2000000
\n.endif
\n.if .amdgcn.gfx_generation_number == 9 &&.amdgcn.gfx_generation_minor >= 4
\n.set TTMP11_TTMPS_SETUP_SHIFT, 31
\n.endif
// ABI between first and second level trap handler:
// ttmp0 = PC[31:0]
// ttmp12 = SQ_WAVE_STATUS
// ttmp14 = TMA[31:0]
// ttmp15 = TMA[63:32]
// gfx9:
// ttmp1 = 0[2:0], PCRewind[3:0], HostTrap[0], TrapId[7:0], PC[47:32]
// gfx906/gfx908/gfx90a:
// ttmp11 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0]
// gfx942/gfx950:
// ttmp13 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[22:0]
// gfx10:
// ttmp1 = 0[0], PCRewind[5:0], HostTrap[0], TrapId[7:0], PC[47:32]
// gfx1010:
// ttmp11 = SQ_WAVE_IB_STS[25], SQ_WAVE_IB_STS[21:15], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0]
// gfx1030/gfx1100:
// ttmp11 = 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0]
\n .globl trap_entry
\n .type trap_entry,@function
\ntrap_entry:
// Branch if not a trap (an exception instead).
\n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE
\n s_cbranch_scc0 .no_skip_debugtrap
// If caused by s_trap then advance PC.
\n s_bitcmp1_b32 ttmp1, SQ_WAVE_PC_HI_HT_SHIFT
\n s_cbranch_scc1 .not_s_trap
\n s_add_u32 ttmp0, ttmp0, 0x4
\n s_addc_u32 ttmp1, ttmp1, 0x0
\n.not_s_trap:
// If llvm.debugtrap and debugger is not attached.
\n s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP
\n s_cbranch_scc0 .no_skip_debugtrap
\n.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || .amdgcn.gfx_generation_number == 10
\n s_bitcmp0_b32 ttmp11, TTMP_DEBUG_ENABLED_SHIFT
\n.else
\n s_bitcmp0_b32 ttmp13, TTMP_DEBUG_ENABLED_SHIFT
\n.endif
\n s_cbranch_scc0 .no_skip_debugtrap
// Ignore llvm.debugtrap.
\n s_branch .exit_trap
\n.no_skip_debugtrap:
// Save trap id and halt status in ttmp6.
\n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK)
\n s_min_u32 ttmp2, ttmp2, 0xF
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATUS_HALT_BFE
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
// Fetch doorbell id for our queue.
\n.if .amdgcn.gfx_generation_number < 11
\n s_mov_b32 ttmp2, exec_lo
\n s_mov_b32 ttmp3, exec_hi
\n s_mov_b32 exec_lo, 0x80000000
\n s_sendmsg sendmsg(MSG_GET_DOORBELL)
\n.wait_sendmsg:
\n s_nop 0x7
\n s_bitcmp0_b32 exec_lo, 0x1F
\n s_cbranch_scc0 .wait_sendmsg
\n s_mov_b32 exec_hi, ttmp3
// Restore exec_lo, move the doorbell_id into ttmp3
\n s_and_b32 ttmp3, exec_lo, DOORBELL_ID_MASK
\n s_mov_b32 exec_lo, ttmp2
\n.else
\n s_sendmsg_rtn_b32 ttmp3, sendmsg(MSG_RTN_GET_DOORBELL)
\n s_waitcnt lgkmcnt(0)
\n s_and_b32 ttmp3, ttmp3, DOORBELL_ID_MASK
\n.endif
// Map trap reason to an exception code.
\n s_getreg_b32 ttmp2, hwreg(HW_REG_TRAPSTS)
\n.set TTMP6_SAVED_STATUS_HALT_MASK , (1 << TTMP6_SAVED_STATUS_HALT_SHIFT)
\n.set TTMP6_SAVED_TRAP_ID_SHIFT , 25
\n.set TTMP6_SAVED_TRAP_ID_SIZE , 4
\n.set TTMP6_SAVED_TRAP_ID_MASK , (((1 << TTMP6_SAVED_TRAP_ID_SIZE) - 1) << TTMP6_SAVED_TRAP_ID_SHIFT)
\n.set TTMP6_SAVED_TRAP_ID_BFE , (TTMP6_SAVED_TRAP_ID_SHIFT | (TTMP6_SAVED_TRAP_ID_SIZE << 16))
\n
\n.set TTMP_PC_HI_SHIFT , 7
\n.set TTMP_DEBUG_ENABLED_SHIFT , 23
\n
\n// ABI between first and second level trap handler:
\n// ttmp0 = PC[31:0]
\n// ttmp6 = 0[6:0], DispatchPktIndx[24:0]
\n// ttmp8 = WorkgroupIdX
\n// ttmp9 = WorkgroupIdY
\n// ttmp10 = WorkgroupIdZ
\n// ttmp11 = 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0]
\n// ttmp12 = SQ_WAVE_STATUS
\n// ttmp14 = TMA[31:0]
\n// ttmp15 = TMA[63:32]
\n
\n .globl trap_entry
\n .type trap_entry,@function
\n .align 256
\n
\ntrap_entry:
\n // Extract trap_id from ttmp2
\n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE
\n s_cbranch_scc0 .not_s_trap // If trap_id == 0, it's not an s_trap nor host trap
\n
\n // Check if the it was an host trap.
\n s_bitcmp1_b32 ttmp1, SQ_WAVE_PC_HI_HT_SHIFT
\n s_cbranch_scc0 .not_host_trap
\n
\n.not_host_trap:
\n // It's an s_trap; advance the PC
\n s_add_u32 ttmp0, ttmp0, 0x4
\n s_addc_u32 ttmp1, ttmp1, 0x0
\n
\n // If llvm.debugtrap and debugger is not attached.
\n s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP
\n s_cbranch_scc0 .no_skip_debugtrap
\n s_bitcmp0_b32 ttmp11, TTMP_DEBUG_ENABLED_SHIFT
\n s_cbranch_scc0 .no_skip_debugtrap
\n
\n // Ignore llvm.debugtrap.
\n s_branch .exit_trap
\n
\n.no_skip_debugtrap:
\n // Save trap id and halt status in ttmp6.
\n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK)
\n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE
\n s_min_u32 ttmp2, ttmp2, 0xF
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATUS_HALT_BFE
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n
\n // Fetch doorbell id for our queue.
\n s_sendmsg_rtn_b32 ttmp3, sendmsg(MSG_RTN_GET_DOORBELL)
\n s_waitcnt lgkmcnt(0)
\n s_and_b32 ttmp3, ttmp3, DOORBELL_ID_MASK
\n
\n // Map trap reason to an exception code.
\n s_getreg_b32 ttmp2, hwreg(HW_REG_TRAPSTS)
\n
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_XNACK_ERROR_SHIFT
\n s_cbranch_scc0 .not_memory_violation
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0
\n
\n // Aperture violation requires XNACK_ERROR == 0.
\n s_branch .not_aperture_violation
\n
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_XNACK_ERROR_SHIFT
\n s_cbranch_scc0 .not_memory_violation
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0
// Aperture violation requires XNACK_ERROR == 0.
\n s_branch .not_aperture_violation
\n.not_memory_violation:
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT
\n s_cbranch_scc0 .not_aperture_violation
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT
\n s_cbranch_scc0 .not_aperture_violation
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0
\n
\n.not_aperture_violation:
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_ILLEGAL_INST_SHIFT
\n s_cbranch_scc0 .not_illegal_instruction
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_ILLEGAL_INST_SHIFT
\n s_cbranch_scc0 .not_illegal_instruction
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0
\n
\n.not_illegal_instruction:
\n s_and_b32 ttmp2, ttmp2, SQ_WAVE_TRAPSTS_MATH_EXCP
\n s_cbranch_scc0 .not_math_exception
\n s_getreg_b32 ttmp7, hwreg(HW_REG_MODE)
\n s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
\n s_and_b32 ttmp2, ttmp2, ttmp7
\n s_cbranch_scc0 .not_math_exception
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0
\n s_and_b32 ttmp2, ttmp2, SQ_WAVE_TRAPSTS_MATH_EXCP
\n s_cbranch_scc0 .not_math_exception
\n s_getreg_b32 ttmp7, hwreg(HW_REG_MODE)
\n s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT
\n s_and_b32 ttmp2, ttmp2, ttmp7
\n s_cbranch_scc0 .not_math_exception
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0
\n
\n.not_math_exception:
\n s_bfe_u32 ttmp2, ttmp6, TTMP6_SAVED_TRAP_ID_BFE
\n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT
\n s_cbranch_scc0 .not_abort_trap
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0
\n s_bfe_u32 ttmp2, ttmp6, TTMP6_SAVED_TRAP_ID_BFE
\n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT
\n s_cbranch_scc0 .not_abort_trap
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0
\n
\n.not_abort_trap:
// If no other exception was flagged then report a generic error.
\n s_andn2_b32 ttmp2, ttmp3, DOORBELL_ID_MASK
\n s_cbranch_scc1 .send_interrupt
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n // If no other exception was flagged then report a generic error.
\n s_andn2_b32 ttmp2, ttmp3, DOORBELL_ID_MASK
\n s_cbranch_scc1 .send_interrupt
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n.send_interrupt:
// m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id
\n s_mov_b32 ttmp2, m0
\n s_mov_b32 m0, ttmp3
\n s_nop 0x0 // Manually inserted wait states
\n s_sendmsg sendmsg(MSG_INTERRUPT)
\n s_waitcnt lgkmcnt(0) // Wait for the message to go out.
\n s_mov_b32 m0, ttmp2
// Parking the wave requires saving the original pc in the preserved ttmps.
// Register layout before parking the wave:
//
// ttmp7: 0[31:0]
// ttmp11: 1st_level_ttmp11[31:23] 0[15:0] 1st_level_ttmp11[6:0]
//
// After parking the wave:
//
// ttmp7: pc_lo[31:0]
// ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0]
\n.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor < 4) || (.amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3)
\n // m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id
\n s_mov_b32 ttmp2, m0
\n s_mov_b32 m0, ttmp3
\n s_nop 0x0 // Manually inserted wait states
\n s_sendmsg sendmsg(MSG_INTERRUPT)
\n s_waitcnt lgkmcnt(0) // Wait for the message to go out.
\n s_mov_b32 m0, ttmp2
\n
\n // Parking the wave requires saving the original pc in the preserved ttmps.
\n // Register layout before parking the wave:
\n //
\n // ttmp7: 0[31:0]
\n // ttmp11: 1st_level_ttmp11[31:23] 0[15:0] 1st_level_ttmp11[6:0]
\n //
\n // After parking the wave:
\n //
\n // ttmp7: pc_lo[31:0]
\n // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0]
\n // Save the PC
\n s_mov_b32 ttmp7, ttmp0
\n s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK
\n s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT
\n s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT)
\n s_or_b32 ttmp11, ttmp11, ttmp1
// Park the wave
\n s_getpc_b64 [ttmp0, ttmp1]
\n s_add_u32 ttmp0, ttmp0, .parked - .
\n s_addc_u32 ttmp1, ttmp1, 0x0
\n.endif
\n s_mov_b32 ttmp7, ttmp0
\n s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK
\n s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT
\n s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT)
\n s_or_b32 ttmp11, ttmp11, ttmp1
\n
\n // Park the wave
\n s_getpc_b64 [ttmp0, ttmp1]
\n s_add_u32 ttmp0, ttmp0, .parked - .
\n s_addc_u32 ttmp1, ttmp1, 0x0
\n
\n.halt_wave:
// Halt the wavefront upon restoring STATUS below.
\n s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT
\n s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_HALT_SHIFT
\n.if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor >= 4)
\n s_bitcmp1_b32 ttmp11, TTMP11_TTMPS_SETUP_SHIFT
\n s_cbranch_scc1 .ttmps_initialized
\n s_mov_b32 ttmp4, 0
\n s_mov_b32 ttmp5, 0
\n s_bitset1_b32 ttmp11, TTMP11_TTMPS_SETUP_SHIFT
\n.ttmps_initialized:
\n.endif
\n // Halt the wavefront upon restoring STATUS below.
\n s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT
\n s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_HALT_SHIFT
\n // Set WAVE.SKIP_EXPORT as a maker so the debugger knows the trap handler was
\n // entered and has decided to halt the wavee.
\n s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_TRAP_SKIP_EXPORT_SHIFT
\n
\n.exit_trap:
// Restore SQ_WAVE_IB_STS.
\n.if .amdgcn.gfx_generation_number == 9
\n.if .amdgcn.gfx_generation_minor < 4
\n s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
\n.else
\n s_lshr_b32 ttmp2, ttmp13, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
\n.endif
\n s_and_b32 ttmp2, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
\n s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2
\n.elseif .amdgcn.gfx_generation_number == 10 && .amdgcn.gfx_generation_minor < 3
\n s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_RCNT_FIRST_REPLAY_SHIFT - SQ_WAVE_IB_STS_FIRST_REPLAY_SHIFT)
\n s_and_b32 ttmp3, ttmp2, SQ_WAVE_IB_STS_RCNT_FIRST_REPLAY_MASK
\n s_lshr_b32 ttmp2, ttmp11, (TTMP_SAVE_REPLAY_W64H_SHIFT - SQ_WAVE_IB_STS_REPLAY_W64H_SHIFT)
\n s_and_b32 ttmp2, ttmp2, SQ_WAVE_IB_STS_REPLAY_W64H_MASK
\n s_or_b32 ttmp2, ttmp2, ttmp3
\n s_setreg_b32 hwreg(HW_REG_IB_STS), ttmp2
\n.endif
// Restore SQ_WAVE_STATUS.
\n s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
\n s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
\n s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12
// Return to original (possibly modified) PC.
\n s_rfe_b64 [ttmp0, ttmp1]
\n // Restore SQ_WAVE_STATUS.
\n s_and_b64 exec, exec, exec // restore STATUS.EXECZ, not writable by s_setreg_b32
\n s_and_b64 vcc, vcc, vcc // restore STATUS.VCCZ, not writable by s_setreg_b32
\n s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12
\n
\n // Return to original (possibly modified) PC.
\n s_rfe_b64 [ttmp0, ttmp1]
\n
\n.parked:
\n s_trap 0x2
\n s_branch .parked
\n.else
\n s_trap 0x2
\n s_branch .parked
\n
\n// For gfx11, add padding instructions so we can ensure instruction cache
\n// prefetch always has something to load.
\n.rept (256 - ((. - trap_entry) % 64)) / 4
\n s_code_end
\n.endr
\n.elseif .amdgcn.gfx_generation_number == 12
\n.set DOORBELL_ID_SIZE , 10
\n.set DOORBELL_ID_MASK , ((1 << DOORBELL_ID_SIZE) - 1)
\n.set EC_QUEUE_WAVE_ABORT_M0 , (1 << (DOORBELL_ID_SIZE + 0))
@@ -269,20 +263,31 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n.set EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 , (1 << (DOORBELL_ID_SIZE + 3))
\n.set EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 4))
\n.set EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 5))
\n
\n.set SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK , (1 << 4) - 1
\n.set SQ_WAVE_EXCP_FLAG_PRIV_MEMVIOL_SHIFT , 4
\n.set SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT , 7
\n.set SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT , 6
\n.set SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT , 8
\n.set SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT , 7
\n.set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT , 8
\n.set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT , 9
\n.set SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_SHIFT , 11
\n.set SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT , 12
\n
\n.set SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT , 0
\n.set SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE , 6
\n.set SQ_WAVE_TRAP_CTRL_MATH_EXCP_SHIFT , 0
\n.set SQ_WAVE_TRAP_CTRL_MATH_EXCP_SIZE , 6
\n.set SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE , 7
\n
\n.set SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK , ((1 << 7) - 1)
\n.set SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT , 7
\n.set SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT , 8
\n.set SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST , 9
\n
\n.set SQ_WAVE_PC_HI_ADDRESS_MASK , 0xFFFF
\n.set SQ_WAVE_PC_HI_TRAP_ID_BFE , (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16))
\n.set SQ_WAVE_PC_HI_TRAP_ID_SHIFT , 28
\n.set SQ_WAVE_PC_HI_TRAP_ID_SIZE , 4
\n.set SQ_WAVE_STATE_PRIV_HALT_BFE , (SQ_WAVE_STATE_PRIV_HALT_SHIFT | (1 << 16))
\n.set SQ_WAVE_STATE_PRIV_HALT_SHIFT , 14
\n.set SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT , 2
\n.set TRAP_ID_ABORT , 2
\n.set TRAP_ID_DEBUGTRAP , 3
\n.set TTMP6_SAVED_STATUS_HALT_MASK , (1 << TTMP6_SAVED_STATUS_HALT_SHIFT)
@@ -303,42 +308,57 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n// ttmp14 = TMA[31:0]
\n// ttmp15 = TMA[63:32]
\n
\n .globl trap_entry
\n .type trap_entry,@function
\n .align 256
\n
\ntrap_entry:
\n // Clear ttmp3 as it will contain the exception code.
\n s_mov_b32 ttmp3, 0
\n
\n // Branch if not a trap (an exception instead).
\n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE
\n s_cbranch_scc0 .no_skip_debugtrap
\n s_cbranch_scc0 .check_exceptions
\n
\n // If caused by s_trap then advance PC.
\n // If caused by s_trap then advance PC, then figure out the trap ID:
\n // - if trapID is DEBUGTRAP and debugger is attach, report WAVE_TRAP,
\n // - if trapID is ABORTTRAP, report WAVE_ABORT,
\n // - report WAVE_TRAP for any other trap ID.
\n s_add_u32 ttmp0, ttmp0, 0x4
\n s_addc_u32 ttmp1, ttmp1, 0x0
\n
\n.not_s_trap:
\n // If llvm.debugtrap and debugger is not attached.
\n s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP
\n s_cbranch_scc0 .no_skip_debugtrap
\n s_cbranch_scc0 .not_debug_trap
\n
\n s_bitcmp0_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT
\n s_cbranch_scc0 .no_skip_debugtrap
\n s_bitcmp1_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT
\n s_cbranch_scc0 .check_exceptions
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n // Ignore llvm.debugtrap.
\n s_branch .exit_trap
\n.not_debug_trap:
\n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT
\n s_cbranch_scc0 .not_abort_trap
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0
\n s_branch .check_exceptions
\n
\n.no_skip_debugtrap:
\n // Save trap id and halt status in ttmp6.
\n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK)
\n s_min_u32 ttmp2, ttmp2, 0xF
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATE_PRIV_HALT_BFE
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n
\n // Fetch doorbell id for our queue.
\n s_sendmsg_rtn_b32 ttmp3, sendmsg(MSG_RTN_GET_DOORBELL)
\n s_wait_kmcnt 0
\n s_and_b32 ttmp3, ttmp3, DOORBELL_ID_MASK
\n.not_abort_trap:
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n // We need to explititly look for all exceptions we want to report to the
\n // host:
\n // - EXCP_FLAG_PRIV.XNACK_ERROR (&& EXCP_FLAG_PRIV.MEMVIOL)
\n // -> WAVE_MEMORY_VIOLATION
\n // - EXCP_FLAG_PRIV.MEMVIOL (and !EXCP_FLAG_PRIV.XNACK_ERROR)
\n // -> WAVE_APERTURE_VIOLATION
\n // - EXCP_FLAG_PRIV.ILLEGAL_INST -> WAVE_ILLEGAL_INSTRUCTION
\n // - EXCP_FLAG_PRIV.WAVE_START -> WAVE_TRAP
\n // - EXCP_FLAG_PRIV.WAVE_END && TRAP_CTRL.WAVE_END -> WAVE_TRAP
\n // - TRAP_CTRL.TRAP_AFTER_INST -> WAVE_TRAP
\n // - EXCP_FLAG_PRIV.ADDR_WATCH && TRAP_CTL.WATCH -> WAVE_TRAP
\n // - (EXCP_FLAG_USER[ALU] & TRAP_CTRL[ALU]) != 0 -> WAVE_MATH_ERROR
\n.check_exceptions:
\n s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV)
\n s_getreg_b32 ttmp13, hwreg(HW_REG_TRAP_CTRL)
\n
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT
\n s_cbranch_scc0 .not_memory_violation
@@ -358,27 +378,59 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0
\n
\n.not_illegal_instruction:
\n s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_USER, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE)
\n s_cbranch_scc0 .not_math_exception
\n s_getreg_b32 ttmp10, hwreg(HW_REG_TRAP_CTRL, SQ_WAVE_TRAP_CTRL_MATH_EXCP_SHIFT, SQ_WAVE_TRAP_CTRL_MATH_EXCP_SIZE)
\n s_and_b32 ttmp2, ttmp2, ttmp10
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT
\n s_cbranch_scc0 .not_wave_end
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n.not_wave_start:
\n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT
\n s_cbranch_scc0 .not_wave_end
\n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT
\n s_cbranch_scc0 .not_wave_end
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n.not_wave_end:
\n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST
\n s_cbranch_scc0 .not_trap_after_inst
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n.not_trap_after_inst:
\n s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK
\n s_cbranch_scc0 .not_addr_watch
\n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT
\n s_cbranch_scc0 .not_addr_watch
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n
\n.not_addr_watch:
\n s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_USER, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE)
\n s_and_b32 ttmp13, ttmp13, SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK
\n s_and_b32 ttmp2, ttmp2, ttmp13
\n s_cbranch_scc0 .not_math_exception
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0
\n
\n.not_math_exception:
\n s_bfe_u32 ttmp2, ttmp6, TTMP6_SAVED_TRAP_ID_BFE
\n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT
\n s_cbranch_scc0 .not_abort_trap
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0
\n
\n.not_abort_trap:
\n // If no other exception was flagged then report a generic error.
\n s_andn2_b32 ttmp2, ttmp3, DOORBELL_ID_MASK
\n s_cbranch_scc1 .send_interrupt
\n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0
\n s_cmp_eq_u32 ttmp3, 0
\n // This was not a s_trap we are interested in or an exception, return to
\n // the user code.
\n s_cbranch_scc1 .exit_trap
\n
\n.send_interrupt:
\n // Fetch doorbell id for our queue.
\n s_sendmsg_rtn_b32 ttmp2, sendmsg(MSG_RTN_GET_DOORBELL)
\n s_wait_kmcnt 0
\n s_and_b32 ttmp2, ttmp2, DOORBELL_ID_MASK
\n s_or_b32 ttmp3, ttmp2, ttmp3
\n
\n // Save trap id and halt status in ttmp6.
\n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK)
\n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE
\n s_min_u32 ttmp2, ttmp2, 0xF
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATE_PRIV_HALT_BFE
\n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT
\n s_or_b32 ttmp6, ttmp6, ttmp2
\n
\n // m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id
\n s_mov_b32 ttmp2, m0
\n s_mov_b32 m0, ttmp3
@@ -428,7 +480,9 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n // Restore SQ_WAVE_STATUS.
\n s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
\n s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
\n s_setreg_b32 hwreg(HW_REG_STATE_PRIV), ttmp12
\n s_setreg_b32 hwreg(HW_REG_STATE_PRIV, 0, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT), ttmp12
\n s_lshr_b32 ttmp12, ttmp12, (SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1)
\n s_setreg_b32 hwreg(HW_REG_STATE_PRIV, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1, 32 - SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT - 1), ttmp12
\n
\n // Return to original (possibly modified) PC.
\n s_rfe_b64 [ttmp0, ttmp1]
@@ -438,10 +492,9 @@ const char* TrapHandlerCode = RUNTIME_KERNEL(
\n s_branch .parked
\n
\n// Add s_code_end padding so instruction prefetch always has something to read.
\n//.rept (256 - ((. - trap_entry) % 64)) / 4
\n .rept 64
\n s_code_end
\n .endr
\n.rept (256 - ((. - trap_entry) % 64)) / 4
\n s_code_end
\n.endr
\n.endif
\n);
} // namespace amd::pal
rocclr/device/pal/paldevice.cpp
+10
Dosyayı Görüntüle
@@ -866,6 +866,8 @@ Device::~Device() {
extern const char* SchedulerSourceCode;
extern const char* SchedulerSourceCode20;
constexpr int TrapHandlerABIVersion = 10;
extern const char* TrapHandlerCode;
// ================================================================================================
@@ -1146,6 +1148,14 @@ bool Device::initializeHeapResources() {
if (iDev()->Finalize(finalizeInfo) != Pal::Result::Success) {
return false;
}
// Override the _amdgpu_r_debug.r_version field with the trap handler
// version. We can't import the definition of r_debug here as it would
// conflict with ELF related definitions pulled-in by platform/program.hpp.
// The layout of r_debug is a stable ABI, so we are guaranteed the
// r_version field will always be at offset 0.
*reinterpret_cast<int *> (_amdgpu_r_debug_ptr) = TrapHandlerABIVersion;
Pal::HipRuntimeSetup setup {.pRdebug = _amdgpu_r_debug_ptr,
.runtimeState = 1, // Always valid debug state
.ttmpSetupHint = GPU_DEBUG_ENABLE};