From 2a2555dd524dcd44dca9198abf33b270c75f1510 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Fri, 18 Aug 2023 21:33:47 +0000 Subject: [PATCH] Update blit shaders for gfx94x Change-Id: Ic8def71aa0c6ab9a9a758877a65ca6b5625e8f1e [ROCm/ROCR-Runtime commit: 6ce1586def315b4cd16c914cbd468933324b9545] --- .../runtime/blit_shaders/blit_copyAligned.s | 68 ++++++++++++++++--- .../blit_shaders/blit_copyMisaligned.s | 25 +++++-- .../core/runtime/blit_shaders/blit_fill.s | 23 ++++++- 3 files changed, 99 insertions(+), 17 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyAligned.s b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyAligned.s index 20dcd87d53..750366ff6b 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyAligned.s +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyAligned.s @@ -71,6 +71,54 @@ .endif .endm +//sc1 sc0 params are only needed for gfx940/gfx941. On gfx942, we use the compiled code for gfx9 +.macro FLAT_LOAD_DWORD dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_load_dword \dst, \src sc1 sc0 + .else + flat_load_dword \dst, \src + .endif +.endm + +.macro FLAT_STORE_DWORD dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_dword \dst, \src sc1 sc0 + .else + flat_store_dword \dst, \src + .endif +.endm + +.macro FLAT_LOAD_DWORDX4 dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_load_dwordx4 \dst, \src sc1 sc0 + .else + flat_load_dwordx4 \dst, \src + .endif +.endm + +.macro FLAT_STORE_DWORDX4 dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_dwordx4 \dst, \src sc1 sc0 + .else + flat_store_dwordx4 \dst, \src + .endif +.endm + +.macro FLAT_LOAD_UBYTE dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_load_ubyte \dst, \src sc1 sc0 + .else + flat_load_ubyte \dst, \src + .endif +.endm + +.macro FLAT_STORE_BYTE dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_byte \dst, \src sc1 sc0 + .else + flat_store_byte \dst, \src + .endif +.endm .p2align 8 @@ -118,13 +166,13 @@ compute_pgm_rsrc1_vgprs = CopyAlignedRsrc1VGPRs s_and_b64 exec, exec, vcc - flat_load_ubyte v1, v[2:3] + FLAT_LOAD_UBYTE v1, v[2:3] s_waitcnt vmcnt(0) V_ADD_CO_U32 v2, v2, s24 V_ADD_CO_CI_U32 v3, v3, 0x0 - flat_store_byte v[4:5], v1 + FLAT_STORE_BYTE v[4:5], v1 V_ADD_CO_U32 v4, v4, s24 V_ADD_CO_CI_U32 v5, v5, 0x0 @@ -162,9 +210,9 @@ compute_pgm_rsrc1_vgprs = CopyAlignedRsrc1VGPRs .macro mCopyAlignedPhase2Load iter iter_end .if kCopyAlignedVecWidth == 4 - flat_load_dwordx4 v[8 + (\iter * 4):8 + (\iter * 4) + 3], v[2:3] + FLAT_LOAD_DWORDX4 v[8 + (\iter * 4):8 + (\iter * 4) + 3], v[2:3] .else - flat_load_dword v[8 + \iter], v[2:3] + FLAT_LOAD_DWORD v[8 + \iter], v[2:3] .endif V_ADD_CO_U32 v2, v2, s25 @@ -181,9 +229,9 @@ mCopyAlignedPhase2Load 0, (kCopyAlignedUnroll - 1) .macro mCopyAlignedPhase2Store iter iter_end .if kCopyAlignedVecWidth == 4 - flat_store_dwordx4 v[4:5], v[8 + (\iter * 4):8 + (\iter * 4) + 3] + FLAT_STORE_DWORDX4 v[4:5], v[8 + (\iter * 4):8 + (\iter * 4) + 3] .else - flat_store_dword v[4:5], v[8 + \iter] + FLAT_STORE_DWORD v[4:5], v[8 + \iter] .endif V_ADD_CO_U32 v4, v4, s25 @@ -219,13 +267,13 @@ mCopyAlignedPhase2Store 0, (kCopyAlignedUnroll - 1) s_and_b64 exec, exec, vcc - flat_load_dword v1, v[2:3] + FLAT_LOAD_DWORD v1, v[2:3] V_ADD_CO_U32 v2, v2, s25 V_ADD_CO_CI_U32 v3, v3, 0x0 s_waitcnt vmcnt(0) - flat_store_dword v[4:5], v1 + FLAT_STORE_DWORD v[4:5], v1 V_ADD_CO_U32 v4, v4, s25 V_ADD_CO_CI_U32 v5, v5, 0x0 @@ -247,10 +295,10 @@ mCopyAlignedPhase2Store 0, (kCopyAlignedUnroll - 1) s_cbranch_vccz L_COPY_ALIGNED_PHASE_4_DONE s_and_b64 exec, exec, vcc - flat_load_ubyte v1, v[2:3] + FLAT_LOAD_UBYTE v1, v[2:3] s_waitcnt vmcnt(0) - flat_store_byte v[4:5], v1 + FLAT_STORE_BYTE v[4:5], v1 L_COPY_ALIGNED_PHASE_4_DONE: s_endpgm diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyMisaligned.s b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyMisaligned.s index dd0b15d10b..a63b2ace55 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyMisaligned.s +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_copyMisaligned.s @@ -71,6 +71,23 @@ .endif .endm +//sc1 sc0 params are only needed for gfx940/gfx941. On gfx942, we use the compiled code for gfx9 +.macro FLAT_LOAD_UBYTE dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_load_ubyte \dst, \src sc1 sc0 + .else + flat_load_ubyte \dst, \src + .endif +.endm + +.macro FLAT_STORE_BYTE dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_byte \dst, \src sc1 sc0 + .else + flat_store_byte \dst, \src + .endif +.endm + .set kCopyMisalignedUnroll, 4 .set kCopyMisalignedNumSGPRs, 17 .set kCopyMisalignedNumVGPRs, 6 + kCopyMisalignedUnroll @@ -118,7 +135,7 @@ CopyMisaligned: .macro mCopyMisalignedPhase1Load iter iter_end - flat_load_ubyte v[6 + \iter], v[2:3] + FLAT_LOAD_UBYTE v[6 + \iter], v[2:3] V_ADD_CO_U32 v2, v2, s16 V_ADD_CO_CI_U32 v3, v3, 0x0 @@ -132,7 +149,7 @@ CopyMisaligned: s_waitcnt vmcnt(0) .macro mCopyMisalignedPhase1Store iter iter_end - flat_store_byte v[4:5], v[6 + \iter] + FLAT_STORE_BYTE v[4:5], v[6 + \iter] V_ADD_CO_U32 v4, v4, s16 V_ADD_CO_CI_U32 v5, v5, 0x0 @@ -162,12 +179,12 @@ CopyMisaligned: s_and_b64 exec, exec, vcc - flat_load_ubyte v1, v[2:3] + FLAT_LOAD_UBYTE v1, v[2:3] V_ADD_CO_U32 v2, v2, s16 V_ADD_CO_CI_U32 v3, v3, 0x0 s_waitcnt vmcnt(0) - flat_store_byte v[4:5], v1 + FLAT_STORE_BYTE v[4:5], v1 V_ADD_CO_U32 v4, v4, s16 V_ADD_CO_CI_U32 v5, v5, 0x0 diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_fill.s b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_fill.s index 859de116bc..bdc4fbcc56 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_fill.s +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/blit_shaders/blit_fill.s @@ -70,6 +70,23 @@ .endif .endm +//sc1 sc0 params are only needed for gfx940/gfx941. On gfx942, we use the compiled code for gfx9 +.macro FLAT_STORE_DWORD dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_dword \dst, \src sc1 sc0 + .else + flat_store_dword \dst, \src + .endif +.endm + +.macro FLAT_STORE_DWORDX4 dst, src + .if (.amdgcn.gfx_generation_number == 9 && .amdgcn.gfx_generation_minor == 4) + flat_store_dwordx4 \dst, \src sc1 sc0 + .else + flat_store_dwordx4 \dst, \src + .endif +.endm + .set kFillVecWidth, 4 .set kFillUnroll, 1 @@ -137,9 +154,9 @@ Fill: .macro mFillPhase1 iter iter_end .if kFillVecWidth == 4 - flat_store_dwordx4 v[2:3], v[4:7] + FLAT_STORE_DWORDX4 v[2:3], v[4:7] .else - flat_store_dword v[2:3], v4 + FLAT_STORE_DWORD v[2:3], v4 .endif V_ADD_CO_U32 v2, v2, s12 @@ -170,7 +187,7 @@ mFillPhase1 0, kFillUnroll - 1 s_and_b64 exec, exec, vcc - flat_store_dword v[2:3], v4 + FLAT_STORE_DWORD v[2:3], v4 V_ADD_CO_U32 v2, v2, s12 V_ADD_CO_CI_U32 v3, v3, 0x0