diff --git a/rocrtst/suites/test_common/CMakeLists.txt b/rocrtst/suites/test_common/CMakeLists.txt index 51842eeecd..919ebf3ca8 100755 --- a/rocrtst/suites/test_common/CMakeLists.txt +++ b/rocrtst/suites/test_common/CMakeLists.txt @@ -82,7 +82,7 @@ if ( PROC_RESULT EQUAL "0" AND NOT EVAL_RESULT STREQUAL "" ) endif() set (CPACK_RPM_PACKAGE_REQUIRES "rocm-core") -set(DEFAULT_TARGETS "gfx700;gfx701;gfx702;gfx801;gfx802;gfx803;gfx805;gfx810;gfx900;gfx902;gfx904;gfx906;gfx908;gfx909;gfx90a;gfx90c;gfx940;gfx1010;gfx1011;gfx1012;gfx1013;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103") +set(DEFAULT_TARGETS "gfx700;gfx701;gfx702;gfx801;gfx802;gfx803;gfx805;gfx810;gfx900;gfx902;gfx904;gfx906;gfx908;gfx909;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1010;gfx1011;gfx1012;gfx1013;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103") # # Currently support for Windows platform is not present diff --git a/runtime/hsa-runtime/core/inc/amd_blit_shaders.h b/runtime/hsa-runtime/core/inc/amd_blit_shaders.h index 583d0bdd08..12c52cc4b3 100644 --- a/runtime/hsa-runtime/core/inc/amd_blit_shaders.h +++ b/runtime/hsa-runtime/core/inc/amd_blit_shaders.h @@ -156,6 +156,62 @@ static const unsigned int kCodeFill8[] = { 0x00001902, 0xD11C6A03, 0x01A90103, 0xBF82FFF5, 0xBF810000, }; +static const unsigned int kCodeCopyAligned940[] = { + 0xc00a0100, 0x00000000, 0xc00a0200, 0x00000010, 0xc00a0300, 0x00000020, + 0xc00a0400, 0x00000030, 0xc00a0500, 0x00000040, 0xc0020600, 0x00000050, + 0xbf8cc07f, 0x8e028602, 0x32000002, 0x7e060205, 0xd1196a02, 0x00000900, + 0xd11c6a03, 0x01a90103, 0x7e0a0207, 0xd1196a04, 0x00000d00, 0xd11c6a05, + 0x01a90105, 0xd0e9006a, 0x00001102, 0xbf86000f, 0x86fe6a7e, 0xde410000, + 0x017f0002, 0xbf8c0f70, 0xd1196a02, 0x00003102, 0xd11c6a03, 0x01a90103, + 0xde610000, 0x007f0104, 0xd1196a04, 0x00003104, 0xd11c6a05, 0x01a90105, + 0xbf82ffee, 0xbefe01c1, 0x8e198418, 0x24020084, 0x7e060209, 0xd1196a02, + 0x00001101, 0xd11c6a03, 0x01a90103, 0x7e0a020b, 0xd1196a04, 0x00001501, + 0xd11c6a05, 0x01a90105, 0xd0e9006a, 0x00001902, 0xbf86000e, 0xde5d0000, + 0x087f0002, 0xd1196a02, 0x00003302, 0xd11c6a03, 0x01a90103, 0xbf8c0f70, + 0xde7d0000, 0x007f0804, 0xd1196a04, 0x00003304, 0xd11c6a05, 0x01a90105, + 0xbf82ffef, 0x8e198218, 0x24020082, 0x7e06020d, 0xd1196a02, 0x00001901, + 0xd11c6a03, 0x01a90103, 0x7e0a020f, 0xd1196a04, 0x00001d01, 0xd11c6a05, + 0x01a90105, 0xd0e9006a, 0x00002102, 0xbf86000f, 0x86fe6a7e, 0xde510000, + 0x017f0002, 0xd1196a02, 0x00003302, 0xd11c6a03, 0x01a90103, 0xbf8c0f70, + 0xde710000, 0x007f0104, 0xd1196a04, 0x00003304, 0xd11c6a05, 0x01a90105, + 0xbf82ffee, 0xbefe01c1, 0x7e060211, 0xd1196a02, 0x00002100, 0xd11c6a03, + 0x01a90103, 0x7e0a0213, 0xd1196a04, 0x00002500, 0xd11c6a05, 0x01a90105, + 0xd0e9006a, 0x00002902, 0xbf860006, 0x86fe6a7e, 0xde410000, 0x017f0002, + 0xbf8c0f70, 0xde610000, 0x007f0104, 0xbf810000, +}; + +static const unsigned int kCodeCopyMisaligned940[] = { + 0xc00a0100, 0x00000000, 0xc00a0200, 0x00000010, 0xc00a0300, 0x00000020, + 0xc0020400, 0x00000030, 0xbf8cc07f, 0x8e028602, 0x32000002, 0x7e060205, + 0xd1196a02, 0x00000900, 0xd11c6a03, 0x01a90103, 0x7e0a0207, 0xd1196a04, + 0x00000d00, 0xd11c6a05, 0x01a90105, 0xd0e9006a, 0x00001102, 0xbf860032, + 0xde410000, 0x067f0002, 0xd1196a02, 0x00002102, 0xd11c6a03, 0x01a90103, + 0xde410000, 0x077f0002, 0xd1196a02, 0x00002102, 0xd11c6a03, 0x01a90103, + 0xde410000, 0x087f0002, 0xd1196a02, 0x00002102, 0xd11c6a03, 0x01a90103, + 0xde410000, 0x097f0002, 0xd1196a02, 0x00002102, 0xd11c6a03, 0x01a90103, + 0xbf8c0f70, 0xde610000, 0x007f0604, 0xd1196a04, 0x00002104, 0xd11c6a05, + 0x01a90105, 0xde610000, 0x007f0704, 0xd1196a04, 0x00002104, 0xd11c6a05, + 0x01a90105, 0xde610000, 0x007f0804, 0xd1196a04, 0x00002104, 0xd11c6a05, + 0x01a90105, 0xde610000, 0x007f0904, 0xd1196a04, 0x00002104, 0xd11c6a05, + 0x01a90105, 0xbf82ffcb, 0x7e060209, 0xd1196a02, 0x00001100, 0xd11c6a03, + 0x01a90103, 0x7e0a020b, 0xd1196a04, 0x00001500, 0xd11c6a05, 0x01a90105, + 0xd0e9006a, 0x00001902, 0xbf86000f, 0x86fe6a7e, 0xde410000, 0x017f0002, + 0xd1196a02, 0x00002102, 0xd11c6a03, 0x01a90103, 0xbf8c0f70, 0xde610000, + 0x007f0104, 0xd1196a04, 0x00002104, 0xd11c6a05, 0x01a90105, 0xbf82ffee, + 0xbf810000, 0x00000000, +}; + +static const unsigned int kCodeFill940[] = { + 0xc00a0100, 0x00000000, 0xc00a0200, 0x00000010, 0xbf8cc07f, 0x8e028602, + 0x32000002, 0x7e08020a, 0x7e0a020a, 0x7e0c020a, 0x7e0e020a, 0x8e0c840b, + 0x24020084, 0x7e060205, 0xd1196a02, 0x00000901, 0xd11c6a03, 0x01a90103, + 0xd0e9006a, 0x00000d02, 0xbf860007, 0xde7d0000, 0x007f0402, 0xd1196a02, + 0x00001902, 0xd11c6a03, 0x01a90103, 0xbf82fff6, 0x8e0c820b, 0x24020082, + 0x7e060207, 0xd1196a02, 0x00000d01, 0xd11c6a03, 0x01a90103, 0xd0e9006a, + 0x00001102, 0xbf860008, 0x86fe6a7e, 0xde710000, 0x007f0402, 0xd1196a02, + 0x00001902, 0xd11c6a03, 0x01a90103, 0xbf82fff5, 0xbf810000, 0x00000000, +}; + static const unsigned int kCodeCopyAligned10[] = { 0xF4080100, 0xFA000000, 0xF4080200, 0xFA000010, 0xF4080300, 0xFA000020, 0xF4080400, 0xFA000030, 0xF4080500, 0xFA000040, 0xF4000600, 0xFA000050, diff --git a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp index 681b310a19..661e3be4db 100644 --- a/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp @@ -491,6 +491,388 @@ static std::string kBlitKernelSource(R"( L_FILL_PHASE_2_DONE: s_endpgm end + +shader CopyAligned_940 + type(CS) + user_sgpr_count(2) + sgpr_count(32) + vgpr_count(8 + (kCopyAlignedUnroll * kCopyAlignedVecWidth)) + + // Retrieve kernel arguments. + s_load_dwordx4 s[4:7], s[0:1], s_load_dword_offset(0x0) + s_load_dwordx4 s[8:11], s[0:1], s_load_dword_offset(0x10) + s_load_dwordx4 s[12:15], s[0:1], s_load_dword_offset(0x20) + s_load_dwordx4 s[16:19], s[0:1], s_load_dword_offset(0x30) + s_load_dwordx4 s[20:23], s[0:1], s_load_dword_offset(0x40) + s_load_dword s24, s[0:1], s_load_dword_offset(0x50) + s_waitcnt lgkmcnt(0) + + // Compute workitem id. + s_lshl_b32 s2, s2, 0x6 + v_add_co_u32 v0, vcc, s2, v0 + + // ===================================================== + // Phase 1: Byte copy up to 0x100 destination alignment. + // ===================================================== + + // Compute phase source address. + v_mov_b32 v3, s5 + v_add_co_u32 v2, vcc, v0, s4 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s7 + v_add_co_u32 v4, vcc, v0, s6 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + +L_COPY_ALIGNED_PHASE_1_LOOP: + // Mask off lanes (or branch out) after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[8:9] + s_cbranch_vccz L_COPY_ALIGNED_PHASE_1_DONE + s_and_b64 exec, exec, vcc + + // Load from/advance the source address. + flat_load_ubyte v1, v[2:3] sc0:1 sc1:1 + s_waitcnt vmcnt(0) + v_add_co_u32 v2, vcc, v2, s24 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Write to/advance the destination address. + flat_store_byte v[4:5], v1 sc0:1 sc1:1 + v_add_co_u32 v4, vcc, v4, s24 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + + // Repeat until branched out. + s_branch L_COPY_ALIGNED_PHASE_1_LOOP + +L_COPY_ALIGNED_PHASE_1_DONE: + // Restore EXEC mask for all lanes. + s_mov_b64 exec, 0xFFFFFFFFFFFFFFFF + + // ======================================================== + // Phase 2: Unrolled dword[x4] copy up to last whole block. + // ======================================================== + + // Compute unrolled dword[x4] stride across all threads. + if kCopyAlignedVecWidth == 4 + s_lshl_b32 s25, s24, 0x4 + else + s_lshl_b32 s25, s24, 0x2 + end + + // Compute phase source address. + if kCopyAlignedVecWidth == 4 + v_lshlrev_b32 v1, 0x4, v0 + else + v_lshlrev_b32 v1, 0x2, v0 + end + + v_mov_b32 v3, s9 + v_add_co_u32 v2, vcc, v1, s8 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s11 + v_add_co_u32 v4, vcc, v1, s10 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + +L_COPY_ALIGNED_PHASE_2_LOOP: + // Branch out after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[12:13] + s_cbranch_vccz L_COPY_ALIGNED_PHASE_2_DONE + + // Load from/advance the source address. + for var i = 0; i < kCopyAlignedUnroll; i ++ + if kCopyAlignedVecWidth == 4 + flat_load_dwordx4 v[8 + (i * 4)], v[2:3] sc0:1 sc1:1 + else + flat_load_dword v[8 + i], v[2:3] sc0:1 sc1:1 + end + + v_add_co_u32 v2, vcc, v2, s25 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + end + + // Write to/advance the destination address. + s_waitcnt vmcnt(0) + + for var i = 0; i < kCopyAlignedUnroll; i ++ + if kCopyAlignedVecWidth == 4 + flat_store_dwordx4 v[4:5], v[8 + (i * 4)] sc0:1 sc1:1 + else + flat_store_dword v[4:5], v[8 + i] sc0:1 sc1:1 + end + + v_add_co_u32 v4, vcc, v4, s25 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + end + + // Repeat until branched out. + s_branch L_COPY_ALIGNED_PHASE_2_LOOP + +L_COPY_ALIGNED_PHASE_2_DONE: + + // =========================================== + // Phase 3: Dword copy up to last whole dword. + // =========================================== + + // Compute dword stride across all threads. + s_lshl_b32 s25, s24, 0x2 + + // Compute phase source address. + v_lshlrev_b32 v1, 0x2, v0 + v_mov_b32 v3, s13 + v_add_co_u32 v2, vcc, v1, s12 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s15 + v_add_co_u32 v4, vcc, v1, s14 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + +L_COPY_ALIGNED_PHASE_3_LOOP: + // Mask off lanes (or branch out) after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[16:17] + s_cbranch_vccz L_COPY_ALIGNED_PHASE_3_DONE + s_and_b64 exec, exec, vcc + + // Load from/advance the source address. + flat_load_dword v1, v[2:3] sc0:1 sc1:1 + v_add_co_u32 v2, vcc, v2, s25 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + s_waitcnt vmcnt(0) + + // Write to/advance the destination address. + flat_store_dword v[4:5], v1 sc0:1 sc1:1 + v_add_co_u32 v4, vcc, v4, s25 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + + // Repeat until branched out. + s_branch L_COPY_ALIGNED_PHASE_3_LOOP + +L_COPY_ALIGNED_PHASE_3_DONE: + // Restore EXEC mask for all lanes. + s_mov_b64 exec, 0xFFFFFFFFFFFFFFFF + + // ============================= + // Phase 4: Byte copy up to end. + // ============================= + + // Compute phase source address. + v_mov_b32 v3, s17 + v_add_co_u32 v2, vcc, v0, s16 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s19 + v_add_co_u32 v4, vcc, v0, s18 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + + // Mask off lanes (or branch out) after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[20:21] + s_cbranch_vccz L_COPY_ALIGNED_PHASE_4_DONE + s_and_b64 exec, exec, vcc + + // Load from the source address. + flat_load_ubyte v1, v[2:3] sc0:1 sc1:1 + s_waitcnt vmcnt(0) + + // Write to the destination address. + flat_store_byte v[4:5], v1 sc0:1 sc1:1 + +L_COPY_ALIGNED_PHASE_4_DONE: + s_endpgm +end + +shader CopyMisaligned_940 + type(CS) + user_sgpr_count(2) + sgpr_count(23) + vgpr_count(6 + kCopyMisalignedUnroll) + + // Retrieve kernel arguments. + s_load_dwordx4 s[4:7], s[0:1], s_load_dword_offset(0x0) + s_load_dwordx4 s[8:11], s[0:1], s_load_dword_offset(0x10) + s_load_dwordx4 s[12:15], s[0:1], s_load_dword_offset(0x20) + s_load_dword s16, s[0:1], s_load_dword_offset(0x30) + s_waitcnt lgkmcnt(0) + + // Compute workitem id. + s_lshl_b32 s2, s2, 0x6 + v_add_co_u32 v0, vcc, s2, v0 + + // =================================================== + // Phase 1: Unrolled byte copy up to last whole block. + // =================================================== + + // Compute phase source address. + v_mov_b32 v3, s5 + v_add_co_u32 v2, vcc, v0, s4 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s7 + v_add_co_u32 v4, vcc, v0, s6 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + +L_COPY_MISALIGNED_PHASE_1_LOOP: + // Branch out after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[8:9] + s_cbranch_vccz L_COPY_MISALIGNED_PHASE_1_DONE + + // Load from/advance the source address. + for var i = 0; i < kCopyMisalignedUnroll; i ++ + flat_load_ubyte v[6 + i], v[2:3] sc0:1 sc1:1 + v_add_co_u32 v2, vcc, v2, s16 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + end + + // Write to/advance the destination address. + s_waitcnt vmcnt(0) + + for var i = 0; i < kCopyMisalignedUnroll; i ++ + flat_store_byte v[4:5], v[6 + i] sc0:1 sc1:1 + v_add_co_u32 v4, vcc, v4, s16 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + end + + // Repeat until branched out. + s_branch L_COPY_MISALIGNED_PHASE_1_LOOP + +L_COPY_MISALIGNED_PHASE_1_DONE: + + // ============================= + // Phase 2: Byte copy up to end. + // ============================= + + // Compute phase source address. + v_mov_b32 v3, s9 + v_add_co_u32 v2, vcc, v0, s8 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Compute phase destination address. + v_mov_b32 v5, s11 + v_add_co_u32 v4, vcc, v0, s10 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + +L_COPY_MISALIGNED_PHASE_2_LOOP: + // Mask off lanes (or branch out) after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[12:13] + s_cbranch_vccz L_COPY_MISALIGNED_PHASE_2_DONE + s_and_b64 exec, exec, vcc + + // Load from/advance the source address. + flat_load_ubyte v1, v[2:3] sc0:1 sc1:1 + v_add_co_u32 v2, vcc, v2, s16 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + s_waitcnt vmcnt(0) + + // Write to/advance the destination address. + flat_store_byte v[4:5], v1 sc0:1 sc1:1 + v_add_co_u32 v4, vcc, v4, s16 + v_addc_co_u32 v5, vcc, v5, 0x0, vcc + + // Repeat until branched out. + s_branch L_COPY_MISALIGNED_PHASE_2_LOOP + +L_COPY_MISALIGNED_PHASE_2_DONE: + s_endpgm +end + +shader Fill_940 + type(CS) + user_sgpr_count(2) + sgpr_count(19) + vgpr_count(8) + + // Retrieve kernel arguments. + s_load_dwordx4 s[4:7], s[0:1], s_load_dword_offset(0x0) + s_load_dwordx4 s[8:11], s[0:1], s_load_dword_offset(0x10) + s_waitcnt lgkmcnt(0) + + // Compute workitem id. + s_lshl_b32 s2, s2, 0x6 + v_add_co_u32 v0, vcc, s2, v0 + + // Copy fill pattern into VGPRs. + for var i = 0; i < kFillVecWidth; i ++ + v_mov_b32 v[4 + i], s10 + end + + // ======================================================== + // Phase 1: Unrolled dword[x4] fill up to last whole block. + // ======================================================== + + // Compute unrolled dword[x4] stride across all threads. + if kFillVecWidth == 4 + s_lshl_b32 s12, s11, 0x4 + else + s_lshl_b32 s12, s11, 0x2 + end + + // Compute phase destination address. + if kFillVecWidth == 4 + v_lshlrev_b32 v1, 0x4, v0 + else + v_lshlrev_b32 v1, 0x2, v0 + end + + v_mov_b32 v3, s5 + v_add_co_u32 v2, vcc, v1, s4 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + +L_FILL_PHASE_1_LOOP: + // Branch out after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[6:7] + s_cbranch_vccz L_FILL_PHASE_1_DONE + + // Write to/advance the destination address. + for var i = 0; i < kFillUnroll; i ++ + if kFillVecWidth == 4 + flat_store_dwordx4 v[2:3], v[4:7] sc0:1 sc1:1 + else + flat_store_dword v[2:3], v4 sc0:1 sc1:1 + end + + v_add_co_u32 v2, vcc, v2, s12 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + end + + // Repeat until branched out. + s_branch L_FILL_PHASE_1_LOOP + +L_FILL_PHASE_1_DONE: + + // ============================== + // Phase 2: Dword fill up to end. + // ============================== + + // Compute dword stride across all threads. + s_lshl_b32 s12, s11, 0x2 + + // Compute phase destination address. + v_lshlrev_b32 v1, 0x2, v0 + v_mov_b32 v3, s7 + v_add_co_u32 v2, vcc, v1, s6 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + +L_FILL_PHASE_2_LOOP: + // Mask off lanes (or branch out) after phase end. + v_cmp_lt_u64 vcc, v[2:3], s[8:9] + s_cbranch_vccz L_FILL_PHASE_2_DONE + s_and_b64 exec, exec, vcc + + // Write to/advance the destination address. + flat_store_dword v[2:3], v4 sc0:1 sc1:1 + v_add_co_u32 v2, vcc, v2, s12 + v_addc_co_u32 v3, vcc, v3, 0x0, vcc + + // Repeat until branched out. + s_branch L_FILL_PHASE_2_LOOP + +L_FILL_PHASE_2_DONE: + s_endpgm +end )"); // Search kernel source for variable definition and return value. diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 59d21bb2e6..78c114a929 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -254,6 +254,7 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar ASICShader compute_9; ASICShader compute_90a; ASICShader compute_940; + ASICShader compute_942; ASICShader compute_1010; ASICShader compute_10; ASICShader compute_11; @@ -262,58 +263,63 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar std::map compiled_shaders = { {"TrapHandler", { - {NULL, 0, 0, 0}, - {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, - {kCodeTrapHandler9, sizeof(kCodeTrapHandler9), 2, 4}, - {kCodeTrapHandler90a, sizeof(kCodeTrapHandler90a), 2, 4}, - {NULL, 0, 0, 0}, - {kCodeTrapHandler1010, sizeof(kCodeTrapHandler1010), 2, 4}, - {kCodeTrapHandler10, sizeof(kCodeTrapHandler10), 2, 4}, - {NULL, 0, 0, 0}, + {NULL, 0, 0, 0}, // gfx7 + {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, // gfx8 + {kCodeTrapHandler9, sizeof(kCodeTrapHandler9), 2, 4}, // gfx9 + {kCodeTrapHandler90a, sizeof(kCodeTrapHandler90a), 2, 4}, // gfx90a + {NULL, 0, 0, 0}, // gfx940 + {NULL, 0, 0, 0}, // gfx942 + {kCodeTrapHandler1010, sizeof(kCodeTrapHandler1010), 2, 4}, // gfx1010 + {kCodeTrapHandler10, sizeof(kCodeTrapHandler10), 2, 4}, // gfx10 + {NULL, 0, 0, 0}, // gfx11 }}, {"TrapHandlerKfdExceptions", { - {NULL, 0, 0, 0}, - {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, - {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, - {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, - {kCodeTrapHandlerV2_940, sizeof(kCodeTrapHandlerV2_940), 2, 4}, - {kCodeTrapHandlerV2_1010, sizeof(kCodeTrapHandlerV2_1010), 2, 4}, - {kCodeTrapHandlerV2_10, sizeof(kCodeTrapHandlerV2_10), 2, 4}, - {kCodeTrapHandlerV2_11, sizeof(kCodeTrapHandlerV2_11), 2, 4}, + {NULL, 0, 0, 0}, // gfx7 + {kCodeTrapHandler8, sizeof(kCodeTrapHandler8), 2, 4}, // gfx8 + {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, // gfx9 + {kCodeTrapHandlerV2_9, sizeof(kCodeTrapHandlerV2_9), 2, 4}, // gfx90a + {kCodeTrapHandlerV2_940, sizeof(kCodeTrapHandlerV2_940), 2, 4}, // gfx940 + {kCodeTrapHandlerV2_940, sizeof(kCodeTrapHandlerV2_940), 2, 4}, // gfx942 + {kCodeTrapHandlerV2_1010, sizeof(kCodeTrapHandlerV2_1010), 2, 4},// gfx1010 + {kCodeTrapHandlerV2_10, sizeof(kCodeTrapHandlerV2_10), 2, 4}, // gfx10 + {kCodeTrapHandlerV2_11, sizeof(kCodeTrapHandlerV2_11), 2, 4}, // gfx11 }}, {"CopyAligned", { - {kCodeCopyAligned7, sizeof(kCodeCopyAligned7), 32, 12}, - {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, - {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, - {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, - {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, - {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, - {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, - {kCodeCopyAligned11, sizeof(kCodeCopyAligned11), 32, 12}, + {kCodeCopyAligned7, sizeof(kCodeCopyAligned7), 32, 12}, // gfx7 + {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, // gfx8 + {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, // gfx9 + {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, // gfx90a + {kCodeCopyAligned940, sizeof(kCodeCopyAligned940), 32, 12}, // gfx940 + {kCodeCopyAligned8, sizeof(kCodeCopyAligned8), 32, 12}, // gfx942 + {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, // gfx1010 + {kCodeCopyAligned10, sizeof(kCodeCopyAligned10), 32, 12}, // gfx10 + {kCodeCopyAligned11, sizeof(kCodeCopyAligned11), 32, 12}, // gfx11 }}, {"CopyMisaligned", { - {kCodeCopyMisaligned7, sizeof(kCodeCopyMisaligned7), 23, 10}, - {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, - {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, - {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, - {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, - {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, - {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, - {kCodeCopyMisaligned11, sizeof(kCodeCopyMisaligned11), 23, 10}, + {kCodeCopyMisaligned7, sizeof(kCodeCopyMisaligned7), 23, 10}, // gfx7 + {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, // gfx8 + {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, // gfx9 + {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, // gfx90a + {kCodeCopyMisaligned940, sizeof(kCodeCopyMisaligned940), 23, 10},// gfx940 + {kCodeCopyMisaligned8, sizeof(kCodeCopyMisaligned8), 23, 10}, // gfx942 + {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, // gfx1010 + {kCodeCopyMisaligned10, sizeof(kCodeCopyMisaligned10), 23, 10}, // gfx10 + {kCodeCopyMisaligned11, sizeof(kCodeCopyMisaligned11), 23, 10}, // gfx11 }}, {"Fill", { - {kCodeFill7, sizeof(kCodeFill7), 19, 8}, - {kCodeFill8, sizeof(kCodeFill8), 19, 8}, - {kCodeFill8, sizeof(kCodeFill8), 19, 8}, - {kCodeFill8, sizeof(kCodeFill8), 19, 8}, - {kCodeFill8, sizeof(kCodeFill8), 19, 8}, - {kCodeFill10, sizeof(kCodeFill10), 19, 8}, - {kCodeFill10, sizeof(kCodeFill10), 19, 8}, - {kCodeFill11, sizeof(kCodeFill11), 19, 8}, + {kCodeFill7, sizeof(kCodeFill7), 19, 8}, // gfx7 + {kCodeFill8, sizeof(kCodeFill8), 19, 8}, // gfx8 + {kCodeFill8, sizeof(kCodeFill8), 19, 8}, // gfx9 + {kCodeFill8, sizeof(kCodeFill8), 19, 8}, // gfx90a + {kCodeFill940, sizeof(kCodeFill940), 19, 8}, // gfx940 + {kCodeFill8, sizeof(kCodeFill8), 19, 8}, // gfx942 + {kCodeFill10, sizeof(kCodeFill10), 19, 8}, // gfx1010 + {kCodeFill10, sizeof(kCodeFill10), 19, 8}, // gfx10 + {kCodeFill11, sizeof(kCodeFill11), 19, 8}, // gfx11 }}}; auto compiled_shader_it = compiled_shaders.find(func_name); @@ -330,12 +336,22 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar asic_shader = &compiled_shader_it->second.compute_8; break; case 9: - if((isa_->GetMinorVersion() == 0) && (isa_->GetStepping() == 10)) + if((isa_->GetMinorVersion() == 0) && (isa_->GetStepping() == 10)) { asic_shader = &compiled_shader_it->second.compute_90a; - else if((isa_->GetMinorVersion() == 4) && (isa_->GetStepping() == 0)) - asic_shader = &compiled_shader_it->second.compute_940; - else + } else if(isa_->GetMinorVersion() == 4) { + switch(isa_->GetStepping()) { + case 0: + case 1: + asic_shader = &compiled_shader_it->second.compute_940; + break; + case 2: + default: + asic_shader = &compiled_shader_it->second.compute_942; + break; + } + } else { asic_shader = &compiled_shader_it->second.compute_9; + } break; case 10: if(isa_->GetMinorVersion() == 1) @@ -386,9 +402,10 @@ void GpuAgent::AssembleShader(const char* func_name, AssembleTarget assemble_tar AMD_HSA_BITS_SET(header->compute_pgm_rsrc2, AMD_COMPUTE_PGM_RSRC_TWO_ENABLE_SGPR_WORKGROUP_ID_X, 1); + // gfx90a, gfx940, gfx941, gfx942 if ((isa_->GetMajorVersion() == 9) && (((isa_->GetMinorVersion() == 0) && (isa_->GetStepping() == 10)) || - ((isa_->GetMinorVersion() == 4) && (isa_->GetStepping() == 0)))) { + (isa_->GetMinorVersion() == 4))) { // Program COMPUTE_PGM_RSRC3.ACCUM_OFFSET for 0 ACC VGPRs on gfx90a. // FIXME: Assemble code objects from source at build time int gran_accvgprs = ((gran_vgprs + 1) * 8) / 4 - 1; diff --git a/runtime/hsa-runtime/core/runtime/isa.cpp b/runtime/hsa-runtime/core/runtime/isa.cpp index 0a2fa5742a..3674848a77 100755 --- a/runtime/hsa-runtime/core/runtime/isa.cpp +++ b/runtime/hsa-runtime/core/runtime/isa.cpp @@ -306,6 +306,24 @@ constexpr size_t hsa_name_size = 63; ISAREG_ENTRY_GEN("gfx940:sramecc-:xnack+", 9, 4, 0, disabled, enabled, 64) ISAREG_ENTRY_GEN("gfx940:sramecc+:xnack-", 9, 4, 0, enabled, disabled, 64) ISAREG_ENTRY_GEN("gfx940:sramecc+:xnack+", 9, 4, 0, enabled, enabled, 64) + ISAREG_ENTRY_GEN("gfx941", 9, 4, 1, any, any, 64) + ISAREG_ENTRY_GEN("gfx941:xnack-", 9, 4, 1, any, disabled, 64) + ISAREG_ENTRY_GEN("gfx941:xnack+", 9, 4, 1, any, enabled, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc-", 9, 4, 1, disabled, any, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc+", 9, 4, 1, enabled, any, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc-:xnack-", 9, 4, 1, disabled, disabled, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc-:xnack+", 9, 4, 1, disabled, enabled, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc+:xnack-", 9, 4, 1, enabled, disabled, 64) + ISAREG_ENTRY_GEN("gfx941:sramecc+:xnack+", 9, 4, 1, enabled, enabled, 64) + ISAREG_ENTRY_GEN("gfx942", 9, 4, 2, any, any, 64) + ISAREG_ENTRY_GEN("gfx942:xnack-", 9, 4, 2, any, disabled, 64) + ISAREG_ENTRY_GEN("gfx942:xnack+", 9, 4, 2, any, enabled, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc-", 9, 4, 2, disabled, any, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc+", 9, 4, 2, enabled, any, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc-:xnack-", 9, 4, 2, disabled, disabled, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc-:xnack+", 9, 4, 2, disabled, enabled, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc+:xnack-", 9, 4, 2, enabled, disabled, 64) + ISAREG_ENTRY_GEN("gfx942:sramecc+:xnack+", 9, 4, 2, enabled, enabled, 64) ISAREG_ENTRY_GEN("gfx1010", 10, 1, 0, unsupported, any, 32) ISAREG_ENTRY_GEN("gfx1010:xnack-", 10, 1, 0, unsupported, disabled, 32) ISAREG_ENTRY_GEN("gfx1010:xnack+", 10, 1, 0, unsupported, enabled, 32) diff --git a/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt b/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt index 929b896bc2..2196cb0e98 100644 --- a/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt +++ b/runtime/hsa-runtime/core/runtime/trap_handler/CMakeLists.txt @@ -46,9 +46,8 @@ cmake_minimum_required ( VERSION 3.7 ) find_package(Clang REQUIRED HINTS ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm ) find_package(LLVM REQUIRED HINTS ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm ) -set (TARGET_DEVS "gfx900;gfx940;gfx1010;gfx1030;gfx1100") -set (POSTFIX "9;940;1010;10;11") - +set (TARGET_DEVS "gfx900;gfx940;gfx941;gfx942;gfx1010;gfx1030;gfx1100") +set (POSTFIX "9;940;941;942;1010;10;11") if(${CMAKE_VERBOSE_MAKEFILE}) get_property(clang_path TARGET clang PROPERTY LOCATION) diff --git a/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s b/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s index a753247a4d..60070e507b 100644 --- a/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s +++ b/runtime/hsa-runtime/core/runtime/trap_handler/trap_handler.s @@ -101,7 +101,7 @@ // ttmp1 = 0[2:0], PCRewind[3:0], HostTrap[0], TrapId[7:0], PC[47:32] // gfx906/gfx908/gfx90a: // ttmp11 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0] -// gfx940: +// gfx940/gfx941/gfx942: // ttmp13 = SQ_WAVE_IB_STS[20:15], 0[1:0], DebugEnabled[0], 0[22:0] // gfx10: // ttmp1 = 0[0], PCRewind[5:0], HostTrap[0], TrapId[7:0], PC[47:32] diff --git a/runtime/hsa-runtime/image/blit_kernel.cpp b/runtime/hsa-runtime/image/blit_kernel.cpp index 6b4bd53e64..067c668f73 100644 --- a/runtime/hsa-runtime/image/blit_kernel.cpp +++ b/runtime/hsa-runtime/image/blit_kernel.cpp @@ -86,6 +86,8 @@ extern uint8_t ocl_blit_object_gfx909[]; extern uint8_t ocl_blit_object_gfx90a[]; extern uint8_t ocl_blit_object_gfx90c[]; extern uint8_t ocl_blit_object_gfx940[]; +extern uint8_t ocl_blit_object_gfx941[]; +extern uint8_t ocl_blit_object_gfx942[]; extern uint8_t ocl_blit_object_gfx1010[]; extern uint8_t ocl_blit_object_gfx1011[]; extern uint8_t ocl_blit_object_gfx1012[]; @@ -1010,6 +1012,10 @@ hsa_status_t BlitKernel::GetPatchedBlitObject(const char* agent_name, *blit_code_object = ocl_blit_object_gfx90c; } else if (sname == "gfx940") { *blit_code_object = ocl_blit_object_gfx940; + } else if (sname == "gfx941") { + *blit_code_object = ocl_blit_object_gfx941; + } else if (sname == "gfx942") { + *blit_code_object = ocl_blit_object_gfx942; } else if (sname == "gfx1010") { *blit_code_object = ocl_blit_object_gfx1010; } else if (sname == "gfx1011") { diff --git a/runtime/hsa-runtime/image/blit_src/CMakeLists.txt b/runtime/hsa-runtime/image/blit_src/CMakeLists.txt index d8d39605d5..1b3c7cfa3c 100644 --- a/runtime/hsa-runtime/image/blit_src/CMakeLists.txt +++ b/runtime/hsa-runtime/image/blit_src/CMakeLists.txt @@ -47,7 +47,7 @@ find_package(Clang REQUIRED HINTS ${CMAKE_PREFIX_PATH}/llvm PATHS /opt/rocm/llvm # Determine the target devices if not specified if (NOT DEFINED TARGET_DEVICES) - set(TARGET_DEVICES "gfx700;gfx701;gfx702;gfx801;gfx802;gfx803;gfx805;gfx810;gfx900;gfx902;gfx904;gfx906;gfx908;gfx909;gfx90a;gfx90c;gfx940;gfx1010;gfx1011;gfx1012;gfx1013;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103") + set(TARGET_DEVICES "gfx700;gfx701;gfx702;gfx801;gfx802;gfx803;gfx805;gfx810;gfx900;gfx902;gfx904;gfx906;gfx908;gfx909;gfx90a;gfx90c;gfx940;gfx941;gfx942;gfx1010;gfx1011;gfx1012;gfx1013;gfx1030;gfx1031;gfx1032;gfx1033;gfx1034;gfx1035;gfx1036;gfx1100;gfx1101;gfx1102;gfx1103") endif() set( TARGET_DEVICES ${TARGET_DEVICES} CACHE STRING "Build targets" FORCE ) diff --git a/runtime/hsa-runtime/inc/amd_hsa_elf.h b/runtime/hsa-runtime/inc/amd_hsa_elf.h index 283d48d754..45962b1ddd 100644 --- a/runtime/hsa-runtime/inc/amd_hsa_elf.h +++ b/runtime/hsa-runtime/inc/amd_hsa_elf.h @@ -126,10 +126,12 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX1036 = 0x045, EF_AMDGPU_MACH_AMDGCN_GFX1101 = 0x046, EF_AMDGPU_MACH_AMDGCN_GFX1102 = 0x047, + EF_AMDGPU_MACH_AMDGCN_GFX941 = 0x04b, + EF_AMDGPU_MACH_AMDGCN_GFX942 = 0x04c, // First/last AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, - EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX1102, + EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX942, // Indicates if the "xnack" target feature is enabled for all code contained // in the object. diff --git a/runtime/hsa-runtime/libamdhsacode/amd_hsa_code.cpp b/runtime/hsa-runtime/libamdhsacode/amd_hsa_code.cpp index 49412aade2..e58b6e2ae4 100644 --- a/runtime/hsa-runtime/libamdhsacode/amd_hsa_code.cpp +++ b/runtime/hsa-runtime/libamdhsacode/amd_hsa_code.cpp @@ -581,6 +581,8 @@ namespace code { case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90A: MI.Name = "gfx90a"; MI.XnackSupported = true; MI.SrameccSupported = true; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX90C: MI.Name = "gfx90c"; MI.XnackSupported = true; MI.SrameccSupported = false; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX940: MI.Name = "gfx940"; MI.XnackSupported = true; MI.SrameccSupported = true; break; + case ELF::EF_AMDGPU_MACH_AMDGCN_GFX941: MI.Name = "gfx941"; MI.XnackSupported = true; MI.SrameccSupported = true; break; + case ELF::EF_AMDGPU_MACH_AMDGCN_GFX942: MI.Name = "gfx942"; MI.XnackSupported = true; MI.SrameccSupported = true; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1010: MI.Name = "gfx1010"; MI.XnackSupported = true; MI.SrameccSupported = false; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1011: MI.Name = "gfx1011"; MI.XnackSupported = true; MI.SrameccSupported = false; break; case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1012: MI.Name = "gfx1012"; MI.XnackSupported = true; MI.SrameccSupported = false; break;