From 4ee01e898225fe73a07ad2372b140f5904aedbd6 Mon Sep 17 00:00:00 2001 From: David Yat Sin Date: Wed, 22 Jun 2022 16:37:10 -0400 Subject: [PATCH] Update scratch register definitions for GFX11 Update scratch register definitions for GFX11 asics. Change-Id: I6195e04b0a099fe84d1015c2f34ca3756a8175ef [ROCm/ROCR-Runtime commit: 90322899fe5fed3965d4cfd8544bc6a117a17959] --- .../hsa-runtime/core/inc/amd_aql_queue.h | 9 + .../runtime/hsa-runtime/core/inc/registers.h | 78 +++++- .../core/runtime/amd_aql_queue.cpp | 222 +++++++++++++----- .../hsa-runtime/image/resource_gfx11.h | 3 + 4 files changed, 243 insertions(+), 69 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_aql_queue.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_aql_queue.h index 0567731225..21c16c21bb 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_aql_queue.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_aql_queue.h @@ -226,6 +226,15 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo /// @brief Define the Scratch Buffer Descriptor and related parameters /// that enable kernel access scratch memory void InitScratchSRD(); + void FillBufRsrcWord0(); + void FillBufRsrcWord1(); + void FillBufRsrcWord1_Gfx11(); + void FillBufRsrcWord2(); + void FillBufRsrcWord3(); + void FillBufRsrcWord3_Gfx10(); + void FillBufRsrcWord3_Gfx11(); + void FillComputeTmpRingSize(); + void FillComputeTmpRingSize_Gfx11(); /// @brief Halt the queue without destroying it or fencing memory. void Suspend(); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/registers.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/registers.h index f436aba6e4..b8ba6aa5d1 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/registers.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/registers.h @@ -117,8 +117,25 @@ SQ_SEL_W = 0x00000007, float f32All; }; + union COMPUTE_TMPRING_SIZE_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int WAVES : 12; + unsigned int WAVESIZE : 15; + unsigned int : 5; +#elif defined(BIGENDIAN_CPU) + unsigned int : 5; + unsigned int WAVESIZE : 15; + unsigned int WAVES : 12; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; - union SQ_BUF_RSRC_WORD0 { + + union SQ_BUF_RSRC_WORD0 { struct { #if defined(LITTLEENDIAN_CPU) unsigned int BASE_ADDRESS : 32; @@ -151,8 +168,25 @@ SQ_SEL_W = 0x00000007, float f32All; }; + union SQ_BUF_RSRC_WORD1_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int BASE_ADDRESS_HI : 16; + unsigned int STRIDE : 14; + unsigned int SWIZZLE_ENABLE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int SWIZZLE_ENABLE : 2; + unsigned int STRIDE : 14; + unsigned int BASE_ADDRESS_HI : 16; +#endif + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; - union SQ_BUF_RSRC_WORD2 { + + union SQ_BUF_RSRC_WORD2 { struct { #if defined(LITTLEENDIAN_CPU) unsigned int NUM_RECORDS : 32; @@ -234,9 +268,43 @@ SQ_SEL_W = 0x00000007, unsigned int DST_SEL_Y : 3; unsigned int DST_SEL_X : 3; #endif - } bitfields, bits; - unsigned int u32All; + } bitfields, bits; + unsigned int u32All; + signed int i32All; + float f32All; + }; + + // From V# Table + union SQ_BUF_RSRC_WORD3_GFX11 { + struct { +#if defined(LITTLEENDIAN_CPU) + unsigned int DST_SEL_X : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_W : 3; + unsigned int FORMAT : 6; + unsigned int RESERVED1 : 3; + unsigned int INDEX_STRIDE : 2; + unsigned int ADD_TID_ENABLE : 1; + unsigned int RESERVED2 : 4; + unsigned int OOB_SELECT : 2; + unsigned int TYPE : 2; +#elif defined(BIGENDIAN_CPU) + unsigned int TYPE : 2; + unsigned int OOB_SELECT : 2; + unsigned int RESERVED2 : 4; + unsigned int ADD_TID_ENABLE : 1; + unsigned int INDEX_STRIDE : 2; + unsigned int RESERVED1 : 3; + unsigned int FORMAT : 6; + unsigned int DST_SEL_W : 3; + unsigned int DST_SEL_Z : 3; + unsigned int DST_SEL_Y : 3; + unsigned int DST_SEL_X : 3; +#endif + } bitfields, bits; + unsigned int u32All; signed int i32All; float f32All; - }; + }; #endif // header guard diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index e8febcfee6..245f4d3032 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -1268,86 +1268,114 @@ void AqlQueue::ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) { } } -// @brief Define the Scratch Buffer Descriptor and related parameters -// that enable kernel access scratch memory -void AqlQueue::InitScratchSRD() { - - // Populate scratch resource descriptor +void AqlQueue::FillBufRsrcWord0() { SQ_BUF_RSRC_WORD0 srd0; - SQ_BUF_RSRC_WORD1 srd1; - SQ_BUF_RSRC_WORD2 srd2; - uint32_t srd3_u32; - - uint32_t scratch_base_hi = 0; uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base); - #ifdef HSA_LARGE_MODEL + + srd0.bits.BASE_ADDRESS = uint32_t(scratch_base); + amd_queue_.scratch_resource_descriptor[0] = srd0.u32All; +} + +void AqlQueue::FillBufRsrcWord1() { + SQ_BUF_RSRC_WORD1 srd1; + uint32_t scratch_base_hi = 0; + +#ifdef HSA_LARGE_MODEL + uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base); scratch_base_hi = uint32_t(scratch_base >> 32); #endif - srd0.bits.BASE_ADDRESS = uint32_t(scratch_base); srd1.bits.BASE_ADDRESS_HI = scratch_base_hi; srd1.bits.STRIDE = 0; srd1.bits.CACHE_SWIZZLE = 0; srd1.bits.SWIZZLE_ENABLE = 1; + amd_queue_.scratch_resource_descriptor[1] = srd1.u32All; +} + +void AqlQueue::FillBufRsrcWord1_Gfx11() { + SQ_BUF_RSRC_WORD1_GFX11 srd1; + uint32_t scratch_base_hi = 0; + +#ifdef HSA_LARGE_MODEL + uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base); + scratch_base_hi = uint32_t(scratch_base >> 32); +#endif + + srd1.bits.BASE_ADDRESS_HI = scratch_base_hi; + srd1.bits.STRIDE = 0; + srd1.bits.SWIZZLE_ENABLE = 1; + + amd_queue_.scratch_resource_descriptor[1] = srd1.u32All; +} + +void AqlQueue::FillBufRsrcWord2() { + SQ_BUF_RSRC_WORD2 srd2; srd2.bits.NUM_RECORDS = uint32_t(queue_scratch_.size); - if (agent_->isa()->GetMajorVersion() < 10) { - SQ_BUF_RSRC_WORD3 srd3; - - srd3.bits.DST_SEL_X = SQ_SEL_X; - srd3.bits.DST_SEL_Y = SQ_SEL_Y; - srd3.bits.DST_SEL_Z = SQ_SEL_Z; - srd3.bits.DST_SEL_W = SQ_SEL_W; - srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT; - srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32; - srd3.bits.ELEMENT_SIZE = 1; // 4 - srd3.bits.INDEX_STRIDE = 3; // 64 - srd3.bits.ADD_TID_ENABLE = 1; - srd3.bits.ATC__CI__VI = (agent_->profile() == HSA_PROFILE_FULL); - srd3.bits.HASH_ENABLE = 0; - srd3.bits.HEAP = 0; - srd3.bits.MTYPE__CI__VI = 0; - srd3.bits.TYPE = SQ_RSRC_BUF; - - srd3_u32 = srd3.u32All; - } else { - SQ_BUF_RSRC_WORD3_GFX10 srd3; - - srd3.bits.DST_SEL_X = SQ_SEL_X; - srd3.bits.DST_SEL_Y = SQ_SEL_Y; - srd3.bits.DST_SEL_Z = SQ_SEL_Z; - srd3.bits.DST_SEL_W = SQ_SEL_W; - srd3.bits.FORMAT = BUF_FORMAT_32_UINT; - srd3.bits.RESERVED1 = 0; - srd3.bits.INDEX_STRIDE = 0; // filled in by CP - srd3.bits.ADD_TID_ENABLE = 1; - srd3.bits.RESOURCE_LEVEL = 1; - srd3.bits.RESERVED2 = 0; - srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode - srd3.bits.TYPE = SQ_RSRC_BUF; - - srd3_u32 = srd3.u32All; - } - - // Update Queue's Scratch descriptor's property - amd_queue_.scratch_resource_descriptor[0] = srd0.u32All; - amd_queue_.scratch_resource_descriptor[1] = srd1.u32All; amd_queue_.scratch_resource_descriptor[2] = srd2.u32All; - amd_queue_.scratch_resource_descriptor[3] = srd3_u32; +} - // Populate flat scratch parameters in amd_queue_. - amd_queue_.scratch_backing_memory_location = - queue_scratch_.queue_process_offset; - amd_queue_.scratch_backing_memory_byte_size = queue_scratch_.size; +void AqlQueue::FillBufRsrcWord3() { + SQ_BUF_RSRC_WORD3 srd3; - // For backwards compatibility this field records the per-lane scratch - // for a 64 lane wavefront. If scratch was allocated for 32 lane waves - // then the effective size for a 64 lane wave is halved. - amd_queue_.scratch_wave64_lane_byte_size = - uint32_t((queue_scratch_.size_per_thread * queue_scratch_.lanes_per_wave) / 64); + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT; + srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32; + srd3.bits.ELEMENT_SIZE = 1; // 4 + srd3.bits.INDEX_STRIDE = 3; // 64 + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.ATC__CI__VI = (agent_->profile() == HSA_PROFILE_FULL); + srd3.bits.HASH_ENABLE = 0; + srd3.bits.HEAP = 0; + srd3.bits.MTYPE__CI__VI = 0; + srd3.bits.TYPE = SQ_RSRC_BUF; - // Set concurrent wavefront limits only when scratch is being used. + amd_queue_.scratch_resource_descriptor[3] = srd3.u32All; +} + +void AqlQueue::FillBufRsrcWord3_Gfx10() { + SQ_BUF_RSRC_WORD3_GFX10 srd3; + + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.FORMAT = BUF_FORMAT_32_UINT; + srd3.bits.RESERVED1 = 0; + srd3.bits.INDEX_STRIDE = 0; // filled in by CP + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.RESOURCE_LEVEL = 1; + srd3.bits.RESERVED2 = 0; + srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode + srd3.bits.TYPE = SQ_RSRC_BUF; + + amd_queue_.scratch_resource_descriptor[3] = srd3.u32All; +} + +void AqlQueue::FillBufRsrcWord3_Gfx11() { + SQ_BUF_RSRC_WORD3_GFX11 srd3; + + srd3.bits.DST_SEL_X = SQ_SEL_X; + srd3.bits.DST_SEL_Y = SQ_SEL_Y; + srd3.bits.DST_SEL_Z = SQ_SEL_Z; + srd3.bits.DST_SEL_W = SQ_SEL_W; + srd3.bits.FORMAT = BUF_FORMAT_32_UINT; + srd3.bits.RESERVED1 = 0; + srd3.bits.INDEX_STRIDE = 0; // filled in by CP + srd3.bits.ADD_TID_ENABLE = 1; + srd3.bits.RESERVED2 = 0; + srd3.bits.OOB_SELECT = 2; // no bounds check in swizzle mode + srd3.bits.TYPE = SQ_RSRC_BUF; + + amd_queue_.scratch_resource_descriptor[3] = srd3.u32All; +} + +// Set concurrent wavefront limits only when scratch is being used. +void AqlQueue::FillComputeTmpRingSize() { COMPUTE_TMPRING_SIZE tmpring_size = {}; if (queue_scratch_.size == 0) { amd_queue_.compute_tmpring_size = tmpring_size.u32All; @@ -1370,6 +1398,72 @@ void AqlQueue::InitScratchSRD() { amd_queue_.compute_tmpring_size = tmpring_size.u32All; assert((tmpring_size.bits.WAVES % agent_props.NumShaderBanks == 0) && "Invalid scratch wave count. Must be divisible by #SEs."); +} + +// Set concurrent wavefront limits only when scratch is being used. +void AqlQueue::FillComputeTmpRingSize_Gfx11() { + COMPUTE_TMPRING_SIZE_GFX11 tmpring_size = {}; + if (queue_scratch_.size == 0) { + amd_queue_.compute_tmpring_size = tmpring_size.u32All; + return; + } + + // Determine the maximum number of waves device can support + const auto& agent_props = agent_->properties(); + uint32_t num_cus = agent_props.NumFComputeCores / agent_props.NumSIMDPerCU; + uint32_t max_scratch_waves = num_cus * agent_props.MaxSlotsScratchCU; + + // Scratch is allocated program COMPUTE_TMPRING_SIZE register + // Scratch Size per Wave is specified in terms of kilobytes + uint32_t wave_scratch = + (((queue_scratch_.lanes_per_wave * queue_scratch_.size_per_thread) + 255) / 256); + tmpring_size.bits.WAVESIZE = wave_scratch; + assert(wave_scratch == tmpring_size.bits.WAVESIZE && "WAVESIZE Overflow."); + uint32_t num_waves = queue_scratch_.size / (tmpring_size.bits.WAVESIZE * 256); + + // For GFX11 we specify number of waves per engine instead of total + num_waves /= agent_->properties().NumShaderBanks; + tmpring_size.bits.WAVES = std::min(num_waves, max_scratch_waves); + amd_queue_.compute_tmpring_size = tmpring_size.u32All; +} + +// @brief Define the Scratch Buffer Descriptor and related parameters +// that enable kernel access scratch memory +void AqlQueue::InitScratchSRD() { + switch (agent_->isa()->GetMajorVersion()) { + case 11: + FillBufRsrcWord0(); + FillBufRsrcWord1_Gfx11(); + FillBufRsrcWord2(); + FillBufRsrcWord3_Gfx11(); + FillComputeTmpRingSize_Gfx11(); + break; + case 10: + FillBufRsrcWord0(); + FillBufRsrcWord1(); + FillBufRsrcWord2(); + FillBufRsrcWord3_Gfx10(); + FillComputeTmpRingSize(); + break; + default: + FillBufRsrcWord0(); + FillBufRsrcWord1(); + FillBufRsrcWord2(); + FillBufRsrcWord3(); + FillComputeTmpRingSize(); + break; + } + + // Populate flat scratch parameters in amd_queue_. + amd_queue_.scratch_backing_memory_location = queue_scratch_.queue_process_offset; + amd_queue_.scratch_backing_memory_byte_size = queue_scratch_.size; + + // For backwards compatibility this field records the per-lane scratch + // for a 64 lane wavefront. If scratch was allocated for 32 lane waves + // then the effective size for a 64 lane wave is halved. + amd_queue_.scratch_wave64_lane_byte_size = + uint32_t((queue_scratch_.size_per_thread * queue_scratch_.lanes_per_wave) / 64); + return; } diff --git a/projects/rocr-runtime/runtime/hsa-runtime/image/resource_gfx11.h b/projects/rocr-runtime/runtime/hsa-runtime/image/resource_gfx11.h index 95efe26c57..bc9d094ed6 100755 --- a/projects/rocr-runtime/runtime/hsa-runtime/image/resource_gfx11.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/image/resource_gfx11.h @@ -75,6 +75,9 @@ union SQ_BUF_RSRC_WORD0 { /***********/ +/* Note: These registers are also defined/used in registers.h + * in SQ_BUF_RSRC_WORD1_GFX11 + */ #define SQ_BUF_RSC_WRD1_REG_SZ 32 #define SQ_BUF_RSC_WRD1_BASE_ADDRESS_HI_SZ 16 #define SQ_BUF_RSC_WRD1_STRIDE_SZ 14