Update scratch register definitions for GFX11

Update scratch register definitions for GFX11 asics.

Change-Id: I6195e04b0a099fe84d1015c2f34ca3756a8175ef


[ROCm/ROCR-Runtime commit: 90322899fe]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_aql_queue.h

@@ -226,6 +226,15 @@ class AqlQueue : public core::Queue, private core::LocalSignal, public core::Doo
   /// @brief Define the Scratch Buffer Descriptor and related parameters
   /// that enable kernel access scratch memory
   void InitScratchSRD();
+  void FillBufRsrcWord0();
+  void FillBufRsrcWord1();
+  void FillBufRsrcWord1_Gfx11();
+  void FillBufRsrcWord2();
+  void FillBufRsrcWord3();
+  void FillBufRsrcWord3_Gfx10();
+  void FillBufRsrcWord3_Gfx11();
+  void FillComputeTmpRingSize();
+  void FillComputeTmpRingSize_Gfx11();
 
   /// @brief Halt the queue without destroying it or fencing memory.
   void Suspend();

projects/rocr-runtime/runtime/hsa-runtime/core/inc/registers.h

@@ -117,8 +117,25 @@ SQ_SEL_W                                 = 0x00000007,
 	float	f32All;
 	};
 
+        union COMPUTE_TMPRING_SIZE_GFX11 {
+          struct {
+#if defined(LITTLEENDIAN_CPU)
+            unsigned int WAVES : 12;
+            unsigned int WAVESIZE : 15;
+            unsigned int : 5;
+#elif defined(BIGENDIAN_CPU)
+            unsigned int : 5;
+            unsigned int WAVESIZE : 15;
+            unsigned int WAVES : 12;
+#endif
+          } bitfields, bits;
+          unsigned int u32All;
+          signed int i32All;
+          float f32All;
+        };
 
-	union SQ_BUF_RSRC_WORD0 {
+
+        union SQ_BUF_RSRC_WORD0 {
 	struct {
 #if		defined(LITTLEENDIAN_CPU)
 		unsigned int                    BASE_ADDRESS : 32;
@@ -151,8 +168,25 @@ SQ_SEL_W                                 = 0x00000007,
 	float	f32All;
 	};
 
+        union SQ_BUF_RSRC_WORD1_GFX11 {
+          struct {
+#if defined(LITTLEENDIAN_CPU)
+            unsigned int BASE_ADDRESS_HI : 16;
+            unsigned int STRIDE : 14;
+            unsigned int SWIZZLE_ENABLE : 2;
+#elif defined(BIGENDIAN_CPU)
+            unsigned int SWIZZLE_ENABLE : 2;
+            unsigned int STRIDE : 14;
+            unsigned int BASE_ADDRESS_HI : 16;
+#endif
+          } bitfields, bits;
+          unsigned int u32All;
+          signed int i32All;
+          float f32All;
+        };
 
-	union SQ_BUF_RSRC_WORD2 {
+
+        union SQ_BUF_RSRC_WORD2 {
 	struct {
 #if		defined(LITTLEENDIAN_CPU)
 		unsigned int                     NUM_RECORDS : 32;
@@ -234,9 +268,43 @@ SQ_SEL_W                                 = 0x00000007,
                 unsigned int                       DST_SEL_Y : 3;
                 unsigned int                       DST_SEL_X : 3;
 #endif
-	} bitfields, bits;
-	unsigned int	u32All;
+        } bitfields, bits;
+        unsigned int u32All;
+        signed int i32All;
+        float f32All;
+        };
+
+        // From V# Table
+        union SQ_BUF_RSRC_WORD3_GFX11 {
+          struct {
+#if defined(LITTLEENDIAN_CPU)
+            unsigned int DST_SEL_X : 3;
+            unsigned int DST_SEL_Y : 3;
+            unsigned int DST_SEL_Z : 3;
+            unsigned int DST_SEL_W : 3;
+            unsigned int FORMAT : 6;
+            unsigned int RESERVED1 : 3;
+            unsigned int INDEX_STRIDE : 2;
+            unsigned int ADD_TID_ENABLE : 1;
+            unsigned int RESERVED2 : 4;
+            unsigned int OOB_SELECT : 2;
+            unsigned int TYPE : 2;
+#elif defined(BIGENDIAN_CPU)
+            unsigned int TYPE : 2;
+            unsigned int OOB_SELECT : 2;
+            unsigned int RESERVED2 : 4;
+            unsigned int ADD_TID_ENABLE : 1;
+            unsigned int INDEX_STRIDE : 2;
+            unsigned int RESERVED1 : 3;
+            unsigned int FORMAT : 6;
+            unsigned int DST_SEL_W : 3;
+            unsigned int DST_SEL_Z : 3;
+            unsigned int DST_SEL_Y : 3;
+            unsigned int DST_SEL_X : 3;
+#endif
+          } bitfields, bits;
+        unsigned int	u32All;
 	signed int	i32All;
 	float	f32All;
-	};
+        };
 #endif  // header guard

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -1268,86 +1268,114 @@ void AqlQueue::ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) {
   }
 }
 
-// @brief Define the Scratch Buffer Descriptor and related parameters
-// that enable kernel access scratch memory
-void AqlQueue::InitScratchSRD() {
-
-  // Populate scratch resource descriptor
+void AqlQueue::FillBufRsrcWord0() {
   SQ_BUF_RSRC_WORD0 srd0;
-  SQ_BUF_RSRC_WORD1 srd1;
-  SQ_BUF_RSRC_WORD2 srd2;
-  uint32_t srd3_u32;
-
-  uint32_t scratch_base_hi = 0;
   uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base);
-  #ifdef HSA_LARGE_MODEL
+
+  srd0.bits.BASE_ADDRESS = uint32_t(scratch_base);
+  amd_queue_.scratch_resource_descriptor[0] = srd0.u32All;
+}
+
+void AqlQueue::FillBufRsrcWord1() {
+  SQ_BUF_RSRC_WORD1 srd1;
+  uint32_t scratch_base_hi = 0;
+
+#ifdef HSA_LARGE_MODEL
+  uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base);
   scratch_base_hi = uint32_t(scratch_base >> 32);
   #endif
-  srd0.bits.BASE_ADDRESS = uint32_t(scratch_base);
 
   srd1.bits.BASE_ADDRESS_HI = scratch_base_hi;
   srd1.bits.STRIDE = 0;
   srd1.bits.CACHE_SWIZZLE = 0;
   srd1.bits.SWIZZLE_ENABLE = 1;
 
+  amd_queue_.scratch_resource_descriptor[1] = srd1.u32All;
+}
+
+void AqlQueue::FillBufRsrcWord1_Gfx11() {
+  SQ_BUF_RSRC_WORD1_GFX11 srd1;
+  uint32_t scratch_base_hi = 0;
+
+#ifdef HSA_LARGE_MODEL
+  uintptr_t scratch_base = uintptr_t(queue_scratch_.queue_base);
+  scratch_base_hi = uint32_t(scratch_base >> 32);
+#endif
+
+  srd1.bits.BASE_ADDRESS_HI = scratch_base_hi;
+  srd1.bits.STRIDE = 0;
+  srd1.bits.SWIZZLE_ENABLE = 1;
+
+  amd_queue_.scratch_resource_descriptor[1] = srd1.u32All;
+}
+
+void AqlQueue::FillBufRsrcWord2() {
+  SQ_BUF_RSRC_WORD2 srd2;
   srd2.bits.NUM_RECORDS = uint32_t(queue_scratch_.size);
 
-  if (agent_->isa()->GetMajorVersion() < 10) {
-    SQ_BUF_RSRC_WORD3 srd3;
-
-    srd3.bits.DST_SEL_X = SQ_SEL_X;
-    srd3.bits.DST_SEL_Y = SQ_SEL_Y;
-    srd3.bits.DST_SEL_Z = SQ_SEL_Z;
-    srd3.bits.DST_SEL_W = SQ_SEL_W;
-    srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT;
-    srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32;
-    srd3.bits.ELEMENT_SIZE = 1;  // 4
-    srd3.bits.INDEX_STRIDE = 3;  // 64
-    srd3.bits.ADD_TID_ENABLE = 1;
-    srd3.bits.ATC__CI__VI = (agent_->profile() == HSA_PROFILE_FULL);
-    srd3.bits.HASH_ENABLE = 0;
-    srd3.bits.HEAP = 0;
-    srd3.bits.MTYPE__CI__VI = 0;
-    srd3.bits.TYPE = SQ_RSRC_BUF;
-
-    srd3_u32 = srd3.u32All;
-  } else {
-    SQ_BUF_RSRC_WORD3_GFX10 srd3;
-
-    srd3.bits.DST_SEL_X = SQ_SEL_X;
-    srd3.bits.DST_SEL_Y = SQ_SEL_Y;
-    srd3.bits.DST_SEL_Z = SQ_SEL_Z;
-    srd3.bits.DST_SEL_W = SQ_SEL_W;
-    srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
-    srd3.bits.RESERVED1 = 0;
-    srd3.bits.INDEX_STRIDE = 0;  // filled in by CP
-    srd3.bits.ADD_TID_ENABLE = 1;
-    srd3.bits.RESOURCE_LEVEL = 1;
-    srd3.bits.RESERVED2 = 0;
-    srd3.bits.OOB_SELECT = 2;  // no bounds check in swizzle mode
-    srd3.bits.TYPE = SQ_RSRC_BUF;
-
-    srd3_u32 = srd3.u32All;
-  }
-
-  // Update Queue's Scratch descriptor's property
-  amd_queue_.scratch_resource_descriptor[0] = srd0.u32All;
-  amd_queue_.scratch_resource_descriptor[1] = srd1.u32All;
   amd_queue_.scratch_resource_descriptor[2] = srd2.u32All;
-  amd_queue_.scratch_resource_descriptor[3] = srd3_u32;
+}
 
-  // Populate flat scratch parameters in amd_queue_.
-  amd_queue_.scratch_backing_memory_location =
-      queue_scratch_.queue_process_offset;
-  amd_queue_.scratch_backing_memory_byte_size = queue_scratch_.size;
+void AqlQueue::FillBufRsrcWord3() {
+  SQ_BUF_RSRC_WORD3 srd3;
 
-  // For backwards compatibility this field records the per-lane scratch
-  // for a 64 lane wavefront. If scratch was allocated for 32 lane waves
-  // then the effective size for a 64 lane wave is halved.
-  amd_queue_.scratch_wave64_lane_byte_size =
-      uint32_t((queue_scratch_.size_per_thread * queue_scratch_.lanes_per_wave) / 64);
+  srd3.bits.DST_SEL_X = SQ_SEL_X;
+  srd3.bits.DST_SEL_Y = SQ_SEL_Y;
+  srd3.bits.DST_SEL_Z = SQ_SEL_Z;
+  srd3.bits.DST_SEL_W = SQ_SEL_W;
+  srd3.bits.NUM_FORMAT = BUF_NUM_FORMAT_UINT;
+  srd3.bits.DATA_FORMAT = BUF_DATA_FORMAT_32;
+  srd3.bits.ELEMENT_SIZE = 1;  // 4
+  srd3.bits.INDEX_STRIDE = 3;  // 64
+  srd3.bits.ADD_TID_ENABLE = 1;
+  srd3.bits.ATC__CI__VI = (agent_->profile() == HSA_PROFILE_FULL);
+  srd3.bits.HASH_ENABLE = 0;
+  srd3.bits.HEAP = 0;
+  srd3.bits.MTYPE__CI__VI = 0;
+  srd3.bits.TYPE = SQ_RSRC_BUF;
 
-  // Set concurrent wavefront limits only when scratch is being used.
+  amd_queue_.scratch_resource_descriptor[3] = srd3.u32All;
+}
+
+void AqlQueue::FillBufRsrcWord3_Gfx10() {
+  SQ_BUF_RSRC_WORD3_GFX10 srd3;
+
+  srd3.bits.DST_SEL_X = SQ_SEL_X;
+  srd3.bits.DST_SEL_Y = SQ_SEL_Y;
+  srd3.bits.DST_SEL_Z = SQ_SEL_Z;
+  srd3.bits.DST_SEL_W = SQ_SEL_W;
+  srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
+  srd3.bits.RESERVED1 = 0;
+  srd3.bits.INDEX_STRIDE = 0;  // filled in by CP
+  srd3.bits.ADD_TID_ENABLE = 1;
+  srd3.bits.RESOURCE_LEVEL = 1;
+  srd3.bits.RESERVED2 = 0;
+  srd3.bits.OOB_SELECT = 2;  // no bounds check in swizzle mode
+  srd3.bits.TYPE = SQ_RSRC_BUF;
+
+  amd_queue_.scratch_resource_descriptor[3] = srd3.u32All;
+}
+
+void AqlQueue::FillBufRsrcWord3_Gfx11() {
+  SQ_BUF_RSRC_WORD3_GFX11 srd3;
+
+  srd3.bits.DST_SEL_X = SQ_SEL_X;
+  srd3.bits.DST_SEL_Y = SQ_SEL_Y;
+  srd3.bits.DST_SEL_Z = SQ_SEL_Z;
+  srd3.bits.DST_SEL_W = SQ_SEL_W;
+  srd3.bits.FORMAT = BUF_FORMAT_32_UINT;
+  srd3.bits.RESERVED1 = 0;
+  srd3.bits.INDEX_STRIDE = 0;  // filled in by CP
+  srd3.bits.ADD_TID_ENABLE = 1;
+  srd3.bits.RESERVED2 = 0;
+  srd3.bits.OOB_SELECT = 2;  // no bounds check in swizzle mode
+  srd3.bits.TYPE = SQ_RSRC_BUF;
+
+  amd_queue_.scratch_resource_descriptor[3] = srd3.u32All;
+}
+
+// Set concurrent wavefront limits only when scratch is being used.
+void AqlQueue::FillComputeTmpRingSize() {
   COMPUTE_TMPRING_SIZE tmpring_size = {};
   if (queue_scratch_.size == 0) {
     amd_queue_.compute_tmpring_size = tmpring_size.u32All;
@@ -1370,6 +1398,72 @@ void AqlQueue::InitScratchSRD() {
   amd_queue_.compute_tmpring_size = tmpring_size.u32All;
   assert((tmpring_size.bits.WAVES % agent_props.NumShaderBanks == 0) &&
          "Invalid scratch wave count.  Must be divisible by #SEs.");
+}
+
+// Set concurrent wavefront limits only when scratch is being used.
+void AqlQueue::FillComputeTmpRingSize_Gfx11() {
+  COMPUTE_TMPRING_SIZE_GFX11 tmpring_size = {};
+  if (queue_scratch_.size == 0) {
+    amd_queue_.compute_tmpring_size = tmpring_size.u32All;
+    return;
+  }
+
+  // Determine the maximum number of waves device can support
+  const auto& agent_props = agent_->properties();
+  uint32_t num_cus = agent_props.NumFComputeCores / agent_props.NumSIMDPerCU;
+  uint32_t max_scratch_waves = num_cus * agent_props.MaxSlotsScratchCU;
+
+  // Scratch is allocated program COMPUTE_TMPRING_SIZE register
+  // Scratch Size per Wave is specified in terms of kilobytes
+  uint32_t wave_scratch =
+      (((queue_scratch_.lanes_per_wave * queue_scratch_.size_per_thread) + 255) / 256);
+  tmpring_size.bits.WAVESIZE = wave_scratch;
+  assert(wave_scratch == tmpring_size.bits.WAVESIZE && "WAVESIZE Overflow.");
+  uint32_t num_waves = queue_scratch_.size / (tmpring_size.bits.WAVESIZE * 256);
+
+  // For GFX11 we specify number of waves per engine instead of total
+  num_waves /= agent_->properties().NumShaderBanks;
+  tmpring_size.bits.WAVES = std::min(num_waves, max_scratch_waves);
+  amd_queue_.compute_tmpring_size = tmpring_size.u32All;
+}
+
+// @brief Define the Scratch Buffer Descriptor and related parameters
+// that enable kernel access scratch memory
+void AqlQueue::InitScratchSRD() {
+  switch (agent_->isa()->GetMajorVersion()) {
+    case 11:
+      FillBufRsrcWord0();
+      FillBufRsrcWord1_Gfx11();
+      FillBufRsrcWord2();
+      FillBufRsrcWord3_Gfx11();
+      FillComputeTmpRingSize_Gfx11();
+      break;
+    case 10:
+      FillBufRsrcWord0();
+      FillBufRsrcWord1();
+      FillBufRsrcWord2();
+      FillBufRsrcWord3_Gfx10();
+      FillComputeTmpRingSize();
+      break;
+    default:
+      FillBufRsrcWord0();
+      FillBufRsrcWord1();
+      FillBufRsrcWord2();
+      FillBufRsrcWord3();
+      FillComputeTmpRingSize();
+      break;
+  }
+
+  // Populate flat scratch parameters in amd_queue_.
+  amd_queue_.scratch_backing_memory_location = queue_scratch_.queue_process_offset;
+  amd_queue_.scratch_backing_memory_byte_size = queue_scratch_.size;
+
+  // For backwards compatibility this field records the per-lane scratch
+  // for a 64 lane wavefront. If scratch was allocated for 32 lane waves
+  // then the effective size for a 64 lane wave is halved.
+  amd_queue_.scratch_wave64_lane_byte_size =
+      uint32_t((queue_scratch_.size_per_thread * queue_scratch_.lanes_per_wave) / 64);
+
   return;
 }
 

projects/rocr-runtime/runtime/hsa-runtime/image/resource_gfx11.h

@@ -75,6 +75,9 @@ union SQ_BUF_RSRC_WORD0 {
 
 /***********/
 
+/* Note: These registers are also defined/used in registers.h
+ * in SQ_BUF_RSRC_WORD1_GFX11
+ */
 #define SQ_BUF_RSC_WRD1_REG_SZ 32
 #define SQ_BUF_RSC_WRD1_BASE_ADDRESS_HI_SZ  16
 #define SQ_BUF_RSC_WRD1_STRIDE_SZ           14