Check max wave scratch limits.

HW has limited bits for wave scratch base address stride.  Enforcement
prevents programs with larger than supported scratch allocations from
running and clobbering neighboring scratch space.

Change-Id: I574da888e9d1d5e290a9c0025ba13b5ef9f1e5c0


[ROCm/ROCR-Runtime commit: 8e4177382a]

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -1054,12 +1054,12 @@ void AqlQueue::InitScratchSRD() {
   // Scratch is allocated program COMPUTE_TMPRING_SIZE register
   // Scratch Size per Wave is specified in terms of kilobytes
   uint32_t wave_size = agent_props.WaveFrontSize;
-  tmpring_size.bits.WAVESIZE =
-        (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024);
+  uint32_t wave_scratch = (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024);
+  tmpring_size.bits.WAVESIZE = wave_scratch;
+  assert(wave_scratch == tmpring_size.bits.WAVESIZE && "WAVESIZE Overflow.");
   uint32_t num_waves = (queue_scratch_.size / (tmpring_size.bits.WAVESIZE * 1024));
   tmpring_size.bits.WAVES = std::min(num_waves, max_scratch_waves);
   amd_queue_.compute_tmpring_size = tmpring_size.u32All;
-
   return;
 }
 }  // namespace amd

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -62,11 +62,12 @@
 #include "core/inc/isa.h"
 #include "core/inc/runtime.h"
 #include "core/util/os.h"
-#include "hsa_ext_image.h"
+#include "inc/hsa_ext_image.h"
 #include "inc/hsa_ven_amd_aqlprofile.h"
 
 // Size of scratch (private) segment pre-allocated per thread, in bytes.
 #define DEFAULT_SCRATCH_BYTES_PER_THREAD 2048
+#define MAX_WAVE_SCRATCH 8387584  // See COMPUTE_TMPRING_SIZE.WAVESIZE
 
 extern core::HsaApiTable hsa_internal_api_table_;
 
@@ -947,15 +948,19 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type,
 }
 
 void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) {
+  assert(scratch.queue_base == nullptr && "AcquireQueueScratch called while holding scratch.");
   bool need_queue_scratch_base = (isa_->GetMajorVersion() > 8);
 
   if (scratch.size == 0) {
     scratch.size = queue_scratch_len_;
     scratch.size_per_thread = scratch_per_thread_;
   }
-
   scratch.retry = false;
 
+  // Fail scratch allocation if per wave limits are exceeded.
+  uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024);
+  if (size_per_wave > MAX_WAVE_SCRATCH) return;
+
   ScopedAcquire<KernelMutex> lock(&scratch_lock_);
   // Limit to 1/8th of scratch pool for small scratch and 1/4 of that for a single queue.
   size_t small_limit = scratch_pool_.size() >> 3;
@@ -1001,7 +1006,6 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) {
   if (core::Runtime::runtime_singleton_->flag().enable_queue_fault_message())
     debug_print("Failed to map requested scratch - reducing queue occupancy.\n");
   uint64_t num_cus = properties_.NumFComputeCores / properties_.NumSIMDPerCU;
-  uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024);
   uint64_t total_waves = scratch.size / size_per_wave;
   uint64_t waves_per_cu = total_waves / num_cus;
   while (waves_per_cu != 0) {
@@ -1044,6 +1048,7 @@ void GpuAgent::ReleaseQueueScratch(ScratchInfo& scratch) {
     }
   }
   scratch_pool_.free(scratch.queue_base);
+  scratch.queue_base = nullptr;
 
   if (scratch.large) scratch_used_large_ -= scratch.size;