Add env key HSA_NO_SCRATCH_THREAD_LIMITER.

Setting to 1 prevents the scratch handler from reducing peak occupancy.
Scratch allocations that would normally reduce peak occupancy will
instead fail.

Diagnostic for TF and PyTorch.

Change-Id: I2d7ea47077eb5cf708251c8aa3fd183ad4261be0


[ROCm/ROCR-Runtime commit: dc165c92bc]

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -1043,6 +1043,9 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) {
     return;
   }
 
+  // Fail scratch allocation if reducing occupancy is disabled.
+  if (core::Runtime::runtime_singleton_->flag().no_scratch_thread_limiter()) return;
+
   // Attempt to trim the maximum number of concurrent waves to allow scratch to fit.
   if (core::Runtime::runtime_singleton_->flag().enable_queue_fault_message())
     debug_print("Failed to map requested scratch (%ld) - reducing queue occupancy.\n",

projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h

@@ -109,6 +109,9 @@ class Flag {
 
     var = os::GetEnvVar("HSA_NO_SCRATCH_RECLAIM");
     no_scratch_reclaim_ = (var == "1") ? true : false;
+
+    var = os::GetEnvVar("HSA_NO_SCRATCH_THREAD_LIMITER");
+    no_scratch_thread_limit_ = (var == "1") ? true : false;
   }
 
   bool check_flat_scratch() const { return check_flat_scratch_; }
@@ -135,6 +138,8 @@ class Flag {
 
   bool no_scratch_reclaim() const { return no_scratch_reclaim_; }
 
+  bool no_scratch_thread_limiter() const { return no_scratch_thread_limit_; }
+
   std::string enable_sdma() const { return enable_sdma_; }
 
   std::string visible_gpus() const { return visible_gpus_; }
@@ -158,6 +163,7 @@ class Flag {
   bool rev_copy_dir_;
   bool fine_grain_pcie_;
   bool no_scratch_reclaim_;
+  bool no_scratch_thread_limit_;
 
   std::string enable_sdma_;