Do not allow occupancy restriction on cooperative groups.

Excessive scratch allocations can normally trigger occupancy
reduction.  This breaks cooperative groups so if occupancy
reduction is required on a cooperative dispatch fail with OOM.

Change-Id: I64612a2e38bf1286f3b74c1c2a68ab0c85452771

runtime/hsa-runtime/core/inc/scratch_cache.h

@@ -94,11 +94,12 @@ class ScratchCache {
     size_t size_per_thread;
     uint32_t lanes_per_wave;
     uint32_t waves_per_group;
+    uint64_t wanted_slots;
+    bool cooperative;
     ptrdiff_t queue_process_offset;
     bool large;
     bool retry;
     hsa_signal_t queue_retry;
-    uint64_t wanted_slots;
     ScratchCache::ref_t scratch_node;
   };
 

runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp

@@ -876,6 +876,8 @@ bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) {
       scratch.dispatch_size =
           scratch.size_per_thread * scratch.wanted_slots * scratch.lanes_per_wave;
 
+      scratch.cooperative = (queue->amd_queue_.hsa_queue.type == HSA_QUEUE_TYPE_COOPERATIVE);
+
       queue->agent_->AcquireQueueScratch(scratch);
 
       if (scratch.retry) {

runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -1230,7 +1230,8 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) {
     }
 
     // Fail scratch allocation if reducing occupancy is disabled.
-    if ((!use_reclaim) || core::Runtime::runtime_singleton_->flag().no_scratch_thread_limiter())
+    if (scratch.cooperative || (!use_reclaim) ||
+        core::Runtime::runtime_singleton_->flag().no_scratch_thread_limiter())
       return;
 
     // Attempt to trim the maximum number of concurrent waves to allow scratch to fit.