From 8a6954c63cc1d3bf6315e55385f4795377442f3e Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Mon, 28 Feb 2022 21:26:18 -0600 Subject: [PATCH] Do not allow occupancy restriction on cooperative groups. Excessive scratch allocations can normally trigger occupancy reduction. This breaks cooperative groups so if occupancy reduction is required on a cooperative dispatch fail with OOM. Change-Id: I64612a2e38bf1286f3b74c1c2a68ab0c85452771 --- runtime/hsa-runtime/core/inc/scratch_cache.h | 3 ++- runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp | 2 ++ runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/runtime/hsa-runtime/core/inc/scratch_cache.h b/runtime/hsa-runtime/core/inc/scratch_cache.h index 6fa6a19af3..9ab50edbf5 100644 --- a/runtime/hsa-runtime/core/inc/scratch_cache.h +++ b/runtime/hsa-runtime/core/inc/scratch_cache.h @@ -94,11 +94,12 @@ class ScratchCache { size_t size_per_thread; uint32_t lanes_per_wave; uint32_t waves_per_group; + uint64_t wanted_slots; + bool cooperative; ptrdiff_t queue_process_offset; bool large; bool retry; hsa_signal_t queue_retry; - uint64_t wanted_slots; ScratchCache::ref_t scratch_node; }; diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index e66ac355a1..55936f83fb 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -876,6 +876,8 @@ bool AqlQueue::DynamicScratchHandler(hsa_signal_value_t error_code, void* arg) { scratch.dispatch_size = scratch.size_per_thread * scratch.wanted_slots * scratch.lanes_per_wave; + scratch.cooperative = (queue->amd_queue_.hsa_queue.type == HSA_QUEUE_TYPE_COOPERATIVE); + queue->agent_->AcquireQueueScratch(scratch); if (scratch.retry) { diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index bc0a578bc8..e61fb368e3 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -1230,7 +1230,8 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) { } // Fail scratch allocation if reducing occupancy is disabled. - if ((!use_reclaim) || core::Runtime::runtime_singleton_->flag().no_scratch_thread_limiter()) + if (scratch.cooperative || (!use_reclaim) || + core::Runtime::runtime_singleton_->flag().no_scratch_thread_limiter()) return; // Attempt to trim the maximum number of concurrent waves to allow scratch to fit.