From d79cd9abf3243df55bda8c0c845c3d9f9271186c Mon Sep 17 00:00:00 2001 From: Sean Keely Date: Thu, 15 Nov 2018 22:18:03 -0600 Subject: [PATCH] Check max wave scratch limits. HW has limited bits for wave scratch base address stride. Enforcement prevents programs with larger than supported scratch allocations from running and clobbering neighboring scratch space. Change-Id: I574da888e9d1d5e290a9c0025ba13b5ef9f1e5c0 [ROCm/ROCR-Runtime commit: 8e4177382a803912640cefc3bb3ce7731cdde5ed] --- .../hsa-runtime/core/runtime/amd_aql_queue.cpp | 6 +++--- .../hsa-runtime/core/runtime/amd_gpu_agent.cpp | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index cd6f341ab7..8ca9faee4a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -1054,12 +1054,12 @@ void AqlQueue::InitScratchSRD() { // Scratch is allocated program COMPUTE_TMPRING_SIZE register // Scratch Size per Wave is specified in terms of kilobytes uint32_t wave_size = agent_props.WaveFrontSize; - tmpring_size.bits.WAVESIZE = - (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024); + uint32_t wave_scratch = (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024); + tmpring_size.bits.WAVESIZE = wave_scratch; + assert(wave_scratch == tmpring_size.bits.WAVESIZE && "WAVESIZE Overflow."); uint32_t num_waves = (queue_scratch_.size / (tmpring_size.bits.WAVESIZE * 1024)); tmpring_size.bits.WAVES = std::min(num_waves, max_scratch_waves); amd_queue_.compute_tmpring_size = tmpring_size.u32All; - return; } } // namespace amd diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 91a279438b..0b862db48f 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -62,11 +62,12 @@ #include "core/inc/isa.h" #include "core/inc/runtime.h" #include "core/util/os.h" -#include "hsa_ext_image.h" +#include "inc/hsa_ext_image.h" #include "inc/hsa_ven_amd_aqlprofile.h" // Size of scratch (private) segment pre-allocated per thread, in bytes. #define DEFAULT_SCRATCH_BYTES_PER_THREAD 2048 +#define MAX_WAVE_SCRATCH 8387584 // See COMPUTE_TMPRING_SIZE.WAVESIZE extern core::HsaApiTable hsa_internal_api_table_; @@ -947,15 +948,19 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type, } void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) { + assert(scratch.queue_base == nullptr && "AcquireQueueScratch called while holding scratch."); bool need_queue_scratch_base = (isa_->GetMajorVersion() > 8); if (scratch.size == 0) { scratch.size = queue_scratch_len_; scratch.size_per_thread = scratch_per_thread_; } - scratch.retry = false; + // Fail scratch allocation if per wave limits are exceeded. + uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024); + if (size_per_wave > MAX_WAVE_SCRATCH) return; + ScopedAcquire lock(&scratch_lock_); // Limit to 1/8th of scratch pool for small scratch and 1/4 of that for a single queue. size_t small_limit = scratch_pool_.size() >> 3; @@ -1001,7 +1006,6 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) { if (core::Runtime::runtime_singleton_->flag().enable_queue_fault_message()) debug_print("Failed to map requested scratch - reducing queue occupancy.\n"); uint64_t num_cus = properties_.NumFComputeCores / properties_.NumSIMDPerCU; - uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024); uint64_t total_waves = scratch.size / size_per_wave; uint64_t waves_per_cu = total_waves / num_cus; while (waves_per_cu != 0) { @@ -1044,6 +1048,7 @@ void GpuAgent::ReleaseQueueScratch(ScratchInfo& scratch) { } } scratch_pool_.free(scratch.queue_base); + scratch.queue_base = nullptr; if (scratch.large) scratch_used_large_ -= scratch.size;