diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index cd6f341ab7..8ca9faee4a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -1054,12 +1054,12 @@ void AqlQueue::InitScratchSRD() { // Scratch is allocated program COMPUTE_TMPRING_SIZE register // Scratch Size per Wave is specified in terms of kilobytes uint32_t wave_size = agent_props.WaveFrontSize; - tmpring_size.bits.WAVESIZE = - (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024); + uint32_t wave_scratch = (((wave_size * queue_scratch_.size_per_thread) + 1023) / 1024); + tmpring_size.bits.WAVESIZE = wave_scratch; + assert(wave_scratch == tmpring_size.bits.WAVESIZE && "WAVESIZE Overflow."); uint32_t num_waves = (queue_scratch_.size / (tmpring_size.bits.WAVESIZE * 1024)); tmpring_size.bits.WAVES = std::min(num_waves, max_scratch_waves); amd_queue_.compute_tmpring_size = tmpring_size.u32All; - return; } } // namespace amd diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 91a279438b..0b862db48f 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -62,11 +62,12 @@ #include "core/inc/isa.h" #include "core/inc/runtime.h" #include "core/util/os.h" -#include "hsa_ext_image.h" +#include "inc/hsa_ext_image.h" #include "inc/hsa_ven_amd_aqlprofile.h" // Size of scratch (private) segment pre-allocated per thread, in bytes. #define DEFAULT_SCRATCH_BYTES_PER_THREAD 2048 +#define MAX_WAVE_SCRATCH 8387584 // See COMPUTE_TMPRING_SIZE.WAVESIZE extern core::HsaApiTable hsa_internal_api_table_; @@ -947,15 +948,19 @@ hsa_status_t GpuAgent::QueueCreate(size_t size, hsa_queue_type32_t queue_type, } void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) { + assert(scratch.queue_base == nullptr && "AcquireQueueScratch called while holding scratch."); bool need_queue_scratch_base = (isa_->GetMajorVersion() > 8); if (scratch.size == 0) { scratch.size = queue_scratch_len_; scratch.size_per_thread = scratch_per_thread_; } - scratch.retry = false; + // Fail scratch allocation if per wave limits are exceeded. + uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024); + if (size_per_wave > MAX_WAVE_SCRATCH) return; + ScopedAcquire lock(&scratch_lock_); // Limit to 1/8th of scratch pool for small scratch and 1/4 of that for a single queue. size_t small_limit = scratch_pool_.size() >> 3; @@ -1001,7 +1006,6 @@ void GpuAgent::AcquireQueueScratch(ScratchInfo& scratch) { if (core::Runtime::runtime_singleton_->flag().enable_queue_fault_message()) debug_print("Failed to map requested scratch - reducing queue occupancy.\n"); uint64_t num_cus = properties_.NumFComputeCores / properties_.NumSIMDPerCU; - uint64_t size_per_wave = AlignUp(scratch.size_per_thread * properties_.WaveFrontSize, 1024); uint64_t total_waves = scratch.size / size_per_wave; uint64_t waves_per_cu = total_waves / num_cus; while (waves_per_cu != 0) { @@ -1044,6 +1048,7 @@ void GpuAgent::ReleaseQueueScratch(ScratchInfo& scratch) { } } scratch_pool_.free(scratch.queue_base); + scratch.queue_base = nullptr; if (scratch.large) scratch_used_large_ -= scratch.size;