From 02b6c65740447118922f877e492cf38b2e7be3b5 Mon Sep 17 00:00:00 2001 From: "Besar Wicaksono (xN/A) TX [TEXT]" Date: Mon, 11 Apr 2016 18:31:45 -0500 Subject: [PATCH] Sdma wraparound optimization. Remove mutex and just make the thread spin again if the queue is wrapping. Remove the wait for the queue to finish wrapping, and just check if there is enough space to recycle when reserving queue space. [git-p4: depot-paths = "//depot/stg/hsa/drivers/hsa/runtime/": change = 1256713] [ROCm/ROCR-Runtime commit: ea67bb8374ad3ee122ab85224eb1f7c9478e6f23] --- .../hsa-runtime/core/inc/amd_blit_sdma.h | 2 -- .../core/runtime/amd_blit_sdma.cpp | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h index e5174ad05f..35f683bc36 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h @@ -210,8 +210,6 @@ class BlitSdma : public core::Blit { /// Max total fill count supported by the queue. size_t max_total_fill_size_; - - std::mutex wrap_lock_; }; } // namespace amd diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index dbecfd54ec..120b0a5fb0 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -650,7 +650,7 @@ char* BlitSdma::AcquireWriteAddress(uint32_t cmd_size) { } while (true) { - uint32_t curr_offset = + const uint32_t curr_offset = atomic::Load(&cached_reserve_offset_, std::memory_order_acquire); const uint32_t end_offset = curr_offset + cmd_size; @@ -660,6 +660,13 @@ char* BlitSdma::AcquireWriteAddress(uint32_t cmd_size) { continue; } + const uint32_t curr_read_ptr_val = + atomic::Load(queue_resource_.Queue_read_ptr, std::memory_order_acquire); + if (curr_offset < curr_read_ptr_val && end_offset > curr_read_ptr_val) { + // Queue is wrapping and there is not enough space to recycle. + continue; + } + if (atomic::Cas(&cached_reserve_offset_, end_offset, curr_offset, std::memory_order_release) == curr_offset) { return queue_start_addr_ + curr_offset; @@ -720,6 +727,8 @@ void BlitSdma::WrapQueue(uint32_t cmd_size) { // Re-determine the offset into queue buffer where NOOP instructions // should be written. while (true) { + const uint32_t full_offset = queue_size_ + 1; + uint32_t curent_offset = atomic::Load(&cached_reserve_offset_, std::memory_order_acquire); const uint32_t end_offset = curent_offset + cmd_size; @@ -727,11 +736,13 @@ void BlitSdma::WrapQueue(uint32_t cmd_size) { return; } - // Only one thread can wrap the queue. - std::lock_guard guard(wrap_lock_); + if (curent_offset == full_offset) { + // Another thread is already wrapping the queue. + continue; + } // Close reservation to queue temporarily by "making" it full. - if (atomic::Cas(&cached_reserve_offset_, queue_size_ + 1, curent_offset, + if (atomic::Cas(&cached_reserve_offset_, full_offset, curent_offset, std::memory_order_release) == curent_offset) { // Wait till all reserved packets are commited. while (atomic::Load(&cached_commit_offset_, std::memory_order_acquire) != @@ -747,12 +758,6 @@ void BlitSdma::WrapQueue(uint32_t cmd_size) { // Update write and doorbell registers to execute NOOP instructions. UpdateWriteAndDoorbellRegister(curent_offset, 0); - // Wait till queue wrapped. - while (atomic::Load(queue_resource_.Queue_read_ptr, - std::memory_order_acquire) != 0) { - os::YieldThread(); - } - // Open access to queue. atomic::Store(&cached_reserve_offset_, 0U, std::memory_order_release); }