diff --git a/runtime/hsa-runtime/core/inc/amd_aql_queue.h b/runtime/hsa-runtime/core/inc/amd_aql_queue.h index 80c462977f..7f8410af53 100644 --- a/runtime/hsa-runtime/core/inc/amd_aql_queue.h +++ b/runtime/hsa-runtime/core/inc/amd_aql_queue.h @@ -47,6 +47,7 @@ #include "core/inc/signal.h" #include "core/inc/queue.h" #include "core/inc/amd_gpu_agent.h" +#include "core/util/locks.h" namespace amd { /// @brief Encapsulates HW Aql Command Processor functionality. It @@ -181,6 +182,9 @@ class AqlQueue : public core::Queue, public core::Signal { hsa_status_t SetCUMasking(const uint32_t num_cu_mask_count, const uint32_t* cu_mask); + // @brief Submits a block of PM4 and waits until it has been executed. + void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) override; + /// @brief This operation is illegal hsa_signal_value_t LoadRelaxed() { assert(false); @@ -396,6 +400,11 @@ class AqlQueue : public core::Queue, public core::Signal { // Is KV device queue bool is_kv_queue_; + // GPU-visible indirect buffer holding PM4 commands. + void* pm4_ib_buf_; + uint32_t pm4_ib_size_b_; + KernelMutex pm4_ib_mutex_; + // Shared event used for queue errors static HsaEvent* queue_event_; diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index 56ddeee704..dea092ff87 100644 --- a/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -136,6 +136,9 @@ class GpuAgentInt : public core::Agent { // @param [out] time Timestamp in agent domain. virtual uint64_t TranslateTime(uint64_t tick) = 0; + // @brief Invalidate caches on the agent which may hold code object data. + virtual void InvalidateCodeCaches() = 0; + // @brief Sets the coherency type of this agent. // // @param [in] type New coherency type. @@ -258,9 +261,11 @@ class GpuAgent : public GpuAgentInt { uint64_t TranslateTime(uint64_t tick) override; // @brief Override from amd::GpuAgentInt. - bool current_coherency_type(hsa_amd_coherency_type_t type) override; + void InvalidateCodeCaches() override; // @brief Override from amd::GpuAgentInt. + bool current_coherency_type(hsa_amd_coherency_type_t type) override; + hsa_amd_coherency_type_t current_coherency_type() const override { return current_coherency_type_; } diff --git a/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h b/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h new file mode 100644 index 0000000000..09d6593d74 --- /dev/null +++ b/runtime/hsa-runtime/core/inc/amd_gpu_pm4.h @@ -0,0 +1,84 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// The University of Illinois/NCSA +// Open Source License (NCSA) +// +// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved. +// +// Developed by: +// +// AMD Research and AMD HSA Software Development +// +// Advanced Micro Devices, Inc. +// +// www.amd.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to +// deal with the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// - Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimers. +// - Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimers in +// the documentation and/or other materials provided with the distribution. +// - Neither the names of Advanced Micro Devices, Inc, +// nor the names of its contributors may be used to endorse or promote +// products derived from this Software without specific prior written +// permission. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS WITH THE SOFTWARE. +// +//////////////////////////////////////////////////////////////////////////////// + +#ifndef HSA_RUNTIME_CORE_INC_AMD_GPU_PM4_H_ +#define HSA_RUNTIME_CORE_INC_AMD_GPU_PM4_H_ + +// PM4 definitions: +// http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2013/10/si_programming_guide_v2.pdf +// +// ACQUIRE_MEM was introduced in Gfx7. + +#define PM4_HDR_IT_OPCODE_NOP 0x10 +#define PM4_HDR_IT_OPCODE_INDIRECT_BUFFER 0x3F +#define PM4_HDR_IT_OPCODE_RELEASE_MEM 0x49 +#define PM4_HDR_IT_OPCODE_ACQUIRE_MEM 0x58 + +#define PM4_HDR_SHADER_TYPE(x) (((x) & 0x1) << 1) +#define PM4_HDR_IT_OPCODE(x) (((x) & 0xFF) << 8) +#define PM4_HDR_COUNT(x) (((x) & 0x3FFF) << 16) +#define PM4_HDR_TYPE(x) (((x) & 0x3) << 30) + +#define PM4_HDR(it_opcode, pkt_size_dw, gfxip_ver) ( \ + PM4_HDR_SHADER_TYPE((gfxip_ver) == 7 ? 1 : 0) | \ + PM4_HDR_IT_OPCODE(it_opcode) | \ + PM4_HDR_COUNT(int(pkt_size_dw) - 2) | \ + PM4_HDR_TYPE(3) \ +) + +#define PM4_INDIRECT_BUFFER_DW1_IB_BASE_LO(x) (((x) & 0x3FFFFFFF) << 2) +#define PM4_INDIRECT_BUFFER_DW2_IB_BASE_HI(x) (((x) & 0xFFFF) << 0) +#define PM4_INDIRECT_BUFFER_DW3_IB_SIZE(x) (((x) & 0xFFFFF) << 0) +#define PM4_INDIRECT_BUFFER_DW3_IB_VALID(x) (((x) & 0x1) << 23) + +#define PM4_ACQUIRE_MEM_DW1_COHER_CNTL(x) (((x) & 0x7FFFFFFF) << 0) +# define PM4_ACQUIRE_MEM_COHER_CNTL_TC_WB_ACTION_ENA (1 << 18) +# define PM4_ACQUIRE_MEM_COHER_CNTL_TC_ACTION_ENA (1 << 23) +# define PM4_ACQUIRE_MEM_COHER_CNTL_SH_KCACHE_ACTION_ENA (1 << 27) +# define PM4_ACQUIRE_MEM_COHER_CNTL_SH_ICACHE_ACTION_ENA (1 << 29) +#define PM4_ACQUIRE_MEM_DW2_COHER_SIZE(x) (((x) & 0xFFFFFFFF) << 0) +#define PM4_ACQUIRE_MEM_DW3_COHER_SIZE_HI(x) (((x) & 0xFF) << 0) + +#define PM4_RELEASE_MEM_DW1_EVENT_INDEX(x) (((x) & 0xF) << 8) +# define PM4_RELEASE_MEM_EVENT_INDEX_AQL 0x7 + +#endif // header guard diff --git a/runtime/hsa-runtime/core/inc/host_queue.h b/runtime/hsa-runtime/core/inc/host_queue.h index d13a2e9028..091f40c938 100644 --- a/runtime/hsa-runtime/core/inc/host_queue.h +++ b/runtime/hsa-runtime/core/inc/host_queue.h @@ -143,6 +143,10 @@ class HostQueue : public Queue { return HSA_STATUS_ERROR; } + void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) override { + assert(false && "HostQueue::ExecutePM4 is unimplemented"); + } + bool active() const { return active_; } void* operator new(size_t size) { diff --git a/runtime/hsa-runtime/core/inc/queue.h b/runtime/hsa-runtime/core/inc/queue.h index c1a56bded5..bc31c7b579 100644 --- a/runtime/hsa-runtime/core/inc/queue.h +++ b/runtime/hsa-runtime/core/inc/queue.h @@ -300,6 +300,9 @@ class Queue : public Checked<0xFA3906A679F9DB49>, virtual hsa_status_t SetCUMasking(const uint32_t num_cu_mask_count, const uint32_t* cu_mask) = 0; + // @brief Submits a block of PM4 and waits until it has been executed. + virtual void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) = 0; + // Handle of AMD Queue struct amd_queue_t& amd_queue_; diff --git a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp index 3999fd5568..b5f94b9a0a 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp @@ -65,6 +65,7 @@ #include "core/inc/registers.h" #include "core/inc/interrupt_signal.h" #include "core/inc/hsa_ext_amd_impl.h" +#include "core/inc/amd_gpu_pm4.h" namespace amd { // Queue::amd_queue_ is cache-aligned for performance. @@ -95,7 +96,9 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, queue_scratch_(scratch), errors_callback_(callback), errors_data_(err_data), - is_kv_queue_(is_kv) { + is_kv_queue_(is_kv), + pm4_ib_buf_(NULL), + pm4_ib_size_b_(0x1000) { if (!Queue::Shared::IsSharedObjectAllocationValid()) { return; } @@ -263,9 +266,18 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id, SignalGuard.Dismiss(); #endif + pm4_ib_buf_ = core::Runtime::runtime_singleton_->system_allocator()( + pm4_ib_size_b_, 0x1000); + if (pm4_ib_buf_ == NULL) return; + + MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() { + core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); + }); + valid_ = true; active_ = 1; + PM4IBGuard.Dismiss(); RingGuard.Dismiss(); QueueGuard.Dismiss(); EventGuard.Dismiss(); @@ -292,6 +304,7 @@ AqlQueue::~AqlQueue() { } } #endif + core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_); } uint64_t AqlQueue::LoadReadIndexAcquire() { @@ -768,6 +781,96 @@ hsa_status_t AqlQueue::SetCUMasking(const uint32_t num_cu_mask_count, return (HSAKMT_STATUS_SUCCESS == ret) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR; } +void AqlQueue::ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) { + // pm4_ib_buf_ is a shared resource, so mutually exclude here. + ScopedAcquire lock(&pm4_ib_mutex_); + + // Obtain a queue slot for a single AQL packet. + uint64_t write_idx = AddWriteIndexAcqRel(1); + + while ((write_idx - LoadReadIndexRelaxed()) > public_handle()->size) { + os::YieldThread(); + } + + uint32_t slot_idx = uint32_t(write_idx % public_handle()->size); + constexpr uint32_t slot_size_b = 0x40; + uint32_t* queue_slot = (uint32_t*)uintptr_t(public_handle()->base_address + + (slot_idx * slot_size_b)); + + // Copy client PM4 command into IB. + assert(cmd_size_b < pm4_ib_size_b_ && "PM4 exceeds IB size"); + memcpy(pm4_ib_buf_, cmd_data, cmd_size_b); + + // Construct a set of PM4 to fit inside the AQL packet slot. + constexpr uint32_t slot_size_dw = uint32_t(slot_size_b / sizeof(uint32_t)); + uint32_t slot_data[slot_size_dw]; + uint32_t slot_dw_idx = 0; + + // Construct a no-op command to pad the queue slot. + constexpr uint32_t ib_jump_size_dw = 4; + constexpr uint32_t rel_mem_size_dw = 7; + constexpr uint32_t nop_pad_size_dw = + slot_size_dw - (ib_jump_size_dw + rel_mem_size_dw); + + uint32_t* nop_pad = &slot_data[slot_dw_idx]; + slot_dw_idx += nop_pad_size_dw; + + nop_pad[0] = PM4_HDR(PM4_HDR_IT_OPCODE_NOP, nop_pad_size_dw, + agent_->isa()->GetMajorVersion()); + + for (int i = 1; i < nop_pad_size_dw; ++i) { + nop_pad[i] = 0; + } + + // Construct a command to execute the IB. + assert(slot_dw_idx + ib_jump_size_dw <= slot_size_dw && + "PM4 exceeded queue slot size"); + uint32_t* ib_jump = &slot_data[slot_dw_idx]; + slot_dw_idx += ib_jump_size_dw; + + ib_jump[0] = PM4_HDR(PM4_HDR_IT_OPCODE_INDIRECT_BUFFER, ib_jump_size_dw, + agent_->isa()->GetMajorVersion()); + ib_jump[1] = + PM4_INDIRECT_BUFFER_DW1_IB_BASE_LO(uint32_t(uintptr_t(pm4_ib_buf_) >> 2)); + ib_jump[2] = PM4_INDIRECT_BUFFER_DW2_IB_BASE_HI( + uint32_t(uintptr_t(pm4_ib_buf_) >> 32)); + ib_jump[3] = + PM4_INDIRECT_BUFFER_DW3_IB_SIZE(uint32_t(cmd_size_b / sizeof(uint32_t))) | + PM4_INDIRECT_BUFFER_DW3_IB_VALID(1); + + // Construct a command to advance the read index and invalidate the packet + // header. This must be the last command since this releases the queue slot + // for writing. + assert(slot_dw_idx + rel_mem_size_dw <= slot_size_dw && + "PM4 exceeded queue slot size"); + uint32_t* rel_mem = &slot_data[slot_dw_idx]; + + rel_mem[0] = PM4_HDR(PM4_HDR_IT_OPCODE_RELEASE_MEM, rel_mem_size_dw, + agent_->isa()->GetMajorVersion()); + rel_mem[1] = PM4_RELEASE_MEM_DW1_EVENT_INDEX(PM4_RELEASE_MEM_EVENT_INDEX_AQL); + rel_mem[2] = 0; + rel_mem[3] = 0; + rel_mem[4] = 0; + rel_mem[5] = 0; + rel_mem[6] = 0; + + // Copy all PM4 commands into the queue slot. + // Overwrite the AQL invalid header (first dword) last. + // This prevents the slot from being read until it's fully written. + memcpy(&queue_slot[1], &slot_data[1], slot_size_b - sizeof(uint32_t)); + atomic::Store(&queue_slot[0], slot_data[0], std::memory_order_release); + + // Submit the packet slot. + core::Signal* doorbell = + core::Signal::Convert(public_handle()->doorbell_signal); + doorbell->StoreRelease(write_idx); + + // Wait for the packet to be consumed. + while (LoadReadIndexRelaxed() <= write_idx) { + os::YieldThread(); + } +} + // @brief Define the Scratch Buffer Descriptor and related parameters // that enable kernel access scratch memory void AqlQueue::InitScratchSRD() { diff --git a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 20e4dc9000..caff6a24aa 100644 --- a/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -53,6 +53,7 @@ #include "core/inc/amd_aql_queue.h" #include "core/inc/amd_blit_kernel.h" #include "core/inc/amd_blit_sdma.h" +#include "core/inc/amd_gpu_pm4.h" #include "core/inc/amd_gpu_shaders.h" #include "core/inc/amd_memory_region.h" #include "core/inc/interrupt_signal.h" @@ -1077,4 +1078,42 @@ void GpuAgent::BindTrapHandler() { assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtSetTrapHandler() failed"); } +void GpuAgent::InvalidateCodeCaches() { + // Check for microcode cache invalidation support. + // This is deprecated in later microcode builds. + if (isa_->GetMajorVersion() == 7) { + if (properties_.EngineId.ui32.uCode < 420) { + // Microcode is handling code cache invalidation. + return; + } + } else if (isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0) { + if (properties_.EngineId.ui32.uCode < 685) { + // Microcode is handling code cache invalidation. + return; + } + } else { + assert(false && "Code cache invalidation not implemented for this agent"); + } + + // Invalidate caches which may hold lines of code object allocation. + constexpr uint32_t cache_inv_size_dw = 7; + uint32_t cache_inv[cache_inv_size_dw]; + + cache_inv[0] = PM4_HDR(PM4_HDR_IT_OPCODE_ACQUIRE_MEM, cache_inv_size_dw, + isa_->GetMajorVersion()); + cache_inv[1] = PM4_ACQUIRE_MEM_DW1_COHER_CNTL( + PM4_ACQUIRE_MEM_COHER_CNTL_SH_ICACHE_ACTION_ENA | + PM4_ACQUIRE_MEM_COHER_CNTL_SH_KCACHE_ACTION_ENA | + PM4_ACQUIRE_MEM_COHER_CNTL_TC_ACTION_ENA | + PM4_ACQUIRE_MEM_COHER_CNTL_TC_WB_ACTION_ENA); + cache_inv[2] = PM4_ACQUIRE_MEM_DW2_COHER_SIZE(0xFFFFFFFF); + cache_inv[3] = PM4_ACQUIRE_MEM_DW3_COHER_SIZE_HI(0xFF); + cache_inv[4] = 0; + cache_inv[5] = 0; + cache_inv[6] = 0; + + // Submit the command to the utility queue and wait for it to complete. + queues_[QueueUtility]->ExecutePM4(cache_inv, sizeof(cache_inv)); +} + } // namespace diff --git a/runtime/hsa-runtime/core/runtime/amd_loader_context.cpp b/runtime/hsa-runtime/core/runtime/amd_loader_context.cpp index 3927d67824..8a51b1d243 100644 --- a/runtime/hsa-runtime/core/runtime/amd_loader_context.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_loader_context.cpp @@ -465,6 +465,10 @@ void* LoaderContext::SegmentAlloc(amdgpu_hsa_elf_segment_t segment, default: assert(false); } + + // Invalidate agent caches which may hold lines of the new allocation. + ((GpuAgentInt*)core::Agent::Convert(agent))->InvalidateCodeCaches(); + break; default: assert(false);