AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Invalidate caches after allocating a code object

Browse Source 
Due to a misinterpretation of the HSA specification the microcode has,
until now, been responsible for ensuring a coherent view of the
amd_kernel_code_t object when acquire_fence_scope is set to agent or system.
To correct this the runtime must instead assume this responsibility.

Introduce GpuAgentInt::InvalidateCodeCaches to perform this operation
on-demand. Invoke this after code object allocation. Extend the Queue
implementations to support PM4 command submission, through which the
PM4 command ACQUIRE_MEM can be submitted to perform cache invalidation.
Submit through a runtime-managed queue shared with the blit implementation.

This change depends on microcode support and this is checked against the
running version. Older microcode builds will perform cache invalidation
themselves, so it is acceptable for this change to do nothing in that case.

Change-Id: I268dd2b83af3decdd9ad07430a81df8a2ecb6bd2
This commit is contained in:
Jay Cornwall
2016-07-27 12:39:40 -05:00
parent d2a4629c55
commit f76577ae43
8 changed files with 253 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/hsa-runtime/core/inc/amd_aql_queue.h
+9
View File
@@ -47,6 +47,7 @@
#include "core/inc/signal.h"
#include "core/inc/queue.h"
#include "core/inc/amd_gpu_agent.h"
#include "core/util/locks.h"
namespace amd {
/// @brief Encapsulates HW Aql Command Processor functionality. It
@@ -181,6 +182,9 @@ class AqlQueue : public core::Queue, public core::Signal {
hsa_status_t SetCUMasking(const uint32_t num_cu_mask_count,
const uint32_t* cu_mask);
// @brief Submits a block of PM4 and waits until it has been executed.
void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) override;
/// @brief This operation is illegal
hsa_signal_value_t LoadRelaxed() {
assert(false);
@@ -396,6 +400,11 @@ class AqlQueue : public core::Queue, public core::Signal {
// Is KV device queue
bool is_kv_queue_;
// GPU-visible indirect buffer holding PM4 commands.
void* pm4_ib_buf_;
uint32_t pm4_ib_size_b_;
KernelMutex pm4_ib_mutex_;
// Shared event used for queue errors
static HsaEvent* queue_event_;
runtime/hsa-runtime/core/inc/amd_gpu_agent.h
+6 -1
View File
@@ -136,6 +136,9 @@ class GpuAgentInt : public core::Agent {
// @param [out] time Timestamp in agent domain.
virtual uint64_t TranslateTime(uint64_t tick) = 0;
// @brief Invalidate caches on the agent which may hold code object data.
virtual void InvalidateCodeCaches() = 0;
// @brief Sets the coherency type of this agent.
//
// @param [in] type New coherency type.
@@ -258,9 +261,11 @@ class GpuAgent : public GpuAgentInt {
uint64_t TranslateTime(uint64_t tick) override;
// @brief Override from amd::GpuAgentInt.
bool current_coherency_type(hsa_amd_coherency_type_t type) override;
void InvalidateCodeCaches() override;
// @brief Override from amd::GpuAgentInt.
bool current_coherency_type(hsa_amd_coherency_type_t type) override;
hsa_amd_coherency_type_t current_coherency_type() const override {
return current_coherency_type_;
}
runtime/hsa-runtime/core/inc/amd_gpu_pm4.h
+84
View File
@@ -0,0 +1,84 @@
////////////////////////////////////////////////////////////////////////////////
//
// The University of Illinois/NCSA
// Open Source License (NCSA)
//
// Copyright (c) 2014-2015, Advanced Micro Devices, Inc. All rights reserved.
//
// Developed by:
//
// AMD Research and AMD HSA Software Development
//
// Advanced Micro Devices, Inc.
//
// www.amd.com
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to
// deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// - Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// - Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimers in
// the documentation and/or other materials provided with the distribution.
// - Neither the names of Advanced Micro Devices, Inc,
// nor the names of its contributors may be used to endorse or promote
// products derived from this Software without specific prior written
// permission.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
// THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS WITH THE SOFTWARE.
//
////////////////////////////////////////////////////////////////////////////////
#ifndef HSA_RUNTIME_CORE_INC_AMD_GPU_PM4_H_
#define HSA_RUNTIME_CORE_INC_AMD_GPU_PM4_H_
// PM4 definitions:
// http://amd-dev.wpengine.netdna-cdn.com/wordpress/media/2013/10/si_programming_guide_v2.pdf
//
// ACQUIRE_MEM was introduced in Gfx7.
#define PM4_HDR_IT_OPCODE_NOP 0x10
#define PM4_HDR_IT_OPCODE_INDIRECT_BUFFER 0x3F
#define PM4_HDR_IT_OPCODE_RELEASE_MEM 0x49
#define PM4_HDR_IT_OPCODE_ACQUIRE_MEM 0x58
#define PM4_HDR_SHADER_TYPE(x) (((x) & 0x1) << 1)
#define PM4_HDR_IT_OPCODE(x) (((x) & 0xFF) << 8)
#define PM4_HDR_COUNT(x) (((x) & 0x3FFF) << 16)
#define PM4_HDR_TYPE(x) (((x) & 0x3) << 30)
#define PM4_HDR(it_opcode, pkt_size_dw, gfxip_ver) ( \
PM4_HDR_SHADER_TYPE((gfxip_ver) == 7 ? 1 : 0) | \
PM4_HDR_IT_OPCODE(it_opcode) | \
PM4_HDR_COUNT(int(pkt_size_dw) - 2) | \
PM4_HDR_TYPE(3) \
)
#define PM4_INDIRECT_BUFFER_DW1_IB_BASE_LO(x) (((x) & 0x3FFFFFFF) << 2)
#define PM4_INDIRECT_BUFFER_DW2_IB_BASE_HI(x) (((x) & 0xFFFF) << 0)
#define PM4_INDIRECT_BUFFER_DW3_IB_SIZE(x) (((x) & 0xFFFFF) << 0)
#define PM4_INDIRECT_BUFFER_DW3_IB_VALID(x) (((x) & 0x1) << 23)
#define PM4_ACQUIRE_MEM_DW1_COHER_CNTL(x) (((x) & 0x7FFFFFFF) << 0)
# define PM4_ACQUIRE_MEM_COHER_CNTL_TC_WB_ACTION_ENA (1 << 18)
# define PM4_ACQUIRE_MEM_COHER_CNTL_TC_ACTION_ENA (1 << 23)
# define PM4_ACQUIRE_MEM_COHER_CNTL_SH_KCACHE_ACTION_ENA (1 << 27)
# define PM4_ACQUIRE_MEM_COHER_CNTL_SH_ICACHE_ACTION_ENA (1 << 29)
#define PM4_ACQUIRE_MEM_DW2_COHER_SIZE(x) (((x) & 0xFFFFFFFF) << 0)
#define PM4_ACQUIRE_MEM_DW3_COHER_SIZE_HI(x) (((x) & 0xFF) << 0)
#define PM4_RELEASE_MEM_DW1_EVENT_INDEX(x) (((x) & 0xF) << 8)
# define PM4_RELEASE_MEM_EVENT_INDEX_AQL 0x7
#endif // header guard
runtime/hsa-runtime/core/inc/host_queue.h
+4
View File
@@ -143,6 +143,10 @@ class HostQueue : public Queue {
return HSA_STATUS_ERROR;
}
void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) override {
assert(false && "HostQueue::ExecutePM4 is unimplemented");
}
bool active() const { return active_; }
void* operator new(size_t size) {
runtime/hsa-runtime/core/inc/queue.h
+3
View File
@@ -300,6 +300,9 @@ class Queue : public Checked<0xFA3906A679F9DB49>,
virtual hsa_status_t SetCUMasking(const uint32_t num_cu_mask_count,
const uint32_t* cu_mask) = 0;
// @brief Submits a block of PM4 and waits until it has been executed.
virtual void ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) = 0;
// Handle of AMD Queue struct
amd_queue_t& amd_queue_;
runtime/hsa-runtime/core/runtime/amd_aql_queue.cpp
+104 -1
View File
@@ -65,6 +65,7 @@
#include "core/inc/registers.h"
#include "core/inc/interrupt_signal.h"
#include "core/inc/hsa_ext_amd_impl.h"
#include "core/inc/amd_gpu_pm4.h"
namespace amd {
// Queue::amd_queue_ is cache-aligned for performance.
@@ -95,7 +96,9 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id,
queue_scratch_(scratch),
errors_callback_(callback),
errors_data_(err_data),
is_kv_queue_(is_kv) {
is_kv_queue_(is_kv),
pm4_ib_buf_(NULL),
pm4_ib_size_b_(0x1000) {
if (!Queue::Shared::IsSharedObjectAllocationValid()) {
return;
}
@@ -263,9 +266,18 @@ AqlQueue::AqlQueue(GpuAgent* agent, size_t req_size_pkts, HSAuint32 node_id,
SignalGuard.Dismiss();
#endif
pm4_ib_buf_ = core::Runtime::runtime_singleton_->system_allocator()(
pm4_ib_size_b_, 0x1000);
if (pm4_ib_buf_ == NULL) return;
MAKE_NAMED_SCOPE_GUARD(PM4IBGuard, [&]() {
core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_);
});
valid_ = true;
active_ = 1;
PM4IBGuard.Dismiss();
RingGuard.Dismiss();
QueueGuard.Dismiss();
EventGuard.Dismiss();
@@ -292,6 +304,7 @@ AqlQueue::~AqlQueue() {
}
}
#endif
core::Runtime::runtime_singleton_->system_deallocator()(pm4_ib_buf_);
}
uint64_t AqlQueue::LoadReadIndexAcquire() {
@@ -768,6 +781,96 @@ hsa_status_t AqlQueue::SetCUMasking(const uint32_t num_cu_mask_count,
return (HSAKMT_STATUS_SUCCESS == ret) ? HSA_STATUS_SUCCESS : HSA_STATUS_ERROR;
}
void AqlQueue::ExecutePM4(uint32_t* cmd_data, size_t cmd_size_b) {
// pm4_ib_buf_ is a shared resource, so mutually exclude here.
ScopedAcquire<KernelMutex> lock(&pm4_ib_mutex_);
// Obtain a queue slot for a single AQL packet.
uint64_t write_idx = AddWriteIndexAcqRel(1);
while ((write_idx - LoadReadIndexRelaxed()) > public_handle()->size) {
os::YieldThread();
}
uint32_t slot_idx = uint32_t(write_idx % public_handle()->size);
constexpr uint32_t slot_size_b = 0x40;
uint32_t* queue_slot = (uint32_t*)uintptr_t(public_handle()->base_address +
(slot_idx * slot_size_b));
// Copy client PM4 command into IB.
assert(cmd_size_b < pm4_ib_size_b_ && "PM4 exceeds IB size");
memcpy(pm4_ib_buf_, cmd_data, cmd_size_b);
// Construct a set of PM4 to fit inside the AQL packet slot.
constexpr uint32_t slot_size_dw = uint32_t(slot_size_b / sizeof(uint32_t));
uint32_t slot_data[slot_size_dw];
uint32_t slot_dw_idx = 0;
// Construct a no-op command to pad the queue slot.
constexpr uint32_t ib_jump_size_dw = 4;
constexpr uint32_t rel_mem_size_dw = 7;
constexpr uint32_t nop_pad_size_dw =
slot_size_dw - (ib_jump_size_dw + rel_mem_size_dw);
uint32_t* nop_pad = &slot_data[slot_dw_idx];
slot_dw_idx += nop_pad_size_dw;
nop_pad[0] = PM4_HDR(PM4_HDR_IT_OPCODE_NOP, nop_pad_size_dw,
agent_->isa()->GetMajorVersion());
for (int i = 1; i < nop_pad_size_dw; ++i) {
nop_pad[i] = 0;
}
// Construct a command to execute the IB.
assert(slot_dw_idx + ib_jump_size_dw <= slot_size_dw &&
"PM4 exceeded queue slot size");
uint32_t* ib_jump = &slot_data[slot_dw_idx];
slot_dw_idx += ib_jump_size_dw;
ib_jump[0] = PM4_HDR(PM4_HDR_IT_OPCODE_INDIRECT_BUFFER, ib_jump_size_dw,
agent_->isa()->GetMajorVersion());
ib_jump[1] =
PM4_INDIRECT_BUFFER_DW1_IB_BASE_LO(uint32_t(uintptr_t(pm4_ib_buf_) >> 2));
ib_jump[2] = PM4_INDIRECT_BUFFER_DW2_IB_BASE_HI(
uint32_t(uintptr_t(pm4_ib_buf_) >> 32));
ib_jump[3] =
PM4_INDIRECT_BUFFER_DW3_IB_SIZE(uint32_t(cmd_size_b / sizeof(uint32_t))) |
PM4_INDIRECT_BUFFER_DW3_IB_VALID(1);
// Construct a command to advance the read index and invalidate the packet
// header. This must be the last command since this releases the queue slot
// for writing.
assert(slot_dw_idx + rel_mem_size_dw <= slot_size_dw &&
"PM4 exceeded queue slot size");
uint32_t* rel_mem = &slot_data[slot_dw_idx];
rel_mem[0] = PM4_HDR(PM4_HDR_IT_OPCODE_RELEASE_MEM, rel_mem_size_dw,
agent_->isa()->GetMajorVersion());
rel_mem[1] = PM4_RELEASE_MEM_DW1_EVENT_INDEX(PM4_RELEASE_MEM_EVENT_INDEX_AQL);
rel_mem[2] = 0;
rel_mem[3] = 0;
rel_mem[4] = 0;
rel_mem[5] = 0;
rel_mem[6] = 0;
// Copy all PM4 commands into the queue slot.
// Overwrite the AQL invalid header (first dword) last.
// This prevents the slot from being read until it's fully written.
memcpy(&queue_slot[1], &slot_data[1], slot_size_b - sizeof(uint32_t));
atomic::Store(&queue_slot[0], slot_data[0], std::memory_order_release);
// Submit the packet slot.
core::Signal* doorbell =
core::Signal::Convert(public_handle()->doorbell_signal);
doorbell->StoreRelease(write_idx);
// Wait for the packet to be consumed.
while (LoadReadIndexRelaxed() <= write_idx) {
os::YieldThread();
}
}
// @brief Define the Scratch Buffer Descriptor and related parameters
// that enable kernel access scratch memory
void AqlQueue::InitScratchSRD() {
runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+39
View File
@@ -53,6 +53,7 @@
#include "core/inc/amd_aql_queue.h"
#include "core/inc/amd_blit_kernel.h"
#include "core/inc/amd_blit_sdma.h"
#include "core/inc/amd_gpu_pm4.h"
#include "core/inc/amd_gpu_shaders.h"
#include "core/inc/amd_memory_region.h"
#include "core/inc/interrupt_signal.h"
@@ -1077,4 +1078,42 @@ void GpuAgent::BindTrapHandler() {
assert(err == HSAKMT_STATUS_SUCCESS && "hsaKmtSetTrapHandler() failed");
}
void GpuAgent::InvalidateCodeCaches() {
// Check for microcode cache invalidation support.
// This is deprecated in later microcode builds.
if (isa_->GetMajorVersion() == 7) {
if (properties_.EngineId.ui32.uCode < 420) {
// Microcode is handling code cache invalidation.
return;
}
} else if (isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0) {
if (properties_.EngineId.ui32.uCode < 685) {
// Microcode is handling code cache invalidation.
return;
}
} else {
assert(false && "Code cache invalidation not implemented for this agent");
}
// Invalidate caches which may hold lines of code object allocation.
constexpr uint32_t cache_inv_size_dw = 7;
uint32_t cache_inv[cache_inv_size_dw];
cache_inv[0] = PM4_HDR(PM4_HDR_IT_OPCODE_ACQUIRE_MEM, cache_inv_size_dw,
isa_->GetMajorVersion());
cache_inv[1] = PM4_ACQUIRE_MEM_DW1_COHER_CNTL(
PM4_ACQUIRE_MEM_COHER_CNTL_SH_ICACHE_ACTION_ENA |
PM4_ACQUIRE_MEM_COHER_CNTL_SH_KCACHE_ACTION_ENA |
PM4_ACQUIRE_MEM_COHER_CNTL_TC_ACTION_ENA |
PM4_ACQUIRE_MEM_COHER_CNTL_TC_WB_ACTION_ENA);
cache_inv[2] = PM4_ACQUIRE_MEM_DW2_COHER_SIZE(0xFFFFFFFF);
cache_inv[3] = PM4_ACQUIRE_MEM_DW3_COHER_SIZE_HI(0xFF);
cache_inv[4] = 0;
cache_inv[5] = 0;
cache_inv[6] = 0;
// Submit the command to the utility queue and wait for it to complete.
queues_[QueueUtility]->ExecutePM4(cache_inv, sizeof(cache_inv));
}
} // namespace
runtime/hsa-runtime/core/runtime/amd_loader_context.cpp
+4
View File
@@ -465,6 +465,10 @@ void* LoaderContext::SegmentAlloc(amdgpu_hsa_elf_segment_t segment,
default:
assert(false);
}
// Invalidate agent caches which may hold lines of the new allocation.
((GpuAgentInt*)core::Agent::Convert(agent))->InvalidateCodeCaches();
break;
default:
assert(false);