AI/rocm-systems
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable sDMA packet HDP Flush on Gfx9 and later devices

Browse Source 
Change-Id: I85922e5266883ef7e9eed3565e2c3b209009d294
This commit is contained in:
Ramesh Errabolu committed 2018-04-02 11:47:59 -04:00
1 parent e4c6a3ae28
commit 987f3f97aa
5 files changed
+175 -44
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp
+123 -15
View File
@@ -360,6 +360,20 @@ typedef struct SDMA_PKT_TRAP_TAG {
} INT_CONTEXT_UNION;
} SDMA_PKT_TRAP;
// Initialize Hdp flush packet for use on sDMA of devices
// from Gfx9 or new family
static const SDMA_PKT_POLL_REGMEM hdp_flush_cmd_ {
{ SDMA_OP_POLL_REGMEM },
{ 0x00 },
{ 0x80000000 },
{ 0x00 },
{ 0x00 },
{ 0x00 },
};
// Version of sDMA microcode supporting Hdp flush
static const uint16_t sdma_version_ = 0x01A5;
inline uint32_t ptrlow32(const void* p) {
return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
}
@@ -377,8 +391,33 @@ const size_t BlitSdmaBase::kCopyPacketSize = sizeof(SDMA_PKT_COPY_LINEAR);
const size_t BlitSdmaBase::kMaxSingleCopySize = 0x3fffe0; // From HW documentation
const size_t BlitSdmaBase::kMaxSingleFillSize = 0x3fffe0;
// Initialize size of various sDMA commands use by this module
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma()
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::fence_command_size_ = sizeof(SDMA_PKT_FENCE);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::flush_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::trap_command_size_ = sizeof(SDMA_PKT_TRAP);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma(bool copy_direction)
: agent_(NULL),
queue_start_addr_(NULL),
fence_base_addr_(NULL),
@@ -386,7 +425,9 @@ BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma()
fence_pool_counter_(0),
cached_reserve_index_(0),
cached_commit_index_(0),
platform_atomic_support_(true) {
sdma_h2d_(copy_direction),
platform_atomic_support_(true),
hdp_flush_support_(false) {
std::memset(&queue_resource_, 0, sizeof(queue_resource_));
}
@@ -407,14 +448,6 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::Initial
return HSA_STATUS_ERROR;
}
linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR);
fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL);
fence_command_size_ = sizeof(SDMA_PKT_FENCE);
poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC);
timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP);
trap_command_size_ = sizeof(SDMA_PKT_TRAP);
const amd::GpuAgentInt& amd_gpu_agent =
static_cast<const amd::GpuAgentInt&>(agent);
@@ -428,6 +461,11 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::Initial
platform_atomic_support_ = false;
}
// Determine if sDMA microcode supports HDP flush command
if (agent_->GetSdmaMicrocodeVersion() >= sdma_version_) {
hdp_flush_support_ = true;
}
// Allocate queue buffer.
queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()(
kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable);
@@ -508,8 +546,16 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
const uint32_t total_copy_command_size =
num_copy_command * linear_copy_command_size_;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_copy_command_size + fence_command_size_;
total_copy_command_size + fence_command_size_ + flush_cmd_size;
const uint32_t kFenceValue = 2015;
uint32_t* fence_addr = ObtainFenceObject();
@@ -522,10 +568,25 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
}
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
// Determine if a Hdp flush cmd is required at the top of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
command_addr += total_copy_command_size;
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
BuildFenceCommand(command_addr, fence_addr, kFenceValue);
ReleaseWriteAddress(curr_index, total_command_size);
@@ -593,9 +654,17 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
? (fence_command_size_ + trap_command_size_)
: 0;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_poll_command_size + total_copy_command_size + sync_command_size +
total_timestamp_command_size + interrupt_command_size;
total_timestamp_command_size + interrupt_command_size + flush_cmd_size;
RingIndexTy curr_index;
char* command_addr = AcquireWriteAddress(total_command_size, curr_index);
@@ -621,11 +690,26 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
command_addr += timestamp_command_size_;
}
// Determine if a Hdp flush cmd is required at the top of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
// Do the transfer after all polls are satisfied.
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
command_addr += total_copy_command_size;
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
if (profiling_enabled) {
assert(IsMultipleOf(end_ts_addr, 32));
BuildGetGlobalTimestampCommand(command_addr,
@@ -685,8 +769,16 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
const uint32_t total_fill_command_size =
num_fill_command * fill_command_size_;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_fill_command_size + fence_command_size_;
total_fill_command_size + fence_command_size_ + flush_cmd_size;
RingIndexTy curr_index;
char* command_addr = AcquireWriteAddress(total_command_size, curr_index);
@@ -724,6 +816,14 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
assert(cur_size == size);
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
const uint32_t kFenceValue = 2015;
uint32_t* fence_addr = ObtainFenceObject();
*fence_addr = 0;
@@ -1018,6 +1118,14 @@ void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BuildTrapComman
packet_addr->HEADER_UNION.op = SDMA_OP_TRAP;
}
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BuildHdpFlushCommand(
char* cmd_addr) {
assert(cmd_addr != NULL);
SDMA_PKT_POLL_REGMEM* addr = reinterpret_cast<SDMA_PKT_POLL_REGMEM*>(cmd_addr);
memcpy(addr, &hdp_flush_cmd_, flush_command_size_);
}
template class BlitSdma<uint32_t, false, 0>;
template class BlitSdma<uint64_t, true, -1>;
runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+13 -8
View File
@@ -515,13 +515,13 @@ core::Queue* GpuAgent::CreateInterceptibleQueue() {
return queue;
}
core::Blit* GpuAgent::CreateBlitSdma() {
core::Blit* GpuAgent::CreateBlitSdma(bool h2d) {
core::Blit* sdma;
if (isa_->GetMajorVersion() <= 8) {
sdma = new BlitSdmaV2V3;
sdma = new BlitSdmaV2V3(h2d);
} else {
sdma = new BlitSdmaV4;
sdma = new BlitSdmaV4(h2d);
}
if (sdma->Initialize(*this) != HSA_STATUS_SUCCESS) {
@@ -561,10 +561,10 @@ void GpuAgent::InitDma() {
// Blits, try create SDMA blit first.
// Disable SDMA on specific ISA targets until they are fully qualified.
auto blit_lambda = [this](lazy_ptr<core::Queue>& queue) {
auto blit_lambda = [this](bool h2d, lazy_ptr<core::Queue>& queue) {
if ((isa_->GetMajorVersion() != 8) && core::Runtime::runtime_singleton_->flag().enable_sdma() &&
(HSA_PROFILE_BASE == profile_)) {
auto ret = CreateBlitSdma();
auto ret = CreateBlitSdma(h2d);
if (ret != nullptr) return ret;
}
auto ret = CreateBlitKernel((*queue).get());
@@ -573,8 +573,8 @@ void GpuAgent::InitDma() {
return ret;
};
blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueBlitOnly]); });
blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueUtility]); });
blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(true, queues_[QueueBlitOnly]); });
blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(false, queues_[QueueUtility]); });
blits_[BlitDevToDev].reset([this]() {
auto ret = CreateBlitKernel((*queues_[QueueUtility]).get());
if (ret == nullptr)
@@ -614,7 +614,8 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
: (src_agent.device_type() == core::Agent::kAmdGpuDevice &&
dst_agent.device_type() == core::Agent::kAmdCpuDevice)
? blits_[BlitDevToHost]
: blits_[BlitDevToDev];
: (src_agent.node_id() == dst_agent.node_id())
? blits_[BlitDevToDev] : blits_[BlitDevToHost];
if (profiling_enabled()) {
// Track the agent so we could translate the resulting timestamp to system
@@ -1063,6 +1064,10 @@ uint16_t GpuAgent::GetMicrocodeVersion() const {
return (properties_.EngineId.ui32.uCode);
}
uint16_t GpuAgent::GetSdmaMicrocodeVersion() const {
return (properties_.uCodeEngineVersions.uCodeSDMA);
}
void GpuAgent::SyncClocks() {
HSAKMT_STATUS err = hsaKmtGetClockCounters(node_id(), &t1_);
assert(err == HSAKMT_STATUS_SUCCESS && "hsaGetClockCounters error");