AI/rocm-systems
2
0
Forka 0
Codice Problemi Pull Request Actions Pacchetti Progetti Rilasci Wiki Attività

Enable sDMA packet HDP Flush on Gfx9 and later devices

Sfoglia il codice sorgente 
Change-Id: I85922e5266883ef7e9eed3565e2c3b209009d294


[ROCm/ROCR-Runtime commit: 987f3f97aa]
This commit is contained in:
Ramesh Errabolu
2018-03-12 16:54:21 -05:00
parent 696bb12601
commit de9064fc44
5 ha cambiato i file con 175 aggiunte e 44 eliminazioni
Scarica il file Patch Scarica il file Diff Expand all files Collapse all files
projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h
+29 -19
Vedi File
@@ -69,7 +69,7 @@ class BlitSdmaBase : public core::Blit {
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
class BlitSdma : public BlitSdmaBase {
public:
explicit BlitSdma();
explicit BlitSdma(bool copy_direction);
virtual ~BlitSdma() override;
@@ -125,7 +125,7 @@ class BlitSdma : public BlitSdmaBase {
virtual hsa_status_t EnableProfiling(bool enable) override;
protected:
private:
/// @brief Acquires the address into queue buffer where a new command
/// packet of specified size could be written. The address that is
/// returned is guaranteed to be unique even in a multi-threaded access
@@ -170,6 +170,9 @@ class BlitSdma : public BlitSdmaBase {
void BuildFenceCommand(char* fence_command_addr, uint32_t* fence,
uint32_t fence_value);
/// @brief Build Hdp Flush command
void BuildHdpFlushCommand(char* cmd_addr);
uint32_t* ObtainFenceObject();
void WaitFence(uint32_t* fence, uint32_t fence_value);
@@ -204,19 +207,25 @@ class BlitSdma : public BlitSdmaBase {
RingIndexTy cached_reserve_index_;
RingIndexTy cached_commit_index_;
uint32_t linear_copy_command_size_;
static const uint32_t linear_copy_command_size_;
uint32_t fill_command_size_;
static const uint32_t fill_command_size_;
uint32_t fence_command_size_;
static const uint32_t fence_command_size_;
uint32_t poll_command_size_;
static const uint32_t poll_command_size_;
uint32_t atomic_command_size_;
static const uint32_t flush_command_size_;
uint32_t timestamp_command_size_;
static const uint32_t atomic_command_size_;
uint32_t trap_command_size_;
static const uint32_t timestamp_command_size_;
static const uint32_t trap_command_size_;
// Flag to indicate if sDMA queue is used for H2D copy operations
// true if used for H2D operations, false otherwise
const bool sdma_h2d_;
// Max copy size of a single linear copy command packet.
size_t max_single_linear_copy_size_;
@@ -232,19 +241,20 @@ class BlitSdma : public BlitSdmaBase {
/// True if platform atomic is supported.
bool platform_atomic_support_;
/// True if sDMA supports HDP flush
bool hdp_flush_support_;
};
class BlitSdmaV2V3
// Ring indices are 32-bit.
// HW ring indices are not monotonic (wrap at end of ring).
// Count fields of SDMA commands are 0-based.
: public BlitSdma<uint32_t, false, 0> {};
// Ring indices are 32-bit.
// HW ring indices are not monotonic (wrap at end of ring).
// Count fields of SDMA commands are 0-based.
typedef BlitSdma<uint32_t, false, 0> BlitSdmaV2V3;
class BlitSdmaV4
// Ring indices are 64-bit.
// HW ring indices are monotonic (do not wrap at end of ring).
// Count fields of SDMA commands are 1-based.
: public BlitSdma<uint64_t, true, -1> {};
// Ring indices are 64-bit.
// HW ring indices are monotonic (do not wrap at end of ring).
// Count fields of SDMA commands are 1-based.
typedef BlitSdma<uint64_t, true, -1> BlitSdmaV4;
} // namespace amd
projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
+3 -1
Vedi File
@@ -192,6 +192,8 @@ class GpuAgent : public GpuAgentInt {
uint16_t GetMicrocodeVersion() const;
uint16_t GetSdmaMicrocodeVersion() const;
// @brief Assembles SP3 shader source into ISA or AQL code object.
//
// @param [in] src_sp3 SP3 shader source text representation.
@@ -325,7 +327,7 @@ class GpuAgent : public GpuAgentInt {
// @brief Create SDMA blit object.
//
// @retval NULL if SDMA blit creation and initialization failed.
core::Blit* CreateBlitSdma();
core::Blit* CreateBlitSdma(bool h2d);
// @brief Create Kernel blit object using provided compute queue.
//
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp
+123 -15
Vedi File
@@ -360,6 +360,20 @@ typedef struct SDMA_PKT_TRAP_TAG {
} INT_CONTEXT_UNION;
} SDMA_PKT_TRAP;
// Initialize Hdp flush packet for use on sDMA of devices
// from Gfx9 or new family
static const SDMA_PKT_POLL_REGMEM hdp_flush_cmd_ {
{ SDMA_OP_POLL_REGMEM },
{ 0x00 },
{ 0x80000000 },
{ 0x00 },
{ 0x00 },
{ 0x00 },
};
// Version of sDMA microcode supporting Hdp flush
static const uint16_t sdma_version_ = 0x01A5;
inline uint32_t ptrlow32(const void* p) {
return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
}
@@ -377,8 +391,33 @@ const size_t BlitSdmaBase::kCopyPacketSize = sizeof(SDMA_PKT_COPY_LINEAR);
const size_t BlitSdmaBase::kMaxSingleCopySize = 0x3fffe0; // From HW documentation
const size_t BlitSdmaBase::kMaxSingleFillSize = 0x3fffe0;
// Initialize size of various sDMA commands use by this module
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma()
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::fence_command_size_ = sizeof(SDMA_PKT_FENCE);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::flush_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
const uint32_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::trap_command_size_ = sizeof(SDMA_PKT_TRAP);
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma(bool copy_direction)
: agent_(NULL),
queue_start_addr_(NULL),
fence_base_addr_(NULL),
@@ -386,7 +425,9 @@ BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BlitSdma()
fence_pool_counter_(0),
cached_reserve_index_(0),
cached_commit_index_(0),
platform_atomic_support_(true) {
sdma_h2d_(copy_direction),
platform_atomic_support_(true),
hdp_flush_support_(false) {
std::memset(&queue_resource_, 0, sizeof(queue_resource_));
}
@@ -407,14 +448,6 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::Initial
return HSA_STATUS_ERROR;
}
linear_copy_command_size_ = sizeof(SDMA_PKT_COPY_LINEAR);
fill_command_size_ = sizeof(SDMA_PKT_CONSTANT_FILL);
fence_command_size_ = sizeof(SDMA_PKT_FENCE);
poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC);
timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP);
trap_command_size_ = sizeof(SDMA_PKT_TRAP);
const amd::GpuAgentInt& amd_gpu_agent =
static_cast<const amd::GpuAgentInt&>(agent);
@@ -428,6 +461,11 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::Initial
platform_atomic_support_ = false;
}
// Determine if sDMA microcode supports HDP flush command
if (agent_->GetSdmaMicrocodeVersion() >= sdma_version_) {
hdp_flush_support_ = true;
}
// Allocate queue buffer.
queue_start_addr_ = (char*)core::Runtime::runtime_singleton_->system_allocator()(
kQueueSize, 0x1000, core::MemoryRegion::AllocateExecutable);
@@ -508,8 +546,16 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
const uint32_t total_copy_command_size =
num_copy_command * linear_copy_command_size_;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_copy_command_size + fence_command_size_;
total_copy_command_size + fence_command_size_ + flush_cmd_size;
const uint32_t kFenceValue = 2015;
uint32_t* fence_addr = ObtainFenceObject();
@@ -522,10 +568,25 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
}
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
// Determine if a Hdp flush cmd is required at the top of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
command_addr += total_copy_command_size;
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
BuildFenceCommand(command_addr, fence_addr, kFenceValue);
ReleaseWriteAddress(curr_index, total_command_size);
@@ -593,9 +654,17 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
? (fence_command_size_ + trap_command_size_)
: 0;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_poll_command_size + total_copy_command_size + sync_command_size +
total_timestamp_command_size + interrupt_command_size;
total_timestamp_command_size + interrupt_command_size + flush_cmd_size;
RingIndexTy curr_index;
char* command_addr = AcquireWriteAddress(total_command_size, curr_index);
@@ -621,11 +690,26 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
command_addr += timestamp_command_size_;
}
// Determine if a Hdp flush cmd is required at the top of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_ == false)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
// Do the transfer after all polls are satisfied.
BuildCopyCommand(command_addr, num_copy_command, dst, src, size);
command_addr += total_copy_command_size;
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_) && (sdma_h2d_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
if (profiling_enabled) {
assert(IsMultipleOf(end_ts_addr, 32));
BuildGetGlobalTimestampCommand(command_addr,
@@ -685,8 +769,16 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
const uint32_t total_fill_command_size =
num_fill_command * fill_command_size_;
// Add space for acquire or release Hdp flush command
uint32_t flush_cmd_size = 0;
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if (HwIndexMonotonic) {
flush_cmd_size = flush_command_size_;
}
}
const uint32_t total_command_size =
total_fill_command_size + fence_command_size_;
total_fill_command_size + fence_command_size_ + flush_cmd_size;
RingIndexTy curr_index;
char* command_addr = AcquireWriteAddress(total_command_size, curr_index);
@@ -724,6 +816,14 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::SubmitL
assert(cur_size == size);
// Determine if a Hdp flush cmd is required at the end of cmd stream
if (core::Runtime::runtime_singleton_->flag().enable_sdma_hdp_flush()) {
if ((HwIndexMonotonic) && (hdp_flush_support_)) {
BuildHdpFlushCommand(command_addr);
command_addr += flush_command_size_;
}
}
const uint32_t kFenceValue = 2015;
uint32_t* fence_addr = ObtainFenceObject();
*fence_addr = 0;
@@ -1018,6 +1118,14 @@ void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BuildTrapComman
packet_addr->HEADER_UNION.op = SDMA_OP_TRAP;
}
template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset>
void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset>::BuildHdpFlushCommand(
char* cmd_addr) {
assert(cmd_addr != NULL);
SDMA_PKT_POLL_REGMEM* addr = reinterpret_cast<SDMA_PKT_POLL_REGMEM*>(cmd_addr);
memcpy(addr, &hdp_flush_cmd_, flush_command_size_);
}
template class BlitSdma<uint32_t, false, 0>;
template class BlitSdma<uint64_t, true, -1>;
projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+13 -8
Vedi File
@@ -515,13 +515,13 @@ core::Queue* GpuAgent::CreateInterceptibleQueue() {
return queue;
}
core::Blit* GpuAgent::CreateBlitSdma() {
core::Blit* GpuAgent::CreateBlitSdma(bool h2d) {
core::Blit* sdma;
if (isa_->GetMajorVersion() <= 8) {
sdma = new BlitSdmaV2V3;
sdma = new BlitSdmaV2V3(h2d);
} else {
sdma = new BlitSdmaV4;
sdma = new BlitSdmaV4(h2d);
}
if (sdma->Initialize(*this) != HSA_STATUS_SUCCESS) {
@@ -561,10 +561,10 @@ void GpuAgent::InitDma() {
// Blits, try create SDMA blit first.
// Disable SDMA on specific ISA targets until they are fully qualified.
auto blit_lambda = [this](lazy_ptr<core::Queue>& queue) {
auto blit_lambda = [this](bool h2d, lazy_ptr<core::Queue>& queue) {
if ((isa_->GetMajorVersion() != 8) && core::Runtime::runtime_singleton_->flag().enable_sdma() &&
(HSA_PROFILE_BASE == profile_)) {
auto ret = CreateBlitSdma();
auto ret = CreateBlitSdma(h2d);
if (ret != nullptr) return ret;
}
auto ret = CreateBlitKernel((*queue).get());
@@ -573,8 +573,8 @@ void GpuAgent::InitDma() {
return ret;
};
blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueBlitOnly]); });
blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(queues_[QueueUtility]); });
blits_[BlitHostToDev].reset([blit_lambda, this]() { return blit_lambda(true, queues_[QueueBlitOnly]); });
blits_[BlitDevToHost].reset([blit_lambda, this]() { return blit_lambda(false, queues_[QueueUtility]); });
blits_[BlitDevToDev].reset([this]() {
auto ret = CreateBlitKernel((*queues_[QueueUtility]).get());
if (ret == nullptr)
@@ -614,7 +614,8 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
: (src_agent.device_type() == core::Agent::kAmdGpuDevice &&
dst_agent.device_type() == core::Agent::kAmdCpuDevice)
? blits_[BlitDevToHost]
: blits_[BlitDevToDev];
: (src_agent.node_id() == dst_agent.node_id())
? blits_[BlitDevToDev] : blits_[BlitDevToHost];
if (profiling_enabled()) {
// Track the agent so we could translate the resulting timestamp to system
@@ -1063,6 +1064,10 @@ uint16_t GpuAgent::GetMicrocodeVersion() const {
return (properties_.EngineId.ui32.uCode);
}
uint16_t GpuAgent::GetSdmaMicrocodeVersion() const {
return (properties_.uCodeEngineVersions.uCodeSDMA);
}
void GpuAgent::SyncClocks() {
HSAKMT_STATUS err = hsaKmtGetClockCounters(node_id(), &t1_);
assert(err == HSAKMT_STATUS_SUCCESS && "hsaGetClockCounters error");
projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
+7 -1
Vedi File
@@ -91,18 +91,23 @@ class Flag {
var = os::GetEnvVar("HSA_DISABLE_FRAGMENT_ALLOCATOR");
disable_fragment_alloc_ = (var == "1") ? true : false;
var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH");
enable_sdma_hdp_flush_ = (var == "0") ? false : true;
}
bool check_flat_scratch() const { return check_flat_scratch_; }
bool enable_vm_fault_message() const { return enable_vm_fault_message_; }
bool enable_queue_fault_message() const { return enable_queue_fault_message_; }
bool enable_interrupt() const { return enable_interrupt_; }
bool enable_sdma() const { return enable_sdma_; }
bool enable_sdma_hdp_flush() const { return enable_sdma_hdp_flush_; }
bool running_valgrind() const { return running_valgrind_; }
bool sdma_wait_idle() const { return sdma_wait_idle_; }
@@ -122,6 +127,7 @@ class Flag {
bool enable_vm_fault_message_;
bool enable_interrupt_;
bool enable_sdma_;
bool enable_sdma_hdp_flush_;
bool running_valgrind_;
bool sdma_wait_idle_;
bool enable_queue_fault_message_;