diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h b/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h index 76720bd79e..fdf89b6258 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/common/shared.h @@ -82,6 +82,8 @@ class Shared : public BaseShared { assert(shared_object_ != NULL && "Failed on allocating shared_object_"); + memset(shared_object_, 0, sizeof(T)); + if (shared_object_ != NULL) new (shared_object_) T; } diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/hsacore.so.def b/projects/rocr-runtime/runtime/hsa-runtime/core/hsacore.so.def index 4e04a8e2a1..130dd0875c 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/hsacore.so.def +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/hsacore.so.def @@ -107,6 +107,8 @@ global: hsa_amd_coherency_set_type; hsa_amd_profiling_set_profiler_enabled; hsa_amd_profiling_get_dispatch_time; + hsa_amd_profiling_async_copy_enable; + hsa_amd_profiling_get_async_copy_time; hsa_amd_profiling_convert_tick_to_system_domain; hsa_amd_signal_wait_any; hsa_amd_signal_async_handler; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/agent.h index abd7acf84b..44f9c7af8c 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/agent.h @@ -240,6 +240,19 @@ class Agent : public Checked<0xF6BC25EB17E6F917> { // @brief Returns node id associated with this agent. __forceinline uint32_t node_id() const { return node_id_; } + // @brief Getter for profiling_enabled_. + __forceinline bool profiling_enabled() const { return profiling_enabled_; } + + // @brief Setter for profiling_enabled_. + virtual hsa_status_t profiling_enabled(bool enable) { + const hsa_status_t stat = EnableDmaProfiling(enable); + if (HSA_STATUS_SUCCESS == stat) { + profiling_enabled_ = enable; + } + + return stat; + } + protected: // Intention here is to have a polymorphic update procedure for public_handle_ // which is callable on any Agent* but only from some class dervied from @@ -254,6 +267,17 @@ class Agent : public Checked<0xF6BC25EB17E6F917> { public_handle_ = handle; } + // @brief Enable profiling of the asynchronous DMA copy. The timestamp + // of each copy request will be stored in the completion signal structure. + // + // @param enable True to enable profiling. False to disable profiling. + // + // @retval HSA_STATUS_SUCCESS The profiling is enabled and the + // timing of subsequent async copy will be measured. + virtual hsa_status_t EnableDmaProfiling(bool enable) { + return HSA_STATUS_SUCCESS; + } + hsa_agent_t public_handle_; private: @@ -262,6 +286,8 @@ class Agent : public Checked<0xF6BC25EB17E6F917> { const uint32_t device_type_; + bool profiling_enabled_; + // Forbid copying and moving of this object DISALLOW_COPY_AND_ASSIGN(Agent); }; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_kernel.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_kernel.h index 73b9ef81cf..f6f7b27e23 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_kernel.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_kernel.h @@ -105,6 +105,8 @@ class BlitKernel : public core::Blit { virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value, size_t count) override; + virtual hsa_status_t EnableProfiling(bool enable) override; + private: union KernelArgs { struct __ALIGNED__(16) { diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h index 56aac90f96..063e1d2f6a 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h @@ -43,10 +43,12 @@ #ifndef HSA_RUNTIME_CORE_INC_AMD_BLIT_SDMA_H_ #define HSA_RUNTIME_CORE_INC_AMD_BLIT_SDMA_H_ +#include #include #include "hsakmt.h" +#include "core/inc/amd_gpu_agent.h" #include "core/inc/blit.h" #include "core/inc/runtime.h" #include "core/inc/signal.h" @@ -109,6 +111,12 @@ class BlitSdma : public core::Blit { virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value, size_t count) override; + virtual hsa_status_t EnableProfiling(bool enable) override; + + static const size_t kQueueSize; + + static const size_t kCopyPacketSize; + protected: /// @brief Acquires the address into queue buffer where a new command /// packet of specified size could be written. The address that is @@ -161,6 +169,11 @@ class BlitSdma : public core::Blit { void BuildAtomicDecrementCommand(char* cmd_addr, void* addr); + void BuildGetGlobalTimestampCommand(char* cmd_addr, void* write_address); + + // Agent object owning the SDMA engine. + GpuAgent* agent_; + /// Indicates size of Queue buffer in bytes. uint32_t queue_size_; @@ -201,6 +214,8 @@ class BlitSdma : public core::Blit { uint32_t atomic_command_size_; + uint32_t timestamp_command_size_; + // Max copy size of a single linear copy command packet. size_t max_single_linear_copy_size_; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h index cbf06be28c..c503b50d16 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h @@ -57,6 +57,8 @@ #include "core/util/locks.h" namespace amd { +class MemoryRegion; + // @brief Contains scratch memory information. struct ScratchInfo { void* queue_base; @@ -113,6 +115,16 @@ class GpuAgentInt : public core::Agent { virtual void TranslateTime(core::Signal* signal, hsa_amd_profiling_dispatch_time_t& time) = 0; + // @brief Translate the async copy start and end timestamp from agent + // domain to host domain. + // + // @param [in] signal Pointer to signal that provides the async copy timing. + // @param [out] time Structure to be populated with the host domain value. + virtual void TranslateTime(core::Signal* signal, + hsa_amd_profiling_async_copy_time_t& time) { + return TranslateTime(signal, (hsa_amd_profiling_dispatch_time_t&)time); + } + // @brief Translate timestamp agent domain to host domain. // // @param [out] time Timestamp in agent domain. @@ -215,6 +227,9 @@ class GpuAgent : public GpuAgentInt { // @brief Override from core::Agent. hsa_status_t DmaFill(void* ptr, uint32_t value, size_t count) override; + // @brief Get the next available end timestamp object. + uint64_t* ObtainEndTsObject(); + // @brief Override from core::Agent. hsa_status_t GetInfo(hsa_agent_info_t attribute, void* value) const override; @@ -320,6 +335,9 @@ class GpuAgent : public GpuAgentInt { // @brief Binds the second-level trap handler to this node. void BindTrapHandler(); + // @brief Override from core::Agent. + hsa_status_t EnableDmaProfiling(bool enable) override; + // @brief Node properties. const HsaNodeProperties properties_; @@ -371,6 +389,8 @@ class GpuAgent : public GpuAgentInt { // @brief Array of regions owned by this agent. std::vector regions_; + MemoryRegion* local_region_; + core::Isa* isa_; // @brief HSA profile. @@ -399,6 +419,10 @@ class GpuAgent : public GpuAgentInt { // @brief Query the driver to get the cache properties. void InitCacheList(); + // @brief Initialize memory pool for end timestamp object. + // @retval True if the memory pool for end timestamp object is initialized. + bool InitEndTsPool(); + // @brief Alternative aperture base address. Only on KV. uintptr_t ape1_base_; @@ -406,7 +430,17 @@ class GpuAgent : public GpuAgentInt { size_t ape1_size_; // @brief True if blit objects are initialized. - bool blit_initialized_; + std::atomic blit_initialized_; + + // Each end ts is 32 bytes. + static const size_t kTsSize = 32; + + // Number of element in the pool. + uint32_t end_ts_pool_size_; + + std::atomic end_ts_pool_counter_; + + std::atomic end_ts_base_addr_; DISALLOW_COPY_AND_ASSIGN(GpuAgent); }; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/blit.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/blit.h index cf1e6da653..48aebaa64b 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/blit.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/blit.h @@ -104,6 +104,15 @@ class Blit { /// @param num Number of uint32_t element to be set to the value. virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value, size_t num) = 0; + + /// @brief Enable profiling of the asynchronous copy command. The timestamp + /// of each copy request will be stored in the completion signal structure. + /// + /// @param enable True to enable profiling. False to disable profiling. + /// + /// @return HSA_STATUS_SUCCESS if the request to enable/disable profiling is + /// successful. + virtual hsa_status_t EnableProfiling(bool enable) = 0; }; } // namespace core diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h index e6509421cc..478034951f 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/signal.h @@ -57,6 +57,7 @@ #include "inc/amd_hsa_signal.h" namespace core { +class Agent; class Signal; /// @brief Helper structure to simplify conversion of amd_signal_t and @@ -75,7 +76,9 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>, public: /// @brief Constructor initializes the signal with initial value. explicit Signal(hsa_signal_value_t initial_value) - : Shared(), signal_(shared_object()->amd_signal) { + : Shared(), + signal_(shared_object()->amd_signal), + async_copy_agent_(NULL) { if (!Shared::IsSharedObjectAllocationValid()) { invalid_ = true; return; @@ -225,6 +228,12 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>, /// @brief Checks if signal is currently in use by a wait API. bool InWaiting() const { return waiting_ != 0; } + __forceinline void async_copy_agent(core::Agent* agent) { + async_copy_agent_ = agent; + } + + __forceinline core::Agent* async_copy_agent() { return async_copy_agent_; } + /// @brief Structure which defines key signal elements like type and value. /// Address of this struct is used as a value for the opaque handle of type /// hsa_signal_t provided to the public API. @@ -246,6 +255,9 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>, volatile uint32_t retained_; + /// @variable Pointer to agent used to perform an async copy. + core::Agent* async_copy_agent_; + private: DISALLOW_COPY_AND_ASSIGN(Signal); }; diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp index 18bf0bcecb..aff60f08ba 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_kernel.cpp @@ -581,6 +581,10 @@ hsa_status_t BlitKernel::Initialize(const core::Agent& agent) { kernel.code_buf_size_); } + if (agent.profiling_enabled()) { + return EnableProfiling(true); + } + return HSA_STATUS_SUCCESS; } @@ -798,6 +802,17 @@ hsa_status_t BlitKernel::SubmitLinearFillCommand(void* ptr, uint32_t value, return HSA_STATUS_SUCCESS; } +hsa_status_t BlitKernel::EnableProfiling(bool enable) { + core::Queue* cmd_queue = core::Queue::Convert(queue_); + if (cmd_queue != NULL) { + AMD_HSA_BITS_SET(cmd_queue->amd_queue_.queue_properties, + AMD_QUEUE_PROPERTIES_ENABLE_PROFILING, enable); + return HSA_STATUS_SUCCESS; + } + + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; +} + uint64_t BlitKernel::AcquireWriteIndex(uint32_t num_packet) { assert(queue_->size >= num_packet); diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp index ff9378e745..ab1cadcf66 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp @@ -48,9 +48,12 @@ #include #include "core/inc/amd_gpu_agent.h" +#include "core/inc/amd_memory_region.h" #include "core/inc/runtime.h" #include "core/inc/signal.h" +#define SDMA_QUEUE_SIZE 1024 * 1024 + namespace amd { // SDMA packet for VI device. // Reference: http://people.freedesktop.org/~agd5f/dma_packets.txt @@ -60,7 +63,9 @@ const unsigned int SDMA_OP_FENCE = 5; const unsigned int SDMA_OP_POLL_REGMEM = 8; const unsigned int SDMA_OP_ATOMIC = 10; const unsigned int SDMA_OP_CONST_FILL = 11; +const unsigned int SDMA_OP_TIMESTAMP = 13; const unsigned int SDMA_SUBOP_COPY_LINEAR = 0; +const unsigned int SDMA_SUBOP_TIMESTAMP_GET_GLOBAL = 2; const unsigned int SDMA_ATOMIC_ADD64 = 47; typedef struct SDMA_PKT_COPY_LINEAR_TAG { @@ -310,6 +315,32 @@ typedef struct SDMA_PKT_ATOMIC_TAG { } LOOP_UNION; } SDMA_PKT_ATOMIC; +typedef struct SDMA_PKT_TIMESTAMP_TAG { + union { + struct { + unsigned int op : 8; + unsigned int sub_op : 8; + unsigned int reserved_0 : 16; + }; + unsigned int DW_0_DATA; + } HEADER_UNION; + + union { + struct { + unsigned int addr_31_0 : 32; + }; + unsigned int DW_1_DATA; + } ADDR_LO_UNION; + + union { + struct { + unsigned int addr_63_32 : 32; + }; + unsigned int DW_2_DATA; + } ADDR_HI_UNION; + +} SDMA_PKT_TIMESTAMP; + inline uint32_t ptrlow32(const void* p) { return static_cast(reinterpret_cast(p)); } @@ -322,8 +353,12 @@ inline uint32_t ptrhigh32(const void* p) { #endif } +const size_t BlitSdma::kQueueSize = SDMA_QUEUE_SIZE; +const size_t BlitSdma::kCopyPacketSize = sizeof(SDMA_PKT_COPY_LINEAR); + BlitSdma::BlitSdma() : core::Blit(), + agent_(NULL), queue_size_(0), queue_start_addr_(NULL), fence_base_addr_(NULL), @@ -337,6 +372,8 @@ BlitSdma::BlitSdma() BlitSdma::~BlitSdma() {} hsa_status_t BlitSdma::Initialize(const core::Agent& agent) { + agent_ = reinterpret_cast(&const_cast(agent)); + if (queue_start_addr_ != NULL && queue_size_ != 0) { // Already initialized. return HSA_STATUS_SUCCESS; @@ -351,6 +388,7 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) { fence_command_size_ = sizeof(SDMA_PKT_FENCE); poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM); atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC); + timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP); const uint32_t sync_command_size = fence_command_size_; const uint32_t max_num_copy_command = @@ -372,18 +410,16 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) { static_cast(max_num_fill_command) * static_cast(max_single_fill_size_))); - const amd::GpuAgent& amd_gpu_agent = static_cast(agent); + const amd::GpuAgentInt& amd_gpu_agent = + static_cast(agent); - if (amd_gpu_agent.isa()->version() != core::Isa::Version(8, 0, 3)) { - assert(false && "Only for Fiji currently"); + if (HSA_PROFILE_FULL == amd_gpu_agent.profile()) { + assert(false && "Only support SDMA for dgpu currently"); return HSA_STATUS_ERROR; } // Allocate queue buffer. - const size_t kPageSize = 4096; - const size_t kSdmaQueueSize = 1024 * 1024; - - queue_size_ = kSdmaQueueSize; + queue_size_ = kQueueSize; HsaMemFlags flags; flags.Value = 0; @@ -413,10 +449,9 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) { // Access kernel driver to initialize the queue control block // This call binds user mode queue object to underlying compute // device. - const GpuAgent& gpu_agent = reinterpret_cast(agent); const HSA_QUEUE_TYPE kQueueType_ = HSA_QUEUE_SDMA; if (HSAKMT_STATUS_SUCCESS != - hsaKmtCreateQueue(gpu_agent.node_id(), kQueueType_, 100, + hsaKmtCreateQueue(amd_gpu_agent.node_id(), kQueueType_, 100, HSA_QUEUE_PRIORITY_MAXIMUM, queue_start_addr_, queue_size_, NULL, &queue_resource_)) { Destroy(agent); @@ -426,8 +461,8 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) { cached_reserve_offset_ = *(queue_resource_.Queue_write_ptr); cached_commit_offset_ = cached_reserve_offset_; - fence_pool_size_ = - static_cast(std::ceil(kSdmaQueueSize / fence_command_size_)); + fence_pool_size_ = static_cast( + (kQueueSize + fence_command_size_ - 1) / fence_command_size_); fence_pool_mask_ = fence_pool_size_ - 1; @@ -479,8 +514,8 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src, // Break the copy into multiple copy operation incase the copy size exceeds // the SDMA linear copy limit. - const uint32_t num_copy_command = static_cast( - std::ceil(static_cast(size) / max_single_linear_copy_size_)); + const uint32_t num_copy_command = + (size + max_single_linear_copy_size_ - 1) / max_single_linear_copy_size_; const uint32_t total_copy_command_size = num_copy_command * linear_copy_command_size_; @@ -528,18 +563,35 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand( // Break the copy into multiple copy operation incase the copy size exceeds // the SDMA linear copy limit. - const uint32_t num_copy_command = static_cast( - std::ceil(static_cast(size) / max_single_linear_copy_size_)); + const uint32_t num_copy_command = + (size + max_single_linear_copy_size_ - 1) / max_single_linear_copy_size_; const uint32_t total_copy_command_size = num_copy_command * linear_copy_command_size_; + // In case the user disable or enable the profiling in the middle of the call. + const bool profiling_enabled = agent_->profiling_enabled(); + + uint64_t* end_ts_addr = NULL; + uint32_t total_timestamp_command_size = 0; + + if (profiling_enabled) { + // SDMA timestamp packet requires 32 byte of aligned memory, but + // amd_signal_t::end_ts is not 32 byte aligned. So an extra copy packet to + // read from a 32 byte aligned bounce buffer is required to avoid changing + // the amd_signal_t ABI. + + end_ts_addr = agent_->ObtainEndTsObject(); + if (end_ts_addr == NULL) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + total_timestamp_command_size = + (2 * timestamp_command_size_) + linear_copy_command_size_; + } + const uint32_t total_command_size = total_poll_command_size + total_copy_command_size + atomic_command_size_ + - fence_command_size_; - - const uint32_t kFenceValue = 2015; - uint32_t* fence_addr = ObtainFenceObject(); - *fence_addr = 0; + total_timestamp_command_size; char* command_addr = AcquireWriteAddress(total_command_size); char* const command_addr_temp = command_addr; @@ -559,18 +611,32 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand( command_addr += poll_command_size_; } + if (profiling_enabled) { + BuildGetGlobalTimestampCommand( + command_addr, reinterpret_cast(&out_signal.signal_.start_ts)); + command_addr += timestamp_command_size_; + } + // Do the transfer after all polls are satisfied. BuildCopyCommand(command_addr, num_copy_command, dst, src, size); command_addr += total_copy_command_size; + if (profiling_enabled) { + assert(IsMultipleOf(end_ts_addr, 32)); + BuildGetGlobalTimestampCommand(command_addr, + reinterpret_cast(end_ts_addr)); + command_addr += timestamp_command_size_; + + BuildCopyCommand(command_addr, 1, + reinterpret_cast(&out_signal.signal_.end_ts), + reinterpret_cast(end_ts_addr), sizeof(uint64_t)); + command_addr += linear_copy_command_size_; + } + // After transfer is completed, decrement the signal. BuildAtomicDecrementCommand(command_addr, out_signal.ValueLocation()); - command_addr += atomic_command_size_; - - BuildFenceCommand(command_addr, fence_addr, kFenceValue); - ReleaseWriteAddress(command_addr_temp, total_command_size); return HSA_STATUS_SUCCESS; @@ -586,8 +652,8 @@ hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value, // Break the copy into multiple copy operation incase the copy size exceeds // the SDMA linear copy limit. - const uint32_t num_fill_command = static_cast( - std::ceil(static_cast(size) / max_single_fill_size_)); + const uint32_t num_fill_command = + (size + max_single_fill_size_ - 1) / max_single_fill_size_; const uint32_t total_fill_command_size = num_fill_command * fill_command_size_; @@ -644,6 +710,10 @@ hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value, return HSA_STATUS_SUCCESS; } +hsa_status_t BlitSdma::EnableProfiling(bool enable) { + return HSA_STATUS_SUCCESS; +} + char* BlitSdma::AcquireWriteAddress(uint32_t cmd_size) { if (cmd_size > queue_size_) { return NULL; @@ -867,4 +937,18 @@ void BlitSdma::BuildAtomicDecrementCommand(char* cmd_addr, void* addr) { packet_addr->SRC_DATA_LO_UNION.src_data_31_0 = 0xffffffff; packet_addr->SRC_DATA_HI_UNION.src_data_63_32 = 0xffffffff; } + +void BlitSdma::BuildGetGlobalTimestampCommand(char* cmd_addr, + void* write_address) { + SDMA_PKT_TIMESTAMP* packet_addr = + reinterpret_cast(cmd_addr); + + memset(packet_addr, 0, sizeof(SDMA_PKT_TIMESTAMP)); + + packet_addr->HEADER_UNION.op = SDMA_OP_TIMESTAMP; + packet_addr->HEADER_UNION.sub_op = SDMA_SUBOP_TIMESTAMP_GET_GLOBAL; + + packet_addr->ADDR_LO_UNION.addr_31_0 = ptrlow32(write_address); + packet_addr->ADDR_HI_UNION.addr_63_32 = ptrhigh32(write_address); +} } // namespace amd diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp index 5aad565789..d278349744 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp @@ -72,6 +72,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) blit_h2d_(NULL), blit_d2h_(NULL), blit_d2d_(NULL), + local_region_(NULL), is_kv_device_(false), trap_code_buf_(NULL), trap_code_buf_size_(0), @@ -79,7 +80,10 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) memory_max_frequency_(0), ape1_base_(0), ape1_size_(0), - blit_initialized_(false) { + blit_initialized_(false), + end_ts_pool_size_(0), + end_ts_pool_counter_(0), + end_ts_base_addr_(NULL) { const bool is_apu_node = (properties_.NumCPUCores > 0); profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE; @@ -91,6 +95,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props) isa_ = (core::Isa*)core::IsaRegistry::GetIsa(core::Isa::Version( node_props.EngineId.ui32.Major, node_props.EngineId.ui32.Minor, node_props.EngineId.ui32.Stepping)); + // Check if the device is Kaveri, only on GPU device. if (isa_->GetMajorVersion() == 7 && isa_->GetMinorVersion() == 0 && isa_->GetStepping() == 0) { @@ -152,6 +157,10 @@ GpuAgent::~GpuAgent() { blit_d2h_ = NULL; } + if (end_ts_base_addr_ != NULL) { + core::Runtime::runtime_singleton_->FreeMemory(end_ts_base_addr_); + } + if (ape1_base_ != 0) { _aligned_free(reinterpret_cast(ape1_base_)); } @@ -296,6 +305,10 @@ void GpuAgent::InitRegionList() { new MemoryRegion(false, false, this, mem_props[mem_idx]); regions_.push_back(region); + + if (region->IsLocalMemory()) { + local_region_ = region; + } break; } case HSA_HEAPTYPE_SYSTEM: @@ -372,6 +385,57 @@ void GpuAgent::InitCacheList() { } } +bool GpuAgent::InitEndTsPool() { + if (HSA_PROFILE_FULL == profile_) { + return true; + } + + if (end_ts_base_addr_.load(std::memory_order_acquire) != NULL) { + return true; + } + + ScopedAcquire lock(&blit_lock_); + + if (end_ts_base_addr_.load(std::memory_order_relaxed) != NULL) { + return true; + } + + end_ts_pool_size_ = static_cast( + (BlitSdma::kQueueSize + BlitSdma::kCopyPacketSize - 1) / + (BlitSdma::kCopyPacketSize)); + + // Allocate end timestamp object for both h2d and d2h DMA. + const size_t alloc_size = 2 * end_ts_pool_size_ * kTsSize; + + core::Runtime* runtime = core::Runtime::runtime_singleton_; + + uint64_t* buff = NULL; + if (HSA_STATUS_SUCCESS != + runtime->AllocateMemory(true, local_region_, alloc_size, + reinterpret_cast(&buff))) { + return false; + } + + end_ts_base_addr_.store(buff, std::memory_order_release); + + return true; +} + +uint64_t* GpuAgent::ObtainEndTsObject() { + if (end_ts_base_addr_ == NULL) { + return NULL; + } + + const uint32_t end_ts_index = + end_ts_pool_counter_.fetch_add(1U, std::memory_order_acq_rel) % + end_ts_pool_size_; + const static size_t kNumU64 = kTsSize / sizeof(uint64_t); + uint64_t* end_ts_addr = &end_ts_base_addr_[end_ts_index * kNumU64]; + assert(IsMultipleOf(end_ts_addr, kTsSize)); + + return end_ts_addr; +} + hsa_status_t GpuAgent::IterateRegion( hsa_status_t (*callback)(hsa_region_t region, void* data), void* data) const { @@ -460,18 +524,17 @@ void GpuAgent::InitDma() { // could give indication of DMA usage in the future. E.g.: // 1. Call to allow access API. // 2. Call to memory lock API. - if (!atomic::Load(&blit_initialized_, std::memory_order_acquire)) { + if (!blit_initialized_.load(std::memory_order_acquire)) { ScopedAcquire lock(&blit_lock_); - if (!atomic::Load(&blit_initialized_, std::memory_order_relaxed)) { + if (!blit_initialized_.load(std::memory_order_relaxed)) { // Try create SDMA blit first. if (core::Runtime::runtime_singleton_->flag().enable_sdma() && - isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 && - isa_->GetStepping() == 3) { + (HSA_PROFILE_BASE == profile_)) { blit_h2d_ = CreateBlitSdma(); blit_d2h_ = CreateBlitSdma(); if (blit_h2d_ != NULL && blit_d2h_ != NULL) { - atomic::Store(&blit_initialized_, true, std::memory_order_release); + blit_initialized_.store(true, std::memory_order_release); return; } } @@ -487,7 +550,7 @@ void GpuAgent::InitDma() { blit_d2h_ = CreateBlitKernel(); } - atomic::Store(&blit_initialized_, true, std::memory_order_release); + blit_initialized_.store(true, std::memory_order_release); } } } @@ -536,7 +599,16 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent, static_cast(out_signal).DisableWaitEvent(); } - return blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal); + hsa_status_t stat = + blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal); + + if (profiling_enabled() && HSA_STATUS_SUCCESS == stat) { + // Track the agent so we could translate the resulting timestamp to system + // domain correctly. + out_signal.async_copy_agent(this); + } + + return stat; } hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) { @@ -547,14 +619,30 @@ hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) { return blit_d2d_->SubmitLinearFillCommand(ptr, value, count); } +hsa_status_t GpuAgent::EnableDmaProfiling(bool enable) { + if (enable && !InitEndTsPool()) { + return HSA_STATUS_ERROR_OUT_OF_RESOURCES; + } + + core::Blit* blit[3] = {blit_h2d_, blit_d2h_, blit_d2d_}; + for (int i = 0; i < 3; ++i) { + if (blit[i] != NULL) { + const hsa_status_t stat = blit[i]->EnableProfiling(enable); + if (stat != HSA_STATUS_SUCCESS) { + return stat; + } + } + } + + return HSA_STATUS_SUCCESS; +} + hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const { const size_t kNameSize = 64; // agent, and vendor name size limit const core::ExtensionEntryPoints& extensions = core::Runtime::runtime_singleton_->extensions_; - hsa_agent_t agent = core::Agent::Convert(this); - const size_t attribute_u = static_cast(attribute); switch (attribute_u) { case HSA_AGENT_INFO_NAME: diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp index 9394c30062..45a64a8c5f 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/hsa_ext_amd.cpp @@ -229,6 +229,17 @@ hsa_status_t HSA_API return HSA_STATUS_SUCCESS; } +hsa_status_t HSA_API hsa_amd_profiling_async_copy_enable(bool enable) { + IS_OPEN(); + + return core::Runtime::runtime_singleton_->IterateAgent( + [](hsa_agent_t agent_handle, void* data) -> hsa_status_t { + const bool enable = *(reinterpret_cast(data)); + return core::Agent::Convert(agent_handle)->profiling_enabled(enable); + }, + reinterpret_cast(&enable)); +} + hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time( hsa_agent_t agent_handle, hsa_signal_t hsa_signal, hsa_amd_profiling_dispatch_time_t* time) { @@ -250,11 +261,40 @@ hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time( amd::GpuAgentInt* gpu_agent = static_cast(agent); + // Translate timestamp from GPU to system domain. gpu_agent->TranslateTime(signal, *time); return HSA_STATUS_SUCCESS; } +hsa_status_t HSA_API hsa_amd_profiling_get_async_copy_time( + hsa_signal_t hsa_signal, hsa_amd_profiling_async_copy_time_t* time) { + IS_OPEN(); + + IS_BAD_PTR(time); + + core::Signal* signal = core::Signal::Convert(hsa_signal); + + IS_VALID(signal); + + core::Agent* agent = signal->async_copy_agent(); + + if (agent == NULL) { + return HSA_STATUS_ERROR; + } + + if (agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice) { + // Translate timestamp from GPU to system domain. + static_cast(agent)->TranslateTime(signal, *time); + return HSA_STATUS_SUCCESS; + } + + // The timestamp is already in system domain. + time->start = signal->signal_.start_ts; + time->end = signal->signal_.end_ts; + return HSA_STATUS_SUCCESS; +} + hsa_status_t HSA_API hsa_amd_profiling_convert_tick_to_system_domain(hsa_agent_t agent_handle, uint64_t agent_tick, diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp index 37c745ef32..240fbfd035 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp +++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp @@ -424,19 +424,37 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent& dst_agent, } // For cpu to cpu, fire and forget a copy thread. - std::thread([](void* dst, const void* src, size_t size, - std::vector dep_signals, - core::Signal* completion_signal) { - for (core::Signal* dep : dep_signals) { - dep->WaitRelaxed(HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, - HSA_WAIT_STATE_BLOCKED); - } + const bool profiling_enabled = + (dst_agent.profiling_enabled() || src_agent.profiling_enabled()); + std::thread( + [](void* dst, const void* src, size_t size, + std::vector dep_signals, + core::Signal* completion_signal, bool profiling_enabled) { + for (core::Signal* dep : dep_signals) { + dep->WaitRelaxed(HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX, + HSA_WAIT_STATE_BLOCKED); + } - memcpy(dst, src, size); + if (profiling_enabled) { + HsaClockCounters clocks = {0}; + core::Runtime::runtime_singleton_->GetSystemInfo( + HSA_SYSTEM_INFO_TIMESTAMP, reinterpret_cast(&clocks)); + completion_signal->signal_.start_ts = clocks.SystemClockCounter; + } - completion_signal->SubRelease(1); - }, - dst, src, size, dep_signals, &completion_signal).detach(); + memcpy(dst, src, size); + + if (profiling_enabled) { + HsaClockCounters clocks = {0}; + core::Runtime::runtime_singleton_->GetSystemInfo( + HSA_SYSTEM_INFO_TIMESTAMP, reinterpret_cast(&clocks)); + completion_signal->signal_.end_ts = clocks.SystemClockCounter; + } + + completion_signal->SubRelease(1); + }, + dst, src, size, dep_signals, &completion_signal, + profiling_enabled).detach(); return HSA_STATUS_SUCCESS; } diff --git a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h index 40d2356df6..4cbc82e7ea 100644 --- a/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h +++ b/projects/rocr-runtime/runtime/hsa-runtime/inc/hsa_ext_amd.h @@ -211,6 +211,23 @@ typedef struct hsa_amd_profiling_dispatch_time_s { uint64_t end; } hsa_amd_profiling_dispatch_time_t; +/** + * @brief Structure containing profiling async copy time information. + * + * Times are reported as ticks in the domain of the HSA system clock. + * The HSA system clock tick and frequency is obtained via hsa_system_get_info. + */ +typedef struct hsa_amd_profiling_async_copy_time_s { + /** + * Async copy processing start time. + */ + uint64_t start; + /** + * Async copy completion time. + */ + uint64_t end; +} hsa_amd_profiling_async_copy_time_t; + /** * @brief Enable or disable profiling capability of a queue. * @@ -230,10 +247,34 @@ typedef struct hsa_amd_profiling_dispatch_time_s { hsa_status_t HSA_API hsa_amd_profiling_set_profiler_enabled(hsa_queue_t* queue, int enable); +/** + * @brief Enable or disable asynchronous memory copy profiling. + * + * @details The runtime will provide the copy processing start timestamp and + * completion timestamp of each call to hsa_amd_memory_async_copy if the + * async copy profiling is enabled prior to the call to + * hsa_amd_memory_async_copy. The completion signal object is used to + * hold the last async copy start and end timestamp. The client can retrieve + * these timestamps via call to hsa_amd_profiling_get_async_copy_time. + * + * @param[in] enable True to enable profiling. False to disable profiling. + * + * @retval ::HSA_STATUS_SUCCESS The function has been executed successfully. + * + * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been + * initialized. + * + * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Failed on allocating resources + * needed to profile the asynchronous copy. + */ +hsa_status_t HSA_API + hsa_amd_profiling_async_copy_enable(bool enable); + /** * @brief Retrieve packet processing time stamps. * - * @param[in] agent The agent with which the signal was last used. For instance, + * @param[in] agent The agent with which the signal was last used. For + *instance, * if the profiled dispatch packet is dispatched on to queue Q, which was * created on agent A, then this parameter must be A. * @@ -261,6 +302,30 @@ hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time( hsa_agent_t agent, hsa_signal_t signal, hsa_amd_profiling_dispatch_time_t* time); +/** + * @brief Retrieve asynchronous copy timestamps. + * + * @details Async copy profiling is enabled via call to + * hsa_amd_profiling_async_copy_enable. + * + * @param[in] signal A signal used as the completion signal of the call to + * hsa_amd_memory_async_copy. + * + * @param[out] time Async copy processing timestamps in the HSA system clock + * domain. + * + * @retval ::HSA_STATUS_SUCCESS The function has been executed successfully. + * + * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been + * initialized. + * + * @retval ::HSA_STATUS_ERROR_INVALID_SIGNAL The signal is invalid. + * + * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT @p time is NULL. + */ +hsa_status_t HSA_API hsa_amd_profiling_get_async_copy_time( + hsa_signal_t signal, hsa_amd_profiling_async_copy_time_t* time); + /** * @brief Computes the frequency ratio and offset between the agent clock and * HSA system clock and converts the agent's tick to HSA system domain tick.