Add profiling support to DMA copy function

Change-Id: Iadeefa2692f35d9305ac1b242284a6220d5830a7 [ROCm/ROCR-Runtime commit: a8b00680b6]
2016-05-17 15:12:41 -05:00
parent 0a5966812c
commit 76dea4613a
14 changed files with 462 additions and 50 deletions
@@ -82,6 +82,8 @@ class Shared : public BaseShared {

    assert(shared_object_ != NULL && "Failed on allocating shared_object_");

+    memset(shared_object_, 0, sizeof(T));
+
    if (shared_object_ != NULL) new (shared_object_) T;
  }

@@ -107,6 +107,8 @@ global:
 	hsa_amd_coherency_set_type;
 	hsa_amd_profiling_set_profiler_enabled;
 	hsa_amd_profiling_get_dispatch_time;
+	hsa_amd_profiling_async_copy_enable;
+	hsa_amd_profiling_get_async_copy_time;
 	hsa_amd_profiling_convert_tick_to_system_domain;
 	hsa_amd_signal_wait_any;
 	hsa_amd_signal_async_handler;
@@ -240,6 +240,19 @@ class Agent : public Checked<0xF6BC25EB17E6F917> {
  // @brief Returns node id associated with this agent.
  __forceinline uint32_t node_id() const { return node_id_; }

+  // @brief Getter for profiling_enabled_.
+  __forceinline bool profiling_enabled() const { return profiling_enabled_; }
+
+  // @brief Setter for profiling_enabled_.
+  virtual hsa_status_t profiling_enabled(bool enable) {
+    const hsa_status_t stat = EnableDmaProfiling(enable);
+    if (HSA_STATUS_SUCCESS == stat) {
+      profiling_enabled_ = enable;
+    }
+
+    return stat;
+  }
+
 protected:
  // Intention here is to have a polymorphic update procedure for public_handle_
  // which is callable on any Agent* but only from some class dervied from
@@ -254,6 +267,17 @@ class Agent : public Checked<0xF6BC25EB17E6F917> {
    public_handle_ = handle;
  }

+  // @brief Enable profiling of the asynchronous DMA copy. The timestamp
+  // of each copy request will be stored in the completion signal structure.
+  //
+  // @param enable True to enable profiling. False to disable profiling.
+  //
+  // @retval HSA_STATUS_SUCCESS The profiling is enabled and the
+  // timing of subsequent async copy will be measured.
+  virtual hsa_status_t EnableDmaProfiling(bool enable) {
+    return HSA_STATUS_SUCCESS;
+  }
+
  hsa_agent_t public_handle_;

 private:
@@ -262,6 +286,8 @@ class Agent : public Checked<0xF6BC25EB17E6F917> {

  const uint32_t device_type_;

+  bool profiling_enabled_;
+
  // Forbid copying and moving of this object
  DISALLOW_COPY_AND_ASSIGN(Agent);
 };
@@ -105,6 +105,8 @@ class BlitKernel : public core::Blit {
  virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value,
                                               size_t count) override;

+  virtual hsa_status_t EnableProfiling(bool enable) override;
+
 private:
  union KernelArgs {
    struct __ALIGNED__(16) {
@@ -43,10 +43,12 @@
 #ifndef HSA_RUNTIME_CORE_INC_AMD_BLIT_SDMA_H_
 #define HSA_RUNTIME_CORE_INC_AMD_BLIT_SDMA_H_

+#include <mutex>
 #include <stdint.h>

 #include "hsakmt.h"

+#include "core/inc/amd_gpu_agent.h"
 #include "core/inc/blit.h"
 #include "core/inc/runtime.h"
 #include "core/inc/signal.h"
@@ -109,6 +111,12 @@ class BlitSdma : public core::Blit {
  virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value,
                                               size_t count) override;

+  virtual hsa_status_t EnableProfiling(bool enable) override;
+
+  static const size_t kQueueSize;
+
+  static const size_t kCopyPacketSize;
+
 protected:
  /// @brief Acquires the address into queue buffer where a new command
  /// packet of specified size could be written. The address that is
@@ -161,6 +169,11 @@ class BlitSdma : public core::Blit {

  void BuildAtomicDecrementCommand(char* cmd_addr, void* addr);

+  void BuildGetGlobalTimestampCommand(char* cmd_addr, void* write_address);
+
+  // Agent object owning the SDMA engine.
+  GpuAgent* agent_;
+
  /// Indicates size of Queue buffer in bytes.
  uint32_t queue_size_;

@@ -201,6 +214,8 @@ class BlitSdma : public core::Blit {

  uint32_t atomic_command_size_;

+  uint32_t timestamp_command_size_;
+
  // Max copy size of a single linear copy command packet.
  size_t max_single_linear_copy_size_;

@@ -57,6 +57,8 @@
 #include "core/util/locks.h"

 namespace amd {
+class MemoryRegion;
+
 // @brief Contains scratch memory information.
 struct ScratchInfo {
  void* queue_base;
@@ -113,6 +115,16 @@ class GpuAgentInt : public core::Agent {
  virtual void TranslateTime(core::Signal* signal,
                             hsa_amd_profiling_dispatch_time_t& time) = 0;

+  // @brief Translate the async copy start and end timestamp from agent
+  // domain to host domain.
+  //
+  // @param [in] signal Pointer to signal that provides the async copy timing.
+  // @param [out] time Structure to be populated with the host domain value.
+  virtual void TranslateTime(core::Signal* signal,
+                             hsa_amd_profiling_async_copy_time_t& time) {
+    return TranslateTime(signal, (hsa_amd_profiling_dispatch_time_t&)time);
+  }
+
  // @brief Translate timestamp agent domain to host domain.
  //
  // @param [out] time Timestamp in agent domain.
@@ -215,6 +227,9 @@ class GpuAgent : public GpuAgentInt {
  // @brief Override from core::Agent.
  hsa_status_t DmaFill(void* ptr, uint32_t value, size_t count) override;

+  // @brief Get the next available end timestamp object.
+  uint64_t* ObtainEndTsObject();
+
  // @brief Override from core::Agent.
  hsa_status_t GetInfo(hsa_agent_info_t attribute, void* value) const override;

@@ -320,6 +335,9 @@ class GpuAgent : public GpuAgentInt {
  // @brief Binds the second-level trap handler to this node.
  void BindTrapHandler();

+  // @brief Override from core::Agent.
+  hsa_status_t EnableDmaProfiling(bool enable) override;
+
  // @brief Node properties.
  const HsaNodeProperties properties_;

@@ -371,6 +389,8 @@ class GpuAgent : public GpuAgentInt {
  // @brief Array of regions owned by this agent.
  std::vector<const core::MemoryRegion*> regions_;

+  MemoryRegion* local_region_;
+
  core::Isa* isa_;

  // @brief HSA profile.
@@ -399,6 +419,10 @@ class GpuAgent : public GpuAgentInt {
  // @brief Query the driver to get the cache properties.
  void InitCacheList();

+  // @brief Initialize memory pool for end timestamp object.
+  // @retval True if the memory pool for end timestamp object is initialized.
+  bool InitEndTsPool();
+
  // @brief Alternative aperture base address. Only on KV.
  uintptr_t ape1_base_;

@@ -406,7 +430,17 @@ class GpuAgent : public GpuAgentInt {
  size_t ape1_size_;

  // @brief True if blit objects are initialized.
-  bool blit_initialized_;
+  std::atomic<bool> blit_initialized_;
+
+  // Each end ts is 32 bytes.
+  static const size_t kTsSize = 32;
+
+  // Number of element in the pool.
+  uint32_t end_ts_pool_size_;
+
+  std::atomic<uint32_t> end_ts_pool_counter_;
+
+  std::atomic<uint64_t*> end_ts_base_addr_;

  DISALLOW_COPY_AND_ASSIGN(GpuAgent);
 };
@@ -104,6 +104,15 @@ class Blit {
  /// @param num Number of uint32_t element to be set to the value.
  virtual hsa_status_t SubmitLinearFillCommand(void* ptr, uint32_t value,
                                               size_t num) = 0;
+
+  /// @brief Enable profiling of the asynchronous copy command. The timestamp
+  /// of each copy request will be stored in the completion signal structure.
+  ///
+  /// @param enable True to enable profiling. False to disable profiling.
+  ///
+  /// @return HSA_STATUS_SUCCESS if the request to enable/disable profiling is
+  /// successful.
+  virtual hsa_status_t EnableProfiling(bool enable) = 0;
 };
 }  // namespace core

@@ -57,6 +57,7 @@
 #include "inc/amd_hsa_signal.h"

 namespace core {
+class Agent;
 class Signal;

 /// @brief Helper structure to simplify conversion of amd_signal_t and
@@ -75,7 +76,9 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>,
 public:
  /// @brief Constructor initializes the signal with initial value.
  explicit Signal(hsa_signal_value_t initial_value)
-      : Shared(), signal_(shared_object()->amd_signal) {
+      : Shared(),
+        signal_(shared_object()->amd_signal),
+        async_copy_agent_(NULL) {
    if (!Shared::IsSharedObjectAllocationValid()) {
      invalid_ = true;
      return;
@@ -225,6 +228,12 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>,
  /// @brief Checks if signal is currently in use by a wait API.
  bool InWaiting() const { return waiting_ != 0; }

+  __forceinline void async_copy_agent(core::Agent* agent) {
+    async_copy_agent_ = agent;
+  }
+
+  __forceinline core::Agent* async_copy_agent() { return async_copy_agent_; }
+
  /// @brief Structure which defines key signal elements like type and value.
  /// Address of this struct is used as a value for the opaque handle of type
  /// hsa_signal_t provided to the public API.
@@ -246,6 +255,9 @@ class Signal : public Checked<0x71FCCA6A3D5D5276>,

  volatile uint32_t retained_;

+  /// @variable Pointer to agent used to perform an async copy.
+  core::Agent* async_copy_agent_;
+
 private:
  DISALLOW_COPY_AND_ASSIGN(Signal);
 };
@@ -581,6 +581,10 @@ hsa_status_t BlitKernel::Initialize(const core::Agent& agent) {
                            kernel.code_buf_size_);
  }

+  if (agent.profiling_enabled()) {
+    return EnableProfiling(true);
+  }
+
  return HSA_STATUS_SUCCESS;
 }

@@ -798,6 +802,17 @@ hsa_status_t BlitKernel::SubmitLinearFillCommand(void* ptr, uint32_t value,
  return HSA_STATUS_SUCCESS;
 }

+hsa_status_t BlitKernel::EnableProfiling(bool enable) {
+  core::Queue* cmd_queue = core::Queue::Convert(queue_);
+  if (cmd_queue != NULL) {
+    AMD_HSA_BITS_SET(cmd_queue->amd_queue_.queue_properties,
+                     AMD_QUEUE_PROPERTIES_ENABLE_PROFILING, enable);
+    return HSA_STATUS_SUCCESS;
+  }
+
+  return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+}
+
 uint64_t BlitKernel::AcquireWriteIndex(uint32_t num_packet) {
  assert(queue_->size >= num_packet);

@@ -48,9 +48,12 @@
 #include <cstring>

 #include "core/inc/amd_gpu_agent.h"
+#include "core/inc/amd_memory_region.h"
 #include "core/inc/runtime.h"
 #include "core/inc/signal.h"

+#define SDMA_QUEUE_SIZE 1024 * 1024
+
 namespace amd {
 // SDMA packet for VI device.
 // Reference: http://people.freedesktop.org/~agd5f/dma_packets.txt
@@ -60,7 +63,9 @@ const unsigned int SDMA_OP_FENCE = 5;
 const unsigned int SDMA_OP_POLL_REGMEM = 8;
 const unsigned int SDMA_OP_ATOMIC = 10;
 const unsigned int SDMA_OP_CONST_FILL = 11;
+const unsigned int SDMA_OP_TIMESTAMP = 13;
 const unsigned int SDMA_SUBOP_COPY_LINEAR = 0;
+const unsigned int SDMA_SUBOP_TIMESTAMP_GET_GLOBAL = 2;
 const unsigned int SDMA_ATOMIC_ADD64 = 47;

 typedef struct SDMA_PKT_COPY_LINEAR_TAG {
@@ -310,6 +315,32 @@ typedef struct SDMA_PKT_ATOMIC_TAG {
  } LOOP_UNION;
 } SDMA_PKT_ATOMIC;

+typedef struct SDMA_PKT_TIMESTAMP_TAG {
+  union {
+    struct {
+      unsigned int op : 8;
+      unsigned int sub_op : 8;
+      unsigned int reserved_0 : 16;
+    };
+    unsigned int DW_0_DATA;
+  } HEADER_UNION;
+
+  union {
+    struct {
+      unsigned int addr_31_0 : 32;
+    };
+    unsigned int DW_1_DATA;
+  } ADDR_LO_UNION;
+
+  union {
+    struct {
+      unsigned int addr_63_32 : 32;
+    };
+    unsigned int DW_2_DATA;
+  } ADDR_HI_UNION;
+
+} SDMA_PKT_TIMESTAMP;
+
 inline uint32_t ptrlow32(const void* p) {
  return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(p));
 }
@@ -322,8 +353,12 @@ inline uint32_t ptrhigh32(const void* p) {
 #endif
 }

+const size_t BlitSdma::kQueueSize = SDMA_QUEUE_SIZE;
+const size_t BlitSdma::kCopyPacketSize = sizeof(SDMA_PKT_COPY_LINEAR);
+
 BlitSdma::BlitSdma()
    : core::Blit(),
+      agent_(NULL),
      queue_size_(0),
      queue_start_addr_(NULL),
      fence_base_addr_(NULL),
@@ -337,6 +372,8 @@ BlitSdma::BlitSdma()
 BlitSdma::~BlitSdma() {}

 hsa_status_t BlitSdma::Initialize(const core::Agent& agent) {
+  agent_ = reinterpret_cast<amd::GpuAgent*>(&const_cast<core::Agent&>(agent));
+
  if (queue_start_addr_ != NULL && queue_size_ != 0) {
    // Already initialized.
    return HSA_STATUS_SUCCESS;
@@ -351,6 +388,7 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) {
  fence_command_size_ = sizeof(SDMA_PKT_FENCE);
  poll_command_size_ = sizeof(SDMA_PKT_POLL_REGMEM);
  atomic_command_size_ = sizeof(SDMA_PKT_ATOMIC);
+  timestamp_command_size_ = sizeof(SDMA_PKT_TIMESTAMP);

  const uint32_t sync_command_size = fence_command_size_;
  const uint32_t max_num_copy_command =
@@ -372,18 +410,16 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) {
               static_cast<uint64_t>(max_num_fill_command) *
                   static_cast<uint64_t>(max_single_fill_size_)));

-  const amd::GpuAgent& amd_gpu_agent = static_cast<const amd::GpuAgent&>(agent);
+  const amd::GpuAgentInt& amd_gpu_agent =
+      static_cast<const amd::GpuAgentInt&>(agent);

-  if (amd_gpu_agent.isa()->version() != core::Isa::Version(8, 0, 3)) {
-    assert(false && "Only for Fiji currently");
+  if (HSA_PROFILE_FULL == amd_gpu_agent.profile()) {
+    assert(false && "Only support SDMA for dgpu currently");
    return HSA_STATUS_ERROR;
  }

  // Allocate queue buffer.
-  const size_t kPageSize = 4096;
-  const size_t kSdmaQueueSize = 1024 * 1024;
-
-  queue_size_ = kSdmaQueueSize;
+  queue_size_ = kQueueSize;

  HsaMemFlags flags;
  flags.Value = 0;
@@ -413,10 +449,9 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) {
  // Access kernel driver to initialize the queue control block
  // This call binds user mode queue object to underlying compute
  // device.
-  const GpuAgent& gpu_agent = reinterpret_cast<const GpuAgent&>(agent);
  const HSA_QUEUE_TYPE kQueueType_ = HSA_QUEUE_SDMA;
  if (HSAKMT_STATUS_SUCCESS !=
-      hsaKmtCreateQueue(gpu_agent.node_id(), kQueueType_, 100,
+      hsaKmtCreateQueue(amd_gpu_agent.node_id(), kQueueType_, 100,
                        HSA_QUEUE_PRIORITY_MAXIMUM, queue_start_addr_,
                        queue_size_, NULL, &queue_resource_)) {
    Destroy(agent);
@@ -426,8 +461,8 @@ hsa_status_t BlitSdma::Initialize(const core::Agent& agent) {
  cached_reserve_offset_ = *(queue_resource_.Queue_write_ptr);
  cached_commit_offset_ = cached_reserve_offset_;

-  fence_pool_size_ =
-      static_cast<uint32_t>(std::ceil(kSdmaQueueSize / fence_command_size_));
+  fence_pool_size_ = static_cast<uint32_t>(
+      (kQueueSize + fence_command_size_ - 1) / fence_command_size_);

  fence_pool_mask_ = fence_pool_size_ - 1;

@@ -479,8 +514,8 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand(void* dst, const void* src,

  // Break the copy into multiple copy operation incase the copy size exceeds
  // the SDMA linear copy limit.
-  const uint32_t num_copy_command = static_cast<uint32_t>(
-      std::ceil(static_cast<double>(size) / max_single_linear_copy_size_));
+  const uint32_t num_copy_command =
+      (size + max_single_linear_copy_size_ - 1) / max_single_linear_copy_size_;

  const uint32_t total_copy_command_size =
      num_copy_command * linear_copy_command_size_;
@@ -528,18 +563,35 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand(

  // Break the copy into multiple copy operation incase the copy size exceeds
  // the SDMA linear copy limit.
-  const uint32_t num_copy_command = static_cast<uint32_t>(
-      std::ceil(static_cast<double>(size) / max_single_linear_copy_size_));
+  const uint32_t num_copy_command =
+      (size + max_single_linear_copy_size_ - 1) / max_single_linear_copy_size_;
  const uint32_t total_copy_command_size =
      num_copy_command * linear_copy_command_size_;

+  // In case the user disable or enable the profiling in the middle of the call.
+  const bool profiling_enabled = agent_->profiling_enabled();
+
+  uint64_t* end_ts_addr = NULL;
+  uint32_t total_timestamp_command_size = 0;
+
+  if (profiling_enabled) {
+    // SDMA timestamp packet requires 32 byte of aligned memory, but
+    // amd_signal_t::end_ts is not 32 byte aligned. So an extra copy packet to
+    // read from a 32 byte aligned bounce buffer is required to avoid changing
+    // the amd_signal_t ABI.
+
+    end_ts_addr = agent_->ObtainEndTsObject();
+    if (end_ts_addr == NULL) {
+      return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+    }
+
+    total_timestamp_command_size =
+        (2 * timestamp_command_size_) + linear_copy_command_size_;
+  }
+
  const uint32_t total_command_size =
      total_poll_command_size + total_copy_command_size + atomic_command_size_ +
-      fence_command_size_;
-
-  const uint32_t kFenceValue = 2015;
-  uint32_t* fence_addr = ObtainFenceObject();
-  *fence_addr = 0;
+      total_timestamp_command_size;

  char* command_addr = AcquireWriteAddress(total_command_size);
  char* const command_addr_temp = command_addr;
@@ -559,18 +611,32 @@ hsa_status_t BlitSdma::SubmitLinearCopyCommand(
    command_addr += poll_command_size_;
  }

+  if (profiling_enabled) {
+    BuildGetGlobalTimestampCommand(
+        command_addr, reinterpret_cast<void*>(&out_signal.signal_.start_ts));
+    command_addr += timestamp_command_size_;
+  }
+
  // Do the transfer after all polls are satisfied.
  BuildCopyCommand(command_addr, num_copy_command, dst, src, size);

  command_addr += total_copy_command_size;

+  if (profiling_enabled) {
+    assert(IsMultipleOf(end_ts_addr, 32));
+    BuildGetGlobalTimestampCommand(command_addr,
+                                   reinterpret_cast<void*>(end_ts_addr));
+    command_addr += timestamp_command_size_;
+
+    BuildCopyCommand(command_addr, 1,
+                     reinterpret_cast<void*>(&out_signal.signal_.end_ts),
+                     reinterpret_cast<void*>(end_ts_addr), sizeof(uint64_t));
+    command_addr += linear_copy_command_size_;
+  }
+
  // After transfer is completed, decrement the signal.
  BuildAtomicDecrementCommand(command_addr, out_signal.ValueLocation());

-  command_addr += atomic_command_size_;
-
-  BuildFenceCommand(command_addr, fence_addr, kFenceValue);
-
  ReleaseWriteAddress(command_addr_temp, total_command_size);

  return HSA_STATUS_SUCCESS;
@@ -586,8 +652,8 @@ hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value,

  // Break the copy into multiple copy operation incase the copy size exceeds
  // the SDMA linear copy limit.
-  const uint32_t num_fill_command = static_cast<uint32_t>(
-      std::ceil(static_cast<double>(size) / max_single_fill_size_));
+  const uint32_t num_fill_command =
+      (size + max_single_fill_size_ - 1) / max_single_fill_size_;

  const uint32_t total_fill_command_size =
      num_fill_command * fill_command_size_;
@@ -644,6 +710,10 @@ hsa_status_t BlitSdma::SubmitLinearFillCommand(void* ptr, uint32_t value,
  return HSA_STATUS_SUCCESS;
 }

+hsa_status_t BlitSdma::EnableProfiling(bool enable) {
+  return HSA_STATUS_SUCCESS;
+}
+
 char* BlitSdma::AcquireWriteAddress(uint32_t cmd_size) {
  if (cmd_size > queue_size_) {
    return NULL;
@@ -867,4 +937,18 @@ void BlitSdma::BuildAtomicDecrementCommand(char* cmd_addr, void* addr) {
  packet_addr->SRC_DATA_LO_UNION.src_data_31_0 = 0xffffffff;
  packet_addr->SRC_DATA_HI_UNION.src_data_63_32 = 0xffffffff;
 }
+
+void BlitSdma::BuildGetGlobalTimestampCommand(char* cmd_addr,
+                                              void* write_address) {
+  SDMA_PKT_TIMESTAMP* packet_addr =
+      reinterpret_cast<SDMA_PKT_TIMESTAMP*>(cmd_addr);
+
+  memset(packet_addr, 0, sizeof(SDMA_PKT_TIMESTAMP));
+
+  packet_addr->HEADER_UNION.op = SDMA_OP_TIMESTAMP;
+  packet_addr->HEADER_UNION.sub_op = SDMA_SUBOP_TIMESTAMP_GET_GLOBAL;
+
+  packet_addr->ADDR_LO_UNION.addr_31_0 = ptrlow32(write_address);
+  packet_addr->ADDR_HI_UNION.addr_63_32 = ptrhigh32(write_address);
+}
 }  // namespace amd
@@ -72,6 +72,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props)
      blit_h2d_(NULL),
      blit_d2h_(NULL),
      blit_d2d_(NULL),
+      local_region_(NULL),
      is_kv_device_(false),
      trap_code_buf_(NULL),
      trap_code_buf_size_(0),
@@ -79,7 +80,10 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props)
      memory_max_frequency_(0),
      ape1_base_(0),
      ape1_size_(0),
-      blit_initialized_(false) {
+      blit_initialized_(false),
+      end_ts_pool_size_(0),
+      end_ts_pool_counter_(0),
+      end_ts_base_addr_(NULL) {
  const bool is_apu_node = (properties_.NumCPUCores > 0);
  profile_ = (is_apu_node) ? HSA_PROFILE_FULL : HSA_PROFILE_BASE;

@@ -91,6 +95,7 @@ GpuAgent::GpuAgent(HSAuint32 node, const HsaNodeProperties& node_props)
  isa_ = (core::Isa*)core::IsaRegistry::GetIsa(core::Isa::Version(
      node_props.EngineId.ui32.Major, node_props.EngineId.ui32.Minor,
      node_props.EngineId.ui32.Stepping));
+
  // Check if the device is Kaveri, only on GPU device.
  if (isa_->GetMajorVersion() == 7 && isa_->GetMinorVersion() == 0 &&
      isa_->GetStepping() == 0) {
@@ -152,6 +157,10 @@ GpuAgent::~GpuAgent() {
    blit_d2h_ = NULL;
  }

+  if (end_ts_base_addr_ != NULL) {
+    core::Runtime::runtime_singleton_->FreeMemory(end_ts_base_addr_);
+  }
+
  if (ape1_base_ != 0) {
    _aligned_free(reinterpret_cast<void*>(ape1_base_));
  }
@@ -296,6 +305,10 @@ void GpuAgent::InitRegionList() {
              new MemoryRegion(false, false, this, mem_props[mem_idx]);

          regions_.push_back(region);
+
+          if (region->IsLocalMemory()) {
+            local_region_ = region;
+          }
          break;
        }
        case HSA_HEAPTYPE_SYSTEM:
@@ -372,6 +385,57 @@ void GpuAgent::InitCacheList() {
  }
 }

+bool GpuAgent::InitEndTsPool() {
+  if (HSA_PROFILE_FULL == profile_) {
+    return true;
+  }
+
+  if (end_ts_base_addr_.load(std::memory_order_acquire) != NULL) {
+    return true;
+  }
+
+  ScopedAcquire<KernelMutex> lock(&blit_lock_);
+
+  if (end_ts_base_addr_.load(std::memory_order_relaxed) != NULL) {
+    return true;
+  }
+
+  end_ts_pool_size_ = static_cast<uint32_t>(
+      (BlitSdma::kQueueSize + BlitSdma::kCopyPacketSize - 1) /
+      (BlitSdma::kCopyPacketSize));
+
+  // Allocate end timestamp object for both h2d and d2h DMA.
+  const size_t alloc_size = 2 * end_ts_pool_size_ * kTsSize;
+
+  core::Runtime* runtime = core::Runtime::runtime_singleton_;
+
+  uint64_t* buff = NULL;
+  if (HSA_STATUS_SUCCESS !=
+      runtime->AllocateMemory(true, local_region_, alloc_size,
+                              reinterpret_cast<void**>(&buff))) {
+    return false;
+  }
+
+  end_ts_base_addr_.store(buff, std::memory_order_release);
+
+  return true;
+}
+
+uint64_t* GpuAgent::ObtainEndTsObject() {
+  if (end_ts_base_addr_ == NULL) {
+    return NULL;
+  }
+
+  const uint32_t end_ts_index =
+      end_ts_pool_counter_.fetch_add(1U, std::memory_order_acq_rel) %
+      end_ts_pool_size_;
+  const static size_t kNumU64 = kTsSize / sizeof(uint64_t);
+  uint64_t* end_ts_addr = &end_ts_base_addr_[end_ts_index * kNumU64];
+  assert(IsMultipleOf(end_ts_addr, kTsSize));
+
+  return end_ts_addr;
+}
+
 hsa_status_t GpuAgent::IterateRegion(
    hsa_status_t (*callback)(hsa_region_t region, void* data),
    void* data) const {
@@ -460,18 +524,17 @@ void GpuAgent::InitDma() {
  // could give indication of DMA usage in the future. E.g.:
  // 1. Call to allow access API.
  // 2. Call to memory lock API.
-  if (!atomic::Load(&blit_initialized_, std::memory_order_acquire)) {
+  if (!blit_initialized_.load(std::memory_order_acquire)) {
    ScopedAcquire<KernelMutex> lock(&blit_lock_);
-    if (!atomic::Load(&blit_initialized_, std::memory_order_relaxed)) {
+    if (!blit_initialized_.load(std::memory_order_relaxed)) {
      // Try create SDMA blit first.
      if (core::Runtime::runtime_singleton_->flag().enable_sdma() &&
-          isa_->GetMajorVersion() == 8 && isa_->GetMinorVersion() == 0 &&
-          isa_->GetStepping() == 3) {
+          (HSA_PROFILE_BASE == profile_)) {
        blit_h2d_ = CreateBlitSdma();
        blit_d2h_ = CreateBlitSdma();

        if (blit_h2d_ != NULL && blit_d2h_ != NULL) {
-          atomic::Store(&blit_initialized_, true, std::memory_order_release);
+          blit_initialized_.store(true, std::memory_order_release);
          return;
        }
      }
@@ -487,7 +550,7 @@ void GpuAgent::InitDma() {
        blit_d2h_ = CreateBlitKernel();
      }

-      atomic::Store(&blit_initialized_, true, std::memory_order_release);
+      blit_initialized_.store(true, std::memory_order_release);
    }
  }
 }
@@ -536,7 +599,16 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
    static_cast<core::InterruptSignal&>(out_signal).DisableWaitEvent();
  }

-  return blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal);
+  hsa_status_t stat =
+      blit->SubmitLinearCopyCommand(dst, src, size, dep_signals, out_signal);
+
+  if (profiling_enabled() && HSA_STATUS_SUCCESS == stat) {
+    // Track the agent so we could translate the resulting timestamp to system
+    // domain correctly.
+    out_signal.async_copy_agent(this);
+  }
+
+  return stat;
 }

 hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) {
@@ -547,14 +619,30 @@ hsa_status_t GpuAgent::DmaFill(void* ptr, uint32_t value, size_t count) {
  return blit_d2d_->SubmitLinearFillCommand(ptr, value, count);
 }

+hsa_status_t GpuAgent::EnableDmaProfiling(bool enable) {
+  if (enable && !InitEndTsPool()) {
+    return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+  }
+
+  core::Blit* blit[3] = {blit_h2d_, blit_d2h_, blit_d2d_};
+  for (int i = 0; i < 3; ++i) {
+    if (blit[i] != NULL) {
+      const hsa_status_t stat = blit[i]->EnableProfiling(enable);
+      if (stat != HSA_STATUS_SUCCESS) {
+        return stat;
+      }
+    }
+  }
+
+  return HSA_STATUS_SUCCESS;
+}
+
 hsa_status_t GpuAgent::GetInfo(hsa_agent_info_t attribute, void* value) const {
  const size_t kNameSize = 64;  // agent, and vendor name size limit

  const core::ExtensionEntryPoints& extensions =
      core::Runtime::runtime_singleton_->extensions_;

-  hsa_agent_t agent = core::Agent::Convert(this);
-
  const size_t attribute_u = static_cast<size_t>(attribute);
  switch (attribute_u) {
    case HSA_AGENT_INFO_NAME:
@@ -229,6 +229,17 @@ hsa_status_t HSA_API
  return HSA_STATUS_SUCCESS;
 }

+hsa_status_t HSA_API hsa_amd_profiling_async_copy_enable(bool enable) {
+  IS_OPEN();
+
+  return core::Runtime::runtime_singleton_->IterateAgent(
+      [](hsa_agent_t agent_handle, void* data) -> hsa_status_t {
+        const bool enable = *(reinterpret_cast<bool*>(data));
+        return core::Agent::Convert(agent_handle)->profiling_enabled(enable);
+      },
+      reinterpret_cast<void*>(&enable));
+}
+
 hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time(
    hsa_agent_t agent_handle, hsa_signal_t hsa_signal,
    hsa_amd_profiling_dispatch_time_t* time) {
@@ -250,11 +261,40 @@ hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time(

  amd::GpuAgentInt* gpu_agent = static_cast<amd::GpuAgentInt*>(agent);

+  // Translate timestamp from GPU to system domain.
  gpu_agent->TranslateTime(signal, *time);

  return HSA_STATUS_SUCCESS;
 }

+hsa_status_t HSA_API hsa_amd_profiling_get_async_copy_time(
+    hsa_signal_t hsa_signal, hsa_amd_profiling_async_copy_time_t* time) {
+  IS_OPEN();
+
+  IS_BAD_PTR(time);
+
+  core::Signal* signal = core::Signal::Convert(hsa_signal);
+
+  IS_VALID(signal);
+
+  core::Agent* agent = signal->async_copy_agent();
+
+  if (agent == NULL) {
+    return HSA_STATUS_ERROR;
+  }
+
+  if (agent->device_type() == core::Agent::DeviceType::kAmdGpuDevice) {
+    // Translate timestamp from GPU to system domain.
+    static_cast<amd::GpuAgentInt*>(agent)->TranslateTime(signal, *time);
+    return HSA_STATUS_SUCCESS;
+  }
+
+  // The timestamp is already in system domain.
+  time->start = signal->signal_.start_ts;
+  time->end = signal->signal_.end_ts;
+  return HSA_STATUS_SUCCESS;
+}
+
 hsa_status_t HSA_API
    hsa_amd_profiling_convert_tick_to_system_domain(hsa_agent_t agent_handle,
                                                    uint64_t agent_tick,
@@ -424,19 +424,37 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent& dst_agent,
  }

  // For cpu to cpu, fire and forget a copy thread.
-  std::thread([](void* dst, const void* src, size_t size,
-                 std::vector<core::Signal*> dep_signals,
-                 core::Signal* completion_signal) {
-                for (core::Signal* dep : dep_signals) {
-                  dep->WaitRelaxed(HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX,
-                                   HSA_WAIT_STATE_BLOCKED);
-                }
+  const bool profiling_enabled =
+      (dst_agent.profiling_enabled() || src_agent.profiling_enabled());
+  std::thread(
+      [](void* dst, const void* src, size_t size,
+         std::vector<core::Signal*> dep_signals,
+         core::Signal* completion_signal, bool profiling_enabled) {
+        for (core::Signal* dep : dep_signals) {
+          dep->WaitRelaxed(HSA_SIGNAL_CONDITION_EQ, 0, UINT64_MAX,
+                           HSA_WAIT_STATE_BLOCKED);
+        }

-                memcpy(dst, src, size);
+        if (profiling_enabled) {
+          HsaClockCounters clocks = {0};
+          core::Runtime::runtime_singleton_->GetSystemInfo(
+              HSA_SYSTEM_INFO_TIMESTAMP, reinterpret_cast<void*>(&clocks));
+          completion_signal->signal_.start_ts = clocks.SystemClockCounter;
+        }

-                completion_signal->SubRelease(1);
-              },
-              dst, src, size, dep_signals, &completion_signal).detach();
+        memcpy(dst, src, size);
+
+        if (profiling_enabled) {
+          HsaClockCounters clocks = {0};
+          core::Runtime::runtime_singleton_->GetSystemInfo(
+              HSA_SYSTEM_INFO_TIMESTAMP, reinterpret_cast<void*>(&clocks));
+          completion_signal->signal_.end_ts = clocks.SystemClockCounter;
+        }
+
+        completion_signal->SubRelease(1);
+      },
+      dst, src, size, dep_signals, &completion_signal,
+      profiling_enabled).detach();

  return HSA_STATUS_SUCCESS;
 }
@@ -211,6 +211,23 @@ typedef struct hsa_amd_profiling_dispatch_time_s {
  uint64_t end;
 } hsa_amd_profiling_dispatch_time_t;

+/**
+ * @brief Structure containing profiling async copy time information.
+ *
+ * Times are reported as ticks in the domain of the HSA system clock.
+ * The HSA system clock tick and frequency is obtained via hsa_system_get_info.
+ */
+typedef struct hsa_amd_profiling_async_copy_time_s {
+  /**
+   * Async copy processing start time.
+   */
+  uint64_t start;
+  /**
+   * Async copy completion time.
+   */
+  uint64_t end;
+} hsa_amd_profiling_async_copy_time_t;
+
 /**
 * @brief Enable or disable profiling capability of a queue.
 *
@@ -230,10 +247,34 @@ typedef struct hsa_amd_profiling_dispatch_time_s {
 hsa_status_t HSA_API
    hsa_amd_profiling_set_profiler_enabled(hsa_queue_t* queue, int enable);

+/**
+ * @brief Enable or disable asynchronous memory copy profiling.
+ *
+ * @details The runtime will provide the copy processing start timestamp and
+ * completion timestamp of each call to hsa_amd_memory_async_copy if the
+ * async copy profiling is enabled prior to the call to
+ * hsa_amd_memory_async_copy. The completion signal object is used to
+ * hold the last async copy start and end timestamp. The client can retrieve
+ * these timestamps via call to hsa_amd_profiling_get_async_copy_time.
+ *
+ * @param[in] enable True to enable profiling. False to disable profiling.
+ *
+ * @retval ::HSA_STATUS_SUCCESS The function has been executed successfully.
+ *
+ * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been
+ * initialized.
+ *
+ * @retval ::HSA_STATUS_ERROR_OUT_OF_RESOURCES Failed on allocating resources
+ * needed to profile the asynchronous copy.
+ */
+hsa_status_t HSA_API
+    hsa_amd_profiling_async_copy_enable(bool enable);
+
 /**
 * @brief Retrieve packet processing time stamps.
 *
- * @param[in] agent The agent with which the signal was last used.  For instance,
+ * @param[in] agent The agent with which the signal was last used.  For
+ *instance,
 * if the profiled dispatch packet is dispatched on to queue Q, which was
 * created on agent A, then this parameter must be A.
 *
@@ -261,6 +302,30 @@ hsa_status_t HSA_API hsa_amd_profiling_get_dispatch_time(
    hsa_agent_t agent, hsa_signal_t signal,
    hsa_amd_profiling_dispatch_time_t* time);

+/**
+ * @brief Retrieve asynchronous copy timestamps.
+ *
+ * @details Async copy profiling is enabled via call to
+ * hsa_amd_profiling_async_copy_enable.
+ *
+ * @param[in] signal A signal used as the completion signal of the call to
+ * hsa_amd_memory_async_copy.
+ *
+ * @param[out] time Async copy processing timestamps in the HSA system clock
+ * domain.
+ *
+ * @retval ::HSA_STATUS_SUCCESS The function has been executed successfully.
+ *
+ * @retval ::HSA_STATUS_ERROR_NOT_INITIALIZED The HSA runtime has not been
+ * initialized.
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_SIGNAL The signal is invalid.
+ *
+ * @retval ::HSA_STATUS_ERROR_INVALID_ARGUMENT @p time is NULL.
+ */
+hsa_status_t HSA_API hsa_amd_profiling_get_async_copy_time(
+    hsa_signal_t signal, hsa_amd_profiling_async_copy_time_t* time);
+
 /**
 * @brief Computes the frequency ratio and offset between the agent clock and
 * HSA system clock and converts the agent's tick to HSA system domain tick.