SQTT local allocation

Change-Id: Ie4a150ad0dc141226f6f1c571916c5a526dd723c

src/core/context.h

@@ -426,11 +426,14 @@ class Context {
         rinfo->data.kind = ROCPROFILER_DATA_KIND_INT64;
       } else if (ainfo_type == HSA_VEN_AMD_AQLPROFILE_INFO_SQTT_DATA) {
         if (rinfo->data.result_bytes.copy) {
+          const bool sqtt_local = SqttProfile::IsLocal();
+          util::HsaRsrcFactory* hsa_rsrc = &util::HsaRsrcFactory::Instance();
           if (sample_id == 0) {
               const uint32_t output_buffer_size = profile->output_buffer.size;
-              util::HsaRsrcFactory* hsa_rsrc = &util::HsaRsrcFactory::Instance();
+              const uint32_t output_buffer_size64 = profile->output_buffer.size / sizeof(uint64_t);
               const util::AgentInfo* agent_info = hsa_rsrc->GetAgentInfo(profile->agent);
-              void* ptr = hsa_rsrc->AllocateSysMemory(agent_info, output_buffer_size);
+              void* ptr = (sqtt_local) ? hsa_rsrc->AllocateSysMemory(agent_info, output_buffer_size) :
+                                         calloc(output_buffer_size64, sizeof(uint64_t));
               rinfo->data.result_bytes.size = output_buffer_size;
               rinfo->data.result_bytes.ptr = ptr;
               callback_data->ptr = reinterpret_cast<char*>(ptr);
@@ -448,13 +451,19 @@ class Context {
             else EXC_RAISING(HSA_STATUS_ERROR, "SQTT data out of output buffer");
           }
 
-          bool suc = util::HsaRsrcFactory::Memcpy(profile->agent, dest, src, size);
+          bool suc = true;
+          if (sqtt_local) {
+            suc = hsa_rsrc->Memcpy(profile->agent, dest, src, size);
+          } else {
+            memcpy(dest, src, size);
+          }
           if (suc) {
             *header = size;
             callback_data->ptr = dest + align_size(size, sizeof(uint32_t));
             rinfo->data.result_bytes.instance_count = sample_id + 1;
             rinfo->data.kind = ROCPROFILER_DATA_KIND_BYTES;
-          }
+          } else
+            EXC_RAISING(HSA_STATUS_ERROR, "Agent Memcpy failed, dst(" << (void*)dest << ") src(" << (void*)src << ") size(" << size << ")");
         } else {
           if (sample_id == 0) {
             rinfo->data.result_bytes.ptr = profile->output_buffer.ptr;

src/core/profile.h

@@ -202,7 +202,7 @@ class PmcProfile : public Profile {
 
   hsa_status_t Allocate(util::HsaRsrcFactory* rsrc) {
     profile_.command_buffer.ptr =
-        rsrc->AllocateSysMemory(agent_info_, profile_.command_buffer.size);
+      rsrc->AllocateSysMemory(agent_info_, profile_.command_buffer.size);
     profile_.output_buffer.ptr = rsrc->AllocateSysMemory(agent_info_, profile_.output_buffer.size);
     return (profile_.command_buffer.ptr && profile_.output_buffer.ptr) ? HSA_STATUS_SUCCESS
                                                                        : HSA_STATUS_ERROR;
@@ -213,6 +213,8 @@ class SqttProfile : public Profile {
  public:
   static inline void SetSize(const uint32_t& size) { output_buffer_size_ = size; }
   static inline uint32_t GetSize() { return output_buffer_size_; }
+  static inline void SetLocal(const bool& b) { output_buffer_local_ = b; }
+  static inline bool IsLocal() { return output_buffer_local_; }
 
   SqttProfile(const util::AgentInfo* agent_info) : Profile(agent_info) {
     profile_.type = HSA_VEN_AMD_AQLPROFILE_EVENT_TYPE_SQTT;
@@ -227,16 +229,18 @@ class SqttProfile : public Profile {
 
   hsa_status_t Allocate(util::HsaRsrcFactory* rsrc) {
     profile_.command_buffer.ptr =
-        rsrc->AllocateSysMemory(agent_info_, profile_.command_buffer.size);
+      rsrc->AllocateSysMemory(agent_info_, profile_.command_buffer.size);
     profile_.output_buffer.size = output_buffer_size_;
-    profile_.output_buffer.ptr =
-        rsrc->AllocateLocalMemory(agent_info_, profile_.output_buffer.size);
+    profile_.output_buffer.ptr = (output_buffer_local_) ?
+      rsrc->AllocateLocalMemory(agent_info_, profile_.output_buffer.size) :
+      rsrc->AllocateSysMemory(agent_info_, profile_.output_buffer.size);
     return (profile_.command_buffer.ptr && profile_.output_buffer.ptr) ? HSA_STATUS_SUCCESS
                                                                        : HSA_STATUS_ERROR;
   }
 
  private:
   static uint32_t output_buffer_size_;
+  static bool output_buffer_local_;
 };
 
 }  // namespace rocprofiler

src/core/rocprofiler.cpp

@@ -127,6 +127,7 @@ bool LoadTool() {
     rocprofiler_settings_t settings{};
     settings.intercept_mode = (intercept_mode) ? 1 : 0;
     settings.sqtt_size = SqttProfile::GetSize();
+    settings.sqtt_local = SqttProfile::IsLocal() ? 1: 0;
     settings.timeout = Context::GetTimeout();
     settings.timestamp_on = InterceptQueue::IsTrackerOn() ? 1 : 0;
 
@@ -135,6 +136,7 @@ bool LoadTool() {
 
     intercept_mode = (settings.intercept_mode != 0);
     SqttProfile::SetSize(settings.sqtt_size);
+    SqttProfile::SetLocal(settings.sqtt_local != 0);
     Context::SetTimeout(settings.timeout);
     InterceptQueue::SetTimeout(settings.timeout);
     InterceptQueue::TrackerOn(settings.timestamp_on != 0);
@@ -186,6 +188,7 @@ const MetricsDict* GetMetrics(const hsa_agent_t& agent) {
 rocprofiler_properties_t rocprofiler_properties;
 uint64_t Context::timeout_ = UINT64_MAX;
 uint32_t SqttProfile::output_buffer_size_ = 0x2000000;  // 32M
+bool SqttProfile::output_buffer_local_ = true;
 Tracker::mutex_t Tracker::mutex_;
 util::Logger::mutex_t util::Logger::mutex_;
 util::Logger* util::Logger::instance_ = NULL;