SWDEV-432445: ATT continuous mode update part2. Added codeobj tracking.

Change-Id: I1b58af70d221bbeb9b4cab960d26357a504045dd


[ROCm/rocprofiler commit: edf93d48ab]

projects/rocprofiler/src/core/hsa/hsa_support.cpp

@@ -503,7 +503,7 @@ hsa_status_t CodeObjectCallback(hsa_executable_t executable,
   if (data.codeobj.unload)
     codeobj_capture_instance::Unload(data.codeobj.load_base);
   else
-    codeobj_capture_instance::Load(data.codeobj.load_base,
+    codeobj_capture_instance::Load(data.codeobj.load_base, data.codeobj.load_size,
             uri_str, data.codeobj.memory_base, data.codeobj.memory_size);
 
   hsa_executable_iterate_agent_symbols(executable, data.codeobj.agent,

projects/rocprofiler/src/core/hsa/packets/packets_generator.cpp

@@ -18,8 +18,6 @@
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE. */
 
-//#define HSA_ATT_MARKER_ENABLE
-
 #include "packets_generator.h"
 #include "src/api/rocprofiler_singleton.h"
 
@@ -618,10 +616,13 @@ hsa_ven_amd_aqlprofile_profile_t* GenerateATTPackets(
 
 // Generate ATT tracer marker packets. Also generate and return
 // the descriptor object which has the PM4 buffer for inserting data
-hsa_ven_amd_aqlprofile_descriptor_t
-GenerateATTMarkerPackets(hsa_agent_t gpu_agent, packet_t& marker_packet, uint32_t data)
+hsa_ven_amd_aqlprofile_descriptor_t GenerateATTMarkerPackets(
+  hsa_agent_t gpu_agent,
+  packet_t& marker_packet,
+  uint32_t data,
+  hsa_ven_amd_aqlprofile_att_marker_channel_t channel
+)
 {
-#ifdef HSA_ATT_MARKER_ENABLE
   // Preparing the profile structure to get the packets
   auto pool = rocprofiler::HSASupport_Singleton::GetInstance()
                                                   .GetHSAAgentInfo(gpu_agent.handle)
@@ -635,7 +636,6 @@ GenerateATTMarkerPackets(hsa_agent_t gpu_agent, packet_t& marker_packet, uint32_
     {}, desc
   };
 
-  hsa_ven_amd_aqlprofile_att_marker_channel_t channel = HSA_VEN_AMD_AQLPROFILE_ATT_CHANNEL_2;
   hsa_status_t status = hsa_ven_amd_aqlprofile_att_marker(&profile, &marker_packet, data, channel);
   if (status != HSA_STATUS_SUCCESS)
   {
@@ -645,9 +645,6 @@ GenerateATTMarkerPackets(hsa_agent_t gpu_agent, packet_t& marker_packet, uint32_
   }
 
   return desc;
-#else
-  return {nullptr,0};
-#endif
 }
 
 void AddVendorSpecificPacket(const packet_t* packet,

projects/rocprofiler/src/core/hsa/packets/packets_generator.h

@@ -61,7 +61,12 @@ hsa_ven_amd_aqlprofile_profile_t* GenerateATTPackets(
     packet_t* stop_packet, size_t att_buffer_size);
 
 hsa_ven_amd_aqlprofile_descriptor_t
-GenerateATTMarkerPackets(hsa_agent_t gpu_agent, packet_t& marker_packet, uint32_t data);
+GenerateATTMarkerPackets(
+  hsa_agent_t gpu_agent,
+  packet_t& marker_packet,
+  uint32_t data,
+  hsa_ven_amd_aqlprofile_att_marker_channel_t channel
+);
 
 uint8_t* AllocateSysMemory(hsa_agent_t gpu_agent, size_t size, hsa_amd_memory_pool_t* cpu_pool);
 

projects/rocprofiler/src/core/isa_capture/code_object_track.cpp

@@ -48,7 +48,7 @@ std::mutex codeobj_record::mutex;
 std::unordered_map<uint64_t, CodeobjPtr> codeobj_record::codeobjs{};
 std::unordered_map<uint64_t, codeobj_record::RecordInstance> codeobj_record::record_id_map{};
 std::unordered_set<codeobj_record*> codeobj_record::listeners;
-std::atomic<uint32_t> codeobj_capture_instance::loadcount{0};
+std::atomic<uint32_t> codeobj_capture_instance::eventcount{0};
 
 // Codeobj Record
 codeobj_record::codeobj_record(rocprofiler_codeobj_capture_mode_t mode) : capture_mode(mode){};
@@ -71,16 +71,17 @@ void codeobj_record::stop_capture() {
 // Codeobj Capture
 void codeobj_capture_instance::Load(
   uint64_t addr,
+  uint64_t load_size,
   const std::string& URI,
   uint64_t mem_addr,
-  uint64_t size
+  uint64_t mem_size
 ) {
-  uint32_t id = loadcount.fetch_add(1, std::memory_order_relaxed)+1;
+  uint32_t id = eventcount.fetch_add(1, std::memory_order_relaxed)+1;
   auto time = rocprofiler::ROCProfiler_Singleton::GetInstance().timestamp_ns().value;
 
   std::lock_guard<std::mutex> lock(codeobj_record::mutex);
 
-  auto inst = std::make_shared<codeobj_capture_instance>(addr, URI, mem_addr, size, time, id);
+  auto inst = std::make_shared<codeobj_capture_instance>(addr, load_size, URI, mem_addr, mem_size, time, id);
   codeobj_record::codeobjs[addr] = inst;
   for (auto* listen : codeobj_record::listeners) listen->addcapture(inst);
 }
@@ -90,6 +91,7 @@ void codeobj_capture_instance::Unload(uint64_t addr) {
 
   if (codeobj_record::codeobjs.find(addr) == codeobj_record::codeobjs.end()) return;
 
+  eventcount.fetch_add(1, std::memory_order_relaxed)+1;
   auto time = rocprofiler::ROCProfiler_Singleton::GetInstance().timestamp_ns().value;
   codeobj_record::codeobjs.at(addr)->end_time = time;
   codeobj_record::codeobjs.erase(addr);

projects/rocprofiler/src/core/isa_capture/code_object_track.hpp

@@ -34,30 +34,44 @@
  */
 class codeobj_capture_instance {
  public:
-  codeobj_capture_instance(uint64_t _addr, const std::string& _uri, uint64_t mem_addr,
-                           uint64_t mem_size, uint64_t start_time, uint32_t id)
-    : addr(_addr), start_time(start_time), URI(_uri),
+  codeobj_capture_instance(
+    uint64_t _addr,
+    uint64_t _load_size,
+    const std::string& _uri,
+    uint64_t mem_addr,
+    uint64_t mem_size,
+    uint64_t start_time,
+    uint32_t id
+  )
+    : addr(_addr), load_size(_load_size), start_time(start_time), URI(_uri),
     mem_addr(mem_addr), mem_size(mem_size), load_id(id) {};
 
   void setmode(rocprofiler_codeobj_capture_mode_t mode);
 
   rocprofiler_intercepted_codeobj_t get() const {
     const char* buf_ptr = buffer.size() ? buffer.data() : nullptr;
-    return {URI.c_str(), addr, mem_size, buf_ptr, buffer.size(), start_time, end_time, load_id};
+    return {URI.c_str(), addr, load_size, buf_ptr, buffer.size(), start_time, end_time, load_id};
   };
 
   const uint64_t addr;
+  const uint64_t load_size;
   const uint64_t start_time;
   const uint32_t load_id;
 
-  static void Load(uint64_t addr, const std::string& URI, uint64_t mem_addr, uint64_t mem_size);
+  static void Load(
+    uint64_t addr,
+    uint64_t load_size,
+    const std::string& URI,
+    uint64_t mem_addr,
+    uint64_t mem_size
+  );
   static void Unload(uint64_t addr);
-  static uint32_t GetLoadCount() { return loadcount.load(std::memory_order_relaxed); }
+  static uint32_t GetEventCount() { return eventcount.load(std::memory_order_relaxed); }
 
  private:
   //! 32 bits ID because this is the natural channel width for ATT Markers.
   //! There is no world in which 4 billions markers can be sent anyway.
-  static std::atomic<uint32_t> loadcount;
+  static std::atomic<uint32_t> eventcount;
   void reset(rocprofiler_codeobj_capture_mode_t mode);
 
   std::pair<size_t, size_t> parse_uri();

projects/rocprofiler/src/core/session/att/att.h

@@ -91,10 +91,21 @@ public:
   );
 
   void InsertMarker(
+    std::vector<packet_t>& transformed_packets,
+    hsa_agent_t agent,
+    uint32_t data,
+    hsa_ven_amd_aqlprofile_att_marker_channel_t channel
+  );
+  void InsertUnloadMarker(
     std::vector<packet_t>& transformed_packets,
     hsa_agent_t agent,
     uint32_t data
   );
+  void InsertLoadMarker(
+    std::vector<packet_t>& transformed_packets,
+    hsa_agent_t agent,
+    rocprofiler_intercepted_codeobj_t codeobj
+  );
 
   void SetParameters(const std::vector<rocprofiler_att_parameter_t>& params) {
     att_parameters_data = params;
@@ -125,7 +136,7 @@ protected:
   static std::mutex att_enable_disable_mutex;
 
 private:
-  uint32_t codeobj_load_cnt = 0;
+  uint32_t codeobj_event_cnt = 0;
 
   static void AddAttRecord(
     rocprofiler_record_att_tracer_t* record,
@@ -176,6 +187,9 @@ private:
 
   std::mutex sessions_pending_signals_lock_;
   std::map<uint32_t, std::vector<att_pending_signal_t>> sessions_pending_signals_;
+
+  rocprofiler_record_id_t capture_id;
+  std::unordered_set<uint32_t> active_capture_event_ids;
 };
 
 }  // namespace att

projects/rocprofiler/src/core/session/att/continuous.cpp

@@ -27,19 +27,71 @@
 
 #define __NR_gettid 186
 
+#define ATT_MARKER_HEADER_CHANNEL HSA_VEN_AMD_AQLPROFILE_ATT_CHANNEL_0
+#define ATT_MARKER_SIZE_CHANNEL HSA_VEN_AMD_AQLPROFILE_ATT_CHANNEL_1
+#define ATT_MARKER_LO_CHANNEL HSA_VEN_AMD_AQLPROFILE_ATT_CHANNEL_2
+#define ATT_MARKER_HI_CHANNEL HSA_VEN_AMD_AQLPROFILE_ATT_CHANNEL_3
+
+enum rocprofiler_att_marker_type_t {
+  ROCPROFILER_ATT_MARKER_LOAD = 0,
+  ROCPROFILER_ATT_MARKER_UNLOAD = 1
+};
+
+union att_header_marker_t
+{
+  uint32_t raw;
+  struct {
+    uint32_t type : 2;
+    uint32_t id : 30;
+  };
+};
+
 namespace rocprofiler {
 
 namespace att {
 
-void AttTracer::InsertMarker(
+void AttTracer::InsertUnloadMarker(
   std::vector<packet_t>& transformed_packets,
   hsa_agent_t agent,
   uint32_t data
+) {
+  att_header_marker_t header{.raw = 0};
+  header.type = ROCPROFILER_ATT_MARKER_UNLOAD;
+  header.id = data;
+  hsa_ven_amd_aqlprofile_att_marker_channel_t channel = ATT_MARKER_HEADER_CHANNEL;
+
+  this->InsertMarker(transformed_packets, agent, header.raw, channel);
+}
+
+void AttTracer::InsertLoadMarker(
+  std::vector<packet_t>& transformed_packets,
+  hsa_agent_t agent,
+  rocprofiler_intercepted_codeobj_t codeobj
+) {
+  this->InsertMarker(transformed_packets, agent, codeobj.mem_size, ATT_MARKER_SIZE_CHANNEL);
+
+  uint64_t addr = codeobj.base_address;
+  this->InsertMarker(transformed_packets, agent, addr & ((1ul << 32)-1), ATT_MARKER_LO_CHANNEL);
+  this->InsertMarker(transformed_packets, agent, addr >> 32, ATT_MARKER_HI_CHANNEL);
+
+  att_header_marker_t header{.raw = 0};
+  header.type = ROCPROFILER_ATT_MARKER_LOAD;
+  header.id = codeobj.att_marker_id;
+  this->InsertMarker(transformed_packets, agent, header.raw, ATT_MARKER_HEADER_CHANNEL);
+}
+
+void AttTracer::InsertMarker(
+  std::vector<packet_t>& transformed_packets,
+  hsa_agent_t agent,
+  uint32_t data,
+  hsa_ven_amd_aqlprofile_att_marker_channel_t channel
 ) {
   packet_t marker_packet{};
-  auto desc = Packet::GenerateATTMarkerPackets(agent, marker_packet, data);
+  auto desc = Packet::GenerateATTMarkerPackets(agent, marker_packet, data, channel);
   if (desc.ptr && desc.size)
     Packet::AddVendorSpecificPacket(&marker_packet, &transformed_packets, hsa_signal_t{.handle = 0});
+  else
+    rocprofiler::warning("Could not add ATT Marker");
 }
 
 
@@ -63,25 +115,26 @@ bool AttTracer::ATTContiguousWriteInterceptor(
 
   // att start
   // Getting Queue Data and Information
+  auto agent_handle = queue_info.GetGPUAgent().handle;
   rocprofiler::HSAAgentInfo& agentInfo = rocprofiler::HSASupport_Singleton::GetInstance()
-                                        .GetHSAAgentInfo(queue_info.GetGPUAgent().handle);
+                                        .GetHSAAgentInfo(agent_handle);
 
   auto dispatchPackets = Packet::ExtractDispatchPackets(packets, pkt_count);
   if (dispatchPackets.size() == 0) return false;
 
   size_t writer_id = WRITER_ID.fetch_add(dispatchPackets.size(), std::memory_order_relaxed);
-  uint32_t new_load_cnt = codeobj_capture_instance::GetLoadCount();
+  uint32_t new_load_cnt = codeobj_capture_instance::GetEventCount();
 
   auto bInsertStart = RequiresStartPacket(writer_id, dispatchPackets.size());
   {
     std::lock_guard<std::mutex> lk(att_enable_disable_mutex);
     // If att_start already exists, don't start again
-    auto agent_pending_packets = pending_stop_packets.find(queue_info.GetGPUAgent().handle);
+    auto agent_pending_packets = pending_stop_packets.find(agent_handle);
     if (agent_pending_packets != pending_stop_packets.end())
       bInsertStart = {};
 
     // If nothing will be added or removed, return
-    if (!bInsertStart && codeobj_load_cnt == new_load_cnt)
+    if (!bInsertStart && codeobj_event_cnt == new_load_cnt)
     {
       if (
         agent_pending_packets == pending_stop_packets.end() ||
@@ -107,7 +160,7 @@ bool AttTracer::ATTContiguousWriteInterceptor(
     }
 
     uint64_t IsGFX9 = HSASupport_Singleton::GetInstance()
-                      .GetHSAAgentInfo(queue_info.GetGPUAgent().handle)
+                      .GetHSAAgentInfo(agent_handle)
                       .GetDeviceInfo()
                       .getName()
                       .find("gfx9") != std::string::npos;
@@ -134,33 +187,58 @@ bool AttTracer::ATTContiguousWriteInterceptor(
       0
     );
 
-    codeobj_record::make_capture(rocprofiler_record_id_t{record_id}, capturem, IsGFX9);
-    codeobj_record::start_capture(rocprofiler_record_id_t{record_id});
+    this->capture_id = rocprofiler_record_id_t{record_id};
+    codeobj_record::make_capture(this->capture_id, capturem, IsGFX9);
+    codeobj_record::start_capture(this->capture_id);
 
     stop_packet.header = HSA_PACKET_TYPE_VENDOR_SPECIFIC << HSA_PACKET_HEADER_TYPE;
     std::lock_guard<std::mutex> lk(att_enable_disable_mutex);
-    pending_stop_packets[queue_info.GetGPUAgent().handle]
+    pending_stop_packets[agent_handle]
                 = {record_id, writer_id, bInsertStart->second, session_id_, stop_packet};
   }
 
-  if (codeobj_load_cnt != new_load_cnt)
+  bool bHasPending = false;
   {
-    codeobj_load_cnt = new_load_cnt;
-    InsertMarker(transformed_packets, queue_info.GetGPUAgent(), new_load_cnt);
+    std::lock_guard<std::mutex> lk(att_enable_disable_mutex);
+    bHasPending = pending_stop_packets.find(agent_handle) != pending_stop_packets.end();
+  }
+
+  if (bHasPending && (bInsertStart || codeobj_event_cnt != new_load_cnt))
+  {
+    codeobj_event_cnt = new_load_cnt;
+
+    auto symbols = codeobj_record::get_capture(this->capture_id);
+    std::unordered_set<uint32_t> current_ids;
+
+    for (size_t s=0; s<symbols.count; s++)
+      current_ids.insert(symbols.symbols[s].att_marker_id);
+
+    for (uint32_t prev_id : active_capture_event_ids)
+      if (current_ids.find(prev_id) == current_ids.end())
+        InsertUnloadMarker(transformed_packets, queue_info.GetGPUAgent(), prev_id);
+
+    for (size_t s=0; s<symbols.count; s++)
+    {
+      auto& symbol = symbols.symbols[s];
+      if (active_capture_event_ids.find(symbol.att_marker_id) == active_capture_event_ids.end())
+        InsertLoadMarker(transformed_packets, queue_info.GetGPUAgent(), symbol);
+    }
+
+    active_capture_event_ids = std::move(current_ids);
   }
 
   // Searching across all the packets given during this write
   for (size_t i = 0; i < pkt_count; ++i)
     transformed_packets.emplace_back(packets_arr[i]);
 
+  if (bHasPending)
   {
     std::lock_guard<std::mutex> lk(att_enable_disable_mutex);
-    auto agent_pending_packets = pending_stop_packets.find(queue_info.GetGPUAgent().handle);
+    auto agent_pending_packets = pending_stop_packets.at(agent_handle);
 
-    if (agent_pending_packets != pending_stop_packets.end() &&
-        agent_pending_packets->second.last_kernel_exec <= writer_id + dispatchPackets.size()
-    ) {
-      const ATTRecordSignal& rsignal = agent_pending_packets->second;
+    if (agent_pending_packets.last_kernel_exec <= writer_id + dispatchPackets.size())
+    {
+      const ATTRecordSignal& rsignal = agent_pending_packets;
       // Adding a barrier packet with the original packet's completion signal.
       hsa_signal_t interrupt_signal;
       CreateSignal(0, &interrupt_signal);
@@ -180,8 +258,10 @@ bool AttTracer::ATTContiguousWriteInterceptor(
           interrupt_signal
       });
 
-      codeobj_record::stop_capture(rocprofiler_record_id_t{rsignal.record_id});
-      pending_stop_packets.erase(queue_info.GetGPUAgent().handle);
+      //codeobj_record::stop_capture(rocprofiler_record_id_t{rsignal.record_id});
+      codeobj_record::stop_capture(this->capture_id);
+      active_capture_event_ids.clear();
+      pending_stop_packets.erase(agent_handle);
     }
   }
 

projects/rocprofiler/src/util/hsa_rsrc_factory.cpp

@@ -316,10 +316,8 @@ hsa_status_t HsaRsrcFactory::LoadAqlProfileLib(aqlprofile_pfn_t* api) {
   api->hsa_ven_amd_aqlprofile_iterate_data =
       (decltype(::hsa_ven_amd_aqlprofile_iterate_data)*)dlsym(
           handle, "hsa_ven_amd_aqlprofile_iterate_data");
-#ifdef HSA_ATT_MARKER_ENABLE
   api->hsa_ven_amd_aqlprofile_att_marker = (decltype(::hsa_ven_amd_aqlprofile_att_marker)*)
                                           dlsym(handle, "hsa_ven_amd_aqlprofile_att_marker");
-#endif
   return HSA_STATUS_SUCCESS;
 }