Added option to control how codeobj is dumped from ATT

Change-Id: Ie76aeea1193c7ba8fe7f51be159516f8a9eab55f [ROCm/rocprofiler commit: 99b14fc9f8]
2023-11-08 17:35:43 -03:00
@@ -65,6 +65,9 @@ class att_plugin_t {

    header.raw = reinterpret_cast<uint64_t>(data);
    header.reserved = 0x11;
+
+    isa_mode = static_cast<decltype(isa_mode)>(header.isadumpmode);
+    header.isadumpmode = 0;
  }

  bool MPI_ENABLE = false;
@@ -72,6 +75,14 @@ class att_plugin_t {
  std::mutex writing_lock;
  bool is_valid_{true};
  rocprofiler::att_header_packet_t header{.raw = 0};
+  rocprofiler::rocprofiler_att_isa_dump_mode isa_mode = rocprofiler::ISA_MODE_DUMP_ALL;
+
+  bool CheckAddrMatches(uint64_t kernel_addr, uint64_t base_address, uint64_t size)
+  {
+    if (isa_mode == rocprofiler::ISA_MODE_DUMP_ALL)
+      return true;
+    return (kernel_addr >= base_address) && (kernel_addr < base_address + size);
+  }

  inline bool att_file_exists(const std::string& name) {
    struct stat buffer;
@@ -133,7 +144,7 @@ class att_plugin_t {
                                 << '\n';

    // iterate over each shader engine att trace
-    header.navi = !att_tracer_record->intercept_list.userdata;
+    header.navi = !att_tracer_record->intercept_list.userdata & 0x1;
    int se_num = att_tracer_record->shader_engine_data_count;
    for (int i = 0; i < se_num; i++) {
      if (!att_tracer_record->shader_engine_data ||
@@ -155,6 +166,11 @@ class att_plugin_t {
      out.write(data_buffer_ptr, se_att_trace->buffer_size);
    }

+    if (isa_mode == rocprofiler::ISA_MODE_DUMP_NONE)
+      return 0;
+
+    uint64_t kernel_addr = att_tracer_record->intercept_list.userdata >> 1;
+
    std::ofstream isafile(outfilepath + "_isa.s");
    if (!isafile.is_open()) {
      std::cerr << "Could not open ISA file: " << outfilepath << "_isa.s" << std::endl;
@@ -166,11 +182,13 @@ class att_plugin_t {
      const rocprofiler_intercepted_codeobj_t& symbol =
          att_tracer_record->intercept_list.symbols[i];

+      if (!CheckAddrMatches(kernel_addr, symbol.base_address, symbol.mem_size)) continue;
+
      std::unique_ptr<CodeObjectBinary> binary;
      std::unique_ptr<code_object_decoder_t> decoder;

-      if (symbol.data && symbol.size) {
-        decoder = std::make_unique<code_object_decoder_t>(symbol.data, symbol.size);
+      if (symbol.data && symbol.data_size) {
+        decoder = std::make_unique<code_object_decoder_t>(symbol.data, symbol.data_size);
      } else if (std::string(symbol.filepath).find("file://") != std::string::npos) {
        binary = std::make_unique<CodeObjectBinary>(symbol.filepath);
        decoder =
@@ -179,6 +197,11 @@ class att_plugin_t {
        continue;
      }

+      if (isa_mode == rocprofiler::ISA_MODE_DUMP_KERNEL)
+        decoder->disassemble_single_kernel(kernel_addr-symbol.base_address);
+      else
+        decoder->disassemble_kernels();
+
      for (auto& instance : decoder->instructions) {
        uint64_t addr = instance.address + symbol.base_address;

@@ -109,7 +109,11 @@ code_object_decoder_t::code_object_decoder_t(const char* codeobj_data, uint64_t
    }
    // load_symbol_map();
  }
-  disassemble_kernels();
+
+  disassembly = std::make_unique<DisassemblyInstance>(*this);
+  m_symbol_map = disassembly->GetKernelMap();
+
+  //disassemble_kernels();
 }


@@ -170,8 +174,11 @@ void code_object_decoder_t::disassemble_kernel(uint64_t faddr, uint64_t vaddr) {
 }

 void code_object_decoder_t::disassemble_kernels() {
-  disassembly = std::make_unique<DisassemblyInstance>(*this);
-  m_symbol_map = disassembly->GetKernelMap();
-
  for (auto& [vaddr, v] : m_symbol_map) disassemble_kernel(v.faddr, vaddr);
 }
+
+void code_object_decoder_t::disassemble_single_kernel(uint64_t kaddr) {
+  for (auto& [vaddr, v] : m_symbol_map)
+    if (kaddr >= vaddr && kaddr < vaddr + v.mem_size)
+      disassemble_kernel(v.faddr, vaddr);
+}
@@ -39,7 +39,9 @@ class code_object_decoder_t {
  ~code_object_decoder_t();

  void disassemble_kernel(uint64_t faddr, uint64_t vaddr);
+  void disassemble_single_kernel(uint64_t kaddr);
  void disassemble_kernels();
+
  int m_fd;

  std::map<uint64_t, std::pair<std::string, size_t>> m_line_number_map;
@@ -238,9 +238,7 @@ class perfetto_plugin_t {
    uint64_t device_id = profiler_record.gpu_id.handle;
    std::unordered_map<uint64_t, perfetto::Track>::iterator device_track_it;
    {
-      std::lock_guard<std::mutex> lock(device_tracks_lock_);
      uint64_t device_track_id = getTrackID(machine_id_, TrackType::DEVICE, device_id);
-
      device_track_it = device_tracks.find(device_track_id);
      if (device_track_it == device_tracks.end()) {
        /* Create a new perfetto::Track (Sub-Track) */
@@ -250,30 +248,25 @@ class perfetto_plugin_t {
        gpu_desc.mutable_process()->set_pid(device_id);
        gpu_desc.mutable_process()->set_chrome_process_type(
            perfetto::protos::gen::ProcessDescriptor::PROCESS_GPU);
-        gpu_desc.mutable_process()->set_process_name("Node: " + std::string(hostname_) +
-                                                     " Device: ");
+        gpu_desc.mutable_process()->set_process_name("Node: " + std::string(hostname_)
+                                                    + std::to_string(GetPid()) + " Device: ");
        perfetto::TrackEvent::SetTrackDescriptor(device_track_it->second, gpu_desc);
        track_ids_used_.emplace_back(device_track_id);
      }
    }
    auto& gpu_track = device_track_it->second;
-    std::pair<int, uint64_t> gpu_queue_id =
-        std::make_pair(device_id, profiler_record.queue_id.handle);
-    auto queue_track_it = queue_tracks_.find(gpu_queue_id.first);
-    {
-      std::lock_guard<std::mutex> lock(stream_tracks_lock_);
-      uint64_t queue_track_id = getTrackID(machine_id_, device_id+TrackType::DEVICE_ID, gpu_queue_id.first);
-      queue_track_it = queue_tracks_.find(queue_track_id);
-      if (queue_track_it == queue_tracks_.end()) {
-        /* Create a new perfetto::Track */
-        queue_track_it =
-            queue_tracks_.emplace(queue_track_id, perfetto::Track(queue_track_id, gpu_track)).first;
+    uint64_t queue_track_id
+      = getTrackID(machine_id_, device_id+TrackType::DEVICE_ID, profiler_record.queue_id.handle);
+    auto queue_track_it = queue_tracks_.find(queue_track_id);
+    if (queue_track_it == queue_tracks_.end()) {
+      /* Create a new perfetto::Track */
+      queue_track_it =
+          queue_tracks_.emplace(queue_track_id, perfetto::Track(queue_track_id, gpu_track)).first;

-        auto queue_desc = queue_track_it->second.Serialize();
-        std::string queue_str = rocprofiler::string_printf("Queue %ld", gpu_queue_id.second);
-        queue_desc.set_name(queue_str);
-        perfetto::TrackEvent::SetTrackDescriptor(queue_track_it->second, queue_desc);
-      }
+      auto queue_desc = queue_track_it->second.Serialize();
+      std::string queue_str = rocprofiler::string_printf("Queue %ld", profiler_record.queue_id.handle);
+      queue_desc.set_name(queue_str);
+      perfetto::TrackEvent::SetTrackDescriptor(queue_track_it->second, queue_desc);
      track_ids_used_.emplace_back(queue_track_id);
    }
    auto& queue_track = queue_track_it->second;
@@ -306,7 +299,6 @@ class perfetto_plugin_t {
      std::pair<int, std::string> gpu_counter_track_id = std::make_pair(device_id, counter_name);
      std::unordered_map<std::string, perfetto::CounterTrack>::iterator counters_track_it;
      {
-        std::lock_guard<std::mutex> lock(counter_tracks_lock_);
        counters_track_it = counter_tracks_.find(gpu_counter_track_id.second);
        if (counters_track_it == counter_tracks_.end()) {
          /* Create a new perfetto::Track */
@@ -643,9 +635,6 @@ class perfetto_plugin_t {
  std::atomic<uint64_t> track_counter_{GetPid()};
  std::vector<uint64_t> track_ids_used_;

-  std::mutex roctx_tracks_lock_, hsa_tracks_lock_, hip_tracks_lock_, device_tracks_lock_;
-  std::mutex stream_tracks_lock_, counter_tracks_lock_, mem_copies_tracks_lock_;
-
  char hostname_[1024];
  uint64_t machine_id_;