Adding rocprofilerv2

Change-Id: Ic0cc280ba207d2b8f6ccae1cd4ac3184152fc1ad [ROCm/rocprofiler commit: 8032adb64f]
2023-02-03 12:31:39 -06:00
parent b23a2f3029
commit de4abd0d0f
263 changed files with 607729 additions and 307 deletions
@@ -0,0 +1,27 @@
+file(GLOB ROCPROFILER_UTIL_SRC_FILES ${PROJECT_SOURCE_DIR}/src/utils/helper.cpp)
+
+add_library(perfetto_plugin
+        ${LIBRARY_TYPE} ${ROCPROFILER_UTIL_SRC_FILES}
+        perfetto.cpp perfetto_sdk/sdk/perfetto.cc)
+
+set_target_properties(perfetto_plugin PROPERTIES
+        CXX_VISIBILITY_PRESET hidden
+        LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../exportmap
+        LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR})
+
+target_compile_definitions(perfetto_plugin
+        PRIVATE HIP_PROF_HIP_API_STRING=1
+        __HIP_PLATFORM_HCC__=1)
+
+target_include_directories(perfetto_plugin
+        PRIVATE ${PROJECT_SOURCE_DIR}/inc ${PROJECT_SOURCE_DIR}
+        ${PROJECT_SOURCE_DIR}/plugin/perfetto/perfetto_sdk/sdk)
+
+target_link_options(perfetto_plugin
+        PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exportmap -Wl,--no-undefined)
+
+target_link_libraries(perfetto_plugin PRIVATE ${ROCPROFILER_TARGET} Threads::Threads systemd stdc++fs amd_comgr)
+
+install(TARGETS perfetto_plugin LIBRARY
+        DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}
+        COMPONENT plugins)
@@ -0,0 +1,804 @@
+/* Copyright (c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE. */
+
+#include "rocprofiler.h"
+
+#include <cassert>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdlib>
+#include <experimental/filesystem>
+#include <fstream>
+#include <memory>
+#include <optional>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <functional>
+#include <iostream>
+
+#include <cxxabi.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <systemd/sd-id128.h>
+
+#include "perfetto_sdk/sdk/perfetto.h"
+#include "rocprofiler_plugin.h"
+#include "../utils.h"
+
+#define STREAM_CONSTANT 98736677
+#define QUEUE_CONSTANT 18746479
+
+namespace fs = std::experimental::filesystem;
+
+PERFETTO_DEFINE_CATEGORIES(
+    perfetto::Category("GENERIC").SetDescription("GENERAL_CATEGORY"),
+    perfetto::Category("ROCTX_API").SetDescription("ACTIVITY_DOMAIN_ROCTX_API"),
+    perfetto::Category("HSA_API").SetDescription("ACTIVITY_DOMAIN_HSA_API"),
+    perfetto::Category("HIP_API").SetDescription("ACTIVITY_DOMAIN_HIP_API"),
+    perfetto::Category("External_API").SetDescription("ACTIVITY_DOMAIN_EXT_API"),
+    perfetto::Category("HIP_OPS").SetDescription("ACTIVITY_DOMAIN_HIP_OPS"),
+    perfetto::Category("HSA_OPS").SetDescription("ACTIVITY_DOMAIN_HSA_OPS"),
+    perfetto::Category("KERNELS").SetDescription("KERNEL_DISPATCHES"),
+    perfetto::Category("COUNTERS").SetDescription("PERFORMANCE_COUNTERS"));
+
+PERFETTO_TRACK_EVENT_STATIC_STORAGE();
+
+namespace {
+
+std::string process_name;
+static std::string output_file_name;
+
+std::string get_kernel_name(rocprofiler_record_profiler_t& profiler_record) {
+  std::string kernel_name = "";
+  size_t name_length = 1;
+  CHECK_ROCMTOOLS(rocprofiler_query_kernel_info_size(ROCPROFILER_KERNEL_NAME, profiler_record.kernel_id,
+                                                   &name_length));
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#pragma GCC diagnostic ignored "-Wstringop-overread"
+  if (name_length > 1) {
+    const char* kernel_name_c = static_cast<const char*>(malloc(name_length * sizeof(char)));
+    CHECK_ROCMTOOLS(rocprofiler_query_kernel_info(ROCPROFILER_KERNEL_NAME, profiler_record.kernel_id,
+                                                &kernel_name_c));
+    if (kernel_name_c && strlen(kernel_name_c) > 1)
+      kernel_name = rocmtools::cxx_demangle(strdup(kernel_name_c));
+  }
+#pragma GCC diagnostic pop
+  return kernel_name;
+}
+
+
+class perfetto_plugin_t {
+ public:
+  perfetto_plugin_t() {
+    const char* output_dir = getenv("OUTPUT_PATH");
+    const char* temp_file_name = getenv("OUT_FILE_NAME");
+    output_file_name = temp_file_name ? std::string(temp_file_name) + "_" : "";
+
+    if (output_dir == nullptr) {
+      stream_.copyfmt(std::cout);
+      stream_.clear(std::cout.rdstate());
+      stream_.basic_ios<char>::rdbuf(std::cout.rdbuf());
+      return;
+    }
+
+    output_prefix_ = output_dir;
+    if (!fs::is_directory(fs::status(output_prefix_))) {
+      if (!stream_.fail()) rocmtools::warning("Cannot open output directory '%s'", output_dir);
+      stream_.setstate(std::ios_base::failbit);
+      return;
+    }
+
+    perfetto::TracingInitArgs args;
+    args.backends |= perfetto::kInProcessBackend;
+
+    perfetto::Tracing::Initialize(args);
+    perfetto::TrackEvent::Register();
+
+    perfetto::protos::gen::TrackEventConfig track_event_cfg;
+    track_event_cfg.add_disabled_categories("*");
+    track_event_cfg.add_enabled_categories("GENERIC");
+    track_event_cfg.add_enabled_categories("ROCTX_API");
+    track_event_cfg.add_enabled_categories("HSA_API");
+    track_event_cfg.add_enabled_categories("HIP_API");
+    track_event_cfg.add_enabled_categories("External_API");
+    track_event_cfg.add_enabled_categories("HIP_OPS");
+    track_event_cfg.add_enabled_categories("HSA_OPS");
+    track_event_cfg.add_enabled_categories("KERNELS");
+    track_event_cfg.add_enabled_categories("COUNTERS");
+
+    perfetto::TraceConfig trace_cfg;
+
+    auto buffer_cfg = trace_cfg.add_buffers();
+    uint32_t max_buffer_size = 10 * 1024 * 1024;  // Default max buffer size is 10 GB
+    const char* max_buffer_size_str = getenv("rocprofiler_PERFETTO_MAX_BUFFER_SIZE_KIB");
+    if (max_buffer_size_str && std::atol(max_buffer_size_str) > 0)
+      max_buffer_size = std::atol(max_buffer_size_str);
+    // Record up to max buffer size determined by user or the 10 GB (default value)
+    buffer_cfg->set_size_kb(max_buffer_size);
+
+    auto* data_source_cfg = trace_cfg.add_data_sources()->mutable_config();
+    data_source_cfg->set_name("track_event");
+    data_source_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
+
+    output_prefix_.append(output_file_name + std::to_string(GetPid()) + "_output.pftrace");
+    file_descriptor_ = open(output_prefix_.string().c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600);
+    if (file_descriptor_ == -1) rocmtools::warning("Can't open output file\n");
+
+    tracing_session_ = perfetto::Tracing::NewTrace();
+    tracing_session_->Setup(trace_cfg, file_descriptor_);
+    tracing_session_->StartBlocking();
+
+
+    hostname_[1023] = '\0';
+    gethostname(hostname_, 1023);
+    sd_id128_t ret;
+    char machine_id[SD_ID128_STRING_MAX];
+    [[maybe_unused]] int status = sd_id128_get_machine(&ret);
+    assert(status == 0 && "Error: Couldn't get machine id!");
+    if (sd_id128_to_string(ret, machine_id)) machine_id_ = std::hash<std::string>{}(machine_id);
+
+    {
+      std::lock_guard<std::mutex> lock(thread_tracks_lock_);
+      process_name =
+          perfetto::ProcessTrack::Current().Serialize().mutable_process()->process_name();
+      auto process_track_desc = perfetto::ProcessTrack::Current().Serialize();
+      uint64_t track_id =
+          track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+      for (uint64_t tid : track_ids_used_) {
+        while (track_id == tid) {
+          track_id =
+              track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+        }
+      }
+      std::string thread_track_str =
+          rocmtools::string_printf("Node: %s Process ID: %lu Thread ID:", hostname_, GetPid());
+      process_track_desc.mutable_process()->set_process_name(thread_track_str);
+      perfetto::TrackEvent::SetTrackDescriptor(perfetto::ProcessTrack::Current(),
+                                               process_track_desc);
+      perfetto::ProcessTrack::Current().Serialize().set_uuid(track_id);
+      thread_tracks_.emplace(GetPid(), perfetto::ProcessTrack::Current());
+    }
+
+    is_valid_ = true;
+  }
+
+  ~perfetto_plugin_t() {
+    if (is_valid_) {
+      tracing_session_->StopBlocking();
+      close(file_descriptor_);
+    }
+  }
+
+  const char* GetDomainName(rocprofiler_tracer_activity_domain_t domain) {
+    switch (domain) {
+      case ACTIVITY_DOMAIN_ROCTX:
+        return "ROCTX_DOMAIN";
+        break;
+      case ACTIVITY_DOMAIN_HIP_API:
+        return "HIP_API_DOMAIN";
+        break;
+      case ACTIVITY_DOMAIN_HIP_OPS:
+        return "HIP_OPS_DOMAIN";
+        break;
+      case ACTIVITY_DOMAIN_HSA_API:
+        return "HSA_API_DOMAIN";
+        break;
+      case ACTIVITY_DOMAIN_HSA_OPS:
+        return "HSA_OPS_DOMAIN";
+        break;
+      case ACTIVITY_DOMAIN_HSA_EVT:
+        return "HSA_EVT_DOMAIN";
+        break;
+      default:
+        return "";
+    }
+  }
+
+  std::mutex writing_lock;
+
+  int FlushProfilerRecord(rocprofiler_record_profiler_t profiler_record,
+                          rocprofiler_session_id_t session_id) {
+    std::lock_guard<std::mutex> lock(writing_lock);
+    // ToDO: rename this variable?
+    if (!tracing_session_) rocmtools::warning("Tracing session is deleted!\n");
+
+    int device_id = profiler_record.gpu_id.handle;
+    std::unordered_map<int, perfetto::Track>::iterator device_track_it;
+    {
+      std::lock_guard<std::mutex> lock(device_tracks_lock_);
+      device_track_it = device_tracks_.find(device_id);
+      if (device_track_it == device_tracks_.end()) {
+        /* Create a new perfetto::Track (Sub-Track) */
+        device_track_it =
+            device_tracks_
+                .emplace(device_id, perfetto::ProcessTrack::Global(((device_id + 1) * machine_id_)))
+                .first;
+        auto gpu_desc = device_track_it->second.Serialize();
+        gpu_desc.mutable_process()->set_pid(device_id);
+        std::string gpu_str = rocmtools::string_printf("Node: %s Device:", hostname_);
+        gpu_desc.mutable_process()->set_process_name(gpu_str);
+        perfetto::TrackEvent::SetTrackDescriptor(device_track_it->second, gpu_desc);
+        track_ids_used_.emplace_back(device_id + 1 + machine_id_);
+      }
+    }
+    auto& gpu_track = device_track_it->second;
+    std::pair<int, uint64_t> gpu_queue_id =
+        std::make_pair(device_id, profiler_record.queue_id.handle);
+    auto queue_track_it = queue_tracks_.find(gpu_queue_id.first);
+    {
+      std::lock_guard<std::mutex> lock(queue_tracks_lock_);
+      queue_track_it = queue_tracks_.find(gpu_queue_id.first);
+      if (queue_track_it == queue_tracks_.end()) {
+        /* Create a new perfetto::Track */
+        queue_track_it = queue_tracks_
+                             .emplace(gpu_queue_id.first,
+                                      perfetto::Track((profiler_record.queue_id.handle + 1 +
+                                                       profiler_record.gpu_id.handle) *
+                                                          QUEUE_CONSTANT * machine_id_ * GetPid(),
+                                                      gpu_track))
+                             .first;
+
+        auto queue_desc = queue_track_it->second.Serialize();
+        std::string queue_str =
+            rocmtools::string_printf("Process ID: %lu Queue %ld", GetPid(), gpu_queue_id.second);
+        queue_desc.set_name(queue_str);
+        perfetto::TrackEvent::SetTrackDescriptor(queue_track_it->second, queue_desc);
+      }
+      track_ids_used_.emplace_back(profiler_record.queue_id.handle + machine_id_ + 1 +
+                                   profiler_record.gpu_id.handle);
+    }
+    auto& queue_track = queue_track_it->second;
+
+    // Taken from rocprofiler: The size hasn't changed in  recent past
+    static const uint32_t lds_block_size = 128 * 4;
+
+    std::string full_kernel_name = get_kernel_name(profiler_record);
+    // std::string truncated_kernel_name = rocmtools::truncate_name(full_kernel_name);
+    // perfetto::StaticString kernel_name(truncated_kernel_name.c_str());
+    TRACE_EVENT_BEGIN("KERNELS", perfetto::StaticString(full_kernel_name.c_str()), queue_track,
+                      profiler_record.timestamps.begin.value, "Full Kernel Name",
+                      full_kernel_name.c_str(), "Agent ID", device_id, "Queue ID",
+                      profiler_record.queue_id.handle, "GRD",
+                      profiler_record.kernel_properties.grid_size, "WGR",
+                      profiler_record.kernel_properties.workgroup_size, "LDS",
+                      (((profiler_record.kernel_properties.lds_size + (lds_block_size - 1)) &
+                        ~(lds_block_size - 1))),
+                      "SCR", profiler_record.kernel_properties.scratch_size, "Arch. VGPR",
+                      profiler_record.kernel_properties.arch_vgpr_count, "Accumilative Vgpr",
+                      profiler_record.kernel_properties.accum_vgpr_count, "SGPR",
+                      profiler_record.kernel_properties.sgpr_count, "Wave Size",
+                      profiler_record.kernel_properties.wave_size, "Signal",
+                      profiler_record.kernel_properties.signal_handle);
+
+    TRACE_EVENT_END("KERNELS", queue_track, profiler_record.timestamps.end.value);
+
+    auto get_counter_track_fn = [&](std::string counter_name) {
+      std ::string counter_track_id =
+          std::to_string(machine_id_) + std::to_string(GetPid()) + counter_name;
+      std::pair<int, std::string> gpu_counter_track_id = std::make_pair(device_id, counter_name);
+      std::unordered_map<std::string, perfetto::CounterTrack>::iterator counters_track_it;
+      {
+        std::lock_guard<std::mutex> lock(counter_tracks_lock_);
+        counters_track_it = counter_tracks_.find(gpu_counter_track_id.second);
+        if (counters_track_it == counter_tracks_.end()) {
+          /* Create a new perfetto::Track */
+          counters_track_it =
+              counter_tracks_
+                  .emplace(gpu_counter_track_id.second,
+                           perfetto::CounterTrack(counter_track_id.c_str(), gpu_track))
+                  .first;
+
+          auto counter_track_desc = counters_track_it->second.Serialize();
+          std::string counter_track_str = "Process ID " + std::to_string(GetPid()) + " - Counter " +
+              gpu_counter_track_id.second;
+          counter_track_desc.set_name(counter_track_str);
+          perfetto::TrackEvent::SetTrackDescriptor(counters_track_it->second, counter_track_desc);
+        }
+      }
+      return counters_track_it->second;
+    };
+
+    // For Counters
+    if (profiler_record.counters) {
+      for (uint64_t i = 0; i < profiler_record.counters_count.value; i++) {
+        if (profiler_record.counters[i].counter_handler.handle > 0) {
+          size_t name_length = 0;
+          CHECK_ROCMTOOLS(rocprofiler_query_counter_info_size(
+              session_id, ROCPROFILER_COUNTER_NAME, profiler_record.counters[i].counter_handler,
+              &name_length));
+          if (name_length > 1) {
+            const char* name_c = static_cast<const char*>(malloc(name_length * sizeof(char)));
+            CHECK_ROCMTOOLS(
+                rocprofiler_query_counter_info(session_id, ROCPROFILER_COUNTER_NAME,
+                                             profiler_record.counters[i].counter_handler, &name_c));
+
+            perfetto::CounterTrack counters_track = get_counter_track_fn(std::string(name_c));
+            TRACE_COUNTER("COUNTERS", counters_track, profiler_record.timestamps.begin.value,
+                          profiler_record.counters[i].value.value);
+            // Added an extra zero event for maintaining start-end of the counter
+            TRACE_COUNTER("COUNTERS", counters_track, profiler_record.timestamps.end.value, 0.001);
+          }
+        }
+      }
+    }
+
+    return 0;
+  }
+
+  int FlushTracerRecord(rocprofiler_record_tracer_t tracer_record,
+                        rocprofiler_session_id_t session_id) {
+    std::lock_guard<std::mutex> lock(writing_lock);
+    if (!tracing_session_) rocmtools::warning("Tracing session is deleted!\n");
+    std::string kernel_name;
+    char* function_name;
+    char* activity_name;
+    std::string roctx_message;
+    uint64_t roctx_id = 0;
+    uint64_t thread_id = tracer_record.thread_id.value;
+    std::unordered_map<uint64_t, perfetto::Track>::iterator thread_track_it;
+    std::unordered_map<int, perfetto::Track>::iterator device_track_it;
+    if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS ||
+        tracer_record.domain == ACTIVITY_DOMAIN_HSA_OPS) {
+      int device_id = tracer_record.agent_id.handle;
+      if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS && device_id > 0) device_id--;
+      {
+        std::lock_guard<std::mutex> lock(device_tracks_lock_);
+        device_track_it = device_tracks_.find(device_id);
+        if (device_track_it == device_tracks_.end()) {
+          /* Create a new perfetto::Track (Sub-Track) */
+          device_track_it =
+              device_tracks_
+                  .emplace(device_id,
+                           perfetto::ProcessTrack::Global(((device_id + 1) * machine_id_)))
+                  .first;
+          auto gpu_desc = device_track_it->second.Serialize();
+          gpu_desc.mutable_process()->set_pid(device_id);
+          std::string gpu_str = rocmtools::string_printf("Node: %s Device:", hostname_);
+          gpu_desc.mutable_process()->set_process_name(gpu_str);
+          perfetto::TrackEvent::SetTrackDescriptor(device_track_it->second, gpu_desc);
+          track_ids_used_.emplace_back(1 + machine_id_ + device_id);
+        }
+      }
+    } else {
+      std::lock_guard<std::mutex> lock(thread_tracks_lock_);
+      thread_track_it = thread_tracks_.find(thread_id);
+      if (thread_track_it == thread_tracks_.end()) {
+        uint64_t track_id =
+            track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+        for (uint64_t tid : track_ids_used_) {
+          while (track_id == tid) {
+            track_id =
+                track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+          }
+        }
+        thread_track_it =
+            thread_tracks_.emplace(thread_id, perfetto::ProcessTrack::Global(track_id)).first;
+        auto thread_track_desc = thread_track_it->second.Serialize();
+        std::string thread_track_str =
+            rocmtools::string_printf("Node: %s Process ID: %lu Thread ID:", hostname_, GetPid());
+        thread_track_desc.mutable_process()->set_pid(thread_id);
+        thread_track_desc.mutable_process()->set_process_name(thread_track_str);
+        perfetto::TrackEvent::SetTrackDescriptor(thread_track_it->second, thread_track_desc);
+      }
+    }
+    auto& thread_track = thread_track_it->second;
+    auto& gpu_track = device_track_it->second;
+    switch (tracer_record.domain) {
+      case ACTIVITY_DOMAIN_ROCTX: {
+        std::unordered_map<uint64_t, perfetto::Track>::iterator roctx_track_it;
+        {
+          std::lock_guard<std::mutex> lock(roctx_tracks_lock_);
+          roctx_track_it = roctx_tracks_.find(thread_id);
+          if (roctx_track_it == roctx_tracks_.end()) {
+            /* Create a new perfetto::Track */
+            uint64_t track_id =
+                track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+            for (uint64_t tid : track_ids_used_) {
+              while (track_id == tid) {
+                track_id = track_counter_.fetch_add((1 + machine_id_) * GetPid(),
+                                                    std::memory_order_acquire);
+              }
+            }
+            roctx_track_it =
+                roctx_tracks_.emplace(thread_id, perfetto::Track(track_id, thread_track)).first;
+
+            auto roctx_track_desc = roctx_track_it->second.Serialize();
+            std::string roctx_track_str = rocmtools::string_printf("ROCTX Markers");
+            roctx_track_desc.set_name(roctx_track_str);
+            perfetto::TrackEvent::SetTrackDescriptor(roctx_track_it->second, roctx_track_desc);
+          }
+        }
+        auto& roctx_track = roctx_track_it->second;
+
+        size_t roctx_message_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_roctx_tracer_api_data_info_size(
+            session_id, ROCPROFILER_ROCTX_MESSAGE, tracer_record.api_data_handle,
+            tracer_record.operation_id, &roctx_message_size));
+        if (roctx_message_size > 1) {
+          char* roctx_message_str = static_cast<char*>(malloc(roctx_message_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_roctx_tracer_api_data_info(
+              session_id, ROCPROFILER_ROCTX_MESSAGE, tracer_record.api_data_handle,
+              tracer_record.operation_id, &roctx_message_str));
+          if (roctx_message_str)
+            roctx_message = rocmtools::cxx_demangle(std::string(strdup(roctx_message_str)));
+        }
+        size_t roctx_id_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_roctx_tracer_api_data_info_size(
+            session_id, ROCPROFILER_ROCTX_ID, tracer_record.api_data_handle,
+            tracer_record.operation_id, &roctx_id_size));
+        if (roctx_id_size > 1) {
+          char* roctx_id_str = static_cast<char*>(malloc(roctx_id_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_roctx_tracer_api_data_info(
+              session_id, ROCPROFILER_ROCTX_ID, tracer_record.api_data_handle,
+              tracer_record.operation_id, &roctx_id_str));
+          if (roctx_id_str) {
+            roctx_id = std::stoll(std::string(strdup(roctx_id_str)));
+            free(roctx_id_str);
+          }
+        }
+
+        if (tracer_record.operation_id.id == 1) {
+          perfetto::StaticString roctx_message_pft(
+              (!roctx_message.empty() ? roctx_message.c_str() : ""));
+          TRACE_EVENT_BEGIN("ROCTX_API", roctx_message_pft, roctx_track,
+                            tracer_record.timestamps.begin.value, "Timestamp(ns)",
+                            tracer_record.timestamps.begin.value, "RocTx ID", roctx_id);
+          roctx_track_entries_++;
+        } else {
+          TRACE_EVENT_END("ROCTX_API", roctx_track, tracer_record.timestamps.begin.value);
+          roctx_track_entries_--;
+        }
+        break;
+      }
+      case ACTIVITY_DOMAIN_HSA_API: {
+        std::unordered_map<uint64_t, perfetto::Track>::iterator hsa_track_it;
+        {
+          std::lock_guard<std::mutex> lock(hsa_tracks_lock_);
+          hsa_track_it = hsa_tracks_.find(thread_id);
+          if (hsa_track_it == hsa_tracks_.end()) {
+            /* Create a new perfetto::Track */
+            uint64_t track_id =
+                track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+            for (uint64_t tid : track_ids_used_) {
+              while (track_id == tid) {
+                track_id = track_counter_.fetch_add((1 + machine_id_) * GetPid(),
+                                                    std::memory_order_acquire);
+              }
+            }
+            hsa_track_it =
+                hsa_tracks_.emplace(thread_id, perfetto::Track(track_id, thread_track)).first;
+            auto hsa_track_desc = hsa_track_it->second.Serialize();
+            std::string hsa_track_str = rocmtools::string_printf("HSA API");
+            hsa_track_desc.set_name(hsa_track_str);
+            perfetto::TrackEvent::SetTrackDescriptor(hsa_track_it->second, hsa_track_desc);
+          }
+        }
+        auto& hsa_track = hsa_track_it->second;
+        size_t function_name_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_hsa_tracer_api_data_info_size(
+            session_id, ROCPROFILER_HSA_FUNCTION_NAME, tracer_record.api_data_handle,
+            tracer_record.operation_id, &function_name_size));
+        if (function_name_size > 1) {
+          function_name = static_cast<char*>(malloc(function_name_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_hsa_tracer_api_data_info(
+              session_id, ROCPROFILER_HSA_FUNCTION_NAME, tracer_record.api_data_handle,
+              tracer_record.operation_id, &function_name));
+        }
+        TRACE_EVENT_BEGIN("HSA_API", perfetto::StaticString(function_name), hsa_track,
+                          tracer_record.timestamps.begin.value,
+                          perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        TRACE_EVENT_END("HSA_API", hsa_track, tracer_record.timestamps.end.value);
+        break;
+      }
+      case ACTIVITY_DOMAIN_HIP_API: {
+        std::unordered_map<uint64_t, perfetto::Track>::iterator hip_track_it;
+        {
+          std::lock_guard<std::mutex> lock(hip_tracks_lock_);
+          hip_track_it = hip_tracks_.find(thread_id);
+          if (hip_track_it == hip_tracks_.end()) {
+            /* Create a new perfetto::Track */
+            uint64_t track_id =
+                track_counter_.fetch_add((1 + machine_id_) * GetPid(), std::memory_order_acquire);
+            for (uint64_t tid : track_ids_used_) {
+              while (track_id == tid) {
+                track_id = track_counter_.fetch_add((1 + machine_id_) * GetPid(),
+                                                    std::memory_order_acquire);
+              }
+            }
+            hip_track_it =
+                hip_tracks_.emplace(thread_id, perfetto::Track(track_id, thread_track)).first;
+
+            auto hip_track_desc = hip_track_it->second.Serialize();
+            std::string hip_track_str = rocmtools::string_printf("HIP API");
+            hip_track_desc.set_name(hip_track_str);
+            perfetto::TrackEvent::SetTrackDescriptor(hip_track_it->second, hip_track_desc);
+          }
+        }
+        auto& hip_track = hip_track_it->second;
+        size_t function_name_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info_size(
+            session_id, ROCPROFILER_HIP_FUNCTION_NAME, tracer_record.api_data_handle,
+            tracer_record.operation_id, &function_name_size));
+        if (function_name_size > 1) {
+          function_name = static_cast<char*>(malloc(function_name_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info(
+              session_id, ROCPROFILER_HIP_FUNCTION_NAME, tracer_record.api_data_handle,
+              tracer_record.operation_id, &function_name));
+        }
+        size_t kernel_name_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info_size(
+            session_id, ROCPROFILER_HIP_KERNEL_NAME, tracer_record.api_data_handle,
+            tracer_record.operation_id, &kernel_name_size));
+        char* kernel_name_str;
+        if (kernel_name_size > 1) {
+          kernel_name_str = static_cast<char*>(malloc(kernel_name_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info(
+              session_id, ROCPROFILER_HIP_KERNEL_NAME, tracer_record.api_data_handle,
+              tracer_record.operation_id, &kernel_name_str));
+          if (kernel_name_str) {
+            kernel_name = rocmtools::cxx_demangle(std::string(kernel_name_str));
+            free(kernel_name_str);
+          }
+        }
+        if (kernel_name.size() > 0) {
+          TRACE_EVENT_BEGIN("HIP_API", perfetto::StaticString(function_name), hip_track,
+                            tracer_record.timestamps.begin.value, "Kernel Name", kernel_name,
+                            perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        } else {
+          TRACE_EVENT_BEGIN("HIP_API", perfetto::StaticString(function_name), hip_track,
+                            tracer_record.timestamps.begin.value,
+                            perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        }
+        TRACE_EVENT_END("HIP_API", hip_track, tracer_record.timestamps.end.value);
+        break;
+      }
+      case ACTIVITY_DOMAIN_EXT_API: {
+        printf("Warning: External API is not supported!\n");
+        break;
+      }
+      case ACTIVITY_DOMAIN_HIP_OPS: {
+        uint64_t stream_id = 0;
+        size_t stream_id_str_size = 0;
+        char* stream_id_str;
+        CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info_size(
+            session_id, ROCPROFILER_HIP_STREAM_ID, rocprofiler_tracer_api_data_handle_t{nullptr, 0},
+            rocprofiler_tracer_operation_id_t{(uint32_t)tracer_record.correlation_id.value},
+            &stream_id_str_size));
+        if (stream_id_str_size > 1) {
+          stream_id_str = static_cast<char*>(malloc(stream_id_str_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info(
+              session_id, ROCPROFILER_HIP_STREAM_ID, rocprofiler_tracer_api_data_handle_t{nullptr, 0},
+              rocprofiler_tracer_operation_id_t{(uint32_t)tracer_record.correlation_id.value},
+              &stream_id_str));
+          if (stream_id_str != nullptr) stream_id = std::stoll(stream_id_str);
+        }
+        std::unordered_map<int, perfetto::Track>::iterator stream_track_it;
+        {
+          std::lock_guard<std::mutex> lock(stream_tracks_lock_);
+          stream_track_it = stream_tracks_.find(stream_id);
+          if (stream_track_it == stream_tracks_.end()) {
+            /* Create a new perfetto::Track */
+            uint64_t track_id = ((1 + stream_id + tracer_record.agent_id.handle) * machine_id_ *
+                                 STREAM_CONSTANT * GetPid());
+            stream_track_it =
+                stream_tracks_.emplace(stream_id, perfetto::Track(track_id, gpu_track)).first;
+
+            auto stream_desc = stream_track_it->second.Serialize();
+            std::string stream_str =
+                rocmtools::string_printf("Process ID: %lu Stream %d", GetPid(), stream_id);
+            stream_desc.set_name(stream_str);
+            perfetto::TrackEvent::SetTrackDescriptor(stream_track_it->second, stream_desc);
+            track_ids_used_.emplace_back(1 + machine_id_ + tracer_record.agent_id.handle);
+          }
+        }
+        auto& stream_track = stream_track_it->second;
+        if (tracer_record.api_data_handle.handle && tracer_record.api_data_handle.size > 1) {
+          kernel_name = rocmtools::cxx_demangle(
+              strdup(reinterpret_cast<const char*>(tracer_record.api_data_handle.handle)));
+          TRACE_EVENT_BEGIN(
+              "HIP_OPS",
+              perfetto::StaticString(strdup(rocmtools::truncate_name(kernel_name).c_str())),
+              stream_track, tracer_record.timestamps.begin.value, "Agent ID",
+              tracer_record.agent_id.handle, "Process ID", GetPid(), "Kernel Name", kernel_name,
+              perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        } else {
+          size_t activity_name_size = 0;
+          CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info_size(
+              session_id, ROCPROFILER_HIP_ACTIVITY_NAME, tracer_record.api_data_handle,
+              tracer_record.operation_id, &activity_name_size));
+          if (activity_name_size > 1) {
+            activity_name = static_cast<char*>(malloc(activity_name_size * sizeof(char)));
+            CHECK_ROCMTOOLS(rocprofiler_query_hip_tracer_api_data_info(
+                session_id, ROCPROFILER_HIP_ACTIVITY_NAME, tracer_record.api_data_handle,
+                tracer_record.operation_id, &activity_name));
+          } else {
+            activity_name = const_cast<char*>(std::string("N/A").c_str());
+          }
+          TRACE_EVENT_BEGIN("HIP_OPS", perfetto::StaticString(activity_name), stream_track,
+                            tracer_record.timestamps.begin.value, "Agent ID",
+                            tracer_record.agent_id.handle, "Process ID", GetPid(),
+                            perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        }
+        TRACE_EVENT_END("HIP_OPS", stream_track, tracer_record.timestamps.end.value);
+        break;
+      }
+      case ACTIVITY_DOMAIN_HSA_OPS: {
+        std::pair<int, uint64_t> gpu_queue_id =
+            std::make_pair(tracer_record.agent_id.handle, tracer_record.queue_id.handle);
+        std::unordered_map<int, perfetto::Track>::iterator queue_track_it;
+        {
+          std::lock_guard<std::mutex> lock(queue_tracks_lock_);
+          queue_track_it = queue_tracks_.find(gpu_queue_id.first);
+          if (queue_track_it == queue_tracks_.end()) {
+            /* Create a new perfetto::Track */
+            uint64_t track_id =
+                ((1 + tracer_record.queue_id.handle + tracer_record.agent_id.handle) * machine_id_ *
+                 QUEUE_CONSTANT * GetPid());
+            queue_track_it =
+                queue_tracks_.emplace(gpu_queue_id.first, perfetto::Track(track_id, gpu_track))
+                    .first;
+
+            auto queue_desc = queue_track_it->second.Serialize();
+            std::string queue_str = rocmtools::string_printf("Process ID: %lu Queue %ld", GetPid(),
+                                                             gpu_queue_id.second);
+            queue_desc.set_name(queue_str);
+            perfetto::TrackEvent::SetTrackDescriptor(queue_track_it->second, queue_desc);
+          }
+          track_ids_used_.emplace_back(tracer_record.queue_id.handle + machine_id_ + 1 +
+                                       tracer_record.agent_id.handle);
+        }
+        auto& queue_track = queue_track_it->second;
+        size_t activity_name_size = 0;
+        CHECK_ROCMTOOLS(rocprofiler_query_hsa_tracer_api_data_info_size(
+            session_id, ROCPROFILER_HSA_ACTIVITY_NAME, tracer_record.api_data_handle,
+            tracer_record.operation_id, &activity_name_size));
+        if (activity_name_size > 1) {
+          activity_name = static_cast<char*>(malloc(activity_name_size * sizeof(char)));
+          CHECK_ROCMTOOLS(rocprofiler_query_hsa_tracer_api_data_info(
+              session_id, ROCPROFILER_HSA_ACTIVITY_NAME, tracer_record.api_data_handle,
+              tracer_record.operation_id, &activity_name));
+        }
+        TRACE_EVENT_BEGIN("HSA_OPS", perfetto::StaticString(activity_name), queue_track,
+                          tracer_record.timestamps.begin.value, "Agent ID",
+                          tracer_record.agent_id.handle, "Queue ID", tracer_record.queue_id.handle,
+                          "Process ID", GetPid(),
+                          perfetto::Flow::ProcessScoped(tracer_record.correlation_id.value));
+        TRACE_EVENT_END("HSA_OPS", queue_track, tracer_record.timestamps.end.value);
+        break;
+      }
+      default: {
+        rocmtools::warning("ignored record for domain %d", tracer_record.domain);
+        break;
+      }
+    }
+    return 0;
+  }
+
+  int WriteBufferRecords(const rocprofiler_record_header_t* begin,
+                         const rocprofiler_record_header_t* end, rocprofiler_session_id_t session_id,
+                         rocprofiler_buffer_id_t buffer_id) {
+    if (!tracing_session_) rocmtools::warning("Tracing session is deleted!\n");
+    while (begin < end) {
+      if (!begin) return 0;
+      switch (begin->kind) {
+        case ROCPROFILER_PROFILER_RECORD: {
+          rocprofiler_record_profiler_t* profiler_record = const_cast<rocprofiler_record_profiler_t*>(
+              reinterpret_cast<const rocprofiler_record_profiler_t*>(begin));
+          FlushProfilerRecord(*profiler_record, session_id);
+          break;
+        }
+        case ROCPROFILER_TRACER_RECORD: {
+          rocprofiler_record_tracer_t* tracer_record = const_cast<rocprofiler_record_tracer_t*>(
+              reinterpret_cast<const rocprofiler_record_tracer_t*>(begin));
+          FlushTracerRecord(*tracer_record, session_id);
+          break;
+        }
+        default:
+          break;
+      }
+      rocprofiler_next_record(begin, &begin, session_id, buffer_id);
+    }
+    return 0;
+  }
+
+  bool IsValid() const { return is_valid_; }
+
+ private:
+  fs::path output_prefix_;
+  std::unique_ptr<perfetto::TracingSession> tracing_session_;
+  int file_descriptor_;
+  bool is_valid_{false};
+  size_t roctx_track_entries_{0};
+
+  // Correlate stream id(s) with correlation id(s) to identify the stream id of every HIP activity
+  std::unordered_map<uint64_t, uint64_t> stream_ids_;
+
+  // Callback Tracks
+  std::unordered_map<uint64_t, perfetto::Track> thread_tracks_;
+  std::unordered_map<uint64_t, perfetto::Track> roctx_tracks_, hsa_tracks_, hip_tracks_,
+      hip_ext_tracks_;
+
+  // Activity Tracks
+  std::unordered_map<int, perfetto::Track> device_tracks_;
+  std::unordered_map<int, perfetto::Track> queue_tracks_, stream_tracks_;
+
+  std::unordered_map<std::string, perfetto::CounterTrack> counter_tracks_;
+
+  std::atomic<uint64_t> track_counter_{GetPid()};
+  std::vector<uint64_t> track_ids_used_;
+
+  std::mutex stream_ids_lock_, thread_tracks_lock_, roctx_tracks_lock_, hsa_tracks_lock_,
+      hip_tracks_lock_, hip_ext_tracks_lock_, device_tracks_lock_, queue_tracks_lock_,
+      stream_tracks_lock_, counter_tracks_lock_;
+
+  char hostname_[1024];
+  uint64_t machine_id_;
+
+  std::ofstream stream_;
+};
+
+perfetto_plugin_t* perfetto_plugin = nullptr;
+
+}  // namespace
+
+int rocprofiler_plugin_initialize(uint32_t rocprofiler_major_version,
+                                uint32_t rocprofiler_minor_version) {
+  if (rocprofiler_major_version != ROCPROFILER_VERSION_MAJOR ||
+      rocprofiler_minor_version > ROCPROFILER_VERSION_MINOR)
+    return -1;
+
+  if (perfetto_plugin != nullptr) return -1;
+
+  perfetto_plugin = new perfetto_plugin_t();
+  if (perfetto_plugin->IsValid()) return 0;
+
+  delete perfetto_plugin;
+  perfetto_plugin = nullptr;
+  return -1;
+}
+
+void rocprofiler_plugin_finalize() {
+  if (!perfetto_plugin) return;
+  delete perfetto_plugin;
+  perfetto_plugin = nullptr;
+}
+
+ROCPROFILER_EXPORT int rocprofiler_plugin_write_buffer_records(const rocprofiler_record_header_t* begin,
+                                                           const rocprofiler_record_header_t* end,
+                                                           rocprofiler_session_id_t session_id,
+                                                           rocprofiler_buffer_id_t buffer_id) {
+  if (!perfetto_plugin || !perfetto_plugin->IsValid()) return -1;
+  return perfetto_plugin->WriteBufferRecords(begin, end, session_id, buffer_id);
+}
+
+ROCPROFILER_EXPORT int rocprofiler_plugin_write_record(rocprofiler_record_tracer_t record,
+                                                   rocprofiler_session_id_t session_id) {
+  if (!perfetto_plugin || !perfetto_plugin->IsValid()) return -1;
+  if (record.header.id.handle == 0) return 0;
+  perfetto_plugin->FlushTracerRecord(record, session_id);
+  return 0;
+}
@@ -0,0 +1,189 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright (c) 2017, The Android Open Source Project
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
@@ -0,0 +1,35 @@
+# Global OWNERS that can approve Perfetto changes.
+# Please look first at OWNERS in the various subdirectories before falling back
+# on this, as the former tend to be more brain-cache-hot.
+
+# Perfetto tracing internals and API/ABI boundaries.
+primiano@google.com
+skyostil@google.com
+
+# UI, Ftrace interop, traced_probes, protozero, Android internals.
+hjd@google.com
+
+# Trace Processor, metrics, infra.
+lalitm@google.com
+
+# Callstack / memory profilers, traced_probes & Linux internals.
+ddiproietto@google.com
+rsavitski@google.com
+
+# Chromium-related things and tracing SDK.
+eseckler@google.com
+nuskos@google.com
+oysteine@google.com
+
+# Most Android-related metrics.
+ilkos@google.com
+
+# fmayer@ left the team. Please try first rsavitski@, ddiproietto@ or primiano@
+# and leave fmayer@ as an emergency-only escalation on profilers.
+fmayer@google.com
+
+# chromium.org aliases for adding DEPS entries from chromium subprojects to
+# third_party/perfetto.
+eseckler@chromium.org
+nuskos@chromium.org
+skyostil@chromium.org
@@ -0,0 +1,394 @@
+# Tracing SDK
+
+The Perfetto Tracing SDK is a C++11 library that allows userspace applications
+to emit trace events and add more app-specific context to a Perfetto trace.
+
+When using the Tracing SDK there are two main aspects to consider:
+
+1. Whether you are interested only in tracing events coming from your own app
+   or want to collect full-stack traces that overlay app trace events with
+   system trace events like scheduler traces, syscalls or any other Perfetto
+   data source.
+
+2. For app-specific tracing, whether you need to trace simple types of timeline
+  events (e.g., slices, counters) or need to define complex data sources with a
+  custom strongly-typed schema (e.g., for dumping the state of a subsystem of
+  your app into the trace).
+
+For Android-only instrumentation, the advice is to keep using the existing
+[android.os.Trace (SDK)][atrace-sdk] / [ATrace_* (NDK)][atrace-ndk] if they
+are sufficient for your use cases. Atrace-based instrumentation is fully
+supported in Perfetto.
+See the [Data Sources -> Android System -> Atrace Instrumentation][atrace-ds]
+for details.
+
+## Getting started
+
+TIP: The code from these examples is also available [in the
+repository](/examples/sdk/README.md).
+
+To start using the Client API, first check out the latest SDK release:
+
+```bash
+git clone https://android.googlesource.com/platform/external/perfetto -b v23.0
+```
+
+The SDK consists of two files, `sdk/perfetto.h` and `sdk/perfetto.cc`. These are
+an amalgamation of the Client API designed to easy to integrate to existing
+build systems. The sources are self-contained and require only a C++11 compliant
+standard library.
+
+For example, to add the SDK to a CMake project, edit your CMakeLists.txt:
+
+```cmake
+cmake_minimum_required(VERSION 3.13)
+project(PerfettoExample)
+find_package(Threads)
+
+# Define a static library for Perfetto.
+include_directories(perfetto/sdk)
+add_library(perfetto STATIC perfetto/sdk/perfetto.cc)
+
+# Link the library to your main executable.
+add_executable(example example.cc)
+target_link_libraries(example perfetto ${CMAKE_THREAD_LIBS_INIT})
+```
+
+Next, initialize Perfetto in your program:
+
+```C++
+#include <perfetto.h>
+
+int main(int argc, char** argv) {
+  perfetto::TracingInitArgs args;
+
+  // The backends determine where trace events are recorded. You may select one
+  // or more of:
+
+  // 1) The in-process backend only records within the app itself.
+  args.backends |= perfetto::kInProcessBackend;
+
+  // 2) The system backend writes events into a system Perfetto daemon,
+  //    allowing merging app and system events (e.g., ftrace) on the same
+  //    timeline. Requires the Perfetto `traced` daemon to be running (e.g.,
+  //    on Android Pie and newer).
+  args.backends |= perfetto::kSystemBackend;
+
+  perfetto::Tracing::Initialize(args);
+}
+```
+
+You are now ready to instrument your app with trace events.
+
+## Custom data sources vs Track events
+
+The SDK offers two abstraction layers to inject tracing data, built on top of
+each other, which trade off code complexity vs expressive power:
+[track events](#track-events) and [custom data sources](#custom-data-sources).
+
+### Track events
+
+Track events are the suggested option when dealing with app-specific tracing as
+they take care of a number of subtleties (e.g., thread safety, flushing, string
+interning).
+Track events are time bounded events (e.g., slices, counter) based on simple
+`TRACE_EVENT` annotation tags in the codebase, like this:
+
+```c++
+#include <perfetto.h>
+
+PERFETTO_DEFINE_CATEGORIES(
+    perfetto::Category("rendering")
+        .SetDescription("Events from the graphics subsystem"),
+    perfetto::Category("network")
+        .SetDescription("Network upload and download statistics"));
+
+...
+
+int main(int argc, char** argv) {
+  ...
+  perfetto::Tracing::Initialize(args);
+  perfetto::TrackEvent::Register();
+}
+
+...
+
+void LayerTreeHost::DoUpdateLayers() {
+  TRACE_EVENT("rendering", "LayerTreeHost::DoUpdateLayers");
+  ...
+  for (PictureLayer& pl : layers) {
+    TRACE_EVENT("rendering", "PictureLayer::Update");
+    pl.Update();
+  }
+}
+```
+
+Which are rendered in the UI as follows:
+
+![Track event example](/docs/images/track-events.png)
+
+Track events are the best default option and serve most tracing use cases with
+very little complexity.
+
+To include your new track events in the trace, ensure that the `track_event`
+data source is included in the trace config. If you do not specify any
+categories then all non-debug categories will be included by default. However,
+you can also add just the categories you are interested in like so:
+
+```protobuf
+data_sources {
+  config {
+    name: "track_event"
+    track_event_config {
+    	enabled_categories: "rendering"
+    }
+  }
+}
+```
+
+See the [Track events page](track-events.md) for full instructions.
+
+### Custom data sources
+
+For most uses, track events are the most straightforward way of instrumenting
+apps for tracing. However, in some rare circumstances they are not
+flexible enough, e.g., when the data doesn't fit the notion of a track or is
+high volume enough that it needs a strongly typed schema to minimize the size of
+each event. In this case, you can implement a *custom data source* for
+Perfetto.
+
+Unlike track events, when working with custom data sources, you will also need
+corresponding changes in [trace processor](/docs/analysis/trace-processor.md)
+to enable importing your data format.
+
+A custom data source is a subclass of `perfetto::DataSource`. Perfetto will
+automatically create one instance of the class for each tracing session it is
+active in (usually just one).
+
+```C++
+class CustomDataSource : public perfetto::DataSource<CustomDataSource> {
+ public:
+  void OnSetup(const SetupArgs&) override {
+    // Use this callback to apply any custom configuration to your data source
+    // based on the TraceConfig in SetupArgs.
+  }
+
+  void OnStart(const StartArgs&) override {
+    // This notification can be used to initialize the GPU driver, enable
+    // counters, etc. StartArgs will contains the DataSourceDescriptor,
+    // which can be extended.
+  }
+
+  void OnStop(const StopArgs&) override {
+    // Undo any initialization done in OnStart.
+  }
+
+  // Data sources can also have per-instance state.
+  int my_custom_state = 0;
+};
+
+PERFETTO_DECLARE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
+```
+
+The data source's static data should be defined in one source file like this:
+
+```C++
+PERFETTO_DEFINE_DATA_SOURCE_STATIC_MEMBERS(CustomDataSource);
+```
+
+Custom data sources need to be registered with Perfetto:
+
+```C++
+int main(int argc, char** argv) {
+  ...
+  perfetto::Tracing::Initialize(args);
+  // Add the following:
+  perfetto::DataSourceDescriptor dsd;
+  dsd.set_name("com.example.custom_data_source");
+  CustomDataSource::Register(dsd);
+}
+```
+
+As with all data sources, the custom data source needs to be specified in the
+trace config to enable tracing:
+
+```C++
+perfetto::TraceConfig cfg;
+auto* ds_cfg = cfg.add_data_sources()->mutable_config();
+ds_cfg->set_name("com.example.custom_data_source");
+```
+
+Finally, call the `Trace()` method to record an event with your custom data
+source. The lambda function passed to that method will only be called if tracing
+is enabled. It is always called synchronously and possibly multiple times if
+multiple concurrent tracing sessions are active.
+
+```C++
+CustomDataSource::Trace([](CustomDataSource::TraceContext ctx) {
+  auto packet = ctx.NewTracePacket();
+  packet->set_timestamp(perfetto::TrackEvent::GetTraceTimeNs());
+  packet->set_for_testing()->set_str("Hello world!");
+});
+```
+
+If necessary the `Trace()` method can access the custom data source state
+(`my_custom_state` in the example above). Doing so, will take a mutex to
+ensure data source isn't destroyed (e.g., because of stopping tracing) while
+the `Trace()` method is called on another thread. For example:
+
+```C++
+CustomDataSource::Trace([](CustomDataSource::TraceContext ctx) {
+  auto safe_handle = trace_args.GetDataSourceLocked();  // Holds a RAII lock.
+  DoSomethingWith(safe_handle->my_custom_state);
+});
+```
+
+## In-process vs System mode
+
+The two modes are not mutually exclusive. An app can be configured to work
+in both modes and respond both to in-process tracing requests and system
+tracing requests. Both modes generate the same trace file format.
+
+### In-process mode
+
+In this mode both the perfetto service and the app-defined data sources are
+hosted fully in-process, in the same process of the profiled app. No connection
+to the system `traced` daemon will be attempted.
+
+In-process mode can be enabled by setting
+`TracingInitArgs.backends = perfetto::kInProcessBackend` when initializing the
+SDK, see examples below.
+
+This mode is used to generate traces that contain only events emitted by
+the app, but not other types of events (e.g. scheduler traces).
+
+The main advantage is that by running fully in-process, it doesn't require any
+special OS privileges and the profiled process can control the lifecycle of
+tracing sessions.
+
+This mode is supported on Android, Linux, MacOS and Windows.
+
+### System mode
+
+In this mode the app-defined data sources will connect to the external `traced`
+service using the [IPC over UNIX socket][ipc].
+
+System mode can be enabled by setting
+`TracingInitArgs.backends = perfetto::kSystemBackend` when initializing the SDK,
+see examples below.
+
+The main advantage of this mode is that it is possible to create fused traces where
+app events are overlaid on the same timeline of OS events. This enables
+full-stack performance investigations, looking all the way through syscalls and
+kernel scheduling events.
+
+The main limitation of this mode is that it requires the external `traced` daemon
+to be up and running and reachable through the UNIX socket connection.
+
+This is suggested for local debugging or lab testing scenarios where the user
+(or the test harness) can control the OS deployment (e.g., sideload binaries on
+Android).
+
+When using system mode, the tracing session must be controlled from the outside,
+using the `perfetto` command-line client
+(See [reference](/docs/reference/perfetto-cli)). This is because when collecting
+system traces, tracing data producers are not allowed to read back the trace
+data as it might disclose information about other processes and allow
+side-channel attacks.
+
+* On Android 9 (Pie) and beyond, traced is shipped as part of the platform.
+* On older versions of Android, traced can be built from sources using the
+  the [standalone NDK-based workflow](/docs/contributing/build-instructions.md)
+  and sideloaded via adb shell.
+* On Linux and MacOS `traced` must be built and run separately. See the
+  [Linux quickstart](/docs/quickstart/linux-tracing.md) for instructions.
+
+_System mode is not yet supported on Windows, due to the lack of an IPC
+implementation_.
+
+## {#recording} Recording traces through the API
+
+_Tracing through the API is currently only supported with the in-process mode.
+When using system mode, use the `perfetto` cmdline client (see quickstart
+guides)._
+
+First initialize a [TraceConfig](/docs/reference/trace-config-proto.autogen)
+message which specifies what type of data to record.
+
+If your app includes [track events](track-events.md) (i.e, `TRACE_EVENT`), you
+typically want to choose the categories which are enabled for tracing.
+
+By default, all non-debug categories are enabled, but you can enable a specific
+one like this:
+
+```C++
+perfetto::protos::gen::TrackEventConfig track_event_cfg;
+track_event_cfg.add_disabled_categories("*");
+track_event_cfg.add_enabled_categories("rendering");
+```
+
+Next, build the main trace config together with the track event part:
+
+```C++
+perfetto::TraceConfig cfg;
+cfg.add_buffers()->set_size_kb(1024);  // Record up to 1 MiB.
+auto* ds_cfg = cfg.add_data_sources()->mutable_config();
+ds_cfg->set_name("track_event");
+ds_cfg->set_track_event_config_raw(track_event_cfg.SerializeAsString());
+```
+
+If your app includes a custom data source, you can also enable it here:
+
+```C++
+ds_cfg = cfg.add_data_sources()->mutable_config();
+ds_cfg->set_name("my_data_source");
+```
+
+After building the trace config, you can begin tracing:
+
+```C++
+std::unique_ptr<perfetto::TracingSession> tracing_session(
+    perfetto::Tracing::NewTrace());
+tracing_session->Setup(cfg);
+tracing_session->StartBlocking();
+```
+
+TIP: API methods with `Blocking` in their name will suspend the calling thread
+     until the respective operation is complete. There are also asynchronous
+     variants that don't have this limitation.
+
+Now that tracing is active, instruct your app to perform the operation you
+want to record. After that, stop tracing and collect the
+protobuf-formatted trace data:
+
+```C++
+tracing_session->StopBlocking();
+std::vector<char> trace_data(tracing_session->ReadTraceBlocking());
+
+// Write the trace into a file.
+std::ofstream output;
+output.open("example.perfetto-trace", std::ios::out | std::ios::binary);
+output.write(&trace_data[0], trace_data.size());
+output.close();
+```
+
+To save memory with longer traces, you can also tell Perfetto to write
+directly into a file by passing a file descriptor into Setup(), remembering
+to close the file after tracing is done:
+
+```C++
+int fd = open("example.perfetto-trace", O_RDWR | O_CREAT | O_TRUNC, 0600);
+tracing_session->Setup(cfg, fd);
+tracing_session->StartBlocking();
+// ...
+tracing_session->StopBlocking();
+close(fd);
+```
+
+The resulting trace file can be directly opened in the [Perfetto
+UI](https://ui.perfetto.dev) or the [Trace Processor](/docs/analysis/trace-processor.md).
+
+[ipc]: /docs/design-docs/api-and-abi.md#socket-protocol
+[atrace-ds]: /docs/data-sources/atrace.md
+[atrace-ndk]: https://developer.android.com/ndk/reference/group/tracing
+[atrace-sdk]: https://developer.android.com/reference/android/os/Trace