PC sampling: online partial PC sampling decoding (#1004)

* PC sampling: online partial PC sampling decoding

PC sampling service decodes a PC sample partially
by replacing the PC with an id of the loaded code object instance
containing PC and the offset of the PC within that code object instance.

* PC sampling: marker records removed

* PC sampling parser: minor doc update in mock

* PC sampling: introducing rocprofiler_pc_t

* NULL value of the code object id introduced.

* Clarifying documenation related to PC offset.

* PC offset documentation improvement

* PC sampling parser benchmark: Reducing the number of samples to recreate half of performance.

samples/pc_sampling/pcs.cpp

@@ -310,7 +310,8 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
                 auto* pc_sample =
                     static_cast<rocprofiler_pc_sampling_record_t*>(cur_header->payload);
 
-                ss << "pc: " << std::hex << pc_sample->pc << ", "
+                ss << "(code_obj_id, offset): (" << pc_sample->pc.loaded_code_object_id << ", 0x"
+                   << std::hex << pc_sample->pc.loaded_code_object_offset << "), "
                    << "timestamp: " << std::dec << pc_sample->timestamp << ", "
                    << "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
                    << "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x
@@ -327,17 +328,9 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
                    << "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}"
                    << std::endl;
             }
-            else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_LOAD_MARKER)
+            else
             {
-                auto* marker = static_cast<rocprofiler_pc_sampling_code_object_load_marker_t*>(
-                    cur_header->payload);
-                ss << "code object loading: " << marker->code_object_id << std::endl;
-            }
-            else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_UNLOAD_MARKER)
-            {
-                auto* marker = static_cast<rocprofiler_pc_sampling_code_object_unload_marker_t*>(
-                    cur_header->payload);
-                ss << "code object unloading: " << marker->code_object_id << std::endl;
+                assert(false);
             }
         }
         else

source/include/rocprofiler-sdk/callback_tracing.h

@@ -130,6 +130,11 @@ typedef struct
     };
 } rocprofiler_callback_tracing_code_object_load_data_t;
 
+/**
+ * @brief The NULL value of a code object id. Used when code object is unknown.
+ */
+#define ROCPROFILER_CODE_OBJECT_ID_NONE ROCPROFILER_UINT64_C(0)
+
 /**
  * @brief ROCProfiler Code Object Kernel Symbol Tracer Callback Record.
  *

source/include/rocprofiler-sdk/fwd.h

@@ -411,9 +411,7 @@ typedef enum
 typedef enum
 {
     ROCPROFILER_PC_SAMPLING_RECORD_NONE = 0,
-    ROCPROFILER_PC_SAMPLING_RECORD_SAMPLE,                   ///< ::rocprofiler_pc_sampling_record_t
-    ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_LOAD_MARKER,  ///< ::rocprofiler_pc_sampling_code_object_load_marker_t
-    ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_UNLOAD_MARKER,  ///< ::rocprofiler_pc_sampling_code_object_unload_marker_t
+    ROCPROFILER_PC_SAMPLING_RECORD_SAMPLE,  ///< ::rocprofiler_pc_sampling_record_t
     ROCPROFILER_PC_SAMPLING_RECORD_LAST,
 } rocprofiler_pc_sampling_record_kind_t;
 

source/include/rocprofiler-sdk/pc_sampling.h

@@ -239,6 +239,31 @@ typedef struct
     uint32_t arb_state_stall   : 10;
 } rocprofiler_pc_sampling_snapshot_v1_t;
 
+/**
+ * @brief Sampled program counter.
+ */
+typedef struct
+{
+    uint64_t loaded_code_object_id;
+    uint64_t loaded_code_object_offset;
+
+    /// @var loaded_code_object_id
+    /// @brief id of the loaded code object instance that contains sampled PC.
+    /// This fields holds the value ::ROCPROFILER_CODE_OBJECT_ID_NONE
+    /// if the code object cannot be determined
+    /// (e.g., sampled PC belongs to code generated by self modifying code).
+    /// @var loaded_code_object_offset
+    /// @brief If @ref loaded_code_object_id is different than ::ROCPROFILER_CODE_OBJECT_ID_NONE,
+    /// then this field contains the offset of the sampled PC relative to the
+    /// ::rocprofiler_callback_tracing_code_object_load_data_t::load_base
+    /// of the code object instance with @ref loaded_code_object_id.
+    /// To calculate the original virtual address of the sampled PC, one can add the value
+    /// of this field to the ::rocprofiler_callback_tracing_code_object_load_data_t::load_base.
+    /// The value of @ref loaded_code_object_offset matches
+    /// the virtual address of the sampled instruction (PC), only if the
+    /// @ref loaded_code_object_id is equal to the ::ROCPROFILER_CODE_OBJECT_ID_NONE.
+} rocprofiler_pc_t;
+
 // TODO: The definition of this structure might change over time
 // to reduce the space needed to represent a single sample.
 /**
@@ -253,12 +278,12 @@ typedef struct
     uint8_t                             wave_issued : 1;
     uint8_t                             reserved    : 7;  ///< reserved 7 bits, must be zero
     uint32_t                            hw_id;            ///< compute unit identifier
-    uint64_t                     pc;  ///< Program counter of the wave of the moment of interruption
-    uint64_t                     exec_mask;
-    rocprofiler_dim3_t           workgroup_id;  ///< wave coordinates within the workgroup
-    uint32_t                     wave_count;
-    uint64_t                     timestamp;  ///< timestamp when sample is generated
-    rocprofiler_correlation_id_t correlation_id;
+    rocprofiler_pc_t                    pc;  ///< information about sampled program counter
+    uint64_t                            exec_mask;
+    rocprofiler_dim3_t                  workgroup_id;  ///< wave coordinates within the workgroup
+    uint32_t                            wave_count;
+    uint64_t                            timestamp;  ///< timestamp when sample is generated
+    rocprofiler_correlation_id_t        correlation_id;
     rocprofiler_pc_sampling_snapshot_v1_t
              snapshot;   ///< @see ::rocprofiler_pc_sampling_snapshot_v1_t
     uint32_t reserved2;  ///< for future use
@@ -274,42 +299,18 @@ typedef struct
     /// @var wave_count
     /// @brief number of active waves on the CU at the moment of sample generation
     /// @var correlation_id
-    /// @brief correlation id of the API call that initiated kernel launch.
-    /// The interrupted wave is executed as part of the kernel.
+    /// @brief correlation id of the API call that initiated a dispatch of the kernel
+    /// during whose execution the wave was interrupted at @ref pc.
 } rocprofiler_pc_sampling_record_t;
 
-/**
- * @brief Marker representing code object loading event.
- *
- * @see rocprofiler_callback_tracing_code_object_load_data_t
- * for more information
- */
-typedef struct
-{
-    uint64_t size;            ///< Size of this struct
-    uint64_t code_object_id;  /// unique code object identifier
-} rocprofiler_pc_sampling_code_object_load_marker_t;
-
-/**
- * @brief Marker representing code object unloading event.
- *
- * @see rocprofiler_callback_tracing_code_object_load_data_t
- * for more information
- */
-typedef struct
-{
-    uint64_t size;            ///< Size of this struct
-    uint64_t code_object_id;  /// unique code object identifier
-} rocprofiler_pc_sampling_code_object_unload_marker_t;
-
 /** @} */
 
 ROCPROFILER_EXTERN_C_FINI
 
 ROCPROFILER_CXX_CODE(
-    static_assert(sizeof(rocprofiler_pc_sampling_record_t) == 80,
+    static_assert(sizeof(rocprofiler_pc_sampling_record_t) == 88,
                   "Increasing the size of the pc sampling record is not permitted."));
 
 ROCPROFILER_CXX_CODE(static_assert(offsetof(rocprofiler_pc_sampling_record_t, chiplet) == 9 &&
-                                       offsetof(rocprofiler_pc_sampling_record_t, reserved2) == 76,
+                                       offsetof(rocprofiler_pc_sampling_record_t, reserved2) == 84,
                                    "PC sampling record layout changed."));

source/lib/rocprofiler-sdk/pc_sampling/code_object.cpp

@@ -26,6 +26,7 @@
 
 #    include "lib/common/container/operators.hpp"
 #    include "lib/common/logging.hpp"
+#    include "lib/common/static_object.hpp"
 #    include "lib/rocprofiler-sdk/code_object/code_object.hpp"
 #    include "lib/rocprofiler-sdk/pc_sampling/service.hpp"
 
@@ -44,6 +45,13 @@ namespace pc_sampling
 {
 namespace code_object
 {
+CodeobjTableTranslatorSynchronized*
+get_code_object_translator()
+{
+    static auto*& _v = common::static_object<CodeobjTableTranslatorSynchronized>::construct();
+    return _v;
+}
+
 namespace
 {
 auto&
@@ -74,14 +82,13 @@ get_destroy_function()
  * @param [in] code_object - loaded/unloaded code object.
  */
 void
-flush_buffers_generate_marker_record(rocprofiler_callback_phase_t                      phase,
-                                     const rocprofiler::code_object::hsa::code_object& code_object)
+flush_pc_sampling_buffers(const rocprofiler::code_object::hsa::code_object& code_object)
 {
     auto agent_id = code_object.rocp_data.rocp_agent;
     if(!is_pc_sample_service_configured(agent_id)) return;
 
-    // The PC sampling service is configured on the agent.
-    // Find the agent's buffer and place marker record.
+    // The PC sampling service is configured on the agent,
+    // so flush its PC sampling buffer
     // TODO: Creating a function that gives the buffer_id based on the agent_id?
     const auto* pcs_service     = get_configured_pc_sampling_service().load();
     const auto* agent_session   = pcs_service->agent_sessions.at(agent_id).get();
@@ -89,46 +96,6 @@ flush_buffers_generate_marker_record(rocprofiler_callback_phase_t
 
     // flush internal PC sampling buffers
     flush_internal_agent_buffers(agent_buffer_id);
-
-    auto* buff = rocprofiler::buffer::get_buffer(agent_buffer_id);
-
-    // create code object load/unload marker record and emplace it into the SDK's PC SAMPLING
-    // buffer.
-    if(phase == ROCPROFILER_CALLBACK_PHASE_LOAD)
-    {
-        auto marker =
-            common::init_public_api_struct(rocprofiler_pc_sampling_code_object_load_marker_t{});
-        marker.code_object_id = code_object.rocp_data.code_object_id;
-        // emplace marker to the SDK's PC sampling buffer
-        buff->emplace(ROCPROFILER_BUFFER_CATEGORY_PC_SAMPLING,
-                      ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_LOAD_MARKER,
-                      marker);
-    }
-    else
-    {
-        auto marker =
-            common::init_public_api_struct(rocprofiler_pc_sampling_code_object_unload_marker_t{});
-        marker.code_object_id = code_object.rocp_data.code_object_id;
-        // emplace marker to the SDK's PC sampling buffer
-        buff->emplace(ROCPROFILER_BUFFER_CATEGORY_PC_SAMPLING,
-                      ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_UNLOAD_MARKER,
-                      marker);
-    }
-
-    // Assuming that the `rocprofiler_pc_sampling_code_object_load_marker_t` and
-    // `rocprofiler_pc_sampling_code_object_unload_marker_t` share the same content,
-    // we could replace the previous if else with the following
-    /*
-        auto marker =
-       common::init_public_api_struct(rocprofiler_pc_sampling_code_object_load_marker_t{});
-        marker.code_object_id = code_object.rocp_data.code_object_id;
-        // emplace marker to the SDK's PC sampling buffer
-        buff->emplace(ROCPROFILER_BUFFER_CATEGORY_PC_SAMPLING,
-                    (phase == ROCPROFILER_CALLBACK_PHASE_LOAD) ?
-                        ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_LOAD_MARKER
-                        : ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_UNLOAD_MARKER,
-                    marker);
-    */
 }
 
 hsa_status_t
@@ -141,7 +108,14 @@ executable_freeze(hsa_executable_t executable, const char* options)
     rocprofiler::code_object::iterate_loaded_code_objects(
         [&](const rocprofiler::code_object::hsa::code_object& code_object) {
             if(code_object.hsa_executable == executable)
-                flush_buffers_generate_marker_record(ROCPROFILER_CALLBACK_PHASE_LOAD, code_object);
+            {
+                const auto& code_object_rocp = code_object.rocp_data;
+                get_code_object_translator()->insert(
+                    address_range_t{code_object_rocp.load_base,
+                                    code_object_rocp.load_size,
+                                    code_object_rocp.code_object_id});
+                flush_pc_sampling_buffers(code_object);
+            }
         });
 
     return HSA_STATUS_SUCCESS;
@@ -153,8 +127,14 @@ executable_destroy(hsa_executable_t executable)
     rocprofiler::code_object::iterate_loaded_code_objects(
         [&](const rocprofiler::code_object::hsa::code_object& code_object) {
             if(code_object.hsa_executable == executable)
-                flush_buffers_generate_marker_record(ROCPROFILER_CALLBACK_PHASE_UNLOAD,
-                                                     code_object);
+            {
+                flush_pc_sampling_buffers(code_object);
+                const auto& code_object_rocp = code_object.rocp_data;
+                get_code_object_translator()->remove(
+                    address_range_t{code_object_rocp.load_base,
+                                    code_object_rocp.load_size,
+                                    code_object_rocp.code_object_id});
+            }
         });
 
     // Call underlying function
@@ -183,7 +163,7 @@ finalize()
 {
     rocprofiler::code_object::iterate_loaded_code_objects(
         [&](const rocprofiler::code_object::hsa::code_object& code_object) {
-            flush_buffers_generate_marker_record(ROCPROFILER_CALLBACK_PHASE_UNLOAD, code_object);
+            flush_pc_sampling_buffers(code_object);
         });
 }
 

source/lib/rocprofiler-sdk/pc_sampling/code_object.hpp

@@ -22,12 +22,19 @@
 
 #pragma once
 
+#include "lib/common/static_object.hpp"
 #include "lib/rocprofiler-sdk/pc_sampling/defines.hpp"
 
 #if ROCPROFILER_SDK_HSA_PC_SAMPLING > 0
 
+#    include <rocprofiler-sdk/callback_tracing.h>
+#    include <rocprofiler-sdk/cxx/codeobj/segment.hpp>
+
 #    include <hsa/hsa_api_trace.h>
 
+#    include <mutex>
+#    include <shared_mutex>
+
 namespace rocprofiler
 {
 namespace pc_sampling
@@ -39,6 +46,49 @@ initialize(HsaApiTable* table);
 
 void
 finalize();
+
+using address_range_t        = rocprofiler::sdk::codeobj::segment::address_range_t;
+using CodeobjTableTranslator = rocprofiler::sdk::codeobj::segment::CodeobjTableTranslator;
+
+class CodeobjTableTranslatorSynchronized : public CodeobjTableTranslator
+{
+    using Super            = CodeobjTableTranslator;
+    using code_object_id_t = uint64_t;
+
+public:
+    // Must acquire write lock
+    void insert(address_range_t addr_range)
+    {
+        auto lock = std::unique_lock{mut};
+        this->Super::insert(addr_range);
+    }
+
+    // Must acquire write lock
+    bool remove(address_range_t addr_range)
+    {
+        auto lock = std::unique_lock{mut};
+        return this->Super::remove(addr_range);
+    }
+
+    // Must acquire read lock
+    address_range_t find_codeobj_in_range(uint64_t addr) const
+    {
+        // TODO: It would be good to have a way to cache search results
+        // (caching could be done easily in the parser)
+        auto lock = std::shared_lock{mut};
+        auto it   = this->find(address_range_t{addr, 0, 0});
+        // `addr` might originate from an unknown code object.
+        if(it == this->end()) return address_range_t{0, 0, ROCPROFILER_CODE_OBJECT_ID_NONE};
+        return *it;
+    }
+
+private:
+    mutable std::shared_mutex mut = {};
+};
+
+CodeobjTableTranslatorSynchronized*
+get_code_object_translator();
+
 }  // namespace code_object
 }  // namespace pc_sampling
 }  // namespace rocprofiler

source/lib/rocprofiler-sdk/pc_sampling/parser/correlation.hpp

@@ -23,6 +23,7 @@
 #pragma once
 
 #include "lib/common/logging.hpp"
+#include "lib/rocprofiler-sdk/pc_sampling/code_object.hpp"
 #include "lib/rocprofiler-sdk/pc_sampling/parser/translation.hpp"
 
 #include <rocprofiler-sdk/fwd.h>
@@ -186,22 +187,37 @@ add_upcoming_samples(const device_handle               device,
                      Parser::CorrelationMap*           corr_map,
                      rocprofiler_pc_sampling_record_t* samples)
 {
-    pcsample_status_t status = PCSAMPLE_STATUS_SUCCESS;
+    pcsample_status_t status     = PCSAMPLE_STATUS_SUCCESS;
+    auto        cache_addr_range = rocprofiler::pc_sampling::code_object::address_range_t{0, 0, 0};
+    const auto* code_object_translator =
+        rocprofiler::pc_sampling::code_object::get_code_object_translator();
     for(uint64_t p = 0; p < available_samples; p++)
     {
         const auto* snap = reinterpret_cast<const perf_sample_snapshot_v1*>(buffer + p);
         samples[p]       = copySample<bHostTrap, GFXIP>((const void*) (buffer + p));
-        samples[p].size  = sizeof(rocprofiler_pc_sampling_record_t);
+
+        auto& pc_sample = samples[p];
+        pc_sample.size  = sizeof(rocprofiler_pc_sampling_record_t);
+
+        // Convert PC -> (loaded code object id containing PC, offset within code object)
+        if(!cache_addr_range.inrange(snap->pc))
+        {
+            cache_addr_range = code_object_translator->find_codeobj_in_range(snap->pc);
+        }
+
+        pc_sample.pc.loaded_code_object_id     = cache_addr_range.id;
+        pc_sample.pc.loaded_code_object_offset = snap->pc - cache_addr_range.addr;
+
         try
         {
             Parser::trap_correlation_id_t trap{.raw = snap->correlation_id};
-            samples[p].correlation_id = corr_map->get(device, trap);
+            pc_sample.correlation_id = corr_map->get(device, trap);
         } catch(std::exception& e)
         {
-            samples[p].correlation_id = {.internal = ROCPROFILER_CORRELATION_ID_INTERNAL_NONE,
-                                         .external = rocprofiler_user_data_t{
-                                             .value = ROCPROFILER_CORRELATION_ID_INTERNAL_NONE}};
-            status                    = PCSAMPLE_STATUS_PARSER_ERROR;
+            pc_sample.correlation_id = {.internal = ROCPROFILER_CORRELATION_ID_INTERNAL_NONE,
+                                        .external = rocprofiler_user_data_t{
+                                            .value = ROCPROFILER_CORRELATION_ID_INTERNAL_NONE}};
+            status                   = PCSAMPLE_STATUS_PARSER_ERROR;
         }
     }
     return status;

source/lib/rocprofiler-sdk/pc_sampling/parser/tests/benchmark_test.cpp

@@ -35,7 +35,7 @@ static bool
 Benchmark(bool bWarmup)
 {
     constexpr size_t SAMPLE_PER_DISPATCH = 8192;
-    constexpr size_t DISP_PER_QUEUE      = 12;
+    constexpr size_t DISP_PER_QUEUE      = 8;
     constexpr size_t NUM_QUEUES          = MockDoorBell::num_unique_bells;
 
     std::shared_ptr<MockRuntimeBuffer> buffer = std::make_shared<MockRuntimeBuffer>();

source/lib/rocprofiler-sdk/pc_sampling/parser/tests/correlation_id_test.cpp

@@ -53,8 +53,10 @@ alloc_callback(rocprofiler_pc_sampling_record_t** buffer, uint64_t size, void* u
 static bool
 check_samples(rocprofiler_pc_sampling_record_t* samples, uint64_t size)
 {
+    // TODO: replace with (code_obj_id, pc)
     for(size_t i = 0; i < size; i++)
-        if(samples[i].correlation_id.internal != samples[i].pc) return false;
+        if(samples[i].correlation_id.internal != samples[i].pc.loaded_code_object_offset)
+            return false;
     return true;
 }
 

source/lib/rocprofiler-sdk/pc_sampling/parser/tests/gfx9test.cpp

@@ -299,7 +299,10 @@ class WaveIssueAndErrorTest : public WaveSnapTest
     {
         rocprofiler_pc_sampling_record_t sample;
         ::memset(&sample, 0, sizeof(sample));
-        sample.pc                      = dispatch->unique_id;
+        // TODO: Since code objects are not mocked, use pc.loaded_code_object_offset
+        // as the absolute physical address of the mocked PC.
+        sample.pc.loaded_code_object_offset = dispatch->unique_id;
+
         sample.correlation_id.internal = dispatch->getMockId().raw;
 
         sample.flags.valid              = valid && !error;

source/lib/rocprofiler-sdk/pc_sampling/parser/translation.hpp

@@ -39,7 +39,6 @@ copySampleHeader(const SType& sample)
     ret.flags      = pcsample_header_v1_t{.raw = 0}.flags;
     ret.flags.type = AMD_SNAPSHOT_V1;
 
-    ret.pc             = sample.pc;
     ret.exec_mask      = sample.exec_mask;
     ret.workgroup_id.x = sample.workgroup_id_x;
     ret.workgroup_id.y = sample.workgroup_id_y;

tests/pc_sampling/pcs.cpp

@@ -339,7 +339,8 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
                     auto* pc_sample =
                         static_cast<rocprofiler_pc_sampling_record_t*>(cur_header->payload);
 
-                    ss << "pc: " << std::hex << pc_sample->pc << ", "
+                    ss << "(code_obj_id, offset): (" << pc_sample->pc.loaded_code_object_id
+                       << ", 0x" << std::hex << pc_sample->pc.loaded_code_object_offset << "), "
                        << "timestamp: " << std::dec << pc_sample->timestamp << ", "
                        << "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
                        << "workgroup_id_(x=" << std::dec << std::setw(5)
@@ -372,30 +373,13 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
                     assert(corr_id.external.value > 0);
 
                     // Decoding the PC
-                    auto inst = translator.get(pc_sample->pc);
+                    auto inst = translator.get(pc_sample->pc.loaded_code_object_id,
+                                               pc_sample->pc.loaded_code_object_offset);
                     flat_profile.add_sample(std::move(inst), pc_sample->exec_mask);
                 }
-                else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_LOAD_MARKER)
+                else
                 {
-                    auto* marker = static_cast<rocprofiler_pc_sampling_code_object_load_marker_t*>(
-                        cur_header->payload);
-                    auto code_object_id = marker->code_object_id;
-                    ss << "code object loading: " << code_object_id << std::endl;
-                    // The code object load event can be reported once per code object id.
-                    assert(pc_sampler->active_code_objects.count(code_object_id) == 0);
-                    pc_sampler->active_code_objects.emplace(code_object_id);
-                }
-                else if(cur_header->kind ==
-                        ROCPROFILER_PC_SAMPLING_RECORD_CODE_OBJECT_UNLOAD_MARKER)
-                {
-                    auto* marker =
-                        static_cast<rocprofiler_pc_sampling_code_object_unload_marker_t*>(
-                            cur_header->payload);
-                    auto code_object_id = marker->code_object_id;
-                    ss << "code object unloading: " << code_object_id << std::endl;
-                    // The code object unload event can be reported once per code object id.
-                    assert(pc_sampler->active_code_objects.count(code_object_id) == 1);
-                    pc_sampler->active_code_objects.erase(code_object_id);
+                    assert(false);
                 }
             }
             else