Gbaraldi/att tool (#766)

* Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Gbaraldi/att tool rebase (#801) * Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Removing python from workflow * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Enabling codeobj and thread trace samples * Updating aqlprofile_v2 header * Codeobj and thread trace samples with output log files * Fixing clang format * Cmake formatting * Adding coverage to codeobj * Comment trace sample * Adding ATT Parser API * Fixing forwarding to aqlprofile * Clang formatting * Clang tidy * Adding option to print memory kernels * Clang format * Remove default from switch case * Separating client/main on codeobj sample for ASAn * Formatting * Fix codeobj library * Allow thread trace in parallel with other service * Zeroing the HSA signals * Adding exception wrappers in ATT sample * Removed force configure * Remove force configure from ISA decode * Removing codecov flag * Gbaraldi/att tool tests (#828) * Adding tests for codeobj ISA decode * Adding ATT tests * Adding ATT integration tests * Formatting * Changing codeobj binary extension * Renaming codeobj library spaces * Fixing samples * Formatting * Formatting * Fixing int test * Fixing linker error * Fixing memory fault * Moving kernel ot inside namespace * ASAN linking fix * Removing unecessary headers * Formatting * Fixing target_cu * Remove codeobj binary * Revert "Remove codeobj binary" This reverts commit 7d286f89d8096bc36925cd79cd742a5e6d10d179. * Enable memory snapshot * adding comgr --------- Co-authored-by: Ammar ELWazir <ammar.elwazir@amd.com> [ROCm/rocprofiler-sdk commit: 099ac7c72d]
2024-05-03 18:45:47 -03:00
@@ -29,5 +29,6 @@ add_subdirectory(api_buffered_tracing)
 add_subdirectory(code_object_tracing)
 add_subdirectory(counter_collection)
 add_subdirectory(intercept_table)
+add_subdirectory(code_object_isa_decode)
+add_subdirectory(advanced_thread_trace)
 add_subdirectory(external_correlation_id_request)
-# add_subdirectory(code_object_isa_decode) add_subdirectory(advanced_thread_trace)
@@ -26,23 +26,23 @@ foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
 endforeach()

 find_package(rocprofiler-sdk REQUIRED)
+find_package(amd_comgr REQUIRED)

 add_library(advanced-thread-trace-client SHARED)
 target_sources(advanced-thread-trace-client PRIVATE client.cpp)
+
 target_link_libraries(
    advanced-thread-trace-client
    PRIVATE rocprofiler::rocprofiler rocprofiler::samples-build-flags
-            rocprofiler-sdk-codeobj rocprofiler::samples-common-library)
+            rocprofiler-sdk-codeobj rocprofiler::samples-common-library amd_comgr dw)

 set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
 set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "-g")
-find_package(Threads REQUIRED)

 add_executable(advanced-thread-trace)
 target_sources(advanced-thread-trace PRIVATE main.cpp)
-target_link_libraries(
-    advanced-thread-trace PRIVATE advanced-thread-trace-client Threads::Threads
-                                  rocprofiler::samples-build-flags)
+target_link_libraries(advanced-thread-trace PRIVATE advanced-thread-trace-client
+                                                    rocprofiler::samples-build-flags)

 rocprofiler_samples_get_preload_env(PRELOAD_ENV advanced-thread-trace-client)

@@ -36,12 +36,11 @@
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/registration.h>
 #include <rocprofiler-sdk/rocprofiler.h>
-#include "lib/rocprofiler-sdk/aql/aql_profile_v2.h"
+#include <rocprofiler-sdk-codeobj/code_printing.hpp>
+#include <shared_mutex>

-#include "code_object_track.hpp"
 #include "common/defines.hpp"
 #include "common/filesystem.hpp"
-#include "lib/rocprofiler-sdk-codeobj/code_printing.hpp"

 #include <cxxabi.h>
 #include <atomic>
@@ -64,19 +63,28 @@
 #include <unordered_map>
 #include <vector>

-#define TARGET_CU   1
-#define SIMD_SELECT 0x3
-#define BUFFER_SIZE 0x6000000
-#define NUM_SE      2
-constexpr bool COPY_MEMORY_CODEOBJ = false;
+#define OUTPUT_OFSTREAM "advanced_thread_trace.log"
+#define TARGET_CU       1
+#define SIMD_SELECT     0x3
+#define BUFFER_SIZE     0x6000000
+#define SE_MASK         0x11
+constexpr bool COPY_MEMORY_CODEOBJ = true;

-template <>
-struct std::hash<pcinfo_t>
-{
-    uint64_t operator()(const pcinfo_t& info) const
+#define C_API_BEGIN                                                                                \
+    try                                                                                            \
    {
-        return info.addr ^ (info.marker_id << 32ul) ^ (info.marker_id >> 32ul);
-    }
+#define C_API_END                                                                                  \
+    }                                                                                              \
+    catch(std::exception & e)                                                                      \
+    {                                                                                              \
+        std::cerr << "Error in " << __FILE__ << ':' << __LINE__ << ' ' << e.what() << std::endl;   \
+    }                                                                                              \
+    catch(...) { std::cerr << "Error in " << __FILE__ << ':' << __LINE__ << std::endl; }
+
+struct pcinfo_t
+{
+    uint64_t marker_id;
+    uint64_t addr;
 };

 bool
@@ -97,6 +105,10 @@ namespace client
 using code_obj_load_data_t = rocprofiler_callback_tracing_code_object_load_data_t;
 using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;

+using Instruction             = rocprofiler::codeobj::disassembly::Instruction;
+using CodeobjAddressTranslate = rocprofiler::codeobj::disassembly::CodeobjAddressTranslate;
+using SymbolInfo              = rocprofiler::codeobj::disassembly::SymbolInfo;
+
 rocprofiler_client_id_t* client_id  = nullptr;
 rocprofiler_context_id_t client_ctx = {};

@@ -109,13 +121,35 @@ struct isa_map_elem_t

 struct ToolData
 {
+    ToolData()
+    {
+        try
+        {
+            output_file.open(OUTPUT_OFSTREAM);
+        } catch(...)
+        {}
+
+        if(output_file.is_open())
+            std::cout << "Writing code-object-isa-decode log to: " << OUTPUT_OFSTREAM << std::endl;
+        else
+            std::cout << "Could not open log file: " << OUTPUT_OFSTREAM << ", writing to stdout\n";
+    };
+
    std::shared_mutex                                   isa_map_mut;
    std::mutex                                          output_mut;
    CodeobjAddressTranslate                             codeobjTranslate;
    std::map<pcinfo_t, std::unique_ptr<isa_map_elem_t>> isa_map;
    std::unordered_map<uint64_t, SymbolInfo>            kernels_in_codeobj           = {};
    std::unordered_map<uint64_t, std::string>           kernel_object_to_kernel_name = {};
-    std::stringstream                                   output;
+    int                                                 num_waves                    = 0;
+
+    std::ostream& output()
+    {
+        if(output_file.is_open())
+            return output_file;
+        else
+            return std::cout;
+    }

    std::stringstream printKernel(uint64_t vaddr)
    {
@@ -133,6 +167,9 @@ struct ToolData
        }
        return ss;
    }
+
+private:
+    std::ofstream output_file;
 };

 struct source_location
@@ -151,15 +188,12 @@ struct trace_data_t
    ToolData* tool;
 };

-std::atomic<int> TRACE_DATA_ID{-1};
-std::atomic<int> KERNEL_ADDR_ID{-1};
-std::atomic<int> OCCUPANCY_ID{-1};
-
 void
 tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
                              rocprofiler_user_data_t*              user_data,
                              void*                                 callback_data)
 {
+    C_API_BEGIN
    if(record.kind != ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT) return;
    if(record.phase != ROCPROFILER_CALLBACK_PHASE_LOAD) return;

@@ -200,19 +234,18 @@ tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,

    (void) user_data;
    (void) callback_data;
+    C_API_END
 }

-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-parameter"
-
 rocprofiler_att_control_flags_t
-dispatch_callback(rocprofiler_queue_id_t              queue_id,
-                  const rocprofiler_agent_t*          agent,
-                  rocprofiler_correlation_id_t        correlation_id,
+dispatch_callback(rocprofiler_queue_id_t /* queue_id  */,
+                  const rocprofiler_agent_t* /* agent  */,
+                  rocprofiler_correlation_id_t /* correlation_id  */,
                  const hsa_kernel_dispatch_packet_t* dispatch_packet,
-                  uint64_t                            kernel_id,
-                  void*                               userdata)
+                  uint64_t /* kernel_id */,
+                  void* userdata)
 {
+    C_API_BEGIN
    assert(userdata && "Dispatch callback passed null!");
    ToolData& tool = *reinterpret_cast<ToolData*>(userdata);

@@ -236,82 +269,55 @@ dispatch_callback(rocprofiler_queue_id_t              queue_id,
                  << std::endl;
    }

+    C_API_END
    return ROCPROFILER_ATT_CONTROL_NONE;
 }

 void
-iterate_trace_types(int id, const char* metadata, void*)
-{
-    if(std::string_view(metadata).find("occupancy") == 0)
-        OCCUPANCY_ID.store(id);
-    else if(std::string_view(metadata).find("kernel_ids_addr") == 0)
-        KERNEL_ADDR_ID.store(id);
-    else if(std::string_view(metadata).find("tracedata") == 0)
-        TRACE_DATA_ID.store(id);
-}
-
-hsa_status_t
-get_trace_data(int      trace_type_id,
-               int      correlation_id,
-               void*    trace_events,
-               uint64_t trace_size,
-               void*    userdata)
+get_trace_data(rocprofiler_att_parser_data_type_t type, void* att_data, void* userdata)
 {
+    C_API_BEGIN
    assert(userdata && "ISA callback passed null!");
    trace_data_t& trace_data = *reinterpret_cast<trace_data_t*>(userdata);
    assert(trace_data.tool && "ISA callback passed null!");
    ToolData& tool = *reinterpret_cast<ToolData*>(trace_data.tool);

-    std::stringstream                   ss;
    std::shared_lock<std::shared_mutex> shared_lock(tool.isa_map_mut);

-    if(trace_type_id == OCCUPANCY_ID)
-    {
-        ss << "Num waves: " << trace_size / 2 << '\n';
-        // auto* occ = reinterpret_cast<att_occupancy_info_t*>(trace_events);
-    }
-    else if(trace_type_id == KERNEL_ADDR_ID)
-    {
-        ss << "Num KRN events: " << trace_size << std::hex << '\n';
-        auto* kaddr = reinterpret_cast<pcinfo_t*>(trace_events);
-        for(size_t i = 0; i < trace_size; i++)
-            if(kaddr[i].addr != 0)
-            {
-                ss << " - ADDR: " << kaddr[i].addr << ' ' << tool.printKernel(kaddr[i].addr).str()
-                   << '\n';
-            }
-        ss << std::dec;
-    }
-    else if(trace_type_id == TRACE_DATA_ID)
-    {
-        ss << "Trace Length: " << trace_size << '\n';
-        auto* tracedata = reinterpret_cast<att_trace_event_t*>(trace_events);
+    if(type == ROCPROFILER_ATT_PARSER_DATA_TYPE_OCCUPANCY) tool.num_waves++;

-        for(size_t i = 0; i < trace_size; i++)
+    if(type != ROCPROFILER_ATT_PARSER_DATA_TYPE_ISA) return;
+
+    auto& event = *reinterpret_cast<rocprofiler_att_data_type_isa_t*>(att_data);
+
+    pcinfo_t pc{event.marker_id, event.offset};
+    auto     it = tool.isa_map.find(pc);
+    if(it == tool.isa_map.end())
+    {
+        shared_lock.unlock();
        {
-            pcinfo_t pc = tracedata[i].pc;
-            auto     it = tool.isa_map.find(pc);
-            if(it == tool.isa_map.end())
+            std::unique_lock<std::shared_mutex> unique_lock(tool.isa_map_mut);
+            auto                                ptr = std::make_unique<isa_map_elem_t>();
+            try
            {
-                shared_lock.unlock();
-                {
-                    std::unique_lock<std::shared_mutex> unique_lock(tool.isa_map_mut);
-                    auto                                ptr = std::make_unique<isa_map_elem_t>();
-                    ptr->code_line = tool.codeobjTranslate.get(pc.marker_id, pc.addr);
-                    it             = tool.isa_map.emplace(pc, std::move(ptr)).first;
-                }
-                shared_lock.lock();
+                ptr->code_line = tool.codeobjTranslate.get(pc.marker_id, pc.addr);
+            } catch(std::exception& e)
+            {
+                std::cerr << pc.marker_id << ":" << pc.addr << ' ' << e.what() << std::endl;
+                return;
+            } catch(...)
+            {
+                std::cerr << "Could not fetch: " << pc.marker_id << ':' << pc.addr << std::endl;
+                return;
            }
-
-            it->second->hitcount.fetch_add(tracedata[i].hitcount, std::memory_order_relaxed);
-            it->second->latency.fetch_add(tracedata[i].latency, std::memory_order_relaxed);
+            it = tool.isa_map.emplace(pc, std::move(ptr)).first;
        }
+        shared_lock.lock();
    }

-    std::unique_lock<std::mutex> lk(tool.output_mut);
-    tool.output << ss.str();
-
-    return HSA_STATUS_SUCCESS;
+    it->second->hitcount.fetch_add(event.hitcount, std::memory_order_relaxed);
+    it->second->latency.fetch_add(event.latency, std::memory_order_relaxed);
+    C_API_END
 }

 uint64_t
@@ -325,16 +331,15 @@ copy_trace_data(int* seid, uint8_t** buffer, uint64_t* buffer_size, void* userda
    return *buffer_size;
 }

-hsa_status_t
+rocprofiler_status_t
 isa_callback(char*     isa_instruction,
-             char*     source_reference,
             uint64_t* isa_memory_size,
             uint64_t* isa_size,
-             uint64_t* source_size,
             uint64_t  marker_id,
             uint64_t  offset,
             void*     userdata)
 {
+    C_API_BEGIN
    assert(userdata && "ISA callback passed null!");
    trace_data_t& trace_data = *reinterpret_cast<trace_data_t*>(userdata);
    assert(trace_data.tool && "ISA callback passed null!");
@@ -347,56 +352,47 @@ isa_callback(char*     isa_instruction,
        instruction = tool.codeobjTranslate.get(marker_id, offset);
    }

-    if(!instruction.get()) return HSA_STATUS_ERROR_INVALID_ARGUMENT;
+    if(!instruction.get()) return ROCPROFILER_STATUS_ERROR_INVALID_ARGUMENT;

    {
-        size_t tmp_isa_size    = *isa_size;
-        size_t tmp_source_size = *source_size;
-        *isa_size              = instruction->inst.size();
-        *source_size           = instruction->comment.size();
+        size_t tmp_isa_size = *isa_size;
+        *isa_size           = instruction->inst.size();

-        if(*isa_size > tmp_isa_size || *source_size > tmp_source_size)
-            return HSA_STATUS_ERROR_OUT_OF_RESOURCES;
+        if(*isa_size > tmp_isa_size) return ROCPROFILER_STATUS_ERROR_OUT_OF_RESOURCES;
    }

    memcpy(isa_instruction, instruction->inst.data(), *isa_size);
-    memcpy(source_reference, instruction->comment.data(), *source_size);
    *isa_memory_size = instruction->size;

    auto ptr       = std::make_unique<isa_map_elem_t>();
    ptr->code_line = std::move(instruction);
-    tool.isa_map.emplace(pcinfo_t{offset, marker_id}, std::move(ptr));
-
-    return HSA_STATUS_SUCCESS;
+    tool.isa_map.emplace(pcinfo_t{marker_id, offset}, std::move(ptr));
+    C_API_END
+    return ROCPROFILER_STATUS_SUCCESS;
 }

 void
-shader_data_callback(int64_t     se_id,
-                     int64_t     data_type_id,
-                     const char* data_type_name,
-                     void*       se_data,
-                     size_t      data_size,
-                     void*       userdata)
+shader_data_callback(int64_t se_id, void* se_data, size_t data_size, void* userdata)
 {
+    C_API_BEGIN
    assert(userdata && "Shader callback passed null!");
    ToolData& tool = *reinterpret_cast<ToolData*>(userdata);

    {
        std::unique_lock<std::mutex> lk(tool.output_mut);
-        tool.output << "SE ID: " << se_id << " with size " << data_size << std::hex << '\n';
+        tool.output() << "SE ID: " << se_id << " with size " << data_size << std::hex << '\n';
    }
    trace_data_t data{.id = se_id, .data = (uint8_t*) se_data, .size = data_size, .tool = &tool};
-    auto status = aqlprofile_att_parse_data(copy_trace_data, get_trace_data, isa_callback, &data);
-    (void) status;
+    auto status = rocprofiler_att_parse_data(copy_trace_data, get_trace_data, isa_callback, &data);
+    if(status != ROCPROFILER_STATUS_SUCCESS)
+        std::cerr << "shader_data_callback failed with status " << status << std::endl;
+    C_API_END
 }

-#pragma GCC diagnostic pop
-
 int
 tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
 {
    (void) fini_func;
-    aqlprofile_att_parser_iterate_event_list(iterate_trace_types, nullptr);
    ROCPROFILER_CALL(rocprofiler_create_context(&client_ctx), "context creation");

    ROCPROFILER_CALL(
@@ -408,22 +404,19 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
                                                       tool_data),
        "code object tracing service configure");

-    rocprofiler_att_parameters_t parameters{};
-    parameters.target_cu   = TARGET_CU;
-    parameters.simd_select = SIMD_SELECT;
-    parameters.buffer_size = BUFFER_SIZE;
+    std::vector<rocprofiler_att_parameter_t> parameters;
+    parameters.push_back({ROCPROFILER_ATT_PARAMETER_TARGET_CU, TARGET_CU});
+    parameters.push_back({ROCPROFILER_ATT_PARAMETER_SIMD_SELECT, SIMD_SELECT});
+    parameters.push_back({ROCPROFILER_ATT_PARAMETER_BUFFER_SIZE, BUFFER_SIZE});
+    parameters.push_back({ROCPROFILER_ATT_PARAMETER_SHADER_ENGINE_MASK, SE_MASK});

-    std::vector<int> shaders;
-    for(size_t i = 0; i < NUM_SE; i++)
-        shaders.push_back(2 * i);  // use shader engines 0, 2
-
-    parameters.shader_ids = shaders.data();
-    parameters.shader_num = shaders.size();
-
-    ROCPROFILER_CALL(
-        rocprofiler_configure_thread_trace_service(
-            client_ctx, parameters, dispatch_callback, shader_data_callback, tool_data),
-        "thread trace service configure");
+    ROCPROFILER_CALL(rocprofiler_configure_thread_trace_service(client_ctx,
+                                                                parameters.data(),
+                                                                parameters.size(),
+                                                                dispatch_callback,
+                                                                shader_data_callback,
+                                                                tool_data),
+                     "thread trace service configure");

    int valid_ctx = 0;
    ROCPROFILER_CALL(rocprofiler_context_is_valid(client_ctx, &valid_ctx),
@@ -476,9 +469,9 @@ tool_fini(void* tool_data)
            size_t latency   = line->latency.load(std::memory_order_relaxed);
            auto&  code_line = line->code_line->inst;

-            tool.output << std::hex << "0x" << addr.addr << std::dec << ' ' << code_line
-                        << empty_space.substr(0, max_inst_size - code_line.size())
-                        << " Hit: " << hitcount << " - Latency: " << latency << '\n';
+            tool.output() << std::hex << "0x" << addr.addr << std::dec << ' ' << code_line
+                          << empty_space.substr(0, max_inst_size - code_line.size())
+                          << " Hit: " << hitcount << " - Latency: " << latency << '\n';

            if(code_line.find("s_waitcnt") == 0)
            {
@@ -511,30 +504,16 @@ tool_fini(void* tool_data)
    float  vmc_fraction   = 100 * vmc_latency / float(total_latency);
    float  lgk_fraction   = 100 * lgk_latency / float(total_latency);

-    tool.output << "Total executed instructions: " << total_exec << '\n'
-                << "Total executed vector instructions: " << vector_exec << " with average "
-                << vector_latency / float(vector_exec) << " cycles.\n"
-                << "Total executed scalar instructions: " << scalar_exec << " with average "
-                << scalar_latency / float(scalar_exec) << " cycles.\n"
-                << "Vector memory ops occupied: " << vmc_fraction << "% of cycles.\n"
-                << "Scalar and LDS memory ops occupied: " << lgk_fraction << "% of cycles.\n";
-
-    std::cout << tool.output.str();
+    tool.output() << "Total executed instructions: " << total_exec << '\n'
+                  << "Total executed vector instructions: " << vector_exec << " with average "
+                  << vector_latency / float(vector_exec) << " cycles.\n"
+                  << "Total executed scalar instructions: " << scalar_exec << " with average "
+                  << scalar_latency / float(scalar_exec) << " cycles.\n"
+                  << "Vector memory ops occupied: " << vmc_fraction << "% of cycles.\n"
+                  << "Scalar and LDS memory ops occupied: " << lgk_fraction << "% of cycles.\n"
+                  << std::endl;
 }

-void
-setup()
-{
-    if(int status = 0;
-       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
-    {
-        ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
-                         "force configuration");
-    }
-}
-
-// force configuration when library is loaded
-bool cfg_on_load = (client::setup(), true);
 }  // namespace client

 extern "C" rocprofiler_tool_configure_result_t*
@@ -26,21 +26,28 @@ foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
 endforeach()

 find_package(rocprofiler-sdk REQUIRED)
-find_package(Threads REQUIRED)
+find_package(amd_comgr REQUIRED)

-add_executable(code-object-isa-decode)
-target_sources(code-object-isa-decode PRIVATE main.cpp client.cpp)
+add_library(code-object-isa-decode-client SHARED)
+target_sources(code-object-isa-decode-client PRIVATE client.cpp)
 set_source_files_properties(main.cpp PROPERTIES LANGUAGE HIP)
 set_source_files_properties(main.cpp PROPERTIES COMPILE_FLAGS "-g")

 target_link_libraries(
-    code-object-isa-decode PRIVATE Threads::Threads rocprofiler::samples-common-library
-                                   rocprofiler-sdk-codeobj rocprofiler::rocprofiler)
+    code-object-isa-decode-client
+    PRIVATE rocprofiler::samples-common-library rocprofiler-sdk-codeobj
+            rocprofiler::rocprofiler amd_comgr dw)
+
+rocprofiler_samples_get_preload_env(PRELOAD_ENV code-object-isa-decode-client)
+
+add_executable(code-object-isa-decode)
+target_sources(code-object-isa-decode PRIVATE main.cpp)
+target_link_libraries(code-object-isa-decode PRIVATE code-object-isa-decode-client
+                                                     rocprofiler::samples-build-flags)

 add_test(NAME code-object-isa-decode COMMAND $<TARGET_FILE:code-object-isa-decode>)

 set_tests_properties(
    code-object-isa-decode
-    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT
-               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}" FAIL_REGULAR_EXPRESSION
-               "threw an exception")
+    PROPERTIES TIMEOUT 45 LABELS "samples" ENVIRONMENT "${PRELOAD_ENV}"
+               FAIL_REGULAR_EXPRESSION "threw an exception")
@@ -25,6 +25,8 @@
 #    undef NDEBUG
 #endif

+#define OUTPUT_OFSTREAM "code_obj_isa_decode.log"
+
 /**
 * @file samples/code_object_isa_decode/client.cpp
 *
@@ -36,7 +38,7 @@
 #include <rocprofiler-sdk/fwd.h>
 #include <rocprofiler-sdk/registration.h>
 #include <rocprofiler-sdk/rocprofiler.h>
-#include "lib/rocprofiler-sdk-codeobj/code_printing.hpp"
+#include <rocprofiler-sdk-codeobj/code_printing.hpp>

 #include "common/defines.hpp"
 #include "common/filesystem.hpp"
@@ -60,16 +62,37 @@
 #include <string_view>
 #include <thread>
 #include <vector>
-#include "code_object_track.hpp"
+
+constexpr bool COPY_MEMORY_CODEOBJ = true;

 namespace client
 {
+std::ostream&
+output_stream()
+{
+    static std::ofstream file(OUTPUT_OFSTREAM);
+
+    static bool file_is_open_check = [&]() {
+        if(!file.is_open())
+            std::cout << "Could not open log file: " << OUTPUT_OFSTREAM << ", writing to stdout\n";
+        else
+            std::cout << "Writing code-object-isa-decode log to: " << OUTPUT_OFSTREAM << std::endl;
+        return file.is_open();
+    }();
+
+    if(!file_is_open_check) return std::cout;
+    return file;
+};
+
 namespace
 {
 using code_obj_load_data_t = rocprofiler_callback_tracing_code_object_load_data_t;
 using kernel_symbol_data_t = rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t;
 using kernel_symbol_map_t  = std::unordered_map<std::string, std::pair<uint64_t, size_t>>;

+using Instruction             = rocprofiler::codeobj::disassembly::Instruction;
+using CodeobjAddressTranslate = rocprofiler::codeobj::disassembly::CodeobjAddressTranslate;
+
 rocprofiler_client_id_t*      client_id          = nullptr;
 rocprofiler_client_finalize_t client_fini_func   = nullptr;
 rocprofiler_context_id_t      client_ctx         = {};
@@ -92,28 +115,41 @@ tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
        if(std::string_view(data->uri).find("file:///") == 0)
        {
            codeobjTranslate.addDecoder(
-                data->uri, data->code_object_id, data->load_base, data->load_size);
-            auto symbolmap = codeobjTranslate.getSymbolMap();
-            for(auto& [vaddr, symbol] : symbolmap)
-                registered_kernels.insert({symbol.name, {vaddr, vaddr + symbol.mem_size}});
+                data->uri, data->code_object_id, data->load_delta, data->load_size);
        }
+        else if(COPY_MEMORY_CODEOBJ)
+        {
+            codeobjTranslate.addDecoder(reinterpret_cast<const void*>(data->memory_base),
+                                        data->memory_size,
+                                        data->code_object_id,
+                                        data->load_delta,
+                                        data->load_size);
+        }
+        else
+        {
+            return;
+        }
+
+        auto symbolmap = codeobjTranslate.getSymbolMap(data->code_object_id);
+        for(auto& [vaddr, symbol] : symbolmap)
+            registered_kernels.insert({symbol.name, {vaddr, vaddr + symbol.mem_size}});
    }
    else if(record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
    {
-        std::cout << std::hex;
+        output_stream() << std::hex;
        auto* data        = static_cast<kernel_symbol_data_t*>(record.payload);
        auto  kernel_name = std::regex_replace(data->kernel_name, std::regex{"(\\.kd)$"}, "");

        if(registered_kernels.find(kernel_name) == registered_kernels.end())
        {
-            std::cout << "Not Found: " << kernel_name << " in codeobj." << std::endl;
+            output_stream() << "Not Found: " << kernel_name << " in codeobj." << std::endl;
            return;
        }

        auto& begin_end = registered_kernels.at(kernel_name);

-        std::cout << std::hex << "Found: " << kernel_name << " at addr: 0x" << begin_end.first
-                  << std::dec << ". Printing first 64 bytes:" << std::endl;
+        output_stream() << std::hex << "Found: " << kernel_name << " at addr: 0x" << begin_end.first
+                        << std::dec << ". Printing first 64 bytes:" << std::endl;

        std::unordered_set<std::string> references{};
        int                             num_waitcnts = 0;
@@ -127,7 +163,7 @@ tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
            auto             inst   = codeobjTranslate.get(vaddr);
            std::string_view source = inst->comment;
            if(source.rfind('/') < source.size()) source = source.substr(source.rfind('/'));
-            if(vaddr < begin_end.first + 64) std::cout << '\t' << inst->inst << '\n';
+            if(vaddr < begin_end.first + 64) output_stream() << '\t' << inst->inst << '\n';

            if(source.rfind(':') < source.size()) source = source.substr(0, source.rfind(':'));

@@ -144,12 +180,13 @@ tool_codeobj_tracing_callback(rocprofiler_callback_tracing_record_t record,
            vaddr += inst->size;
        }

-        std::cout << "  --- Num Scalar: " << num_scalar << "\n  --- Num Vector: " << num_vector
-                  << "\n  --- Num Waitcnts: " << num_waitcnts
-                  << "\n  --- Other instructions: " << num_other
-                  << "\nKernel has source references to: " << std::endl;
+        output_stream() << "  --- Num Scalar: " << num_scalar
+                        << "\n  --- Num Vector: " << num_vector
+                        << "\n  --- Num Waitcnts: " << num_waitcnts
+                        << "\n  --- Other instructions: " << num_other
+                        << "\nKernel has source references to: " << std::endl;
        for(auto& ref : references)
-            std::cout << '\t' << ref << std::endl;
+            output_stream() << '\t' << ref << std::endl;
    }

    (void) user_data;
@@ -195,20 +232,8 @@ tool_fini(void* tool_data)
    (void) tool_data;
 }

-void
-setup()
-{
-    if(int status = 0;
-       rocprofiler_is_initialized(&status) == ROCPROFILER_STATUS_SUCCESS && status == 0)
-    {
-        ROCPROFILER_CALL(rocprofiler_force_configure(&rocprofiler_configure),
-                         "force configuration");
-    }
-}
 }  // namespace

-// force configuration when library is loaded
-bool cfg_on_load = (client::setup(), true);
 }  // namespace client

 extern "C" rocprofiler_tool_configure_result_t*