AI/rocm-systems
2
0
Förgrening 0
Kod Ärenden Pull-förfrågningar Actions Packages Projekt Släpp Wiki Aktiviteter

Correlation ID Retirement + misc (#527)

Bläddra i källkod 
* Correlation ID Retirement

- include/rocprofiler-sdk/buffer_tracing.h
  - add rocprofiler_buffer_tracing_correlation_id_retirement_record_t
- include/rocprofiler-sdk/fwd.h
  - ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT
- lib/rocprofiler-sdk/buffer_tracing.cpp
  - kind string for correlation id retirement
- lib/rocprofiler-sdk/buffer.hpp
  - emplace returns bool
- lib/rocprofiler-sdk/registration.cpp
  - pass lib_instance to copy_table functions
- lib/rocprofiler-sdk/context/context.*
  - update correlation_id struct
    - make ref_count private
    - {get,add,sub}_ref_count() functions
      - sub_ref_count() performs correlation id retirement
    - use stack for "latest" thread-local correlation id
- lib/rocprofiler-sdk/hip/hip.*
  - migrate to new {get,add,sub}_ref_count() for correlation ids
  - return in iterate_args
  - handle table instance in copy_table
- lib/rocprofiler-sdk/hsa/hsa.*
  - migrate to new {get,add,sub}_ref_count() for correlation ids
  - return in iterate_args
  - handle table instance in copy_table
- lib/rocprofiler-sdk/marker/marker.*
  - migrate to new {get,add,sub}_ref_count() for correlation ids
  - return in iterate_args
  - handle table instance in copy_table
- lib/rocprofiler-sdk/hsa/async_copy.cpp
  - migrate to new {get,add,sub}_ref_count() for correlation ids
  - handle table instance in async_copy_init / async_copy_save
- lib/rocprofiler-sdk/hsa/queue.cpp
  - migrate to new {get,add,sub}_ref_count() for correlation ids
  - tweak to external correlation id mapping in WriteInterceptor
- tests/async-copy-tracing/validate.py
  - check retired_correlation_ids
- tests/common/serialization.hpp
  - support rocprofiler_buffer_tracing_correlation_id_retirement_record_t
- tests/kernel-tracing/validate.py
  - check retired_correlation_ids
- tests/common/CMakeLists.txt
  - perfetto external project
- tests/common/perfetto.hpp
  - perfetto categories + aliases
  - add_perfetto_annotation
  - metaprogramming helpers
- tests/tools/CMakeLists.txt
  - link to tests-perfetto
- tests/tools/json-tool.cpp
  - demangling functions
  - serialization of marker API callback args
  - reduce parallel bottleneck in tool_tracing_callback
  - support correlation id retirement
  - Multiple threads for buffers
  - Support ROCPROFILER_TOOL_CONTEXTS_EXCLUDE env variable
  - write_perfetto() function

* Update tests/rocprofv3/tracing/validate.py

- tweak test_hsa_api_trace

* Update PTL submodule

- fixes for data race during destruction of task

* Update lib/rocprofiler-sdk/buffer.*

- unique_buffer_vec_t uses std::unique_ptr instead of allocator::unique_static_ptr_t

* Reduce timeouts in counter collection samples [skip ci]

* Update tests/tools/json-tool.cpp

- tweak demangle(string_view, int*) -> demangle(string_view, int&)

* Update lib/rocprofiler-sdk/hsa/async_copy.cpp

- move sub_ref_count() to later in async_copy_handler to delay retirement slightly more
This commit is contained in:
Jonathan R. Madsen
2024-02-23 10:30:33 -06:00
committad av GitHub
förälder 2d71520953
incheckning 875f53b608
27 ändrade filer med 1217 tillägg och 184 borttagningar
Download Patch File Download Diff File Expand all files Collapse all files
source/include/rocprofiler-sdk/buffer_tracing.h
+19
Visa fil
@@ -164,6 +164,25 @@ typedef struct
// Not Sure What is the info needed here?
} rocprofiler_buffer_tracing_scratch_memory_record_t;
/**
* @brief ROCProfiler Buffer Correlation ID Retirement Tracer Record.
*/
typedef struct
{
uint64_t size; ///< size of this struct
rocprofiler_buffer_tracing_kind_t kind;
rocprofiler_timestamp_t timestamp;
uint64_t internal_correlation_id;
/// @var kind
/// @brief ::ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT
/// @var timestamp
/// @brief Timestamp (in nanosec) of when rocprofiler detected the correlation ID could be
/// retired
/// @var internal_correlation_id
/// @brief Only internal correlation ID is provided
} rocprofiler_buffer_tracing_correlation_id_retirement_record_t;
/**
* @brief Callback function for mapping @ref rocprofiler_buffer_tracing_kind_t ids to
* string names. @see rocprofiler_iterate_buffer_trace_kind_names.
source/include/rocprofiler-sdk/fwd.h
+1
Visa fil
@@ -171,6 +171,7 @@ typedef enum // NOLINT(performance-enum-size)
ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH, ///< Buffer kernel dispatch info
ROCPROFILER_BUFFER_TRACING_PAGE_MIGRATION, ///< Buffer page migration info
ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY, ///< Buffer scratch memory reclaimation info
ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, ///< Correlation ID in no longer in use
ROCPROFILER_BUFFER_TRACING_LAST,
} rocprofiler_buffer_tracing_kind_t;
source/lib/rocprofiler-sdk/buffer.cpp
+13 -7
Visa fil
@@ -22,9 +22,6 @@
#include "lib/rocprofiler-sdk/buffer.hpp"
#include <glog/logging.h>
#include <rocprofiler-sdk/rocprofiler.h>
#include "lib/common/container/stable_vector.hpp"
#include "lib/common/static_object.hpp"
#include "lib/common/utility.hpp"
@@ -34,6 +31,11 @@
#include "lib/rocprofiler-sdk/internal_threading.hpp"
#include "lib/rocprofiler-sdk/registration.hpp"
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/rocprofiler.h>
#include <glog/logging.h>
#include <atomic>
#include <exception>
#include <mutex>
@@ -121,7 +123,7 @@ allocate_buffer()
// create an entry in the registered
auto& _cfg_v = CHECK_NOTNULL(get_buffers())->back();
_cfg_v = allocator::make_unique_static<buffer::instance>();
_cfg_v = std::make_unique<buffer::instance>();
auto* _cfg = _cfg_v.get();
if(!_cfg) return std::nullopt;
@@ -150,6 +152,8 @@ flush(rocprofiler_buffer_id_t buffer_id, bool wait)
auto* buff = get_buffer(buffer_id);
if(!buff) return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
auto* task_group =
internal_threading::get_task_group(rocprofiler_callback_thread_t{buff->task_group_id});
@@ -211,11 +215,10 @@ flush(rocprofiler_buffer_id_t buffer_id, bool wait)
buff_v->syncer.clear();
};
task_group->exec(_task);
task_group->exec(std::move(_task));
if(wait)
{
while(task_group->size() > 0)
task_group->join();
task_group->join();
}
return ROCPROFILER_STATUS_SUCCESS;
@@ -283,6 +286,8 @@ rocprofiler_destroy_buffer(rocprofiler_buffer_id_t buffer_id)
auto* buffers = CHECK_NOTNULL(rocprofiler::buffer::get_buffers());
auto& buff = buffers->at(buffer_id.handle - offset);
if(!buff) return ROCPROFILER_STATUS_ERROR_BUFFER_NOT_FOUND;
// buffer is currently being flushed or destroyed
if(buff->syncer.test_and_set()) return ROCPROFILER_STATUS_ERROR_BUFFER_BUSY;
@@ -290,6 +295,7 @@ rocprofiler_destroy_buffer(rocprofiler_buffer_id_t buffer_id)
itr.reset();
buff->syncer.clear();
buff.reset();
return ROCPROFILER_STATUS_SUCCESS;
}
source/lib/rocprofiler-sdk/buffer.hpp
+9 -9
Visa fil
@@ -28,7 +28,6 @@
#include "lib/common/container/record_header_buffer.hpp"
#include "lib/common/container/stable_vector.hpp"
#include "lib/common/demangle.hpp"
#include "lib/rocprofiler-sdk/allocator.hpp"
#include <array>
#include <atomic>
@@ -56,14 +55,13 @@ struct instance
rocprofiler_buffer_policy_t policy = ROCPROFILER_BUFFER_POLICY_NONE;
template <typename Tp>
void emplace(uint32_t, uint32_t, Tp&);
bool emplace(uint32_t, uint32_t, Tp&);
buffer_t& get_internal_buffer();
buffer_t& get_internal_buffer(size_t);
};
using unique_buffer_vec_t =
common::container::stable_vector<allocator::unique_static_ptr_t<instance>, 4>;
using unique_buffer_vec_t = common::container::stable_vector<std::unique_ptr<instance>, 4>;
bool
is_valid_buffer_id(rocprofiler_buffer_id_t id);
@@ -114,14 +112,15 @@ rocprofiler::buffer::flush(uint64_t buffer_idx, bool wait)
}
template <typename Tp>
inline void
inline bool
rocprofiler::buffer::instance::emplace(uint32_t category, uint32_t kind, Tp& value)
{
// get the index of the current buffer
auto get_idx = [this]() { return buffer_idx.load(std::memory_order_acquire) % buffers.size(); };
auto idx = get_idx();
if(!buffers.at(idx).emplace(category, kind, value))
auto idx = get_idx();
auto success = buffers.at(idx).emplace(category, kind, value);
if(!success)
{
if(buffers.at(idx).capacity() < sizeof(value))
{
@@ -135,10 +134,9 @@ rocprofiler::buffer::instance::emplace(uint32_t category, uint32_t kind, Tp& val
if(policy == ROCPROFILER_BUFFER_POLICY_LOSSLESS)
{
// blocks until buffer is flushed
bool success = false;
while(!success)
{
buffer::flush(buffer_id, true);
buffer::flush(buffer_id, false);
idx = get_idx();
success = buffers.at(idx).emplace(category, kind, value);
}
@@ -154,4 +152,6 @@ rocprofiler::buffer::instance::emplace(uint32_t category, uint32_t kind, Tp& val
// flush without syncing
buffer::flush(buffer_id, false);
}
return success;
}
source/lib/rocprofiler-sdk/buffer_tracing.cpp
+3
Visa fil
@@ -77,6 +77,7 @@ ROCPROFILER_BUFFER_TRACING_KIND_STRING(MEMORY_COPY)
ROCPROFILER_BUFFER_TRACING_KIND_STRING(KERNEL_DISPATCH)
ROCPROFILER_BUFFER_TRACING_KIND_STRING(PAGE_MIGRATION)
ROCPROFILER_BUFFER_TRACING_KIND_STRING(SCRATCH_MEMORY)
ROCPROFILER_BUFFER_TRACING_KIND_STRING(CORRELATION_ID_RETIREMENT)
template <size_t Idx, size_t... Tail>
std::pair<const char*, size_t>
@@ -222,6 +223,7 @@ rocprofiler_query_buffer_tracing_kind_operation_name(rocprofiler_buffer_tracing_
case ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH:
case ROCPROFILER_BUFFER_TRACING_PAGE_MIGRATION:
case ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY:
case ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT:
{
return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
}
@@ -320,6 +322,7 @@ rocprofiler_iterate_buffer_tracing_kind_operations(
case ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH:
case ROCPROFILER_BUFFER_TRACING_PAGE_MIGRATION:
case ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY:
case ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT:
{
return ROCPROFILER_STATUS_ERROR_NOT_IMPLEMENTED;
}
source/lib/rocprofiler-sdk/context/context.cpp
+75 -7
Visa fil
@@ -24,6 +24,7 @@
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/rocprofiler.h>
#include "lib/common/container/small_vector.hpp"
#include "lib/common/container/stable_vector.hpp"
#include "lib/common/static_object.hpp"
#include "lib/common/synchronized.hpp"
@@ -118,10 +119,10 @@ get_correlation_id_map()
return _v;
}
auto*&
auto&
get_latest_correlation_id_impl()
{
static thread_local correlation_id* _v = nullptr;
static thread_local auto _v = common::container::small_vector<correlation_id*, 16>{};
return _v;
}
@@ -133,6 +134,53 @@ get_unique_internal_id()
}
} // namespace
uint32_t
correlation_id::add_ref_count()
{
return m_ref_count.fetch_add(1);
}
uint32_t
correlation_id::sub_ref_count()
{
auto _ret = m_ref_count.fetch_sub(1);
LOG_IF(FATAL, _ret == 0) << "correlation id underflow";
if(_ret == 1)
{
auto ctxs = get_active_contexts([](const context* ctx) {
return (ctx->buffered_tracer &&
(ctx->buffered_tracer->domains(
ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT)));
});
auto record = rocprofiler_buffer_tracing_correlation_id_retirement_record_t{
.size = sizeof(rocprofiler_buffer_tracing_correlation_id_retirement_record_t),
.kind = ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT,
.timestamp = common::timestamp_ns(),
.internal_correlation_id = internal};
if(!ctxs.empty())
{
for(const auto* itr : ctxs)
{
auto* _buffer = buffer::get_buffer(itr->buffered_tracer->buffer_data.at(
ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT));
auto success = CHECK_NOTNULL(_buffer)->emplace(
ROCPROFILER_BUFFER_CATEGORY_TRACING,
ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT,
record);
LOG_IF(FATAL, !success) << "failed to emplace correlation id retirement";
}
}
}
return _ret;
}
correlation_id*
correlation_tracing_service::construct(uint32_t _init_ref_count)
{
@@ -142,7 +190,7 @@ correlation_tracing_service::construct(uint32_t _init_ref_count)
auto& ret = corr_id_map->wlock([](auto& data) -> auto& { return data.emplace_back(); });
ret = std::make_unique<correlation_id>(_init_ref_count, common::get_tid(), _internal_id);
get_latest_correlation_id_impl() = ret.get();
get_latest_correlation_id_impl().emplace_back(ret.get());
return ret.get();
}
@@ -150,13 +198,33 @@ correlation_tracing_service::construct(uint32_t _init_ref_count)
correlation_id*
get_latest_correlation_id()
{
return get_latest_correlation_id_impl();
return (get_latest_correlation_id_impl().empty()) ? nullptr
: get_latest_correlation_id_impl().back();
}
void
pop_latest_correlation_id(const correlation_id* val)
const correlation_id*
pop_latest_correlation_id(correlation_id* val)
{
if(get_latest_correlation_id_impl() == val) get_latest_correlation_id_impl() = nullptr;
if(!val)
{
LOG(ERROR) << "passed nullptr to correlation id";
return nullptr;
}
if(get_latest_correlation_id_impl().empty())
{
LOG(ERROR) << "empty thread-local correlation id stack";
return nullptr;
}
LOG_IF(ERROR, get_latest_correlation_id_impl().back() != val)
<< "pop_latest_correlation_id is happening out of order for " << val->internal
<< ". top of stack is " << get_latest_correlation_id_impl().back()->internal;
get_latest_correlation_id_impl().pop_back();
return (get_latest_correlation_id_impl().empty()) ? nullptr
: get_latest_correlation_id_impl().back();
}
context_array_t&
source/lib/rocprofiler-sdk/context/context.hpp
+11 -5
Visa fil
@@ -59,9 +59,9 @@ struct correlation_id
// thus, after two HSA buffer flushes, we will have received all the PC samples for
// the
correlation_id(uint32_t _cnt, rocprofiler_thread_id_t _tid, uint64_t _internal) noexcept
: ref_count{_cnt}
, thread_idx{_tid}
: thread_idx{_tid}
, internal{_internal}
, m_ref_count{_cnt}
{}
correlation_id() = default;
@@ -72,9 +72,15 @@ struct correlation_id
correlation_id& operator=(const correlation_id&) = delete;
correlation_id& operator=(correlation_id&&) noexcept = delete;
std::atomic<uint32_t> ref_count = {};
rocprofiler_thread_id_t thread_idx = 0;
uint64_t internal = 0;
uint32_t get_ref_count() const { return m_ref_count.load(); }
uint32_t add_ref_count();
uint32_t sub_ref_count();
private:
std::atomic<uint32_t> m_ref_count = {};
};
correlation_id*
@@ -84,8 +90,8 @@ get_correlation_id(rocprofiler_thread_id_t tid, uint64_t internal_id);
correlation_id*
get_latest_correlation_id();
void
pop_latest_correlation_id(const correlation_id*);
const correlation_id*
pop_latest_correlation_id(correlation_id*);
/// permits tools opportunity to modify the correlation id based on the domain, op, and
/// the rocprofiler generated correlation id
source/lib/rocprofiler-sdk/hip/hip.cpp
+30 -20
Visa fil
@@ -308,7 +308,7 @@ hip_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count before invoking
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
auto _ret = exec(info_type::get_table_func(), std::forward<Args>(args)...);
@@ -360,7 +360,7 @@ hip_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count after usage in the callback/buffers
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
context::pop_latest_correlation_id(corr_id);
@@ -454,6 +454,7 @@ iterate_args(const uint32_t id,
user_data);
if(ret != 0) break;
}
return;
}
if constexpr(sizeof...(OpIdxTail) > 0)
iterate_args<TableIdx>(id, data, func, user_data, std::index_sequence<OpIdxTail...>{});
@@ -485,7 +486,7 @@ should_wrap_functor(rocprofiler_callback_tracing_kind_t _callback_domain,
template <size_t TableIdx, typename Tp, size_t OpIdx>
void
copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::integral_constant<size_t, OpIdx>)
{
using table_type = typename hip_table_lookup<TableIdx>::type;
@@ -493,24 +494,34 @@ copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
{
auto _info = hip_api_info<TableIdx, OpIdx>{};
LOG(INFO) << "copying table entry for " << _info.name;
// make sure we don't access a field that doesn't exist in input table
if(_info.offset() >= _orig->size) return;
// 1. get the sub-table containing the function pointer in original table
// 2. get reference to function pointer in sub-table in original table
auto& _table = _info.get_table(_orig);
auto& _func = _info.get_table_func(_table);
auto& _orig_table = _info.get_table(_orig);
auto& _orig_func = _info.get_table_func(_orig_table);
// 3. get the sub-table containing the function pointer in saved table
// 4. get reference to function pointer in sub-table in saved table
// 5. save the original function in the saved table
auto& _saved = _info.get_table(get_table());
auto& _ofunc = _info.get_table_func(_saved);
_ofunc = _func;
}
auto& _copy_table = _info.get_table(get_table());
auto& _copy_func = _info.get_table_func(_copy_table);
(void) _orig;
LOG_IF(FATAL, _copy_func && _tbl_instance == 0)
<< _info.name << " has non-null function pointer " << _copy_func
<< " despite this being the first instance of the library being copies";
if(!_copy_func)
{
LOG(INFO) << "copying table entry for " << _info.name;
_copy_func = _orig_func;
}
else
{
LOG(INFO) << "skipping copying table entry for " << _info.name
<< " from table instance " << _tbl_instance;
}
}
}
template <size_t TableIdx, typename Tp, size_t OpIdx>
@@ -540,17 +551,15 @@ update_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
auto& _func = _info.get_table_func(_table);
_func = _info.get_functor(_func);
}
(void) _orig;
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
void
copy_table(Tp* _orig, std::index_sequence<OpIdx, OpIdxTail...>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::index_sequence<OpIdx, OpIdxTail...>)
{
copy_table<TableIdx>(_orig, std::integral_constant<size_t, OpIdx>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::integral_constant<size_t, OpIdx>{});
if constexpr(sizeof...(OpIdxTail) > 0)
copy_table<TableIdx>(_orig, std::index_sequence<OpIdxTail...>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::index_sequence<OpIdxTail...>{});
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
@@ -617,11 +626,12 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig)
copy_table(TableT* _orig, uint64_t _tbl_instance)
{
constexpr auto TableIdx = hip_table_id_lookup<TableT>::value;
if(_orig)
copy_table<TableIdx>(_orig, std::make_index_sequence<hip_domain_info<TableIdx>::last>{});
copy_table<TableIdx>(
_orig, _tbl_instance, std::make_index_sequence<hip_domain_info<TableIdx>::last>{});
}
template <typename TableT>
@@ -637,7 +647,7 @@ using hip_api_data_t = rocprofiler_callback_tracing_hip_api_data_t;
using hip_op_args_cb_t = rocprofiler_callback_tracing_operation_args_cb_t;
#define INSTANTIATE_HIP_TABLE_FUNC(TABLE_TYPE, TABLE_IDX) \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl, uint64_t _instv); \
template void update_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
template const char* name_by_id<TABLE_IDX>(uint32_t); \
template uint32_t id_by_name<TABLE_IDX>(const char*); \
source/lib/rocprofiler-sdk/hip/hip.hpp
+1 -1
Visa fil
@@ -103,7 +103,7 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig);
copy_table(TableT* _orig, uint64_t _tbl_instance);
template <typename TableT>
void
source/lib/rocprofiler-sdk/hsa/async_copy.cpp
+31 -12
Visa fil
@@ -249,8 +249,6 @@ async_copy_handler(hsa_signal_value_t signal_value, void* arg)
}
}
if(_corr_id) _corr_id->ref_count.fetch_sub(1);
auto* orig_amd_signal = convert_hsa_handle<amd_signal_t>(_data->orig_signal);
// Original intercepted signal completion
@@ -277,6 +275,8 @@ async_copy_handler(hsa_signal_value_t signal_value, void* arg)
get_active_signals()->fetch_sub(1);
}
if(_corr_id) _corr_id->sub_ref_count();
return (signal_value > 0);
}
@@ -487,7 +487,7 @@ async_copy_impl(Args... args)
for(const auto* ctx : _data->contexts)
extern_corr_ids.emplace(ctx,
ctx->correlation_tracer.external_correlator.get(_data->tid));
_data->correlation_id->ref_count.fetch_add(1);
_data->correlation_id->add_ref_count();
}
_data->orig_signal = _completion_signal;
@@ -505,27 +505,46 @@ auto get_async_copy_impl(RetT (*)(Args...))
template <size_t TableIdx, size_t OpIdx>
void
async_copy_save(hsa_amd_ext_table_t* _orig)
async_copy_save(hsa_amd_ext_table_t* _orig, uint64_t _tbl_instance)
{
static_assert(
std::is_same<hsa_amd_ext_table_t, typename hsa_table_lookup<TableIdx>::type>::value,
"unexpected type");
auto _meta = hsa_api_meta<TableIdx, OpIdx>{};
auto& _table = _meta.get_table(_orig);
auto& _func = _meta.get_table_func(_table);
get_next_dispatch<TableIdx, OpIdx>() = _func;
auto _meta = hsa_api_meta<TableIdx, OpIdx>{};
// original table and function
auto& _orig_table = _meta.get_table(_orig);
auto& _orig_func = _meta.get_table_func(_orig_table);
// table with copy function
auto& _copy_func = get_next_dispatch<TableIdx, OpIdx>();
LOG_IF(FATAL, _copy_func && _tbl_instance == 0)
<< _meta.name << " has non-null function pointer " << _copy_func
<< " despite this being the first instance of the library being copies";
if(!_copy_func)
{
LOG(INFO) << "copying table entry for " << _meta.name;
_copy_func = _orig_func;
}
else
{
LOG(INFO) << "skipping copying table entry for " << _meta.name << " from table instance "
<< _tbl_instance;
}
}
template <size_t TableIdx, size_t... OpIdx>
void
async_copy_save(hsa_amd_ext_table_t* _orig, std::index_sequence<OpIdx...>)
async_copy_save(hsa_amd_ext_table_t* _orig, uint64_t _tbl_instance, std::index_sequence<OpIdx...>)
{
static_assert(
std::is_same<hsa_amd_ext_table_t, typename hsa_table_lookup<TableIdx>::type>::value,
"unexpected type");
(async_copy_save<TableIdx, OpIdx>(_orig), ...);
(async_copy_save<TableIdx, OpIdx>(_orig, _tbl_instance), ...);
}
template <size_t TableIdx, size_t OpIdx>
@@ -594,12 +613,12 @@ get_names()
} // namespace async_copy
void
async_copy_init(hsa_api_table_t* _orig)
async_copy_init(hsa_api_table_t* _orig, uint64_t _tbl_instance)
{
if(_orig && _orig->amd_ext_)
{
async_copy::async_copy_save<ROCPROFILER_HSA_TABLE_ID_AmdExt>(
_orig->amd_ext_, async_copy::async_copy_index_seq_t{});
_orig->amd_ext_, _tbl_instance, async_copy::async_copy_index_seq_t{});
auto ctxs = context::get_registered_contexts(async_copy::context_filter);
if(!ctxs.empty())
source/lib/rocprofiler-sdk/hsa/async_copy.hpp
+1 -1
Visa fil
@@ -46,7 +46,7 @@ get_ids();
} // namespace async_copy
void
async_copy_init(hsa_api_table_t* _orig);
async_copy_init(hsa_api_table_t* _orig, uint64_t _tbl_instance);
void
async_copy_fini();
source/lib/rocprofiler-sdk/hsa/hsa.cpp
+38 -26
Visa fil
@@ -35,6 +35,8 @@
#include <rocprofiler-sdk/callback_tracing.h>
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/hsa/api_id.h>
#include <rocprofiler-sdk/hsa/core_api_id.h>
#include <rocprofiler-sdk/hsa/table_id.h>
#include <glog/logging.h>
@@ -364,7 +366,7 @@ hsa_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count before invoking
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
auto _ret = exec(info_type::get_table_func(), std::forward<Args>(args)...);
@@ -416,7 +418,7 @@ hsa_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count after usage in the callback/buffers
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
context::pop_latest_correlation_id(corr_id);
@@ -510,6 +512,7 @@ iterate_args(const uint32_t id,
user_data);
if(ret != 0) break;
}
return;
}
if constexpr(sizeof...(IdxTail) > 0)
iterate_args<TableIdx>(id, data, func, user_data, std::index_sequence<IdxTail...>{});
@@ -542,7 +545,7 @@ should_wrap_functor(const context::context_array_t& _contexts,
template <size_t TableIdx, typename Tp, size_t OpIdx>
void
copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::integral_constant<size_t, OpIdx>)
{
using table_type = typename hsa_table_lookup<TableIdx>::type;
@@ -550,24 +553,34 @@ copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
{
auto _info = hsa_api_info<TableIdx, OpIdx>{};
LOG(INFO) << "copying table entry for " << _info.name;
// make sure we don't access a field that doesn't exist in input table
if(_info.offset() >= _orig->version.minor_id) return;
// 1. get the sub-table containing the function pointer in original table
// 2. get reference to function pointer in sub-table in original table
auto& _table = _info.get_table(_orig);
auto& _func = _info.get_table_func(_table);
auto& _orig_table = _info.get_table(_orig);
auto& _orig_func = _info.get_table_func(_orig_table);
// 3. get the sub-table containing the function pointer in saved table
// 4. get reference to function pointer in sub-table in saved table
// 5. save the original function in the saved table
auto& _saved = _info.get_table(hsa_table_lookup<TableIdx>{}());
auto& _ofunc = _info.get_table_func(_saved);
_ofunc = _func;
}
auto& _copy_table = _info.get_table(hsa_table_lookup<TableIdx>{}());
auto& _copy_func = _info.get_table_func(_copy_table);
(void) _orig;
LOG_IF(FATAL, _copy_func && _tbl_instance == 0)
<< _info.name << " has non-null function pointer " << _copy_func
<< " despite this being the first instance of the library being copies";
if(!_copy_func)
{
LOG(INFO) << "copying table entry for " << _info.name;
_copy_func = _orig_func;
}
else
{
LOG(INFO) << "skipping copying table entry for " << _info.name
<< " from table instance " << _tbl_instance;
}
}
}
template <size_t TableIdx, typename Tp, size_t OpIdx>
@@ -602,17 +615,15 @@ update_table(const context::context_array_t& _contexts,
auto& _func = _info.get_table_func(_table);
_func = _info.get_functor(_func);
}
(void) _orig;
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
void
copy_table(Tp* _orig, std::index_sequence<OpIdx, OpIdxTail...>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::index_sequence<OpIdx, OpIdxTail...>)
{
copy_table<TableIdx>(_orig, std::integral_constant<size_t, OpIdx>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::integral_constant<size_t, OpIdx>{});
if constexpr(sizeof...(OpIdxTail) > 0)
copy_table<TableIdx>(_orig, std::index_sequence<OpIdxTail...>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::index_sequence<OpIdxTail...>{});
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
@@ -681,11 +692,12 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig)
copy_table(TableT* _orig, uint64_t _tbl_instance)
{
constexpr auto TableIdx = hsa_table_id_lookup<TableT>::value;
if(_orig)
copy_table<TableIdx>(_orig, std::make_index_sequence<hsa_domain_info<TableIdx>::last>{});
copy_table<TableIdx>(
_orig, _tbl_instance, std::make_index_sequence<hsa_domain_info<TableIdx>::last>{});
}
template <typename TableT>
@@ -704,8 +716,8 @@ update_table(TableT* _orig)
using iterate_args_data_t = rocprofiler_callback_tracing_hsa_api_data_t;
using iterate_args_cb_t = rocprofiler_callback_tracing_operation_args_cb_t;
#define INSTANTIATE_MARKER_TABLE_FUNC(TABLE_TYPE, TABLE_IDX) \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
#define INSTANTIATE_HSA_TABLE_FUNC(TABLE_TYPE, TABLE_IDX) \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl, uint64_t _instv); \
template void update_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
template const char* name_by_id<TABLE_IDX>(uint32_t); \
template uint32_t id_by_name<TABLE_IDX>(const char*); \
@@ -714,11 +726,11 @@ using iterate_args_cb_t = rocprofiler_callback_tracing_operation_args_cb_t;
template void iterate_args<TABLE_IDX>( \
uint32_t, const iterate_args_data_t&, iterate_args_cb_t, void*);
INSTANTIATE_MARKER_TABLE_FUNC(hsa_core_table_t, ROCPROFILER_HSA_TABLE_ID_Core)
INSTANTIATE_MARKER_TABLE_FUNC(hsa_amd_ext_table_t, ROCPROFILER_HSA_TABLE_ID_AmdExt)
INSTANTIATE_MARKER_TABLE_FUNC(hsa_img_ext_table_t, ROCPROFILER_HSA_TABLE_ID_ImageExt)
INSTANTIATE_MARKER_TABLE_FUNC(hsa_fini_ext_table_t, ROCPROFILER_HSA_TABLE_ID_FinalizeExt)
INSTANTIATE_HSA_TABLE_FUNC(hsa_core_table_t, ROCPROFILER_HSA_TABLE_ID_Core)
INSTANTIATE_HSA_TABLE_FUNC(hsa_amd_ext_table_t, ROCPROFILER_HSA_TABLE_ID_AmdExt)
INSTANTIATE_HSA_TABLE_FUNC(hsa_img_ext_table_t, ROCPROFILER_HSA_TABLE_ID_ImageExt)
INSTANTIATE_HSA_TABLE_FUNC(hsa_fini_ext_table_t, ROCPROFILER_HSA_TABLE_ID_FinalizeExt)
#undef INSTANTIATE_MARKER_TABLE_FUNC
#undef INSTANTIATE_HSA_TABLE_FUNC
} // namespace hsa
} // namespace rocprofiler
source/lib/rocprofiler-sdk/hsa/hsa.hpp
+1 -1
Visa fil
@@ -125,7 +125,7 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig);
copy_table(TableT* _orig, uint64_t _tbl_instance);
template <typename TableT>
void
source/lib/rocprofiler-sdk/hsa/queue.cpp
+8 -13
Visa fil
@@ -181,10 +181,10 @@ AsyncSignalHandler(hsa_signal_value_t /*signal_v*/, void* data)
if(_corr_id)
{
LOG_IF(FATAL, _corr_id->ref_count.load() == 0)
LOG_IF(FATAL, _corr_id->get_ref_count() == 0)
<< "reference counter for correlation id " << _corr_id->internal << " from thread "
<< _corr_id->thread_idx << " has no reference count";
_corr_id->ref_count.fetch_sub(1);
_corr_id->sub_ref_count();
}
queue_info_session.queue.async_complete();
@@ -257,24 +257,17 @@ WriteInterceptor(const void* packets,
auto* corr_id = context::get_latest_correlation_id();
// use thread-local value to reuse allocation
static thread_local auto extern_corr_ids_tl =
Queue::queue_info_session_t::external_corr_id_map_t{};
auto extern_corr_ids = Queue::queue_info_session_t::external_corr_id_map_t{};
// increase the reference count to denote that this correlation id is being used in a kernel
if(corr_id)
{
extern_corr_ids_tl.clear(); // clear it so that it only contains the current contexts
extern_corr_ids_tl.reserve(ctxs.size()); // reserve for performance
extern_corr_ids.clear(); // clear it so that it only contains the current contexts
extern_corr_ids.reserve(ctxs.size()); // reserve for performance
for(const auto* ctx : ctxs)
extern_corr_ids_tl.emplace(ctx,
ctx->correlation_tracer.external_correlator.get(thr_id));
corr_id->ref_count.fetch_add(1);
extern_corr_ids.emplace(ctx, ctx->correlation_tracer.external_correlator.get(thr_id));
}
// move to local variable
auto extern_corr_ids = std::move(extern_corr_ids_tl);
// hsa_ext_amd_aql_pm4_packet_t
const auto* packets_arr = static_cast<const rocprofiler_packet*>(packets);
auto transformed_packets = std::vector<rocprofiler_packet>{};
@@ -375,6 +368,8 @@ WriteInterceptor(const void* packets,
LOG_IF(FATAL, packet_type != HSA_PACKET_TYPE_KERNEL_DISPATCH)
<< "get_kernel_id below might need to be updated";
if(corr_id) corr_id->add_ref_count();
// Enqueue the signal into the handler. Will call completed_cb when
// signal completes.
queue.async_started();
source/lib/rocprofiler-sdk/marker/marker.cpp
+31 -22
Visa fil
@@ -191,7 +191,6 @@ roctx_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
auto thr_id = common::get_tid();
auto callback_contexts = std::vector<callback_context_data>{};
auto buffered_contexts = std::vector<buffered_context_data>{};
auto has_pc_sampling = false;
populate_contexts(info_type::callback_domain_idx,
info_type::buffered_domain_idx,
@@ -208,7 +207,7 @@ roctx_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
return 0;
}
auto ref_count = (has_pc_sampling) ? 4 : 2;
auto ref_count = 2;
auto buffer_record = common::init_public_api_struct(buffer_marker_api_record_t{});
auto tracer_data = callback_api_data_t{.size = sizeof(callback_api_data_t)};
auto* corr_id = correlation_service::construct(ref_count);
@@ -274,7 +273,7 @@ roctx_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count before invoking
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
auto _ret = exec(info_type::get_table_func(), std::forward<Args>(args)...);
@@ -326,7 +325,7 @@ roctx_api_impl<TableIdx, OpIdx>::functor(Args&&... args)
}
// decrement the reference count after usage in the callback/buffers
corr_id->ref_count.fetch_sub(1);
corr_id->sub_ref_count();
context::pop_latest_correlation_id(corr_id);
@@ -420,6 +419,7 @@ iterate_args(const uint32_t id,
user_data);
if(ret != 0) break;
}
return;
}
if constexpr(sizeof...(OpIdxTail) > 0)
iterate_args<TableIdx>(id, data, func, user_data, std::index_sequence<OpIdxTail...>{});
@@ -451,7 +451,7 @@ should_wrap_functor(rocprofiler_callback_tracing_kind_t _callback_domain,
template <size_t TableIdx, typename Tp, size_t OpIdx>
void
copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::integral_constant<size_t, OpIdx>)
{
using table_type = typename roctx_table_lookup<TableIdx>::type;
@@ -459,24 +459,34 @@ copy_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
{
auto _info = roctx_api_info<TableIdx, OpIdx>{};
LOG(INFO) << "copying table entry for " << _info.name;
// make sure we don't access a field that doesn't exist in input table
if(_info.offset() >= _orig->size) return;
// 1. get the sub-table containing the function pointer in original table
// 2. get reference to function pointer in sub-table in original table
auto& _table = _info.get_table(_orig);
auto& _func = _info.get_table_func(_table);
auto& _orig_table = _info.get_table(_orig);
auto& _orig_func = _info.get_table_func(_orig_table);
// 3. get the sub-table containing the function pointer in saved table
// 4. get reference to function pointer in sub-table in saved table
// 5. save the original function in the saved table
auto& _saved = _info.get_table(*get_table<TableIdx>());
auto& _ofunc = _info.get_table_func(_saved);
_ofunc = _func;
}
auto& _copy_table = _info.get_table(*get_table<TableIdx>());
auto& _copy_func = _info.get_table_func(_copy_table);
(void) _orig;
LOG_IF(FATAL, _copy_func && _tbl_instance == 0)
<< _info.name << " has non-null function pointer " << _copy_func
<< " despite this being the first instance of the library being copies";
if(!_copy_func)
{
LOG(INFO) << "copying table entry for " << _info.name;
_copy_func = _orig_func;
}
else
{
LOG(INFO) << "skipping copying table entry for " << _info.name
<< " from table instance " << _tbl_instance;
}
}
}
template <size_t TableIdx, typename Tp, size_t OpIdx>
@@ -507,17 +517,15 @@ update_table(Tp* _orig, std::integral_constant<size_t, OpIdx>)
auto& _func = _info.get_table_func(_table);
_func = _info.get_functor(_func);
}
(void) _orig;
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
void
copy_table(Tp* _orig, std::index_sequence<OpIdx, OpIdxTail...>)
copy_table(Tp* _orig, uint64_t _tbl_instance, std::index_sequence<OpIdx, OpIdxTail...>)
{
copy_table<TableIdx>(_orig, std::integral_constant<size_t, OpIdx>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::integral_constant<size_t, OpIdx>{});
if constexpr(sizeof...(OpIdxTail) > 0)
copy_table<TableIdx>(_orig, std::index_sequence<OpIdxTail...>{});
copy_table<TableIdx>(_orig, _tbl_instance, std::index_sequence<OpIdxTail...>{});
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>
@@ -585,11 +593,12 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig)
copy_table(TableT* _orig, uint64_t _tbl_instance)
{
constexpr auto TableIdx = roctx_table_id_lookup<TableT>::value;
if(_orig)
copy_table<TableIdx>(_orig, std::make_index_sequence<roctx_domain_info<TableIdx>::last>{});
copy_table<TableIdx>(
_orig, _tbl_instance, std::make_index_sequence<roctx_domain_info<TableIdx>::last>{});
}
template <typename TableT>
@@ -606,7 +615,7 @@ using iterate_args_data_t = rocprofiler_callback_tracing_marker_api_data_t;
using iterate_args_cb_t = rocprofiler_callback_tracing_operation_args_cb_t;
#define INSTANTIATE_MARKER_TABLE_FUNC(TABLE_TYPE, TABLE_IDX) \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
template void copy_table<TABLE_TYPE>(TABLE_TYPE * _tbl, uint64_t _instv); \
template void update_table<TABLE_TYPE>(TABLE_TYPE * _tbl); \
template const char* name_by_id<TABLE_IDX>(uint32_t); \
template uint32_t id_by_name<TABLE_IDX>(const char*); \
source/lib/rocprofiler-sdk/marker/marker.hpp
+1 -1
Visa fil
@@ -91,7 +91,7 @@ iterate_args(uint32_t id,
template <typename TableT>
void
copy_table(TableT* _orig);
copy_table(TableT* _orig, uint64_t _tbl_instance);
template <typename TableT>
void
source/lib/rocprofiler-sdk/registration.cpp
+10 -10
Visa fil
@@ -652,7 +652,7 @@ rocprofiler_set_api_table(const char* name,
// any internal modifications to the HipDispatchTable need to be done before we make the
// copy or else those modifications will be lost when HIP API tracing is enabled
// because the HIP API tracing invokes the function pointers from the copy below
rocprofiler::hip::copy_table(hip_runtime_api_table);
rocprofiler::hip::copy_table(hip_runtime_api_table, lib_instance);
// install rocprofiler API wrappers
rocprofiler::hip::update_table(hip_runtime_api_table);
@@ -675,7 +675,7 @@ rocprofiler_set_api_table(const char* name,
// any internal modifications to the HipCompilerDispatchTable need to be done before we make
// the copy or else those modifications will be lost when HIP API tracing is enabled because
// the HIP API tracing invokes the function pointers from the copy below
rocprofiler::hip::copy_table(hip_compiler_api_table);
rocprofiler::hip::copy_table(hip_compiler_api_table, lib_instance);
// install rocprofiler API wrappers
rocprofiler::hip::update_table(hip_compiler_api_table);
@@ -703,15 +703,15 @@ rocprofiler_set_api_table(const char* name,
rocprofiler::agent::construct_agent_cache(hsa_api_table);
rocprofiler::hsa::queue_controller_init(hsa_api_table);
rocprofiler::hsa::code_object_init(hsa_api_table);
rocprofiler::hsa::async_copy_init(hsa_api_table);
rocprofiler::hsa::async_copy_init(hsa_api_table, lib_instance);
// any internal modifications to the HsaApiTable need to be done before we make the
// copy or else those modifications will be lost when HSA API tracing is enabled
// because the HSA API tracing invokes the function pointers from the copy below
rocprofiler::hsa::copy_table(hsa_api_table->core_);
rocprofiler::hsa::copy_table(hsa_api_table->amd_ext_);
rocprofiler::hsa::copy_table(hsa_api_table->image_ext_);
rocprofiler::hsa::copy_table(hsa_api_table->finalizer_ext_);
rocprofiler::hsa::copy_table(hsa_api_table->core_, lib_instance);
rocprofiler::hsa::copy_table(hsa_api_table->amd_ext_, lib_instance);
rocprofiler::hsa::copy_table(hsa_api_table->image_ext_, lib_instance);
rocprofiler::hsa::copy_table(hsa_api_table->finalizer_ext_, lib_instance);
// install rocprofiler API wrappers
rocprofiler::hsa::update_table(hsa_api_table->core_);
@@ -736,9 +736,9 @@ rocprofiler_set_api_table(const char* name,
// any internal modifications to the roctxApiTable_t need to be done before we make
// the copy or else those modifications will be lost when ROCTx tracing is enabled because
// the ROCTx tracing invokes the function pointers from the copy below
rocprofiler::marker::copy_table(roctx_core);
rocprofiler::marker::copy_table(roctx_ctrl);
rocprofiler::marker::copy_table(roctx_name);
rocprofiler::marker::copy_table(roctx_core, lib_instance);
rocprofiler::marker::copy_table(roctx_ctrl, lib_instance);
rocprofiler::marker::copy_table(roctx_name, lib_instance);
// install rocprofiler API wrappers
rocprofiler::marker::update_table(roctx_core);