AI/rocm-systems
2
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

SWDEV-492625 memory free functions (#11)

Ver Código-Fonte 
* SWDEV-492625: Track free memory HSA functions to help determine total amount of memory allocated on the system at any one time

* Minor fixes to address comments

* Update allocation size description

* Moved get function back to specialization, minor typo fixes

* Removed memory_operation_type field, removed memory_pool allocation enum, converted starting address to hex string for json format.

* Made conversion to hex_string a function, changed address to use union rocprofiler_address_t type, changed VMEM descriptors

* Removed as_hex from the global namespace

* Formatting

* Removed TRACK_EVENT for memory allocation, now TRACK_COUNTER for memory allocation is being performed

* Check if address was recorded before retrieving allocation size in generate Perfetto

* Formatting

* Update source/lib/output/generatePerfetto.cpp

* Explicitly disable app-abort tests

* Remove excluding app-abort test from workflow CI

- redundant bc these tests are explicitly marked as disabled now

---------

Co-authored-by: Madsen, Jonathan <Jonathan.Madsen@amd.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>

[ROCm/rocprofiler-sdk commit: 79006bb896]
Esse commit está contido em:
Trowbridge, Ian
2024-12-06 00:05:30 -06:00
commit de GitHub
pai a79f8a0198
commit 792329fefd
23 arquivos alterados com 564 adições e 248 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
projects/rocprofiler-sdk/source/lib/output/generateCSV.cpp
+13 -3
Ver Arquivo
@@ -33,6 +33,7 @@
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/marker/api_id.h>
#include <rocprofiler-sdk/cxx/operators.hpp>
#include <rocprofiler-sdk/cxx/utility.hpp>
#include <unistd.h>
#include <cstdint>
@@ -456,7 +457,7 @@ generate_csv(const output_config&
"Operation",
"Agent_Id",
"Allocation_Size",
"Starting_Address",
"Address",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
@@ -464,15 +465,24 @@ generate_csv(const output_config&
{
for(auto record : data.get(ditr))
{
uint64_t agent_info{0};
// Free functions currently do not track agent information. Only set it on allocation
// operations, otherwise set it to 0 currently
if(record.operation == ROCPROFILER_MEMORY_ALLOCATION_ALLOCATE ||
record.operation == ROCPROFILER_MEMORY_ALLOCATION_VMEM_ALLOCATE)
{
agent_info = tool_metadata.get_node_id(record.agent_id);
}
auto api_name = tool_metadata.get_operation_name(record.kind, record.operation);
auto row_ss = std::stringstream{};
rocprofiler::tool::csv::memory_allocation_csv_encoder::write_row(
row_ss,
tool_metadata.get_kind_name(record.kind),
api_name,
tool_metadata.get_node_id(record.agent_id),
agent_info,
record.allocation_size,
record.starting_address,
rocprofiler::sdk::utility::as_hex(record.address.value, 16),
record.correlation_id.internal,
record.start_timestamp,
record.end_timestamp);
projects/rocprofiler-sdk/source/lib/output/generateOTF2.cpp
+21 -7
Ver Arquivo
@@ -486,17 +486,25 @@ write_otf2(
{
for(auto& [agent, evt] : itr)
{
const auto* _agent = _get_agent(agent);
auto _type_name = std::string_view{"UNK"};
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
// Free functions do not track agent information. Below handles case where
// null rocprof agent id is passed to generate OTF2
constexpr auto null_rocp_agent_id =
rocprofiler_agent_id_t{.handle = std::numeric_limits<uint64_t>::max()};
const rocprofiler_agent_t* _agent = nullptr;
if(agent != null_rocp_agent_id)
{
_agent = _get_agent(agent);
}
auto _type_name = std::string_view{"UNK"};
if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_type_name = "CPU";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
else if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_type_name = "GPU";
evt.name = fmt::format("Thread {}, Memory Allocation at {} {}",
evt.name = fmt::format("Thread {}, Memory Operation at {} {}",
tid,
_type_name,
_agent->logical_node_type_id);
_agent == nullptr ? 0 : _agent->logical_node_type_id);
}
}
@@ -860,6 +868,12 @@ write_otf2(
for(auto& [agent, evt] : itr)
{
auto _hash = get_hash_id(evt.name);
// Using max numeric limits results in an out-of-bound runtime error for OTF2
// and perfetto for agent ids. Setting handle to 0 for free functions.
constexpr auto null_rocp_agent_id =
rocprofiler_agent_id_t{.handle = std::numeric_limits<uint64_t>::max()};
auto handle = agent.handle;
if(agent == null_rocp_agent_id) handle = 0;
add_write_string(_hash, evt.name);
OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocation(global_def_writer,
@@ -867,7 +881,7 @@ write_otf2(
_hash,
OTF2_LOCATION_TYPE_ACCELERATOR_STREAM,
2 * evt.event_count, // # events
agent.handle // location group
handle // location group
));
}
}
projects/rocprofiler-sdk/source/lib/output/generatePerfetto.cpp
+93 -68
Ver Arquivo
@@ -241,33 +241,6 @@ write_perfetto(
}
}
for(const auto& itr : agent_thread_ids_alloc)
{
const auto* _agent = _get_agent(itr.first);
for(auto titr : itr.second)
{
auto _namess = std::stringstream{};
_namess << "MEMORY ALLOCATION on AGENT [" << _agent->logical_node_id << "] THREAD ["
<< thread_indexes.at(titr) << "] ";
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_namess << "(CPU)";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_namess << "(GPU)";
else
_namess << "(UNK)";
auto _track = ::perfetto::Track{get_hash_id(_namess.str())};
auto _desc = _track.Serialize();
_desc.set_name(_namess.str());
perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
agent_thread_tracks_alloc[itr.first].emplace(titr, _track);
}
}
for(const auto& aitr : agent_queue_ids)
{
uint32_t nqueue = 0;
@@ -463,47 +436,6 @@ write_perfetto(
tracing_session->FlushBlocking();
}
for(auto ditr : memory_allocation_gen)
for(auto itr : memory_allocation_gen.get(ditr))
{
auto name = buffer_names.at(itr.kind, itr.operation);
auto& track = agent_thread_tracks_alloc.at(itr.agent_id).at(itr.thread_id);
std::stringstream hex_stream;
hex_stream << "0x" << std::hex << std::setw(16) << std::setfill('0')
<< itr.starting_address;
std::string hex_starting_address(hex_stream.str());
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::memory_allocation>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"end_ns",
itr.end_timestamp,
"delta_ns",
(itr.end_timestamp - itr.start_timestamp),
"kind",
itr.kind,
"operation",
itr.operation,
"agent",
agents_map.at(itr.agent_id).logical_node_id,
"allocation_size",
itr.allocation_size,
"starting_address",
hex_starting_address,
"corr_id",
itr.correlation_id.internal,
"tid",
itr.thread_id);
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::memory_allocation>::name,
track,
itr.end_timestamp);
tracing_session->FlushBlocking();
}
for(auto ditr : kernel_dispatch_gen)
for(auto itr : kernel_dispatch_gen.get(ditr))
{
@@ -634,6 +566,99 @@ write_perfetto(
tracing_session->FlushBlocking();
}
}
// memory allocation counter track
auto mem_alloc_endpoints = std::map<rocprofiler_agent_id_t, std::map<uint64_t, uint64_t>>{};
auto mem_alloc_extremes = std::pair<uint64_t, uint64_t>{};
auto address_to_size = std::unordered_map<uint64_t, uint64_t>{};
for(auto ditr : memory_allocation_gen)
for(auto itr : memory_allocation_gen.get(ditr))
{
uint64_t _mean_timestamp =
itr.start_timestamp + (0.5 * (itr.end_timestamp - itr.start_timestamp));
mem_alloc_endpoints[itr.agent_id].emplace(itr.start_timestamp - 1000, 0);
mem_alloc_endpoints[itr.agent_id].emplace(itr.start_timestamp, 0);
mem_alloc_endpoints[itr.agent_id].emplace(_mean_timestamp, 0);
mem_alloc_endpoints[itr.agent_id].emplace(itr.end_timestamp, 0);
mem_alloc_endpoints[itr.agent_id].emplace(itr.end_timestamp + 1000, 0);
mem_alloc_extremes =
std::make_pair(std::min(mem_alloc_extremes.first, itr.start_timestamp),
std::max(mem_alloc_extremes.second, itr.end_timestamp));
if(itr.operation == ROCPROFILER_MEMORY_ALLOCATION_ALLOCATE ||
itr.operation == ROCPROFILER_MEMORY_ALLOCATION_VMEM_ALLOCATE)
{
address_to_size.emplace(itr.address.value, itr.allocation_size);
}
}
for(auto ditr : memory_allocation_gen)
for(auto itr : memory_allocation_gen.get(ditr))
{
auto alloc_beg =
mem_alloc_endpoints.at(itr.agent_id).lower_bound(itr.start_timestamp);
auto alloc_end =
mem_alloc_endpoints.at(itr.agent_id).upper_bound(itr.end_timestamp);
LOG_IF(FATAL, alloc_beg == alloc_end)
<< "Missing range for timestamp [" << itr.start_timestamp << ", "
<< itr.end_timestamp << "]";
for(auto alloc_itr = alloc_beg; alloc_itr != alloc_end; ++alloc_itr)
{
if(address_to_size.count(itr.address.value) > 0)
{
alloc_itr->second += address_to_size.at(itr.address.value);
}
}
}
auto mem_alloc_tracks =
std::unordered_map<rocprofiler_agent_id_t, ::perfetto::CounterTrack>{};
auto mem_alloc_cnt_names = std::vector<std::string>{};
constexpr auto null_rocp_agent_id =
rocprofiler_agent_id_t{.handle = std::numeric_limits<uint64_t>::max()};
mem_alloc_cnt_names.reserve(mem_alloc_endpoints.size());
for(auto& alloc_itr : mem_alloc_endpoints)
{
mem_alloc_endpoints[alloc_itr.first].emplace(mem_alloc_extremes.first - 5000, 0);
mem_alloc_endpoints[alloc_itr.first].emplace(mem_alloc_extremes.second + 5000, 0);
auto _track_name = std::stringstream{};
const rocprofiler_agent_t* _agent = nullptr;
if(alloc_itr.first != null_rocp_agent_id)
{
_agent = _get_agent(alloc_itr.first);
}
if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_track_name << "ALLOCATE BYTES on AGENT [" << _agent->logical_node_id << "] (CPU)";
else if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_track_name << "ALLOCATE BYTES on AGENT [" << _agent->logical_node_id << "] (GPU)";
else
_track_name << "FREE BYTES";
constexpr auto _unit = ::perfetto::CounterTrack::Unit::UNIT_SIZE_BYTES;
auto& _name = mem_alloc_cnt_names.emplace_back(_track_name.str());
mem_alloc_tracks.emplace(alloc_itr.first,
::perfetto::CounterTrack{_name.c_str()}
.set_unit(_unit)
.set_unit_multiplier(bytes_multiplier)
.set_is_incremental(false));
}
for(auto& alloc_itr : mem_alloc_endpoints)
{
for(auto itr : alloc_itr.second)
{
TRACE_COUNTER(sdk::perfetto_category<sdk::category::memory_allocation>::name,
mem_alloc_tracks.at(alloc_itr.first),
itr.first,
itr.second / bytes_multiplier);
tracing_session->FlushBlocking();
}
}
}
::perfetto::TrackEvent::Flush();
projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/hsa.def.cpp
+3
Ver Arquivo
@@ -481,6 +481,9 @@ HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_AmdExt, ROCPROFILER_HSA_AMD_EXT
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_Core, ROCPROFILER_HSA_CORE_API_ID_hsa_memory_allocate, hsa_memory_allocate, hsa_memory_allocate_fn)
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_AmdExt, ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_memory_pool_allocate, hsa_amd_memory_pool_allocate, hsa_amd_memory_pool_allocate_fn)
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_AmdExt, ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_vmem_handle_create, hsa_amd_vmem_handle_create, hsa_amd_vmem_handle_create_fn)
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_Core, ROCPROFILER_HSA_CORE_API_ID_hsa_memory_free, hsa_memory_free, hsa_memory_free_fn)
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_AmdExt, ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_memory_pool_free, hsa_amd_memory_pool_free, hsa_amd_memory_pool_free_fn)
HSA_API_META_DEFINITION(ROCPROFILER_HSA_TABLE_ID_AmdExt, ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_vmem_handle_release, hsa_amd_vmem_handle_release, hsa_amd_vmem_handle_release_fn)
// clang-format on
#else
projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/memory_allocation.cpp
+260 -50
Ver Arquivo
@@ -68,49 +68,125 @@ using memory_pool_to_agent_map = std::unordered_map<hsa_amd_memory_pool_t, rocpr
using region_to_agent_pair = std::pair<region_to_agent_map*, rocprofiler_agent_id_t>;
using map_pool_to_agent_pair = std::pair<memory_pool_to_agent_map*, rocprofiler_agent_id_t>;
template <size_t TableIdx, size_t OpIdx, typename... Args>
hsa_status_t
memory_allocation_impl(Args... args);
template <size_t TableIdx, size_t OpIdx, typename... Args>
hsa_status_t
memory_free_impl(Args... args);
// Local enum to specify implementation of memory function wrappers
typedef enum
{
HSA_NONE = 0, ///< Unknown memory allocation function
HSA_MEMORY_ALLOCATE, ///< Allocate memory function
HSA_AMD_MEMORY_POOL_ALLOCATE, ///< Allocate memory pool
HSA_AMD_VMEM_ALLOCATE, ///< Allocate vmem memory handle
HSA_MEMORY_FREE, ///< Free memory function
HSA_AMD_MEMORY_POOL_FREE, ///< Free memory pool
HSA_AMD_VMEM_FREE, ///< Release vmem memory handle
HSA_LAST,
} hsa_memory_operation_functions_t;
// Set up information to identify agent from regions/pool
template <size_t OpIdx>
struct memory_allocation_info;
#define SPECIALIZE_MEMORY_ALLOCATION_INFO(FUNCTION, MAPTYPE, PAIRTYPE, SEARCHTYPE, ITERATEFUNC) \
#define SPECIALIZE_MEMORY_ALLOCATION_INFO( \
FUNCTION, ENUM, MAPTYPE, PAIRTYPE, SEARCHTYPE, ITERATEFUNC, IMPLEMENTATION) \
template <> \
struct memory_allocation_info<ROCPROFILER_MEMORY_ALLOCATION_##FUNCTION> \
struct memory_allocation_info<FUNCTION> \
{ \
using maptype = MAPTYPE; \
using pairtype = PAIRTYPE; \
using searchtype = SEARCHTYPE; \
auto& operator()() const { return ITERATEFUNC; } \
static constexpr auto operation_idx = ROCPROFILER_MEMORY_ALLOCATION_##FUNCTION; \
static constexpr auto name = "MEMORY_ALLOCATION_" #FUNCTION; \
static constexpr auto operation_idx = ROCPROFILER_MEMORY_ALLOCATION_##ENUM; \
static constexpr auto name = "MEMORY_ALLOCATION_" #ENUM; \
\
template <size_t TableIdx, size_t OpIdx, typename RetT, typename... Args> \
static auto get_memory_allocation_impl(RetT (*)(Args...)) \
{ \
return &IMPLEMENTATION<TableIdx, OpIdx, Args...>; \
} \
};
SPECIALIZE_MEMORY_ALLOCATION_INFO(NONE,
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_NONE,
NONE,
region_to_agent_map,
region_to_agent_pair,
hsa_region_t,
get_core_table()->hsa_agent_iterate_regions_fn)
SPECIALIZE_MEMORY_ALLOCATION_INFO(ALLOCATE,
get_core_table()->hsa_agent_iterate_regions_fn,
memory_allocation_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_MEMORY_ALLOCATE,
ALLOCATE,
region_to_agent_map,
region_to_agent_pair,
hsa_region_t,
get_core_table()->hsa_agent_iterate_regions_fn)
SPECIALIZE_MEMORY_ALLOCATION_INFO(MEMORY_POOL_ALLOCATE,
get_core_table()->hsa_agent_iterate_regions_fn,
memory_allocation_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_AMD_MEMORY_POOL_ALLOCATE,
ALLOCATE,
memory_pool_to_agent_map,
map_pool_to_agent_pair,
hsa_amd_memory_pool_t,
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn)
SPECIALIZE_MEMORY_ALLOCATION_INFO(VMEM_HANDLE_CREATE,
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn,
memory_allocation_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_AMD_VMEM_ALLOCATE,
VMEM_ALLOCATE,
memory_pool_to_agent_map,
map_pool_to_agent_pair,
hsa_amd_memory_pool_t,
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn)
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn,
memory_allocation_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_MEMORY_FREE,
FREE,
region_to_agent_map,
region_to_agent_pair,
hsa_region_t,
get_core_table()->hsa_agent_iterate_regions_fn,
memory_free_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_AMD_MEMORY_POOL_FREE,
FREE,
memory_pool_to_agent_map,
map_pool_to_agent_pair,
hsa_amd_memory_pool_t,
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn,
memory_free_impl)
SPECIALIZE_MEMORY_ALLOCATION_INFO(HSA_AMD_VMEM_FREE,
VMEM_FREE,
memory_pool_to_agent_map,
map_pool_to_agent_pair,
hsa_amd_memory_pool_t,
get_amd_ext_table()->hsa_amd_agent_iterate_memory_pools_fn,
memory_free_impl)
#undef SPECIALIZE_MEMORY_ALLOCATION_INFO
// Map rocprofiler_memory_allocation_operation_t to respective name
template <size_t OpIdx>
struct memory_allocation_name;
#define MEMORY_ALLOCATION_NAME(ENUM) \
template <> \
struct memory_allocation_name<ROCPROFILER_MEMORY_ALLOCATION_##ENUM> \
{ \
static constexpr auto name = "MEMORY_ALLOCATION_" #ENUM; \
static constexpr auto operation_idx = ROCPROFILER_MEMORY_ALLOCATION_##ENUM; \
};
MEMORY_ALLOCATION_NAME(NONE)
MEMORY_ALLOCATION_NAME(ALLOCATE)
MEMORY_ALLOCATION_NAME(VMEM_ALLOCATE)
MEMORY_ALLOCATION_NAME(FREE)
MEMORY_ALLOCATION_NAME(VMEM_FREE)
#undef MEMORY_ALLOCATION_NAME
template <size_t Idx, size_t... IdxTail>
const char*
name_by_id(const uint32_t id, std::index_sequence<Idx, IdxTail...>)
{
if(Idx == id) return memory_allocation_info<Idx>::name;
if(Idx == id) return memory_allocation_name<Idx>::name;
if constexpr(sizeof...(IdxTail) > 0)
return name_by_id(id, std::index_sequence<IdxTail...>{});
else
@@ -121,8 +197,8 @@ template <size_t Idx, size_t... IdxTail>
uint32_t
id_by_name(const char* name, std::index_sequence<Idx, IdxTail...>)
{
if(std::string_view{memory_allocation_info<Idx>::name} == std::string_view{name})
return memory_allocation_info<Idx>::operation_idx;
if(std::string_view{memory_allocation_name<Idx>::name} == std::string_view{name})
return memory_allocation_name<Idx>::operation_idx;
if constexpr(sizeof...(IdxTail) > 0)
return id_by_name(name, std::index_sequence<IdxTail...>{});
else
@@ -137,7 +213,7 @@ get_ids(std::vector<uint32_t>& _id_list, std::index_sequence<Idx...>)
if(_v < static_cast<uint32_t>(ROCPROFILER_MEMORY_ALLOCATION_LAST)) _vec.emplace_back(_v);
};
(_emplace(_id_list, memory_allocation_info<Idx>::operation_idx), ...);
(_emplace(_id_list, memory_allocation_name<Idx>::operation_idx), ...);
}
template <size_t... Idx>
@@ -148,7 +224,7 @@ get_names(std::vector<const char*>& _name_list, std::index_sequence<Idx...>)
if(_v != nullptr && strnlen(_v, 1) > 0) _vec.emplace_back(_v);
};
(_emplace(_name_list, memory_allocation_info<Idx>::name), ...);
(_emplace(_name_list, memory_allocation_name<Idx>::name), ...);
}
bool
@@ -169,17 +245,24 @@ context_filter(const context::context* ctx)
enum memory_allocation_core_id
{
memory_allocation_core_allocate_id = ROCPROFILER_HSA_CORE_API_ID_hsa_memory_allocate,
memory_allocation_core_free_id = ROCPROFILER_HSA_CORE_API_ID_hsa_memory_free,
};
using memory_allocation_core_index_seq_t = std::index_sequence<memory_allocation_core_allocate_id>;
using memory_allocation_core_index_seq_t =
std::index_sequence<memory_allocation_core_allocate_id, memory_allocation_core_free_id>;
enum memory_allocation_amd_ext_id
{
memory_allocation_amd_ext_allocate_id =
ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_memory_pool_allocate,
memory_allocation_vmem_allocate_id = ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_vmem_handle_create
memory_allocation_vmem_allocate_id = ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_vmem_handle_create,
memory_allocation_amd_ext_free_id = ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_memory_pool_free,
memory_allocation_vmem_release_id = ROCPROFILER_HSA_AMD_EXT_API_ID_hsa_amd_vmem_handle_release,
};
using memory_allocation_amd_ext_index_seq_t =
std::index_sequence<memory_allocation_amd_ext_allocate_id, memory_allocation_vmem_allocate_id>;
std::index_sequence<memory_allocation_amd_ext_allocate_id,
memory_allocation_vmem_allocate_id,
memory_allocation_amd_ext_free_id,
memory_allocation_vmem_release_id>;
template <size_t TableIdx>
struct memory_allocation_seq;
@@ -203,14 +286,17 @@ struct arg_indices;
template <> \
struct arg_indices<ENUM_ID> \
{ \
static constexpr auto starting_address_idx = STARTING_ADDRESS_IDX; \
static constexpr auto size_idx = SIZE_IDX; \
static constexpr auto region_idx = REGION_IDX; \
static constexpr auto address_idx = STARTING_ADDRESS_IDX; \
static constexpr auto size_idx = SIZE_IDX; \
static constexpr auto region_idx = REGION_IDX; \
};
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_core_allocate_id, 2, 1, 0)
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_amd_ext_allocate_id, 3, 1, 0)
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_vmem_allocate_id, 4, 1, 0)
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_core_free_id, 0, 0, 0)
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_amd_ext_free_id, 0, 0, 0)
HSA_MEMORY_ALLOCATE_DEFINE_ARG_INDICES(memory_allocation_vmem_release_id, 0, 0, 0)
// Define operation indices for each tracked functions
template <size_t Idx>
@@ -220,12 +306,15 @@ struct memory_allocation_op;
template <> \
struct memory_allocation_op<ENUM_ID> \
{ \
static constexpr auto operation_idx = ROCPROFILER_MEMORY_ALLOCATION_##FUNCTION; \
static constexpr auto operation_idx = FUNCTION; \
};
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_core_allocate_id, ALLOCATE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_amd_ext_allocate_id, MEMORY_POOL_ALLOCATE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_vmem_allocate_id, VMEM_HANDLE_CREATE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_core_allocate_id, HSA_MEMORY_ALLOCATE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_amd_ext_allocate_id, HSA_AMD_MEMORY_POOL_ALLOCATE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_vmem_allocate_id, HSA_AMD_VMEM_ALLOCATE)
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_core_free_id, HSA_MEMORY_FREE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_amd_ext_free_id, HSA_AMD_MEMORY_POOL_FREE);
MEMORY_ALLOCATE_OPERATION_IDX(memory_allocation_vmem_release_id, HSA_AMD_VMEM_FREE);
template <typename FuncT, typename ArgsT, size_t... Idx>
decltype(auto)
@@ -255,7 +344,7 @@ struct memory_allocation_data
rocprofiler_thread_id_t tid = common::get_tid();
rocprofiler_agent_id_t agent = null_rocp_agent_id;
uint64_t size_allocated = 0;
uint64_t starting_addr = 0;
rocprofiler_address_t address = {.value = 0};
uint64_t start_ts = 0;
context::correlation_id* correlation_id = nullptr;
tracing::tracing_data tracing_data = {};
@@ -271,7 +360,7 @@ memory_allocation_data::callback_data_t
memory_allocation_data::get_callback_data(timestamp_t _beg, timestamp_t _end) const
{
return common::init_public_api_struct(
callback_data_t{}, _beg, _end, agent, starting_addr, size_allocated);
callback_data_t{}, _beg, _end, agent, address, size_allocated);
}
memory_allocation_data::buffered_data_t
@@ -291,7 +380,7 @@ memory_allocation_data::get_buffered_record(const context_t* _ctx,
_beg,
_end,
agent,
starting_addr,
address,
size_allocated);
}
@@ -336,16 +425,32 @@ get_agent(T val, IterateFunc iterate_func, CallbackFunc callback)
return existing.count(val) == 0 ? null_rocp_agent_id : existing.at(val);
}
uint64_t
void*
handle_starting_addr(void** starting_addr_pointer)
{
return reinterpret_cast<uint64_t>(*starting_addr_pointer);
return *starting_addr_pointer;
}
uint64_t
// The handle field of hsa_amd_vmem_alloc_handle_t is the starting address
// cast as uint64_t, so returning the handle field after casting to void* suffices
void*
handle_starting_addr(hsa_amd_vmem_alloc_handle_t* vmem_alloc_handle)
{
return vmem_alloc_handle->handle;
return reinterpret_cast<void*>(vmem_alloc_handle->handle);
}
// Handling starting address for free memory operations
void*
handle_starting_addr(void* starting_addr_pointer)
{
return starting_addr_pointer;
}
// Handles starting address for releasing handle
void*
handle_starting_addr(hsa_amd_vmem_alloc_handle_t vmem_alloc_handle)
{
return reinterpret_cast<void*>(vmem_alloc_handle.handle);
}
// Wrapper implementation that stores memory allocation information
@@ -353,11 +458,12 @@ template <size_t TableIdx, size_t OpIdx, typename... Args>
hsa_status_t
memory_allocation_impl(Args... args)
{
constexpr auto N = sizeof...(Args);
constexpr auto starting_address_idx = arg_indices<OpIdx>::starting_address_idx;
constexpr auto size_idx = arg_indices<OpIdx>::size_idx;
constexpr auto region_idx = arg_indices<OpIdx>::region_idx;
constexpr auto operation = memory_allocation_op<OpIdx>::operation_idx;
constexpr auto N = sizeof...(Args);
constexpr auto address_idx = arg_indices<OpIdx>::address_idx;
constexpr auto size_idx = arg_indices<OpIdx>::size_idx;
constexpr auto region_idx = arg_indices<OpIdx>::region_idx;
constexpr auto operation = memory_allocation_op<OpIdx>::operation_idx;
constexpr auto rocprofiler_enum = memory_allocation_info<operation>::operation_idx;
auto&& _tied_args = std::tie(args...);
memory_allocation_data _data{};
@@ -380,7 +486,7 @@ memory_allocation_impl(Args... args)
}
auto& tracing_data = _data.tracing_data;
auto starting_addr_pointer = std::get<starting_address_idx>(_tied_args);
auto starting_addr_pointer = std::get<address_idx>(_tied_args);
auto region_or_pool = std::get<region_idx>(_tied_args);
_data.tid = common::get_tid();
@@ -389,7 +495,7 @@ memory_allocation_impl(Args... args)
memory_allocation_info<operation>{}(),
callback_populate_map<operation, typename memory_allocation_info<operation>::searchtype>);
_data.size_allocated = std::get<size_idx>(_tied_args);
_data.func = operation;
_data.func = rocprofiler_enum;
_data.correlation_id = context::get_latest_correlation_id();
if(!_data.correlation_id)
@@ -405,7 +511,7 @@ memory_allocation_impl(Args... args)
tracing_data.external_correlation_ids,
thr_id,
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_ALLOCATION,
operation,
rocprofiler_enum,
_data.correlation_id->internal);
if(!tracing_data.callback_contexts.empty())
@@ -417,7 +523,7 @@ memory_allocation_impl(Args... args)
_data.correlation_id->internal,
tracing_data.external_correlation_ids,
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
operation,
rocprofiler_enum,
_tracer_data);
// enter callback may update the external correlation id field
tracing::update_external_correlation_ids(
@@ -433,7 +539,7 @@ memory_allocation_impl(Args... args)
// checks before retrieving starting address?
if(starting_addr_pointer != nullptr)
{
_data.starting_addr = handle_starting_addr(starting_addr_pointer);
_data.address.ptr = handle_starting_addr(starting_addr_pointer);
}
if(!tracing_data.empty())
@@ -445,7 +551,7 @@ memory_allocation_impl(Args... args)
tracing::execute_phase_exit_callbacks(_data.tracing_data.callback_contexts,
_data.tracing_data.external_correlation_ids,
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
operation,
rocprofiler_enum,
_tracer_data);
}
@@ -458,7 +564,7 @@ memory_allocation_impl(Args... args)
_data.correlation_id->internal,
_data.tracing_data.external_correlation_ids,
ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION,
operation,
rocprofiler_enum,
record);
}
}
@@ -468,11 +574,113 @@ memory_allocation_impl(Args... args)
return _ret;
}
template <size_t TableIdx, size_t OpIdx, typename RetT, typename... Args>
auto get_memory_allocation_impl(RetT (*)(Args...))
// Wrapper implementation that stores memory free operation information
template <size_t TableIdx, size_t OpIdx, typename... Args>
hsa_status_t
memory_free_impl(Args... args)
{
return &memory_allocation_impl<TableIdx, OpIdx, Args...>;
constexpr auto N = sizeof...(Args);
constexpr auto address_idx = arg_indices<OpIdx>::address_idx;
constexpr auto operation = memory_allocation_op<OpIdx>::operation_idx;
constexpr auto rocprofiler_enum = memory_allocation_info<operation>::operation_idx;
auto&& _tied_args = std::tie(args...);
memory_allocation_data _data{};
{
auto tracing_data = tracing::tracing_data{};
tracing::populate_contexts(ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION,
OpIdx,
tracing_data);
// if no contexts are tracing memory copies for this direction, execute as usual
if(tracing_data.empty())
{
return invoke(get_next_dispatch<TableIdx, OpIdx>(),
std::move(_tied_args),
std::make_index_sequence<N>{});
}
_data.tracing_data = std::move(tracing_data);
}
auto& tracing_data = _data.tracing_data;
_data.tid = common::get_tid();
_data.func = rocprofiler_enum;
_data.correlation_id = context::get_latest_correlation_id();
_data.address.ptr = handle_starting_addr(std::get<address_idx>(_tied_args));
if(!_data.correlation_id)
{
constexpr auto ref_count = 1;
_data.correlation_id = context::correlation_tracing_service::construct(ref_count);
}
// increase the reference count to denote that this correlation id is being used in a kernel
_data.correlation_id->add_ref_count();
auto thr_id = _data.correlation_id->thread_idx;
tracing::populate_external_correlation_ids(
tracing_data.external_correlation_ids,
thr_id,
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_ALLOCATION,
rocprofiler_enum,
_data.correlation_id->internal);
if(!tracing_data.callback_contexts.empty())
{
auto _tracer_data = _data.get_callback_data();
tracing::execute_phase_enter_callbacks(tracing_data.callback_contexts,
thr_id,
_data.correlation_id->internal,
tracing_data.external_correlation_ids,
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
rocprofiler_enum,
_tracer_data);
// enter callback may update the external correlation id field
tracing::update_external_correlation_ids(
tracing_data.external_correlation_ids,
thr_id,
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_ALLOCATION);
}
auto start_ts = common::timestamp_ns();
auto _ret = invoke(
get_next_dispatch<TableIdx, OpIdx>(), std::move(_tied_args), std::make_index_sequence<N>{});
auto end_ts = common::timestamp_ns();
if(!tracing_data.empty())
{
if(!_data.tracing_data.callback_contexts.empty())
{
auto _tracer_data = _data.get_callback_data(start_ts, end_ts);
tracing::execute_phase_exit_callbacks(_data.tracing_data.callback_contexts,
_data.tracing_data.external_correlation_ids,
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
rocprofiler_enum,
_tracer_data);
}
if(!_data.tracing_data.buffered_contexts.empty())
{
auto record = _data.get_buffered_record(nullptr, start_ts, end_ts);
tracing::execute_buffer_record_emplace(_data.tracing_data.buffered_contexts,
_data.tid,
_data.correlation_id->internal,
_data.tracing_data.external_correlation_ids,
ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION,
rocprofiler_enum,
record);
}
}
// decrement the reference count after usage in the callback/buffers
_data.correlation_id->sub_ref_count();
return _ret;
}
} // namespace
// check out the assembly here... this compiles to a switch statement
const char*
@@ -573,7 +781,9 @@ memory_allocation_wrap(Tp* _orig, std::integral_constant<size_t, OpIdx>)
auto& _dispatch = get_next_dispatch<TableIdx, OpIdx>();
CHECK_NOTNULL(_dispatch);
_func = get_memory_allocation_impl<TableIdx, OpIdx>(_func);
constexpr auto LocalIdx = memory_allocation_op<OpIdx>::operation_idx;
_func = memory_allocation_info<LocalIdx>::template get_memory_allocation_impl<TableIdx, OpIdx>(
_func);
}
template <size_t TableIdx, typename Tp, size_t OpIdx, size_t... OpIdxTail>