AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Memory Allocation Tracking (#1142)

Browse Source 
* Initial commit: Need to implement wrapper function to collect data and test that wrapper function is correctly replacing core HSA functions

* Attempted to implement wrapper implementation for hsa memory allocation functions. Need to modify generate record files and test if implementation is working as expected

* Debugging and implementing generateCSV function

* Memory allocation size and starting address outputted to csv and json file formats

* Formatting

* Initial setup for OTF2 and Perfetto generation

* Collecting agent id for memory_allocation and formatting

* Modified memory_allocation.cpp to set up code for AMD_EXT commands

* Support for memory_pool_allocate added

* Removed accidently added file

* Made flag optional and added more OTF2 and Perfetto code. Needs testing to ensure perfetto and OTF2 works

* Formatting

* Fixed perfetto and otf2 output

* Fixed flag issue due to incorrect buffer use

* Updated documentation

* Small cleaning and comments

* Added test for HSA memory allocation tracing

* Fixed summary test validation errors due to allocation tracing. Added type to location_base to create unique event ids for allocation due to OTF2 trace error

* Decreased lower limit of hip calls for test

* Modified summary tests to vary number of allocate requests

* Minor fixes to address comments. Still need to address OTF2 comments

* Fix docs and changed OTF2 to use enum for type specified in location_base construction

* Fixed schema error

* Added vmem command tracking. Need to add test

* Updated test to work with vmem command and updated generateCSV to output int instead of hex string.

* OTF2 enum update and mispelling fix

* CI does not support Virtual Memory API. Removed vmem test. Will add back if CI is modifed to suport vmem API

* Update CMakeLists.txt for memory allocation test

* Updated summary test

* Minor fixes to address comments

* Moved domain_type.hpp enum to before LAST

* Fixed compile errors and formatting

* Fixed stats summary domain name error

* Added rocprofv3 test

* Page migration test fix

* Undo page migration test changes. Failures do not appear to have to do with memory allocation
This commit is contained in:
itrowbri
2024-11-18 20:22:14 -06:00
committed by GitHub
parent 0d764eb3c5
commit 3bd7773cf7
53 changed files with 2387 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/lib/output/buffered_output.hpp
+3
View File
@@ -153,5 +153,8 @@ using counter_collection_buffered_output_t =
using scratch_memory_buffered_output_t =
buffered_output<rocprofiler_buffer_tracing_scratch_memory_record_t,
domain_type::SCRATCH_MEMORY>;
using memory_allocation_buffered_output_t =
buffered_output<rocprofiler_buffer_tracing_memory_allocation_record_t,
domain_type::MEMORY_ALLOCATION>;
} // namespace tool
} // namespace rocprofiler
source/lib/output/csv.hpp
+1
View File
@@ -104,6 +104,7 @@ using agent_info_csv_encoder = csv_encoder<53>;
using kernel_trace_csv_encoder = csv_encoder<18>;
using counter_collection_csv_encoder = csv_encoder<18>;
using memory_copy_csv_encoder = csv_encoder<7>;
using memory_allocation_csv_encoder = csv_encoder<8>;
using marker_csv_encoder = csv_encoder<7>;
using list_basic_metrics_csv_encoder = csv_encoder<5>;
using list_derived_metrics_csv_encoder = csv_encoder<5>;
source/lib/output/domain_type.cpp
+4
View File
@@ -52,6 +52,10 @@ DEFINE_BUFFER_TYPE_NAME(COUNTER_COLLECTION,
"counter_collection",
"counter_collection_stats")
DEFINE_BUFFER_TYPE_NAME(RCCL, "RCCL_API", "rccl_api_trace", "rccl_api_stats")
DEFINE_BUFFER_TYPE_NAME(MEMORY_ALLOCATION,
"MEMORY_ALLOCATION",
"memory_allocation",
"memory_allocation_stats")
#undef DEFINE_BUFFER_TYPE_NAME
source/lib/output/domain_type.hpp
+1
View File
@@ -34,6 +34,7 @@ enum class domain_type
SCRATCH_MEMORY,
COUNTER_COLLECTION,
RCCL,
MEMORY_ALLOCATION,
LAST,
};
source/lib/output/generateCSV.cpp
+44
View File
@@ -437,6 +437,50 @@ generate_csv(const output_config& c
}
}
void
generate_csv(const output_config& cfg,
const metadata& tool_metadata,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& data,
const stats_entry_t& stats)
{
if(data.empty()) return;
if(cfg.stats && stats)
write_stats(get_stats_output_file(cfg, domain_type::MEMORY_ALLOCATION), stats.entries);
auto ofs = tool::csv_output_file{cfg,
domain_type::MEMORY_ALLOCATION,
tool::csv::memory_allocation_csv_encoder{},
{"Kind",
"Operation",
"Agent_Id",
"Allocation_Size",
"Starting_Address",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
for(auto ditr : data)
{
for(auto record : data.get(ditr))
{
auto api_name = tool_metadata.get_operation_name(record.kind, record.operation);
auto row_ss = std::stringstream{};
rocprofiler::tool::csv::memory_allocation_csv_encoder::write_row(
row_ss,
tool_metadata.get_kind_name(record.kind),
api_name,
tool_metadata.get_node_id(record.agent_id),
record.allocation_size,
record.starting_address,
record.correlation_id.internal,
record.start_timestamp,
record.end_timestamp);
ofs << row_ss.str();
}
}
}
void
generate_csv(const output_config& cfg,
const metadata& tool_metadata,
source/lib/output/generateCSV.hpp
+6
View File
@@ -87,6 +87,12 @@ generate_csv(const output_config& cfg,
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& data,
const stats_entry_t& stats);
void
generate_csv(const output_config& cfg,
const metadata& tool_metadata,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& data,
const stats_entry_t& stats);
void
generate_csv(const output_config& cfg,
const metadata& tool_metadata,
source/lib/output/generateJSON.cpp
+11 -9
View File
@@ -169,15 +169,16 @@ void
write_json(json_output& json_ar,
const output_config& /*cfg*/,
const metadata& /*tool_metadata*/,
const domain_stats_vec_t& domain_stats,
generator<rocprofiler_buffer_tracing_hip_api_record_t>&& hip_api_gen,
generator<rocprofiler_buffer_tracing_hsa_api_record_t> hsa_api_gen,
generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t> kernel_dispatch_gen,
generator<rocprofiler_buffer_tracing_memory_copy_record_t> memory_copy_gen,
generator<tool_counter_record_t> counter_collection_gen,
generator<rocprofiler_buffer_tracing_marker_api_record_t> marker_api_gen,
generator<rocprofiler_buffer_tracing_scratch_memory_record_t> scratch_memory_gen,
generator<rocprofiler_buffer_tracing_rccl_api_record_t> rccl_api_gen)
const domain_stats_vec_t& domain_stats,
generator<rocprofiler_buffer_tracing_hip_api_record_t>&& hip_api_gen,
generator<rocprofiler_buffer_tracing_hsa_api_record_t> hsa_api_gen,
generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t> kernel_dispatch_gen,
generator<rocprofiler_buffer_tracing_memory_copy_record_t> memory_copy_gen,
generator<tool_counter_record_t> counter_collection_gen,
generator<rocprofiler_buffer_tracing_marker_api_record_t> marker_api_gen,
generator<rocprofiler_buffer_tracing_scratch_memory_record_t> scratch_memory_gen,
generator<rocprofiler_buffer_tracing_rccl_api_record_t> rccl_api_gen,
generator<rocprofiler_buffer_tracing_memory_allocation_record_t> memory_allocation_gen)
{
// summary
@@ -216,6 +217,7 @@ write_json(json_output& json_ar,
json_ar(cereal::make_nvp("marker_api", marker_api_gen));
json_ar(cereal::make_nvp("rccl_api", rccl_api_gen));
json_ar(cereal::make_nvp("memory_copy", memory_copy_gen));
json_ar(cereal::make_nvp("memory_allocation", memory_allocation_gen));
json_ar(cereal::make_nvp("scratch_memory", scratch_memory_gen));
json_ar.finishNode();
}
source/lib/output/generateJSON.hpp
+13 -12
View File
@@ -81,17 +81,18 @@ void
write_json(json_output&, const output_config& cfg, const metadata& tool_metadata, uint64_t pid);
void
write_json(json_output& json_ar,
const output_config& cfg,
const metadata& tool_metadata,
const domain_stats_vec_t& domain_stats,
generator<rocprofiler_buffer_tracing_hip_api_record_t>&& hip_api_gen,
generator<rocprofiler_buffer_tracing_hsa_api_record_t> hsa_api_gen,
generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t> kernel_dispatch_gen,
generator<rocprofiler_buffer_tracing_memory_copy_record_t> memory_copy_gen,
generator<tool_counter_record_t> counter_collection_gen,
generator<rocprofiler_buffer_tracing_marker_api_record_t> marker_api_gen,
generator<rocprofiler_buffer_tracing_scratch_memory_record_t> scratch_memory_gen,
generator<rocprofiler_buffer_tracing_rccl_api_record_t> rccl_api_gen);
write_json(json_output& json_ar,
const output_config& cfg,
const metadata& tool_metadata,
const domain_stats_vec_t& domain_stats,
generator<rocprofiler_buffer_tracing_hip_api_record_t>&& hip_api_gen,
generator<rocprofiler_buffer_tracing_hsa_api_record_t> hsa_api_gen,
generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t> kernel_dispatch_gen,
generator<rocprofiler_buffer_tracing_memory_copy_record_t> memory_copy_gen,
generator<tool_counter_record_t> counter_collection_gen,
generator<rocprofiler_buffer_tracing_marker_api_record_t> marker_api_gen,
generator<rocprofiler_buffer_tracing_scratch_memory_record_t> scratch_memory_gen,
generator<rocprofiler_buffer_tracing_rccl_api_record_t> rccl_api_gen,
generator<rocprofiler_buffer_tracing_memory_allocation_record_t> memory_allocation_gen);
} // namespace tool
} // namespace rocprofiler
source/lib/output/generateOTF2.cpp
+123 -32
View File
@@ -127,43 +127,55 @@ auto main_tid = common::get_tid();
archive_t* archive = nullptr;
auto flush_callbacks = OTF2_FlushCallbacks{pre_flush, post_flush};
enum rocprofiler_location_type_t
{
ROCPROFILER_AGENT_NO_TYPE = 0,
ROCPROFILER_AGENT_MEMORY_COPY_TYPE,
ROCPROFILER_AGENT_DISPATCH_TYPE,
ROCPROFILER_AGENT_MEMORY_ALLOC_TYPE
};
struct location_base
{
uint64_t pid = 0;
rocprofiler_thread_id_t tid = 0;
rocprofiler_agent_id_t agent = {.handle = 0};
rocprofiler_queue_id_t queue = {.handle = 0};
uint64_t pid = 0;
rocprofiler_thread_id_t tid = 0;
rocprofiler_agent_id_t agent = {.handle = 0};
rocprofiler_queue_id_t queue = {.handle = 0};
rocprofiler_location_type_t type = ROCPROFILER_AGENT_NO_TYPE;
location_base(uint64_t _pid,
rocprofiler_thread_id_t _tid,
rocprofiler_agent_id_t _agent = {.handle = 0},
rocprofiler_queue_id_t _queue = {.handle = 0})
location_base(uint64_t _pid,
rocprofiler_thread_id_t _tid,
rocprofiler_agent_id_t _agent = {.handle = 0},
rocprofiler_location_type_t _type = ROCPROFILER_AGENT_NO_TYPE,
rocprofiler_queue_id_t _queue = {.handle = 0})
: pid{_pid}
, tid{_tid}
, agent{_agent}
, queue{_queue}
, type{_type}
{}
auto hash() const
{
return array_hash<uint64_t, 4>{}(pid, tid, agent.handle + 1, queue.handle + 1);
return array_hash<uint64_t, 5>{}(pid, tid, agent.handle + 1, queue.handle + 1, type);
}
};
bool
operator<(const location_base& lhs, const location_base& rhs)
{
return std::tie(lhs.pid, lhs.tid, lhs.agent.handle, lhs.queue.handle) <
std::tie(rhs.pid, rhs.tid, rhs.agent.handle, rhs.queue.handle);
return std::tie(lhs.pid, lhs.tid, lhs.agent.handle, lhs.queue.handle, lhs.type) <
std::tie(rhs.pid, rhs.tid, rhs.agent.handle, rhs.queue.handle, rhs.type);
}
struct location_data : location_base
{
location_data(uint64_t _pid,
rocprofiler_thread_id_t _tid,
rocprofiler_agent_id_t _agent = {.handle = 0},
rocprofiler_queue_id_t _queue = {.handle = 0})
: location_base{_pid, _tid, _agent, _queue}
location_data(uint64_t _pid,
rocprofiler_thread_id_t _tid,
rocprofiler_agent_id_t _agent = {.handle = 0},
rocprofiler_location_type_t _type = ROCPROFILER_AGENT_NO_TYPE,
rocprofiler_queue_id_t _queue = {.handle = 0})
: location_base{_pid, _tid, _agent, _type, _queue}
, index{++index_counter}
, event_writer{OTF2_Archive_GetEvtWriter(CHECK_NOTNULL(archive), index)}
{
@@ -206,7 +218,7 @@ get_location(const location_base& _location, bool _init = false)
if(_init)
return get_locations()
.emplace_back(std::make_unique<location_data>(
_location.pid, _location.tid, _location.agent, _location.queue))
_location.pid, _location.tid, _location.agent, _location.type, _location.queue))
.get();
return nullptr;
@@ -299,7 +311,7 @@ setup(const output_config& cfg)
OTF2_CHECK(OTF2_Pthread_Archive_SetLockingCallbacks(archive, nullptr));
OTF2_CHECK(OTF2_Archive_OpenEvtFiles(archive));
ROCP_ERROR << "Opened result file: " << _filename << ".oft2";
ROCP_ERROR << "Opened result file: " << _filename << ".otf2";
}
void
@@ -343,17 +355,19 @@ create_attribute_list()
} // namespace
void
write_otf2(const output_config& cfg,
const metadata& tool_metadata,
uint64_t pid,
const std::vector<agent_info>& agent_data,
std::deque<rocprofiler_buffer_tracing_hip_api_record_t>* hip_api_data,
std::deque<rocprofiler_buffer_tracing_hsa_api_record_t>* hsa_api_data,
std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>* kernel_dispatch_data,
std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>* memory_copy_data,
std::deque<rocprofiler_buffer_tracing_marker_api_record_t>* marker_api_data,
std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>* /*scratch_memory_data*/,
std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>* rccl_api_data)
write_otf2(
const output_config& cfg,
const metadata& tool_metadata,
uint64_t pid,
const std::vector<agent_info>& agent_data,
std::deque<rocprofiler_buffer_tracing_hip_api_record_t>* hip_api_data,
std::deque<rocprofiler_buffer_tracing_hsa_api_record_t>* hsa_api_data,
std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>* kernel_dispatch_data,
std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>* memory_copy_data,
std::deque<rocprofiler_buffer_tracing_marker_api_record_t>* marker_api_data,
std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>* /*scratch_memory_data*/,
std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>* rccl_api_data,
std::deque<rocprofiler_buffer_tracing_memory_allocation_record_t>* memory_allocation_data)
{
namespace sdk = ::rocprofiler::sdk;
@@ -366,6 +380,8 @@ write_otf2(const output_config& cfg,
const auto& buffer_names = tool_metadata.buffer_names;
auto tids = std::set<rocprofiler_thread_id_t>{};
auto agent_thread_ids = std::map<rocprofiler_thread_id_t, std::set<rocprofiler_agent_id_t>>{};
auto agent_thread_ids_alloc =
std::map<rocprofiler_thread_id_t, std::set<rocprofiler_agent_id_t>>{};
auto agent_queue_ids =
std::map<rocprofiler_thread_id_t,
std::map<rocprofiler_agent_id_t, std::unordered_set<rocprofiler_queue_id_t>>>{};
@@ -373,6 +389,8 @@ write_otf2(const output_config& cfg,
auto thread_event_info = std::map<rocprofiler_thread_id_t, event_info>{};
auto agent_memcpy_info =
std::map<rocprofiler_thread_id_t, std::map<rocprofiler_agent_id_t, event_info>>{};
auto agent_memalloc_info =
std::map<rocprofiler_thread_id_t, std::map<rocprofiler_agent_id_t, event_info>>{};
auto agent_dispatch_info =
std::map<rocprofiler_thread_id_t,
std::map<rocprofiler_agent_id_t, std::map<rocprofiler_queue_id_t, event_info>>>{};
@@ -407,6 +425,12 @@ write_otf2(const output_config& cfg,
agent_thread_ids[itr.thread_id].emplace(itr.dst_agent_id);
}
for(auto itr : *memory_allocation_data)
{
tids.emplace(itr.thread_id);
agent_thread_ids_alloc[itr.thread_id].emplace(itr.agent_id);
}
for(auto itr : *kernel_dispatch_data)
{
tids.emplace(itr.thread_id);
@@ -421,13 +445,20 @@ write_otf2(const output_config& cfg,
for(const auto& [tid, itr] : agent_thread_ids)
for(auto agent : itr)
agent_memcpy_info[tid].emplace(agent, location_base{pid, tid, agent});
agent_memcpy_info[tid].emplace(
agent, location_base{pid, tid, agent, ROCPROFILER_AGENT_MEMORY_COPY_TYPE});
for(const auto& [tid, itr] : agent_thread_ids_alloc)
for(auto agent : itr)
agent_memalloc_info[tid].emplace(
agent, location_base{pid, tid, agent, ROCPROFILER_AGENT_MEMORY_ALLOC_TYPE});
for(const auto& [tid, itr] : agent_queue_ids)
for(const auto& [agent, qitr] : itr)
for(auto queue : qitr)
agent_dispatch_info[tid][agent].emplace(queue,
location_base{pid, tid, agent, queue});
agent_dispatch_info[tid][agent].emplace(
queue,
location_base{pid, tid, agent, ROCPROFILER_AGENT_DISPATCH_TYPE, queue});
}
for(auto& [tid, evt] : thread_event_info)
@@ -451,6 +482,24 @@ write_otf2(const output_config& cfg,
}
}
for(auto& [tid, itr] : agent_memalloc_info)
{
for(auto& [agent, evt] : itr)
{
const auto* _agent = _get_agent(agent);
auto _type_name = std::string_view{"UNK"};
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_type_name = "CPU";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_type_name = "GPU";
evt.name = fmt::format("Thread {}, Memory Allocation at {} {}",
tid,
_type_name,
_agent->logical_node_type_id);
}
}
auto _queue_ids = std::map<rocprofiler_queue_id_t, uint64_t>{};
for(auto& [tid, itr] : agent_dispatch_info)
for(auto& [agent, qitr] : itr)
@@ -583,6 +632,30 @@ write_otf2(const output_config& cfg,
nullptr});
}
for(auto itr : *memory_allocation_data)
{
auto name = buffer_names.at(itr.kind, itr.operation);
_hash_data.emplace(
get_hash_id(name),
region_info{std::string{name}, OTF2_REGION_ROLE_ALLOCATE, OTF2_PARADIGM_HIP});
// TODO: add attributes for memory allocation parameters
auto& _evt_info = agent_memalloc_info.at(itr.thread_id).at(itr.agent_id);
_evt_info.event_count += 1;
_data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_ENTER,
name,
_evt_info.get_location(),
itr.start_timestamp,
get_attr(sdk::category::memory_allocation{})});
_data.emplace_back(evt_data{ROCPROFILER_CALLBACK_PHASE_EXIT,
name,
_evt_info.get_location(),
itr.end_timestamp,
nullptr});
}
for(auto itr : *kernel_dispatch_data)
{
const auto& info = itr.dispatch_info;
@@ -781,6 +854,24 @@ write_otf2(const output_config& cfg,
}
}
// Memalloc Events
for(auto& [tid, itr] : agent_memalloc_info)
{
for(auto& [agent, evt] : itr)
{
auto _hash = get_hash_id(evt.name);
add_write_string(_hash, evt.name);
OTF2_CHECK(OTF2_GlobalDefWriter_WriteLocation(global_def_writer,
evt.id(), // id
_hash,
OTF2_LOCATION_TYPE_ACCELERATOR_STREAM,
2 * evt.event_count, // # events
agent.handle // location group
));
}
}
// Dispatch Events
for(auto& [tid, itr] : agent_dispatch_info)
{
source/lib/output/generateOTF2.hpp
+13 -11
View File
@@ -34,16 +34,18 @@ namespace rocprofiler
namespace tool
{
void
write_otf2(const output_config& cfg,
const metadata& tool_metadata,
uint64_t pid,
const std::vector<agent_info>& agent_data,
std::deque<rocprofiler_buffer_tracing_hip_api_record_t>* hip_api_data,
std::deque<rocprofiler_buffer_tracing_hsa_api_record_t>* hsa_api_data,
std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>* kernel_dispatch_data,
std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>* memory_copy_data,
std::deque<rocprofiler_buffer_tracing_marker_api_record_t>* marker_api_data,
std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>* scratch_memory_data,
std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>* rccl_api_data);
write_otf2(
const output_config& cfg,
const metadata& tool_metadata,
uint64_t pid,
const std::vector<agent_info>& agent_data,
std::deque<rocprofiler_buffer_tracing_hip_api_record_t>* hip_api_data,
std::deque<rocprofiler_buffer_tracing_hsa_api_record_t>* hsa_api_data,
std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>* kernel_dispatch_data,
std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>* memory_copy_data,
std::deque<rocprofiler_buffer_tracing_marker_api_record_t>* marker_api_data,
std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>* scratch_memory_data,
std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>* rccl_api_data,
std::deque<rocprofiler_buffer_tracing_memory_allocation_record_t>* memory_allocation_data);
} // namespace tool
} // namespace rocprofiler
source/lib/output/generatePerfetto.cpp
+84 -4
View File
@@ -71,7 +71,8 @@ write_perfetto(
const generator<rocprofiler_buffer_tracing_memory_copy_record_t>& memory_copy_gen,
const generator<rocprofiler_buffer_tracing_marker_api_record_t>& marker_api_gen,
const generator<rocprofiler_buffer_tracing_scratch_memory_record_t>& /*scratch_memory_gen*/,
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& rccl_api_gen)
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& rccl_api_gen,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& memory_allocation_gen)
{
namespace sdk = ::rocprofiler::sdk;
@@ -127,9 +128,10 @@ write_perfetto(
tracing_session->Setup(cfg);
tracing_session->StartBlocking();
auto tids = std::set<rocprofiler_thread_id_t>{};
auto demangled = std::unordered_map<std::string_view, std::string>{};
auto agent_thread_ids = std::unordered_map<rocprofiler_agent_id_t, std::set<uint64_t>>{};
auto tids = std::set<rocprofiler_thread_id_t>{};
auto demangled = std::unordered_map<std::string_view, std::string>{};
auto agent_thread_ids = std::unordered_map<rocprofiler_agent_id_t, std::set<uint64_t>>{};
auto agent_thread_ids_alloc = std::unordered_map<rocprofiler_agent_id_t, std::set<uint64_t>>{};
auto agent_queue_ids =
std::unordered_map<rocprofiler_agent_id_t, std::unordered_set<rocprofiler_queue_id_t>>{};
auto thread_indexes = std::unordered_map<rocprofiler_thread_id_t, uint64_t>{};
@@ -138,6 +140,9 @@ write_perfetto(
auto agent_thread_tracks =
std::unordered_map<rocprofiler_agent_id_t,
std::unordered_map<uint64_t, ::perfetto::Track>>{};
auto agent_thread_tracks_alloc =
std::unordered_map<rocprofiler_agent_id_t,
std::unordered_map<uint64_t, ::perfetto::Track>>{};
auto agent_queue_tracks =
std::unordered_map<rocprofiler_agent_id_t,
std::unordered_map<rocprofiler_queue_id_t, ::perfetto::Track>>{};
@@ -171,6 +176,13 @@ write_perfetto(
agent_thread_ids[itr.dst_agent_id].emplace(itr.thread_id);
}
for(auto ditr : memory_allocation_gen)
for(auto itr : memory_allocation_gen.get(ditr))
{
tids.emplace(itr.thread_id);
agent_thread_ids_alloc[itr.agent_id].emplace(itr.thread_id);
}
for(auto ditr : kernel_dispatch_gen)
for(auto itr : kernel_dispatch_gen.get(ditr))
{
@@ -229,6 +241,33 @@ write_perfetto(
}
}
for(const auto& itr : agent_thread_ids_alloc)
{
const auto* _agent = _get_agent(itr.first);
for(auto titr : itr.second)
{
auto _namess = std::stringstream{};
_namess << "MEMORY ALLOCATION on AGENT [" << _agent->logical_node_id << "] THREAD ["
<< thread_indexes.at(titr) << "] ";
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_namess << "(CPU)";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_namess << "(GPU)";
else
_namess << "(UNK)";
auto _track = ::perfetto::Track{get_hash_id(_namess.str())};
auto _desc = _track.Serialize();
_desc.set_name(_namess.str());
perfetto::TrackEvent::SetTrackDescriptor(_track, _desc);
agent_thread_tracks_alloc[itr.first].emplace(titr, _track);
}
}
for(const auto& aitr : agent_queue_ids)
{
uint32_t nqueue = 0;
@@ -424,6 +463,47 @@ write_perfetto(
tracing_session->FlushBlocking();
}
for(auto ditr : memory_allocation_gen)
for(auto itr : memory_allocation_gen.get(ditr))
{
auto name = buffer_names.at(itr.kind, itr.operation);
auto& track = agent_thread_tracks_alloc.at(itr.agent_id).at(itr.thread_id);
std::stringstream hex_stream;
hex_stream << "0x" << std::hex << std::setw(16) << std::setfill('0')
<< itr.starting_address;
std::string hex_starting_address(hex_stream.str());
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::memory_allocation>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"end_ns",
itr.end_timestamp,
"delta_ns",
(itr.end_timestamp - itr.start_timestamp),
"kind",
itr.kind,
"operation",
itr.operation,
"agent",
agents_map.at(itr.agent_id).logical_node_id,
"allocation_size",
itr.allocation_size,
"starting_address",
hex_starting_address,
"corr_id",
itr.correlation_id.internal,
"tid",
itr.thread_id);
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::memory_allocation>::name,
track,
itr.end_timestamp);
tracing_session->FlushBlocking();
}
for(auto ditr : kernel_dispatch_gen)
for(auto itr : kernel_dispatch_gen.get(ditr))
{
source/lib/output/generatePerfetto.hpp
+11 -10
View File
@@ -36,15 +36,16 @@ namespace tool
{
void
write_perfetto(
const output_config& cfg,
const metadata& tool_metadata,
std::vector<agent_info> agent_data,
const generator<rocprofiler_buffer_tracing_hip_api_record_t>& hip_api_gen,
const generator<rocprofiler_buffer_tracing_hsa_api_record_t>& hsa_api_gen,
const generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t>& kernel_dispatch_gen,
const generator<rocprofiler_buffer_tracing_memory_copy_record_t>& memory_copy_gen,
const generator<rocprofiler_buffer_tracing_marker_api_record_t>& marker_api_gen,
const generator<rocprofiler_buffer_tracing_scratch_memory_record_t>& scratch_memory_gen,
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& rccl_api_gen);
const output_config& cfg,
const metadata& tool_metadata,
std::vector<agent_info> agent_data,
const generator<rocprofiler_buffer_tracing_hip_api_record_t>& hip_api_gen,
const generator<rocprofiler_buffer_tracing_hsa_api_record_t>& hsa_api_gen,
const generator<rocprofiler_buffer_tracing_kernel_dispatch_record_t>& kernel_dispatch_gen,
const generator<rocprofiler_buffer_tracing_memory_copy_record_t>& memory_copy_gen,
const generator<rocprofiler_buffer_tracing_marker_api_record_t>& marker_api_gen,
const generator<rocprofiler_buffer_tracing_scratch_memory_record_t>& scratch_memory_gen,
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& rccl_api_gen,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& memory_allocation_gen);
} // namespace tool
} // namespace rocprofiler
source/lib/output/generateStats.cpp
+18
View File
@@ -166,6 +166,24 @@ generate_stats(const output_config& /*cfg*/,
return get_stats(marker_stats);
}
stats_entry_t
generate_stats(const output_config& /*cfg*/,
const metadata& tool_metadata,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& data)
{
auto memory_allocation_stats = stats_map_t{};
for(auto ditr : data)
{
for(auto record : data.get(ditr))
{
auto api_name = tool_metadata.get_operation_name(record.kind, record.operation);
memory_allocation_stats[api_name] += (record.end_timestamp - record.start_timestamp);
}
}
return get_stats(memory_allocation_stats);
}
stats_entry_t
generate_stats(const output_config& /*cfg*/,
const metadata& /*tool_metadata*/,
source/lib/output/generateStats.hpp
+5
View File
@@ -70,6 +70,11 @@ generate_stats(const output_config& cf
const metadata& tool_metadata,
const generator<rocprofiler_buffer_tracing_rccl_api_record_t>& data);
stats_entry_t
generate_stats(const output_config& cfg,
const metadata& tool_metadata,
const generator<rocprofiler_buffer_tracing_memory_allocation_record_t>& data);
void
generate_stats(const output_config& cfg,
const metadata& tool_metadata,