AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding agent-index (#189)

Browse Source 
* Adding agent-index

* review changes

* review comments addressed

* minor fix

* fix CI failure

* review comments

* Fix agent index test and address review comments

* Build Fixes

---------

Co-authored-by: Benjamin Welton <bewelton@amd.com>
This commit is contained in:
Nagaraj, Sriraksha
2025-03-14 02:51:32 -05:00
committed by GitHub
parent 2fe63d873e
commit c30bb7cbda
32 changed files with 604 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/bin/rocprofv3.py
+17
View File
@@ -568,6 +568,20 @@ For MPI applications (or other job launchers such as SLURM), place rocprofv3 ins
"--realpath",
help=argparse.SUPPRESS,
)
advanced_options.add_argument(
"-A",
"--agent-index",
choices=("absolute", "relative", "type-relative"),
help="""absolute == node_id, e.g. Agent-0, Agent-2, Agent-4- absolute index of the agent regardless of cgroups masking.
This is a monotonically increasing number that is incremented for every folder in /sys/class/kfd/kfd/topology/nodes.
relative == logical_node_id,
e.g. Agent-0, Agent-1, Agent-2- relative index of the agent accounting for cgroups masking.
This is a monotonically increasing number which is incremented for every folder in /sys/class/kfd/kfd/topology/nodes/ whose properties file was non-empty.
type-relative == logical_node_type_id,
e.g. CPU-0, GPU-0, GPU-1- relative index of the agent accounting for cgroups masking where indexing starts at zero for each agent type.
It is a monotonically increasing number for each agent type and is incremented for every type folder in /sys/class/kfd/kfd/topology/nodes/ whose properties file is non-empty.
If agent-index is not provided then the default value for it is relative.""",
)
# below is available for CI because LD_PRELOADing a library linked to a sanitizer library
# causes issues in apps where HIP is part of shared library.
add_parser_bool_argument(
@@ -1202,6 +1216,9 @@ def run(app_args, args, **kwargs):
if args.kernel_iteration_range:
update_env("ROCPROF_KERNEL_FILTER_RANGE", ", ".join(args.kernel_iteration_range))
if args.agent_index:
update_env("ROCPROF_AGENT_INDEX", args.agent_index)
if args.extra_counters is not None:
with open(args.extra_counters, "r") as e_file:
e_file_contents = e_file.read()
source/include/rocprofiler-sdk/cxx/serialization.hpp
+20
View File
@@ -954,6 +954,23 @@ save(ArchiveT& ar, rocprofiler_pc_sampling_configuration_t data)
ROCP_SDK_SAVE_DATA_FIELD(flags);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_agent_runtime_visiblity_t data)
{
ROCP_SDK_SAVE_DATA_BITFIELD("hsa", hsa);
ROCP_SDK_SAVE_DATA_BITFIELD("hip", hip);
ROCP_SDK_SAVE_DATA_BITFIELD("rccl", rccl);
ROCP_SDK_SAVE_DATA_BITFIELD("rocdecode", rocdecode);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_uuid_t data)
{
ROCP_SDK_SAVE_DATA_FIELD(value);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, const rocprofiler_agent_v0_t& data)
@@ -1012,6 +1029,9 @@ save(ArchiveT& ar, const rocprofiler_agent_v0_t& data)
ROCP_SDK_SAVE_DATA_CSTR(model_name);
ROCP_SDK_SAVE_DATA_FIELD(node_id);
ROCP_SDK_SAVE_DATA_FIELD(logical_node_id);
ROCP_SDK_SAVE_DATA_FIELD(logical_node_type_id);
ROCP_SDK_SAVE_DATA_FIELD(runtime_visibility);
ROCP_SDK_SAVE_DATA_FIELD(uuid);
auto generate = [&](auto name, const auto* value, uint64_t size) {
using value_type = std::remove_const_t<std::remove_pointer_t<decltype(value)>>;
source/lib/output/generateCSV.cpp
+25 -15
View File
@@ -288,10 +288,12 @@ generate_csv(const output_config&
auto row_ss = std::stringstream{};
auto kernel_name = tool_metadata.get_kernel_name(record.dispatch_info.kernel_id,
record.correlation_id.external.value);
rocprofiler::tool::csv::kernel_trace_csv_encoder::write_row(
row_ss,
tool_metadata.get_kind_name(record.kind),
tool_metadata.get_node_id(record.dispatch_info.agent_id),
tool_metadata.get_agent_index(record.dispatch_info.agent_id, cfg.agent_index_value)
.as_string(),
record.dispatch_info.queue_id.handle,
record.thread_id,
record.dispatch_info.dispatch_id,
@@ -418,6 +420,7 @@ generate_csv(const output_config& c
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
for(auto ditr : data)
{
for(auto record : data.get(ditr))
@@ -428,8 +431,10 @@ generate_csv(const output_config& c
row_ss,
tool_metadata.get_kind_name(record.kind),
api_name,
tool_metadata.get_node_id(record.src_agent_id),
tool_metadata.get_node_id(record.dst_agent_id),
tool_metadata.get_agent_index(record.src_agent_id, cfg.agent_index_value)
.as_string(),
tool_metadata.get_agent_index(record.dst_agent_id, cfg.agent_index_value)
.as_string(),
record.correlation_id.internal,
record.start_timestamp,
record.end_timestamp);
@@ -465,13 +470,14 @@ generate_csv(const output_config&
{
for(auto record : data.get(ditr))
{
uint64_t agent_info{0};
auto agent_info = std::string{};
// Free functions currently do not track agent information. Only set it on allocation
// operations, otherwise set it to 0 currently
if(record.operation == ROCPROFILER_MEMORY_ALLOCATION_ALLOCATE ||
record.operation == ROCPROFILER_MEMORY_ALLOCATION_VMEM_ALLOCATE)
{
agent_info = tool_metadata.get_node_id(record.agent_id);
agent_info = tool_metadata.get_agent_index(record.agent_id, cfg.agent_index_value)
.as_string();
}
auto api_name = tool_metadata.get_operation_name(record.kind, record.operation);
auto row_ss = std::stringstream{};
@@ -611,7 +617,10 @@ generate_csv(const output_config& cfg,
row_ss,
correlation_id.internal,
record.dispatch_data.dispatch_info.dispatch_id,
tool_metadata.get_node_id(record.dispatch_data.dispatch_info.agent_id),
tool_metadata
.get_agent_index(record.dispatch_data.dispatch_info.agent_id,
cfg.agent_index_value)
.as_string(),
record.dispatch_data.dispatch_info.queue_id.handle,
tool_metadata.process_id,
record.thread_id,
@@ -667,15 +676,16 @@ generate_csv(const output_config&
auto kind_name = tool_metadata.get_kind_name(record.kind);
auto op_name = tool_metadata.get_operation_name(record.kind, record.operation);
tool::csv::scratch_memory_encoder::write_row(row_ss,
kind_name,
op_name,
tool_metadata.get_node_id(record.agent_id),
record.queue_id.handle,
record.thread_id,
record.flags,
record.start_timestamp,
record.end_timestamp);
tool::csv::scratch_memory_encoder::write_row(
row_ss,
kind_name,
op_name,
tool_metadata.get_agent_index(record.agent_id, cfg.agent_index_value).as_string(),
record.queue_id.handle,
record.thread_id,
record.flags,
record.start_timestamp,
record.end_timestamp);
ofs << row_ss.str();
}
source/lib/output/generateOTF2.cpp
+26 -28
View File
@@ -476,15 +476,13 @@ write_otf2(
{
for(auto& [agent, evt] : itr)
{
const auto* _agent = _get_agent(agent);
auto _type_name = std::string_view{"UNK"};
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_type_name = "CPU";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_type_name = "GPU";
evt.name = fmt::format(
"Thread {}, Copy to {} {}", tid, _type_name, _agent->logical_node_type_id);
const auto* _agent = _get_agent(agent);
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, cfg.agent_index_value);
evt.name = fmt::format("Thread {}, Copy to {} {}",
tid,
agent_index_info.type,
agent_index_info.as_string("-"));
}
}
@@ -501,16 +499,20 @@ write_otf2(
{
_agent = _get_agent(agent);
}
auto _type_name = std::string_view{"UNK"};
if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_type_name = "CPU";
else if(_agent != nullptr && _agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_type_name = "GPU";
evt.name = fmt::format("Thread {}, Memory Operation at {} {}",
tid,
_type_name,
_agent == nullptr ? 0 : _agent->logical_node_type_id);
if(_agent)
{
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, cfg.agent_index_value);
evt.name = fmt::format("Thread {}, Memory Operation at {} {}",
tid,
agent_index_info.type,
agent_index_info.as_string("-"));
}
else
{
auto _type_name = std::string_view{"UNK"};
evt.name = fmt::format("Thread {}, Memory Operation at {} {}", tid, _type_name, 0);
}
}
}
@@ -532,17 +534,13 @@ write_otf2(
{
for(auto& [queue, evt] : qitr)
{
const auto* _agent = _get_agent(agent);
auto _type_name = std::string_view{"UNK"};
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_type_name = "CPU";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_type_name = "GPU";
const auto* _agent = _get_agent(agent);
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, cfg.agent_index_value);
evt.name = fmt::format("Thread {}, Compute on {} {}, Queue {}",
tid,
_type_name,
_agent->logical_node_type_id,
agent_index_info.type,
agent_index_info.as_string("-"),
_queue_ids.at(queue));
}
}
source/lib/output/generatePerfetto.cpp
+50 -44
View File
@@ -260,15 +260,11 @@ write_perfetto(
const auto* _agent = _get_agent(aitr.first);
auto _namess = std::stringstream{};
_namess << "COMPUTE AGENT [" << _agent->logical_node_id << "] QUEUE [" << nqueue++
<< "] ";
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_namess << "(CPU)";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_namess << "(GPU)";
else
_namess << "(UNK)";
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, ocfg.agent_index_value);
_namess << "COMPUTE " << agent_index_info.label << " [" << agent_index_info.index
<< "] QUEUE [" << nqueue++ << "] ";
_namess << agent_index_info.type;
auto _track = ::perfetto::Track{get_hash_id(_namess.str())};
auto _desc = _track.Serialize();
@@ -478,31 +474,34 @@ write_perfetto(
auto name = buffer_names.at(itr.kind, itr.operation);
auto& track = agent_thread_tracks.at(itr.dst_agent_id).at(itr.thread_id);
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::memory_copy>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"end_ns",
itr.end_timestamp,
"delta_ns",
(itr.end_timestamp - itr.start_timestamp),
"kind",
itr.kind,
"operation",
itr.operation,
"src_agent",
agents_map.at(itr.src_agent_id).logical_node_id,
"dst_agent",
agents_map.at(itr.dst_agent_id).logical_node_id,
"copy_bytes",
itr.bytes,
"corr_id",
itr.correlation_id.internal,
"tid",
itr.thread_id);
TRACE_EVENT_BEGIN(
sdk::perfetto_category<sdk::category::memory_copy>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"end_ns",
itr.end_timestamp,
"delta_ns",
(itr.end_timestamp - itr.start_timestamp),
"kind",
itr.kind,
"operation",
itr.operation,
"src_agent",
tool_metadata.get_agent_index(itr.src_agent_id, ocfg.agent_index_value)
.as_string("-"),
"dst_agent",
tool_metadata.get_agent_index(itr.dst_agent_id, ocfg.agent_index_value)
.as_string("-"),
"copy_bytes",
itr.bytes,
"corr_id",
itr.correlation_id.internal,
"tid",
itr.thread_id);
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::memory_copy>::name,
track,
itr.end_timestamp);
@@ -592,7 +591,10 @@ write_perfetto(
"kind",
current.kind,
"agent",
tool_metadata.get_node_id(info.agent_id),
tool_metadata
.get_agent_index(agents_map.at(info.agent_id).id,
ocfg.agent_index_value)
.as_string("-"),
"corr_id",
current.correlation_id.internal,
"queue",
@@ -676,11 +678,10 @@ write_perfetto(
auto _track_name = std::stringstream{};
const auto* _agent = _get_agent(mitr.first);
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_track_name << "COPY BYTES to AGENT [" << _agent->logical_node_id << "] (CPU)";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_track_name << "COPY BYTES to AGENT [" << _agent->logical_node_id << "] (GPU)";
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, ocfg.agent_index_value);
_track_name << "COPY BYTES to " << agent_index_info.label << " ["
<< agent_index_info.index << "] (" << agent_index_info.type << ")";
constexpr auto _unit = ::perfetto::CounterTrack::Unit::UNIT_SIZE_BYTES;
auto& _name = mem_cpy_cnt_names.emplace_back(_track_name.str());
@@ -847,10 +848,15 @@ write_perfetto(
auto _track_name = std::stringstream{};
const rocprofiler_agent_t* _agent = _get_agent(alloc_itr.first);
if(_agent->type == ROCPROFILER_AGENT_TYPE_CPU)
_track_name << "ALLOCATE BYTES on AGENT [" << _agent->logical_node_id << "] (CPU)";
else if(_agent->type == ROCPROFILER_AGENT_TYPE_GPU)
_track_name << "ALLOCATE BYTES on AGENT [" << _agent->logical_node_id << "] (GPU)";
if(_agent != nullptr)
{
auto agent_index_info =
tool_metadata.get_agent_index(_agent->id, ocfg.agent_index_value);
_track_name << "ALLOCATE BYTES on " << agent_index_info.label << " ["
<< agent_index_info.index << "] (" << agent_index_info.type << ")";
}
else
_track_name << "FREE BYTES";
constexpr auto _unit = ::perfetto::CounterTrack::Unit::UNIT_SIZE_BYTES;
auto& _name = mem_alloc_cnt_names.emplace_back(_track_name.str());
source/lib/output/metadata.cpp
+30 -3
View File
@@ -33,6 +33,7 @@
#include <rocprofiler-sdk/rocprofiler.h>
#include <rocprofiler-sdk/cxx/details/tokenize.hpp>
#include <fmt/core.h>
#include <fmt/format.h>
#include <unistd.h>
@@ -534,10 +535,36 @@ metadata::get_operation_name(rocprofiler_buffer_tracing_kind_t kind,
return buffer_names.at(kind, op);
}
uint64_t
metadata::get_node_id(rocprofiler_agent_id_t _val) const
agent_index
metadata::get_agent_index(rocprofiler_agent_id_t id, agent_indexing index) const
{
return CHECK_NOTNULL(get_agent(_val))->logical_node_id;
const auto* _agent = get_agent(id);
ROCP_FATAL_IF(!_agent) << "Information of the agent with handle: " << id.handle
<< " is not present";
// stringify agent type
auto get_type = [_agent]() -> std::string_view {
switch(_agent->type)
{
case ROCPROFILER_AGENT_TYPE_CPU: return "CPU";
case ROCPROFILER_AGENT_TYPE_GPU: return "GPU";
case ROCPROFILER_AGENT_TYPE_NONE:
case ROCPROFILER_AGENT_TYPE_LAST: break;
}
return "UNK";
};
switch(index)
{
case agent_indexing::node: return agent_index{"Agent", _agent->node_id, get_type()};
case agent_indexing::logical_node_type:
return agent_index{
get_type(), static_cast<uint32_t>(_agent->logical_node_type_id), get_type()};
case agent_indexing::logical_node:
default:
return agent_index{"Agent", static_cast<uint32_t>(_agent->logical_node_id), get_type()};
}
}
const std::string*
source/lib/output/metadata.hpp
+23 -1
View File
@@ -44,6 +44,8 @@
#include <rocprofiler-sdk/cxx/name_info.hpp>
#include <rocprofiler-sdk/cxx/operators.hpp>
#include <fmt/core.h>
#include <cstdint>
#include <map>
#include <string>
@@ -86,6 +88,26 @@ using code_object_load_info_vec_t = std::vector<rocprofiler::att_wrapper::Codeob
template <typename Tp>
using synced_map = common::Synchronized<Tp, true>;
enum class agent_indexing
{
node = 0,
logical_node,
logical_node_type,
};
struct agent_index
{
std::string_view label = {}; // e.g. Agent, CPU, GPU
uint32_t index = 0; // the numerical index
std::string_view type = {}; // e.g. CPU, GPU, etc.
// returns label + index
std::string as_string(std::string_view sep = " ") const
{
return fmt::format("{}{}{}", label, sep, index);
}
};
struct metadata
{
using agent_info_ptr_vec_t = common::container::small_vector<const agent_info*, 16>;
@@ -165,7 +187,7 @@ struct metadata
rocprofiler_tracing_operation_t op) const;
std::string_view get_operation_name(rocprofiler_buffer_tracing_kind_t kind,
rocprofiler_tracing_operation_t op) const;
uint64_t get_node_id(rocprofiler_agent_id_t _val) const;
agent_index get_agent_index(rocprofiler_agent_id_t agent, agent_indexing index) const;
const std::string* get_string_entry(size_t key) const;
private:
source/lib/output/output_config.cpp
+8
View File
@@ -87,6 +87,14 @@ output_config::parse_env()
<< "Unsupported output format type: " << itr;
}
std::string agent_index = common::get_env("ROCPROF_AGENT_INDEX", "relative");
if(agent_index == "type-relative")
agent_index_value = agent_indexing::logical_node_type;
else if(agent_index == "absolute")
agent_index_value = agent_indexing::node;
else
agent_index_value = agent_indexing::logical_node;
const auto supported_perfetto_backends = std::set<std::string_view>{"inprocess", "system"};
LOG_IF(FATAL, supported_perfetto_backends.count(perfetto_backend) == 0)
<< "Unsupported perfetto backend type: " << perfetto_backend;
source/lib/output/output_config.hpp
+2
View File
@@ -23,6 +23,7 @@
#pragma once
#include "format_path.hpp"
#include "metadata.hpp"
#include "lib/common/environment.hpp"
#include "lib/common/filesystem.hpp"
@@ -71,6 +72,7 @@ struct output_config
uint64_t stats_summary_unit_value = 1;
size_t perfetto_shmem_size_hint = defaults::perfetto_shmem_size_hint_kb;
size_t perfetto_buffer_size = defaults::perfetto_buffer_size_kb;
agent_indexing agent_index_value = agent_indexing::logical_node;
std::string stats_summary_unit = "nsec";
std::string output_path = "%cwd%";
std::string output_file = "%hostname%/%pid%";