AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[SDK][rocprofv3] MI300 Stochastic PC sampling (#92)

Просмотр исходного кода 
* MI300 Stochastic PC sampling SDK API implementation

* ROCProfV3: Stochastic PC sampling Support (#94)

* ROCProfV3: MI300 Stochastic PC sampling initial draft

* ROCProfV3: Initial Stochastic PC sampling Tests (#95)

ROCProfV3: Initial Stochastic PC sampling tests

* Update rocprofiler_pc_sampling_record_stochastic_v0_t

- update doxygen docs for members
- replace rocprofiler_correlation_id_t with rocprofiler_async_correlation_id_t

* Relax the check in JSON tests

* drain PC sampling buffer during finalize_rocprofv3

* Increase timeout for "Test Install Build" step

- 10 minutes -> 20 minutes
- "Test Installed Packages" has 20 minutes so "Test Install Build" should also

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
Этот коммит содержится в:
Indic, Vladimir
2025-03-21 20:40:45 +01:00
коммит произвёл GitHub
родитель c06feccf2a
Коммит 49ce79a5b5
98 изменённых файлов: 5266 добавлений и 1031 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
samples/pc_sampling/pcs.cpp
+127 -33
Просмотреть файл
@@ -43,7 +43,7 @@ namespace pcs
// TODO: Since this is used only within the `tool_init`,
// we are safe using static constructor.
// It would be nice to make this consistent with the `buffer_ids`.
tool_agent_info_vec_t gpu_agents;
tool_agent_info_vec_t gpu_agents = {};
// The reason for using raw pointers is the following.
// Sometimes, statically created objects of the client::pcs
// namespace might be freed prior to the `tool_fini`,
@@ -55,6 +55,12 @@ tool_agent_info_vec_t gpu_agents;
// `pcs` namespace and export functions for registering/flushing/destroying buffers.
pc_sampling_buffer_id_vec_t* buffer_ids = nullptr;
namespace
{
constexpr uint64_t host_trap_interval = 10000; // 10ms
constexpr uint64_t stochastic_interval = 1048576; // 2 ^ 20 cycles
} // namespace
void
init()
{
@@ -67,6 +73,7 @@ fini()
// Clear the data
buffer_ids->clear();
delete buffer_ids;
buffer_ids = nullptr;
}
pc_sampling_buffer_id_vec_t*
@@ -112,7 +119,7 @@ find_all_gpu_agents_supporting_pc_sampling_impl(rocprofiler_agent_version_t vers
<< "type=" << _agents[i]->type << "\n";
}
*utils::get_output_stream() << ss.str() << std::endl;
*utils::get_output_stream() << ss.str() << "\n";
return ROCPROFILER_STATUS_SUCCESS;
}
@@ -161,8 +168,8 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
// The query operation failed, so consider the PC sampling is unsupported at the agent.
// This can happen if the PC sampling service is invoked within the ROCgdb.
ss << "Querying PC sampling capabilities failed with status=" << status
<< " :: " << rocprofiler_get_status_string(status) << std::endl;
*utils::get_output_stream() << ss.str() << std::endl;
<< " :: " << rocprofiler_get_status_string(status) << "\n";
*utils::get_output_stream() << ss.str() << "\n";
return false;
}
else if(agent_info->avail_configs->empty())
@@ -172,7 +179,8 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
}
ss << "The agent with the id: " << agent_info->agent_id.handle << " supports the "
<< agent_info->avail_configs->size() << " configurations: " << std::endl;
<< agent_info->avail_configs->size() << " configurations: "
<< "\n";
size_t ind = 0;
for(auto& cfg : *agent_info->avail_configs)
{
@@ -181,7 +189,11 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
<< "unit: " << cfg.unit << ", "
<< "min_interval: " << cfg.min_interval << ", "
<< "max_interval: " << cfg.max_interval << ", "
<< "flags: " << std::hex << cfg.flags << std::dec << std::endl;
<< "flags: " << std::hex << cfg.flags << std::dec
<< ((cfg.flags == ROCPROFILER_PC_SAMPLING_CONFIGURATION_FLAGS_INTERVAL_POW2)
? " (an interval value must be power of 2)"
: "")
<< "\n";
}
*utils::get_output_stream() << ss.str() << std::flush;
@@ -194,8 +206,9 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
rocprofiler_context_id_t context_id,
rocprofiler_buffer_id_t buffer_id)
{
int failures = 10;
size_t interval = 0;
auto stochastic_picked = false;
int failures = 10;
size_t interval = 0;
do
{
// Update the list of available configurations
@@ -216,9 +229,9 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
{
if(cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_STOCHASTIC)
{
// Temporarily disable stochastic sampling as it's not fully supported.
// first_stochastic_config = &cfg;
// break;
first_stochastic_config = &cfg;
stochastic_picked = true;
break;
}
else if(!first_host_trap_config &&
cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_HOST_TRAP)
@@ -238,7 +251,7 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
}
else
{
interval = 10000;
interval = stochastic_picked ? stochastic_interval : host_trap_interval;
}
auto status = rocprofiler_configure_pc_sampling_service(context_id,
@@ -251,8 +264,10 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
if(status == ROCPROFILER_STATUS_SUCCESS)
{
*utils::get_output_stream()
<< ">>> We chose PC sampling interval: " << interval
<< " on the agent: " << agent_info->agent->id.handle << std::endl;
<< ">>> We chose " << (stochastic_picked ? "stochastic" : "Host-Trap")
<< " PC sampling with the interval: " << interval << " "
<< (stochastic_picked ? "clock-cycles" : "micro seconds")
<< " on the agent: " << agent_info->agent->id.handle << "\n";
return;
}
else if(status != ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE)
@@ -279,6 +294,87 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
ROCPROFILER_CHECK(ROCPROFILER_STATUS_ERROR);
}
template <typename PcSamplingRecordT>
void
print_sample_common_fields(std::ostream& os, const PcSamplingRecordT* pc_sample)
{
os << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x" << std::hex
<< pc_sample->pc.code_object_offset << "), "
<< "timestamp: " << std::dec << pc_sample->timestamp << ", "
<< "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
<< "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x << ", "
<< "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
<< "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
<< "wave_in_group: " << std::setw(2) << static_cast<unsigned int>(pc_sample->wave_in_group)
<< ", "
<< "chiplet: " << std::setw(2) << static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
<< "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
<< "correlation: {internal=" << std::setw(7) << pc_sample->correlation_id.internal << ", "
<< "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}, ";
}
void
print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_host_trap_v0_t* sample)
{
print_sample_common_fields(os, sample);
os << "\n";
}
void
print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_stochastic_v0_t* sample)
{
print_sample_common_fields(os, sample);
if(sample->wave_issued)
{
auto* inst_c_str = rocprofiler_get_pc_sampling_instruction_type_name(
static_cast<rocprofiler_pc_sampling_instruction_type_t>(sample->inst_type));
utils::pcs_assert(inst_c_str != nullptr, "Invalid instruction type");
os << "wave issued " << std::string(inst_c_str) << " instruction, ";
}
else
{
auto* reason_c_str = rocprofiler_get_pc_sampling_instruction_not_issued_reason_name(
static_cast<rocprofiler_pc_sampling_instruction_not_issued_reason_t>(
sample->snapshot.reason_not_issued));
utils::pcs_assert(reason_c_str != nullptr, "Invalid not issued reason");
os << "wave is stalled due to: " << std::string(reason_c_str) << " reason, ";
}
auto snapshot = sample->snapshot;
os << "two VALU instructions issued: " << static_cast<unsigned int>(snapshot.dual_issue_valu)
<< ", ";
os << "arbiter state: {pipe issued: ("
<< "VALU: " << static_cast<unsigned int>(snapshot.arb_state_issue_valu) << ", "
<< "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_issue_matrix) << ", "
<< "LDS: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds) << ", "
<< "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds_direct) << ", "
<< "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_issue_scalar) << ", "
<< "TEX: " << static_cast<unsigned int>(snapshot.arb_state_issue_vmem_tex) << ", "
<< "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_issue_flat) << ", "
<< "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_issue_exp) << ", "
<< "MISC: " << static_cast<unsigned int>(snapshot.arb_state_issue_misc) << "), "
<< "pipe stalled: ("
<< "VALU: " << static_cast<unsigned int>(snapshot.arb_state_stall_valu) << ", "
<< "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_stall_matrix) << ", "
<< "LDS: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds) << ", "
<< "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds_direct) << ", "
<< "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_stall_scalar) << ", "
<< "TEX: " << static_cast<unsigned int>(snapshot.arb_state_stall_vmem_tex) << ", "
<< "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_stall_flat) << ", "
<< "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_stall_exp) << ", "
<< "MISC: " << static_cast<unsigned int>(snapshot.arb_state_stall_misc) << ")}";
os << "\n";
}
void
print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_invalid_t* /*sample*/)
{
os << "Invalid sample detected.\n";
}
void
rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
rocprofiler_buffer_id_t /*buffer_id*/,
@@ -289,7 +385,7 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
{
std::stringstream ss;
ss << "The number of delivered samples is: " << num_headers << ", "
<< "while the number of dropped samples is: " << drop_count << std::endl;
<< "while the number of dropped samples is: " << drop_count << "\n";
for(size_t i = 0; i < num_headers; i++)
{
@@ -312,23 +408,21 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_host_trap_v0_t*>(
cur_header->payload);
ss << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x"
<< std::hex << pc_sample->pc.code_object_offset << "), "
<< "timestamp: " << std::dec << pc_sample->timestamp << ", "
<< "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
<< "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x
<< ", "
<< "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
<< "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
<< "wave_in_group: " << std::setw(2)
<< static_cast<unsigned int>(pc_sample->wave_in_group) << ", "
<< "chiplet: " << std::setw(2)
<< static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
<< "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
<< "correlation: {internal=" << std::setw(7)
<< pc_sample->correlation_id.internal << ", "
<< "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}"
<< std::endl;
print_sample(ss, pc_sample);
}
else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_STOCHASTIC_V0_SAMPLE)
{
auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_stochastic_v0_t*>(
cur_header->payload);
print_sample(ss, pc_sample);
}
else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_INVALID_SAMPLE)
{
auto* pc_sample =
static_cast<rocprofiler_pc_sampling_record_invalid_t*>(cur_header->payload);
print_sample(ss, pc_sample);
}
else
{
@@ -341,7 +435,7 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
}
}
*utils::get_output_stream() << ss.str() << std::endl;
*utils::get_output_stream() << ss.str() << "\n";
}
} // namespace pcs
} // namespace client