AI/rocm-systems
2
0
Atdalīts 0
Kods Problēmas Izmaiņu pieprasījumi Darbības Pakotnes Projekti Laidieni Vikivietne Aktivitāte

[SDK][rocprofv3] MI300 Stochastic PC sampling (#92)

Pārlūkot izejas kodu 
* MI300 Stochastic PC sampling SDK API implementation

* ROCProfV3: Stochastic PC sampling Support (#94)

* ROCProfV3: MI300 Stochastic PC sampling initial draft

* ROCProfV3: Initial Stochastic PC sampling Tests (#95)

ROCProfV3: Initial Stochastic PC sampling tests

* Update rocprofiler_pc_sampling_record_stochastic_v0_t

- update doxygen docs for members
- replace rocprofiler_correlation_id_t with rocprofiler_async_correlation_id_t

* Relax the check in JSON tests

* drain PC sampling buffer during finalize_rocprofv3

* Increase timeout for "Test Install Build" step

- 10 minutes -> 20 minutes
- "Test Installed Packages" has 20 minutes so "Test Install Build" should also

---------

Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
Šī revīzija ir iekļauta:
Indic, Vladimir
2025-03-21 20:40:45 +01:00
revīziju iesūtīja GitHub
vecāks c06feccf2a
revīzija 49ce79a5b5
98 mainīti faili ar 5266 papildinājumiem un 1031 dzēšanām
Lejupielādēt ielāpa failu Lejupielādēt izmaiņu failu Izvērst visus failus Savērst visus failus
tests/pc_sampling/pcs.cpp
+155 -68
Parādīt failu
@@ -53,6 +53,11 @@ using avail_configs_vec_t = std::vector<rocprofiler_pc_sampling_configur
using tool_agent_info_vec_t = std::vector<std::unique_ptr<tool_agent_info>>;
using pc_sampling_buffer_id_vec_t = std::vector<rocprofiler_buffer_id_t>;
namespace
{
constexpr uint64_t stochastic_interval = 1048576; // 2 ^ 20 cycles
} // namespace
struct tool_agent_info
{
rocprofiler_agent_id_t agent_id;
@@ -79,16 +84,14 @@ public:
}
// GPU agents supporting PC sampling
tool_agent_info_vec_t gpu_agents;
// The total number of collected samples
std::atomic<uint64_t> total_samples_num{0};
tool_agent_info_vec_t gpu_agents = {};
// ROCProfiler-SDK PC sampling buffers
pc_sampling_buffer_id_vec_t buffer_ids;
pc_sampling_buffer_id_vec_t buffer_ids = {};
// The set that keeps track of reported code object loading/unloading events.
// At the end of the test, the sets needs to be empty.
// Namely, each loading event will insert a code object id into the set,
// while each unloading event will delete a code ojbect id from the set.
code_object_id_set_t active_code_objects;
code_object_id_set_t active_code_objects = {};
};
// The reason for using raw pointers is the following.
@@ -139,7 +142,7 @@ find_all_gpu_agents_supporting_pc_sampling_impl(rocprofiler_agent_version_t vers
<< "type=" << _agents[i]->type << "\n";
}
*utils::get_output_stream() << ss.str() << std::endl;
*utils::get_output_stream() << ss.str() << "\n";
return ROCPROFILER_STATUS_SUCCESS;
}
@@ -188,8 +191,8 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
{
// The query operation failed, so consider the PC sampling is unsupported at the agent.
// This can happen if the PC sampling service is invoked within the ROCgdb.
ss << "Querying PC sampling capabilities failed with status: " << status << std::endl;
*utils::get_output_stream() << ss.str() << std::endl;
ss << "Querying PC sampling capabilities failed with status: " << status << "\n";
*utils::get_output_stream() << ss.str() << "\n";
return false;
}
else if(agent_info->avail_configs->size() == 0)
@@ -199,7 +202,8 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
}
ss << "The agent with the id: " << agent_info->agent_id.handle << " supports the "
<< agent_info->avail_configs->size() << " configurations: " << std::endl;
<< agent_info->avail_configs->size() << " configurations: "
<< "\n";
size_t ind = 0;
for(auto& cfg : *agent_info->avail_configs)
{
@@ -208,7 +212,11 @@ query_avail_configs_for_agent(tool_agent_info* agent_info)
<< "unit: " << cfg.unit << ", "
<< "min_interval: " << cfg.min_interval << ", "
<< "max_interval: " << cfg.max_interval << ", "
<< "flags: " << std::hex << cfg.flags << std::dec << std::endl;
<< "flags: " << std::hex << cfg.flags << std::dec
<< ((cfg.flags == ROCPROFILER_PC_SAMPLING_CONFIGURATION_FLAGS_INTERVAL_POW2)
? " (an interval value must be power of 2)"
: "")
<< "\n";
}
*utils::get_output_stream() << ss.str() << std::flush;
@@ -221,8 +229,9 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
rocprofiler_context_id_t context_id,
rocprofiler_buffer_id_t buffer_id)
{
int failures = MAX_FAILURES;
size_t interval = 0;
auto stochastic_picked = false;
int failures = MAX_FAILURES;
size_t interval = 0;
do
{
// Update the list of available configurations
@@ -245,9 +254,9 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
{
if(cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_STOCHASTIC)
{
// Temporarily disable stochastic sampling as it's not fully supported.
// first_stochastic_config = &cfg;
// break;
first_stochastic_config = &cfg;
stochastic_picked = true;
break;
}
else if(!first_host_trap_config &&
cfg.method == ROCPROFILER_PC_SAMPLING_METHOD_HOST_TRAP)
@@ -260,7 +269,7 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
const rocprofiler_pc_sampling_configuration_t* picked_cfg =
(first_stochastic_config != nullptr) ? first_stochastic_config : first_host_trap_config;
interval = picked_cfg->min_interval;
interval = (stochastic_picked) ? stochastic_interval : picked_cfg->min_interval;
auto status = rocprofiler_configure_pc_sampling_service(context_id,
agent_info->agent_id,
@@ -272,8 +281,10 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
if(status == ROCPROFILER_STATUS_SUCCESS)
{
*utils::get_output_stream()
<< ">>> We chose PC sampling interval: " << interval
<< " on the agent: " << agent_info->agent->id.handle << std::endl;
<< ">>> We chose " << (stochastic_picked ? "stochastic" : "Host-Trap")
<< " PC sampling with the interval: " << interval << " "
<< (stochastic_picked ? "clock-cycles" : "micro seconds")
<< " on the agent: " << agent_info->agent->id.handle << "\n";
return;
}
else if(status != ROCPROFILER_STATUS_ERROR_NOT_AVAILABLE)
@@ -301,6 +312,106 @@ configure_pc_sampling_prefer_stochastic(tool_agent_info* agent_info,
"Failed too many times configuring PC sampling service");
}
template <typename PcSamplingRecordT>
void
print_sample_common_fields(std::ostream& os, const PcSamplingRecordT* pc_sample)
{
os << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x" << std::hex
<< pc_sample->pc.code_object_offset << "), "
<< "timestamp: " << std::dec << pc_sample->timestamp << ", "
<< "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
<< "workgroup_id_(x=" << std::dec << std::setw(5) << pc_sample->workgroup_id.x << ", "
<< "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
<< "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
<< "wave_in_group: " << std::setw(2) << static_cast<unsigned int>(pc_sample->wave_in_group)
<< ", "
<< "chiplet: " << std::setw(2) << static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
<< "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
<< "correlation: {internal=" << std::setw(7) << pc_sample->correlation_id.internal << ", "
<< "external=" << std::setw(5) << pc_sample->correlation_id.external.value << "}, ";
}
void
print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_host_trap_v0_t* sample)
{
print_sample_common_fields(os, sample);
os << "\n";
}
void
print_sample(std::ostream& os, const rocprofiler_pc_sampling_record_stochastic_v0_t* sample)
{
print_sample_common_fields(os, sample);
if(sample->wave_issued)
{
auto* inst_c_str = rocprofiler_get_pc_sampling_instruction_type_name(
static_cast<rocprofiler_pc_sampling_instruction_type_t>(sample->inst_type));
utils::pcs_assert(inst_c_str != nullptr, "Invalid instruction type");
os << "wave issued " << std::string(inst_c_str) << " instruction, ";
}
else
{
auto* reason_c_str = rocprofiler_get_pc_sampling_instruction_not_issued_reason_name(
static_cast<rocprofiler_pc_sampling_instruction_not_issued_reason_t>(
sample->snapshot.reason_not_issued));
utils::pcs_assert(reason_c_str != nullptr, "Invalid not issued reason");
os << "wave is stalled due to: " << std::string(reason_c_str) << " reason, ";
}
auto snapshot = sample->snapshot;
os << "two VALU instructions issued: " << static_cast<unsigned int>(snapshot.dual_issue_valu)
<< ", ";
os << "arbiter state: {pipe issued: ("
<< "VALU: " << static_cast<unsigned int>(snapshot.arb_state_issue_valu) << ", "
<< "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_issue_matrix) << ", "
<< "LDS: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds) << ", "
<< "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_issue_lds_direct) << ", "
<< "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_issue_scalar) << ", "
<< "TEX: " << static_cast<unsigned int>(snapshot.arb_state_issue_vmem_tex) << ", "
<< "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_issue_flat) << ", "
<< "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_issue_exp) << ", "
<< "MISC: " << static_cast<unsigned int>(snapshot.arb_state_issue_misc) << "), "
<< "pipe stalled: ("
<< "VALU: " << static_cast<unsigned int>(snapshot.arb_state_stall_valu) << ", "
<< "MATRIX: " << static_cast<unsigned int>(snapshot.arb_state_stall_matrix) << ", "
<< "LDS: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds) << ", "
<< "LDS_DIRECT: " << static_cast<unsigned int>(snapshot.arb_state_stall_lds_direct) << ", "
<< "SCALAR: " << static_cast<unsigned int>(snapshot.arb_state_stall_scalar) << ", "
<< "TEX: " << static_cast<unsigned int>(snapshot.arb_state_stall_vmem_tex) << ", "
<< "FLAT: " << static_cast<unsigned int>(snapshot.arb_state_stall_flat) << ", "
<< "EXPORT: " << static_cast<unsigned int>(snapshot.arb_state_stall_exp) << ", "
<< "MISC: " << static_cast<unsigned int>(snapshot.arb_state_stall_misc) << ")}";
os << "\n";
}
template <typename PcSamplingRecordT>
static inline void
process_sample(const PcSamplingRecordT* pc_sample,
address_translation::CodeobjAddressTranslate& translator,
address_translation::FlatProfile& flat_profile)
{
// Ignore samples from blit kernels or self-modifying code.
if(pc_sample->correlation_id.internal == ROCPROFILER_CORRELATION_ID_INTERNAL_NONE) return;
auto corr_id = pc_sample->correlation_id;
// Internal correlation IDs are generated by the ROCProfiler-SDK for
// kernel dispatches only. Similarly, the test tool generate external
// correlation IDs for the kernel dispatches only.
// Thus, we should expect them to be equal.
assert(corr_id.internal == corr_id.external.value);
assert(corr_id.external.value > 0);
// Decoding the PC
auto inst = translator.get(pc_sample->pc.code_object_id, pc_sample->pc.code_object_offset);
flat_profile.add_sample(std::move(inst), pc_sample->exec_mask);
// TODO: introduce checks specific to stochastic sampling
// TODO: print an instruction inside print_sample
}
void
rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
rocprofiler_buffer_id_t /*buffer_id*/,
@@ -311,7 +422,7 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
{
std::stringstream ss;
ss << "The number of delivered samples is: " << num_headers << ", "
<< "while the number of dropped samples is: " << drop_count << std::endl;
<< "while the number of dropped samples is: " << drop_count << "\n";
auto& flat_profile = client::address_translation::get_flat_profile();
auto& translator = client::address_translation::get_address_translator();
@@ -340,48 +451,26 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
{
auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_host_trap_v0_t*>(
cur_header->payload);
ss << "(code_obj_id, offset): (" << pc_sample->pc.code_object_id << ", 0x"
<< std::hex << pc_sample->pc.code_object_offset << "), "
<< "timestamp: " << std::dec << pc_sample->timestamp << ", "
<< "exec: " << std::hex << std::setw(16) << pc_sample->exec_mask << ", "
<< "workgroup_id_(x=" << std::dec << std::setw(5)
<< pc_sample->workgroup_id.x << ", "
<< "y=" << std::setw(5) << pc_sample->workgroup_id.y << ", "
<< "z=" << std::setw(5) << pc_sample->workgroup_id.z << "), "
<< "wave_in_group: " << std::setw(2)
<< static_cast<unsigned int>(pc_sample->wave_in_group) << ", "
<< "chiplet: " << std::setw(2)
<< static_cast<unsigned int>(pc_sample->hw_id.chiplet) << ", "
<< "dispatch_id: " << std::setw(7) << pc_sample->dispatch_id << ","
<< "correlation: {internal=" << std::setw(7)
<< pc_sample->correlation_id.internal << ", "
<< "external=" << std::setw(5) << pc_sample->correlation_id.external.value
<< "}" << std::endl;
// Ignore samples from blit kernels.
if(pc_sample->correlation_id.internal ==
ROCPROFILER_CORRELATION_ID_INTERNAL_NONE)
continue;
total_samples_num() += 1;
auto corr_id = pc_sample->correlation_id;
// Internal correlation IDs are generated by the ROCProfiler-SDK for
// kernel dispatches only. Similarly, the test tool generate external
// correlation IDs for the kernel dispatches only.
// Thus, we should expect them to be equal.
assert(corr_id.internal == corr_id.external.value);
assert(corr_id.external.value > 0);
// Decoding the PC
auto inst = translator.get(pc_sample->pc.code_object_id,
pc_sample->pc.code_object_offset);
flat_profile.add_sample(std::move(inst), pc_sample->exec_mask);
print_sample(ss, pc_sample);
process_sample(pc_sample, translator, flat_profile);
}
else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_STOCHASTIC_V0_SAMPLE)
{
auto* pc_sample = static_cast<rocprofiler_pc_sampling_record_stochastic_v0_t*>(
cur_header->payload);
print_sample(ss, pc_sample);
process_sample(pc_sample, translator, flat_profile);
}
else if(cur_header->kind == ROCPROFILER_PC_SAMPLING_RECORD_INVALID_SAMPLE)
{
// tracking number of invalid samples
flat_profile.add_invalid_sample();
}
else
{
assert(false);
std::cerr << "Unexpected kind of PC sampling record: " << cur_header->kind
<< "\n";
exit(-1);
}
}
else
@@ -391,7 +480,7 @@ rocprofiler_pc_sampling_callback(rocprofiler_context_id_t /*context_id*/,
}
// TODO: do we need some sync here?
*utils::get_output_stream() << ss.str() << std::endl;
*utils::get_output_stream() << ss.str() << "\n";
}
}
} // namespace
@@ -406,12 +495,7 @@ void
fini()
{
delete pc_sampler;
}
std::atomic<uint64_t>&
total_samples_num()
{
return pc_sampler->total_samples_num;
pc_sampler = nullptr;
}
void
@@ -421,9 +505,11 @@ configure_pc_sampling_on_all_agents(rocprofiler_context_id_t context)
if(pc_sampler->gpu_agents.empty())
{
*utils::get_output_stream() << "No availabe gpu agents supporting PC sampling" << std::endl;
*utils::get_output_stream() << "No availabe gpu agents supporting PC sampling"
<< "\n";
// Emit the message to skip the test.
std::cerr << "PC sampling unavailable" << std::endl;
std::cerr << "PC sampling unavailable"
<< "\n";
// Exit with no error if none of the GPUs support PC sampling.
exit(0);
}
@@ -481,7 +567,8 @@ flush_and_destroy_buffers()
if(status == ROCPROFILER_STATUS_ERROR_BUFFER_BUSY)
{
*utils::get_output_stream()
<< "The buffer is busy, so we cannot destroy it at the moment." << std::endl;
<< "The buffer is busy, so we cannot destroy it at the moment."
<< "\n";
}
else
{