AI/rocm-systems
2
0
フォーク 0
コード イシュー プルリクエスト Actions パッケージ プロジェクト リリース Wiki アクティビティ

SWDEV-406401: Adding Kernel Duration as a Metric

ソースを参照 
Change-Id: If5137ce0bdb65d87ca9d86fa860cb95b240081ce


[ROCm/rocprofiler commit: f1f5c4485d]
このコミットが含まれているのは:
Ammar ELWazir
2023-06-15 19:21:19 +00:00
committed by Ammar Elwazir
親 41223f8297
コミット 5f784afbe5
6個のファイルの変更、121行の追加、96行の削除
Patchファイルをダウンロード Diffファイルをダウンロード すべてのファイルを展開 すべてのファイルを折り畳む
projects/rocprofiler/src/core/counters/metrics/derived_counters.xml
+5 -1
ファイルの表示
@@ -1,6 +1,7 @@
#include "basic_counters.xml"
<gfx8_expr>
<metric name="KERNEL_DURATION" expr=1 descr="The duration of the kernel dispatch"></metric>
<metric name="TA_BUSY_avr" expr=avr(TA_TA_BUSY,16) descr="TA block is busy. Average over TA instances."></metric>
<metric name="TA_BUSY_max" expr=max(TA_TA_BUSY,16) descr="TA block is busy. Max over TA instances."></metric>
<metric name="TA_BUSY_min" expr=min(TA_TA_BUSY,16) descr="TA block is busy. Min over TA instances."></metric>
@@ -35,6 +36,7 @@
</gfx8_expr>
<gfx9_expr>
<metric name="KERNEL_DURATION" expr=1 descr="The duration of the kernel dispatch"></metric>
<metric name="TA_BUSY_avr" expr=avr(TA_TA_BUSY,16) descr="TA block is busy. Average over TA instances."></metric>
<metric name="TA_BUSY_max" expr=max(TA_TA_BUSY,16) descr="TA block is busy. Max over TA instances."></metric>
<metric name="TA_BUSY_min" expr=min(TA_TA_BUSY,16) descr="TA block is busy. Min over TA instances."></metric>
@@ -325,7 +327,7 @@
<metric name="TA_BUFFER_ATOMIC_WAVEFRONTS_sum" expr=sum(TA_BUFFER_ATOMIC_WAVEFRONTS,16) descr="Number of buffer atomic wavefronts processed by TA. Sum over TA instances."></metric>
<metric name="TA_BUFFER_TOTAL_CYCLES_sum" expr=sum(TA_BUFFER_TOTAL_CYCLES,16) descr="Number of buffer cycles issued to TC. Sum over TA instances."></metric>
<metric name="TA_BUFFER_COALESCED_READ_CYCLES_sum" expr=sum(TA_BUFFER_COALESCED_READ_CYCLES,16) descr="Number of buffer coalesced read cycles issued to TC. Sum over TA instances."></metric>
<metric name="TA_BUFFER_COALESCED_WRITE_CYCLES_sum" expr=sum(TA_BUFFER_COALESCED_WRITE_CYCLES,16) descr="Number of buffer coalesced write cycles issued to TC. Sum over TA instances."></metric>
<metric name="TA_BUFFER_COALESCED_WRITE_CYCLES_sum" expr=sum(TA_BUFFER_COALESCED_WRITE_CYCLES,16) descr="Number of buffer coalesced write cycles issued to TC. Sum over TA instances."></metric>
<metric name="TD_TD_BUSY_sum" expr=sum(TD_TD_BUSY,16) descr="TD is processing or waiting for data. Perf_Windowing not supported for this counter. Sum over TD instances."></metric>
<metric name="TD_TC_STALL_sum" expr=sum(TD_TC_STALL,16) descr="TD is stalled waiting for TC data. Sum over TD instances."></metric>
<metric name="TD_LOAD_WAVEFRONT_sum" expr=sum(TD_LOAD_WAVEFRONT,16) descr="Count the wavefronts with opcode = load, include atomics and store. Sum over TD instances."></metric>
@@ -426,6 +428,7 @@
</gfx940_expr>
<gfx10_expr>
<metric name="KERNEL_DURATION" expr=1 descr="The duration of the kernel dispatch"></metric>
<metric name="MeanOccupancyPerCU" expr=GRBM_COUNT*0+SQ_LEVEL_WAVES*0+SQ_ACCUM_PREV/GRBM_GUI_ACTIVE/CU_NUM descr="Mean occupancy per compute unit."></metric>
<metric name="MeanOccupancyPerActiveCU" expr=GRBM_COUNT*0+SQ_LEVEL_WAVES*0+SQ_ACCUM_PREV*4/SQ_BUSY_CYCLES/CU_NUM descr="Mean occupancy per active compute unit."></metric>
@@ -470,6 +473,7 @@
</gfx1032_expr>
<gfx11_expr>
<metric name="KERNEL_DURATION" expr=1 descr="The duration of the kernel dispatch"></metric>
<metric name="GPU_UTIL" expr=100*GRBM_GUI_ACTIVE/GRBM_COUNT descr="Percentage of the time that GUI is active"></metric>
<metric name="WAVE_DEP_WAIT" expr=100*SQ_WAIT_ANY/SQ_WAVE_CYCLES descr="Percentage of the SQ_WAVE_CYCLE time spent waiting for anything."></metric>
<metric name="WAVE_ISSUE_WAIT" expr=100*SQ_WAIT_INST_ANY/SQ_WAVE_CYCLES descr="Percentage of the SQ_WAVE_CYCLE time spent waiting for any instruction issue."></metric>
projects/rocprofiler/src/core/counters/metrics/eval_metrics.cpp
+13 -3
ファイルの表示
@@ -108,6 +108,12 @@ bool metrics::ExtractMetricEvents(
// adding result object for derived metric
std::lock_guard<std::mutex> lock(extract_metric_events_lock);
if(metric_names[i].compare("KERNEL_DURATION")==0) {
if (results_map.find(metric_names[i]) == results_map.end()) {
results_map[metric_names[i]] = new results_t(metric_names[i], {}, xcc_count);
}
continue;
}
counters_vec = metric->GetCounters();
if (counters_vec.empty())
rocprofiler::fatal("bad metric '%s' is empty", metric_names[i].c_str());
@@ -178,7 +184,7 @@ bool metrics::GetCounterData(hsa_ven_amd_aqlprofile_profile_t* profile, hsa_agen
}
bool metrics::GetMetricsData(std::map<std::string, results_t*>& results_map,
std::vector<const Metric*>& metrics_list) {
std::vector<const Metric*>& metrics_list, uint64_t kernel_duration) {
MetricArgs<std::map<std::string, results_t*>> args(&results_map);
for (auto& metric : metrics_list) {
const xml::Expr* expr = metric->GetExpr();
@@ -186,6 +192,10 @@ bool metrics::GetMetricsData(std::map<std::string, results_t*>& results_map,
auto it = results_map.find(metric->GetName());
if (it == results_map.end()) rocprofiler::fatal("metric results not found ");
results_t* res = it->second;
if(metric->GetName().compare("KERNEL_DURATION") == 0) {
res->val_double = kernel_duration;
continue;
}
res->val_double = expr->Eval(args);
}
}
@@ -196,7 +206,7 @@ bool metrics::GetMetricsData(std::map<std::string, results_t*>& results_map,
void metrics::GetCountersAndMetricResultsByXcc(uint32_t xcc_index,
std::vector<results_t*>& results_list,
std::map<std::string, results_t*>& results_map,
std::vector<const Metric*>& metrics_list) {
std::vector<const Metric*>& metrics_list, uint64_t kernel_duration) {
for (auto it = results_list.begin(); it != results_list.end(); it++) {
(*it)->val_double =
(*it)->xcc_vals[xcc_index]; // set val_double to hold value for specific xcc
@@ -207,5 +217,5 @@ void metrics::GetCountersAndMetricResultsByXcc(uint32_t xcc_index,
it->second->xcc_vals[xcc_index]; // set val_double to hold value for specific xcc
}
GetMetricsData(results_map, metrics_list);
GetMetricsData(results_map, metrics_list, kernel_duration);
}
projects/rocprofiler/src/core/counters/metrics/eval_metrics.h
+4 -4
ファイルの表示
@@ -37,7 +37,7 @@ typedef std::vector<double> xcc_results_t;
class results_t{
public:
results_t(std::string in_name, event_t in_event, uint32_t xcc_count):
results_t(std::string in_name, event_t in_event, uint32_t xcc_count):
name(in_name), val_double(0), event(in_event) {
xcc_vals.resize(xcc_count);
std::fill(xcc_vals.begin(), xcc_vals.end(), 0);
@@ -75,11 +75,11 @@ bool GetCounterData(hsa_ven_amd_aqlprofile_profile_t* profile, hsa_agent_t gpu_a
std::vector<results_t*>& results_list);
bool GetMetricsData(std::map<std::string, results_t*>& results_map,
std::vector<const Metric*>& metrics_list);
std::vector<const Metric*>& metrics_list, uint64_t kernel_duration = 0);
void GetCountersAndMetricResultsByXcc(uint32_t xcc_index, std::vector<results_t*>& results_list,
std::map<std::string, results_t*>& results_map,
std::vector<const Metric*>& metrics_list);
std::map<std::string, results_t*>& results_map,
std::vector<const Metric*>& metrics_list, uint64_t kernel_duration = 0);
} // namespace metrics
} // namespace rocprofiler
projects/rocprofiler/src/core/counters/metrics/metrics.h
+1
ファイルの表示
@@ -213,6 +213,7 @@ class MetricsDict {
xml_->AddConst("top.const.metric", "SIMD_NUM",
agent_info->getSimdCountPerCU() * agent_info->getCUCount());
xml_->AddConst("top.const.metric", "SE_NUM", agent_info->getShaderEngineCount());
xml_->AddConst("top.const.metric", "LDS_BANKS", 32);
ImportMetrics(agent_info, "const");
agent_name_ = agent_info->getName();
projects/rocprofiler/src/core/hsa/packets/packets_generator.cpp
+86 -73
ファイルの表示
@@ -181,19 +181,17 @@ InitializeAqlPackets(hsa_agent_t cpu_agent, hsa_agent_t gpu_agent,
// TODO: validate needs to be called on each events_list[i]
// Validating the events array for the specified gpu agent
bool validate_event_result;
status =
hsa_ven_amd_aqlprofile_validate_event(gpu_agent, &events_list[0], &validate_event_result);
CHECK_HSA_STATUS("Error: Validating Counters", status);
if (!validate_event_result) {
std::cerr << "Error: Events are not valid for the current gpu agent" << std::endl;
abort();
if(events_list.size() > 0) {
bool validate_event_result;
status =
hsa_ven_amd_aqlprofile_validate_event(gpu_agent, &events_list[0], &validate_event_result);
CHECK_HSA_STATUS("Error: Validating Counters", status);
if (!validate_event_result) {
std::cerr << "Error: Events are not valid for the current gpu agent" << std::endl;
abort();
}
}
std::vector<std::pair<rocprofiler::profiling_context_t*, hsa_ven_amd_aqlprofile_profile_t*>>
profiles = std::vector<
std::pair<rocprofiler::profiling_context_t*, hsa_ven_amd_aqlprofile_profile_t*>>();
// do {
rocprofiler::profiling_context_t* context = new rocprofiler::profiling_context_t();
context->gpu_agent = gpu_agent;
@@ -236,6 +234,14 @@ InitializeAqlPackets(hsa_agent_t cpu_agent, hsa_agent_t gpu_agent,
}
}
for(auto& cname : counter_names) {
if(cname.compare("KERNEL_DURATION")==0) {
rocprofiler::Metric* metric = const_cast<rocprofiler::Metric*>(metricsDict[gpu_agent.handle]->Get(cname));
if (metric == nullptr) std::cout << cname << " not found in metricsDict\n";
context->metrics_list.push_back(metric);
}
}
std::set<std::string> metrics_taken;
for (auto result : results_map) {
@@ -282,10 +288,15 @@ InitializeAqlPackets(hsa_agent_t cpu_agent, hsa_agent_t gpu_agent,
packet_t* stop_packet = new packet_t();
packet_t* read_packet = new packet_t();
if (context->events_list.size() <= 0) {
std::cerr << "Error: No events to profile" << std::endl;
abort();
}
std::vector<std::pair<rocprofiler::profiling_context_t*, hsa_ven_amd_aqlprofile_profile_t*>>
profiles = std::vector<
std::pair<rocprofiler::profiling_context_t*, hsa_ven_amd_aqlprofile_profile_t*>>();
// if (context->events_list.size() <= 0) {
// std::cerr << "Error: No events to profile" << std::endl;
// abort();
// }
// Preparing the profile structure to get the packets
hsa_ven_amd_aqlprofile_event_type_t profile_type = HSA_VEN_AMD_AQLPROFILE_EVENT_TYPE_PMC;
@@ -304,83 +315,85 @@ InitializeAqlPackets(hsa_agent_t cpu_agent, hsa_agent_t gpu_agent,
hsa_agent_t ag_list[ag_list_count];
ag_list[0] = gpu_agent;
// Preparing an Getting the size of the command and output buffers
status = hsa_ven_amd_aqlprofile_start(profile, NULL);
// CHECK_HSA_STATUS("Error: Getting Buffers Size", status);
if(context->events_list.size() > 0) {
// Preparing an Getting the size of the command and output buffers
status = hsa_ven_amd_aqlprofile_start(profile, NULL);
// CHECK_HSA_STATUS("Error: Getting Buffers Size", status);
if (profile->command_buffer.size > 0 && profile->output_buffer.size > 0) {
status = HSA_STATUS_ERROR;
size_t size = profile->command_buffer.size;
size = (size + MEM_PAGE_MASK) & ~MEM_PAGE_MASK;
if (size <= 0) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: Command buffer given size is " << size << std::endl;
abort();
}
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_memory_pool_allocate_fn(
agentInfo.cpu_pool, size, 0, reinterpret_cast<void**>(&(profile->command_buffer.ptr)));
if (status != HSA_STATUS_SUCCESS) {
profile->command_buffer.ptr = malloc(size);
/*numa_alloc_onnode(
size,
rocprofiler::hsa_support::GetAgentInfo(agentInfo.getNearCpuAgent().handle).getNumaNode());*/
if (profile->command_buffer.ptr == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: allocating memory for command buffer using NUMA" << std::endl;
abort();
}
} else {
// Both the CPU and GPU can access the memory
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_agents_allow_access_fn(
ag_list_count, ag_list, NULL, profile->command_buffer.ptr);
CHECK_HSA_STATUS("Error: Allowing access to Command Buffer", status);
}
if (!is_spm) {
if (profile->command_buffer.size > 0 && profile->output_buffer.size > 0) {
status = HSA_STATUS_ERROR;
size_t size = profile->output_buffer.size;
size_t size = profile->command_buffer.size;
size = (size + MEM_PAGE_MASK) & ~MEM_PAGE_MASK;
if (size <= 0) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: Output buffer given size is " << size << std::endl;
<< "Error: Command buffer given size is " << size << std::endl;
abort();
}
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_memory_pool_allocate_fn(
agentInfo.kernarg_pool, size, 0, reinterpret_cast<void**>(&profile->output_buffer.ptr));
agentInfo.cpu_pool, size, 0, reinterpret_cast<void**>(&(profile->command_buffer.ptr)));
if (status != HSA_STATUS_SUCCESS) {
profile->output_buffer.ptr = malloc(size);
profile->command_buffer.ptr = malloc(size);
/*numa_alloc_onnode(
size,
rocprofiler::hsa_support::GetAgentInfo(agentInfo.getNearCpuAgent().handle)
.getNumaNode());*/
if (profile->output_buffer.ptr == NULL) {
rocprofiler::hsa_support::GetAgentInfo(agentInfo.getNearCpuAgent().handle).getNumaNode());*/
if (profile->command_buffer.ptr == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: allocating memory for output buffer using NUMA" << std::endl;
<< "Error: allocating memory for command buffer using NUMA" << std::endl;
abort();
}
} else {
// Both the CPU and GPU can access the memory
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_agents_allow_access_fn(
ag_list_count, ag_list, NULL, profile->output_buffer.ptr);
CHECK_HSA_STATUS("Error: GPU Agent can't have output buffer access", status);
memset(profile->output_buffer.ptr, 0x0, profile->output_buffer.size);
ag_list_count, ag_list, NULL, profile->command_buffer.ptr);
CHECK_HSA_STATUS("Error: Allowing access to Command Buffer", status);
}
} else {
profile->output_buffer.size = 0;
if (!is_spm) {
status = HSA_STATUS_ERROR;
size_t size = profile->output_buffer.size;
size = (size + MEM_PAGE_MASK) & ~MEM_PAGE_MASK;
if (size <= 0) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: Output buffer given size is " << size << std::endl;
abort();
}
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_memory_pool_allocate_fn(
agentInfo.kernarg_pool, size, 0, reinterpret_cast<void**>(&profile->output_buffer.ptr));
if (status != HSA_STATUS_SUCCESS) {
profile->output_buffer.ptr = malloc(size);
/*numa_alloc_onnode(
size,
rocprofiler::hsa_support::GetAgentInfo(agentInfo.getNearCpuAgent().handle)
.getNumaNode());*/
if (profile->output_buffer.ptr == NULL) {
std::cerr << __FILE__ << ":" << __LINE__ << " "
<< "Error: allocating memory for output buffer using NUMA" << std::endl;
abort();
}
} else {
status = rocprofiler::hsa_support::GetAmdExtTable().hsa_amd_agents_allow_access_fn(
ag_list_count, ag_list, NULL, profile->output_buffer.ptr);
CHECK_HSA_STATUS("Error: GPU Agent can't have output buffer access", status);
memset(profile->output_buffer.ptr, 0x0, profile->output_buffer.size);
}
} else {
profile->output_buffer.size = 0;
}
status = hsa_ven_amd_aqlprofile_start(profile, start_packet);
// CHECK_HSA_STATUS("Error: Creating Start Packet\n", status);
status = hsa_ven_amd_aqlprofile_stop(profile, stop_packet);
// CHECK_HSA_STATUS("Error: Creating Stop Packet\n", status);
status = hsa_ven_amd_aqlprofile_read(profile, read_packet);
// CHECK_HSA_STATUS("Error: Creating Read Packet\n", status);
context->start_packet = start_packet;
context->stop_packet = stop_packet;
context->read_packet = read_packet;
}
status = hsa_ven_amd_aqlprofile_start(profile, start_packet);
// CHECK_HSA_STATUS("Error: Creating Start Packet\n", status);
status = hsa_ven_amd_aqlprofile_stop(profile, stop_packet);
// CHECK_HSA_STATUS("Error: Creating Stop Packet\n", status);
status = hsa_ven_amd_aqlprofile_read(profile, read_packet);
// CHECK_HSA_STATUS("Error: Creating Read Packet\n", status);
context->start_packet = start_packet;
context->stop_packet = stop_packet;
context->read_packet = read_packet;
// add profiles
profiles.emplace_back(std::make_pair(context, profile));
}
// add profiles
profiles.emplace_back(std::make_pair(context, profile));
// } while (events_list.size() > 0);
return profiles;
}
projects/rocprofiler/src/core/hsa/queues/queue.cpp
+12 -15
ファイルの表示
@@ -432,18 +432,17 @@ bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) {
if (pending->session_id.handle == 0) {
pending->session_id = GetROCProfilerSingleton()->GetCurrentSessionId();
}
if (pending->counters_count > 0 && pending->context->metrics_list.size() > 0 &&
pending->profile) {
if (xcc_id == 0) // call to GetCounterData() is required only once for a dispatch
if (pending->counters_count > 0) {
if (xcc_id == 0 && pending->context && pending->context->metrics_list.size() > 0 && pending->profile) // call to GetCounterData() is required only once for a dispatch
rocprofiler::metrics::GetCounterData(pending->profile, queue_info_session->agent,
pending->context->results_list);
if (is_individual_xcc_mode)
rocprofiler::metrics::GetCountersAndMetricResultsByXcc(
xcc_id, pending->context->results_list, pending->context->results_map,
pending->context->metrics_list);
pending->context->metrics_list, time.end-time.start);
else
rocprofiler::metrics::GetMetricsData(pending->context->results_map,
pending->context->metrics_list);
pending->context->metrics_list, time.end-time.start);
AddRecordCounters(&record, pending);
} else {
if (session->FindBuffer(pending->buffer_id)) {
@@ -721,7 +720,7 @@ void WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t user_pkt
uint32_t writer_id = WRITER_ID.fetch_add(1, std::memory_order_release);
if (session_data_count > 0 && is_counter_collection_mode && profiles.size() > 0 &&
replay_mode_count > 0) {
replay_mode_count > 0 && profile.first && profile.first->start_packet) {
// Adding start packet and its barrier with a dummy signal
hsa_signal_t dummy_signal{};
dummy_signal.handle = 0;
@@ -745,18 +744,16 @@ void WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t user_pkt
uint64_t record_id = GetROCProfilerSingleton()->GetUniqueRecordId();
AddKernelNameWithDispatchID(GetKernelNameFromKsymbols(dispatch_packet.kernel_object),
record_id);
if (profiles.size() > 0 && replay_mode_count > 0) {
if (session_data_count > 0 && profile.second) {
session->GetProfiler()->AddPendingSignals(
writer_id, record_id, original_packet.completion_signal,
dispatch_packet.completion_signal, session_id_snapshot, buffer_id, profile.first,
profile.first->metrics_list.size(), profile.second, kernel_properties,
writer_id, record_id, original_packet.completion_signal, dispatch_packet.completion_signal, session_id, buffer_id,
profile.first, session_data_count, profile.second, kernel_properties,
(uint32_t)syscall(__NR_gettid), user_pkt_index, correlation_id);
} else {
session->GetProfiler()->AddPendingSignals(
writer_id, record_id, original_packet.completion_signal,
dispatch_packet.completion_signal, session_id_snapshot, buffer_id, nullptr, 0,
nullptr, kernel_properties, (uint32_t)syscall(__NR_gettid), user_pkt_index,
correlation_id);
writer_id, record_id, original_packet.completion_signal, dispatch_packet.completion_signal, session_id, buffer_id,
nullptr, session_data_count, nullptr, kernel_properties, (uint32_t)syscall(__NR_gettid),
user_pkt_index, correlation_id);
}
}
@@ -776,7 +773,7 @@ void WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t user_pkt
CreateSignal(0, &interrupt_signal);
// Adding Stop and Read PM4 Packets
if (session_data_count > 0 && is_counter_collection_mode) {
if (session_data_count > 0 && is_counter_collection_mode && profiles.size() > 0 && profile.first && profile.first->stop_packet) {
hsa_signal_t dummy_signal{};
profile.first->stop_packet->header = HSA_PACKET_TYPE_VENDOR_SPECIFIC
<< HSA_PACKET_HEADER_TYPE;