AI/rocm-systems
2
0
Rozštěpit 0
Zdrojový kód Úkoly Pull requesty Akce Balíčky Projekty Vydání Wiki Aktivita

SWDEV-391589: Fixing File plugin output text

Procházet zdrojové kódy 
Change-Id: I5e1d46431ca2b93661772b062996ed62574c36f9
Tento commit je obsažen v:
Ammar ELWazir
2023-05-19 17:53:17 +00:00
odevzdal Ammar Elwazir
rodič d11ebc6da6
revize d3c39dca88
24 změnil soubory, kde provedl 1135 přidání a 546 odebrání
Stáhněte soubor záplaty Stáhněte rozdílový soubor Rozbalit všechny soubory Sbalit všechny soubory
plugin/file/file.cpp
+240 -131
Zobrazit soubor
@@ -26,6 +26,7 @@
#include <sys/types.h>
#include <unistd.h>
#include <atomic>
#include <cassert>
#include <cstddef>
#include <cstdint>
@@ -49,6 +50,36 @@ namespace fs = std::experimental::filesystem;
namespace {
std::vector<std::string> GetCounterNames() {
std::vector<std::string> counters;
const char* line_c_str = getenv("ROCPROFILER_COUNTERS");
if (line_c_str) {
std::string line = line_c_str;
// skip commented lines
auto found = line.find_first_not_of(" \t");
if (found != std::string::npos) {
if (line[found] == '#') return {};
}
if (line.find("pmc") == std::string::npos) return counters;
char seperator = ' ';
std::string::size_type prev_pos = 0, pos = line.find(seperator, prev_pos);
prev_pos = ++pos;
if (pos != std::string::npos) {
while ((pos = line.find(seperator, pos)) != std::string::npos) {
std::string substring(line.substr(prev_pos, pos - prev_pos));
if (substring.length() > 0 && substring != ":") {
counters.push_back(substring);
}
prev_pos = ++pos;
}
if (!line.substr(prev_pos, pos - prev_pos).empty()) {
counters.push_back(line.substr(prev_pos, pos - prev_pos));
}
}
}
return counters;
}
static std::string output_file_name;
class file_plugin_t {
private:
@@ -78,7 +109,7 @@ class file_plugin_t {
if (fail()) return;
const char* output_dir = getenv("OUTPUT_PATH");
output_file_name = getenv("OUT_FILE_NAME") ? std::string(getenv("OUT_FILE_NAME")) + "_" : "";
output_file_name = getenv("OUT_FILE_NAME") ? std::string(getenv("OUT_FILE_NAME")) : "";
if (output_dir == nullptr && getenv("OUT_FILE_NAME") == nullptr) {
stream_.copyfmt(std::cout);
@@ -96,10 +127,12 @@ class file_plugin_t {
return;
}
std::stringstream ss;
output_file_name = replace_MPI_macros(output_file_name);
ss << output_file_name << GetPid() << "_" << name_;
std::stringstream ss;
ss << name_ << "_" << ((output_file_name.empty()) ? std::to_string(GetPid()) : "")
<< output_file_name << ".csv";
std::cout << "Results File: " << output_prefix / ss.str() << std::endl;
stream_.open(output_prefix / ss.str());
}
@@ -163,45 +196,90 @@ class file_plugin_t {
}
public:
file_plugin_t() {
output_file_t hsa_handles("hsa_handles.txt", true);
[[maybe_unused]] hsa_status_t status = hsa_iterate_agents(
[](hsa_agent_t agent, void* user_data) {
auto* file = static_cast<decltype(hsa_handles)*>(user_data);
hsa_device_type_t type;
if (hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &type) != HSA_STATUS_SUCCESS)
return HSA_STATUS_ERROR;
if (!file->isStdOut())
*file << std::hex << std::showbase << agent.handle << " agent "
<< ((type == HSA_DEVICE_TYPE_CPU) ? "cpu" : "gpu") << std::endl;
return HSA_STATUS_SUCCESS;
},
&hsa_handles);
assert(status == HSA_STATUS_SUCCESS && "failed to iterate HSA agents");
if (hsa_handles.fail()) {
rocprofiler::warning("Cannot write to '%s'", hsa_handles.name().c_str());
return;
}
// App begin timestamp begin_ts_file.txt
output_file_t begin_ts("begin_ts_file.txt", true);
[[maybe_unused]] rocprofiler_timestamp_t app_begin_timestamp = {};
CHECK_ROCPROFILER(rocprofiler_get_timestamp(&app_begin_timestamp));
if (!begin_ts.isStdOut()) begin_ts << std::dec << app_begin_timestamp.value << std::endl;
if (begin_ts.fail()) {
rocprofiler::warning("Cannot write to '%s'", begin_ts.name().c_str());
return;
}
file_plugin_t(void* data) {
if (data) counter_names_ = GetCounterNames();
valid_ = true;
}
void WriteHeader(output_type_t type, rocprofiler_tracer_activity_domain_t domain) {
output_file_t* output_file;
switch (domain) {
case ACTIVITY_DOMAIN_HSA_API: {
if (hsa_api_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::TRACER, ACTIVITY_DOMAIN_HSA_API);
*output_file << "Record_ID,Domain,Function,Start_Timestamp,End_Timestamp,Correlation_ID"
<< std::endl;
*output_file << std::endl;
hsa_api_header_written_.exchange(true, std::memory_order_release);
return;
}
case ACTIVITY_DOMAIN_HIP_API: {
if (hip_api_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::TRACER, ACTIVITY_DOMAIN_HIP_API);
*output_file
<< "Record_ID,Domain,Function,Kernel_Name,Start_Timestamp,End_Timestamp,Correlation_ID"
<< std::endl;
*output_file << std::endl;
hip_api_header_written_.exchange(true, std::memory_order_release);
return;
}
case ACTIVITY_DOMAIN_ROCTX: {
if (roctx_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::TRACER, ACTIVITY_DOMAIN_ROCTX);
*output_file << "Record_ID,Domain,ROCTX_ID,Message,Timestamp" << std::endl;
*output_file << std::endl;
roctx_header_written_.exchange(true, std::memory_order_release);
return;
}
case ACTIVITY_DOMAIN_HSA_OPS: {
if (hsa_async_copy_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::TRACER, ACTIVITY_DOMAIN_HSA_OPS);
*output_file << "Record_ID,Domain,Operation,Start_Timestamp,Stop_Timestamp,Correlation_ID"
<< std::endl;
*output_file << std::endl;
hsa_async_copy_header_written_.exchange(true, std::memory_order_release);
return;
}
case ACTIVITY_DOMAIN_HIP_OPS: {
if (hip_activity_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::TRACER, ACTIVITY_DOMAIN_HIP_OPS);
*output_file << "Record_ID,Domain,Operation,Kernel_Name,Start_Timestamp,Stop_Timestamp,"
"Correlation_ID"
<< std::endl;
*output_file << std::endl;
hip_activity_header_written_.exchange(true, std::memory_order_release);
return;
}
default: {
if (type == output_type_t::COUNTER) {
if (kernel_dispatches_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::COUNTER);
*output_file
<< "Dispatch_ID,GPU_ID,Queue_ID,Queue_Index,PID,TID,GRD,WGR,LDS,SCR,Arch_VGPR,"
"ACCUM_VGPR,SGPR,Wave_Size,SIG,OBJ,Kernel_Name,Start_Timestamp,End_Timestamp";
if (counter_names_.size() > 0) {
for (uint32_t i = 0; i < counter_names_.size(); i++)
*output_file << "," << counter_names_[i];
}
*output_file << std::endl;
*output_file << std::endl;
kernel_dispatches_header_written_.exchange(true, std::memory_order_release);
return;
} else if (type == output_type_t::PC_SAMPLING) {
if (pc_sample_header_written_.load(std::memory_order_relaxed)) return;
output_file = get_output_file(output_type_t::PC_SAMPLING);
*output_file << "Dispatch_ID,Timestamp,GPU_ID,PC_Sample,Shader_Engines" << std::endl;
*output_file << std::endl;
pc_sample_header_written_.exchange(true, std::memory_order_release);
return;
}
return;
}
}
}
std::mutex writing_lock;
const char* GetDomainName(rocprofiler_tracer_activity_domain_t domain) {
@@ -235,13 +313,21 @@ class file_plugin_t {
std::lock_guard<std::mutex> lock(writing_lock);
if (tracer_record.timestamps.end.value <= 0 && tracer_record.domain != ACTIVITY_DOMAIN_ROCTX)
return;
WriteHeader(output_type_t::TRACER, tracer_record.domain);
std::string function_name;
std::string kernel_name;
std::string roctx_message;
uint64_t roctx_id;
if (tracer_record.name) {
if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_API)
if ((tracer_record.operation_id.id == 0 && tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS)) {
if (tracer_record.name) {
kernel_name = rocprofiler::cxx_demangle(tracer_record.name);
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX) roctx_message = tracer_record.name;
std::string key = "\"";
std::size_t found = kernel_name.rfind(key);
while (found != std::string::npos) {
kernel_name.replace(found, key.length(), "'");
found = kernel_name.rfind(key, found - 1);
}
}
}
size_t function_name_size = 0;
char* function_name_c = nullptr;
@@ -253,6 +339,7 @@ class file_plugin_t {
CHECK_ROCPROFILER(rocprofiler_query_hsa_tracer_api_data_info(
rocprofiler_session_id_t{0}, ROCPROFILER_HSA_FUNCTION_NAME,
tracer_record.api_data_handle, tracer_record.operation_id, &function_name_c));
if (function_name_c) function_name = std::string(function_name_c);
}
}
if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_API) {
@@ -261,51 +348,87 @@ class file_plugin_t {
tracer_record.operation_id, &function_name_size));
if (function_name_size > 1) {
CHECK_ROCPROFILER(rocprofiler_query_hip_tracer_api_data_info(
rocprofiler_session_id_t{0}, ROCPROFILER_HIP_FUNCTION_NAME,
tracer_record.api_data_handle, tracer_record.operation_id, &function_name_c));
session_id, ROCPROFILER_HIP_FUNCTION_NAME, tracer_record.api_data_handle,
tracer_record.operation_id, &function_name_c));
if (function_name_c) function_name = std::string(function_name_c);
}
}
output_file_t* output_file = get_output_file(output_type_t::TRACER, tracer_record.domain);
*output_file << "Record(" << tracer_record.header.id.handle << "), Domain("
<< GetDomainName(tracer_record.domain) << "),";
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX && roctx_id >= 0)
*output_file << " ROCTX_ID(" << tracer_record.operation_id.id << "),";
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX && tracer_record.name)
*output_file << " ROCTX_Message(" << reinterpret_cast<const char*>(tracer_record.name)
<< "),";
if (function_name_c) *output_file << " Function(" << function_name_c << "),";
if (kernel_name.size() > 1) *output_file << " Kernel_Name(" << kernel_name.c_str() << "),";
if (tracer_record.domain == ACTIVITY_DOMAIN_HSA_OPS ||
tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS) {
switch (tracer_record.operation_id.id) {
case 0:
*output_file << " Operation(DISPATCH_OP),";
break;
case 1:
*output_file << " Operation(COPY_OP),";
break;
case 2:
*output_file << " Operation(BARRIER_OP),";
break;
default:
break;
if (tracer_record.name) {
kernel_name = rocprofiler::cxx_demangle(std::string(tracer_record.name));
std::string key = "\"";
std::size_t found = kernel_name.rfind(key);
while (found != std::string::npos) {
kernel_name.replace(found, key.length(), "'");
found = kernel_name.rfind(key, found - 1);
}
// TODO: Change how this API returns a string.
}
}
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX) {
*output_file << " timestamp(" << tracer_record.timestamps.begin.value << ")";
} else if (tracer_record.phase == ROCPROFILER_PHASE_EXIT ||
tracer_record.phase == ROCPROFILER_PHASE_NONE) {
*output_file << " Begin(" << tracer_record.timestamps.begin.value << "), End("
<< tracer_record.timestamps.end.value << ")";
if (tracer_record.name) roctx_message = rocprofiler::cxx_demangle(tracer_record.name);
roctx_id = tracer_record.operation_id.id;
}
if (tracer_record.domain != ACTIVITY_DOMAIN_ROCTX)
*output_file << ", Correlation_ID(" << tracer_record.correlation_id.value << ")";
*output_file << '\n';
char* activity_name = nullptr;
if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS) {
if (tracer_record.api_data_handle.handle) {
kernel_name = rocprofiler::cxx_demangle(
const_cast<char*>(reinterpret_cast<const char*>(tracer_record.api_data_handle.handle)));
}
size_t activity_name_size = 0;
CHECK_ROCPROFILER(rocprofiler_query_hip_tracer_api_data_info_size(
session_id, ROCPROFILER_HIP_ACTIVITY_NAME, tracer_record.api_data_handle,
tracer_record.operation_id, &activity_name_size));
if (activity_name_size > 1) {
activity_name = nullptr;
CHECK_ROCPROFILER(rocprofiler_query_hip_tracer_api_data_info(
session_id, ROCPROFILER_HIP_ACTIVITY_NAME, tracer_record.api_data_handle,
tracer_record.operation_id, &activity_name));
}
}
if (tracer_record.domain == ACTIVITY_DOMAIN_HSA_OPS) {
size_t activity_name_size = 0;
CHECK_ROCPROFILER(rocprofiler_query_hsa_tracer_api_data_info_size(
session_id, ROCPROFILER_HSA_ACTIVITY_NAME, tracer_record.api_data_handle,
tracer_record.operation_id, &activity_name_size));
if (activity_name_size > 1) {
activity_name = nullptr;
CHECK_ROCPROFILER(rocprofiler_query_hsa_tracer_api_data_info(
session_id, ROCPROFILER_HSA_ACTIVITY_NAME, tracer_record.api_data_handle,
tracer_record.operation_id, &activity_name));
}
}
// return;
output_file_t* output_file = get_output_file(output_type_t::TRACER, tracer_record.domain);
*output_file << "" << tracer_record.header.id.handle << ","
<< GetDomainName(tracer_record.domain);
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX && roctx_id >= 0)
*output_file << "," << roctx_id;
if (tracer_record.domain == ACTIVITY_DOMAIN_ROCTX) {
if (roctx_message.size() > 1)
*output_file << ",\"" << roctx_message << "\"";
else
*output_file << ",";
}
if (function_name.size() > 1) *output_file << ",\"" << function_name << "\"";
if (activity_name) *output_file << ",\"" << activity_name << "\"";
if (kernel_name.size() > 1)
*output_file << ",\"" << kernel_name.c_str() << "\"";
else if (tracer_record.domain == ACTIVITY_DOMAIN_HIP_API ||
tracer_record.domain == ACTIVITY_DOMAIN_HIP_OPS)
*output_file << ",";
if (tracer_record.domain != ACTIVITY_DOMAIN_ROCTX) {
*output_file << "," << tracer_record.timestamps.begin.value << ","
<< tracer_record.timestamps.end.value;
*output_file << "," << tracer_record.correlation_id.value;
} else {
*output_file << "," << tracer_record.timestamps.begin.value;
}
*output_file << std::endl;
}
void FlushProfilerRecord(const rocprofiler_record_profiler_t* profiler_record,
rocprofiler_session_id_t session_id, rocprofiler_buffer_id_t buffer_id) {
std::lock_guard<std::mutex> lock(writing_lock);
WriteHeader(output_type_t::COUNTER, ACTIVITY_DOMAIN_NUMBER);
size_t name_length = 0;
output_file_t* output_file{nullptr};
output_file = get_output_file(output_type_t::COUNTER);
@@ -318,62 +441,42 @@ class file_plugin_t {
CHECK_ROCPROFILER(rocprofiler_query_kernel_info(ROCPROFILER_KERNEL_NAME,
profiler_record->kernel_id, &kernel_name_c));
}
*output_file << std::string("dispatch[") << std::to_string(profiler_record->header.id.handle)
<< "], " << std::string("gpu_id(")
<< std::to_string(profiler_record->gpu_id.handle) << "), "
<< std::string("queue_id(") << std::to_string(profiler_record->queue_id.handle)
<< "), " << std::string("queue_index(")
<< std::to_string(profiler_record->queue_idx.value) << "), " << std::string("pid(")
<< std::to_string(GetPid()) << "), " << std::string("tid(")
<< std::to_string(profiler_record->thread_id.value) << ")";
*output_file << ", " << std::string("grd(")
<< std::to_string(profiler_record->kernel_properties.grid_size) << "), "
<< std::string("wgr(")
<< std::to_string(profiler_record->kernel_properties.workgroup_size) << "), "
<< std::string("lds(")
*output_file << std::to_string(profiler_record->header.id.handle) << ","
<< std::to_string(profiler_record->gpu_id.handle) << ","
<< std::to_string(profiler_record->queue_id.handle) << ","
<< std::to_string(profiler_record->queue_idx.value) << ","
<< std::to_string(GetPid()) << ","
<< std::to_string(profiler_record->thread_id.value);
*output_file << "," << std::to_string(profiler_record->kernel_properties.grid_size) << ","
<< std::to_string(profiler_record->kernel_properties.workgroup_size) << ","
<< std::to_string(
((profiler_record->kernel_properties.lds_size + (lds_block_size - 1)) &
~(lds_block_size - 1)))
<< "), " << std::string("scr(")
<< std::to_string(profiler_record->kernel_properties.scratch_size) << "), "
<< std::string("arch_vgpr(")
<< std::to_string(profiler_record->kernel_properties.arch_vgpr_count) << "), "
<< std::string("accum_vgpr(")
<< std::to_string(profiler_record->kernel_properties.accum_vgpr_count) << "), "
<< std::string("sgpr(")
<< std::to_string(profiler_record->kernel_properties.sgpr_count) << "), "
<< std::string("wave_size(")
<< std::to_string(profiler_record->kernel_properties.wave_size) << "), "
<< std::string("sig(")
<< "," << std::to_string(profiler_record->kernel_properties.scratch_size) << ","
<< std::to_string(profiler_record->kernel_properties.arch_vgpr_count) << ","
<< std::to_string(profiler_record->kernel_properties.accum_vgpr_count) << ","
<< std::to_string(profiler_record->kernel_properties.sgpr_count) << ","
<< std::to_string(profiler_record->kernel_properties.wave_size) << ","
<< std::to_string(profiler_record->kernel_properties.signal_handle);
std::string kernel_name = "";
if (name_length > 1) {
kernel_name = rocprofiler::truncate_name(rocprofiler::cxx_demangle(kernel_name_c));
kernel_name = rocprofiler::cxx_demangle(kernel_name_c);
std::string key = "\"";
std::size_t found = kernel_name.rfind(key);
while (found != std::string::npos) {
kernel_name.replace(found, key.length(), "'");
found = kernel_name.rfind(key, found - 1);
}
}
*output_file << "), " << std::string("obj(")
<< std::to_string(profiler_record->kernel_id.handle) << "), "
<< std::string("kernel-name(\"") << kernel_name << "\")"
<< std::string(", start_time(")
<< std::to_string(profiler_record->timestamps.begin.value) << ")"
<< std::string(", end_time(")
<< std::to_string(profiler_record->timestamps.end.value) << ")";
*output_file << "," << std::to_string(profiler_record->kernel_id.handle) << ",\"" << kernel_name
<< "\"," << std::to_string(profiler_record->timestamps.begin.value) << ","
<< std::to_string(profiler_record->timestamps.end.value);
// For Counters
if (profiler_record->counters) {
for (uint64_t i = 0; i < profiler_record->counters_count.value; i++) {
if (profiler_record->counters[i].counter_handler.handle > 0) {
size_t counter_name_length = 0;
CHECK_ROCPROFILER(rocprofiler_query_counter_info_size(
session_id, ROCPROFILER_COUNTER_NAME, profiler_record->counters[i].counter_handler,
&counter_name_length));
if (counter_name_length > 1) {
const char* name_c = nullptr;
CHECK_ROCPROFILER(rocprofiler_query_counter_info(
session_id, ROCPROFILER_COUNTER_NAME, profiler_record->counters[i].counter_handler,
&name_c));
*output_file << ", " << name_c << " ("
<< std::to_string(profiler_record->counters[i].value.value) << ')';
}
*output_file << "," << std::to_string(profiler_record->counters[i].value.value);
}
}
}
@@ -384,14 +487,13 @@ class file_plugin_t {
}
void FlushPCSamplingRecord(const rocprofiler_record_pc_sample_t* pc_sampling_record) {
WriteHeader(output_type_t::PC_SAMPLING, ACTIVITY_DOMAIN_NUMBER);
output_file_t* output_file{nullptr};
output_file = get_output_file(output_type_t::PC_SAMPLING);
const auto& sample = pc_sampling_record->pc_sample;
*output_file << "dispatch[" << sample.dispatch_id.value << "], "
<< "timestamp(" << sample.timestamp.value << "), "
<< "gpu_id(" << sample.gpu_id.handle << "), "
<< "pc-sample(" << std::hex << std::showbase << sample.pc << "), "
<< "se(" << sample.se << ')' << std::endl;
*output_file << sample.dispatch_id.value << "," << sample.timestamp.value << ","
<< sample.gpu_id.handle << "," << std::hex << std::showbase << sample.pc << ","
<< sample.se << std::endl;
}
int WriteBufferRecords(const rocprofiler_record_header_t* begin,
const rocprofiler_record_header_t* end,
@@ -432,11 +534,17 @@ class file_plugin_t {
private:
bool valid_{false};
std::vector<std::string> counter_names_;
output_file_t roctx_file_{"roctx_trace.txt"}, hsa_api_file_{"hsa_api_trace.txt"},
hip_api_file_{"hip_api_trace.txt"}, hip_activity_file_{"hcc_ops_trace.txt"},
hsa_async_copy_file_{"async_copy_trace.txt"}, pc_sample_file_{"pcs_trace.txt"},
output_file_{"results.txt"};
std::atomic<bool> roctx_header_written_{false}, hsa_api_header_written_{false},
hip_api_header_written_{false}, hip_activity_header_written_{false},
hsa_async_copy_header_written_{false}, pc_sample_header_written_{false},
kernel_dispatches_header_written_{false};
output_file_t roctx_file_{"roctx_trace"}, hsa_api_file_{"hsa_api_trace"},
hip_api_file_{"hip_api_trace"}, hip_activity_file_{"hcc_ops_trace"},
hsa_async_copy_file_{"async_copy_trace"}, pc_sample_file_{"pcs_trace"},
output_file_{"results"};
};
file_plugin_t* file_plugin = nullptr;
@@ -444,14 +552,15 @@ file_plugin_t* file_plugin = nullptr;
} // namespace
ROCPROFILER_EXPORT int rocprofiler_plugin_initialize(uint32_t rocprofiler_major_version,
uint32_t rocprofiler_minor_version) {
uint32_t rocprofiler_minor_version,
void* data) {
if (rocprofiler_major_version != ROCPROFILER_VERSION_MAJOR ||
rocprofiler_minor_version < ROCPROFILER_VERSION_MINOR)
return -1;
if (file_plugin != nullptr) return -1;
file_plugin = new file_plugin_t();
file_plugin = new file_plugin_t(data);
if (file_plugin->is_valid()) return 0;
// The plugin failed to initialized, destroy it and return an error.