diff --git a/projects/roctracer/plugin/CMakeLists.txt b/projects/roctracer/plugin/CMakeLists.txt index abcb287c39..0d0717c5d6 100644 --- a/projects/roctracer/plugin/CMakeLists.txt +++ b/projects/roctracer/plugin/CMakeLists.txt @@ -20,6 +20,4 @@ ## IN THE SOFTWARE. ################################################################################ -# Uncomment the following lines to add new plugin -# Replace with the name of the plugin -# add_subdirectory() \ No newline at end of file +add_subdirectory(file) \ No newline at end of file diff --git a/projects/roctracer/plugin/file/CMakeLists.txt b/projects/roctracer/plugin/file/CMakeLists.txt new file mode 100644 index 0000000000..78f75b5e99 --- /dev/null +++ b/projects/roctracer/plugin/file/CMakeLists.txt @@ -0,0 +1,42 @@ +################################################################################ +## Copyright (c) 2022 Advanced Micro Devices, Inc. +## +## Permission is hereby granted, free of charge, to any person obtaining a copy +## of this software and associated documentation files (the "Software"), to +## deal in the Software without restriction, including without limitation the +## rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +## sell copies of the Software, and to permit persons to whom the Software is +## furnished to do so, subject to the following conditions: +## +## The above copyright notice and this permission notice shall be included in +## all copies or substantial portions of the Software. +## +## THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +## IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +## FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +## AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +## LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +## FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +## IN THE SOFTWARE. +################################################################################ + +file(GLOB FILE_SOURCES "*.cpp") +add_library(file_plugin ${LIBRARY_TYPE} ${FILE_SOURCES}) + +set_target_properties(file_plugin PROPERTIES + CXX_VISIBILITY_PRESET hidden + LINK_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../exportmap + LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) + +target_compile_definitions(file_plugin + PRIVATE HIP_PROF_HIP_API_STRING=1 __HIP_PLATFORM_HCC__=1) + +target_include_directories(file_plugin PRIVATE ${PROJECT_SOURCE_DIR}/inc) + +target_link_options(file_plugin PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/../exportmap -Wl,--no-undefined) + +target_link_libraries(file_plugin PRIVATE roctracer hsa-runtime64::hsa-runtime64 stdc++fs) + +install(TARGETS file_plugin LIBRARY + DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME} + COMPONENT runtime) \ No newline at end of file diff --git a/projects/roctracer/plugin/file/file.cpp b/projects/roctracer/plugin/file/file.cpp new file mode 100644 index 0000000000..a9548dacf8 --- /dev/null +++ b/projects/roctracer/plugin/file/file.cpp @@ -0,0 +1,397 @@ +/* Copyright (c) 2022 Advanced Micro Devices, Inc. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +// Macro to check ROCtracer calls status +#define CHECK_ROCTRACER(call) \ + do { \ + if ((call) != 0) fatal("%s\n", roctracer_error_string()); \ + } while (false) + +namespace fs = std::experimental::filesystem; + +namespace { + +void warning(const char* format, ...) +#if defined(__GNUC__) + __attribute__((format(printf, 1, 2))) +#endif /* defined (__GNUC__) */ + ; + +void fatal [[noreturn]] (const char* format, ...) +#if defined(__GNUC__) +__attribute__((format(printf, 1, 2))) +#endif /* defined (__GNUC__) */ +; + +void warning(const char* format, ...) { + va_list va; + va_start(va, format); + vfprintf(stderr, format, va); + va_end(va); +} + +void fatal(const char* format, ...) { + va_list va; + va_start(va, format); + vfprintf(stderr, format, va); + va_end(va); + abort(); +} + +uint32_t GetPid() { + static uint32_t pid = syscall(__NR_getpid); + return pid; +} + +/* The function extracts the kernel name from +input string. By using the iterators it finds the +window in the string which contains only the kernel name. +For example 'Foo::foo(a[], int (int))' -> 'foo'*/ +std::string truncate_name(const std::string& name) { + auto rit = name.rbegin(); + auto rend = name.rend(); + uint32_t counter = 0; + char open_token = 0; + char close_token = 0; + while (rit != rend) { + if (counter == 0) { + switch (*rit) { + case ')': + counter = 1; + open_token = ')'; + close_token = '('; + break; + case '>': + counter = 1; + open_token = '>'; + close_token = '<'; + break; + case ']': + counter = 1; + open_token = ']'; + close_token = '['; + break; + case ' ': + ++rit; + continue; + } + if (counter == 0) break; + } else { + if (*rit == open_token) counter++; + if (*rit == close_token) counter--; + } + ++rit; + } + auto rbeg = rit; + while ((rit != rend) && (*rit != ' ') && (*rit != ':')) rit++; + return name.substr(rend - rit, rit - rbeg); +} + +// C++ symbol demangle +std::string cxx_demangle(const std::string& symbol) { + int status; + char* demangled = abi::__cxa_demangle(symbol.c_str(), nullptr, nullptr, &status); + if (status != 0) return symbol; + std::string ret(demangled); + free(demangled); + return ret; +} + +class file_plugin_t { + private: + class output_file_t { + public: + output_file_t(std::string name) : name_(std::move(name)) {} + ~output_file_t() { close(); } + + std::string path() const { + std::stringstream ss; + ss << GetPid() << "_" << name_; + return output_prefix_ / ss.str(); + } + + template output_file_t& operator<<(T&& value) { + if (!is_open()) open(); + stream() << std::forward(value); + return *this; + } + + output_file_t& operator<<(std::ostream& (*func)(std::ostream&)) { + if (!is_open()) open(); + stream() << func; + return *this; + } + + bool is_open() const { return output_prefix_.empty() || stream_.is_open(); } + bool fail() const { return stream().fail(); } + + void open() { + if (output_prefix_.empty()) return; + stream_.open(path()); + } + void close() { + if (stream_.is_open()) stream_.close(); + } + + private: + std::ostream& stream() { return output_prefix_.empty() ? std::cout : stream_; } + const std::ostream& stream() const { + return output_prefix_.empty() ? const_cast(std::cout) : stream_; + } + + const std::string name_; + std::ofstream stream_; + }; + + output_file_t* get_output_file(uint32_t domain, uint32_t op = 0) { + switch (domain) { + case ACTIVITY_DOMAIN_ROCTX: + return &roctx_file_; + case ACTIVITY_DOMAIN_HSA_API: + return &hsa_api_file_; + case ACTIVITY_DOMAIN_HIP_API: + case ACTIVITY_DOMAIN_EXT_API: + return &hip_api_file_; + case ACTIVITY_DOMAIN_HIP_OPS: + return &hip_activity_file_; + case ACTIVITY_DOMAIN_HSA_OPS: + if (op == HSA_OP_ID_COPY) { + return &hsa_async_copy_file_; + } else if (op == HSA_OP_ID_RESERVED1) { + return &pc_sample_file_; + } + default: + assert(!"domain/op not supported!"); + break; + } + return nullptr; + } + + public: + file_plugin_t() { + // Dumping HSA handles for agents + + output_file_t hsa_handles("hsa_handles.txt"); + + [[maybe_unused]] hsa_status_t status = hsa_iterate_agents( + [](hsa_agent_t agent, void* user_data) { + auto* file = static_cast(user_data); + hsa_device_type_t type; + + if (hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &type) != HSA_STATUS_SUCCESS) + return HSA_STATUS_ERROR; + + *file << std::hex << std::showbase << agent.handle << " agent " + << ((type == HSA_DEVICE_TYPE_CPU) ? "cpu" : "gpu") << std::endl; + return HSA_STATUS_SUCCESS; + }, + &hsa_handles); + assert(status == HSA_STATUS_SUCCESS && "failed to iterate HSA agents"); + if (hsa_handles.fail()) { + warning("Cannot write to '%s'\n", hsa_handles.path().c_str()); + return; + } + + // App begin timestamp begin_ts_file.txt + output_file_t begin_ts("begin_ts_file.txt"); + + roctracer_timestamp_t app_begin_timestamp; + CHECK_ROCTRACER(roctracer_get_timestamp(&app_begin_timestamp)); + begin_ts << app_begin_timestamp << std::endl; + if (begin_ts.fail()) { + warning("Cannot write to '%s'\n", begin_ts.path().c_str()); + return; + } + is_valid_ = true; + } + + int write_callback_record(const roctracer_record_t* record, const void* callback_data) { + output_file_t* output_file{nullptr}; + switch (record->domain) { + case ACTIVITY_DOMAIN_ROCTX: { + const roctx_api_data_t* data = reinterpret_cast(callback_data); + output_file = get_output_file(ACTIVITY_DOMAIN_ROCTX); + *output_file << std::dec << record->begin_ns << " " << record->process_id << ":" + << record->thread_id << " " << record->op << ":" << data->args.id << ":\"" + << (data->args.message ? data->args.message : "") << "\"" << std::endl; + break; + } + case ACTIVITY_DOMAIN_HSA_API: { + const hsa_api_data_t* data = reinterpret_cast(callback_data); + output_file = get_output_file(ACTIVITY_DOMAIN_HSA_API); + *output_file << std::dec << record->begin_ns << ":" + << ((record->op == HSA_API_ID_hsa_shut_down) ? record->begin_ns + : record->end_ns) + << " " << record->process_id << ":" << record->thread_id << " " + << hsa_api_data_pair_t(record->op, *data) << " :" << data->correlation_id + << std::endl; + break; + } + case ACTIVITY_DOMAIN_HIP_API: { + const hip_api_data_t* data = reinterpret_cast(callback_data); + + std::string kernel_name; + if (record->kernel_name) { + static bool truncate = []() { + const char* env_var = getenv("ROCP_TRUNCATE_NAMES"); + return env_var && std::atoi(env_var) != 0; + }(); + kernel_name = cxx_demangle(record->kernel_name); + if (truncate) kernel_name = truncate_name(kernel_name); + kernel_name = " kernel=" + kernel_name; + } + + output_file = get_output_file(ACTIVITY_DOMAIN_HIP_API); + *output_file << std::dec << record->begin_ns << ":" << record->end_ns << " " + << record->process_id << ":" << record->thread_id << " " + << hipApiString((hip_api_id_t)record->op, data) << kernel_name << " :" + << data->correlation_id << std::endl; + break; + } + case ACTIVITY_DOMAIN_EXT_API: { + output_file = get_output_file(ACTIVITY_DOMAIN_EXT_API); + *output_file << std::dec << record->begin_ns << ":" << record->end_ns << " " + << record->process_id << ":" << record->thread_id << " MARK" + << "(name(" << record->mark_message << "))" << std::endl; + break; + [[fallthrough]]; + } + default: + warning("write_callback_record: ignored record for domain %d", record->domain); + break; + } + + return (output_file && output_file->fail()) ? -1 : 0; + } + + int write_activity_records(const roctracer_record_t* begin, const roctracer_record_t* end) { + while (begin != end) { + output_file_t* output_file{nullptr}; + const char* name = roctracer_op_string(begin->domain, begin->op, begin->kind); + + switch (begin->domain) { + case ACTIVITY_DOMAIN_HIP_OPS: { + output_file = get_output_file(ACTIVITY_DOMAIN_HIP_OPS); + *output_file << std::dec << begin->begin_ns << ":" << begin->end_ns << " " + << begin->device_id << ":" << begin->queue_id << " " << name << ":" + << begin->correlation_id << ":" << GetPid() << std::endl; + break; + } + case ACTIVITY_DOMAIN_HSA_OPS: + output_file = get_output_file(ACTIVITY_DOMAIN_HSA_OPS, begin->op); + if (begin->op == HSA_OP_ID_COPY) { + *output_file << std::dec << begin->begin_ns << ":" << begin->end_ns + << " async-copy:" << begin->correlation_id << " " << GetPid() << std::endl; + break; + } else if (begin->op == HSA_OP_ID_RESERVED1) { + *output_file << std::dec << begin->pc_sample.se << " " << begin->pc_sample.cycle << " " + << std::hex << std::showbase << begin->pc_sample.pc << " " << name + << std::endl; + break; + } + [[fallthrough]]; + default: { + warning("write_activity_records: ignored activity for domain %d", begin->domain); + break; + } + } + if (output_file && output_file->fail()) return -1; + CHECK_ROCTRACER(roctracer_next_record(begin, &begin)); + } + return 0; + } + + bool is_valid() const { return is_valid_; } + + private: + bool is_valid_{false}; + + output_file_t roctx_file_{"roctx_trace.txt"}, hsa_api_file_{"hsa_api_trace.txt"}, + hip_api_file_{"hip_api_trace.txt"}, hip_activity_file_{"hcc_ops_trace.txt"}, + hsa_async_copy_file_{"async_copy_trace.txt"}, pc_sample_file_{"pcs_trace.txt"}; + + static fs::path output_prefix_; +}; + +fs::path file_plugin_t::output_prefix_ = []() { + const char* output_dir = getenv("ROCP_OUTPUT_DIR"); + if (output_dir == nullptr) return ""; + + if (!fs::is_directory(fs::status(output_dir))) + fatal("Cannot open output directory specified by ROCP_OUTPUT_DIR!\n"); + return output_dir; +}(); + +} // namespace + +std::unique_ptr file_plugin; + +int roctracer_plugin_initialize(uint32_t roctracer_major_version, + uint32_t roctracer_minor_version) { + if (roctracer_major_version != ROCTRACER_VERSION_MAJOR || + roctracer_minor_version < ROCTRACER_VERSION_MINOR) + return -1; + + if (file_plugin) return -1; + file_plugin = std::make_unique(); + if (!file_plugin->is_valid()) { + file_plugin.reset(); + return -1; + } + return 0; +} + +void roctracer_plugin_finalize() { file_plugin.reset(); } + +int roctracer_plugin_write_callback_record(const roctracer_record_t* record, + const void* callback_data) { + if (!file_plugin || !file_plugin->is_valid()) return -1; + return file_plugin->write_callback_record(record, callback_data); +} + +int roctracer_plugin_write_activity_records(const roctracer_record_t* begin, + const roctracer_record_t* end) { + if (!file_plugin || !file_plugin->is_valid()) return -1; + return file_plugin->write_activity_records(begin, end); +} diff --git a/projects/roctracer/src/tracer_tool/tracer_tool.cpp b/projects/roctracer/src/tracer_tool/tracer_tool.cpp index e43d02e909..67d5c0fdc1 100644 --- a/projects/roctracer/src/tracer_tool/tracer_tool.cpp +++ b/projects/roctracer/src/tracer_tool/tracer_tool.cpp @@ -60,36 +60,11 @@ namespace fs = std::experimental::filesystem; } \ } while (0) -#define ONLOAD_TRACE(str) \ - if (getenv("ROCP_ONLOAD_TRACE")) do { \ - std::cout << "PID(" << GetPid() << "): TRACER_TOOL::" << __FUNCTION__ << " " << str \ - << std::endl; \ - } while (0); - -#define ONLOAD_TRACE_BEG() ONLOAD_TRACE("begin") -#define ONLOAD_TRACE_END() ONLOAD_TRACE("end") - LOADER_INSTANTIATE(); TRACE_BUFFER_INSTANTIATE(); namespace { -#if DEBUG_TRACE_ON -inline static void DEBUG_TRACE(const char* fmt, ...) { - constexpr int size = 256; - char buf[size]; - - va_list valist; - va_start(valist, fmt); - vsnprintf(buf, size, fmt, valist); - printf("%u:%u %s", GetPid(), GetTid(), buf); - fflush(stdout); - va_end(valist); -} -#else -#define DEBUG_TRACE(...) -#endif - void warning(const std::string& msg) { std::cerr << msg << std::endl; } [[noreturn]] void fatal(const std::string& msg) { @@ -97,32 +72,6 @@ void warning(const std::string& msg) { std::cerr << msg << std::endl; } abort(); } -// Global output file handle -FILE* begin_ts_file_handle = nullptr; -FILE* roctx_file_handle = nullptr; -FILE* hsa_api_file_handle = nullptr; -FILE* hsa_async_copy_file_handle = nullptr; -FILE* hip_api_file_handle = nullptr; -FILE* hip_activity_file_handle = nullptr; -FILE* pc_sample_file_handle = nullptr; - -void close_output_file(FILE* file_handle) { - if (file_handle != nullptr) { - fflush(file_handle); - if (file_handle != stdout) fclose(file_handle); - } -} - -void close_file_handles() { - if (begin_ts_file_handle) close_output_file(begin_ts_file_handle); - if (roctx_file_handle) close_output_file(roctx_file_handle); - if (hsa_api_file_handle) close_output_file(hsa_api_file_handle); - if (hsa_async_copy_file_handle) close_output_file(hsa_async_copy_file_handle); - if (hip_api_file_handle) close_output_file(hip_api_file_handle); - if (hip_activity_file_handle) close_output_file(hip_activity_file_handle); - if (pc_sample_file_handle) close_output_file(pc_sample_file_handle); -} - thread_local roctracer_timestamp_t hsa_begin_timestamp = 0; thread_local roctracer_timestamp_t hip_begin_timestamp = 0; @@ -151,59 +100,6 @@ uint32_t GetTid() { return tid; } -/* The function extracts the kernel name from -input string. By using the iterators it finds the -window in the string which contains only the kernel name. -For example 'Foo::foo(a[], int (int))' -> 'foo'*/ -std::string truncate_name(const std::string& name) { - auto rit = name.rbegin(); - auto rend = name.rend(); - uint32_t counter = 0; - char open_token = 0; - char close_token = 0; - while (rit != rend) { - if (counter == 0) { - switch (*rit) { - case ')': - counter = 1; - open_token = ')'; - close_token = '('; - break; - case '>': - counter = 1; - open_token = '>'; - close_token = '<'; - break; - case ']': - counter = 1; - open_token = ']'; - close_token = '['; - break; - case ' ': - ++rit; - continue; - } - if (counter == 0) break; - } else { - if (*rit == open_token) counter++; - if (*rit == close_token) counter--; - } - ++rit; - } - auto rbeg = rit; - while ((rit != rend) && (*rit != ' ') && (*rit != ':')) rit++; - return name.substr(rend - rit, rit - rbeg); -} -// C++ symbol demangle -static inline std::string cxx_demangle(const std::string& symbol) { - int status; - char* demangled = abi::__cxa_demangle(symbol.c_str(), nullptr, nullptr, &status); - if (status != 0) return symbol; - std::string ret(demangled); - free(demangled); - return ret; -} - // Tracing control thread uint32_t control_delay_us = 0; uint32_t control_len_us = 0; @@ -294,6 +190,8 @@ class roctracer_plugin_t { decltype(roctracer_plugin_write_activity_records)* roctracer_plugin_write_activity_records_; }; +std::optional plugin; + } // namespace /////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -324,21 +222,11 @@ struct roctx_trace_entry_t { } }; -// rocTX buffer flush function -void roctx_flush_cb(roctx_trace_entry_t* entry) { - std::ostringstream os; - os << entry->record.begin_ns << " " << entry->record.process_id << ":" << entry->record.thread_id - << " " << entry->record.op << ":" << entry->data.args.id; - if (entry->data.args.message) - os << ":\"" << entry->data.args.message << "\""; - else - os << ":\"\""; - fprintf(roctx_file_handle, "%s\n", os.str().c_str()); - fflush(roctx_file_handle); -} - roctracer::TraceBuffer roctx_trace_buffer("rocTX API", 0x200000, - roctx_flush_cb); + [](roctx_trace_entry_t* entry) { + plugin->write_callback_record( + &entry->record, &entry->data); + }); // rocTX callback function void roctx_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, @@ -375,17 +263,10 @@ struct hsa_api_trace_entry_t { ~hsa_api_trace_entry_t() {} }; -void hsa_api_flush_cb(hsa_api_trace_entry_t* entry) { - std::ostringstream os; - os << entry->record.begin_ns << ":" << entry->record.end_ns << " " << entry->record.process_id - << ":" << entry->record.thread_id << " " << hsa_api_data_pair_t(entry->record.op, entry->data) - << " :" << entry->data.correlation_id; - fprintf(hsa_api_file_handle, "%s\n", os.str().c_str()); - fflush(hsa_api_file_handle); -} - -roctracer::TraceBuffer hsa_api_trace_buffer("HSA API", 0x200000, - hsa_api_flush_cb); +roctracer::TraceBuffer hsa_api_trace_buffer( + "HSA API", 0x200000, [](hsa_api_trace_entry_t* entry) { + plugin->write_callback_record(&entry->record, &entry->data); + }); // HSA API callback function @@ -517,53 +398,10 @@ static std::optional getKernelName(uint32_t cid, const hip_api_data function); } -void hip_api_flush_cb(hip_api_trace_entry_t* entry) { - const uint32_t domain = entry->record.domain; - const uint32_t cid = entry->record.op; - const hip_api_data_t* data = &entry->data; - const uint64_t correlation_id = data->correlation_id; - const roctracer_timestamp_t begin_timestamp = entry->record.begin_ns; - const roctracer_timestamp_t end_timestamp = entry->record.end_ns; - std::ostringstream rec_ss; - std::ostringstream oss; - - const char* str = - (domain != ACTIVITY_DOMAIN_EXT_API) ? roctracer_op_string(domain, cid, 0) : "MARK"; - rec_ss << std::dec << begin_timestamp << ":" << end_timestamp << " " << GetPid() << ":" - << entry->record.thread_id; - oss << std::dec << rec_ss.str() << " " << str; - - DEBUG_TRACE( - "hip_api_flush_cb(\"%s\"): domain(%u) cid(%u) entry(%p) kernel_name(\"%s\" " - "correlation_id(%lu) " - "beg(%lu) end(%lu))\n", - roctracer_op_string(entry->record.domain, entry->record.op, entry->record.kind), - entry->record.domain, entry->record.op, entry, entry->record.kernel_name, correlation_id, - begin_timestamp, end_timestamp); - - if (domain == ACTIVITY_DOMAIN_HIP_API) { - const char* str = hipApiString((hip_api_id_t)cid, data); - rec_ss << " " << str; - if (entry->record.kernel_name) { - static bool truncate = []() { - const char* env_var = getenv("ROCP_TRUNCATE_NAMES"); - return env_var && std::atoi(env_var) != 0; - }(); - - std::string kernel_name(cxx_demangle(entry->record.kernel_name)); - if (truncate) kernel_name = truncate_name(kernel_name); - rec_ss << " kernel=" << kernel_name; - } - rec_ss << " :" << correlation_id; - fprintf(hip_api_file_handle, "%s\n", rec_ss.str().c_str()); - } else { - fprintf(hip_api_file_handle, "%s(name(%s))\n", oss.str().c_str(), entry->record.mark_message); - } - fflush(hip_api_file_handle); -} - -roctracer::TraceBuffer hip_api_trace_buffer("HIP API", 0x200000, - hip_api_flush_cb); +roctracer::TraceBuffer hip_api_trace_buffer( + "HIP API", 0x200000, [](hip_api_trace_entry_t* entry) { + plugin->write_callback_record(&entry->record, &entry->data); + }); void hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) { (void)arg; @@ -582,12 +420,6 @@ void hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, GetTid(), *data, kernel_name ? kernel_name->c_str() : nullptr); entry.valid.store(roctracer::TRACE_ENTRY_COMPLETE, std::memory_order_release); } - - DEBUG_TRACE( - "hip_api_callback(\"%s\") phase(%d): cid(%u) data(%p) entry(%p) name(\"%s\") " - "correlation_id(%lu) timestamp(%lu)\n", - roctracer_op_string(domain, cid, 0), data->phase, cid, data, entry, - kernel_name ? kernel_name->c_str() : "", data->correlation_id, timestamp); } void mark_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) { @@ -602,47 +434,6 @@ void mark_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, } /////////////////////////////////////////////////////////////////////////////////////////////////////// -// Activity tracing callback -// hipMalloc id(3) correlation_id(1): begin_ns(1525888652762640464) end_ns(1525888652762877067) -void pool_activity_callback(const char* begin, const char* end, void* arg) { - const roctracer_record_t* record = reinterpret_cast(begin); - const roctracer_record_t* end_record = reinterpret_cast(end); - - while (record < end_record) { - const char* name = roctracer_op_string(record->domain, record->op, record->kind); - DEBUG_TRACE( - "pool_activity_callback(\"%s\"): domain(%u) op(%u) kind(%u) record(%p) correlation_id(%lu) " - "beg(%lu) end(%lu)\n", - name, record->domain, record->op, record->kind, record, record->correlation_id, - record->begin_ns, record->end_ns); - - switch (record->domain) { - case ACTIVITY_DOMAIN_HIP_OPS: - fprintf(hip_activity_file_handle, "%lu:%lu %d:%lu %s:%lu:%u\n", record->begin_ns, - record->end_ns, record->device_id, record->queue_id, name, record->correlation_id, - GetPid()); - fflush(hip_activity_file_handle); - break; - case ACTIVITY_DOMAIN_HSA_OPS: - if (record->op == HSA_OP_ID_COPY) { - fprintf(hsa_async_copy_file_handle, "%lu:%lu async-copy:%lu:%u\n", record->begin_ns, - record->end_ns, record->correlation_id, GetPid()); - fflush(hsa_async_copy_file_handle); - } else if (record->op == HSA_OP_ID_RESERVED1) { - fprintf(pc_sample_file_handle, "%u %lu 0x%lx %s\n", record->pc_sample.se, - record->pc_sample.cycle, record->pc_sample.pc, name); - fflush(pc_sample_file_handle); - } - break; - default: - assert(!"Unexpected activity"); - } - CHECK_ROCTRACER(roctracer_next_record(record, &record)); - } -} - -/////////////////////////////////////////////////////////////////////////////////////////////////////// - // Input parser std::string normalize_token(const std::string& token, bool not_empty, const std::string& label) { const std::string space_chars_set = " \t"; @@ -698,34 +489,15 @@ int get_xml_array(const xml::Xml::level_t* node, const std::string& field, const return parse_iter; } -// Open output file -FILE* open_output_file(const char* prefix, const char* name, const char** path = nullptr) { - FILE* file_handle = nullptr; - if (path != nullptr) *path = nullptr; - - if (prefix != nullptr) { - std::ostringstream oss; - oss << prefix << "/" << GetPid() << "_" << name; - file_handle = fopen(oss.str().c_str(), "w"); - if (file_handle == nullptr) { - std::ostringstream errmsg; - errmsg << "ROCTracer: fopen error, file '" << oss.str().c_str() << "'"; - perror(errmsg.str().c_str()); - abort(); - } - - if (path != nullptr) *path = strdup(oss.str().c_str()); - } else - file_handle = stdout; - return file_handle; -} - // Allocating tracing pool void open_tracing_pool() { if (roctracer_default_pool() == nullptr) { roctracer_properties_t properties{}; properties.buffer_size = 0x80000; - properties.buffer_callback_fun = pool_activity_callback; + properties.buffer_callback_fun = [](const char* begin, const char* end, void* /* arg */) { + plugin->write_activity_records(reinterpret_cast(begin), + reinterpret_cast(end)); + }; CHECK_ROCTRACER(roctracer_open_pool(&properties)); } } @@ -743,8 +515,6 @@ static bool is_loaded = false; // tool unload method void tool_unload() { - ONLOAD_TRACE("begin, loaded(" << is_loaded << ")"); - if (is_loaded == false) return; is_loaded = false; @@ -781,23 +551,20 @@ void tool_unload() { close_tracing_pool(); roctracer::TraceBufferBase::FlushAll(); - close_file_handles(); - - ONLOAD_TRACE_END(); + plugin.reset(); } // tool load method void tool_load() { - ONLOAD_TRACE("begin, loaded(" << is_loaded << ")"); - if (is_loaded == true) return; is_loaded = true; - // Output file - const char* output_prefix = getenv("ROCP_OUTPUT_DIR"); - if (output_prefix != nullptr) { - DIR* dir = opendir(output_prefix); - if (dir == nullptr) fatal("ROCTracer: Cannot open output directory"); + // Load output plugin + const char* plugin_name = getenv("ROCTRACER_PLUGIN_LIB"); + if (plugin_name == nullptr) plugin_name = "libfile_plugin.so"; + if (Dl_info dl_info; dladdr((void*)tool_load, &dl_info) != 0) { + if (!plugin.emplace(fs::path(dl_info.dli_fname).replace_filename(plugin_name)).is_valid()) + plugin.reset(); } // API traces switches @@ -886,8 +653,6 @@ void tool_load() { // Enable rpcTX callbacks if (trace_roctx) { - roctx_file_handle = open_output_file(output_prefix, "roctx_trace.txt"); - // initialize HSA tracing fprintf(stdout, " rocTX-trace()\n"); fflush(stdout); @@ -934,8 +699,6 @@ void tool_load() { fflush(stdout); flush_thread = new std::thread(flush_thr_fun); } - - ONLOAD_TRACE_END(); } extern "C" { @@ -947,8 +710,6 @@ ROCTRACER_EXPORT extern const uint32_t HSA_AMD_TOOL_PRIORITY = 1050; // HSA-runtime tool on-load method ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, uint64_t failed_tool_count, const char* const* failed_tool_names) { - ONLOAD_TRACE_BEG(); - if (roctracer_version_major() != ROCTRACER_VERSION_MAJOR || roctracer_version_minor() < ROCTRACER_VERSION_MINOR) { warning("The ROCtracer API version is not compatible with this tool"); @@ -961,40 +722,8 @@ ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, // application before exiting, so register an atexit handler to unload the tool. std::atexit(tool_unload); - const char* output_prefix = getenv("ROCP_OUTPUT_DIR"); - - // Dumping HSA handles for agents - FILE* handles_file_handle = open_output_file(output_prefix, "hsa_handles.txt"); - auto iterate_agent_data = std::make_pair(table, handles_file_handle); - - [[maybe_unused]] hsa_status_t status = table->core_->hsa_iterate_agents_fn( - [](hsa_agent_t agent, void* user_data) { - auto [hsa_api_table, hsa_handles_file] = - *reinterpret_cast(user_data); - hsa_device_type_t type; - - if (hsa_api_table->core_->hsa_agent_get_info_fn(agent, HSA_AGENT_INFO_DEVICE, &type) != - HSA_STATUS_SUCCESS) - return HSA_STATUS_ERROR; - - fprintf(hsa_handles_file, "0x%lx agent %s\n", agent.handle, - (type == HSA_DEVICE_TYPE_CPU) ? "cpu" : "gpu"); - return HSA_STATUS_SUCCESS; - }, - &iterate_agent_data); - assert(status == HSA_STATUS_SUCCESS && "failed to iterate HSA agents"); - - close_output_file(handles_file_handle); - - // App begin timestamp begin_ts_file.txt - begin_ts_file_handle = open_output_file(output_prefix, "begin_ts_file.txt"); - const roctracer_timestamp_t app_start_time = timestamp_ns(); - fprintf(begin_ts_file_handle, "%lu\n", app_start_time); - // Enable HSA API callbacks/activity if (trace_hsa_api) { - hsa_api_file_handle = open_output_file(output_prefix, "hsa_api_trace.txt"); - fprintf(stdout, " HSA-trace("); fflush(stdout); if (hsa_api_vec.size() != 0) { @@ -1015,8 +744,6 @@ ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, // Enable HSA GPU activity if (trace_hsa_activity) { - hsa_async_copy_file_handle = open_output_file(output_prefix, "async_copy_trace.txt"); - // Allocating tracing pool open_tracing_pool(); @@ -1036,7 +763,6 @@ ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, // Enable tracing if (trace_hip_api) { - hip_api_file_handle = open_output_file(output_prefix, "hip_api_trace.txt"); if (hip_api_vec.size() != 0) { for (unsigned i = 0; i < hip_api_vec.size(); ++i) { uint32_t cid = HIP_API_ID_NONE; @@ -1053,7 +779,6 @@ ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, } if (trace_hip_activity) { - hip_activity_file_handle = open_output_file(output_prefix, "hcc_ops_trace.txt"); CHECK_ROCTRACER(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)); } } @@ -1063,19 +788,12 @@ ROCTRACER_EXPORT bool OnLoad(HsaApiTable* table, uint64_t runtime_version, fprintf(stdout, " PCS-trace()\n"); fflush(stdout); open_tracing_pool(); - pc_sample_file_handle = open_output_file(output_prefix, "pcs_trace.txt"); CHECK_ROCTRACER(roctracer_enable_op_activity(ACTIVITY_DOMAIN_HSA_OPS, HSA_OP_ID_RESERVED1)); } - - ONLOAD_TRACE_END(); return true; } // HSA-runtime on-unload method -ROCTRACER_EXPORT void OnUnload() { - ONLOAD_TRACE_BEG(); - tool_unload(); - ONLOAD_TRACE_END(); -} +ROCTRACER_EXPORT void OnUnload() { tool_unload(); } } // extern "C"