From 38ba63030d0e751a1c14cff66367bd663606d64c Mon Sep 17 00:00:00 2001 From: Benjamin Welton Date: Wed, 18 Oct 2023 05:00:56 +0000 Subject: [PATCH] Added ROCTRACER_BUFFER_SIZE to set buffer size [SWDEV-418917] reported that timing skew was being introduced by roctracer. Most of the cause of this problem seems to stem from outrunning the double buffering scheme that we use in memory_pool (part of the reason for this outrun is due to File writing being slow). A semi-quick fix that may be able to last until RocProf v2 is complete is to allow adjustment of the buffer size. ROCTRACER_BUFFER_SIZE env variable was introduced here which allows setting the buffer size of tracer tool. By increasing the buffer size, an ~8% reduction in execution time when timing on the program side. This should also reduce the frequency of large delays when we outrun the buffer. Note: increasing this size dramatically can cause slow startups (i.e. above 50MB). Change-Id: I98c4316cfe93a043623ae2669cfe1a5abb55c990 --- src/tracer_tool/tracer_tool.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/tracer_tool/tracer_tool.cpp b/src/tracer_tool/tracer_tool.cpp index fb74c70edc..83bc4ae4ee 100644 --- a/src/tracer_tool/tracer_tool.cpp +++ b/src/tracer_tool/tracer_tool.cpp @@ -94,6 +94,13 @@ uint32_t GetTid() { return tid; } +size_t GetBufferSize() { + auto bufSize = getenv("ROCTRACER_BUFFER_SIZE"); + // Default size if not set + if (!bufSize) return 0x200000; + return std::stoll({bufSize}); +} + // Tracing control thread uint32_t control_delay_us = 0; uint32_t control_len_us = 0; @@ -217,7 +224,7 @@ struct roctx_trace_entry_t { }; roctracer::TraceBuffer roctx_trace_buffer( - "rocTX API", 0x200000, [](roctx_trace_entry_t* entry) { + "rocTX API", GetBufferSize(), [](roctx_trace_entry_t* entry) { assert(plugin && "plugin is not initialized"); plugin->write_callback_record(&entry->record, &entry->data); }); @@ -258,7 +265,7 @@ struct hsa_api_trace_entry_t { }; roctracer::TraceBuffer hsa_api_trace_buffer( - "HSA API", 0x200000, [](hsa_api_trace_entry_t* entry) { + "HSA API", GetBufferSize(), [](hsa_api_trace_entry_t* entry) { assert(plugin && "plugin is not initialized"); plugin->write_callback_record(&entry->record, &entry->data); }); @@ -397,7 +404,7 @@ static std::optional getKernelName(uint32_t cid, const hip_api_data } roctracer::TraceBuffer hip_api_trace_buffer( - "HIP API", 0x200000, [](hip_api_trace_entry_t* entry) { + "HIP API", GetBufferSize(), [](hip_api_trace_entry_t* entry) { assert(plugin && "plugin is not initialized"); plugin->write_callback_record(&entry->record, &entry->data); }); @@ -481,7 +488,7 @@ int get_xml_array(const xml::Xml::level_t* node, const std::string& field, const void open_tracing_pool() { if (roctracer_default_pool() == nullptr) { roctracer_properties_t properties{}; - properties.buffer_size = 0x80000; + properties.buffer_size = GetBufferSize(); properties.buffer_callback_fun = [](const char* begin, const char* end, void* /* arg */) { assert(plugin && "plugin is not initialized"); plugin->write_activity_records(reinterpret_cast(begin),