Updates/fixes for CI, docs, tests, samples, and common library (#528)

- .github/workflows/continuous_integration.yml
  - apt-get update before apt-get install
  - remove libgtest-dev
  - actions-comment-pull-request: v2.4.3 -> v2.5.0
- .github/workflows/formatting.yml
  - create-pull-request: v5 -> v6
- cmake/rocprofiler_options.cmake
  - remove unused ROCPROFILER_DEBUG_TRACE and ROCPROFILER_LD_AQLPROFILE options
- samples/counter_collection/callback_client.cpp
  - corr_id field renamed to correlation_id
- samples/counter_collection/client.cpp
  - corr_id field renamed to correlation_id
- include/rocprofiler-sdk/fwd.h
  - In rocprofiler_record_counter_t: rename corr_id field to correlation_id
  - doxygen fixes
- lib/common/utility.*
  - remove get_accurate_clock_id_impl
  - timestamp_ns() defaults to CLOCK_BOOTTIME
- lib/rocprofiler-sdk/counters/core.cpp
  - fix spelling mistake: extrenal -> external
  - corr_id field renamed to correlation_id
- lib/rocprofiler-sdk-tool/tool.cpp
  - fix destruction of static tool::output_file before finalization
- scripts/update-docs.sh
  - define PROJECT_NAME
- tests/async-copy-tracing/validate.py
  - init_time and fini_time checks
  - hip_api_traces, marker_api_tracing
- tests/common/serialization.hpp
  - fix save function for rocprofiler_record_counter_t following rename of corr_id to correlation_id
- tests/kernel-tracing/validate.py
  - init_time and fini_time checks
  - relax test_total_runtime range
- tests/rocprofv3/tracing/CMakeLists.txt
  - remove -M from rocprofv3-test-systrace-execute
  - exclude test_hsa_api_trace in rocprofv3-test-systrace-validate due to HIP API tracing
- tests/rocprofv3/tracing/validate.py
  - update test_kernel_trace to accept mangled or demangled
- tests/tools/json-tool.cpp
  - remove use of GLOG
  - include init_time and fini_time
  - write_json(...) function
This commit is contained in:
Jonathan R. Madsen
2024-02-22 00:16:43 -06:00
committed by GitHub
parent 7adffd5b22
commit 0d939edbba
17 changed files with 292 additions and 233 deletions
+14 -9
View File
@@ -344,8 +344,8 @@ typedef uint64_t rocprofiler_counter_dimension_id_t;
*/
typedef union rocprofiler_user_data_t
{
uint64_t value;
void* ptr;
uint64_t value; ///< usage example: set to process id, thread id, etc.
void* ptr; ///< usage example: set to address of data allocation
} rocprofiler_user_data_t;
//--------------------------------------------------------------------------------------//
@@ -504,10 +504,12 @@ rocprofiler_record_header_compute_hash(uint32_t category, uint32_t kind)
*/
typedef struct
{
const char* name;
size_t instance_size;
rocprofiler_counter_dimension_id_t
id; //<< Id for this dimension used by @ref rocprofiler_query_record_dimension_position
const char* name;
size_t instance_size;
rocprofiler_counter_dimension_id_t id;
/// @var id
/// @brief Id for this dimension used by @ref rocprofiler_query_record_dimension_position
} rocprofiler_record_dimension_info_t;
/**
@@ -515,9 +517,12 @@ typedef struct
*/
typedef struct
{
rocprofiler_counter_instance_id_t id;
double counter_value; //<< counter value
rocprofiler_correlation_id_t corr_id;
rocprofiler_counter_instance_id_t id; ///< counter identifier
double counter_value; ///< counter value
rocprofiler_correlation_id_t correlation_id;
/// @var correlation_id
/// @brief Used to correlate the kernel data to an API call
} rocprofiler_record_counter_t;
/**
+1 -26
View File
@@ -62,35 +62,10 @@ get_clock_name(clockid_t _id)
default: break;
}
return "CLOCK_UNKNOWN";
#undef CLOCK_NAME_CASE_STATEMENT
}
} // namespace
clockid_t
get_accurate_clock_id_impl()
{
auto clock = CLOCK_MONOTONIC;
utsname kernelInfo;
if(uname(&kernelInfo) == 0)
{
try
{
std::string ver = kernelInfo.release;
size_t idx;
int major = std::stoi(ver, &idx);
int minor = std::stoi(ver.substr(idx + 1));
if(major > 4 || ((major == 4) && (minor >= 4)))
{
clock = CLOCK_MONOTONIC_RAW;
}
} catch(...)
{
// Kernel version string doesn't conform to the standard pattern.
// Keep using the "safe" (non-RAW) clock.
}
}
return clock;
}
uint64_t
get_clock_period_ns_impl(clockid_t _clk_id)
{
+7 -31
View File
@@ -48,9 +48,6 @@ namespace rocprofiler
{
namespace common
{
clockid_t
get_accurate_clock_id_impl();
uint64_t
get_clock_period_ns_impl(clockid_t _clk_id);
@@ -62,20 +59,6 @@ get_tid()
return _v;
}
inline clockid_t
get_accurate_clock_id()
{
static auto clk_id = get_accurate_clock_id_impl();
return clk_id;
}
inline uint64_t
get_accurate_clock_period_ns()
{
static auto clk_period = get_clock_period_ns_impl(get_accurate_clock_id());
return clk_period;
}
inline uint64_t
get_ticks(clockid_t clk_id_v) noexcept
{
@@ -92,24 +75,17 @@ get_ticks(clockid_t clk_id_v) noexcept
return (static_cast<uint64_t>(ts.tv_sec) * nanosec) + static_cast<uint64_t>(ts.tv_nsec);
}
// this equates to HSA-runtime library implementation of os::ReadAccurateClock()
// CLOCK_MONOTONIC_RAW equates to HSA-runtime library implementation of os::ReadAccurateClock()
// CLOCK_BOOTTIME equates to HSA-runtime library implementation of os::ReadSystemClock()
template <int ClockT = CLOCK_BOOTTIME>
inline uint64_t
timestamp_ns()
{
auto&& clk_period = get_accurate_clock_period_ns();
if(ROCPROFILER_LIKELY(clk_period == 1)) return get_ticks(get_accurate_clock_id());
return get_ticks(get_accurate_clock_id()) / clk_period;
}
constexpr auto _clk = ClockT;
static auto _clk_period = get_clock_period_ns_impl(_clk);
// this equates to HSA-runtime library implementation of os::ReadSystemClock()
inline uint64_t
system_timestamp_ns()
{
constexpr auto boottime_clk = CLOCK_BOOTTIME;
static auto boottime_clk_period = get_clock_period_ns_impl(boottime_clk);
if(ROCPROFILER_LIKELY(boottime_clk_period == 1)) return get_ticks(boottime_clk);
return get_ticks(boottime_clk) / boottime_clk_period;
if(ROCPROFILER_LIKELY(_clk_period == 1)) return get_ticks(_clk);
return get_ticks(_clk) / _clk_period;
}
std::vector<std::string>
+155 -102
View File
@@ -52,104 +52,154 @@
#include <unordered_set>
#include <vector>
namespace common = ::rocprofiler::common;
namespace tool = ::rocprofiler::tool;
static const uint32_t lds_block_size = 128 * 4;
namespace common = ::rocprofiler::common;
namespace tool = ::rocprofiler::tool;
namespace
{} // namespace
{
constexpr uint32_t lds_block_size = 128 * 4;
auto&
auto destructors = new std::vector<std::function<void()>>{};
template <typename Tp>
Tp&
get_dereference(Tp* ptr)
{
return *CHECK_NOTNULL(ptr);
}
template <typename Tp>
void
add_destructor(Tp*& ptr)
{
static auto _mutex = std::mutex{};
auto _lk = std::unique_lock<std::mutex>{_mutex};
destructors->emplace_back([&ptr]() {
delete ptr;
ptr = nullptr;
});
}
#define ADD_DESTRUCTOR(PTR) \
{ \
static auto _once = std::once_flag{}; \
std::call_once(_once, []() { add_destructor(PTR); }); \
}
tool::output_file*&
get_hsa_api_file()
{
static auto _v = tool::output_file{"hsa_api_trace",
tool::csv::api_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
static auto* _v = new tool::output_file{"hsa_api_trace",
tool::csv::api_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
ADD_DESTRUCTOR(_v);
return _v;
}
auto&
tool::output_file*&
get_hip_api_file()
{
static auto _v = tool::output_file{"hip_api_trace",
tool::csv::api_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
static auto* _v = new tool::output_file{"hip_api_trace",
tool::csv::api_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
ADD_DESTRUCTOR(_v);
return _v;
}
auto&
tool::output_file*&
get_kernel_trace_file()
{
static auto _v = tool::output_file{"kernel_trace",
tool::csv::kernel_trace_csv_encoder{},
{"Kind",
"Agent_Id",
"Queue_Id",
"Kernel_Id",
"Kernel_Name",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp",
"Private_Segment_Size",
"Group_Segment_Size",
"Workgroup_Size_X",
"Workgroup_Size_Y",
"Workgroup_Size_Z",
"Grid_Size_X",
"Grid_Size_Y",
"Grid_Size_Z"}};
static auto* _v = new tool::output_file{"kernel_trace",
tool::csv::kernel_trace_csv_encoder{},
{"Kind",
"Agent_Id",
"Queue_Id",
"Kernel_Id",
"Kernel_Name",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp",
"Private_Segment_Size",
"Group_Segment_Size",
"Workgroup_Size_X",
"Workgroup_Size_Y",
"Workgroup_Size_Z",
"Grid_Size_X",
"Grid_Size_Y",
"Grid_Size_Z"}};
ADD_DESTRUCTOR(_v);
return _v;
}
auto&
tool::output_file*&
get_counter_collection_file()
{
static auto _v = tool::output_file{"counter_collection",
tool::csv::counter_collection_csv_encoder{},
{"Counter_Id",
"Agent_Id",
"Queue_Id",
"Process_Id",
"Thread_Id",
"Grid_Size",
"Kernel-Name",
"Workgroup_Size",
"LDS_Block_Size",
"Scratch_Size",
"VGPR_Count",
"SGPR_Count",
"Counter_Name",
"Counter_Value"}};
static auto* _v = new tool::output_file{"counter_collection",
tool::csv::counter_collection_csv_encoder{},
{"Counter_Id",
"Agent_Id",
"Queue_Id",
"Process_Id",
"Thread_Id",
"Grid_Size",
"Kernel-Name",
"Workgroup_Size",
"LDS_Block_Size",
"Scratch_Size",
"VGPR_Count",
"SGPR_Count",
"Counter_Name",
"Counter_Value"}};
ADD_DESTRUCTOR(_v);
return _v;
}
auto&
tool::output_file*&
get_memory_copy_trace_file()
{
static auto _v = tool::output_file{"memory_copy_trace",
tool::csv::memory_copy_csv_encoder{},
{"Kind",
"Direction",
"Source_Agent_Id",
"Destination_Agent_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
static auto* _v = new tool::output_file{"memory_copy_trace",
tool::csv::memory_copy_csv_encoder{},
{"Kind",
"Direction",
"Source_Agent_Id",
"Destination_Agent_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
ADD_DESTRUCTOR(_v);
return _v;
}
tool::output_file*&
get_marker_api_file()
{
static auto* _v = new tool::output_file{"marker_api_trace",
tool::csv::marker_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
ADD_DESTRUCTOR(_v);
return _v;
}
#undef ADD_DESTRUCTOR
struct marker_entry
{
uint64_t cid = 0;
@@ -159,21 +209,6 @@ struct marker_entry
std::string message = {};
};
auto&
get_marker_api_file()
{
static auto _v = tool::output_file{"marker_api_trace",
tool::csv::marker_csv_encoder{},
{"Domain",
"Function",
"Process_Id",
"Thread_Id",
"Correlation_Id",
"Start_Timestamp",
"End_Timestamp"}};
return _v;
}
struct buffer_ids
{
rocprofiler_buffer_id_t hsa_api_trace = {};
@@ -283,7 +318,7 @@ cntrl_tracing_callback(rocprofiler_callback_tracing_record_t record,
record.correlation_id.internal,
user_data->value,
ts);
get_marker_api_file() << ss.str();
get_dereference(get_marker_api_file()) << ss.str();
}
}
}
@@ -319,7 +354,7 @@ callback_tracing_callback(rocprofiler_callback_tracing_record_t record,
record.correlation_id.internal,
ts,
ts);
get_marker_api_file() << ss.str();
get_dereference(get_marker_api_file()) << ss.str();
}
}
else if(record.operation == ROCPROFILER_MARKER_CORE_API_ID_roctxRangePushA)
@@ -349,7 +384,7 @@ callback_tracing_callback(rocprofiler_callback_tracing_record_t record,
auto ss = std::stringstream{};
tool::csv::marker_csv_encoder::write_row(
ss, kind_name, val.message, val.pid, val.tid, val.cid, val.data.value, ts);
get_marker_api_file() << ss.str();
get_dereference(get_marker_api_file()) << ss.str();
}
}
else if(record.operation == ROCPROFILER_MARKER_CORE_API_ID_roctxRangeStartA)
@@ -384,7 +419,7 @@ callback_tracing_callback(rocprofiler_callback_tracing_record_t record,
_entry.cid,
_entry.data.value,
ts);
get_marker_api_file() << ss.str();
get_dereference(get_marker_api_file()) << ss.str();
global_range.wlock([](auto& map, auto _key) { return map.erase(_key); }, _id);
}
@@ -408,7 +443,7 @@ callback_tracing_callback(rocprofiler_callback_tracing_record_t record,
record.correlation_id.internal,
user_data->value,
ts);
get_marker_api_file() << ss.str();
get_dereference(get_marker_api_file()) << ss.str();
}
}
}
@@ -504,7 +539,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
record->grid_size.y,
record->grid_size.z);
get_kernel_trace_file() << kernel_trace_ss.str();
get_dereference(get_kernel_trace_file()) << kernel_trace_ss.str();
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_HSA_CORE_API ||
header->kind == ROCPROFILER_BUFFER_TRACING_HSA_AMD_EXT_API ||
@@ -525,7 +560,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
record->start_timestamp,
record->end_timestamp);
get_hsa_api_file() << hsa_trace_ss.str();
get_dereference(get_hsa_api_file()) << hsa_trace_ss.str();
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_MEMORY_COPY)
{
@@ -543,7 +578,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
record->start_timestamp,
record->end_timestamp);
get_memory_copy_trace_file() << memory_copy_trace_ss.str();
get_dereference(get_memory_copy_trace_file()) << memory_copy_trace_ss.str();
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API ||
header->kind == ROCPROFILER_BUFFER_TRACING_HIP_COMPILER_API)
@@ -562,7 +597,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
record->start_timestamp,
record->end_timestamp);
get_hip_api_file() << hip_trace_ss.str();
get_dereference(get_hip_api_file()) << hip_trace_ss.str();
}
else
{
@@ -575,7 +610,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
{
auto* profiler_record = static_cast<rocprofiler_record_counter_t*>(header->payload);
rocprofiler_tool_kernel_properties_t kernel_properties =
GetKernelProperties(profiler_record->corr_id.internal);
GetKernelProperties(profiler_record->correlation_id.internal);
rocprofiler_counter_id_t counter_id;
size_t pos;
rocprofiler_counter_info_v0_t version;
@@ -606,7 +641,7 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
fmt::format("{}[{}]", version.name, pos),
profiler_record->counter_value);
get_counter_collection_file() << counter_collection_ss.str();
get_dereference(get_counter_collection_file()) << counter_collection_ss.str();
}
}
}
@@ -925,27 +960,45 @@ tool_fini(void* tool_data)
flush();
rocprofiler_stop_context(get_client_ctx());
if(destructors)
{
for(const auto& itr : *destructors)
itr();
delete destructors;
destructors = nullptr;
}
(void) (tool_data);
}
} // namespace
extern "C" rocprofiler_tool_configure_result_t*
rocprofiler_configure(uint32_t /*version*/,
const char* /*runtime_version*/,
rocprofiler_configure(uint32_t version,
const char* runtime_version,
uint32_t priority,
rocprofiler_client_id_t* id)
{
common::init_logging("ROCPROF_LOG_LEVEL");
FLAGS_colorlogtostderr = true;
// only activate if main tool
if(priority > 0) return nullptr;
// set the client name
id->name = "rocprofiler-tool";
id->name = "rocprofv3";
// store client info
client_identifier = id;
// note that rocprofv3 is not the primary tool
LOG_IF(WARNING, priority > 0) << id->name << " has a priority of " << priority
<< " (not primary tool)";
// compute major/minor/patch version info
uint32_t major = version / 10000;
uint32_t minor = (version % 10000) / 100;
uint32_t patch = version % 100;
LOG(INFO) << id->name << " is using rocprofiler-sdk v" << major << "." << minor << "." << patch
<< " (" << runtime_version << ")";
// create configure data
static auto cfg = rocprofiler_tool_configure_result_t{
sizeof(rocprofiler_tool_configure_result_t), &tool_init, &tool_fini, nullptr};
+5 -5
View File
@@ -265,10 +265,10 @@ queue_cb(const std::shared_ptr<counter_callback_info>& info,
if(const auto* _corr_id = correlation_id)
{
_corr_id_v.internal = _corr_id->internal;
if(const auto* extrenal =
if(const auto* external =
rocprofiler::common::get_val(extern_corr_ids, info->internal_context))
{
_corr_id_v.external = *extrenal;
_corr_id_v.external = *external;
}
}
@@ -380,10 +380,10 @@ completed_cb(const std::shared_ptr<counter_callback_info>& info,
if(const auto* _corr_id = session.correlation_id)
{
_corr_id_v.internal = _corr_id->internal;
if(const auto* extrenal =
if(const auto* external =
rocprofiler::common::get_val(session.extern_corr_ids, info->internal_context))
{
_corr_id_v.external = *extrenal;
_corr_id_v.external = *external;
}
}
@@ -396,7 +396,7 @@ completed_cb(const std::shared_ptr<counter_callback_info>& info,
for(auto& val : *ret)
{
val.corr_id = _corr_id_v;
val.correlation_id = _corr_id_v;
if(buf)
buf->emplace(ROCPROFILER_BUFFER_CATEGORY_COUNTERS, 0, val);
else
+1 -1
View File
@@ -21,7 +21,7 @@ message "Changing directory to ${WORK_DIR}"
cd ${WORK_DIR}
message "Generating rocprofiler-sdk.dox"
cmake -DSOURCE_DIR=${SOURCE_DIR} -P ${WORK_DIR}/generate-doxyfile.cmake
cmake -DSOURCE_DIR=${SOURCE_DIR} -DPROJECT_NAME="Rocprofiler SDK" -P ${WORK_DIR}/generate-doxyfile.cmake
message "Generating doxygen xml files"
doxygen rocprofiler-sdk.dox