Add support for scratch reporting (#523)
* Add ToolsApiTable
Add ToolsApiTable wrapping for
scratch memory tracking
* Add initial support for scratch memory tracking
Buffering is implemented
* cmake formatting (cmake-format) (#525)
Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com>
* source formatting (clang-format v11) (#524)
Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com>
* Add callback tracing for scratch
Fixed the error where scratch tracking init was called irrespective of whether any client requested for it
* Apply suggestions from code review
Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com>
* Fix tools api copy/update
Table were saved/updated incorrectly in previous
commit. Also adds passing user data through the callback
* Fix OpKind sequence for scratch tracking
Previously scratch was using OpKind from rocprofiler-sdk, but
templates were instantiated using API ID. These differ by 1
* Integration tests for scratch reporting
Added buffer and callback integration tests for scratch reporting
* source formatting (clang-format v11) (#550)
Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com>
* cmake formatting (cmake-format) (#551)
Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com>
* python formatting (black) (#549)
Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com>
* CI fixes
* source formatting (clang-format v11) (#554)
Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com>
* Update api
Rebase on main and updates based on PR feedback
* Update scratch reporting and address PR comments
- Added agent id to buffer records
- Updated `test_internal_correlation_ids` - Is almost identical to
one in async-copy
- Updated scratch test to check for agent id
- Updated queue id serialization in callback records (prints
handle as nested key)
- Remove `marker_api_traces` from scratch `test_internal_correlation_ids`
validation test
- Rename `amd_tools_api` to `scratch_memory`
- Added doxygen comments
- Remove scratch callback from `tool.cpp`
- Replace assert with `LOF_IF` in `scratch_memory.cpp`
* Update tools table
Changed to match up with changes to hsa tables in main branch
* Rework scratch memory structure
* Update tests
- Added suggestions from PR review, and updated tests accordingly
* Misc cleanup
* Update scratch test
As of Apr 4th, `hsa_amd_agent_set_async_scratch_limit` is disabled.
Note,
> This API: `hsa_amd_agent_set_async_scratch_limit` is currently
> disabled. We need some changes in CP firmware to be able to do this
> and these changes are not ready yet.
> With the current code, you will also not get notifications for
> alternate-scratch allocations because this feature has been disabled
> while CP firmware is making additional changes
> We are hoping to have that feature enabled by ROCm-6.3
* Minor update to lib/rocprofiler-sdk/internal_threading.*
- delay destruction of shared_ptrs of the tasks to prevent rare (but possible) data race on the destruction of the shared_ptr
---------
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com>
Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com>
Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
[ROCm/rocprofiler-sdk commit: 4fa165ec1a]
This commit is contained in:
Mythreya
GitHub
@@ -208,6 +208,25 @@ get_memory_copy_trace_file()
|
||||
return _v;
|
||||
}
|
||||
|
||||
auto&
|
||||
get_scratch_memory_trace_file()
|
||||
{
|
||||
static auto* _v = new tool::output_file{"scratch_memory_trace",
|
||||
tool::csv::scratch_memory_encoder{},
|
||||
{
|
||||
"Kind",
|
||||
"Operation",
|
||||
"Agent_Id",
|
||||
"Queue_Id",
|
||||
"Thread_Id",
|
||||
"Alloc_flags",
|
||||
"Start_Timestamp",
|
||||
"End_Timestamp",
|
||||
}};
|
||||
ADD_DESTRUCTOR(_v);
|
||||
return _v;
|
||||
}
|
||||
|
||||
tool::output_file*&
|
||||
get_marker_api_file()
|
||||
{
|
||||
@@ -264,11 +283,16 @@ struct buffer_ids
|
||||
rocprofiler_buffer_id_t kernel_trace = {};
|
||||
rocprofiler_buffer_id_t memory_copy_trace = {};
|
||||
rocprofiler_buffer_id_t counter_collection = {};
|
||||
rocprofiler_buffer_id_t scratch_memory = {};
|
||||
|
||||
auto as_array() const
|
||||
{
|
||||
return std::array<rocprofiler_buffer_id_t, 5>{
|
||||
hsa_api_trace, hip_api_trace, kernel_trace, memory_copy_trace, counter_collection};
|
||||
return std::array<rocprofiler_buffer_id_t, 6>{hsa_api_trace,
|
||||
hip_api_trace,
|
||||
kernel_trace,
|
||||
memory_copy_trace,
|
||||
counter_collection,
|
||||
scratch_memory};
|
||||
}
|
||||
};
|
||||
|
||||
@@ -728,6 +752,27 @@ buffered_tracing_callback(rocprofiler_context_id_t /*context*/,
|
||||
|
||||
get_dereference(get_memory_copy_trace_file()) << memory_copy_trace_ss.str();
|
||||
}
|
||||
else if(header->kind == ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY)
|
||||
{
|
||||
auto* record = static_cast<rocprofiler_buffer_tracing_scratch_memory_record_t*>(
|
||||
header->payload);
|
||||
|
||||
auto scratch_memory_trace = std::stringstream{};
|
||||
tool::csv::scratch_memory_encoder::write_row(
|
||||
scratch_memory_trace,
|
||||
CHECK_NOTNULL(buffered_name_info)->kind_names.at(record->kind),
|
||||
CHECK_NOTNULL(buffered_name_info)
|
||||
->operation_names.at(record->kind)
|
||||
.at(record->operation),
|
||||
agent_info->at(record->agent_id)->node_id,
|
||||
record->queue_id.handle,
|
||||
record->thread_id,
|
||||
record->flags,
|
||||
record->start_timestamp,
|
||||
record->end_timestamp);
|
||||
|
||||
get_dereference(get_scratch_memory_trace_file()) << scratch_memory_trace.str();
|
||||
}
|
||||
else if(header->kind == ROCPROFILER_BUFFER_TRACING_HIP_RUNTIME_API ||
|
||||
header->kind == ROCPROFILER_BUFFER_TRACING_HIP_COMPILER_API)
|
||||
{
|
||||
@@ -1162,6 +1207,26 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
|
||||
"buffer tracing service for memory copy configure");
|
||||
}
|
||||
|
||||
if(tool::get_config().scratch_memory)
|
||||
{
|
||||
ROCPROFILER_CALL(rocprofiler_create_buffer(get_client_ctx(),
|
||||
buffer_size,
|
||||
buffer_watermark,
|
||||
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
|
||||
buffered_tracing_callback,
|
||||
tool_data,
|
||||
&get_buffers().scratch_memory),
|
||||
"buffer creation");
|
||||
|
||||
ROCPROFILER_CALL(
|
||||
rocprofiler_configure_buffer_tracing_service(get_client_ctx(),
|
||||
ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY,
|
||||
nullptr,
|
||||
0,
|
||||
get_buffers().scratch_memory),
|
||||
"buffer tracing service for scratch memory configure");
|
||||
}
|
||||
|
||||
if(tool::get_config().hsa_core_api_trace || tool::get_config().hsa_amd_ext_api_trace ||
|
||||
tool::get_config().hsa_image_ext_api_trace || tool::get_config().hsa_finalizer_ext_api_trace)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user