Callback tracing for kernel dispatches + External correlation ID request service (#682)
* Support ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH * Fix doxygen * Update callback tracing - temporary hacks for kind operation name and iterate kind operations * Update source/include/rocprofiler-sdk - introduce sequence id for kernel dispatches * Update lib/rocprofiler-sdk (seq id) - support sequence id passing * Update tests (seq id) - testing for sequence ids * Cleanup include/rocprofiler-sdk/fwd.h * Misc cleanup * External Correlation ID Request Service (#699) * External correlation ID request service - callback requesting an external correlation ID instead of fetching from top of pushed external correlation ID stack * Update external correlation id request support - pass internal correlation ID in callback - async copy generates a correlation ID if none already exists - added external correlation ID request support for scratch memory tracing - updated scratch memory tracing to use tracing:: functions * Update hsa/queue.hpp - new line at EOF * Misc tweaks - remove unnecessary logging in agent.cpp - correlation_id::add_ref_count check for retirement - finalization check in HSA queue AsyncSignalHandler * Improve assertion failure logging in misc tests * Update include/rocprofiler-sdk/fwd.h - remove rocprofiler_record_counter_header_t * Move lib/rocprofiler-sdk/tracing.hpp into lib/rocprofiler-sdk/tracing/ folder * Update lib/rocprofiler-sdk/hsa/* - hsa::get_hsa_status_string - queue_info_session.hpp header - rocprofiler_packet.hpp * Update lib/rocprofiler-sdk/{counters,hip,marker} - execute_phase_exit_callbacks tweaks - queue_info_session tweaks * Move rocprofiler_kernel_dispatch_operation_t to include/rocprofiler-sdk/fwd.h * Update rocprofiler_buffer_tracing_kernel_dispatch_record_t - add operation field and thread_id field * Add lib/rocprofiler-sdk/kernel_dispatch - enum <-> string mapping for kernel dispatch - tracing implementations * Update lib/rocprofiler-sdk/CMakeLists.txt - tracing and kernel dispatch sub-directories * Update lib/rocprofiler-sdk/{buffer,callback}_tracing.cpp - invoke rocprofiler::kernel_tracing functions * Update tests/common/serialization.hpp - support operation and thread_id fields for rocprofiler_buffer_tracing_kernel_dispatch_record_t * Update tests/tools/json-tool.cpp - use external correlation id request service * Rename sequence_id to dispatch_id
This commit is contained in:
Jonathan R. Madsen
committed by
GitHub
GitHub
parent
0f5c575435
commit
56030018dc
@@ -76,13 +76,21 @@ def test_timestamps(input_data):
|
||||
|
||||
for titr in ["kernel_dispatches", "memory_copies"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["start_timestamp"] < itr["end_timestamp"]
|
||||
assert itr["correlation_id"]["internal"] > 0
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
|
||||
assert sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
|
||||
assert sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
|
||||
assert sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
|
||||
assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
|
||||
assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
|
||||
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
|
||||
assert (
|
||||
sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
|
||||
), f"[{titr}] {itr}"
|
||||
assert (
|
||||
sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
|
||||
), f"[{titr}] {itr}"
|
||||
assert (
|
||||
sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
|
||||
), f"[{titr}] {itr}"
|
||||
assert (
|
||||
sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
|
||||
), f"[{titr}] {itr}"
|
||||
|
||||
# TODO(Is this check applicable for scratch, which doesn't use any correlation id?)
|
||||
# api_start = cb_start[itr["correlation_id"]["internal"]]
|
||||
@@ -133,22 +141,20 @@ def test_external_correlation_ids(input_data):
|
||||
extern_corr_ids = list(set(sorted(extern_corr_ids)))
|
||||
for titr in ["hsa_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["thread_id"] == itr["correlation_id"]["external"]
|
||||
assert itr["thread_id"] in extern_corr_ids
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
|
||||
assert (
|
||||
itr["thread_id"] == itr["correlation_id"]["external"]
|
||||
), f"[{titr}] {itr}"
|
||||
assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
|
||||
|
||||
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
for itr in sdk_data["buffer_records"]["memory_copies"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
for titr in ["kernel_dispatches", "memory_copies"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
|
||||
|
||||
|
||||
def op_name(op_name, record):
|
||||
found_op = False
|
||||
op_key = None
|
||||
|
||||
for kind_node in record["names"]["kind_names"]:
|
||||
@@ -159,6 +165,8 @@ def op_name(op_name, record):
|
||||
if op_node["key"] == op_key:
|
||||
return op_node
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# Tests above are identical to async-copy. Update as needed
|
||||
|
||||
@@ -168,8 +176,10 @@ def test_scratch_memory_tracking(input_data):
|
||||
callback_records = sdk_data["callback_records"]
|
||||
buffer_records = sdk_data["buffer_records"]
|
||||
|
||||
scratch_callback_data = sdk_data["callback_records"]["scratch_memory_traces"]
|
||||
scratch_buffer_data = sdk_data["buffer_records"]["scratch_memory_traces"]
|
||||
scratch_callback_data = callback_records["scratch_memory_traces"]
|
||||
scratch_buffer_data = buffer_records["scratch_memory_traces"]
|
||||
|
||||
assert len(scratch_callback_data) == 2 * len(scratch_buffer_data)
|
||||
|
||||
cb_op_names = op_name("SCRATCH_MEMORY", callback_records)["value"]
|
||||
bf_op_names = op_name("SCRATCH_MEMORY", buffer_records)["value"]
|
||||
@@ -226,14 +236,23 @@ def test_scratch_memory_tracking(input_data):
|
||||
for thread_id, nodes in cb_threads.items():
|
||||
assert thread_id > 0
|
||||
|
||||
# sort based on timestamp
|
||||
nodes = sorted(nodes, key=lambda x: x["timestamp"])
|
||||
|
||||
# start must be followed by end
|
||||
for inx in range(0, len(nodes), 2):
|
||||
this_node = nodes[inx]
|
||||
next_node = nodes[inx + 1]
|
||||
|
||||
assert rc(this_node)["phase"] + 1 == rc(next_node)["phase"]
|
||||
assert rc(this_node)["thread_id"] == rc(next_node)["thread_id"]
|
||||
assert this_node["timestamp"] < next_node["timestamp"]
|
||||
assert (
|
||||
rc(this_node)["phase"] + 1 == rc(next_node)["phase"]
|
||||
), f"this:\n{this_node}\n\nnext:\n{next_node}"
|
||||
assert (
|
||||
rc(this_node)["thread_id"] == rc(next_node)["thread_id"]
|
||||
), f"this:\n{this_node}\n\nnext:\n{next_node}"
|
||||
assert (
|
||||
this_node["timestamp"] < next_node["timestamp"]
|
||||
), f"this:\n{this_node}\n\nnext:\n{next_node}"
|
||||
|
||||
# alloc has more data vs free and async reclaim
|
||||
scratch_alloc_node = (
|
||||
|
||||
Reference in New Issue
Block a user