Async memory copy tracing (#317)
* Update samples/api_buffered_tracing/client.cpp
- support ROCPROFILER_BUFFER_TRACING_MEMORY_COPY
* Update include/rocprofiler-sdk/{buffer_tracing,fwd}.h
- update rocprofiler_buffer_tracing_memory_copy_record_t
- add ROCPROFILER_BUFFER_TRACING_MEMORY_COPY_HOST_TO_HOST to rocprofiler_memory_copy_operation_t
* Update lib/rocprofiler-sdk/context/context.*
- get_registered_contexts functions (local copy)
* Update tests/apps/reproducible-runtime/reproducible-runtime.cpp
- include some memory allocations and memory copies for better testing
* Update tests/common/serialization.hpp
- update serialization save function for rocprofiler_buffer_tracing_memory_copy_record_t
* Update lib/rocprofiler-sdk/hsa/hsa.*
- remove stale set_callback / activity_functor_t code
- forward decl hsa_api_meta
- template struct hsa_api_func for getting function return type and args
* Update tests/kernel-tracing/validate.py
- enforce memory_copies data size
- test timestamps in memory copies data
- improve internal and external correlation id validation
* Update lib/rocprofiler-sdk/hsa/defines.hpp
- HSA_API_META_DEFINITION macro
* Update lib/rocprofiler/hsa/rocprofiler-sdk/hsa/hsa.def.cpp
- HSA_API_META_DEFINITION specializations for async copy functions
* Add lib/rocprofiler-sdk/hsa/async_copy.{hpp,cpp}
- implements buffer memory tracing
* Update lib/rocprofiler-sdk/registration.cpp
- invoke rocprofiler::hsa::async_copy_init
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- logging improvements
- improve hsa <-> rocp agent mapping
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- load original signal in async signal handler before store_screlease
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- use store_relaxed instead of store_screlease
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- logging
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- logging
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- misc changes
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- misc changes
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- misc changes
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- return function pointer instead of lambda
* Update reproducible-runtime.cpp
- device sync
* Update tests/apps/reproducible-runtime/reproducible-runtime.cpp
- use *Async variants of hipMalloc and hipMemcpy
* Update lib/rocprofiler-sdk/hsa/async_copy.cpp
- populate async data properly
* Update tests/kernel-tracing/validate.py
- verification of async copy direction
* Update tests/apps/reproducible-runtime/reproducible-runtime.cpp
- temporarily disable async memcpy functions
* Create tests/tools
- directory containing tool libraries used for collecting data in integration tests
* Update tests/kernel-tracing
- remove kernel-tracing-test-tool library (now rocprofiler-sdk-json-tool)
- update cmake, validate.py, conftest.py accordingly
* Add tests/async-copy-tracing
- integration test validating async copy tracing in transpose example
* Update tests/CMakeLists.txt
- updates for restructuring
* Revert tests/apps/reproducible-runtime
- restore code to semi-original state (no memory copying)
* Update tests/async-copy-tracing/validate.py
- fix comment in test_async_copy_direction
* Fix building tests against installation
[ROCm/rocprofiler-sdk commit: 936816f762]
Esse commit está contido em:
Jonathan R. Madsen
GitHub
@@ -15,21 +15,21 @@ def test_data_structure(input_data):
|
||||
"""verify minimum amount of expected data is present"""
|
||||
data = input_data
|
||||
|
||||
node_exists("kernel-tracing-test-tool", data)
|
||||
node_exists("agents", data["kernel-tracing-test-tool"])
|
||||
node_exists("call_stack", data["kernel-tracing-test-tool"])
|
||||
node_exists("callback_records", data["kernel-tracing-test-tool"])
|
||||
node_exists("buffer_records", data["kernel-tracing-test-tool"])
|
||||
node_exists("rocprofiler-sdk-json-tool", data)
|
||||
node_exists("agents", data["rocprofiler-sdk-json-tool"])
|
||||
node_exists("call_stack", data["rocprofiler-sdk-json-tool"])
|
||||
node_exists("callback_records", data["rocprofiler-sdk-json-tool"])
|
||||
node_exists("buffer_records", data["rocprofiler-sdk-json-tool"])
|
||||
|
||||
node_exists("names", data["kernel-tracing-test-tool"]["callback_records"])
|
||||
node_exists("code_objects", data["kernel-tracing-test-tool"]["callback_records"])
|
||||
node_exists("kernel_symbols", data["kernel-tracing-test-tool"]["callback_records"])
|
||||
node_exists("hsa_api_traces", data["kernel-tracing-test-tool"]["callback_records"])
|
||||
node_exists("names", data["rocprofiler-sdk-json-tool"]["callback_records"])
|
||||
node_exists("code_objects", data["rocprofiler-sdk-json-tool"]["callback_records"])
|
||||
node_exists("kernel_symbols", data["rocprofiler-sdk-json-tool"]["callback_records"])
|
||||
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["callback_records"])
|
||||
|
||||
node_exists("names", data["kernel-tracing-test-tool"]["buffer_records"])
|
||||
node_exists("kernel_dispatches", data["kernel-tracing-test-tool"]["buffer_records"])
|
||||
node_exists("memory_copies", data["kernel-tracing-test-tool"]["buffer_records"], 0)
|
||||
node_exists("hsa_api_traces", data["kernel-tracing-test-tool"]["buffer_records"])
|
||||
node_exists("names", data["rocprofiler-sdk-json-tool"]["buffer_records"])
|
||||
node_exists("kernel_dispatches", data["rocprofiler-sdk-json-tool"]["buffer_records"])
|
||||
node_exists("memory_copies", data["rocprofiler-sdk-json-tool"]["buffer_records"], 0)
|
||||
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["buffer_records"])
|
||||
|
||||
|
||||
def test_timestamps(input_data):
|
||||
@@ -37,7 +37,7 @@ def test_timestamps(input_data):
|
||||
|
||||
cb_start = {}
|
||||
cb_end = {}
|
||||
for itr in data["kernel-tracing-test-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
cid = itr["record"]["correlation_id"]["internal"]
|
||||
phase = itr["record"]["phase"]
|
||||
if phase == 1:
|
||||
@@ -48,10 +48,13 @@ def test_timestamps(input_data):
|
||||
else:
|
||||
assert phase == 1 or phase == 2
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
assert itr["start_timestamp"] <= itr["end_timestamp"]
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
|
||||
assert itr["start_timestamp"] <= itr["end_timestamp"]
|
||||
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["start_timestamp"] < itr["end_timestamp"]
|
||||
assert itr["correlation_id"]["internal"] > 0
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
@@ -66,15 +69,21 @@ def test_internal_correlation_ids(input_data):
|
||||
data = input_data
|
||||
|
||||
api_corr_ids = []
|
||||
for itr in data["kernel-tracing-test-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
api_corr_ids.append(itr["correlation_id"]["internal"])
|
||||
|
||||
api_corr_ids_sorted = sorted(api_corr_ids)
|
||||
api_corr_ids_unique = list(set(api_corr_ids))
|
||||
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
|
||||
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
|
||||
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
|
||||
|
||||
len_corr_id_unq = len(api_corr_ids_unique)
|
||||
assert len(api_corr_ids) != len_corr_id_unq
|
||||
assert max(api_corr_ids_sorted) == len_corr_id_unq
|
||||
@@ -83,18 +92,33 @@ def test_internal_correlation_ids(input_data):
|
||||
def test_external_correlation_ids(input_data):
|
||||
data = input_data
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
extern_corr_ids = []
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
|
||||
assert itr["record"]["correlation_id"]["external"] > 0
|
||||
assert itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
|
||||
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
extern_corr_ids = list(set(sorted(extern_corr_ids)))
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["thread_id"] == itr["correlation_id"]["external"]
|
||||
assert itr["thread_id"] in extern_corr_ids
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
|
||||
def test_kernel_ids(input_data):
|
||||
data = input_data
|
||||
|
||||
symbol_info = {}
|
||||
for itr in data["kernel-tracing-test-tool"]["callback_records"]["kernel_symbols"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["kernel_symbols"]:
|
||||
phase = itr["record"]["phase"]
|
||||
payload = itr["payload"]
|
||||
kern_id = payload["kernel_id"]
|
||||
@@ -108,10 +132,33 @@ def test_kernel_ids(input_data):
|
||||
assert payload["kernel_id"] in symbol_info.keys()
|
||||
assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
|
||||
|
||||
for itr in data["kernel-tracing-test-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["kernel_id"] in symbol_info.keys()
|
||||
|
||||
|
||||
def test_async_copy_direction(input_data):
|
||||
data = input_data
|
||||
|
||||
# Direction values:
|
||||
# -1 == ??? (unknown)
|
||||
# 0 == H2H (host to host)
|
||||
# 1 == H2D (host to device)
|
||||
# 2 == D2H (device to host)
|
||||
# 3 == D2D (device to device)
|
||||
async_dir_cnt = dict([(idx, 0) for idx in range(-1, 4)])
|
||||
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
|
||||
op_id = itr["operation"]
|
||||
async_dir_cnt[op_id] += 1
|
||||
|
||||
# in the reproducible-runtime test which generates the input file,
|
||||
# we don't expect any async memory copy operations
|
||||
assert async_dir_cnt[-1] == 0
|
||||
assert async_dir_cnt[0] == 0
|
||||
assert async_dir_cnt[1] == 0
|
||||
assert async_dir_cnt[2] == 0
|
||||
assert async_dir_cnt[3] == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
|
||||
sys.exit(exit_code)
|
||||
|
||||
Referência em uma Nova Issue
Bloquear um usuário