ROCTx Library Tracing (#390)

* Update include/rocprofiler-sdk/marker/*

- Update rocprofiler_marker_api_args_t for all API functions
- Add ROCPROFILER_MARKER_API_ID_roctxGetThreadId to rocprofiler_marker_api_id_t

* Update include/rocprofiler-sdk/marker/api_args.h

- fix include

* Update lib/common/mpl.hpp

- is_pair
- is_type_complete_v

* Update include/rocprofiler-sdk/marker/*

- fix rocprofiler_marker_api_retval_t
- add roctxGetThreadId to rocprofiler_marker_api_args_t
- fix type in enum: HsaDevice -> HsaAgent
- add table_api_id.h

* Update include/rocprofiler-sdk/marker.h

- include marker/table_api_id.h

* Update include/rocprofiler-sdk/buffer_tracing.h

- Buffer marker tracer records have begin and end timestamp

* Add lib/rocprofiler-sdk/marker

- tracing implementation for marker (roctx) library

* Update include/rocprofiler-sdk/{buffer_tracing,marker/table_api_id}.h

- rocprofiler_buffer_tracing_marker_record_t -> rocprofiler_buffer_tracing_marker_api_record_t

* Update lib/rocprofiler-sdk/buffer_tracing.cpp

- support for ROCPROFILER_BUFFER_TRACING_MARKER_API

* Update lib/rocprofiler-sdk/callback_tracing.cpp

- support for ROCPROFILER_CALLBACK_TRACING_MARKER_API

* Update lib/rocprofiler-sdk/intercept_table.cpp

- template instantiation for notify_runtime_api_registration

* Update lib/rocprofiler-sdk/registration.cpp

- enable roctx in rocprofiler_set_api_table

* Update lib/rocprofiler-sdk/marker/marker.cpp

- rocprofiler_buffer_tracing_marker_record_t -> rocprofiler_buffer_tracing_marker_api_record_t

* Update lib/rocprofiler/tests for roctx testing

- add roctx.cpp
  - unit tests for roctx callback and buffer tracing
- support marker API in get_{buffer,callback}_tracing_names()

* Update lib/common/logging.cpp

- logging initialized message mentions env variable

* Update lib/common/mpl.hpp

- NOLINT for misc-definitions-in-headers

* Update lib/rocprofiler-sdk/tests/CMakeLists.txt

- include LD_LIBRARY_PATH in rocprofiler-lib-tests-shared tests

* Update lib/rocprofiler-sdk/registration.cpp

- client_library_vec_t is now vector of option<client_library>
  - enables resetting the client_library after finalization
- removed acquiring registration lock when invoke_client_finalizers called via atexit
  - this was causing some lock-order-inversion warnings (potential deadlock)

* Update lib/rocprofiler-sdk/agent.cpp

- model name for agent supports spaces

* Update tests/common/serialization.hpp

- add serialization support for marker tracing data structures

* Update tests/apps

- Add ROCTx markers into reproducible-runtime and transpose

* Update tests/tools/json-tools.cpp

- add marker tracing support
- remove strdup (no longer necessary)

* Update tests/kernel-tracing/validate.py

- validate marker API tracing data

* Update tests/async-copy-tracing/validate.py

- validate marker API tracing data

* Update cmake for load path resolution during testing

* Update tests/async-copy-tracing/CMakeLists.txt

- fix test LD_LIBRARY_PATH

* Update cmake/Templates/rocprofiler-sdk-roctx/config.cmake.in

- fix constructing rocprofiler-sdk-roctx::rocprofiler-sdk-roctx
This commit is contained in:
Jonathan R. Madsen
2024-01-18 09:48:06 -06:00
committed by GitHub
parent b55cea0e98
commit 21dd088c8e
37 changed files with 2270 additions and 310 deletions
+4
View File
@@ -5,6 +5,10 @@ cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
project(rocprofiler-test-apps LANGUAGES C CXX)
set(CMAKE_BUILD_RPATH
"\$ORIGIN:\$ORIGIN/../lib:$<TARGET_FILE_DIR:rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library>"
)
# applications used by integration tests
add_subdirectory(reproducible-runtime)
add_subdirectory(transpose)
@@ -43,6 +43,10 @@ target_compile_options(reproducible-runtime PRIVATE -W -Wall -Wextra -Wpedantic
find_package(Threads REQUIRED)
target_link_libraries(reproducible-runtime PRIVATE Threads::Threads)
find_package(rocprofiler-sdk-roctx REQUIRED)
target_link_libraries(reproducible-runtime
PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
if(REPRODUCIBLE_RUNTIME_USE_MPI)
find_package(MPI REQUIRED)
target_compile_definitions(reproducible-runtime PRIVATE USE_MPI)
@@ -21,6 +21,7 @@
// THE SOFTWARE.
#include "hip/hip_runtime.h"
#include "rocprofiler-sdk-roctx/roctx.h"
#include <unistd.h>
#include <chrono>
@@ -124,6 +125,8 @@ reproducible_runtime(uint32_t nspin_v)
void
run(int tid, int devid)
{
auto roctx_range_id = roctxRangeStart("run");
constexpr int min_sa = 8;
constexpr int min_avail_simd = 24;
dim3 grid(min_sa * min_avail_simd);
@@ -140,6 +143,7 @@ run(int tid, int devid)
do
{
roctxMark("iteration");
uint32_t cyclesleft = 2000 * 1000 * (nruntime - static_cast<double>(time));
HIP_API_CALL(hipEventRecord(start, stream));
reproducible_runtime<<<grid, block, 0, stream>>>(std::min<uint32_t>(nspin, cyclesleft));
@@ -166,6 +170,8 @@ run(int tid, int devid)
HIP_API_CALL(hipStreamSynchronize(stream));
HIP_API_CALL(hipStreamDestroy(stream));
roctxRangeStop(roctx_range_id);
}
namespace
+3
View File
@@ -42,6 +42,9 @@ target_compile_options(transpose PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -W
find_package(Threads REQUIRED)
target_link_libraries(transpose PRIVATE Threads::Threads)
find_package(rocprofiler-sdk-roctx REQUIRED)
target_link_libraries(transpose PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
if(TRANSPOSE_USE_MPI)
find_package(MPI REQUIRED)
target_compile_definitions(transpose PRIVATE USE_MPI)
+5
View File
@@ -21,6 +21,7 @@
// THE SOFTWARE.
#include "hip/hip_runtime.h"
#include "rocprofiler-sdk-roctx/roctx.h"
#include <chrono>
#include <cstdio>
@@ -173,6 +174,8 @@ transpose(const int* in, int* out, int M, int N)
void
run(int rank, int tid, int devid, int argc, char** argv)
{
roctxRangePush("run");
constexpr unsigned int M = 4960 * 2;
constexpr unsigned int N = 4960 * 2;
@@ -243,6 +246,8 @@ run(int rank, int tid, int devid, int argc, char** argv)
delete[] inp_matrix;
delete[] out_matrix;
roctxRangePop();
}
namespace
+9 -9
View File
@@ -19,17 +19,17 @@ endif()
add_test(NAME test-async-copy-tracing-execute COMMAND $<TARGET_FILE:transpose>)
set(async-copy-tracing-env
"${PRELOAD_ENV}"
"HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>"
"ROCPROFILER_TOOL_OUTPUT_FILE=async-copy-tracing-test.json"
"LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler::rocprofiler-shared-library>:$ENV{LD_LIBRARY_PATH}"
)
set_tests_properties(
test-async-copy-tracing-execute
PROPERTIES
TIMEOUT
45
LABELS
"integration-tests"
ENVIRONMENT
"${PRELOAD_ENV};HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>;ROCPROFILER_TOOL_OUTPUT_FILE=async-copy-tracing-test.json"
FAIL_REGULAR_EXPRESSION
"threw an exception")
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT
"${async-copy-tracing-env}" FAIL_REGULAR_EXPRESSION "threw an exception")
foreach(FILENAME validate.py pytest.ini conftest.py)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
+61 -47
View File
@@ -16,45 +16,50 @@ def test_data_structure(input_data):
data = input_data
node_exists("rocprofiler-sdk-json-tool", data)
node_exists("agents", data["rocprofiler-sdk-json-tool"])
node_exists("call_stack", data["rocprofiler-sdk-json-tool"])
node_exists("callback_records", data["rocprofiler-sdk-json-tool"])
node_exists("buffer_records", data["rocprofiler-sdk-json-tool"])
node_exists("names", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("code_objects", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("kernel_symbols", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["callback_records"])
sdk_data = data["rocprofiler-sdk-json-tool"]
node_exists("names", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("kernel_dispatches", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("memory_copies", data["rocprofiler-sdk-json-tool"]["buffer_records"], 4)
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("agents", sdk_data)
node_exists("call_stack", sdk_data)
node_exists("callback_records", sdk_data)
node_exists("buffer_records", sdk_data)
node_exists("names", sdk_data["callback_records"])
node_exists("code_objects", sdk_data["callback_records"])
node_exists("kernel_symbols", sdk_data["callback_records"])
node_exists("hsa_api_traces", sdk_data["callback_records"])
node_exists("names", sdk_data["buffer_records"])
node_exists("kernel_dispatches", sdk_data["buffer_records"])
node_exists("memory_copies", sdk_data["buffer_records"], 4)
node_exists("hsa_api_traces", sdk_data["buffer_records"])
def test_timestamps(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
cb_start = {}
cb_end = {}
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
cid = itr["record"]["correlation_id"]["internal"]
phase = itr["record"]["phase"]
if phase == 1:
cb_start[cid] = itr["timestamp"]
elif phase == 2:
cb_end[cid] = itr["timestamp"]
assert cb_start[cid] <= itr["timestamp"]
else:
assert phase == 1 or phase == 2
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
cid = itr["record"]["correlation_id"]["internal"]
phase = itr["record"]["phase"]
if phase == 1:
cb_start[cid] = itr["timestamp"]
elif phase == 2:
cb_end[cid] = itr["timestamp"]
assert cb_start[cid] <= itr["timestamp"]
else:
assert phase == 1 or phase == 2
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["start_timestamp"] < itr["end_timestamp"]
assert itr["correlation_id"]["internal"] > 0
assert itr["correlation_id"]["external"] > 0
@@ -67,21 +72,23 @@ def test_timestamps(input_data):
def test_internal_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
api_corr_ids = []
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
api_corr_ids.append(itr["correlation_id"]["internal"])
for itr in sdk_data["buffer_records"][titr]:
api_corr_ids.append(itr["correlation_id"]["internal"])
api_corr_ids_sorted = sorted(api_corr_ids)
api_corr_ids_unique = list(set(api_corr_ids))
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
len_corr_id_unq = len(api_corr_ids_unique)
@@ -91,34 +98,40 @@ def test_internal_correlation_ids(input_data):
def test_external_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
extern_corr_ids = []
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
assert itr["record"]["correlation_id"]["external"] > 0
assert itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
assert itr["record"]["correlation_id"]["external"] > 0
assert (
itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
)
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
extern_corr_ids = list(set(sorted(extern_corr_ids)))
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
assert itr["correlation_id"]["external"] > 0
assert itr["thread_id"] == itr["correlation_id"]["external"]
assert itr["thread_id"] in extern_corr_ids
assert itr["correlation_id"]["external"] in extern_corr_ids
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["correlation_id"]["external"] > 0
assert itr["thread_id"] == itr["correlation_id"]["external"]
assert itr["thread_id"] in extern_corr_ids
assert itr["correlation_id"]["external"] in extern_corr_ids
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["correlation_id"]["external"] > 0
assert itr["correlation_id"]["external"] in extern_corr_ids
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["correlation_id"]["external"] > 0
assert itr["correlation_id"]["external"] in extern_corr_ids
def test_kernel_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
symbol_info = {}
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["kernel_symbols"]:
for itr in sdk_data["callback_records"]["kernel_symbols"]:
phase = itr["record"]["phase"]
payload = itr["payload"]
kern_id = payload["kernel_id"]
@@ -132,12 +145,13 @@ def test_kernel_ids(input_data):
assert payload["kernel_id"] in symbol_info.keys()
assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["kernel_id"] in symbol_info.keys()
def test_async_copy_direction(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
# Direction values:
# -1 == ??? (unknown)
@@ -146,7 +160,7 @@ def test_async_copy_direction(input_data):
# 2 == D2H (device to host)
# 3 == D2D (device to device)
async_dir_cnt = dict([(idx, 0) for idx in range(-1, 4)])
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
op_id = itr["operation"]
assert op_id > 0
assert op_id < 3
+16
View File
@@ -158,6 +158,13 @@ save(ArchiveT& ar, rocprofiler_hsa_api_retval_t data)
SAVE_DATA_FIELD(uint64_t_retval);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_marker_api_retval_t data)
{
SAVE_DATA_FIELD(int64_t_retval);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_callback_tracing_hsa_api_data_t data)
@@ -167,6 +174,15 @@ save(ArchiveT& ar, rocprofiler_callback_tracing_hsa_api_data_t data)
SAVE_DATA_FIELD(retval);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_callback_tracing_marker_api_data_t data)
{
SAVE_DATA_FIELD(size);
// SAVE_DATA_FIELD(args);
SAVE_DATA_FIELD(retval);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_callback_tracing_record_t data)
+9 -9
View File
@@ -19,17 +19,17 @@ endif()
add_test(NAME test-kernel-tracing-execute COMMAND $<TARGET_FILE:reproducible-runtime>)
set(kernel-tracing-env
"${PRELOAD_ENV}"
"HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>"
"ROCPROFILER_TOOL_OUTPUT_FILE=kernel-tracing-test.json"
"LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler::rocprofiler-shared-library>:$ENV{LD_LIBRARY_PATH}"
)
set_tests_properties(
test-kernel-tracing-execute
PROPERTIES
TIMEOUT
45
LABELS
"integration-tests"
ENVIRONMENT
"${PRELOAD_ENV};HSA_TOOLS_LIB=$<TARGET_FILE:rocprofiler::rocprofiler-shared-library>;ROCPROFILER_TOOL_OUTPUT_FILE=kernel-tracing-test.json"
FAIL_REGULAR_EXPRESSION
"threw an exception")
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${kernel-tracing-env}"
FAIL_REGULAR_EXPRESSION "threw an exception")
foreach(FILENAME validate.py pytest.ini conftest.py)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
+63 -47
View File
@@ -16,45 +16,52 @@ def test_data_structure(input_data):
data = input_data
node_exists("rocprofiler-sdk-json-tool", data)
node_exists("agents", data["rocprofiler-sdk-json-tool"])
node_exists("call_stack", data["rocprofiler-sdk-json-tool"])
node_exists("callback_records", data["rocprofiler-sdk-json-tool"])
node_exists("buffer_records", data["rocprofiler-sdk-json-tool"])
node_exists("names", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("code_objects", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("kernel_symbols", data["rocprofiler-sdk-json-tool"]["callback_records"])
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["callback_records"])
sdk_data = data["rocprofiler-sdk-json-tool"]
node_exists("names", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("kernel_dispatches", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("memory_copies", data["rocprofiler-sdk-json-tool"]["buffer_records"], 0)
node_exists("hsa_api_traces", data["rocprofiler-sdk-json-tool"]["buffer_records"])
node_exists("agents", sdk_data)
node_exists("call_stack", sdk_data)
node_exists("callback_records", sdk_data)
node_exists("buffer_records", sdk_data)
node_exists("names", sdk_data["callback_records"])
node_exists("code_objects", sdk_data["callback_records"])
node_exists("kernel_symbols", sdk_data["callback_records"])
node_exists("hsa_api_traces", sdk_data["callback_records"])
node_exists("marker_api_traces", sdk_data["callback_records"])
node_exists("names", sdk_data["buffer_records"])
node_exists("kernel_dispatches", sdk_data["buffer_records"])
node_exists("memory_copies", sdk_data["buffer_records"], 0)
node_exists("hsa_api_traces", sdk_data["buffer_records"])
node_exists("marker_api_traces", sdk_data["buffer_records"])
def test_timestamps(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
cb_start = {}
cb_end = {}
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
cid = itr["record"]["correlation_id"]["internal"]
phase = itr["record"]["phase"]
if phase == 1:
cb_start[cid] = itr["timestamp"]
elif phase == 2:
cb_end[cid] = itr["timestamp"]
assert cb_start[cid] <= itr["timestamp"]
else:
assert phase == 1 or phase == 2
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
cid = itr["record"]["correlation_id"]["internal"]
phase = itr["record"]["phase"]
if phase == 1:
cb_start[cid] = itr["timestamp"]
elif phase == 2:
cb_end[cid] = itr["timestamp"]
assert cb_start[cid] <= itr["timestamp"]
else:
assert phase == 1 or phase == 2
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["start_timestamp"] < itr["end_timestamp"]
assert itr["correlation_id"]["internal"] > 0
assert itr["correlation_id"]["external"] > 0
@@ -67,21 +74,23 @@ def test_timestamps(input_data):
def test_internal_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
api_corr_ids = []
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
api_corr_ids.append(itr["correlation_id"]["internal"])
for itr in sdk_data["buffer_records"][titr]:
api_corr_ids.append(itr["correlation_id"]["internal"])
api_corr_ids_sorted = sorted(api_corr_ids)
api_corr_ids_unique = list(set(api_corr_ids))
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
len_corr_id_unq = len(api_corr_ids_unique)
@@ -91,34 +100,40 @@ def test_internal_correlation_ids(input_data):
def test_external_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
extern_corr_ids = []
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["hsa_api_traces"]:
assert itr["record"]["correlation_id"]["external"] > 0
assert itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
assert itr["record"]["correlation_id"]["external"] > 0
assert (
itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
)
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
extern_corr_ids = list(set(sorted(extern_corr_ids)))
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["hsa_api_traces"]:
assert itr["correlation_id"]["external"] > 0
assert itr["thread_id"] == itr["correlation_id"]["external"]
assert itr["thread_id"] in extern_corr_ids
assert itr["correlation_id"]["external"] in extern_corr_ids
for titr in ["hsa_api_traces", "marker_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["correlation_id"]["external"] > 0
assert itr["thread_id"] == itr["correlation_id"]["external"]
assert itr["thread_id"] in extern_corr_ids
assert itr["correlation_id"]["external"] in extern_corr_ids
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["correlation_id"]["external"] > 0
assert itr["correlation_id"]["external"] in extern_corr_ids
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
assert itr["correlation_id"]["external"] > 0
assert itr["correlation_id"]["external"] in extern_corr_ids
def test_kernel_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
symbol_info = {}
for itr in data["rocprofiler-sdk-json-tool"]["callback_records"]["kernel_symbols"]:
for itr in sdk_data["callback_records"]["kernel_symbols"]:
phase = itr["record"]["phase"]
payload = itr["payload"]
kern_id = payload["kernel_id"]
@@ -132,12 +147,13 @@ def test_kernel_ids(input_data):
assert payload["kernel_id"] in symbol_info.keys()
assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["kernel_dispatches"]:
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
assert itr["kernel_id"] in symbol_info.keys()
def test_async_copy_direction(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
# Direction values:
# -1 == ??? (unknown)
@@ -146,7 +162,7 @@ def test_async_copy_direction(input_data):
# 2 == D2H (device to host)
# 3 == D2D (device to device)
async_dir_cnt = dict([(idx, 0) for idx in range(-1, 4)])
for itr in data["rocprofiler-sdk-json-tool"]["buffer_records"]["memory_copies"]:
for itr in sdk_data["buffer_records"]["memory_copies"]:
op_id = itr["operation"]
async_dir_cnt[op_id] += 1
+3 -10
View File
@@ -18,20 +18,13 @@ target_link_libraries(
rocprofiler::tests-common-library)
set_target_properties(
rocprofiler-sdk-json-tool
PROPERTIES INSTALL_RPATH "\$ORIGIN:\$ORIGIN/.."
INSTALL_RPATH_USE_LINK_PATH ON
PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib/rocprofiler-sdk"
SOVERSION ${PROJECT_VERSION_MINOR}
VERSION
${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH})
${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}
INSTALL_RPATH "\$ORIGIN:\$ORIGIN/..")
install(
TARGETS rocprofiler-sdk-json-tool
DESTINATION lib/rocprofiler-sdk
COMPONENT tests)
if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
set(PRELOAD_ENV
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
else()
set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
endif()
+99 -53
View File
@@ -128,6 +128,9 @@ rocprofiler_client_finalize_t client_fini_func = nullptr;
callback_name_info
get_callback_tracing_names()
{
static const auto supported = std::unordered_set<rocprofiler_callback_tracing_kind_t>{
ROCPROFILER_CALLBACK_TRACING_HSA_API, ROCPROFILER_CALLBACK_TRACING_MARKER_API};
auto cb_name_info = callback_name_info{};
//
// callback for each kind operation
@@ -136,7 +139,7 @@ get_callback_tracing_names()
[](rocprofiler_callback_tracing_kind_t kindv, uint32_t operation, void* data_v) {
auto* name_info_v = static_cast<callback_name_info*>(data_v);
if(kindv == ROCPROFILER_CALLBACK_TRACING_HSA_API)
if(supported.count(kindv) > 0)
{
const char* name = nullptr;
ROCPROFILER_CALL(rocprofiler_query_callback_tracing_kind_operation_name(
@@ -158,7 +161,7 @@ get_callback_tracing_names()
"query buffer tracing kind operation name");
if(name) name_info_v->kind_names[kind] = name;
if(kind == ROCPROFILER_CALLBACK_TRACING_HSA_API)
if(supported.count(kind) > 0)
{
ROCPROFILER_CALL(rocprofiler_iterate_callback_tracing_kind_operations(
kind, tracing_kind_operation_cb, static_cast<void*>(data)),
@@ -177,6 +180,9 @@ get_callback_tracing_names()
buffer_name_info
get_buffer_tracing_names()
{
static const auto supported = std::unordered_set<rocprofiler_buffer_tracing_kind_t>{
ROCPROFILER_BUFFER_TRACING_HSA_API, ROCPROFILER_BUFFER_TRACING_MARKER_API};
auto cb_name_info = buffer_name_info{};
//
// callback for each kind operation
@@ -185,7 +191,7 @@ get_buffer_tracing_names()
[](rocprofiler_buffer_tracing_kind_t kindv, uint32_t operation, void* data_v) {
auto* name_info_v = static_cast<buffer_name_info*>(data_v);
if(kindv == ROCPROFILER_BUFFER_TRACING_HSA_API)
if(supported.count(kindv) > 0)
{
const char* name = nullptr;
ROCPROFILER_CALL(rocprofiler_query_buffer_tracing_kind_operation_name(
@@ -207,7 +213,7 @@ get_buffer_tracing_names()
"query buffer tracing kind operation name");
if(name) name_info_v->kind_names[kind] = name;
if(kind == ROCPROFILER_BUFFER_TRACING_HSA_API)
if(supported.count(kind) > 0)
{
ROCPROFILER_CALL(rocprofiler_iterate_buffer_tracing_kind_operations(
kind, tracing_kind_operation_cb, static_cast<void*>(data)),
@@ -226,7 +232,8 @@ get_buffer_tracing_names()
using callback_payload_t =
std::variant<rocprofiler_callback_tracing_code_object_load_data_t,
rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t,
rocprofiler_callback_tracing_hsa_api_data_t>;
rocprofiler_callback_tracing_hsa_api_data_t,
rocprofiler_callback_tracing_marker_api_data_t>;
struct code_object_callback_record_t
{
@@ -273,9 +280,25 @@ struct hsa_api_callback_record_t
}
};
struct marker_api_callback_record_t
{
uint64_t timestamp = 0;
rocprofiler_callback_tracing_record_t record = {};
rocprofiler_callback_tracing_marker_api_data_t payload = {};
template <typename ArchiveT>
void save(ArchiveT& ar) const
{
ar(cereal::make_nvp("timestamp", timestamp));
ar(cereal::make_nvp("record", record));
ar(cereal::make_nvp("payload", payload));
}
};
auto code_object_records = std::deque<code_object_callback_record_t>{};
auto kernel_symbol_records = std::deque<kernel_symbol_callback_record_t>{};
auto hsa_api_cb_records = std::deque<hsa_api_callback_record_t>{};
auto marker_api_cb_records = std::deque<marker_api_callback_record_t>{};
rocprofiler_thread_id_t
push_external_correlation();
@@ -303,14 +326,12 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
{
auto data_v =
*static_cast<rocprofiler_callback_tracing_code_object_load_data_t*>(record.payload);
data_v.uri = ::strdup(data_v.uri);
code_object_records.emplace_back(code_object_callback_record_t{ts, record, data_v});
}
else if(record.operation ==
ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER)
{
auto data_v = *static_cast<kernel_symbol_data_t*>(record.payload);
data_v.kernel_name = ::strdup(data_v.kernel_name);
auto data_v = *static_cast<kernel_symbol_data_t*>(record.payload);
kernel_symbol_records.emplace_back(kernel_symbol_callback_record_t{ts, record, data_v});
}
}
@@ -319,6 +340,11 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
auto* data = static_cast<rocprofiler_callback_tracing_hsa_api_data_t*>(record.payload);
hsa_api_cb_records.emplace_back(hsa_api_callback_record_t{ts, record, *data});
}
else if(record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_API)
{
auto* data = static_cast<rocprofiler_callback_tracing_marker_api_data_t*>(record.payload);
marker_api_cb_records.emplace_back(marker_api_callback_record_t{ts, record, *data});
}
else
{
throw std::runtime_error{"unsupported callback kind"};
@@ -326,6 +352,7 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
}
auto hsa_api_bf_records = std::deque<rocprofiler_buffer_tracing_hsa_api_record_t>{};
auto marker_api_bf_records = std::deque<rocprofiler_buffer_tracing_marker_api_record_t>{};
auto kernel_dispatch_records = std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>{};
auto memory_copy_records = std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>{};
@@ -377,6 +404,13 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
hsa_api_bf_records.emplace_back(*record);
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_MARKER_API)
{
auto* record =
static_cast<rocprofiler_buffer_tracing_marker_api_record_t*>(header->payload);
marker_api_bf_records.emplace_back(*record);
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_KERNEL_DISPATCH)
{
auto* record = static_cast<rocprofiler_buffer_tracing_kernel_dispatch_record_t*>(
@@ -444,23 +478,29 @@ void
flush();
// contexts
rocprofiler_context_id_t api_callback_ctx = {};
rocprofiler_context_id_t code_object_ctx = {};
rocprofiler_context_id_t api_buffered_ctx = {};
rocprofiler_context_id_t kernel_dispatch_ctx = {};
rocprofiler_context_id_t memory_copy_ctx = {};
rocprofiler_context_id_t hsa_api_callback_ctx = {};
rocprofiler_context_id_t marker_api_callback_ctx = {};
rocprofiler_context_id_t code_object_ctx = {};
rocprofiler_context_id_t hsa_api_buffered_ctx = {};
rocprofiler_context_id_t marker_api_buffered_ctx = {};
rocprofiler_context_id_t kernel_dispatch_ctx = {};
rocprofiler_context_id_t memory_copy_ctx = {};
// buffers
rocprofiler_buffer_id_t api_buffered_buffer = {};
rocprofiler_buffer_id_t kernel_dispatch_buffer = {};
rocprofiler_buffer_id_t memory_copy_buffer = {};
rocprofiler_buffer_id_t hsa_api_buffered_buffer = {};
rocprofiler_buffer_id_t marker_api_buffered_buffer = {};
rocprofiler_buffer_id_t kernel_dispatch_buffer = {};
rocprofiler_buffer_id_t memory_copy_buffer = {};
auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
{"API_CALLBACK", &api_callback_ctx},
{"HSA_API_CALLBACK", &hsa_api_callback_ctx},
{"MARKER_API_CALLBACK", &marker_api_callback_ctx},
{"CODE_OBJECT", &code_object_ctx},
{"API_BUFFERED", &api_buffered_ctx},
{"HSA_API_BUFFERED", &hsa_api_buffered_ctx},
{"MARKER_API_BUFFERED", &marker_api_buffered_ctx},
{"KERNEL_DISPATCH", &kernel_dispatch_ctx},
{"MEMORY_COPY", &memory_copy_ctx}};
auto buffers = std::array<rocprofiler_buffer_id_t*, 3>{&api_buffered_buffer,
auto buffers = std::array<rocprofiler_buffer_id_t*, 4>{&hsa_api_buffered_buffer,
&marker_api_buffered_buffer,
&kernel_dispatch_buffer,
&memory_copy_buffer};
@@ -475,14 +515,7 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
[](const rocprofiler_agent_t** agents_arr, size_t num_agents, void* user_data) {
auto* agents_v = static_cast<std::vector<rocprofiler_agent_t>*>(user_data);
for(size_t i = 0; i < num_agents; ++i)
{
const auto* agent = agents_arr[i];
auto& val = agents_v->emplace_back(*agent);
val.name = ::strdup(agent->name);
val.vendor_name = ::strdup(agent->vendor_name);
val.product_name = ::strdup(agent->product_name);
val.model_name = ::strdup(agent->model_name);
}
agents_v->emplace_back(*agents_arr[i]);
return ROCPROFILER_STATUS_SUCCESS;
};
@@ -504,7 +537,7 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
}
ROCPROFILER_CALL(
rocprofiler_configure_callback_tracing_service(api_callback_ctx,
rocprofiler_configure_callback_tracing_service(hsa_api_callback_ctx,
ROCPROFILER_CALLBACK_TRACING_HSA_API,
nullptr,
0,
@@ -521,16 +554,34 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
nullptr),
"code object tracing service configure");
ROCPROFILER_CALL(
rocprofiler_configure_callback_tracing_service(marker_api_callback_ctx,
ROCPROFILER_CALLBACK_TRACING_MARKER_API,
nullptr,
0,
tool_tracing_callback,
nullptr),
"hsa api tracing service configure");
constexpr auto buffer_size = 8192;
constexpr auto watermark = 7936;
ROCPROFILER_CALL(rocprofiler_create_buffer(api_buffered_ctx,
ROCPROFILER_CALL(rocprofiler_create_buffer(hsa_api_buffered_ctx,
buffer_size,
watermark,
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
tool_tracing_buffered,
tool_data,
&api_buffered_buffer),
&hsa_api_buffered_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(marker_api_buffered_ctx,
buffer_size,
watermark,
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
tool_tracing_buffered,
tool_data,
&marker_api_buffered_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(kernel_dispatch_ctx,
@@ -552,8 +603,19 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
"buffer creation");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(
api_buffered_ctx, ROCPROFILER_BUFFER_TRACING_HSA_API, nullptr, 0, api_buffered_buffer),
rocprofiler_configure_buffer_tracing_service(hsa_api_buffered_ctx,
ROCPROFILER_BUFFER_TRACING_HSA_API,
nullptr,
0,
hsa_api_buffered_buffer),
"buffer tracing service configure");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(marker_api_buffered_ctx,
ROCPROFILER_BUFFER_TRACING_MARKER_API,
nullptr,
0,
marker_api_buffered_buffer),
"buffer tracing service configure");
ROCPROFILER_CALL(
@@ -650,9 +712,11 @@ tool_fini(void* tool_data)
<< ", code_object_callback_records=" << code_object_records.size()
<< ", kernel_symbol_callback_records=" << kernel_symbol_records.size()
<< ", hsa_api_callback_records=" << hsa_api_cb_records.size()
<< ", marker_api_callback_records=" << marker_api_cb_records.size()
<< ", kernel_dispatch_records=" << kernel_dispatch_records.size()
<< ", memory_copy_records=" << memory_copy_records.size()
<< ", hsa_api_bf_records=" << hsa_api_bf_records.size() << " ...\n"
<< ", hsa_api_bf_records=" << hsa_api_bf_records.size()
<< ", marker_api_bf_records=" << marker_api_bf_records.size() << " ...\n"
<< std::flush;
auto* _call_stack = static_cast<call_stack_t*>(tool_data);
@@ -713,6 +777,7 @@ tool_fini(void* tool_data)
json_ar(cereal::make_nvp("code_objects", code_object_records));
json_ar(cereal::make_nvp("kernel_symbols", kernel_symbol_records));
json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_cb_records));
json_ar(cereal::make_nvp("marker_api_traces", marker_api_cb_records));
} catch(std::exception& e)
{
std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -729,6 +794,7 @@ tool_fini(void* tool_data)
json_ar(cereal::make_nvp("kernel_dispatches", kernel_dispatch_records));
json_ar(cereal::make_nvp("memory_copies", memory_copy_records));
json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_bf_records));
json_ar(cereal::make_nvp("marker_api_traces", marker_api_bf_records));
} catch(std::exception& e)
{
std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -748,26 +814,6 @@ tool_fini(void* tool_data)
<< std::flush;
delete _call_stack;
auto free_cstr = [](const char*& val) {
::free(const_cast<char*>(val));
val = nullptr;
};
// clean up our strdups to avoid triggering our leak sanitizer during CI
for(auto& itr : code_object_records)
free_cstr(itr.payload.uri);
for(auto& itr : kernel_symbol_records)
free_cstr(itr.payload.kernel_name);
for(auto& itr : agents)
{
free_cstr(itr.name);
free_cstr(itr.vendor_name);
free_cstr(itr.product_name);
free_cstr(itr.model_name);
}
}
void