AI/rocm-systems
2
0
Fork 0
Código Issues Pull Requests Ações Pacotes Projetos Versões Wiki Atividade

rocprofv3: summary reports + more JSON metadata (#1029)

Ver Código-Fonte 
* Move include/rocprofiler-sdk/cxx/details/delimit.hpp to tokenize.hpp

* Update docs/how-to/using-rocprofv3.rst

- fix code block indents
- reorder rocprofv3 options, limit them to important options
- add docs for `--runtime-trace`

* Update rocprofv3.py

- parser argument groups
- new `--runtime-trace` option
- new `--summary` option
- new `--summary-per-domain` option
- new `--summary-groups` option
- new `--summary-output-file` option
- new `--summary-units` option

* Update lib/rocprofiler-sdk/hsa/async_copy.cpp

- fix async copy operation names: add "MEMORY_COPY_" prefix

* lib/rocprofiler-sdk-tool: update statistics.{hpp,cpp}

- statistics<>::get_percent function
- stats_entry_t struct
- stats_formatter struct
- percentage struct
- std::to_string(::rocprofiler::tool::percentage)

* lib/rocprofiler-sdk-tool: update domain_type.{hpp,cpp}

- reorder domain_type enum values

* lib/rocprofiler-sdk-tool: update generateCSV.{hpp,cpp}

- separate writing CSV from accumulating statistics
- a lot of functionality was moved to statistics.{hpp,cpp}

* lib/rocprofiler-sdk-tool: update output_file.{hpp,cpp}

- output_stream_t struct
- get_output_stream(...) returns output_stream_t instance

* lib/rocprofiler-sdk-tool: update generateJSON.cpp

- update get_output_stream usage to output_stream_t

* lib/rocprofiler-sdk-tool: update generateOTF2.cpp

- header include order tweak

* lib/rocprofiler-sdk-tool: update buffered_output.hpp

- stats_data_t was renamed to stats_entry_t

* lib/rocprofiler-sdk-tool: update generatePerfetto.cpp

- header include tweak

* lib/rocprofiler-sdk-tool: update tmp_file_buffer.hpp

- emit warning message if write_ring_buffer fails after offloading instead of aborting
- prefer placement new instead of assignment in write_ring_buffer

* lib/rocprofiler-sdk-tool: add generateStats.{hpp,cpp}

- functions for accumulating statistics

* Update tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt

- accommodate tweak to CSV output file name for HIP and HSA traces

* lib/rocprofiler-sdk-tool: update config.{hpp,cpp}

- new config variables
  - stats_summary
  - stats_summary_per_domain
  - summary_output
  - stats_summary_unit_value
  - stats_summary_unit
  - stats_summary_file
  - stats_summary_groups
- support output keys for hostname: %hostname% / %h

* lib/rocprofiler-sdk-tool: update tool.cpp

- support summary output

* Documentation fixes

* Test for summary output

* Update tests/bin/transpose to use more ROCTx

- also support building with the roctracer ROCTx

* Remove roctxMark from OTF2 + fix kernel-rename tests

- following more ROCTx calls in transpose, kernel-rename validation had to be updated

* JSON metadata + JSON summary

- add serialization support for config
- add serialization support for statistics
- additions to json spec
  - rocprofiler-sdk-tool/metadata/config
  - rocprofiler-sdk-tool/metadata/command
  - rocprofiler-sdk-tool/summary
- config output_keys support for NVIDIA %q{<ENV-VAR>} syntax
- config output_keys support keys within keys

* rocprofv3 --summary-groups warning if no domain matches

- emit warning if a regex in for summary groups did not match any domain names

* Compile fix for lib/rocprofiler-sdk-tool/tool.cpp

- get_config().scratch_memory_trace
- pass contributions to write_json

* Update rocprofv3.py to preload rocprofiler-sdk-roctx

- appended to LD_PRELOAD when args.marker_trace is enabled

* Fix ReST link errors about subtitle underline being too short

* Patch tokenization of config::stats_summary_groups

- guard against array values of empty strings

* Tweak rocprofv3 summary test

- input-summary.yaml (used by rocprofv3-test-summary-inp-yaml-execute) only provides one summary group regex

* Disable LD_PRELOAD of librocprofiler-sdk-roctx.so

- this causes problems in the sanitizers, will be addressed in another PR
Esse commit está contido em:
Jonathan R. Madsen
2024-09-09 11:20:55 -05:00
commit de GitHub
pai 1354452cf0
commit 395f01b689
40 arquivos alterados com 2293 adições e 580 exclusões
Baixar Arquivo de Patch Baixar Arquivo de Diff Expandir todos os arquivos Colapsar todos os arquivos
tests/bin/transpose/CMakeLists.txt
+57 -10
Ver Arquivo
@@ -35,18 +35,65 @@ set(CMAKE_HIP_EXTENSIONS OFF)
set(CMAKE_HIP_STANDARD_REQUIRED ON)
set_source_files_properties(transpose.cpp PROPERTIES LANGUAGE HIP)
add_executable(transpose)
target_sources(transpose PRIVATE transpose.cpp)
target_compile_options(transpose PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
function(transpose_build_target _NAME _COMPILE_DEFS _LINK_TARGETS)
add_executable(${_NAME})
target_sources(${_NAME} PRIVATE transpose.cpp)
target_compile_definitions(${_NAME} PRIVATE ${_COMPILE_DEFS})
target_compile_options(${_NAME} PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
target_link_libraries(
${_NAME} PRIVATE Threads::Threads rocprofiler-sdk::tests-build-flags
${_LINK_TARGETS})
if(TRANSPOSE_USE_MPI)
find_package(MPI REQUIRED)
target_compile_definitions(${_NAME} PRIVATE USE_MPI=1)
target_link_libraries(${_NAME} PRIVATE MPI::MPI_C)
endif()
endfunction()
find_package(Threads REQUIRED)
target_link_libraries(transpose PRIVATE Threads::Threads)
find_package(rocprofiler-sdk-roctx REQUIRED)
target_link_libraries(transpose PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
if(TRANSPOSE_USE_MPI)
find_package(MPI REQUIRED)
target_compile_definitions(transpose PRIVATE USE_MPI)
target_link_libraries(transpose PRIVATE MPI::MPI_C)
transpose_build_target(transpose "" rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
#
# using old roctracer roctx
#
find_path(
roctracer_roctx_ROOT_DIR
NAMES include/roctracer/roctx.h
HINTS ${hip_DIR}
PATHS ${hip_DIR})
find_path(
roctracer_roctx_INCLUDE_DIR
NAMES roctracer/roctx.h
HINTS ${roctracer_roctx_ROOT_DIR} ${hip_DIR}
PATHS ${roctracer_roctx_ROOT_DIR} ${hip_DIR}
PATH_SUFFIXES include)
find_library(
roctracer_roctx_LIBRARY
NAMES roctx64
HINTS ${roctracer_roctx_ROOT_DIR} ${hip_DIR}
PATHS ${roctracer_roctx_ROOT_DIR} ${hip_DIR}
PATH_SUFFIXES lib lib64)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(roctracer-roctx DEFAULT_MSG roctracer_roctx_ROOT_DIR
roctracer_roctx_INCLUDE_DIR roctracer_roctx_LIBRARY)
if(roctracer-roctx_FOUND)
add_library(roctracer-roctx INTERFACE IMPORTED)
add_library(roctracer-roctx::roctracer-roctx ALIAS roctracer-roctx)
target_include_directories(roctracer-roctx INTERFACE ${roctracer_roctx_INCLUDE_DIR})
target_link_libraries(roctracer-roctx INTERFACE ${roctracer_roctx_LIBRARY})
endif()
if(TARGET roctracer-roctx::roctracer-roctx)
transpose_build_target(transpose-roctracer-roctx "USE_ROCTRACER_ROCTX=1"
roctracer-roctx::roctracer-roctx)
endif()
tests/bin/transpose/transpose.cpp
+37 -5
Ver Arquivo
@@ -20,8 +20,14 @@
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
#include "hip/hip_runtime.h"
#include "rocprofiler-sdk-roctx/roctx.h"
#include <hip/hip_runtime.h>
#include <sstream>
#if defined(USE_ROCTRACER_ROCTX)
# include <roctracer/roctx.h>
#else
# include <rocprofiler-sdk-roctx/roctx.h>
#endif
#include <chrono>
#include <cstdio>
@@ -99,6 +105,15 @@ main(int argc, char** argv)
printf("[transpose] Number of iterations: %zu\n", nitr);
printf("[transpose] Syncing every %zu iterations\n", nsync);
#if defined(USE_ROCTRACER_ROCTX)
{
auto _roctracer_roctx_ss = std::stringstream{};
_roctracer_roctx_ss << "roctracer/roctx v" << roctx_version_major() << "."
<< roctx_version_minor();
roctxMark(_roctracer_roctx_ss.str().c_str());
}
#endif
#if defined(USE_MPI)
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -174,7 +189,15 @@ transpose(const int* in, int* out, int M, int N)
void
run(int rank, int tid, int devid, int argc, char** argv)
{
roctxRangePush("run");
auto roctx_run_id = roctxRangeStart("run");
const auto mark = [rank, tid, devid](std::string_view suffix) {
auto _ss = std::stringstream{};
_ss << "run/rank-" << rank << "/thread-" << tid << "/device-" << devid << "/" << suffix;
roctxMark(_ss.str().c_str());
};
mark("begin");
constexpr unsigned int M = 4960 * 2;
constexpr unsigned int N = 4960 * 2;
@@ -219,9 +242,16 @@ run(int rank, int tid, int devid, int argc, char** argv)
auto t1 = std::chrono::high_resolution_clock::now();
for(size_t i = 0; i < nitr; ++i)
{
roctxRangePush("run/iteration");
transpose<<<grid, block, 0, stream>>>(in, out, M, N);
check_hip_error();
if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
if(i % nsync == (nsync - 1))
{
roctxRangePush("run/iteration/sync");
HIP_API_CALL(hipStreamSynchronize(stream));
roctxRangePop();
}
roctxRangePop();
}
auto t2 = std::chrono::high_resolution_clock::now();
HIP_API_CALL(hipStreamSynchronize(stream));
@@ -251,7 +281,9 @@ run(int rank, int tid, int devid, int argc, char** argv)
delete[] inp_matrix;
delete[] out_matrix;
roctxRangePop();
mark("end");
roctxRangeStop(roctx_run_id);
}
namespace
tests/pytest-packages/tests/rocprofv3.py
+24
Ver Arquivo
@@ -54,6 +54,16 @@ def test_otf2_data(
otf2_data, json_data, categories=("hip", "hsa", "marker", "kernel", "memory_copy")
):
def get_operation_name(kind_id, op_id):
return json_data["rocprofiler-sdk-tool"]["strings"]["buffer_records"][kind_id][
"operations"
][op_id]
def get_kind_name(kind_id):
return json_data["rocprofiler-sdk-tool"]["strings"]["buffer_records"][kind_id][
"kind"
]
mapping = {
"hip": ("hip_api", "hip_api"),
"hsa": ("hsa_api", "hsa_api"),
@@ -72,6 +82,20 @@ def test_otf2_data(
_otf2_data = otf2_data.loc[otf2_data["category"] == otf2_category]
_json_data = json_data["rocprofiler-sdk-tool"]["buffer_records"][json_category]
# we do not encode the roctxMark "regions" in OTF2 because
# they don't map to the OTF2_REGION_ROLE_FUNCTION well
if json_category == "marker_api":
def roctx_mark_filter(val):
return (
None
if get_kind_name(val.kind) == "MARKER_CORE_API"
and get_operation_name(val.kind, val.operation) == "roctxMarkA"
else val
)
_json_data = [itr for itr in _json_data if roctx_mark_filter(itr) is not None]
assert len(_otf2_data) == len(
_json_data
), f"{otf2_category} ({len(_otf2_data)}):\n\t{_otf2_data}\n{json_category} ({len(_json_data)}):\n\t{_json_data}"
tests/rocprofv3/CMakeLists.txt
+1
Ver Arquivo
@@ -30,3 +30,4 @@ add_subdirectory(counter-collection)
add_subdirectory(hsa-queue-dependency)
add_subdirectory(kernel-rename)
add_subdirectory(aborted-app)
add_subdirectory(summary)
tests/rocprofv3/kernel-rename/validate.py
+18 -2
Ver Arquivo
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import re
import sys
import pytest
@@ -83,7 +84,15 @@ def test_kernel_trace(json_data):
def get_kind_name(kind_id):
return data["strings"]["buffer_records"][kind_id]["kind"]
valid_kernel_names = ("run",)
valid_kernel_names = "|".join(
(
"run",
"run/iteration",
"run/rank-([0-9]+)/thread-([0-9]+)/device-([0-9]+)/(begin|end)",
)
)
valid_kernel_regex = re.compile("^({})$".format(valid_kernel_names))
kernel_dispatch_data = data["buffer_records"]["kernel_dispatch"]
for dispatch in kernel_dispatch_data:
assert get_kind_name(dispatch["kind"]) == "KERNEL_DISPATCH"
@@ -97,6 +106,10 @@ def test_kernel_trace(json_data):
assert dispatch["end_timestamp"] >= dispatch["start_timestamp"]
kernel_name = get_kernel_name(dispatch_info["kernel_id"])
assert (
re.search(valid_kernel_regex, kernel_name) is None
), f"kernel '{kernel_name}' matches regular expression '{valid_kernel_names}'"
assert kernel_name not in valid_kernel_names
external_corr_id = dispatch["correlation_id"]["external"]
@@ -104,7 +117,10 @@ def test_kernel_trace(json_data):
kernel_rename = get_kernel_rename(external_corr_id)
assert kernel_rename is not None, f"{dispatch}"
assert kernel_rename in valid_kernel_names
assert kernel_rename != kernel_name, f"{dispatch}"
assert (
re.search(valid_kernel_regex, kernel_rename) is not None
), f"renamed kernel '{kernel_rename}' does not match regular expression '{valid_kernel_names}'"
def test_memory_copy_json_trace(json_data):
tests/rocprofv3/summary/CMakeLists.txt
+127
Ver Arquivo
@@ -0,0 +1,127 @@
#
# rocprofv3 tool tests for summaries
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
project(
rocprofiler-tests-rocprofv3-summary
LANGUAGES CXX
VERSION 0.0.0)
find_package(rocprofiler-sdk REQUIRED)
if(ROCPROFILER_MEMCHECK STREQUAL "LeakSanitizer")
set(LOG_LEVEL "warning") # info produces memory leak
else()
set(LOG_LEVEL "info")
endif()
string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
set(summary-env "${PRELOAD_ENV}")
rocprofiler_configure_pytest_files(CONFIG pytest.ini input-summary.yaml COPY validate.py
conftest.py)
##########################################################################################
#
# Command line input
#
##########################################################################################
set(summary-cmdl-env ${summary-env} "ARBITRARY_ENV_VARIABLE=%tag%-trace-cmd-line")
add_test(
NAME rocprofv3-test-summary-cmd-line-execute
COMMAND
$<TARGET_FILE:rocprofiler-sdk::rocprofv3> -d
${CMAKE_CURRENT_BINARY_DIR}/%tag%-trace-cmd-line -o out --output-format pftrace
json otf2 --log-level env --runtime-trace --kernel-rename --summary
--summary-output-file summary --summary-per-domain --summary-groups
"KERNEL_DISPATCH|MEMORY_COPY" ".*_API" -- $<TARGET_FILE:transpose> 2 500 10)
set_tests_properties(
rocprofv3-test-summary-cmd-line-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${summary-cmdl-env}"
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
add_test(
NAME rocprofv3-test-summary-cmd-line-validate
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json
--pftrace-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace
--otf2-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.otf2
--summary-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_summary.txt)
set(VALIDATION_FILES
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_summary.txt)
set_tests_properties(
rocprofv3-test-summary-cmd-line-validate
PROPERTIES TIMEOUT
45
LABELS
"integration-tests"
DEPENDS
"rocprofv3-test-summary-execute"
FAIL_REGULAR_EXPRESSION
"AssertionError"
ATTACHED_FILES_ON_FAIL
"${VALIDATION_FILES}")
##########################################################################################
#
# YAML input
#
##########################################################################################
set(summary-yaml-env ${summary-env} "ARBITRARY_ENV_VARIABLE=%tag%-trace-inp-yaml")
add_test(
NAME rocprofv3-test-summary-inp-yaml-execute
COMMAND
$<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
${CMAKE_CURRENT_BINARY_DIR}/input-summary.yaml -- $<TARGET_FILE:transpose> 2 500
10)
set_tests_properties(
rocprofv3-test-summary-inp-yaml-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${summary-yaml-env}"
FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
add_test(
NAME rocprofv3-test-summary-inp-yaml-validate
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json
--pftrace-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace
--otf2-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.otf2
--summary-input
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_summary.txt)
set(VALIDATION_FILES
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json
${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_summary.txt)
set_tests_properties(
rocprofv3-test-summary-inp-yaml-validate
PROPERTIES TIMEOUT
45
LABELS
"integration-tests"
DEPENDS
"rocprofv3-test-summary-inp-yaml-execute"
FAIL_REGULAR_EXPRESSION
"AssertionError"
ATTACHED_FILES_ON_FAIL
"${VALIDATION_FILES}")
tests/rocprofv3/summary/conftest.py
+108
Ver Arquivo
@@ -0,0 +1,108 @@
#!/usr/bin/env python3
import pandas as pd
import pytest
import json
import os
import io
from rocprofiler_sdk.pytest_utils.dotdict import dotdict
from rocprofiler_sdk.pytest_utils import collapse_dict_list
from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader
def pytest_addoption(parser):
parser.addoption(
"--json-input",
action="store",
help="Path to JSON file.",
)
parser.addoption(
"--pftrace-input",
action="store",
help="Path to Perfetto trace file.",
)
parser.addoption(
"--otf2-input",
action="store",
help="Path to OTF2 trace file.",
)
parser.addoption(
"--summary-input",
action="store",
help="Path to summary markdown file.",
)
@pytest.fixture
def json_data(request):
filename = request.config.getoption("--json-input")
with open(filename, "r") as inp:
return dotdict(collapse_dict_list(json.load(inp)))
@pytest.fixture
def pftrace_data(request):
filename = request.config.getoption("--pftrace-input")
return PerfettoReader(filename).read()[0]
@pytest.fixture
def otf2_data(request):
filename = request.config.getoption("--otf2-input")
if not os.path.exists(filename):
raise FileExistsError(f"{filename} does not exist")
return OTF2Reader(filename).read()[0]
@pytest.fixture
def summary_data(request):
filename = request.config.getoption("--summary-input")
if not os.path.exists(filename):
raise FileExistsError(f"{filename} does not exist")
domains = {}
with open(filename, "r") as inp:
lines = [itr.strip() for itr in inp.readlines()]
lines = [itr for itr in lines if itr and not itr.startswith("|--")]
def rework(x):
tmp = [itr.strip(" ") for itr in x.split("|")]
tmp = [itr.strip() for itr in tmp if len(itr.strip()) > 0]
if tmp[0] == "NAME":
tmp = [f'"{itr}"' for itr in tmp]
else:
tmp[0] = f'"{tmp[0]}"'
tmp[1] = f'"{tmp[1]}"'
return ",".join(tmp)
def process_current_domain(_name, _list):
if _name and _list:
_list = [rework(itr) for itr in _list]
_contents = "{}\n".format("\n".join(_list))
ifs = io.StringIO(_contents)
df = pd.read_csv(ifs)
domains[_name] = df
_name = None
_list = []
return (None, [])
current_name = None
current_list = []
for itr in lines:
if not itr.startswith("|") or itr.startswith("ROCPROFV3 "):
current_name, current_list = process_current_domain(
current_name, current_list
)
current_name = itr.strip().strip(":").replace("ROCPROFV3 ", "", 1)
rpos = current_name.rfind(" SUMMARY")
if rpos >= 0:
current_name = current_name[:rpos]
else:
current_list += [itr]
process_current_domain(current_name, current_list)
return domains
tests/rocprofv3/summary/input-summary.yaml
+11
Ver Arquivo
@@ -0,0 +1,11 @@
jobs:
- output_directory: "@CMAKE_CURRENT_BINARY_DIR@/%env{ARBITRARY_ENV_VARIABLE}%"
output_file: out
output_format: [pftrace, json, otf2]
log_level: env
runtime_trace: True
kernel_rename: True
summary: True
summary_per_domain: True
summary_groups: ["KERNEL_DISPATCH|MEMORY_COPY"]
summary_output_file: "summary"
tests/rocprofv3/summary/pytest.ini
+5
Ver Arquivo
@@ -0,0 +1,5 @@
[pytest]
addopts = --durations=20 -rA -s -vv
testpaths = validate.py
pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages
tests/rocprofv3/summary/validate.py
+267
Ver Arquivo
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
import os
import re
import sys
import pytest
def test_hip_api_trace(json_data):
data = json_data["rocprofiler-sdk-tool"]
def get_operation_name(kind_id, op_id):
return data["strings"]["buffer_records"][kind_id]["operations"][op_id]
def get_kind_name(kind_id):
return data["strings"]["buffer_records"][kind_id]["kind"]
valid_domain_names = ("HIP_RUNTIME_API",)
hip_api_data = data["buffer_records"]["hip_api"]
functions = []
for api in hip_api_data:
kind = get_kind_name(api["kind"])
assert kind in valid_domain_names
assert api["end_timestamp"] >= api["start_timestamp"]
functions.append(get_operation_name(api["kind"], api["operation"]))
functions = list(set(functions))
for itr in (
"__hipPushCallConfiguration",
"__hipPopCallConfiguration",
"__hipRegisterFatBinary",
"__hipRegisterFunction",
):
assert itr not in functions, f"{itr}"
for itr in (
"hipMallocAsync",
"hipMemcpyAsync",
"hipMemsetAsync",
"hipFreeAsync",
"hipLaunchKernel",
):
assert itr in functions, f"{itr}"
def test_kernel_trace(json_data):
data = json_data["rocprofiler-sdk-tool"]
def get_kernel_name(kernel_id):
return data["kernel_symbols"][kernel_id]["formatted_kernel_name"]
def get_kernel_rename(corr_id):
for itr in data.strings.correlation_id.external:
if itr.key == corr_id:
return itr.value
return None
def get_kind_name(kind_id):
return data["strings"]["buffer_records"][kind_id]["kind"]
valid_kernel_names = "|".join(
(
"run",
"run/iteration",
"run/rank-([0-9]+)/thread-([0-9]+)/device-([0-9]+)/(begin|end)",
)
)
valid_kernel_regex = re.compile("^({})$".format(valid_kernel_names))
kernel_dispatch_data = data["buffer_records"]["kernel_dispatch"]
for dispatch in kernel_dispatch_data:
assert get_kind_name(dispatch["kind"]) == "KERNEL_DISPATCH"
assert dispatch["correlation_id"]["internal"] > 0
assert dispatch["correlation_id"]["external"] > 0
dispatch_info = dispatch["dispatch_info"]
assert dispatch_info["agent_id"]["handle"] > 0
assert dispatch_info["queue_id"]["handle"] > 0
assert dispatch_info["kernel_id"] > 0
assert dispatch["end_timestamp"] >= dispatch["start_timestamp"]
kernel_name = get_kernel_name(dispatch_info["kernel_id"])
assert (
re.search(valid_kernel_regex, kernel_name) is None
), f"kernel '{kernel_name}' matches regular expression '{valid_kernel_names}'"
assert kernel_name not in valid_kernel_names
external_corr_id = dispatch["correlation_id"]["external"]
assert external_corr_id > 0
kernel_rename = get_kernel_rename(external_corr_id)
assert kernel_rename is not None, f"{dispatch}"
assert kernel_rename != kernel_name, f"{dispatch}"
assert (
re.search(valid_kernel_regex, kernel_rename) is not None
), f"renamed kernel '{kernel_rename}' does not match regular expression '{valid_kernel_names}'"
def test_memory_copy_trace(json_data):
data = json_data["rocprofiler-sdk-tool"]
buffer_records = data["buffer_records"]
agent_data = data["agents"]
memory_copy_data = buffer_records["memory_copy"]
def get_kind_name(kind_id):
return data["strings"]["buffer_records"][kind_id]["kind"]
def get_agent(node_id):
for agent in agent_data:
if agent["id"]["handle"] == node_id["handle"]:
return agent
return None
assert len(memory_copy_data) == 12
for row in memory_copy_data:
src_agent = get_agent(row["src_agent_id"])
dst_agent = get_agent(row["dst_agent_id"])
assert get_kind_name(row["kind"]) == "MEMORY_COPY"
assert src_agent is not None, f"{row}"
assert dst_agent is not None, f"{row}"
assert row["correlation_id"]["internal"] > 0
assert row["end_timestamp"] >= row["start_timestamp"]
def test_metadata_data(json_data):
data = json_data["rocprofiler-sdk-tool"]
# patch summary groups for testing purposes
# (it appears sometimes these contain single quotes and other times it doesn't)
data.metadata.config.summary_groups = [
f"{itr}".strip("'") for itr in data.metadata.config.summary_groups
]
num_summary_grps = len(data.metadata.config.summary_groups)
expected_summary_grps = (
[
"KERNEL_DISPATCH|MEMORY_COPY",
]
if num_summary_grps == 1
else [
"KERNEL_DISPATCH|MEMORY_COPY",
".*_API",
]
)
assert data.metadata.config.summary is True
assert data.metadata.config.summary_per_domain is True
assert data.metadata.config.summary_unit == "nsec"
assert data.metadata.config.summary_file == "summary"
assert data.metadata.config.summary_groups == expected_summary_grps
assert len(data.metadata.command) == 4
# patch command to make it easier to test
data.metadata.command[0] = os.path.basename(data.metadata.command[0])
assert data.metadata.command == ["transpose", "2", "500", "10"]
def test_summary_data(json_data):
data = json_data["rocprofiler-sdk-tool"]
domains = []
for itr in data.summary:
domains.append(itr.domain)
if itr.domain == "KERNEL_DISPATCH":
assert itr.stats.count == 1004
expected = dict([["run/iteration", 1000]])
for oitr in itr.stats.operations:
if oitr.key in expected.keys():
assert oitr.value.count == expected[oitr.key]
else:
assert oitr.key.startswith("run/rank-")
assert (
re.match(
r"run/rank-([0-9]+)/thread-([0-9]+)/device-([0-9]+)/begin",
oitr.key,
)
is not None
)
assert oitr.value.count == 2
elif itr.domain == "HIP_API":
assert itr.stats.count == 2135
elif itr.domain == "MEMORY_COPY":
assert itr.stats.count == 12
elif itr.domain == "MARKER_API":
assert itr.stats.count == 1106
expected = dict(
[
["run", 2],
["run/iteration", 1000],
["run/iteration/sync", 100],
]
)
for oitr in itr.stats.operations:
if oitr.key in expected.keys():
assert oitr.value.count == expected[oitr.key]
else:
assert oitr.key.startswith("run/rank-")
assert (
re.match(
r"run/rank-([0-9]+)/thread-([0-9]+)/device-([0-9]+)/(begin|end)",
oitr.key,
)
is not None
)
assert oitr.value.count == 1
else:
assert False, f"unhandled domain: {itr.domain}"
assert len(list(set(domains))) == len(domains)
def test_summary_display_data(json_data, summary_data):
data = json_data["rocprofiler-sdk-tool"]
num_summary_grps = len(data.metadata.config.summary_groups)
def get_df(domain):
return summary_data[domain]
def get_dims(df):
# return rows x cols
return [df.shape[0], df.shape[1]] if df is not None else [0, 0]
hip = get_df("HIP_API")
marker = get_df("MARKER_API")
dispatch = get_df("KERNEL_DISPATCH")
memcpy = get_df("MEMORY_COPY")
dispatch_and_copy = get_df("KERNEL_DISPATCH + MEMORY_COPY")
hip_and_marker = get_df("HIP_API + MARKER_API") if num_summary_grps > 1 else None
total = get_df("SUMMARY")
expected_hip_and_marker_dims = [20, 9] if hip_and_marker is not None else [0, 0]
assert get_dims(hip) == [13, 9]
assert get_dims(marker) == [7, 9]
assert get_dims(dispatch) == [3, 9]
assert get_dims(memcpy) == [2, 9]
assert get_dims(dispatch_and_copy) == [5, 9]
assert get_dims(hip_and_marker) == expected_hip_and_marker_dims
assert get_dims(total) == [22, 9]
def test_perfetto_data(pftrace_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_perfetto_data(
pftrace_data, json_data, ("hip", "marker", "kernel", "memory_copy")
)
def test_otf2_data(otf2_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_otf2_data(
otf2_data, json_data, ("hip", "marker", "kernel", "memory_copy")
)
if __name__ == "__main__":
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
sys.exit(exit_code)
tests/rocprofv3/tracing-hip-in-libraries/CMakeLists.txt
+6 -4
Ver Arquivo
@@ -53,8 +53,10 @@ add_test(
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_agent_info.csv
--kernel-stats
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_stats.csv
--hip-stats ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_stats.csv
--hsa-stats ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_stats.csv
--hip-stats
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_api_stats.csv
--hsa-stats
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_api_stats.csv
--memory-copy-stats
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv
--json-input ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.json
@@ -71,8 +73,8 @@ set(VALIDATION_FILES
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_agent_info.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_stats.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_stats.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_stats.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_api_stats.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_api_stats.csv
${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv)
set_tests_properties(
tests/rocprofv3/tracing-hip-in-libraries/validate.py
+6 -3
Ver Arquivo
@@ -392,16 +392,19 @@ def test_memory_copy_trace(
for row in memory_copy_input_data:
assert row["Kind"] == "MEMORY_COPY"
assert row["Direction"] in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
assert row["Direction"] in (
"MEMORY_COPY_HOST_TO_DEVICE",
"MEMORY_COPY_DEVICE_TO_HOST",
)
src_agent = get_agent(row["Source_Agent_Id"])
dst_agent = get_agent(row["Destination_Agent_Id"])
assert src_agent is not None and dst_agent is not None, f"{agent_info_input_data}"
if row["Direction"] == "HOST_TO_DEVICE":
if row["Direction"] == "MEMORY_COPY_HOST_TO_DEVICE":
assert src_agent["Agent_Type"] == "CPU"
assert dst_agent["Agent_Type"] == "GPU"
elif row["Direction"] == "DEVICE_TO_HOST":
elif row["Direction"] == "MEMORY_COPY_DEVICE_TO_HOST":
assert src_agent["Agent_Type"] == "GPU"
assert dst_agent["Agent_Type"] == "CPU"
tests/rocprofv3/tracing/validate.py
+8 -8
Ver Arquivo
@@ -170,13 +170,13 @@ def test_memory_copy_trace(agent_info_input_data, memory_copy_input_data):
assert len(memory_copy_input_data) == 2
def test_row(idx, direction):
assert direction in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
assert direction in ("MEMORY_COPY_HOST_TO_DEVICE", "MEMORY_COPY_DEVICE_TO_HOST")
row = memory_copy_input_data[idx]
assert row["Direction"] == direction
src_agent = get_agent(row["Source_Agent_Id"])
dst_agent = get_agent(row["Destination_Agent_Id"])
assert src_agent is not None and dst_agent is not None, f"{agent_info_input_data}"
if direction == "HOST_TO_DEVICE":
if direction == "MEMORY_COPY_HOST_TO_DEVICE":
assert src_agent["Agent_Type"] == "CPU"
assert dst_agent["Agent_Type"] == "GPU"
else:
@@ -185,8 +185,8 @@ def test_memory_copy_trace(agent_info_input_data, memory_copy_input_data):
assert int(row["Correlation_Id"]) > 0
assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
test_row(0, "HOST_TO_DEVICE")
test_row(1, "DEVICE_TO_HOST")
test_row(0, "MEMORY_COPY_HOST_TO_DEVICE")
test_row(1, "MEMORY_COPY_DEVICE_TO_HOST")
def test_memory_copy_json_trace(json_data):
@@ -208,14 +208,14 @@ def test_memory_copy_json_trace(json_data):
assert len(memory_copy_data) == 2
def test_row(idx, direction):
assert direction in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
assert direction in ("MEMORY_COPY_HOST_TO_DEVICE", "MEMORY_COPY_DEVICE_TO_HOST")
row = memory_copy_data[idx]
src_agent = get_agent(row["src_agent_id"])
dst_agent = get_agent(row["dst_agent_id"])
assert get_kind_name(row["kind"]) == "MEMORY_COPY"
assert src_agent is not None, f"{row}"
assert dst_agent is not None, f"{row}"
if direction == "HOST_TO_DEVICE":
if direction == "MEMORY_COPY_HOST_TO_DEVICE":
assert src_agent["type"] == 1
assert dst_agent["type"] == 2
else:
@@ -224,8 +224,8 @@ def test_memory_copy_json_trace(json_data):
assert row["correlation_id"]["internal"] > 0
assert row["end_timestamp"] >= row["start_timestamp"]
test_row(0, "HOST_TO_DEVICE")
test_row(1, "DEVICE_TO_HOST")
test_row(0, "MEMORY_COPY_HOST_TO_DEVICE")
test_row(1, "MEMORY_COPY_DEVICE_TO_HOST")
def test_marker_api_trace(marker_input_data):