Page migration reporting (#651)
* Page migration reporting support * Page migration: Update parser and reporting Container does not lave latest KFD header, so CI might fail * Add kfd_ioctl.h * Formatting * Update get_key - get key was not used (and shouldn't be), so delete it * clang-tidy fixes * Tests for page migration * Apply suggestions from code review Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update tests/bin/page-migration/CMakeLists.txt Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update page-migration test app - add hipHostRegister to register mmap'ed allocation with HIP - misc cleanup and reorg - remove HSA_XNACK=1 from test env * Update lib/rocprofiler-sdk/tests/page_migration.cpp - fix compilation error * Minor updates (reorg, rename) * Page migration reporting support * Page migration: Update parser and reporting Container does not lave latest KFD header, so CI might fail * Update page migration tests, fix trigger types * Page Migration Tracing Support Refactoring (#753) * Reorganization * Update page migration init/fini * Formatting * Update page_migration.cpp - change logging severity * Skip test if KFD does not support page migration reporting * Rework skipping test if KFD does not support page migration * Fix event trigger enum values * Fix clang-diagnostic-unused-const-variable --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com> Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com>
Этот коммит содержится в:
Mythreya
GitHub
@@ -0,0 +1,65 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
project(
|
||||
rocprofiler-tests-page-migration
|
||||
LANGUAGES CXX
|
||||
VERSION 0.0.0)
|
||||
|
||||
find_package(rocprofiler-sdk REQUIRED)
|
||||
|
||||
if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
|
||||
set(PRELOAD_ENV
|
||||
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
|
||||
else()
|
||||
set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
|
||||
endif()
|
||||
|
||||
add_test(NAME test-page-migration-execute COMMAND $<TARGET_FILE:page-migration>)
|
||||
|
||||
set(page-migration-env
|
||||
"${PRELOAD_ENV}"
|
||||
"ROCPROFILER_TOOL_OUTPUT_FILE=page-migration-test.json"
|
||||
"LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler::rocprofiler-shared-library>:$ENV{LD_LIBRARY_PATH}"
|
||||
)
|
||||
|
||||
set_tests_properties(
|
||||
test-page-migration-execute
|
||||
PROPERTIES TIMEOUT
|
||||
45
|
||||
LABELS
|
||||
"integration-tests"
|
||||
ENVIRONMENT
|
||||
"${page-migration-env}"
|
||||
FAIL_REGULAR_EXPRESSION
|
||||
"${ROCPROFILER_DEFAULT_FAIL_REGEX}"
|
||||
SKIP_REGULAR_EXPRESSION
|
||||
"KFD does not support SVM event reporting"
|
||||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
foreach(FILENAME validate.py pytest.ini conftest.py)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILENAME}
|
||||
${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} COPYONLY)
|
||||
endforeach()
|
||||
|
||||
add_test(NAME test-page-migration-validate
|
||||
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
|
||||
${CMAKE_CURRENT_BINARY_DIR}/page-migration-test.json)
|
||||
|
||||
set_tests_properties(
|
||||
test-page-migration-validate
|
||||
PROPERTIES TIMEOUT
|
||||
45
|
||||
LABELS
|
||||
"integration-tests"
|
||||
DEPENDS
|
||||
test-page-migration-execute
|
||||
FAIL_REGULAR_EXPRESSION
|
||||
"${ROCPROFILER_DEFAULT_FAIL_REGEX}"
|
||||
SKIP_REGULAR_EXPRESSION
|
||||
"KFD does not support SVM event reporting"
|
||||
WORKING_DIRECTORY
|
||||
${CMAKE_CURRENT_BINARY_DIR})
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import json
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption(
|
||||
"--input",
|
||||
action="store",
|
||||
default="page-migration-test.json",
|
||||
help="Input JSON",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def input_data(request):
|
||||
filename = request.config.getoption("--input")
|
||||
data = None
|
||||
with open(filename, "r") as inp:
|
||||
data = json.load(inp)
|
||||
|
||||
if data["rocprofiler-sdk-json-tool"]["metadata"]["validate_page_migration"] is False:
|
||||
return pytest.skip(
|
||||
"Skipping test because KFD does not support SVM event reporting"
|
||||
)
|
||||
return data
|
||||
@@ -0,0 +1,4 @@
|
||||
|
||||
[pytest]
|
||||
addopts = --durations=20 -rA -s -vv
|
||||
testpaths = validate.py
|
||||
@@ -0,0 +1,344 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from collections import defaultdict
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
|
||||
|
||||
# helper function
|
||||
def node_exists(name, data, min_len=1):
|
||||
assert name in data
|
||||
assert data[name] is not None
|
||||
if isinstance(data[name], (list, tuple, dict, set)):
|
||||
assert len(data[name]) >= min_len
|
||||
|
||||
|
||||
def to_dict(key_values):
|
||||
a = defaultdict()
|
||||
for kv in key_values:
|
||||
a[kv["key"]] = kv["value"]
|
||||
return a
|
||||
|
||||
|
||||
def op_name(op_name, record):
|
||||
found_op = False
|
||||
op_key = None
|
||||
|
||||
for kind_node in record["names"]["kind_names"]:
|
||||
if kind_node["value"] == op_name:
|
||||
op_key = kind_node["key"]
|
||||
|
||||
for op_node in record["names"]["operation_names"]:
|
||||
if op_node["key"] == op_key:
|
||||
return op_key, to_dict(op_node["value"])
|
||||
|
||||
|
||||
def dict_from_value_key(d):
|
||||
ret_d = defaultdict()
|
||||
|
||||
for k, v in d.items():
|
||||
assert v not in ret_d
|
||||
ret_d[v] = k
|
||||
return ret_d
|
||||
|
||||
|
||||
def sort_by_timestamp(lines):
|
||||
timestamp_line_map = {}
|
||||
|
||||
for log_line in lines:
|
||||
timestamp = log_line.split(" ")[1]
|
||||
timestamp_line_map[timestamp] = log_line
|
||||
|
||||
timestamps_sorted = sorted([l.split(" ")[1] for l in lines])
|
||||
return timestamps_sorted, timestamp_line_map
|
||||
|
||||
|
||||
# ------------------------------ Tests ------------------------------ #
|
||||
|
||||
|
||||
def test_data_structure(input_data):
|
||||
"""verify minimum amount of expected data is present"""
|
||||
data = input_data
|
||||
|
||||
node_exists("rocprofiler-sdk-json-tool", data)
|
||||
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
node_exists("metadata", sdk_data)
|
||||
node_exists("pid", sdk_data["metadata"])
|
||||
node_exists("main_tid", sdk_data["metadata"])
|
||||
node_exists("init_time", sdk_data["metadata"])
|
||||
node_exists("fini_time", sdk_data["metadata"])
|
||||
node_exists("validate_page_migration", sdk_data["metadata"])
|
||||
|
||||
assert sdk_data["metadata"]["validate_page_migration"] is True
|
||||
|
||||
node_exists("agents", sdk_data)
|
||||
node_exists("call_stack", sdk_data)
|
||||
node_exists("callback_records", sdk_data)
|
||||
node_exists("buffer_records", sdk_data)
|
||||
|
||||
node_exists("names", sdk_data["callback_records"])
|
||||
node_exists("code_objects", sdk_data["callback_records"])
|
||||
node_exists("kernel_symbols", sdk_data["callback_records"])
|
||||
node_exists("hsa_api_traces", sdk_data["callback_records"])
|
||||
node_exists("hip_api_traces", sdk_data["callback_records"], 0)
|
||||
node_exists("marker_api_traces", sdk_data["callback_records"], 0)
|
||||
|
||||
node_exists("names", sdk_data["buffer_records"])
|
||||
node_exists("kernel_dispatches", sdk_data["buffer_records"])
|
||||
node_exists("memory_copies", sdk_data["buffer_records"], 0)
|
||||
node_exists("hsa_api_traces", sdk_data["buffer_records"])
|
||||
node_exists("hip_api_traces", sdk_data["buffer_records"], 0)
|
||||
node_exists("marker_api_traces", sdk_data["buffer_records"], 0)
|
||||
node_exists("retired_correlation_ids", sdk_data["buffer_records"])
|
||||
node_exists("page_migration", sdk_data["buffer_records"])
|
||||
|
||||
|
||||
def test_timestamps(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
cb_start = {}
|
||||
cb_end = {}
|
||||
for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["callback_records"][titr]:
|
||||
cid = itr["record"]["correlation_id"]["internal"]
|
||||
phase = itr["record"]["phase"]
|
||||
if phase == 1:
|
||||
cb_start[cid] = itr["timestamp"]
|
||||
elif phase == 2:
|
||||
cb_end[cid] = itr["timestamp"]
|
||||
assert cb_start[cid] <= itr["timestamp"]
|
||||
else:
|
||||
assert phase == 1 or phase == 2
|
||||
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["start_timestamp"] <= itr["end_timestamp"]
|
||||
|
||||
for titr in ["kernel_dispatches", "memory_copies"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["start_timestamp"] < itr["end_timestamp"]
|
||||
assert itr["correlation_id"]["internal"] > 0
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
|
||||
assert sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
|
||||
assert sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
|
||||
assert sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
|
||||
|
||||
# api_start = cb_start[itr["correlation_id"]["internal"]]
|
||||
# api_end = cb_end[itr["correlation_id"]["internal"]]
|
||||
# assert api_start < itr["start_timestamp"]
|
||||
# assert api_end <= itr["end_timestamp"]
|
||||
|
||||
|
||||
def test_internal_correlation_ids(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
api_corr_ids = []
|
||||
for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["callback_records"][titr]:
|
||||
api_corr_ids.append(itr["record"]["correlation_id"]["internal"])
|
||||
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
api_corr_ids.append(itr["correlation_id"]["internal"])
|
||||
|
||||
api_corr_ids_sorted = sorted(api_corr_ids)
|
||||
api_corr_ids_unique = list(set(api_corr_ids))
|
||||
|
||||
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
|
||||
|
||||
for itr in sdk_data["buffer_records"]["memory_copies"]:
|
||||
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
|
||||
|
||||
len_corr_id_unq = len(api_corr_ids_unique)
|
||||
assert len(api_corr_ids) != len_corr_id_unq
|
||||
assert max(api_corr_ids_sorted) == len_corr_id_unq
|
||||
|
||||
|
||||
def test_external_correlation_ids(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
extern_corr_ids = []
|
||||
for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["callback_records"][titr]:
|
||||
assert itr["record"]["correlation_id"]["external"] > 0
|
||||
assert (
|
||||
itr["record"]["thread_id"] == itr["record"]["correlation_id"]["external"]
|
||||
)
|
||||
extern_corr_ids.append(itr["record"]["correlation_id"]["external"])
|
||||
|
||||
extern_corr_ids = list(set(sorted(extern_corr_ids)))
|
||||
for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["thread_id"] == itr["correlation_id"]["external"]
|
||||
assert itr["thread_id"] in extern_corr_ids
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
for itr in sdk_data["buffer_records"]["memory_copies"]:
|
||||
assert itr["correlation_id"]["external"] > 0
|
||||
assert itr["correlation_id"]["external"] in extern_corr_ids
|
||||
|
||||
|
||||
def test_kernel_ids(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
symbol_info = {}
|
||||
for itr in sdk_data["callback_records"]["kernel_symbols"]:
|
||||
phase = itr["record"]["phase"]
|
||||
payload = itr["payload"]
|
||||
kern_id = payload["kernel_id"]
|
||||
|
||||
assert phase == 1 or phase == 2
|
||||
assert kern_id > 0
|
||||
if phase == 1:
|
||||
assert len(payload["kernel_name"]) > 0
|
||||
symbol_info[kern_id] = payload
|
||||
elif phase == 2:
|
||||
assert payload["kernel_id"] in symbol_info.keys()
|
||||
assert payload["kernel_name"] == symbol_info[kern_id]["kernel_name"]
|
||||
|
||||
for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
|
||||
assert itr["kernel_id"] in symbol_info.keys()
|
||||
|
||||
|
||||
def test_retired_correlation_ids(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
|
||||
def _sort_dict(inp):
|
||||
return dict(sorted(inp.items()))
|
||||
|
||||
api_corr_ids = {}
|
||||
for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
corr_id = itr["correlation_id"]["internal"]
|
||||
assert corr_id not in api_corr_ids.keys()
|
||||
api_corr_ids[corr_id] = itr
|
||||
|
||||
async_corr_ids = {}
|
||||
for titr in ["kernel_dispatches", "memory_copies"]:
|
||||
for itr in sdk_data["buffer_records"][titr]:
|
||||
corr_id = itr["correlation_id"]["internal"]
|
||||
assert corr_id not in async_corr_ids.keys()
|
||||
async_corr_ids[corr_id] = itr
|
||||
|
||||
retired_corr_ids = {}
|
||||
for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
|
||||
corr_id = itr["internal_correlation_id"]
|
||||
assert corr_id not in retired_corr_ids.keys()
|
||||
retired_corr_ids[corr_id] = itr
|
||||
|
||||
api_corr_ids = _sort_dict(api_corr_ids)
|
||||
async_corr_ids = _sort_dict(async_corr_ids)
|
||||
retired_corr_ids = _sort_dict(retired_corr_ids)
|
||||
|
||||
for cid, itr in async_corr_ids.items():
|
||||
assert cid in retired_corr_ids.keys()
|
||||
ts = retired_corr_ids[cid]["timestamp"]
|
||||
assert (ts - itr["end_timestamp"]) > 0, f"correlation-id: {cid}, data: {itr}"
|
||||
|
||||
for cid, itr in api_corr_ids.items():
|
||||
assert cid in retired_corr_ids.keys()
|
||||
ts = retired_corr_ids[cid]["timestamp"]
|
||||
assert (ts - itr["end_timestamp"]) > 0, f"correlation-id: {cid}, data: {itr}"
|
||||
|
||||
assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
|
||||
|
||||
|
||||
def get_allocated_pages(callback_records):
|
||||
# Get how many pages we allocated
|
||||
hip_api_traces = callback_records["hip_api_traces"]
|
||||
_, op_dict = op_name("HIP_RUNTIME_API", callback_records)
|
||||
op_key = [k for k, v in op_dict.items() if v == "hipHostRegister"][0]
|
||||
|
||||
host_register_record = []
|
||||
for r in hip_api_traces:
|
||||
if (
|
||||
r["record"]["operation"] == op_key
|
||||
and "sizeBytes" in r["args"]
|
||||
and "hostPtr" in r["args"]
|
||||
):
|
||||
host_register_record.append(r)
|
||||
|
||||
assert len(host_register_record) == 1
|
||||
alloc_size = int(host_register_record[0]["args"]["sizeBytes"], 10)
|
||||
start_addr = int(host_register_record[0]["args"]["hostPtr"], 16)
|
||||
end_addr = start_addr + alloc_size
|
||||
|
||||
return start_addr, end_addr, alloc_size
|
||||
|
||||
|
||||
def test_page_migration_data(input_data):
|
||||
data = input_data
|
||||
sdk_data = data["rocprofiler-sdk-json-tool"]
|
||||
buffer_records = sdk_data["buffer_records"]
|
||||
callback_records = sdk_data["callback_records"]
|
||||
page_migtation_buffers = buffer_records["page_migration"]
|
||||
|
||||
bf_op_id, bf_op_names = op_name("PAGE_MIGRATION", buffer_records)
|
||||
assert bf_op_names[0] == "NONE"
|
||||
assert "PAGE_MIGRATE" in str(bf_op_names)
|
||||
assert len(bf_op_names) == 5
|
||||
|
||||
node_ids = set(x["gpu_id"] for x in sdk_data["agents"])
|
||||
start_addr, end_addr, alloc_size = get_allocated_pages(callback_records)
|
||||
|
||||
assert start_addr < end_addr and start_addr + alloc_size == end_addr
|
||||
assert int(alloc_size) == 16 * 4096 # We allocated 16 pages in the test
|
||||
|
||||
# PID must be same
|
||||
assert len(set(r["pid"] for r in page_migtation_buffers)) == 1
|
||||
|
||||
for r in page_migtation_buffers:
|
||||
op = r["operation"]
|
||||
|
||||
assert r["size"] == 136
|
||||
assert r["kind"] == bf_op_id
|
||||
assert op != 0 and bf_op_names[op] != "NONE"
|
||||
assert bf_op_names[op].lower() in r
|
||||
|
||||
if "page_migrate" in r:
|
||||
assert r["page_migrate"]["from_node"] in node_ids
|
||||
assert r["page_migrate"]["to_node"] in node_ids
|
||||
assert r["page_migrate"]["prefetch_node"] in node_ids
|
||||
assert r["page_migrate"]["preferred_node"] in node_ids
|
||||
assert r["page_migrate"]["trigger"] >= 0
|
||||
|
||||
if "queue_suspend" in r:
|
||||
assert r["queue_suspend"]["trigger"] >= 0
|
||||
assert r["queue_suspend"]["node_id"] in node_ids
|
||||
|
||||
if "unmap_from_gpu" in r:
|
||||
assert r["unmap_from_gpu"]["trigger"] >= 0
|
||||
# unmap is "instantaneous"
|
||||
assert 0 < r["start_timestamp"] == r["end_timestamp"]
|
||||
else:
|
||||
assert 0 < r["start_timestamp"] < r["end_timestamp"]
|
||||
|
||||
# Check for events with our page
|
||||
for r in page_migtation_buffers:
|
||||
|
||||
if "page_migrate" in r and r["page_migrate"]["start_addr"] == start_addr:
|
||||
assert end_addr == r["page_migrate"]["end_addr"]
|
||||
|
||||
if "unmap_from_gpu" in r and r["unmap_from_gpu"]["start_addr"] == start_addr:
|
||||
assert end_addr == r["unmap_from_gpu"]["end_addr"]
|
||||
|
||||
# TODO: Check if a migrate a->b is paired up with b->a
|
||||
# It may not always be reported towards app finalization
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
|
||||
sys.exit(exit_code)
|
||||
Ссылка в новой задаче
Block a user