AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Memory Allocation Tracking (#1142)

Browse Source 
* Initial commit: Need to implement wrapper function to collect data and test that wrapper function is correctly replacing core HSA functions

* Attempted to implement wrapper implementation for hsa memory allocation functions. Need to modify generate record files and test if implementation is working as expected

* Debugging and implementing generateCSV function

* Memory allocation size and starting address outputted to csv and json file formats

* Formatting

* Initial setup for OTF2 and Perfetto generation

* Collecting agent id for memory_allocation and formatting

* Modified memory_allocation.cpp to set up code for AMD_EXT commands

* Support for memory_pool_allocate added

* Removed accidently added file

* Made flag optional and added more OTF2 and Perfetto code. Needs testing to ensure perfetto and OTF2 works

* Formatting

* Fixed perfetto and otf2 output

* Fixed flag issue due to incorrect buffer use

* Updated documentation

* Small cleaning and comments

* Added test for HSA memory allocation tracing

* Fixed summary test validation errors due to allocation tracing. Added type to location_base to create unique event ids for allocation due to OTF2 trace error

* Decreased lower limit of hip calls for test

* Modified summary tests to vary number of allocate requests

* Minor fixes to address comments. Still need to address OTF2 comments

* Fix docs and changed OTF2 to use enum for type specified in location_base construction

* Fixed schema error

* Added vmem command tracking. Need to add test

* Updated test to work with vmem command and updated generateCSV to output int instead of hex string.

* OTF2 enum update and mispelling fix

* CI does not support Virtual Memory API. Removed vmem test. Will add back if CI is modifed to suport vmem API

* Update CMakeLists.txt for memory allocation test

* Updated summary test

* Minor fixes to address comments

* Moved domain_type.hpp enum to before LAST

* Fixed compile errors and formatting

* Fixed stats summary domain name error

* Added rocprofv3 test

* Page migration test fix

* Undo page migration test changes. Failures do not appear to have to do with memory allocation
This commit is contained in:
itrowbri
2024-11-18 20:22:14 -06:00
committed by GitHub
parent 0d764eb3c5
commit 3bd7773cf7
53 changed files with 2387 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/CMakeLists.txt
+1
View File
@@ -54,6 +54,7 @@ add_subdirectory(bin)
# validation tests
add_subdirectory(kernel-tracing)
add_subdirectory(async-copy-tracing)
add_subdirectory(hsa-memory-allocation)
add_subdirectory(scratch-memory-tracing)
add_subdirectory(c-tool)
add_subdirectory(page-migration)
tests/bin/CMakeLists.txt
+1
View File
@@ -24,3 +24,4 @@ add_subdirectory(scratch-memory)
add_subdirectory(page-migration)
add_subdirectory(hsa-queue-dependency)
add_subdirectory(hip-graph)
add_subdirectory(hsa-memory-allocation)
tests/bin/hsa-memory-allocation/CMakeLists.txt
+43
View File
@@ -0,0 +1,43 @@
#
#
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
if(NOT CMAKE_HIP_COMPILER)
find_program(
amdclangpp_EXECUTABLE
NAMES amdclang++
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
PATH_SUFFIXES bin llvm/bin NO_CACHE)
mark_as_advanced(amdclangpp_EXECUTABLE)
if(amdclangpp_EXECUTABLE)
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
endif()
endif()
project(rocprofiler-tool-test-app-hsa-memory-allocation LANGUAGES CXX HIP)
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
endif()
endforeach()
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_HIP_STANDARD 17)
set(CMAKE_HIP_EXTENSIONS OFF)
set(CMAKE_HIP_STANDARD_REQUIRED ON)
set_source_files_properties(hsa-memory-allocation.cpp PROPERTIES LANGUAGE HIP)
add_executable(hsa-memory-allocation)
target_sources(hsa-memory-allocation PRIVATE hsa-memory-allocation.cpp)
target_compile_options(hsa-memory-allocation PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
-Werror)
find_package(Threads REQUIRED)
target_link_libraries(hsa-memory-allocation PRIVATE Threads::Threads hsa-runtime64
rocprofiler-sdk::tests-common-library)
tests/bin/hsa-memory-allocation/hsa-memory-allocation.cpp
+267
View File
@@ -0,0 +1,267 @@
#include <hip/hip_runtime.h>
#include <hsa/hsa.h>
#include <hsa/hsa_ext_amd.h>
#include <cstdio>
#include <fstream>
#include <iostream>
#include <vector>
#define RET_IF_HSA_ERR(err) \
{ \
if((err) != HSA_STATUS_SUCCESS) \
{ \
char err_val[12]; \
char* err_str = nullptr; \
if(hsa_status_string(err, (const char**) &err_str) != HSA_STATUS_SUCCESS) \
{ \
sprintf(&(err_val[0]), "%#x", (uint32_t) err); \
err_str = &(err_val[0]); \
} \
printf("hsa api call failure at: %s:%d\n", __FILE__, __LINE__); \
printf("Call returned %s\n", err_str); \
abort(); \
} \
}
// Callback function to get the list of agents
hsa_status_t
get_agents(hsa_agent_t agent, void* data)
{
hsa_agent_t** agent_list = (hsa_agent_t**) data;
**agent_list = agent;
++(*agent_list);
return HSA_STATUS_SUCCESS;
}
// Callback function to get the number of agents
hsa_status_t
get_num_agents(hsa_agent_t agent, void* data)
{
(void) agent;
int* num_agents = (int*) data;
++(*num_agents);
return HSA_STATUS_SUCCESS;
}
// Callback function to get the number of regions of an agent
hsa_status_t
callback_get_num_regions(hsa_region_t region, void* data)
{
(void) region;
int* num_regions = (int*) data;
++(*num_regions);
return HSA_STATUS_SUCCESS;
}
// Callback function to get the number of memory pools of an agent
hsa_status_t
callback_get_num_pools(hsa_amd_memory_pool_t memory_pool, void* data)
{
(void) memory_pool;
int* num_pools = (int*) data;
++(*num_pools);
return HSA_STATUS_SUCCESS;
}
// Callback function to get the list of regions of an agent
hsa_status_t
callback_get_regions(hsa_region_t region, void* data)
{
hsa_region_t** region_list = (hsa_region_t**) data;
**region_list = region;
++(*region_list);
return HSA_STATUS_SUCCESS;
}
// Callback function to get the list of memory pools of an agent
hsa_status_t
callback_get_memory_pools(hsa_amd_memory_pool_t memory_pool, void* data)
{
hsa_amd_memory_pool_t** pool_list = (hsa_amd_memory_pool_t**) data;
**pool_list = memory_pool;
++(*pool_list);
return HSA_STATUS_SUCCESS;
}
std::vector<hsa_agent_t>
get_agent_list()
{
size_t num_agents = 0;
hsa_status_t status;
// Get number of agents
status = hsa_iterate_agents(get_num_agents, &num_agents);
RET_IF_HSA_ERR(status)
if(num_agents < 2)
{
printf("Not enough HSA agents available\n");
abort();
}
// Create a array of size num_agents to store the agent list
std::vector<hsa_agent_t> agents(num_agents);
// Get the agent list
hsa_agent_t* agent_iter = &agents[0];
status = hsa_iterate_agents(get_agents, &agent_iter);
RET_IF_HSA_ERR(status)
return agents;
}
hsa_agent_t
get_cpu_agent(std::vector<hsa_agent_t>& agents)
{
for(hsa_agent_t agent : agents)
{
hsa_device_type_t ag_type;
hsa_status_t status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
RET_IF_HSA_ERR(status)
if(ag_type == HSA_DEVICE_TYPE_CPU)
{
return agent;
}
}
std::cerr << "No CPU agents available" << std::endl;
abort();
}
hsa_agent_t
get_gpu_agent(std::vector<hsa_agent_t>& agents)
{
for(hsa_agent_t agent : agents)
{
hsa_device_type_t ag_type;
hsa_status_t status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
RET_IF_HSA_ERR(status)
if(ag_type == HSA_DEVICE_TYPE_GPU)
{
return agent;
}
}
std::cerr << "No GPU agents available" << std::endl;
abort();
}
void
call_hsa_memory_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
{
// Getting total number of regions for the agent
int num_regions = 0;
hsa_status_t status = hsa_agent_iterate_regions(agent, callback_get_num_regions, &num_regions);
RET_IF_HSA_ERR(status)
if(num_regions < 1)
{
printf("No HSA regions available\n");
abort();
}
// Allocate memory to hold region list of an agent
std::vector<hsa_region_t> region_list(num_regions);
hsa_region_t* ptr_reg = &region_list[0];
status = hsa_agent_iterate_regions(agent, callback_get_regions, &ptr_reg);
RET_IF_HSA_ERR(status)
for(size_t j = 0; j < i; ++j)
{
void* addr = 0;
status = hsa_memory_allocate(region_list[0], base_size, &addr);
RET_IF_HSA_ERR(status)
}
}
void
call_hsa_memory_pool_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
{
// Getting total number of regions for the agent
int num_pools = 0;
hsa_status_t status =
hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
RET_IF_HSA_ERR(status)
if(num_pools < 1)
{
printf("No memory pools available\n");
abort();
}
// Allocate memory to hold region list of an agent
std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
hsa_amd_memory_pool_t* ptr_memory_pool = &memory_pool_list[0];
status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
RET_IF_HSA_ERR(status)
for(size_t j = 0; j < i; ++j)
{
void* addr = 0;
uint32_t flags = 0;
status = hsa_amd_memory_pool_allocate(memory_pool_list[0], base_size, flags, &addr);
RET_IF_HSA_ERR(status)
}
}
void
call_hsa_vmem_allocate(const size_t i, hsa_agent_t agent)
{
// Getting total number of regions for the agent
int num_pools = 0;
hsa_status_t status =
hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
RET_IF_HSA_ERR(status)
if(num_pools < 1)
{
printf("No memory pools available\n");
abort();
}
// Allocate memory to hold region list of an agent
std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
hsa_amd_memory_pool_t* ptr_memory_pool = &memory_pool_list[0];
status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
RET_IF_HSA_ERR(status)
// Ensure Virtual Memory API is supported
bool supp = false;
status = hsa_system_get_info(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED, (void*) &supp);
RET_IF_HSA_ERR(status)
if(!supp)
{
std::cerr << "Virtual Memory API not supported" << std::endl;
abort();
}
// Get runtime allocation granule size. Required for vmem_handle_create
int size;
status = hsa_amd_memory_pool_get_info(
memory_pool_list[0], HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, (void*) &size);
RET_IF_HSA_ERR(status)
for(size_t j = 0; j < i; ++j)
{
hsa_amd_vmem_alloc_handle_t memory_handle{};
status = hsa_amd_vmem_handle_create(
memory_pool_list[0], size, MEMORY_TYPE_NONE, 0, &memory_handle);
RET_IF_HSA_ERR(status)
}
}
int
main()
{
hsa_status_t status;
status = hsa_init();
RET_IF_HSA_ERR(status)
std::vector<hsa_agent_t> agents = get_agent_list();
hsa_agent_t cpu_agent = get_cpu_agent(agents);
hsa_agent_t gpu_agent = get_gpu_agent(agents);
call_hsa_memory_allocate(6, 1024, cpu_agent);
call_hsa_memory_pool_allocate(9, 512, gpu_agent);
// Virtual memory API not supported in CI. Will add back if this changes
// call_hsa_vmem_allocate(3, gpu_agent);
status = hsa_shut_down();
RET_IF_HSA_ERR(status)
return 0;
}
tests/hsa-memory-allocation/CMakeLists.txt
+46
View File
@@ -0,0 +1,46 @@
#
#
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
project(
rocprofiler-tests-memory-allocation-tracing
LANGUAGES CXX
VERSION 0.0.0)
find_package(rocprofiler-sdk REQUIRED)
if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
set(PRELOAD_ENV
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
else()
set(PRELOAD_ENV "LD_PRELOAD=$<TARGET_FILE:rocprofiler-sdk-json-tool>")
endif()
add_test(NAME test-memory-allocation-tracing-execute
COMMAND $<TARGET_FILE:hsa-memory-allocation>)
set(memory-allocation-tracing-env
"${PRELOAD_ENV}"
"ROCPROFILER_TOOL_OUTPUT_FILE=memory-allocation-tracing-test.json"
"LD_LIBRARY_PATH=$<TARGET_FILE_DIR:rocprofiler-sdk::rocprofiler-sdk-shared-library>:$ENV{LD_LIBRARY_PATH}"
)
set_tests_properties(
test-memory-allocation-tracing-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT
"${memory-allocation-tracing-env}" FAIL_REGULAR_EXPRESSION
"${ROCPROFILER_DEFAULT_FAIL_REGEX}")
# copy to binary directory
rocprofiler_configure_pytest_files(COPY validate.py conftest.py CONFIG pytest.ini)
add_test(NAME test-memory-allocation-tracing-validate
COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
${CMAKE_CURRENT_BINARY_DIR}/memory-allocation-tracing-test.json)
set_tests_properties(
test-memory-allocation-tracing-validate
PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
test-memory-allocation-tracing-execute FAIL_REGULAR_EXPRESSION
"${ROCPROFILER_DEFAULT_FAIL_REGEX}")
tests/hsa-memory-allocation/conftest.py
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env python3
import json
import pytest
from rocprofiler_sdk.pytest_utils.dotdict import dotdict
def pytest_addoption(parser):
parser.addoption(
"--input",
action="store",
default="memory-allocation-tracing-test.json",
help="Input JSON",
)
@pytest.fixture
def input_data(request):
filename = request.config.getoption("--input")
with open(filename, "r") as inp:
return dotdict(json.load(inp))
tests/hsa-memory-allocation/pytest.ini
+5
View File
@@ -0,0 +1,5 @@
[pytest]
addopts = --durations=20 -rA -s -vv
testpaths = validate.py
pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages
tests/hsa-memory-allocation/validate.py
+289
View File
@@ -0,0 +1,289 @@
#!/usr/bin/env python3
import sys
import pytest
# helper function
def node_exists(name, data, min_len=1):
assert name in data
assert data[name] is not None
if isinstance(data[name], (list, tuple, dict, set)):
assert len(data[name]) >= min_len, f"{name}:\n{data}"
def test_data_structure(input_data):
"""verify minimum amount of expected data is present"""
data = input_data
node_exists("rocprofiler-sdk-json-tool", data)
sdk_data = data["rocprofiler-sdk-json-tool"]
node_exists("metadata", sdk_data)
node_exists("pid", sdk_data["metadata"])
node_exists("main_tid", sdk_data["metadata"])
node_exists("init_time", sdk_data["metadata"])
node_exists("fini_time", sdk_data["metadata"])
node_exists("agents", sdk_data)
node_exists("call_stack", sdk_data)
node_exists("callback_records", sdk_data)
node_exists("buffer_records", sdk_data)
node_exists("names", sdk_data["callback_records"])
node_exists("hsa_api_traces", sdk_data["callback_records"])
node_exists("memory_allocations", sdk_data["callback_records"])
node_exists("names", sdk_data["buffer_records"])
node_exists("hsa_api_traces", sdk_data["callback_records"])
node_exists("memory_allocations", sdk_data["buffer_records"])
def test_size_entries(input_data):
# check that size fields are > 0 but account for function arguments
# which are named "size"
def check_size(data, bt):
if "size" in data.keys():
if isinstance(data["size"], str) and bt.endswith('["args"]'):
pass
else:
assert data["size"] > 0, f"origin: {bt}"
# recursively check the entire data structure
def iterate_data(data, bt):
if isinstance(data, (list, tuple)):
for i, itr in enumerate(data):
if isinstance(itr, dict):
check_size(itr, f"{bt}[{i}]")
iterate_data(itr, f"{bt}[{i}]")
elif isinstance(data, dict):
check_size(data, f"{bt}")
for key, itr in data.items():
iterate_data(itr, f'{bt}["{key}"]')
# start recursive check over entire JSON dict
iterate_data(input_data, "input_data")
def test_timestamps(input_data):
"""Verify starting timestamps are less than ending timestamps"""
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
cb_start = {}
cb_end = {}
for titr in ["hsa_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
cid = itr["correlation_id"]["internal"]
phase = itr["phase"]
if phase == 1:
cb_start[cid] = itr["timestamp"]
elif phase == 2:
cb_end[cid] = itr["timestamp"]
assert cb_start[cid] <= itr["timestamp"]
else:
assert phase == 1 or phase == 2
for itr in sdk_data["buffer_records"][titr]:
assert itr["start_timestamp"] <= itr["end_timestamp"]
for titr in ["memory_allocations"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
assert (
sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
), f"[{titr}] {itr}"
assert (
sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
), f"[{titr}] {itr}"
assert (
sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
), f"[{titr}] {itr}"
assert (
sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
), f"[{titr}] {itr}"
api_start = cb_start[itr["correlation_id"]["internal"]]
# api_end = cb_end[itr["correlation_id"]["internal"]]
assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
# assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
def test_internal_correlation_ids(input_data):
"""Assure correlation ids are unique"""
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
api_corr_ids = []
for titr in ["hsa_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
api_corr_ids.append(itr["correlation_id"]["internal"])
for itr in sdk_data["buffer_records"][titr]:
api_corr_ids.append(itr["correlation_id"]["internal"])
api_corr_ids_sorted = sorted(api_corr_ids)
api_corr_ids_unique = list(set(api_corr_ids))
for itr in sdk_data["buffer_records"]["memory_allocations"]:
assert itr["correlation_id"]["internal"] in api_corr_ids_unique
len_corr_id_unq = len(api_corr_ids_unique)
assert len(api_corr_ids) != len_corr_id_unq
assert max(api_corr_ids_sorted) == len_corr_id_unq
def test_external_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
extern_corr_ids = []
for titr in ["hsa_api_traces"]:
for itr in sdk_data["callback_records"][titr]:
assert itr["correlation_id"]["external"] > 0
assert itr["thread_id"] == itr["correlation_id"]["external"]
extern_corr_ids.append(itr["correlation_id"]["external"])
extern_corr_ids = list(set(sorted(extern_corr_ids)))
for titr in ["hsa_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
assert (
itr["thread_id"] == itr["correlation_id"]["external"]
), f"[{titr}] {itr}"
assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
for titr in ["memory_allocations"]:
for itr in sdk_data["buffer_records"][titr]:
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
for itr in sdk_data["callback_records"][titr]:
assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
def test_memory_alloc_sizes(input_data):
"""Ensure trace file memory allocation operations match up with the memory allocation operations performed in hsa-memory-allocation"""
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
# Op values:
# 0 == ??? (unknown)
# 1 == hsa_memory_allocate
# 2 == hsa_amd_ext_memory_allocate
# 3 == hsa_amd_vmem_handle_create
memory_alloc_cnt = dict(
[
(idx, {"agent": set(), "starting_addr": set(), "size": set(), "count": 0})
for idx in range(1, 4)
]
)
for itr in sdk_data["buffer_records"]["memory_allocations"]:
op_id = itr["operation"]
assert op_id > 0 and op_id <= 3, f"{itr}"
memory_alloc_cnt[op_id]["count"] += 1
memory_alloc_cnt[op_id]["starting_addr"].add(itr.starting_address)
memory_alloc_cnt[op_id]["size"].add(itr.allocation_size)
memory_alloc_cnt[op_id]["agent"].add(itr.agent_id.handle)
for itr in sdk_data["callback_records"]["memory_copies"]:
op_id = itr.operation
assert op_id > 0 and op_id <= 3, f"{itr}"
memory_alloc_cnt[op_id]["count"] += 1
phase = itr.phase
pitr = itr.payload
assert phase is not None, f"{itr}"
assert pitr is not None, f"{itr}"
if phase == 1:
assert pitr.start_timestamp == 0, f"{itr}"
assert pitr.end_timestamp == 0, f"{itr}"
elif phase == 2:
assert pitr.start_timestamp > 0, f"{itr}"
assert pitr.end_timestamp > 0, f"{itr}"
assert pitr.end_timestamp >= pitr.start_timestamp, f"{itr}"
memory_alloc_cnt[op_id]["starting_addr"].add(pitr.starting_address)
memory_alloc_cnt[op_id]["size"].add(pitr.allocation_size)
memory_alloc_cnt[op_id]["agent"].add(pitr.agent_id.handle)
else:
assert phase == 1 or phase == 2, f"{itr}"
# In the memory allocation test which generates this file
# 6 hsa_memory_allocation calls with 1024 bytes were called
# and 9 hsa_amd_memory_pool_allocations with 512 bytes
# were called
assert memory_alloc_cnt[1]["count"] == 6
assert memory_alloc_cnt[2]["count"] == 9
# assert memory_alloc_cnt[3]["count"] == 3
assert len(memory_alloc_cnt[1]["starting_addr"]) == 6
assert len(memory_alloc_cnt[2]["starting_addr"]) == 9
# assert len(memory_alloc_cnt[3]["starting_addr"]) == 3
assert len(memory_alloc_cnt[1]["size"]) == 1
assert len(memory_alloc_cnt[2]["size"]) == 1
# assert len(memory_alloc_cnt[3]["size"]) == 1
assert 1024 in memory_alloc_cnt[1]["size"]
assert 512 in memory_alloc_cnt[2]["size"]
assert len(memory_alloc_cnt[1]["agent"]) == 1
assert len(memory_alloc_cnt[2]["agent"]) == 1
# assert len(memory_alloc_cnt[3]["agent"]) == 1
assert memory_alloc_cnt[1]["agent"] != memory_alloc_cnt[2]["agent"]
# assert memory_alloc_cnt[2]["agent"] == memory_alloc_cnt[3]["agent"]
def test_retired_correlation_ids(input_data):
data = input_data
sdk_data = data["rocprofiler-sdk-json-tool"]
def _sort_dict(inp):
return dict(sorted(inp.items()))
api_corr_ids = {}
for titr in ["hsa_api_traces"]:
for itr in sdk_data["buffer_records"][titr]:
corr_id = itr["correlation_id"]["internal"]
assert corr_id not in api_corr_ids.keys()
api_corr_ids[corr_id] = itr
alloc_corr_ids = {}
for titr in ["memory_allocations"]:
for itr in sdk_data["buffer_records"][titr]:
corr_id = itr["correlation_id"]["internal"]
assert corr_id not in alloc_corr_ids.keys()
alloc_corr_ids[corr_id] = itr
retired_corr_ids = {}
for itr in sdk_data["buffer_records"]["retired_correlation_ids"]:
corr_id = itr["internal_correlation_id"]
assert corr_id not in retired_corr_ids.keys()
retired_corr_ids[corr_id] = itr
api_corr_ids = _sort_dict(api_corr_ids)
alloc_corr_ids = _sort_dict(alloc_corr_ids)
retired_corr_ids = _sort_dict(retired_corr_ids)
for cid, itr in alloc_corr_ids.items():
assert cid in retired_corr_ids.keys()
retired_ts = retired_corr_ids[cid]["timestamp"]
end_ts = itr["end_timestamp"]
assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
for cid, itr in api_corr_ids.items():
assert cid in retired_corr_ids.keys()
retired_ts = retired_corr_ids[cid]["timestamp"]
end_ts = itr["end_timestamp"]
assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
if __name__ == "__main__":
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
sys.exit(exit_code)
tests/pytest-packages/tests/rocprofv3.py
+5 -1
View File
@@ -24,7 +24,9 @@ from __future__ import absolute_import
def test_perfetto_data(
pftrace_data, json_data, categories=("hip", "hsa", "marker", "kernel", "memory_copy")
pftrace_data,
json_data,
categories=("hip", "hsa", "marker", "kernel", "memory_copy", "memory_allocation"),
):
mapping = {
@@ -33,6 +35,7 @@ def test_perfetto_data(
"marker": ("marker_api", "marker_api"),
"kernel": ("kernel_dispatch", "kernel_dispatch"),
"memory_copy": ("memory_copy", "memory_copy"),
"memory_allocation": ("memory_allocation", "memory_allocation"),
}
# make sure they specified valid categories
@@ -70,6 +73,7 @@ def test_otf2_data(
"marker": ("marker_api", "marker_api"),
"kernel": ("kernel_dispatch", "kernel_dispatch"),
"memory_copy": ("memory_copy", "memory_copy"),
"memory_allocation": ("memory_allocation", "memory_allocation"),
}
# make sure they specified valid categories
tests/rocprofv3/CMakeLists.txt
+1
View File
@@ -29,6 +29,7 @@ add_subdirectory(tracing-hip-in-libraries)
add_subdirectory(counter-collection)
add_subdirectory(hsa-queue-dependency)
add_subdirectory(kernel-rename)
add_subdirectory(memory-allocation)
add_subdirectory(aborted-app)
add_subdirectory(summary)
add_subdirectory(roctracer-roctx)
tests/rocprofv3/memory-allocation/CMakeLists.txt
+47
View File
@@ -0,0 +1,47 @@
#
#
#
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
project(
rocprofiler-tests-rocprofv3-memory-allocation-tracing
LANGUAGES CXX
VERSION 0.0.0)
find_package(rocprofiler-sdk REQUIRED)
rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py)
string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
"${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
set(memory-allocation-tracing-env "${PRELOAD_ENV}")
add_test(
NAME rocprofv3-test-memory-allocation-tracing-execute
COMMAND
$<TARGET_FILE:rocprofiler-sdk::rocprofv3> --memory-allocation-trace -d
${CMAKE_CURRENT_BINARY_DIR}/%tag%-trace -o out --output-format json pftrace otf2
--log-level env -- $<TARGET_FILE:hsa-memory-allocation>)
set_tests_properties(
rocprofv3-test-memory-allocation-tracing-execute
PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT
"${memory-allocation-tracing-env}" FAIL_REGULAR_EXPRESSION
"threw an exception")
add_test(
NAME rocprofv3-test-memory-allocation-tracing-validate
COMMAND
${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.json
--pftrace-input
${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.pftrace
--otf2-input
${CMAKE_CURRENT_BINARY_DIR}/hsa-memory-allocation-trace/out_results.otf2)
set_tests_properties(
rocprofv3-test-memory-allocation-tracing-validate
PROPERTIES TIMEOUT 45 LABELS "integration-tests" DEPENDS
rocprofv3-test-memory-allocation-tracing-execute FAIL_REGULAR_EXPRESSION
"AssertionError")
tests/rocprofv3/memory-allocation/conftest.py
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
import json
import os
import pytest
from rocprofiler_sdk.pytest_utils.dotdict import dotdict
from rocprofiler_sdk.pytest_utils import collapse_dict_list
from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader
def pytest_addoption(parser):
parser.addoption(
"--json-input",
action="store",
default="memory-allocation-tracing/out_results.json",
help="Input JSON",
)
parser.addoption(
"--pftrace-input",
action="store",
default="memory-allocation-tracing/out_results.pftrace",
help="Input JSON",
)
parser.addoption(
"--otf2-input",
action="store",
default="memory-allocation-tracing/out_results.otf2",
help="Input JSON",
)
@pytest.fixture
def json_data(request):
filename = request.config.getoption("--json-input")
with open(filename, "r") as inp:
return dotdict(collapse_dict_list(json.load(inp)))
@pytest.fixture
def pftrace_data(request):
filename = request.config.getoption("--pftrace-input")
return PerfettoReader(filename).read()[0]
@pytest.fixture
def otf2_data(request):
filename = request.config.getoption("--otf2-input")
if not os.path.exists(filename):
raise FileExistsError(f"{filename} does not exist")
return OTF2Reader(filename).read()[0]
tests/rocprofv3/memory-allocation/pytest.ini
+5
View File
@@ -0,0 +1,5 @@
[pytest]
addopts = --durations=20 -rA -s -vv
testpaths = validate.py
pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages
tests/rocprofv3/memory-allocation/validate.py
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
import sys
import pytest
import json
from collections import defaultdict
# helper function
def node_exists(name, data, min_len=1):
assert name in data
assert data[name] is not None
if isinstance(data[name], (list, tuple, dict, set)):
assert len(data[name]) >= min_len
def get_operation(record, kind_name, op_name=None):
for idx, itr in enumerate(record["strings"]["buffer_records"]):
if kind_name == itr["kind"]:
if op_name is None:
return idx, itr["operations"]
else:
for oidx, oname in enumerate(itr["operations"]):
if op_name == oname:
return oidx
return None
def test_memory_allocation(json_data):
data = json_data["rocprofiler-sdk-tool"]
buffer_records = data["buffer_records"]
memory_allocation_data = buffer_records["memory_allocation"]
_, bf_op_names = get_operation(data, "MEMORY_ALLOCATION")
assert len(bf_op_names) == 4
allocation_reported_agent_ids = set()
# check buffering data
for node in memory_allocation_data:
assert "size" in node
assert "kind" in node
assert "operation" in node
assert "correlation_id" in node
assert "end_timestamp" in node
assert "start_timestamp" in node
assert "thread_id" in node
assert "agent_id" in node
assert "starting_address" in node
assert "allocation_size" in node
assert node.size > 0
assert node.allocation_size > 0
assert node.starting_address > 0
assert node.thread_id > 0
assert node.agent_id.handle > 0
assert node.start_timestamp > 0
assert node.end_timestamp > 0
assert node.start_timestamp < node.end_timestamp
assert data.strings.buffer_records[node.kind].kind == "MEMORY_ALLOCATION"
assert (
data.strings.buffer_records[node.kind].operations[node.operation]
in bf_op_names
)
allocation_reported_agent_ids.add(node["agent_id"]["handle"])
assert 2**64 - 1 not in allocation_reported_agent_ids
def test_perfetto_data(pftrace_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_perfetto_data(pftrace_data, json_data, ("memory_allocation",))
def test_otf2_data(otf2_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_otf2_data(otf2_data, json_data, ("memory_allocation",))
if __name__ == "__main__":
exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
sys.exit(exit_code)
tests/rocprofv3/summary/validate.py
+11 -3
View File
@@ -188,6 +188,8 @@ def test_summary_data(json_data):
assert itr.stats.count >= 2130 and itr.stats.count <= 2150
elif itr.domain == "MEMORY_COPY":
assert itr.stats.count == 12
elif itr.domain == "MEMORY_ALLOCATION":
assert itr.stats.count >= 10 and itr.stats.count <= 30
elif itr.domain == "MARKER_API":
assert itr.stats.count == 1106
expected = dict(
@@ -231,6 +233,7 @@ def test_summary_display_data(json_data, summary_data):
marker = get_df("MARKER_API")
dispatch = get_df("KERNEL_DISPATCH")
memcpy = get_df("MEMORY_COPY")
memalloc = get_df("MEMORY_ALLOCATION")
dispatch_and_copy = get_df("KERNEL_DISPATCH + MEMORY_COPY")
hip_and_marker = get_df("HIP_API + MARKER_API") if num_summary_grps > 1 else None
total = get_df("SUMMARY")
@@ -239,18 +242,21 @@ def test_summary_display_data(json_data, summary_data):
assert get_dims(marker) == [7, 9], f"{marker}"
assert get_dims(memcpy) == [2, 9], f"{memcpy}"
assert get_dims(memalloc) == [1, 9], f"{memalloc}"
assert get_dims(dispatch) == [3, 9], f"{dispatch}"
assert get_dims(dispatch_and_copy) == [5, 9], f"{dispatch_and_copy}"
assert get_dims(hip) == [14, 9], f"{hip}"
assert get_dims(hip_and_marker) == expected_hip_and_marker_dims, f"{hip_and_marker}"
assert get_dims(total) == [23, 9], f"{total}"
assert get_dims(total) == [24, 9], f"{total}"
def test_perfetto_data(pftrace_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_perfetto_data(
pftrace_data, json_data, ("hip", "marker", "kernel", "memory_copy")
pftrace_data,
json_data,
("hip", "marker", "kernel", "memory_copy", "memory_allocation"),
)
@@ -258,7 +264,9 @@ def test_otf2_data(otf2_data, json_data):
import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
rocprofv3.test_otf2_data(
otf2_data, json_data, ("hip", "marker", "kernel", "memory_copy")
otf2_data,
json_data,
("hip", "marker", "kernel", "memory_copy", "memory_allocation"),
)
tests/tools/json-tool.cpp
+126 -20
View File
@@ -391,6 +391,21 @@ struct memory_copy_callback_record_t
}
};
struct memory_allocation_callback_record_t
{
uint64_t timestamp = 0;
rocprofiler_callback_tracing_record_t record = {};
rocprofiler_callback_tracing_memory_allocation_data_t payload = {};
template <typename ArchiveT>
void save(ArchiveT& ar) const
{
ar(cereal::make_nvp("timestamp", timestamp));
cereal::save(ar, record);
ar(cereal::make_nvp("payload", payload));
}
};
struct scratch_memory_callback_record_t
{
uint64_t timestamp = 0;
@@ -483,6 +498,7 @@ auto hip_api_cb_records = std::deque<hip_api_callback_record_t>{};
auto scratch_memory_cb_records = std::deque<scratch_memory_callback_record_t>{};
auto kernel_dispatch_cb_records = std::deque<kernel_dispatch_callback_record_t>{};
auto memory_copy_cb_records = std::deque<memory_copy_callback_record_t>{};
auto memory_allocation_cb_records = std::deque<memory_allocation_callback_record_t>{};
auto rccl_api_cb_records = std::deque<rccl_api_callback_record_t>{};
int
@@ -696,6 +712,16 @@ tool_tracing_callback(rocprofiler_callback_tracing_record_t record,
auto _lk = std::unique_lock<std::mutex>{_mutex};
memory_copy_cb_records.emplace_back(memory_copy_callback_record_t{ts, record, *data});
}
else if(record.kind == ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION)
{
auto* data =
static_cast<rocprofiler_callback_tracing_memory_allocation_data_t*>(record.payload);
static auto _mutex = std::mutex{};
auto _lk = std::unique_lock<std::mutex>{_mutex};
memory_allocation_cb_records.emplace_back(
memory_allocation_callback_record_t{ts, record, *data});
}
else if(record.kind == ROCPROFILER_CALLBACK_TRACING_RCCL_API)
{
auto* data = static_cast<rocprofiler_callback_tracing_rccl_api_data_t*>(record.payload);
@@ -720,8 +746,10 @@ auto marker_api_bf_records = std::deque<rocprofiler_buffer_tracing_marker_a
auto hip_api_bf_records = std::deque<rocprofiler_buffer_tracing_hip_api_record_t>{};
auto kernel_dispatch_bf_records = std::deque<rocprofiler_buffer_tracing_kernel_dispatch_record_t>{};
auto memory_copy_bf_records = std::deque<rocprofiler_buffer_tracing_memory_copy_record_t>{};
auto scratch_memory_records = std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>{};
auto page_migration_records = std::deque<rocprofiler_buffer_tracing_page_migration_record_t>{};
auto memory_allocation_bf_records =
std::deque<rocprofiler_buffer_tracing_memory_allocation_record_t>{};
auto scratch_memory_records = std::deque<rocprofiler_buffer_tracing_scratch_memory_record_t>{};
auto page_migration_records = std::deque<rocprofiler_buffer_tracing_page_migration_record_t>{};
auto corr_id_retire_records =
std::deque<rocprofiler_buffer_tracing_correlation_id_retirement_record_t>{};
auto rccl_api_bf_records = std::deque<rocprofiler_buffer_tracing_rccl_api_record_t>{};
@@ -800,6 +828,13 @@ tool_tracing_buffered(rocprofiler_context_id_t /*context*/,
memory_copy_bf_records.emplace_back(*record);
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION)
{
auto* record = static_cast<rocprofiler_buffer_tracing_memory_allocation_record_t*>(
header->payload);
memory_allocation_bf_records.emplace_back(*record);
}
else if(header->kind == ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY)
{
auto* record = static_cast<rocprofiler_buffer_tracing_scratch_memory_record_t*>(
@@ -904,29 +939,32 @@ void
pop_external_correlation();
// contexts
rocprofiler_context_id_t hsa_api_callback_ctx = {0};
rocprofiler_context_id_t hip_api_callback_ctx = {0};
rocprofiler_context_id_t marker_api_callback_ctx = {0};
rocprofiler_context_id_t code_object_ctx = {0};
rocprofiler_context_id_t rccl_api_callback_ctx = {0};
rocprofiler_context_id_t hsa_api_buffered_ctx = {0};
rocprofiler_context_id_t hip_api_buffered_ctx = {0};
rocprofiler_context_id_t marker_api_buffered_ctx = {0};
rocprofiler_context_id_t memory_copy_callback_ctx = {0};
rocprofiler_context_id_t memory_copy_buffered_ctx = {0};
rocprofiler_context_id_t rccl_api_buffered_ctx = {0};
rocprofiler_context_id_t counter_collection_ctx = {0};
rocprofiler_context_id_t scratch_memory_ctx = {0};
rocprofiler_context_id_t corr_id_retire_ctx = {0};
rocprofiler_context_id_t kernel_dispatch_callback_ctx = {0};
rocprofiler_context_id_t kernel_dispatch_buffered_ctx = {0};
rocprofiler_context_id_t page_migration_ctx = {0};
rocprofiler_context_id_t hsa_api_callback_ctx = {0};
rocprofiler_context_id_t hip_api_callback_ctx = {0};
rocprofiler_context_id_t marker_api_callback_ctx = {0};
rocprofiler_context_id_t code_object_ctx = {0};
rocprofiler_context_id_t rccl_api_callback_ctx = {0};
rocprofiler_context_id_t hsa_api_buffered_ctx = {0};
rocprofiler_context_id_t hip_api_buffered_ctx = {0};
rocprofiler_context_id_t marker_api_buffered_ctx = {0};
rocprofiler_context_id_t memory_copy_callback_ctx = {0};
rocprofiler_context_id_t memory_copy_buffered_ctx = {0};
rocprofiler_context_id_t memory_allocation_callback_ctx = {0};
rocprofiler_context_id_t memory_allocation_buffered_ctx = {0};
rocprofiler_context_id_t rccl_api_buffered_ctx = {0};
rocprofiler_context_id_t counter_collection_ctx = {0};
rocprofiler_context_id_t scratch_memory_ctx = {0};
rocprofiler_context_id_t corr_id_retire_ctx = {0};
rocprofiler_context_id_t kernel_dispatch_callback_ctx = {0};
rocprofiler_context_id_t kernel_dispatch_buffered_ctx = {0};
rocprofiler_context_id_t page_migration_ctx = {0};
// buffers
rocprofiler_buffer_id_t hsa_api_buffered_buffer = {};
rocprofiler_buffer_id_t hip_api_buffered_buffer = {};
rocprofiler_buffer_id_t marker_api_buffered_buffer = {};
rocprofiler_buffer_id_t kernel_dispatch_buffer = {};
rocprofiler_buffer_id_t memory_copy_buffer = {};
rocprofiler_buffer_id_t memory_allocation_buffer = {};
rocprofiler_buffer_id_t page_migration_buffer = {};
rocprofiler_buffer_id_t counter_collection_buffer = {};
rocprofiler_buffer_id_t scratch_memory_buffer = {};
@@ -940,12 +978,14 @@ auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
{"CODE_OBJECT", &code_object_ctx},
{"KERNEL_DISPATCH_CALLBACK", &kernel_dispatch_callback_ctx},
{"MEMORY_COPY_CALLBACK", &memory_copy_callback_ctx},
{"MEMORY_ALLOCATION_CALLBACK", &memory_allocation_callback_ctx},
{"RCCL_API_CALLBACK", &rccl_api_callback_ctx},
{"HSA_API_BUFFERED", &hsa_api_buffered_ctx},
{"HIP_API_BUFFERED", &hip_api_buffered_ctx},
{"MARKER_API_BUFFERED", &marker_api_buffered_ctx},
{"KERNEL_DISPATCH_BUFFERED", &kernel_dispatch_buffered_ctx},
{"MEMORY_COPY_BUFFERED", &memory_copy_buffered_ctx},
{"MEMORY_ALLOCATION_BUFFERED", &memory_allocation_buffered_ctx},
{"PAGE_MIGRATION", &page_migration_ctx},
{"COUNTER_COLLECTION", &counter_collection_ctx},
{"SCRATCH_MEMORY", &scratch_memory_ctx},
@@ -953,11 +993,12 @@ auto contexts = std::unordered_map<std::string_view, rocprofiler_context_id_t*>{
{"RCCL_API_BUFFERED", &rccl_api_buffered_ctx},
};
auto buffers = std::array<rocprofiler_buffer_id_t*, 10>{&hsa_api_buffered_buffer,
auto buffers = std::array<rocprofiler_buffer_id_t*, 11>{&hsa_api_buffered_buffer,
&hip_api_buffered_buffer,
&marker_api_buffered_buffer,
&kernel_dispatch_buffer,
&memory_copy_buffer,
&memory_allocation_buffer,
&scratch_memory_buffer,
&page_migration_buffer,
&counter_collection_buffer,
@@ -1092,6 +1133,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
nullptr),
"memory copy callback tracing service configure");
ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
memory_allocation_callback_ctx,
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION,
nullptr,
0,
tool_tracing_callback,
nullptr),
"memory allocation callback tracing service configure");
ROCPROFILER_CALL(
rocprofiler_configure_callback_tracing_service(scratch_memory_ctx,
ROCPROFILER_CALLBACK_TRACING_SCRATCH_MEMORY,
@@ -1158,6 +1208,15 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
&memory_copy_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(memory_allocation_buffered_ctx,
buffer_size,
watermark,
ROCPROFILER_BUFFER_POLICY_LOSSLESS,
tool_tracing_buffered,
tool_data,
&memory_allocation_buffer),
"buffer creation");
ROCPROFILER_CALL(rocprofiler_create_buffer(scratch_memory_ctx,
buffer_size,
watermark,
@@ -1261,6 +1320,14 @@ tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
memory_copy_buffer),
"buffer tracing service for memory copy configure");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(memory_allocation_buffered_ctx,
ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION,
nullptr,
0,
memory_allocation_buffer),
"buffer tracing service for memory allocation configure");
ROCPROFILER_CALL(
rocprofiler_configure_buffer_tracing_service(scratch_memory_ctx,
ROCPROFILER_BUFFER_TRACING_SCRATCH_MEMORY,
@@ -1448,9 +1515,11 @@ tool_fini(void* tool_data)
<< ", scratch_memory_callback_records=" << scratch_memory_cb_records.size()
<< ", kernel_dispatch_callback_records=" << kernel_dispatch_cb_records.size()
<< ", memory_copy_callback_records=" << memory_copy_cb_records.size()
<< ", memory_allocation_callback_records=" << memory_allocation_cb_records.size()
<< ", rccl_api_callback_records=" << rccl_api_cb_records.size()
<< ", kernel_dispatch_bf_records=" << kernel_dispatch_bf_records.size()
<< ", memory_copy_bf_records=" << memory_copy_bf_records.size()
<< ", memory_allocation_bf_records=" << memory_allocation_bf_records.size()
<< ", scratch_memory_records=" << scratch_memory_records.size()
<< ", page_migration=" << page_migration_records.size()
<< ", hsa_api_bf_records=" << hsa_api_bf_records.size()
@@ -1551,6 +1620,7 @@ write_json(call_stack_t* _call_stack)
json_ar(cereal::make_nvp("scratch_memory_traces", scratch_memory_cb_records));
json_ar(cereal::make_nvp("kernel_dispatch", kernel_dispatch_cb_records));
json_ar(cereal::make_nvp("memory_copies", memory_copy_cb_records));
json_ar(cereal::make_nvp("memory_allocations", memory_allocation_cb_records));
} catch(std::exception& e)
{
std::cerr << "[" << getpid() << "][" << __FUNCTION__
@@ -1566,6 +1636,7 @@ write_json(call_stack_t* _call_stack)
json_ar(cereal::make_nvp("names", buffer_names));
json_ar(cereal::make_nvp("kernel_dispatch", kernel_dispatch_bf_records));
json_ar(cereal::make_nvp("memory_copies", memory_copy_bf_records));
json_ar(cereal::make_nvp("memory_allocations", memory_allocation_bf_records));
json_ar(cereal::make_nvp("scratch_memory_traces", scratch_memory_records));
json_ar(cereal::make_nvp("page_migration", page_migration_records));
json_ar(cereal::make_nvp("hsa_api_traces", hsa_api_bf_records));
@@ -1650,6 +1721,12 @@ write_perfetto()
agent_ids.emplace(itr.src_agent_id.handle);
}
for(auto itr : memory_allocation_bf_records)
{
tids.emplace(itr.thread_id);
agent_ids.emplace(itr.agent_id.handle);
}
for(auto itr : kernel_dispatch_bf_records)
{
tids.emplace(itr.thread_id);
@@ -1888,6 +1965,35 @@ write_perfetto()
itr.end_timestamp);
}
for(auto itr : memory_allocation_bf_records)
{
auto name = buffer_names.at(itr.kind, itr.operation);
auto& track = agent_tracks.at(itr.agent_id.handle);
TRACE_EVENT_BEGIN(sdk::perfetto_category<sdk::category::memory_allocation>::name,
::perfetto::StaticString(name.data()),
track,
itr.start_timestamp,
::perfetto::Flow::ProcessScoped(itr.correlation_id.internal),
"begin_ns",
itr.start_timestamp,
"kind",
itr.kind,
"operation",
itr.operation,
"agent",
agents_map.at(itr.agent_id).logical_node_id,
"Allocation_size",
itr.allocation_size,
"Starting_address",
itr.starting_address);
TRACE_EVENT_END(sdk::perfetto_category<sdk::category::memory_allocation>::name,
track,
itr.end_timestamp,
"end_ns",
itr.end_timestamp);
}
auto demangled = std::unordered_map<std::string_view, std::string>{};
for(auto itr : kernel_dispatch_bf_records)
{