Memory Allocation Tracking (#1142)
* Initial commit: Need to implement wrapper function to collect data and test that wrapper function is correctly replacing core HSA functions * Attempted to implement wrapper implementation for hsa memory allocation functions. Need to modify generate record files and test if implementation is working as expected * Debugging and implementing generateCSV function * Memory allocation size and starting address outputted to csv and json file formats * Formatting * Initial setup for OTF2 and Perfetto generation * Collecting agent id for memory_allocation and formatting * Modified memory_allocation.cpp to set up code for AMD_EXT commands * Support for memory_pool_allocate added * Removed accidently added file * Made flag optional and added more OTF2 and Perfetto code. Needs testing to ensure perfetto and OTF2 works * Formatting * Fixed perfetto and otf2 output * Fixed flag issue due to incorrect buffer use * Updated documentation * Small cleaning and comments * Added test for HSA memory allocation tracing * Fixed summary test validation errors due to allocation tracing. Added type to location_base to create unique event ids for allocation due to OTF2 trace error * Decreased lower limit of hip calls for test * Modified summary tests to vary number of allocate requests * Minor fixes to address comments. Still need to address OTF2 comments * Fix docs and changed OTF2 to use enum for type specified in location_base construction * Fixed schema error * Added vmem command tracking. Need to add test * Updated test to work with vmem command and updated generateCSV to output int instead of hex string. * OTF2 enum update and mispelling fix * CI does not support Virtual Memory API. Removed vmem test. Will add back if CI is modifed to suport vmem API * Update CMakeLists.txt for memory allocation test * Updated summary test * Minor fixes to address comments * Moved domain_type.hpp enum to before LAST * Fixed compile errors and formatting * Fixed stats summary domain name error * Added rocprofv3 test * Page migration test fix * Undo page migration test changes. Failures do not appear to have to do with memory allocation
This commit is contained in:
itrowbri
GitHub
@@ -0,0 +1,43 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tool-test-app-hsa-memory-allocation LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(hsa-memory-allocation.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(hsa-memory-allocation)
|
||||
target_sources(hsa-memory-allocation PRIVATE hsa-memory-allocation.cpp)
|
||||
target_compile_options(hsa-memory-allocation PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
|
||||
-Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(hsa-memory-allocation PRIVATE Threads::Threads hsa-runtime64
|
||||
rocprofiler-sdk::tests-common-library)
|
||||
@@ -0,0 +1,267 @@
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <hsa/hsa.h>
|
||||
#include <hsa/hsa_ext_amd.h>
|
||||
#include <cstdio>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
#define RET_IF_HSA_ERR(err) \
|
||||
{ \
|
||||
if((err) != HSA_STATUS_SUCCESS) \
|
||||
{ \
|
||||
char err_val[12]; \
|
||||
char* err_str = nullptr; \
|
||||
if(hsa_status_string(err, (const char**) &err_str) != HSA_STATUS_SUCCESS) \
|
||||
{ \
|
||||
sprintf(&(err_val[0]), "%#x", (uint32_t) err); \
|
||||
err_str = &(err_val[0]); \
|
||||
} \
|
||||
printf("hsa api call failure at: %s:%d\n", __FILE__, __LINE__); \
|
||||
printf("Call returned %s\n", err_str); \
|
||||
abort(); \
|
||||
} \
|
||||
}
|
||||
|
||||
// Callback function to get the list of agents
|
||||
hsa_status_t
|
||||
get_agents(hsa_agent_t agent, void* data)
|
||||
{
|
||||
hsa_agent_t** agent_list = (hsa_agent_t**) data;
|
||||
**agent_list = agent;
|
||||
++(*agent_list);
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Callback function to get the number of agents
|
||||
hsa_status_t
|
||||
get_num_agents(hsa_agent_t agent, void* data)
|
||||
{
|
||||
(void) agent;
|
||||
int* num_agents = (int*) data;
|
||||
++(*num_agents);
|
||||
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Callback function to get the number of regions of an agent
|
||||
hsa_status_t
|
||||
callback_get_num_regions(hsa_region_t region, void* data)
|
||||
{
|
||||
(void) region;
|
||||
int* num_regions = (int*) data;
|
||||
++(*num_regions);
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Callback function to get the number of memory pools of an agent
|
||||
hsa_status_t
|
||||
callback_get_num_pools(hsa_amd_memory_pool_t memory_pool, void* data)
|
||||
{
|
||||
(void) memory_pool;
|
||||
int* num_pools = (int*) data;
|
||||
++(*num_pools);
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Callback function to get the list of regions of an agent
|
||||
hsa_status_t
|
||||
callback_get_regions(hsa_region_t region, void* data)
|
||||
{
|
||||
hsa_region_t** region_list = (hsa_region_t**) data;
|
||||
**region_list = region;
|
||||
++(*region_list);
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Callback function to get the list of memory pools of an agent
|
||||
hsa_status_t
|
||||
callback_get_memory_pools(hsa_amd_memory_pool_t memory_pool, void* data)
|
||||
{
|
||||
hsa_amd_memory_pool_t** pool_list = (hsa_amd_memory_pool_t**) data;
|
||||
**pool_list = memory_pool;
|
||||
++(*pool_list);
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
std::vector<hsa_agent_t>
|
||||
get_agent_list()
|
||||
{
|
||||
size_t num_agents = 0;
|
||||
hsa_status_t status;
|
||||
// Get number of agents
|
||||
status = hsa_iterate_agents(get_num_agents, &num_agents);
|
||||
RET_IF_HSA_ERR(status)
|
||||
if(num_agents < 2)
|
||||
{
|
||||
printf("Not enough HSA agents available\n");
|
||||
abort();
|
||||
}
|
||||
|
||||
// Create a array of size num_agents to store the agent list
|
||||
std::vector<hsa_agent_t> agents(num_agents);
|
||||
|
||||
// Get the agent list
|
||||
hsa_agent_t* agent_iter = &agents[0];
|
||||
status = hsa_iterate_agents(get_agents, &agent_iter);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
return agents;
|
||||
}
|
||||
|
||||
hsa_agent_t
|
||||
get_cpu_agent(std::vector<hsa_agent_t>& agents)
|
||||
{
|
||||
for(hsa_agent_t agent : agents)
|
||||
{
|
||||
hsa_device_type_t ag_type;
|
||||
hsa_status_t status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
if(ag_type == HSA_DEVICE_TYPE_CPU)
|
||||
{
|
||||
return agent;
|
||||
}
|
||||
}
|
||||
std::cerr << "No CPU agents available" << std::endl;
|
||||
abort();
|
||||
}
|
||||
|
||||
hsa_agent_t
|
||||
get_gpu_agent(std::vector<hsa_agent_t>& agents)
|
||||
{
|
||||
for(hsa_agent_t agent : agents)
|
||||
{
|
||||
hsa_device_type_t ag_type;
|
||||
hsa_status_t status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &ag_type);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
if(ag_type == HSA_DEVICE_TYPE_GPU)
|
||||
{
|
||||
return agent;
|
||||
}
|
||||
}
|
||||
std::cerr << "No GPU agents available" << std::endl;
|
||||
abort();
|
||||
}
|
||||
|
||||
void
|
||||
call_hsa_memory_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
|
||||
{
|
||||
// Getting total number of regions for the agent
|
||||
int num_regions = 0;
|
||||
hsa_status_t status = hsa_agent_iterate_regions(agent, callback_get_num_regions, &num_regions);
|
||||
RET_IF_HSA_ERR(status)
|
||||
if(num_regions < 1)
|
||||
{
|
||||
printf("No HSA regions available\n");
|
||||
abort();
|
||||
}
|
||||
// Allocate memory to hold region list of an agent
|
||||
std::vector<hsa_region_t> region_list(num_regions);
|
||||
hsa_region_t* ptr_reg = ®ion_list[0];
|
||||
status = hsa_agent_iterate_regions(agent, callback_get_regions, &ptr_reg);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
for(size_t j = 0; j < i; ++j)
|
||||
{
|
||||
void* addr = 0;
|
||||
|
||||
status = hsa_memory_allocate(region_list[0], base_size, &addr);
|
||||
RET_IF_HSA_ERR(status)
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
call_hsa_memory_pool_allocate(const size_t i, const size_t base_size, hsa_agent_t agent)
|
||||
{
|
||||
// Getting total number of regions for the agent
|
||||
int num_pools = 0;
|
||||
hsa_status_t status =
|
||||
hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
|
||||
RET_IF_HSA_ERR(status)
|
||||
if(num_pools < 1)
|
||||
{
|
||||
printf("No memory pools available\n");
|
||||
abort();
|
||||
}
|
||||
// Allocate memory to hold region list of an agent
|
||||
std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
|
||||
hsa_amd_memory_pool_t* ptr_memory_pool = &memory_pool_list[0];
|
||||
status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
for(size_t j = 0; j < i; ++j)
|
||||
{
|
||||
void* addr = 0;
|
||||
uint32_t flags = 0;
|
||||
|
||||
status = hsa_amd_memory_pool_allocate(memory_pool_list[0], base_size, flags, &addr);
|
||||
RET_IF_HSA_ERR(status)
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
call_hsa_vmem_allocate(const size_t i, hsa_agent_t agent)
|
||||
{
|
||||
// Getting total number of regions for the agent
|
||||
int num_pools = 0;
|
||||
hsa_status_t status =
|
||||
hsa_amd_agent_iterate_memory_pools(agent, callback_get_num_pools, &num_pools);
|
||||
RET_IF_HSA_ERR(status)
|
||||
if(num_pools < 1)
|
||||
{
|
||||
printf("No memory pools available\n");
|
||||
abort();
|
||||
}
|
||||
// Allocate memory to hold region list of an agent
|
||||
std::vector<hsa_amd_memory_pool_t> memory_pool_list(num_pools);
|
||||
hsa_amd_memory_pool_t* ptr_memory_pool = &memory_pool_list[0];
|
||||
status = hsa_amd_agent_iterate_memory_pools(agent, callback_get_memory_pools, &ptr_memory_pool);
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
// Ensure Virtual Memory API is supported
|
||||
bool supp = false;
|
||||
status = hsa_system_get_info(HSA_AMD_SYSTEM_INFO_VIRTUAL_MEM_API_SUPPORTED, (void*) &supp);
|
||||
RET_IF_HSA_ERR(status)
|
||||
if(!supp)
|
||||
{
|
||||
std::cerr << "Virtual Memory API not supported" << std::endl;
|
||||
abort();
|
||||
}
|
||||
|
||||
// Get runtime allocation granule size. Required for vmem_handle_create
|
||||
int size;
|
||||
status = hsa_amd_memory_pool_get_info(
|
||||
memory_pool_list[0], HSA_AMD_MEMORY_POOL_INFO_RUNTIME_ALLOC_GRANULE, (void*) &size);
|
||||
RET_IF_HSA_ERR(status)
|
||||
for(size_t j = 0; j < i; ++j)
|
||||
{
|
||||
hsa_amd_vmem_alloc_handle_t memory_handle{};
|
||||
|
||||
status = hsa_amd_vmem_handle_create(
|
||||
memory_pool_list[0], size, MEMORY_TYPE_NONE, 0, &memory_handle);
|
||||
RET_IF_HSA_ERR(status)
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main()
|
||||
{
|
||||
hsa_status_t status;
|
||||
status = hsa_init();
|
||||
RET_IF_HSA_ERR(status)
|
||||
|
||||
std::vector<hsa_agent_t> agents = get_agent_list();
|
||||
hsa_agent_t cpu_agent = get_cpu_agent(agents);
|
||||
hsa_agent_t gpu_agent = get_gpu_agent(agents);
|
||||
call_hsa_memory_allocate(6, 1024, cpu_agent);
|
||||
call_hsa_memory_pool_allocate(9, 512, gpu_agent);
|
||||
// Virtual memory API not supported in CI. Will add back if this changes
|
||||
// call_hsa_vmem_allocate(3, gpu_agent);
|
||||
|
||||
status = hsa_shut_down();
|
||||
RET_IF_HSA_ERR(status)
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user