AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Memory Allocation Tracking (#1142)

Browse Source 
* Initial commit: Need to implement wrapper function to collect data and test that wrapper function is correctly replacing core HSA functions

* Attempted to implement wrapper implementation for hsa memory allocation functions. Need to modify generate record files and test if implementation is working as expected

* Debugging and implementing generateCSV function

* Memory allocation size and starting address outputted to csv and json file formats

* Formatting

* Initial setup for OTF2 and Perfetto generation

* Collecting agent id for memory_allocation and formatting

* Modified memory_allocation.cpp to set up code for AMD_EXT commands

* Support for memory_pool_allocate added

* Removed accidently added file

* Made flag optional and added more OTF2 and Perfetto code. Needs testing to ensure perfetto and OTF2 works

* Formatting

* Fixed perfetto and otf2 output

* Fixed flag issue due to incorrect buffer use

* Updated documentation

* Small cleaning and comments

* Added test for HSA memory allocation tracing

* Fixed summary test validation errors due to allocation tracing. Added type to location_base to create unique event ids for allocation due to OTF2 trace error

* Decreased lower limit of hip calls for test

* Modified summary tests to vary number of allocate requests

* Minor fixes to address comments. Still need to address OTF2 comments

* Fix docs and changed OTF2 to use enum for type specified in location_base construction

* Fixed schema error

* Added vmem command tracking. Need to add test

* Updated test to work with vmem command and updated generateCSV to output int instead of hex string.

* OTF2 enum update and mispelling fix

* CI does not support Virtual Memory API. Removed vmem test. Will add back if CI is modifed to suport vmem API

* Update CMakeLists.txt for memory allocation test

* Updated summary test

* Minor fixes to address comments

* Moved domain_type.hpp enum to before LAST

* Fixed compile errors and formatting

* Fixed stats summary domain name error

* Added rocprofv3 test

* Page migration test fix

* Undo page migration test changes. Failures do not appear to have to do with memory allocation
This commit is contained in:
itrowbri
2024-11-18 20:22:14 -06:00
committed by GitHub
parent 0d764eb3c5
commit 3bd7773cf7
53 changed files with 2387 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
source/include/rocprofiler-sdk/buffer_tracing.h
+22
View File
@@ -203,6 +203,28 @@ typedef struct
/// ::rocprofiler_memory_copy_operation_t)
} rocprofiler_buffer_tracing_memory_copy_record_t;
/**
* @brief ROCProfiler Buffer Memory Allocation Tracer Record.
*/
typedef struct
{
uint64_t size; ///< size of this struct
rocprofiler_buffer_tracing_kind_t kind;
rocprofiler_memory_allocation_operation_t operation;
rocprofiler_correlation_id_t correlation_id; ///< correlation ids for record
rocprofiler_thread_id_t thread_id; ///< id for thread that triggered copy
rocprofiler_timestamp_t start_timestamp; ///< start time in nanoseconds
rocprofiler_timestamp_t end_timestamp; ///< end time in nanoseconds
rocprofiler_agent_id_t agent_id; ///< agent information for memory allocation
uint64_t starting_address; ///< starting address for memory allocation
uint64_t allocation_size; ///< size for memory allocation
/// @var kind
/// @brief ::ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION
/// @var operation
/// @brief Specification of the memory allocation function (@see
/// ::rocprofiler_memory_allocation_operation_t
} rocprofiler_buffer_tracing_memory_allocation_record_t;
/**
* @brief ROCProfiler Buffer Kernel Dispatch Tracer Record.
*/
source/include/rocprofiler-sdk/callback_tracing.h
+13
View File
@@ -210,6 +210,19 @@ typedef struct
uint64_t bytes; ///< bytes copied
} rocprofiler_callback_tracing_memory_copy_data_t;
/**
* @brief ROCProfiler Memory Copy Allocation Tracer Record.
*/
typedef struct
{
uint64_t size; ///< size of this struct
rocprofiler_timestamp_t start_timestamp; ///< start time in nanoseconds
rocprofiler_timestamp_t end_timestamp; ///< end time in nanoseconds
rocprofiler_agent_id_t agent_id; ///< agent id for memory allocation
uint64_t starting_address; ///< starting address for memory allocation
uint64_t allocation_size; ///< size of memory allocation
} rocprofiler_callback_tracing_memory_allocation_data_t;
/**
* @brief ROCProfiler Scratch Memory Callback Data.
*/
source/include/rocprofiler-sdk/cxx/hash.hpp
+2
View File
@@ -66,6 +66,8 @@ ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(rocprofiler_callback_thread_t)
ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(hsa_agent_t)
ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(hsa_signal_t)
ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(hsa_executable_t)
ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(hsa_region_t)
ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER(hsa_amd_memory_pool_t)
#undef ROCPROFILER_CXX_SPECIALIZE_HANDLE_HASHER
} // namespace std
source/include/rocprofiler-sdk/cxx/operators.hpp
+11
View File
@@ -26,6 +26,7 @@
#include <rocprofiler-sdk/agent.h>
#include <rocprofiler-sdk/defines.h>
#include <rocprofiler-sdk/fwd.h>
#include <rocprofiler-sdk/hsa.h>
#include <rocprofiler-sdk/internal_threading.h>
#include <tuple>
@@ -104,6 +105,8 @@ ROCPROFILER_CXX_DECLARE_OPERATORS(hsa_signal_t)
ROCPROFILER_CXX_DECLARE_OPERATORS(hsa_executable_t)
ROCPROFILER_CXX_DECLARE_OPERATORS(const rocprofiler_agent_v0_t&)
ROCPROFILER_CXX_DECLARE_OPERATORS(rocprofiler_dim3_t)
ROCPROFILER_CXX_DECLARE_OPERATORS(hsa_region_t)
ROCPROFILER_CXX_DECLARE_OPERATORS(hsa_amd_memory_pool_t)
// definitions of operator==
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(rocprofiler_context_id_t)
@@ -116,6 +119,8 @@ ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(rocprofiler_callback_thread_t)
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(hsa_agent_t)
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(hsa_signal_t)
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(hsa_executable_t)
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(hsa_region_t)
ROCPROFILER_CXX_DEFINE_EQ_HANDLE_OPERATOR(hsa_amd_memory_pool_t)
inline bool
operator==(const rocprofiler_agent_v0_t& lhs, const rocprofiler_agent_v0_t& rhs)
@@ -142,6 +147,8 @@ ROCPROFILER_CXX_DEFINE_NE_OPERATOR(hsa_signal_t)
ROCPROFILER_CXX_DEFINE_NE_OPERATOR(hsa_executable_t)
ROCPROFILER_CXX_DEFINE_NE_OPERATOR(const rocprofiler_agent_v0_t&)
ROCPROFILER_CXX_DEFINE_NE_OPERATOR(rocprofiler_dim3_t)
ROCPROFILER_CXX_DEFINE_NE_OPERATOR(hsa_region_t)
ROCPROFILER_CXX_DEFINE_NE_OPERATOR(hsa_amd_memory_pool_t)
// definitions of operator<
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(rocprofiler_context_id_t)
@@ -154,6 +161,8 @@ ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(rocprofiler_callback_thread_t)
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(hsa_agent_t)
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(hsa_signal_t)
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(hsa_executable_t)
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(hsa_region_t)
ROCPROFILER_CXX_DEFINE_LT_HANDLE_OPERATOR(hsa_amd_memory_pool_t)
inline bool
operator<(const rocprofiler_agent_v0_t& lhs, const rocprofiler_agent_v0_t& rhs)
@@ -185,6 +194,8 @@ ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(hsa_signal_t)
ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(hsa_executable_t)
ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(const rocprofiler_agent_v0_t&)
ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(rocprofiler_dim3_t)
ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(hsa_region_t)
ROCPROFILER_CXX_DEFINE_COMPARE_OPERATORS(hsa_amd_memory_pool_t)
// cleanup defines
#undef ROCPROFILER_CXX_DECLARE_OPERATORS
source/include/rocprofiler-sdk/cxx/perfetto.hpp
+3 -1
View File
@@ -80,6 +80,7 @@ ROCPROFILER_DEFINE_CATEGORY(category, marker_api, "Marker API region")
ROCPROFILER_DEFINE_CATEGORY(category, rccl_api, "RCCL API function")
ROCPROFILER_DEFINE_CATEGORY(category, kernel_dispatch, "GPU kernel dispatch")
ROCPROFILER_DEFINE_CATEGORY(category, memory_copy, "Async memory copy")
ROCPROFILER_DEFINE_CATEGORY(category, memory_allocation, "Memory Allocation")
#define ROCPROFILER_PERFETTO_CATEGORIES \
ROCPROFILER_PERFETTO_CATEGORY(category::hsa_api), \
@@ -87,7 +88,8 @@ ROCPROFILER_DEFINE_CATEGORY(category, memory_copy, "Async memory copy")
ROCPROFILER_PERFETTO_CATEGORY(category::marker_api), \
ROCPROFILER_PERFETTO_CATEGORY(category::rccl_api), \
ROCPROFILER_PERFETTO_CATEGORY(category::kernel_dispatch), \
ROCPROFILER_PERFETTO_CATEGORY(category::memory_copy)
ROCPROFILER_PERFETTO_CATEGORY(category::memory_copy), \
ROCPROFILER_PERFETTO_CATEGORY(category::memory_allocation)
#include <perfetto.h>
source/include/rocprofiler-sdk/cxx/serialization.hpp
+28
View File
@@ -330,6 +330,18 @@ save(ArchiveT& ar, rocprofiler_callback_tracing_memory_copy_data_t data)
ROCP_SDK_SAVE_DATA_FIELD(bytes);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_callback_tracing_memory_allocation_data_t data)
{
ROCP_SDK_SAVE_DATA_FIELD(size);
ROCP_SDK_SAVE_DATA_FIELD(start_timestamp);
ROCP_SDK_SAVE_DATA_FIELD(end_timestamp);
ROCP_SDK_SAVE_DATA_FIELD(agent_id);
ROCP_SDK_SAVE_DATA_FIELD(starting_address);
ROCP_SDK_SAVE_DATA_FIELD(allocation_size);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_rccl_api_retval_t data)
@@ -461,6 +473,22 @@ save(ArchiveT& ar, rocprofiler_buffer_tracing_memory_copy_record_t data)
ROCP_SDK_SAVE_DATA_FIELD(bytes);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, rocprofiler_buffer_tracing_memory_allocation_record_t data)
{
ROCP_SDK_SAVE_DATA_FIELD(size);
ROCP_SDK_SAVE_DATA_FIELD(kind);
ROCP_SDK_SAVE_DATA_FIELD(operation);
ROCP_SDK_SAVE_DATA_FIELD(thread_id);
ROCP_SDK_SAVE_DATA_FIELD(correlation_id);
ROCP_SDK_SAVE_DATA_FIELD(start_timestamp);
ROCP_SDK_SAVE_DATA_FIELD(end_timestamp);
ROCP_SDK_SAVE_DATA_FIELD(agent_id);
ROCP_SDK_SAVE_DATA_FIELD(starting_address);
ROCP_SDK_SAVE_DATA_FIELD(allocation_size);
}
template <typename ArchiveT>
void
save(ArchiveT& ar, const rocprofiler_page_migration_page_fault_start_t& data)
source/include/rocprofiler-sdk/external_correlation.h
+1
View File
@@ -68,6 +68,7 @@ typedef enum // NOLINT(performance-enum-size)
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_SCRATCH_MEMORY, ///<
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_RCCL_API, ///<
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_OPENMP, ///<
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_MEMORY_ALLOCATION, ///<
ROCPROFILER_EXTERNAL_CORRELATION_REQUEST_LAST,
} rocprofiler_external_correlation_id_request_kind_t;
source/include/rocprofiler-sdk/fwd.h
+20 -4
View File
@@ -167,10 +167,12 @@ typedef enum // NOLINT(performance-enum-size)
ROCPROFILER_CALLBACK_TRACING_MARKER_NAME_API, ///< @see ::rocprofiler_marker_name_api_id_t
ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT, ///< @see ::rocprofiler_code_object_operation_t
ROCPROFILER_CALLBACK_TRACING_SCRATCH_MEMORY, ///< @see ::rocprofiler_scratch_memory_operation_t
ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH, ///< Callbacks for kernel dispatches
ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY, ///< @see ::rocprofiler_memory_copy_operation_t
ROCPROFILER_CALLBACK_TRACING_RCCL_API, ///< @RCCL tracing
ROCPROFILER_CALLBACK_TRACING_OPENMP, ///< @see ::rocprofiler_ompt_operation_t
ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH, ///< Callbacks for kernel dispatches
ROCPROFILER_CALLBACK_TRACING_MEMORY_COPY, ///< @see ::rocprofiler_memory_copy_operation_t
ROCPROFILER_CALLBACK_TRACING_RCCL_API, ///< @RCCL tracing
ROCPROFILER_CALLBACK_TRACING_OPENMP, ///< @see ::rocprofiler_ompt_operation_t
ROCPROFILER_CALLBACK_TRACING_MEMORY_ALLOCATION, ///< @see
///< ::rocprofiler_memory_allocation_operation_t
ROCPROFILER_CALLBACK_TRACING_LAST,
} rocprofiler_callback_tracing_kind_t;
@@ -197,6 +199,8 @@ typedef enum // NOLINT(performance-enum-size)
ROCPROFILER_BUFFER_TRACING_CORRELATION_ID_RETIREMENT, ///< Correlation ID in no longer in use
ROCPROFILER_BUFFER_TRACING_RCCL_API, ///< RCCL tracing
ROCPROFILER_BUFFER_TRACING_OPENMP, ///< @see ::rocprofiler_ompt_operation_t
ROCPROFILER_BUFFER_TRACING_MEMORY_ALLOCATION, ///< @see
///< ::rocprofiler_memory_allocation_operation_t
ROCPROFILER_BUFFER_TRACING_LAST,
} rocprofiler_buffer_tracing_kind_t;
@@ -224,6 +228,18 @@ typedef enum // NOLINT(performance-enum-size)
ROCPROFILER_MEMORY_COPY_LAST,
} rocprofiler_memory_copy_operation_t;
/**
* @brief Memory Allocation Operation.
*/
typedef enum // NOLINT(performance-enum-size)
{
ROCPROFILER_MEMORY_ALLOCATION_NONE = 0, ///< Unknown memory allocation function
ROCPROFILER_MEMORY_ALLOCATION_ALLOCATE, ///< Allocate memory function
ROCPROFILER_MEMORY_ALLOCATION_MEMORY_POOL_ALLOCATE, ///< Allocate memory pool
ROCPROFILER_MEMORY_ALLOCATION_VMEM_HANDLE_CREATE, ///< Allocate vmem memory handle
ROCPROFILER_MEMORY_ALLOCATION_LAST,
} rocprofiler_memory_allocation_operation_t;
/**
* @brief ROCProfiler Kernel Dispatch Tracing Operation Types.
*/