Add support for scratch reporting (#523)
* Add ToolsApiTable Add ToolsApiTable wrapping for scratch memory tracking * Add initial support for scratch memory tracking Buffering is implemented * cmake formatting (cmake-format) (#525) Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com> * source formatting (clang-format v11) (#524) Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com> * Add callback tracing for scratch Fixed the error where scratch tracking init was called irrespective of whether any client requested for it * Apply suggestions from code review Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com> * Fix tools api copy/update Table were saved/updated incorrectly in previous commit. Also adds passing user data through the callback * Fix OpKind sequence for scratch tracking Previously scratch was using OpKind from rocprofiler-sdk, but templates were instantiated using API ID. These differ by 1 * Integration tests for scratch reporting Added buffer and callback integration tests for scratch reporting * source formatting (clang-format v11) (#550) Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com> * cmake formatting (cmake-format) (#551) Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com> * python formatting (black) (#549) Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com> * CI fixes * source formatting (clang-format v11) (#554) Co-authored-by: MythreyaK <26112391+MythreyaK@users.noreply.github.com> * Update api Rebase on main and updates based on PR feedback * Update scratch reporting and address PR comments - Added agent id to buffer records - Updated `test_internal_correlation_ids` - Is almost identical to one in async-copy - Updated scratch test to check for agent id - Updated queue id serialization in callback records (prints handle as nested key) - Remove `marker_api_traces` from scratch `test_internal_correlation_ids` validation test - Rename `amd_tools_api` to `scratch_memory` - Added doxygen comments - Remove scratch callback from `tool.cpp` - Replace assert with `LOF_IF` in `scratch_memory.cpp` * Update tools table Changed to match up with changes to hsa tables in main branch * Rework scratch memory structure * Update tests - Added suggestions from PR review, and updated tests accordingly * Misc cleanup * Update scratch test As of Apr 4th, `hsa_amd_agent_set_async_scratch_limit` is disabled. Note, > This API: `hsa_amd_agent_set_async_scratch_limit` is currently > disabled. We need some changes in CP firmware to be able to do this > and these changes are not ready yet. > With the current code, you will also not get notifications for > alternate-scratch allocations because this feature has been disabled > while CP firmware is making additional changes > We are hoping to have that feature enabled by ROCm-6.3 * Minor update to lib/rocprofiler-sdk/internal_threading.* - delay destruction of shared_ptrs of the tasks to prevent rare (but possible) data race on the destruction of the shared_ptr --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: MythreyaK <MythreyaK@users.noreply.github.com> Co-authored-by: Jonathan R. Madsen <jrmadsen@users.noreply.github.com> Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
このコミットが含まれているのは:
Mythreya
GitHub
@@ -12,6 +12,7 @@ add_subdirectory(simple-transpose)
|
||||
add_subdirectory(multistream)
|
||||
add_subdirectory(vector-operations)
|
||||
add_subdirectory(hip-in-libraries)
|
||||
add_subdirectory(scratch-memory)
|
||||
|
||||
set(CMAKE_BUILD_RPATH
|
||||
"\$ORIGIN:\$ORIGIN/../lib:$<TARGET_FILE_DIR:rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library>"
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tool-test-app-scratch-memory LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(scratch-memory.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(scratch-memory)
|
||||
target_sources(scratch-memory PRIVATE scratch-memory.cpp)
|
||||
target_compile_options(scratch-memory PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
|
||||
-Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(scratch-memory PRIVATE Threads::Threads hsa-runtime64)
|
||||
|
||||
install(
|
||||
TARGETS scratch-memory
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,239 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <hsa/hsa.h>
|
||||
#include <hsa/hsa_ext_amd.h>
|
||||
#include <cstdio>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
#define hipCheckErr(errval) \
|
||||
do \
|
||||
{ \
|
||||
hipCheckAndFail((errval), __FILE__, __LINE__); \
|
||||
} while(0)
|
||||
|
||||
#define hipCheckLastError() \
|
||||
do \
|
||||
{ \
|
||||
hipCheckErr(hipGetLastError()); \
|
||||
} while(0)
|
||||
|
||||
#define HSA_CALL2(cmd) \
|
||||
do \
|
||||
{ \
|
||||
hsa_status_t error = (cmd); \
|
||||
if(error != HSA_STATUS_SUCCESS) \
|
||||
{ \
|
||||
const char* errorStr; \
|
||||
hsa_status_string(error, &errorStr); \
|
||||
std::cout << "Encountered HSA error (" << errorStr << ") at line " << __LINE__ \
|
||||
<< " in file " << __FILE__ << "\n"; \
|
||||
exit(-1); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
namespace
|
||||
{
|
||||
inline void
|
||||
hipCheckAndFail(hipError_t errval, const char* file, int line)
|
||||
{
|
||||
if(errval != hipSuccess)
|
||||
{
|
||||
std::cerr << "hip error: " << hipGetErrorString(errval) << std::endl;
|
||||
std::cerr << " Location: " << file << ":" << line << std::endl;
|
||||
exit(errval);
|
||||
}
|
||||
}
|
||||
|
||||
hsa_status_t
|
||||
find_gpu_agents(hsa_agent_t agent, void* data)
|
||||
{
|
||||
hsa_status_t status;
|
||||
hsa_device_type_t device_type;
|
||||
status = hsa_agent_get_info(agent, HSA_AGENT_INFO_DEVICE, &device_type);
|
||||
if(status == HSA_STATUS_SUCCESS && device_type == HSA_DEVICE_TYPE_GPU)
|
||||
{
|
||||
std::vector<hsa_agent_t>* agents = reinterpret_cast<std::vector<hsa_agent_t>*>(data);
|
||||
agents->push_back(agent);
|
||||
}
|
||||
return HSA_STATUS_SUCCESS;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
__global__ void
|
||||
test_kern_large(uint64_t* output)
|
||||
{
|
||||
uint64_t result = 0;
|
||||
int test[4000];
|
||||
memset(test, 5, 4000);
|
||||
for(int& i : test)
|
||||
{
|
||||
i = i + 7;
|
||||
*output += i;
|
||||
result += i;
|
||||
}
|
||||
*output ^= result;
|
||||
*output ^= result;
|
||||
}
|
||||
|
||||
__global__ void
|
||||
test_kern_medium(uint64_t* output)
|
||||
{
|
||||
uint64_t result = 0;
|
||||
int test[175];
|
||||
memset(test, 5, 175);
|
||||
for(int& i : test)
|
||||
{
|
||||
i = i + 7;
|
||||
*output += i;
|
||||
result += i;
|
||||
}
|
||||
*output ^= result;
|
||||
*output ^= result;
|
||||
}
|
||||
|
||||
__global__ void
|
||||
test_kern_small(uint64_t* output)
|
||||
{
|
||||
uint64_t result = 0;
|
||||
int test[2];
|
||||
for(int& i : test)
|
||||
{
|
||||
i = i + 7;
|
||||
*output += i;
|
||||
result += i;
|
||||
}
|
||||
*output ^= result;
|
||||
*output ^= result;
|
||||
}
|
||||
|
||||
// Checks whether we get a request-more-scratch when grid-x is incremented
|
||||
int
|
||||
test_gridx(uint64_t* data_ptr)
|
||||
{
|
||||
*data_ptr = 0;
|
||||
printf("Running Medium\n");
|
||||
test_kern_medium<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Medium - done\n");
|
||||
|
||||
printf("Running Medium-2 - should trigger more-scratch requests\n");
|
||||
test_kern_medium<<<1500, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
|
||||
printf("Running Medium-2 - done\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// 1st allocation should go to primary, then large should still trigger a USO
|
||||
int
|
||||
test_primary_then_uso(uint64_t* data_ptr)
|
||||
{
|
||||
printf("Running Medium - all slots\n");
|
||||
test_kern_medium<<<10000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Medium - done\n");
|
||||
|
||||
printf("Running Large - should trigger USO\n");
|
||||
test_kern_large<<<1100, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Large - done\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
test_scratch()
|
||||
{
|
||||
uint64_t* data_ptr;
|
||||
hipCheckErr(hipHostMalloc(&data_ptr, sizeof(uint64_t), 0));
|
||||
|
||||
std::vector<float> host_floats(1024);
|
||||
float* dev;
|
||||
|
||||
hipCheckErr(hipMalloc((void**) &dev, host_floats.size() * sizeof(float)));
|
||||
hipCheckErr(hipMemcpy(
|
||||
dev, host_floats.data(), host_floats.size() * sizeof(float), hipMemcpyHostToDevice));
|
||||
|
||||
*data_ptr = 0;
|
||||
|
||||
printf("Running test_primary_then_uso========================\n");
|
||||
test_primary_then_uso(data_ptr);
|
||||
printf("=====================================================\n");
|
||||
|
||||
printf("Running test_gridx===================================\n");
|
||||
test_gridx(data_ptr);
|
||||
printf("=====================================================\n");
|
||||
|
||||
printf("Running Small\n");
|
||||
test_kern_small<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Small - done\n");
|
||||
|
||||
printf("Running Medium\n");
|
||||
test_kern_medium<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Medium - done\n");
|
||||
|
||||
printf("Running Small\n");
|
||||
test_kern_small<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Small - done\n");
|
||||
|
||||
printf("Running Large\n");
|
||||
test_kern_large<<<1100, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Large - done\n");
|
||||
|
||||
printf("Running Large\n");
|
||||
test_kern_large<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Large - done\n");
|
||||
|
||||
printf("Running Large\n");
|
||||
test_kern_large<<<1000, 1>>>(data_ptr);
|
||||
hipCheckErr(hipFree(dev));
|
||||
hipCheckErr(hipDeviceSynchronize());
|
||||
printf("Running Large - done\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
main()
|
||||
{
|
||||
hipCheckErr(hipInit(0));
|
||||
|
||||
std::vector<hsa_agent_t> agents;
|
||||
HSA_CALL2(hsa_iterate_agents(find_gpu_agents, &agents));
|
||||
size_t numAgents = agents.size();
|
||||
printf("Detected %ld agents\n", numAgents);
|
||||
|
||||
for(size_t i = 0; i < agents.size(); ++i)
|
||||
{
|
||||
hipCheckErr(hipSetDevice(i));
|
||||
test_scratch();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
新しいイシューから参照
ユーザーをブロックする