Shared Library Constructor (rocprofv3 deadlock fix) (#599)
* Moved tests/apps to tests/bin * Renamed cmake project in tests/bin * Update samples - Use ROCPROFILER_DEFAULT_FAIL_REGEX - tweaks to stdout messages * Update tests - Use ROCPROFILER_DEFAULT_FAIL_REGEX * Add tests/lib - libraries with HIP code * Update PTL submodule - remove atexit delete of thread_id_map * Update cmake/rocprofiler_options.cmake - Set ROCPROFILER_DEFAULT_FAIL_REGEX * Update common lib: env + logging - improved customization of logging settings - default to disabling logging to files - install failure handler for rocprofv3 - set_env support in environment.* * Add lib/rocprofiler-sdk/shared_library.cpp - shared library constructor * Update lib/rocprofiler-sdk-tool/tool.cpp - destructor thread safety - convert callback_name_info and buffered_name_info to pointers - install failure handler for logging * Add tests/bin/hip-in-libraries - hip-in-libraries is an exe which uses two shared libraries where each shared library contains HIP kernels - used for testing deadlocking within __hipRegisterFatBinary * Update bin/rocprofv3 - reorganized the env variables - use exec to launch command - set ROCPROFILER_LIBRARY_CTOR=1 * Add tests/rocprofv3/tracing-hip-in-libraries - uses hip-in-libraries exe for exe which uses shared libraries to launch HIP kernels * Update bin/rocprofv3 - fix counter collection (no exec) * Update lib/rocprofiler-sdk-tool/tool.cpp - replace "Kernel-Name" with "Kernel_Name" * Update lib/rocprofiler-sdk/registration.cpp Use RTLD_LOCAL instead of RTLD_GLOBAL for env libraries * Update tests/rocprofv3 - replace "Kernel-Name" with "Kernel_Name" * Update tests - vector-ops (bin) stream syncs + runs with 4 queues per device - improve counter-collection/input1 validation - rocprofv3/tracing-hip-in-libraries does not do sys-trace - improved validation script for tracing-hip-in-libraries - updated dispatch_callback in json-tool.cpp following reworking of prototypes for counter collection * Update samples/counter_collection - updated dispatch_callback(s) and record_callback(s) following reworking of prototypes * Update bin/rocprofv3 - reorganized help menu - added options for sub-HSA tables - added --hip-runtime-trace - changed --hip-trace to include --hip-compiler-trace * Update lib/rocprofiler-sdk-tool - improved kernel filtering - removed arch_vgpr, accum_vgpr, sgpr code (in rocprofiler-sdk) - fixed issue with counter-collection w/o tracing - added support for fine grained HSA API tracing - removed directly linking to HSA-runtime * Update lib/rocprofiler-sdk/agent.cpp - rocp_agents != hsa_agents is non-fatal when ROCPROFILER_BUILD_CI=OFF (CMake option) * GPR (vector and scalar) info in kernel symbol data - rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t contains general purpose register info * Header include order fix - Include repo headers first - Third party library headers next - standard library headers last * Update dispatch profiling public API - introduce rocprofiler_profile_counting_dispatch_data_t - change signature of rocprofiler_profile_counting_dispatch_callback_t and rocprofiler_profile_counting_record_callback_t - provide rocprofiler_user_data_t pointer in dispatch callback - provide rocprofiler_user_data_t value (from dispatch cb) in record callback * Update tests/bin/CMakeLists.txt - fix add_subdirectory(hip-in-libraries) order * Update VERSION - bump to 0.2.0 in prep for AFAR
Этот коммит содержится в:
Jonathan R. Madsen authored and 
GitHub committed
1 родитель
665c546e65
Коммит
7b6d3c70bd
85 files changed
+2497
-856
No files matched your search
@@ -0,0 +1,22 @@
|
||||
#
|
||||
# Integration test applications
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
project(rocprofiler-tests-bin LANGUAGES C CXX)
|
||||
|
||||
set(CMAKE_BUILD_RPATH "\$ORIGIN:\$ORIGIN/../lib")
|
||||
|
||||
# applications used by integration tests which DO NOT link to rocprofiler-sdk-roctx
|
||||
add_subdirectory(simple-transpose)
|
||||
add_subdirectory(multistream)
|
||||
add_subdirectory(vector-operations)
|
||||
add_subdirectory(hip-in-libraries)
|
||||
|
||||
set(CMAKE_BUILD_RPATH
|
||||
"\$ORIGIN:\$ORIGIN/../lib:$<TARGET_FILE_DIR:rocprofiler-sdk-roctx::rocprofiler-sdk-roctx-shared-library>"
|
||||
)
|
||||
|
||||
# applications used by integration tests which DO link to rocprofiler-sdk-roctx
|
||||
add_subdirectory(reproducible-runtime)
|
||||
add_subdirectory(transpose)
|
||||
@@ -0,0 +1,34 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
project(rocprofiler-tests-bin-hip-in-libraries LANGUAGES CXX)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
add_executable(hip-in-libraries)
|
||||
target_sources(hip-in-libraries PRIVATE hip-in-libraries.cpp)
|
||||
target_compile_options(hip-in-libraries PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
|
||||
-Werror)
|
||||
target_link_libraries(hip-in-libraries PRIVATE transpose-shared-library
|
||||
vector-ops-shared-library)
|
||||
|
||||
find_package(hip REQUIRED)
|
||||
target_link_libraries(hip-in-libraries PRIVATE hip::host)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(hip-in-libraries PRIVATE Threads::Threads)
|
||||
|
||||
if(TRANSPOSE_USE_MPI)
|
||||
find_package(MPI REQUIRED)
|
||||
target_compile_definitions(hip-in-libraries PRIVATE USE_MPI)
|
||||
target_link_libraries(hip-in-libraries PRIVATE MPI::MPI_C)
|
||||
endif()
|
||||
|
||||
install(
|
||||
TARGETS hip-in-libraries
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,151 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
#include "transpose.hpp"
|
||||
#include "vector-ops.hpp"
|
||||
|
||||
#include <hip/hip_runtime_api.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
|
||||
#if defined(USE_MPI)
|
||||
# include <mpi.h>
|
||||
#endif
|
||||
|
||||
#define HIP_API_CALL(CALL) \
|
||||
{ \
|
||||
hipError_t error_ = (CALL); \
|
||||
if(error_ != hipSuccess) \
|
||||
{ \
|
||||
auto _hip_api_print_lk = auto_lock_t{print_lock}; \
|
||||
fprintf(stderr, \
|
||||
"%s:%d :: HIP error : %s\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
hipGetErrorString(error_)); \
|
||||
throw std::runtime_error("hip_api_call"); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using auto_lock_t = std::unique_lock<std::mutex>;
|
||||
auto print_lock = std::mutex{};
|
||||
size_t nqueues = 8;
|
||||
size_t nthreads = 4;
|
||||
size_t nitr = 500;
|
||||
size_t nsync = 10;
|
||||
} // namespace
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
int rank = 0;
|
||||
int size = 1;
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Init(&argc, &argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &size);
|
||||
#else
|
||||
(void) size;
|
||||
#endif
|
||||
|
||||
for(int i = 1; i < argc; ++i)
|
||||
{
|
||||
auto _arg = std::string{argv[i]};
|
||||
if(_arg == "?" || _arg == "-h" || _arg == "--help")
|
||||
{
|
||||
if(rank == 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"usage: hip-in-libraries [NUM_QUEUES (%zu)] [NUM_THREADS (%zu)] "
|
||||
"[NUM_ITERATION (%zu)] "
|
||||
"[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
|
||||
nqueues,
|
||||
nthreads,
|
||||
nitr,
|
||||
nsync);
|
||||
}
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
if(argc > 1) nqueues = atoll(argv[1]);
|
||||
if(argc > 2) nthreads = atoll(argv[2]);
|
||||
if(argc > 3) nitr = atoll(argv[3]);
|
||||
if(argc > 4) nsync = atoll(argv[4]);
|
||||
|
||||
int ndevice = 0;
|
||||
HIP_API_CALL(hipGetDeviceCount(&ndevice));
|
||||
|
||||
printf("[hip-in-libraries] Number of devices found: %i\n", ndevice);
|
||||
printf("[hip-in-libraries] Number of queues: %zu\n", nqueues);
|
||||
printf("[hip-in-libraries] Number of threads: %zu\n", nthreads);
|
||||
printf("[hip-in-libraries] Number of iterations: %zu\n", nitr);
|
||||
printf("[hip-in-libraries] Syncing every %zu iterations\n", nsync);
|
||||
|
||||
{
|
||||
auto vector_ops_thread = std::thread{run_vector_ops, nthreads, nqueues};
|
||||
auto transpose_thread = std::thread{run_transpose, nthreads, nitr, nsync};
|
||||
|
||||
vector_ops_thread.join();
|
||||
transpose_thread.join();
|
||||
}
|
||||
|
||||
// this is a temporary workaround in omnitrace when HIP + MPI is enabled
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
for(int i = 0; i < ndevice; ++i)
|
||||
{
|
||||
HIP_API_CALL(hipSetDevice(i));
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
}
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
if(rank == 0)
|
||||
{
|
||||
for(int i = 0; i < ndevice; ++i)
|
||||
{
|
||||
HIP_API_CALL(hipSetDevice(i));
|
||||
HIP_API_CALL(hipDeviceReset());
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tests-bin-multistream LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(multistream_app.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(multistream)
|
||||
target_sources(multistream PRIVATE multistream_app.cpp)
|
||||
target_compile_options(multistream PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(multistream PRIVATE Threads::Threads)
|
||||
|
||||
install(
|
||||
TARGETS multistream
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,116 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <vector>
|
||||
#define HIP_ASSERT(call) \
|
||||
do \
|
||||
{ \
|
||||
hipError_t err = call; \
|
||||
if(err != hipSuccess) \
|
||||
{ \
|
||||
fprintf(stderr, "%s\n", hipGetErrorString(err)); \
|
||||
abort(); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
__device__ int counter = 0;
|
||||
__global__ void
|
||||
add(int n, float* x, float* y)
|
||||
{
|
||||
if(__hip_atomic_load(&counter, __ATOMIC_ACQUIRE, __HIP_MEMORY_SCOPE_AGENT) != 0)
|
||||
{
|
||||
abort();
|
||||
}
|
||||
__hip_atomic_fetch_add(&counter, 1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
|
||||
|
||||
int index = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = blockDim.x * gridDim.x;
|
||||
for(int i = index; i < n; i += stride)
|
||||
y[i] = x[i] + y[i];
|
||||
__hip_atomic_fetch_add(&counter, -1, __ATOMIC_RELEASE, __HIP_MEMORY_SCOPE_SYSTEM);
|
||||
}
|
||||
|
||||
void
|
||||
LaunchMultiStreamKernels()
|
||||
{
|
||||
int N = 1 << 4;
|
||||
float* x = new float[N];
|
||||
float* y = new float[N];
|
||||
float* d_x;
|
||||
float* d_y;
|
||||
// Allocate Unified Memory -- accessible from CPU or GPU
|
||||
HIP_ASSERT(hipMallocManaged(&d_x, N * sizeof(float)));
|
||||
HIP_ASSERT(hipMallocManaged(&d_y, N * sizeof(float)));
|
||||
|
||||
// initialize x and y arrays on the host
|
||||
for(int i = 0; i < N; i++)
|
||||
{
|
||||
x[i] = 1.0f;
|
||||
y[i] = 2.0f;
|
||||
}
|
||||
std::vector<hipStream_t> hip_streams;
|
||||
for(int i = 0; i < 100; i++)
|
||||
{
|
||||
hipStream_t stream;
|
||||
HIP_ASSERT(hipStreamCreate(&stream));
|
||||
hip_streams.push_back(stream);
|
||||
}
|
||||
HIP_ASSERT(hipMemcpy(d_x, x, N * sizeof(float), hipMemcpyHostToDevice));
|
||||
HIP_ASSERT(hipMemcpy(d_y, y, N * sizeof(float), hipMemcpyHostToDevice));
|
||||
|
||||
// Launch kernel on 1M elements on the GPU
|
||||
int blockSize = 64;
|
||||
// This Kernel will always be launched with one wave
|
||||
int numBlocks = 1;
|
||||
for(int i = 0; i < 100; i++)
|
||||
{
|
||||
for(size_t j = 0; j < hip_streams.size(); j++)
|
||||
{
|
||||
hipLaunchKernelGGL(add, numBlocks, blockSize, 0, hip_streams[j], N, d_x, d_y);
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for GPU to finish before accessing on host
|
||||
HIP_ASSERT(hipDeviceSynchronize());
|
||||
|
||||
HIP_ASSERT(hipMemcpy(x, d_x, N * sizeof(float), hipMemcpyDeviceToHost));
|
||||
HIP_ASSERT(hipMemcpy(y, d_y, N * sizeof(float), hipMemcpyDeviceToHost));
|
||||
|
||||
// Free memory
|
||||
HIP_ASSERT(hipFree(d_x));
|
||||
HIP_ASSERT(hipFree(d_y));
|
||||
|
||||
delete[] x;
|
||||
delete[] y;
|
||||
|
||||
for(size_t i = 0; i < hip_streams.size(); i++)
|
||||
{
|
||||
HIP_ASSERT(hipStreamDestroy(hip_streams[i]));
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main()
|
||||
{
|
||||
LaunchMultiStreamKernels();
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tests-bin-reproducible-runtime LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
option(REPRODUCIBLE_RUNTIME_USE_MPI "Enable MPI support in reproducible-runtime exe" OFF)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(reproducible-runtime.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(reproducible-runtime)
|
||||
target_sources(reproducible-runtime PRIVATE reproducible-runtime.cpp)
|
||||
target_compile_options(reproducible-runtime PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
|
||||
-Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(reproducible-runtime PRIVATE Threads::Threads)
|
||||
|
||||
find_package(rocprofiler-sdk-roctx REQUIRED)
|
||||
target_link_libraries(reproducible-runtime
|
||||
PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
|
||||
|
||||
if(REPRODUCIBLE_RUNTIME_USE_MPI)
|
||||
find_package(MPI REQUIRED)
|
||||
target_compile_definitions(reproducible-runtime PRIVATE USE_MPI)
|
||||
target_link_libraries(reproducible-runtime PRIVATE MPI::MPI_C)
|
||||
endif()
|
||||
|
||||
install(
|
||||
TARGETS reproducible-runtime
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,190 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
#include "hip/hip_runtime.h"
|
||||
#include "rocprofiler-sdk-roctx/roctx.h"
|
||||
|
||||
#include <unistd.h>
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <iomanip>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
|
||||
#if defined(USE_MPI)
|
||||
# include <mpi.h>
|
||||
#endif
|
||||
|
||||
#define HIP_API_CALL(CALL) \
|
||||
{ \
|
||||
hipError_t error_ = (CALL); \
|
||||
if(error_ != hipSuccess) \
|
||||
{ \
|
||||
auto _hip_api_print_lk = auto_lock_t{print_lock}; \
|
||||
fprintf(stderr, \
|
||||
"%s:%d :: HIP error %i : %s\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
static_cast<int>(error_), \
|
||||
hipGetErrorString(error_)); \
|
||||
throw std::runtime_error("hip_api_call"); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using auto_lock_t = std::unique_lock<std::mutex>;
|
||||
auto print_lock = std::mutex{};
|
||||
double nruntime = 500.0; // ms
|
||||
uint32_t nspin = 1000000;
|
||||
size_t nthreads = 2;
|
||||
|
||||
void
|
||||
check_hip_error(void);
|
||||
} // namespace
|
||||
|
||||
__global__ void
|
||||
reproducible_runtime(uint32_t nspin);
|
||||
|
||||
void
|
||||
run(int tid, int devid);
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
for(int i = 1; i < argc; ++i)
|
||||
{
|
||||
auto _arg = std::string{argv[i]};
|
||||
if(_arg == "?" || _arg == "-h" || _arg == "--help")
|
||||
{
|
||||
fprintf(stderr,
|
||||
"usage: reproducible-runtime [KERNEL RUNTIME PER THREAD (default: %f msec)] "
|
||||
"[SPIN CYCLES PER KERNEL LAUNCH (default: %u)] [NUM_THREADS (default: %zu)]\n",
|
||||
nruntime,
|
||||
nspin,
|
||||
nthreads);
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
}
|
||||
|
||||
if(argc > 1) nruntime = std::stod(argv[1]);
|
||||
if(argc > 2) nspin = std::stoll(argv[2]);
|
||||
if(argc > 3) nthreads = std::stoll(argv[3]);
|
||||
|
||||
printf("[reproducible-runtime] Kernel runtime per thread: %.3f msec\n", nruntime);
|
||||
printf("[reproducible-runtime] Spin time per kernel: %u cycles\n", nspin);
|
||||
printf("[reproducible-runtime] Number of threads: %zu\n", nthreads);
|
||||
|
||||
// this is a temporary workaround in omnitrace when HIP + MPI is enabled
|
||||
int ndevice = 0;
|
||||
HIP_API_CALL(hipGetDeviceCount(&ndevice));
|
||||
printf("[reproducible-runtime] Number of devices found: %i\n", ndevice);
|
||||
auto _threads = std::vector<std::thread>{};
|
||||
for(size_t i = 0; i < nthreads; ++i)
|
||||
_threads.emplace_back(run, i, i % ndevice);
|
||||
for(auto& itr : _threads)
|
||||
itr.join();
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
HIP_API_CALL(hipDeviceReset());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
__global__ void
|
||||
reproducible_runtime(uint32_t nspin_v)
|
||||
{
|
||||
for(uint32_t i = 0; i < nspin_v / 2048; i++)
|
||||
asm volatile("s_sleep 32"); // ~2048 cycles -> ~1us
|
||||
uint32_t remainder = nspin_v % 2048;
|
||||
for(uint32_t i = 0; i < remainder / 64; i++)
|
||||
asm volatile("s_sleep 1");
|
||||
}
|
||||
|
||||
void
|
||||
run(int tid, int devid)
|
||||
{
|
||||
auto roctx_range_id = roctxRangeStart("run");
|
||||
|
||||
constexpr int min_sa = 8;
|
||||
constexpr int min_avail_simd = 24;
|
||||
dim3 grid(min_sa * min_avail_simd);
|
||||
dim3 block(32);
|
||||
double time = 0.0;
|
||||
hipStream_t stream = {};
|
||||
hipEvent_t start = {};
|
||||
hipEvent_t stop = {};
|
||||
|
||||
HIP_API_CALL(hipSetDevice(devid));
|
||||
HIP_API_CALL(hipStreamCreate(&stream));
|
||||
HIP_API_CALL(hipEventCreate(&start));
|
||||
HIP_API_CALL(hipEventCreate(&stop));
|
||||
|
||||
do
|
||||
{
|
||||
roctxMark("iteration");
|
||||
uint32_t cyclesleft = 2000 * 1000 * (nruntime - static_cast<double>(time));
|
||||
HIP_API_CALL(hipEventRecord(start, stream));
|
||||
reproducible_runtime<<<grid, block, 0, stream>>>(std::min<uint32_t>(nspin, cyclesleft));
|
||||
HIP_API_CALL(hipEventRecord(stop, stream));
|
||||
check_hip_error();
|
||||
HIP_API_CALL(hipEventSynchronize(stop));
|
||||
float elapsed = 0.0f;
|
||||
HIP_API_CALL(hipEventElapsedTime(&elapsed, start, stop));
|
||||
time += static_cast<double>(elapsed);
|
||||
} while(time < nruntime);
|
||||
|
||||
HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
HIP_API_CALL(hipEventDestroy(start));
|
||||
HIP_API_CALL(hipEventDestroy(stop));
|
||||
|
||||
{
|
||||
auto _msg = std::stringstream{};
|
||||
_msg << '[' << getpid() << "][" << tid << "] Runtime of reproducible-runtime is "
|
||||
<< std::setprecision(2) << std::fixed << time << " ms (" << std::setprecision(3)
|
||||
<< (time / 1000.0f) << " sec)\n";
|
||||
auto_lock_t _lk{print_lock};
|
||||
std::cout << _msg.str() << std::flush;
|
||||
}
|
||||
|
||||
HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
HIP_API_CALL(hipStreamDestroy(stream));
|
||||
|
||||
roctxRangeStop(roctx_range_id);
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
void
|
||||
check_hip_error(void)
|
||||
{
|
||||
hipError_t err = hipGetLastError();
|
||||
if(err != hipSuccess)
|
||||
{
|
||||
auto_lock_t _lk{print_lock};
|
||||
std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
|
||||
throw std::runtime_error("hip_api_call");
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
@@ -0,0 +1,51 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tests-bin-transpose LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(simple-transpose.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(simple-transpose)
|
||||
target_sources(simple-transpose PRIVATE simple-transpose.cpp)
|
||||
target_compile_options(simple-transpose PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow
|
||||
-Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(simple-transpose PRIVATE Threads::Threads)
|
||||
|
||||
find_package(rocprofiler-sdk-roctx REQUIRED)
|
||||
target_link_libraries(simple-transpose
|
||||
PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
|
||||
|
||||
install(
|
||||
TARGETS simple-transpose
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
Copyright (c) 2015 - 2021 Advanced Micro Devices, Inc. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
|
||||
// hip header file
|
||||
#include <hip/hip_runtime.h>
|
||||
|
||||
// roctx header file
|
||||
#include <rocprofiler-sdk-roctx/roctx.h>
|
||||
|
||||
#define WIDTH 1024
|
||||
|
||||
#define NUM (WIDTH * WIDTH)
|
||||
|
||||
#define THREADS_PER_BLOCK_X 4
|
||||
#define THREADS_PER_BLOCK_Y 4
|
||||
#define THREADS_PER_BLOCK_Z 1
|
||||
|
||||
#define HIP_API_CALL(CALL) \
|
||||
{ \
|
||||
hipError_t error_ = (CALL); \
|
||||
if(error_ != hipSuccess) \
|
||||
{ \
|
||||
auto _hip_api_print_lk = auto_lock_t{print_lock}; \
|
||||
fprintf(stderr, \
|
||||
"%s:%d :: HIP error : %s\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
hipGetErrorString(error_)); \
|
||||
throw std::runtime_error("hip_api_call"); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using auto_lock_t = std::unique_lock<std::mutex>;
|
||||
auto print_lock = std::mutex{};
|
||||
} // namespace
|
||||
|
||||
// Device (Kernel) function, it must be void
|
||||
__global__ void
|
||||
matrixTranspose(float* out, float* in, const int width)
|
||||
{
|
||||
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
int y = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
|
||||
out[y * width + x] = in[x * width + y];
|
||||
}
|
||||
|
||||
// CPU implementation of matrix transpose
|
||||
void
|
||||
matrixTransposeCPUReference(float* output, float* input, const unsigned int width)
|
||||
{
|
||||
for(unsigned int j = 0; j < width; j++)
|
||||
{
|
||||
for(unsigned int i = 0; i < width; i++)
|
||||
{
|
||||
output[i * width + j] = input[j * width + i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int
|
||||
main()
|
||||
{
|
||||
roctxRangePush("main");
|
||||
|
||||
float* Matrix;
|
||||
float* TransposeMatrix;
|
||||
float* cpuTransposeMatrix;
|
||||
|
||||
float* gpuMatrix;
|
||||
float* gpuTransposeMatrix;
|
||||
|
||||
hipDeviceProp_t devProp;
|
||||
HIP_API_CALL(hipGetDeviceProperties(&devProp, 0));
|
||||
|
||||
std::cout << "Device name " << devProp.name << std::endl;
|
||||
|
||||
int i;
|
||||
int errors;
|
||||
|
||||
Matrix = (float*) malloc(NUM * sizeof(float));
|
||||
TransposeMatrix = (float*) malloc(NUM * sizeof(float));
|
||||
cpuTransposeMatrix = (float*) malloc(NUM * sizeof(float));
|
||||
|
||||
// initialize the input data
|
||||
for(i = 0; i < NUM; i++)
|
||||
{
|
||||
Matrix[i] = (float) i * 10.0f;
|
||||
}
|
||||
|
||||
// allocate the memory on the device side
|
||||
HIP_API_CALL(hipMalloc((void**) &gpuMatrix, NUM * sizeof(float)));
|
||||
HIP_API_CALL(hipMalloc((void**) &gpuTransposeMatrix, NUM * sizeof(float)));
|
||||
|
||||
// Memory transfer from host to device
|
||||
HIP_API_CALL(hipMemcpy(gpuMatrix, Matrix, NUM * sizeof(float), hipMemcpyHostToDevice));
|
||||
|
||||
auto tid = roctx_thread_id_t{};
|
||||
roctxGetThreadId(&tid);
|
||||
roctxProfilerPause(tid);
|
||||
// Memory transfer that should be hidden by profiling tool
|
||||
HIP_API_CALL(
|
||||
hipMemcpy(gpuTransposeMatrix, gpuMatrix, NUM * sizeof(float), hipMemcpyDeviceToDevice));
|
||||
roctxProfilerResume(tid);
|
||||
|
||||
roctxMark("pre-kernel-launch");
|
||||
// Lauching kernel from host
|
||||
hipLaunchKernelGGL(matrixTranspose,
|
||||
dim3(WIDTH / THREADS_PER_BLOCK_X, WIDTH / THREADS_PER_BLOCK_Y),
|
||||
dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
|
||||
0,
|
||||
0,
|
||||
gpuTransposeMatrix,
|
||||
gpuMatrix,
|
||||
WIDTH);
|
||||
roctxMark("post-kernel-launch");
|
||||
|
||||
// Memory transfer from device to host
|
||||
HIP_API_CALL(
|
||||
hipMemcpy(TransposeMatrix, gpuTransposeMatrix, NUM * sizeof(float), hipMemcpyDeviceToHost));
|
||||
|
||||
// CPU MatrixTranspose computation
|
||||
matrixTransposeCPUReference(cpuTransposeMatrix, Matrix, WIDTH);
|
||||
|
||||
// verify the results
|
||||
errors = 0;
|
||||
double eps = 1.0E-6;
|
||||
for(i = 0; i < NUM; i++)
|
||||
{
|
||||
if(std::abs(TransposeMatrix[i] - cpuTransposeMatrix[i]) > eps)
|
||||
{
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
if(errors != 0)
|
||||
{
|
||||
printf("FAILED: %d errors\n", errors);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("PASSED!\n");
|
||||
}
|
||||
|
||||
// free the resources on device side
|
||||
HIP_API_CALL(hipFree(gpuMatrix));
|
||||
HIP_API_CALL(hipFree(gpuTransposeMatrix));
|
||||
|
||||
// free the resources on host side
|
||||
free(Matrix);
|
||||
free(TransposeMatrix);
|
||||
free(cpuTransposeMatrix);
|
||||
|
||||
roctxRangePop();
|
||||
|
||||
return errors;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tests-bin-transpose LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
option(TRANSPOSE_USE_MPI "Enable MPI support in transpose exe" OFF)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(transpose.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(transpose)
|
||||
target_sources(transpose PRIVATE transpose.cpp)
|
||||
target_compile_options(transpose PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(transpose PRIVATE Threads::Threads)
|
||||
|
||||
find_package(rocprofiler-sdk-roctx REQUIRED)
|
||||
target_link_libraries(transpose PRIVATE rocprofiler-sdk-roctx::rocprofiler-sdk-roctx)
|
||||
|
||||
if(TRANSPOSE_USE_MPI)
|
||||
find_package(MPI REQUIRED)
|
||||
target_compile_definitions(transpose PRIVATE USE_MPI)
|
||||
target_link_libraries(transpose PRIVATE MPI::MPI_C)
|
||||
endif()
|
||||
|
||||
install(
|
||||
TARGETS transpose
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,284 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
#include "hip/hip_runtime.h"
|
||||
#include "rocprofiler-sdk-roctx/roctx.h"
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
|
||||
#if defined(USE_MPI)
|
||||
# include <mpi.h>
|
||||
#endif
|
||||
|
||||
#define HIP_API_CALL(CALL) \
|
||||
{ \
|
||||
hipError_t error_ = (CALL); \
|
||||
if(error_ != hipSuccess) \
|
||||
{ \
|
||||
auto _hip_api_print_lk = auto_lock_t{print_lock}; \
|
||||
fprintf(stderr, \
|
||||
"%s:%d :: HIP error : %s\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
hipGetErrorString(error_)); \
|
||||
throw std::runtime_error("hip_api_call"); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using auto_lock_t = std::unique_lock<std::mutex>;
|
||||
auto print_lock = std::mutex{};
|
||||
size_t nthreads = 2;
|
||||
size_t nitr = 500;
|
||||
size_t nsync = 10;
|
||||
constexpr unsigned shared_mem_tile_dim = 32;
|
||||
|
||||
void
|
||||
check_hip_error(void);
|
||||
|
||||
void
|
||||
verify(int* in, int* out, int M, int N);
|
||||
} // namespace
|
||||
|
||||
__global__ void
|
||||
transpose(const int* in, int* out, int M, int N);
|
||||
|
||||
void
|
||||
run(int rank, int tid, int ndevice, int argc, char** argv);
|
||||
|
||||
int
|
||||
main(int argc, char** argv)
|
||||
{
|
||||
int rank = 0;
|
||||
int size = 1;
|
||||
for(int i = 1; i < argc; ++i)
|
||||
{
|
||||
auto _arg = std::string{argv[i]};
|
||||
if(_arg == "?" || _arg == "-h" || _arg == "--help")
|
||||
{
|
||||
fprintf(stderr,
|
||||
"usage: transpose [NUM_THREADS (%zu)] [NUM_ITERATION (%zu)] "
|
||||
"[SYNC_EVERY_N_ITERATIONS (%zu)]\n",
|
||||
nthreads,
|
||||
nitr,
|
||||
nsync);
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
}
|
||||
if(argc > 1) nthreads = atoll(argv[1]);
|
||||
if(argc > 2) nitr = atoll(argv[2]);
|
||||
if(argc > 3) nsync = atoll(argv[3]);
|
||||
|
||||
printf("[transpose] Number of threads: %zu\n", nthreads);
|
||||
printf("[transpose] Number of iterations: %zu\n", nitr);
|
||||
printf("[transpose] Syncing every %zu iterations\n", nsync);
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Init(&argc, &argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &size);
|
||||
#else
|
||||
(void) size;
|
||||
#endif
|
||||
// this is a temporary workaround in omnitrace when HIP + MPI is enabled
|
||||
int ndevice = 0;
|
||||
HIP_API_CALL(hipGetDeviceCount(&ndevice));
|
||||
printf("[transpose] Number of devices found: %i\n", ndevice);
|
||||
auto devids = std::vector<int>{};
|
||||
devids.resize(size * nthreads, 0);
|
||||
int devid = 0;
|
||||
for(size_t i = 0; i < nthreads; ++i)
|
||||
{
|
||||
for(int j = 0; j < size; ++j)
|
||||
{
|
||||
auto idx = (j * nthreads) + i;
|
||||
devids.at(idx) = devid++ % ndevice;
|
||||
}
|
||||
}
|
||||
auto devid_offset = (rank * nthreads);
|
||||
auto _threads = std::vector<std::thread>{};
|
||||
for(size_t i = 1; i < nthreads; ++i)
|
||||
_threads.emplace_back(run, rank, i, devids.at(devid_offset + i), argc, argv);
|
||||
run(rank, 0, devids.at(devid_offset + 0), argc, argv);
|
||||
for(auto& itr : _threads)
|
||||
itr.join();
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
for(int i = 0; i < ndevice; ++i)
|
||||
{
|
||||
HIP_API_CALL(hipSetDevice(i));
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
}
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
if(rank == 0)
|
||||
{
|
||||
for(int i = 0; i < ndevice; ++i)
|
||||
{
|
||||
HIP_API_CALL(hipSetDevice(i));
|
||||
HIP_API_CALL(hipDeviceReset());
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(USE_MPI)
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
__global__ void
|
||||
transpose(const int* in, int* out, int M, int N)
|
||||
{
|
||||
__shared__ int tile[shared_mem_tile_dim][shared_mem_tile_dim];
|
||||
|
||||
int idx = (blockIdx.y * blockDim.y + threadIdx.y) * M + blockIdx.x * blockDim.x + threadIdx.x;
|
||||
tile[threadIdx.y][threadIdx.x] = in[idx];
|
||||
__syncthreads();
|
||||
idx = (blockIdx.x * blockDim.x + threadIdx.y) * N + blockIdx.y * blockDim.y + threadIdx.x;
|
||||
out[idx] = tile[threadIdx.x][threadIdx.y];
|
||||
}
|
||||
|
||||
void
|
||||
run(int rank, int tid, int devid, int argc, char** argv)
|
||||
{
|
||||
roctxRangePush("run");
|
||||
|
||||
constexpr unsigned int M = 4960 * 2;
|
||||
constexpr unsigned int N = 4960 * 2;
|
||||
|
||||
if(argc > 2) nitr = atoll(argv[2]);
|
||||
if(argc > 3) nsync = atoll(argv[3]);
|
||||
|
||||
hipStream_t stream = {};
|
||||
|
||||
printf("[transpose] Rank %i, thread %i assigned to device %i\n", rank, tid, devid);
|
||||
HIP_API_CALL(hipSetDevice(devid));
|
||||
HIP_API_CALL(hipStreamCreate(&stream));
|
||||
|
||||
auto_lock_t _lk{print_lock};
|
||||
std::cout << "[transpose][" << rank << "][" << tid << "] M: " << M << " N: " << N << std::endl;
|
||||
_lk.unlock();
|
||||
|
||||
std::default_random_engine _engine{std::random_device{}() * (rank + 1) * (tid + 1)};
|
||||
std::uniform_int_distribution<int> _dist{0, 1000};
|
||||
|
||||
size_t size = sizeof(int) * M * N;
|
||||
int* inp_matrix = new int[size];
|
||||
int* out_matrix = new int[size];
|
||||
for(size_t i = 0; i < M * N; i++)
|
||||
{
|
||||
inp_matrix[i] = _dist(_engine);
|
||||
out_matrix[i] = 0;
|
||||
}
|
||||
int* in = nullptr;
|
||||
int* out = nullptr;
|
||||
|
||||
HIP_API_CALL(hipMalloc(&in, size));
|
||||
HIP_API_CALL(hipMalloc(&out, size));
|
||||
HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
|
||||
HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
|
||||
HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
|
||||
HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
|
||||
dim3 grid(M / 32, N / 32, 1);
|
||||
dim3 block(32, 32, 1); // transpose
|
||||
|
||||
auto t1 = std::chrono::high_resolution_clock::now();
|
||||
for(size_t i = 0; i < nitr; ++i)
|
||||
{
|
||||
transpose<<<grid, block, 0, stream>>>(in, out, M, N);
|
||||
check_hip_error();
|
||||
if(i % nsync == (nsync - 1)) HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
}
|
||||
auto t2 = std::chrono::high_resolution_clock::now();
|
||||
HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
HIP_API_CALL(hipMemcpyAsync(out_matrix, out, size, hipMemcpyDeviceToHost, stream));
|
||||
double time = std::chrono::duration_cast<std::chrono::duration<double>>(t2 - t1).count();
|
||||
float GB = (float) size * nitr * 2 / (1 << 30);
|
||||
|
||||
print_lock.lock();
|
||||
std::cout << "[transpose][" << rank << "][" << tid << "] Runtime of transpose is " << time
|
||||
<< " sec\n";
|
||||
std::cout << "[transpose][" << rank << "][" << tid
|
||||
<< "] The average performance of transpose is " << GB / time << " GBytes/sec"
|
||||
<< std::endl;
|
||||
print_lock.unlock();
|
||||
|
||||
HIP_API_CALL(hipStreamSynchronize(stream));
|
||||
HIP_API_CALL(hipStreamDestroy(stream));
|
||||
|
||||
// cpu_transpose(matrix, out_matrix, M, N);
|
||||
verify(inp_matrix, out_matrix, M, N);
|
||||
|
||||
HIP_API_CALL(hipFree(in));
|
||||
HIP_API_CALL(hipFree(out));
|
||||
|
||||
delete[] inp_matrix;
|
||||
delete[] out_matrix;
|
||||
|
||||
roctxRangePop();
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
void
|
||||
check_hip_error(void)
|
||||
{
|
||||
hipError_t err = hipGetLastError();
|
||||
if(err != hipSuccess)
|
||||
{
|
||||
auto_lock_t _lk{print_lock};
|
||||
std::cerr << "Error: " << hipGetErrorString(err) << std::endl;
|
||||
throw std::runtime_error("hip_api_call");
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
verify(int* in, int* out, int M, int N)
|
||||
{
|
||||
for(int i = 0; i < 10; i++)
|
||||
{
|
||||
int row = rand() % M;
|
||||
int col = rand() % N;
|
||||
if(in[row * N + col] != out[col * M + row])
|
||||
{
|
||||
auto_lock_t _lk{print_lock};
|
||||
std::cout << "mismatch: " << row << ", " << col << " : " << in[row * N + col] << " | "
|
||||
<< out[col * M + row] << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
@@ -0,0 +1,46 @@
|
||||
#
|
||||
#
|
||||
#
|
||||
cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
|
||||
|
||||
if(NOT CMAKE_HIP_COMPILER)
|
||||
find_program(
|
||||
amdclangpp_EXECUTABLE
|
||||
NAMES amdclang++
|
||||
HINTS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATHS ${ROCM_PATH} ENV ROCM_PATH /opt/rocm
|
||||
PATH_SUFFIXES bin llvm/bin NO_CACHE)
|
||||
mark_as_advanced(amdclangpp_EXECUTABLE)
|
||||
|
||||
if(amdclangpp_EXECUTABLE)
|
||||
set(CMAKE_HIP_COMPILER "${amdclangpp_EXECUTABLE}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
project(rocprofiler-tests-bin-vector-operations LANGUAGES CXX HIP)
|
||||
|
||||
foreach(_TYPE DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
|
||||
if("${CMAKE_HIP_FLAGS_${_TYPE}}" STREQUAL "")
|
||||
set(CMAKE_HIP_FLAGS_${_TYPE} "${CMAKE_CXX_FLAGS_${_TYPE}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_EXTENSIONS OFF)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_HIP_STANDARD 17)
|
||||
set(CMAKE_HIP_EXTENSIONS OFF)
|
||||
set(CMAKE_HIP_STANDARD_REQUIRED ON)
|
||||
|
||||
set_source_files_properties(vector-ops.cpp PROPERTIES LANGUAGE HIP)
|
||||
add_executable(vector-ops)
|
||||
target_sources(vector-ops PRIVATE vector-ops.cpp)
|
||||
target_compile_options(vector-ops PRIVATE -W -Wall -Wextra -Wpedantic -Wshadow -Werror)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(vector-ops PRIVATE Threads::Threads)
|
||||
|
||||
install(
|
||||
TARGETS vector-ops
|
||||
DESTINATION bin
|
||||
COMPONENT tests)
|
||||
@@ -0,0 +1,280 @@
|
||||
// MIT License
|
||||
//
|
||||
// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
#include <assert.h>
|
||||
#include <hip/hip_runtime.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <vector>
|
||||
|
||||
#define HIP_API_CALL(CALL) \
|
||||
{ \
|
||||
hipError_t error_ = (CALL); \
|
||||
if(error_ != hipSuccess) \
|
||||
{ \
|
||||
auto _hip_api_print_lk = auto_lock_t{print_lock}; \
|
||||
fprintf(stderr, \
|
||||
"%s:%d :: HIP error : %s\n", \
|
||||
__FILE__, \
|
||||
__LINE__, \
|
||||
hipGetErrorString(error_)); \
|
||||
throw std::runtime_error("hip_api_call"); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
using auto_lock_t = std::unique_lock<std::mutex>;
|
||||
auto print_lock = std::mutex{};
|
||||
} // namespace
|
||||
|
||||
#define WIDTH (1024)
|
||||
#define HEIGHT (1024)
|
||||
|
||||
#define NUM (WIDTH * HEIGHT)
|
||||
|
||||
#define THREADS_PER_BLOCK_X 64
|
||||
#define THREADS_PER_BLOCK_Y 1
|
||||
#define THREADS_PER_BLOCK_Z 1
|
||||
|
||||
// Computes vectorAdd with matrix-multiply
|
||||
template <typename T>
|
||||
__global__ void
|
||||
addition_kernel(T* __restrict__ a,
|
||||
const float* __restrict__ b,
|
||||
const float* __restrict__ c,
|
||||
int width,
|
||||
[[maybe_unused]] int height)
|
||||
{
|
||||
// printf("addition kernel\n");
|
||||
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
int y = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
|
||||
if(x >= WIDTH || y >= HEIGHT) return;
|
||||
int index = y * width + x;
|
||||
|
||||
a[index] = b[index] + c[index];
|
||||
}
|
||||
|
||||
__global__ void
|
||||
subtract_kernel(float* __restrict__ a,
|
||||
const float* __restrict__ b,
|
||||
const float* __restrict__ c,
|
||||
int width,
|
||||
[[maybe_unused]] int height)
|
||||
{
|
||||
// printf("subtract kernel\n");
|
||||
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
int y = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
|
||||
if(x >= WIDTH || y >= HEIGHT) return;
|
||||
int index = y * width + x;
|
||||
|
||||
a[index] = abs(b[index] - c[index]);
|
||||
}
|
||||
|
||||
__global__ void
|
||||
multiply_kernel(float* __restrict__ a,
|
||||
const float* __restrict__ b,
|
||||
const float* __restrict__ c,
|
||||
int width,
|
||||
[[maybe_unused]] int height)
|
||||
{
|
||||
// printf("multiply kernel\n");
|
||||
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
int y = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
|
||||
if(x >= WIDTH || y >= HEIGHT) return;
|
||||
int index = y * width + x;
|
||||
|
||||
a[index] = (b[index] - 1) * (c[index] - 1) + 1;
|
||||
}
|
||||
|
||||
__global__ void
|
||||
divide_kernel(float* __restrict__ a,
|
||||
const float* __restrict__ b,
|
||||
const float* __restrict__ c,
|
||||
int width,
|
||||
[[maybe_unused]] int height)
|
||||
{
|
||||
// printf("divide kernel\n");
|
||||
int x = blockDim.x * blockIdx.x + threadIdx.x;
|
||||
int y = blockDim.y * blockIdx.y + threadIdx.y;
|
||||
|
||||
if(x >= WIDTH || y >= HEIGHT) return;
|
||||
int index = y * width + x;
|
||||
|
||||
a[index] = (b[index] - c[index]) / abs(c[index] + b[index]) + 1;
|
||||
}
|
||||
|
||||
using namespace std;
|
||||
|
||||
void
|
||||
run(int NUM_QUEUE, int DEVICE_ID)
|
||||
{
|
||||
HIP_API_CALL(hipSetDevice(DEVICE_ID));
|
||||
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
|
||||
std::vector<float*> hostA(NUM_QUEUE);
|
||||
std::vector<float*> hostB(NUM_QUEUE);
|
||||
std::vector<float*> hostC(NUM_QUEUE);
|
||||
|
||||
std::vector<float*> deviceA(NUM_QUEUE);
|
||||
std::vector<float*> deviceB(NUM_QUEUE);
|
||||
std::vector<float*> deviceC(NUM_QUEUE);
|
||||
|
||||
std::vector<hipStream_t> streams(NUM_QUEUE);
|
||||
|
||||
auto sync_stream = [NUM_QUEUE, streams](int q) {
|
||||
if(q < 0 || q >= NUM_QUEUE)
|
||||
throw std::runtime_error{std::string{"invalid stream id: "} + std::to_string(q)};
|
||||
|
||||
HIP_API_CALL(hipStreamSynchronize(streams.at(q)));
|
||||
};
|
||||
|
||||
auto sync_streams = [NUM_QUEUE, sync_stream]() {
|
||||
for(int i = 0; i < NUM_QUEUE; ++i)
|
||||
sync_stream(i);
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
};
|
||||
|
||||
for(int q = 0; q < NUM_QUEUE; q++)
|
||||
{
|
||||
HIP_API_CALL(hipStreamCreateWithFlags(&streams[q], hipStreamNonBlocking));
|
||||
|
||||
HIP_API_CALL(hipHostMalloc(&hostA[q], NUM * sizeof(float), 0));
|
||||
HIP_API_CALL(hipHostMalloc(&hostB[q], NUM * sizeof(float), 0));
|
||||
HIP_API_CALL(hipHostMalloc(&hostC[q], NUM * sizeof(float), 0));
|
||||
|
||||
// initialize the input data
|
||||
for(int i = 0; i < NUM; i++)
|
||||
{
|
||||
hostB[q][i] = static_cast<float>(i);
|
||||
hostC[q][i] = static_cast<float>(i * 100.0f);
|
||||
}
|
||||
|
||||
HIP_API_CALL(hipMallocAsync(&deviceA[q], NUM * sizeof(float), streams[q]));
|
||||
HIP_API_CALL(hipMallocAsync(&deviceB[q], NUM * sizeof(float), streams[q]));
|
||||
HIP_API_CALL(hipMallocAsync(&deviceC[q], NUM * sizeof(float), streams[q]));
|
||||
|
||||
HIP_API_CALL(hipMemcpyAsync(
|
||||
deviceB[q], hostB[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
|
||||
HIP_API_CALL(hipMemcpyAsync(
|
||||
deviceC[q], hostC[q], NUM * sizeof(float), hipMemcpyHostToDevice, streams[q]));
|
||||
}
|
||||
|
||||
sync_streams();
|
||||
|
||||
for(int q = 0; q < NUM_QUEUE; q++)
|
||||
{
|
||||
hipLaunchKernelGGL(addition_kernel,
|
||||
dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
|
||||
dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
|
||||
0,
|
||||
streams[q],
|
||||
deviceA[q],
|
||||
deviceB[q],
|
||||
deviceC[q],
|
||||
WIDTH,
|
||||
HEIGHT);
|
||||
|
||||
HIP_API_CALL(hipGetLastError());
|
||||
|
||||
hipLaunchKernelGGL(subtract_kernel,
|
||||
dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
|
||||
dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
|
||||
0,
|
||||
streams[q],
|
||||
deviceA[q],
|
||||
deviceB[q],
|
||||
deviceC[q],
|
||||
WIDTH,
|
||||
HEIGHT);
|
||||
|
||||
HIP_API_CALL(hipGetLastError());
|
||||
|
||||
hipLaunchKernelGGL(multiply_kernel,
|
||||
dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
|
||||
dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
|
||||
0,
|
||||
streams[q],
|
||||
deviceA[q],
|
||||
deviceB[q],
|
||||
deviceC[q],
|
||||
WIDTH,
|
||||
HEIGHT);
|
||||
|
||||
HIP_API_CALL(hipGetLastError());
|
||||
|
||||
hipLaunchKernelGGL(divide_kernel,
|
||||
dim3(WIDTH / THREADS_PER_BLOCK_X, HEIGHT / THREADS_PER_BLOCK_Y),
|
||||
dim3(THREADS_PER_BLOCK_X, THREADS_PER_BLOCK_Y),
|
||||
0,
|
||||
streams[q],
|
||||
deviceB[q],
|
||||
deviceA[q],
|
||||
deviceC[q],
|
||||
WIDTH,
|
||||
HEIGHT);
|
||||
|
||||
HIP_API_CALL(hipGetLastError());
|
||||
}
|
||||
|
||||
sync_streams();
|
||||
|
||||
for(int q = 0; q < NUM_QUEUE; q++)
|
||||
{
|
||||
HIP_API_CALL(hipMemcpyAsync(
|
||||
hostA[q], deviceA[q], NUM * sizeof(float), hipMemcpyDeviceToHost, streams[q]));
|
||||
|
||||
sync_stream(q);
|
||||
|
||||
HIP_API_CALL(hipFree(deviceA[q]));
|
||||
HIP_API_CALL(hipFree(deviceB[q]));
|
||||
HIP_API_CALL(hipFree(deviceC[q]));
|
||||
|
||||
HIP_API_CALL(hipHostFree(hostA[q]));
|
||||
HIP_API_CALL(hipHostFree(hostB[q]));
|
||||
HIP_API_CALL(hipHostFree(hostC[q]));
|
||||
|
||||
HIP_API_CALL(hipStreamDestroy(streams[q]));
|
||||
}
|
||||
|
||||
HIP_API_CALL(hipDeviceSynchronize());
|
||||
}
|
||||
|
||||
int
|
||||
main()
|
||||
{
|
||||
int device_count = 0;
|
||||
HIP_API_CALL(hipGetDeviceCount(&device_count));
|
||||
|
||||
for(int i = 0; i < device_count; ++i)
|
||||
run(4, i);
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user