AI/rocm-systems
2
0 Tükrözés 0
Kód Hibajegyek Egyesítési kérések Actions Packages Projektek Kiadások Wiki Tevékenység

SWDEV-288707 - Merge HIP refactored code to staging

Forráskód böngészése 
These changes move-
- rocclr based implementation inside src/hipamd/src
- platform specific (both nvidia and amd) headers inside
src/hipamd/include/hip

Change-Id: Ia29791a727244952591fe1d813dcef0303b73a9e


[ROCm/hip commit: ce76ec8b88]
This commit is contained in:
Rahul Garg committed 2021-06-07 00:07:24 +00:00
1 szülő 7a932782e2
commit 4cd9afebe9
119 files changed
+3943 -97
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

projects/hip/src/hipamd/CMakeLists.txt
Executable
+286
Fájl megtekintése
@@ -0,0 +1,286 @@
# Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
cmake_minimum_required(VERSION 3.5.1)
include(GNUInstallDirs)
if(ADDRESS_SANITIZER)
set(ASAN_LINKER_FLAGS "-fsanitize=address")
set(ASAN_COMPILER_FLAGS "-fno-omit-frame-pointer -fsanitize=address")
if(NOT CMAKE_COMPILER_IS_GNUCC)
if(BUILD_SHARED_LIBS)
set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -shared-libsan")
else()
set(ASAN_LINKER_FLAGS "${ASAN_LINKER_FLAGS} -static-libsan")
endif()
endif()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${ASAN_COMPILER_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ASAN_COMPILER_FLAGS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_LINKER_FLAGS} -s")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${ASAN_LINKER_FLAGS}")
endif()
option(BUILD_SHARED_LIBS "Build the shared library" ON)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
find_package(ROCclr)
if(BUILD_SHARED_LIBS)
add_library(amdhip64 SHARED)
# Windows doesn't have a strip utility, so CMAKE_STRIP won't be set.
if((CMAKE_BUILD_TYPE STREQUAL "Release") AND NOT ("${CMAKE_STRIP}" STREQUAL ""))
add_custom_command(TARGET amdhip64 POST_BUILD COMMAND ${CMAKE_STRIP} $<TARGET_FILE:amdhip64>)
endif()
else()
add_library(amdhip64 STATIC $<TARGET_OBJECTS:rocclr>)
endif()
set_target_properties(amdhip64 PROPERTIES
CXX_STANDARD 14
CXX_STANDARD_REQUIRED ON
CXX_EXTENSIONS OFF
POSITION_INDEPENDENT_CODE ON
# Workaround for many places in the HIP project
# having hardcoded references to build/lib/libamdhip64.so
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
if(CMAKE_SIZEOF_VOID_P EQUAL 8)
set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip64")
else()
set_target_properties(amdhip64 PROPERTIES OUTPUT_NAME "amdhip32")
endif()
# Disable versioning for Windows
# as currently HIP_LIB_VERSION_STRING and HIP_LIB_VERSION_MAJOR
# are not being populated
if(NOT WIN32)
if(BUILD_SHARED_LIBS)
set_target_properties(amdhip64 PROPERTIES
VERSION ${HIP_LIB_VERSION_STRING}
SOVERSION ${HIP_LIB_VERSION_MAJOR})
endif()
endif()
target_sources(amdhip64 PRIVATE
src/cl_gl.cpp
src/cl_lqdflash_amd.cpp
src/fixme.cpp
src/hip_activity.cpp
src/hip_code_object.cpp
src/hip_context.cpp
src/hip_device_runtime.cpp
src/hip_device.cpp
src/hip_error.cpp
src/hip_event.cpp
src/hip_fatbin.cpp
src/hip_global.cpp
src/hip_graph_internal.cpp
src/hip_graph.cpp
src/hip_hmm.cpp
src/hip_intercept.cpp
src/hip_memory.cpp
src/hip_module.cpp
src/hip_peer.cpp
src/hip_platform.cpp
src/hip_profile.cpp
src/hip_rtc.cpp
src/hip_stream_ops.cpp
src/hip_stream.cpp
src/hip_surface.cpp
src/hip_texture.cpp)
if(WIN32)
target_sources(amdhip64 PRIVATE
src/cl_d3d9.cpp
src/cl_d3d10.cpp
src/cl_d3d11.cpp)
endif()
if(BUILD_SHARED_LIBS)
if(WIN32)
target_sources(amdhip64 PRIVATE amdhip.def)
else()
target_link_libraries(amdhip64 PRIVATE "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/src/hip_hcc.map.in")
set_target_properties(amdhip64 PROPERTIES LINK_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/src/hip_hcc.map.in")
endif()
endif()
target_include_directories(amdhip64
PRIVATE
${PROJECT_SOURCE_DIR}/src/hipamd/include
${PROJECT_SOURCE_DIR}/include
${PROJECT_BINARY_DIR}/include)
target_compile_definitions(amdhip64 PRIVATE __HIP_PLATFORM_AMD__)
target_link_libraries(amdhip64 PRIVATE ${CMAKE_DL_LIBS})
# Additional dependencies for hipRTC
if(WIN32)
target_link_libraries(amdhip64 PRIVATE Dbghelp.lib)
endif()
# Note in static case we cannot link against rocclr.
# If we would, we'd also have to export rocclr and have hipcc pass it to the linker.
if(BUILD_SHARED_LIBS)
target_link_libraries(amdhip64 PRIVATE rocclr)
else()
target_compile_definitions(amdhip64 PRIVATE $<TARGET_PROPERTY:rocclr,COMPILE_DEFINITIONS>)
target_include_directories(amdhip64 PRIVATE $<TARGET_PROPERTY:rocclr,INCLUDE_DIRECTORIES>)
endif()
# Short-Term solution for pre-compiled headers for online compilation
# Enable pre compiled header
if(__HIP_ENABLE_PCH)
find_package(LLVM REQUIRED CONFIG
PATHS
/opt/rocm/llvm)
# find_package(LLVM) returns the lib/cmake/llvm location. We require the root.
set(HIP_LLVM_ROOT "${LLVM_DIR}/../../..")
# execute_process(COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${HIP_LLVM_ROOT}" COMMAND_ECHO STDERR RESULT_VARIABLE EMBED_PCH_RC)
execute_process(COMMAND sh -c "${CMAKE_CURRENT_SOURCE_DIR}/../../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src/hipamd/include ${HIP_LLVM_ROOT}" COMMAND_ECHO STDERR RESULT_VARIABLE EMBED_PCH_RC)
if (EMBED_PCH_RC AND NOT EMBED_PCH_RC EQUAL 0)
message(FATAL_ERROR "Failed to embed PCH")
endif()
target_compile_definitions(amdhip64 PRIVATE __HIP_ENABLE_PCH)
target_sources(amdhip64 PRIVATE ${CMAKE_BINARY_DIR}/hip_pch.o)
endif()
# Enable preprocessed hiprtc-builtins library
if(__HIP_ENABLE_RTC)
find_package(LLVM REQUIRED CONFIG
PATHS
/opt/rocm/llvm)
# find_package(LLVM) returns the lib/cmake/llvm location. We require the root.
set(HIP_LLVM_ROOT "${LLVM_DIR}/../../..")
if(WIN32)
set(HIPRTC_LIB_NAME "hiprtc-builtins64_${HIP_LIB_VERSION_MAJOR}${HIP_LIB_VERSION_MINOR}.dll")
else()
set(HIPRTC_LIB_NAME "libhiprtc-builtins.so.${HIP_LIB_VERSION_MAJOR}.${HIP_LIB_VERSION_MINOR}")
endif()
execute_process(
COMMAND sh -c "mkdir -p ${PROJECT_BINARY_DIR}/lib; ${CMAKE_CURRENT_SOURCE_DIR}/../../bin/hip_embed_pch.sh ${PROJECT_BINARY_DIR}/include ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src/hipamd/include ${HIP_LLVM_ROOT} -r ${PROJECT_BINARY_DIR}/lib/${HIPRTC_LIB_NAME}"
COMMAND_ECHO STDERR
RESULT_VARIABLE EMBED_RTC_RC
)
if (EMBED_RTC_RC AND NOT EMBED_RTC_RC EQUAL 0)
message(FATAL_ERROR "Failed to create hiprtc shared lib")
endif()
install(FILES ${PROJECT_BINARY_DIR}/lib/${HIPRTC_LIB_NAME} DESTINATION lib)
endif()
#############################
# Profiling API support
#############################
# Generate profiling API macros/structures header
set(PROF_API_STR "${PROJECT_BINARY_DIR}/include/hip/amd_detail/hip_prof_str.h")
set(PROF_API_HDR "${PROJECT_SOURCE_DIR}/include/hip/hip_runtime_api.h")
set(PROF_API_SRC "${CMAKE_CURRENT_SOURCE_DIR}/src")
set(PROF_API_GEN "${CMAKE_CURRENT_SOURCE_DIR}/src/hip_prof_gen.py")
set(PROF_API_LOG "${PROJECT_BINARY_DIR}/hip_prof_gen.log.txt")
find_package(PythonInterp REQUIRED)
add_custom_command(OUTPUT ${PROF_API_STR}
COMMAND ${PYTHON_EXECUTABLE} ${PROF_API_GEN} -v -t --priv ${OPT_PROF_API} ${PROF_API_HDR} ${PROF_API_SRC} ${PROF_API_STR}
OUTPUT_FILE ${PROF_API_LOG}
DEPENDS ${PROF_API_HDR} ${PROF_API_GEN}
COMMENT "Generating profiling primitives: ${PROF_API_STR}")
add_custom_target(gen-prof-api-str-header ALL
DEPENDS ${PROF_API_STR}
SOURCES ${PROF_API_HDR})
set_target_properties(amdhip64 PROPERTIES PUBLIC_HEADER ${PROF_API_STR})
option(USE_PROF_API ON "Enable roctracer integration")
# Enable profiling API
if(USE_PROF_API)
find_path(PROF_API_HEADER_DIR prof_protocol.h
HINTS
${PROF_API_HEADER_PATH}
PATHS
${ROCM_PATH}/roctracer
PATH_SUFFIXES
include/ext)
if(NOT PROF_API_HEADER_DIR)
message(WARNING "Profiling API header not found. Disabling roctracer integration. Use -DPROF_API_HEADER_PATH=<path to prof_protocol.h header>")
else()
target_compile_definitions(amdhip64 PUBLIC USE_PROF_API=1)
target_include_directories(amdhip64 PUBLIC ${PROF_API_HEADER_DIR})
message(STATUS "Profiling API: ${PROF_API_HEADER_DIR}")
endif()
endif()
add_dependencies(amdhip64 gen-prof-api-str-header)
add_custom_command(TARGET amdhip64 POST_BUILD COMMAND
${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/.hipInfo ${PROJECT_BINARY_DIR}/lib/.hipInfo)
add_custom_command(TARGET amdhip64 POST_BUILD COMMAND
${CMAKE_COMMAND} -E copy_directory ${PROJECT_SOURCE_DIR}/include ${PROJECT_BINARY_DIR}/include)
add_library(host INTERFACE)
target_link_libraries(host INTERFACE amdhip64)
add_library(device INTERFACE)
target_link_libraries(device INTERFACE host)
INSTALL(TARGETS amdhip64 host device
EXPORT hip-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
INSTALL(EXPORT hip-targets DESTINATION ${CONFIG_PACKAGE_INSTALL_DIR} NAMESPACE hip::)
INSTALL(TARGETS amdhip64 host device
EXPORT hip-lang-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
INSTALL(EXPORT hip-lang-targets DESTINATION ${CONFIG_LANG_PACKAGE_INSTALL_DIR} NAMESPACE hip-lang::)
if(NOT WIN32)
include(CMakePackageConfigHelpers)
configure_package_config_file(
${PROJECT_SOURCE_DIR}/hip-lang-config.cmake.in
${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake
INSTALL_DESTINATION ${CONFIG_LANG_PACKAGE_INSTALL_DIR}
PATH_VARS LIB_INSTALL_DIR INCLUDE_INSTALL_DIR BIN_INSTALL_DIR)
write_basic_package_version_file(
${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake
VERSION "${HIP_VERSION_MAJOR}.${HIP_VERSION_MINOR}.${HIP_VERSION_GITDATE}"
COMPATIBILITY SameMajorVersion)
install(
FILES
${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config.cmake
${CMAKE_CURRENT_BINARY_DIR}/hip-lang-config-version.cmake
DESTINATION
${CONFIG_LANG_PACKAGE_INSTALL_DIR}/
)
endif()
projects/hip/src/hipamd/cmake/FindROCclr.cmake
+51
Fájl megtekintése
@@ -0,0 +1,51 @@
# Copyright (c) 2020-2021 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
if(ROCCLR_FOUND)
return()
endif()
find_path(ROCCLR_INCLUDE_DIR top.hpp
HINTS
${ROCCLR_PATH}
PATHS
# gerrit repo name
${CMAKE_SOURCE_DIR}/vdi
${CMAKE_SOURCE_DIR}/../vdi
${CMAKE_SOURCE_DIR}/../../vdi
# github repo name
${CMAKE_SOURCE_DIR}/ROCclr
${CMAKE_SOURCE_DIR}/../ROCclr
${CMAKE_SOURCE_DIR}/../../ROCclr
# jenkins repo name
${CMAKE_SOURCE_DIR}/rocclr
${CMAKE_SOURCE_DIR}/../rocclr
${CMAKE_SOURCE_DIR}/../../rocclr
PATH_SUFFIXES
include)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(ROCclr
"\nROCclr not found"
ROCCLR_INCLUDE_DIR)
mark_as_advanced(ROCCLR_INCLUDE_DIR)
list(APPEND CMAKE_MODULE_PATH "${ROCCLR_INCLUDE_DIR}/../cmake")
include(ROCclr)
projects/hip/src/hipamd/include/hip/amd_detail/amd_channel_descriptor.h
+348
Fájl megtekintése
@@ -0,0 +1,348 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_CHANNEL_DESCRIPTOR_H
#include <hip/hip_common.h>
#include <hip/amd_detail/amd_driver_types.h>
#include <hip/amd_detail/amd_hip_vector_types.h>
#ifdef __cplusplus
extern "C" HIP_PUBLIC_API
hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w, hipChannelFormatKind f);
static inline hipChannelFormatDesc hipCreateChannelDescHalf() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
}
static inline hipChannelFormatDesc hipCreateChannelDescHalf1() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
}
static inline hipChannelFormatDesc hipCreateChannelDescHalf2() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
}
template <typename T>
static inline hipChannelFormatDesc hipCreateChannelDesc() {
return hipCreateChannelDesc(0, 0, 0, 0, hipChannelFormatKindNone);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<char>() {
int e = (int)sizeof(char) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<signed char>() {
int e = (int)sizeof(signed char) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<unsigned char>() {
int e = (int)sizeof(unsigned char) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uchar1>() {
int e = (int)sizeof(unsigned char) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<char1>() {
int e = (int)sizeof(signed char) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uchar2>() {
int e = (int)sizeof(unsigned char) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<char2>() {
int e = (int)sizeof(signed char) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
}
#ifndef __GNUC__ // vector3 is the same as vector4
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uchar3>() {
int e = (int)sizeof(unsigned char) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<char3>() {
int e = (int)sizeof(signed char) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
}
#endif
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uchar4>() {
int e = (int)sizeof(unsigned char) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<char4>() {
int e = (int)sizeof(signed char) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<unsigned short>() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<signed short>() {
int e = (int)sizeof(signed short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ushort1>() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<short1>() {
int e = (int)sizeof(signed short) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ushort2>() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<short2>() {
int e = (int)sizeof(signed short) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
}
#ifndef __GNUC__
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ushort3>() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<short3>() {
int e = (int)sizeof(signed short) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
}
#endif
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ushort4>() {
int e = (int)sizeof(unsigned short) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<short4>() {
int e = (int)sizeof(signed short) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<unsigned int>() {
int e = (int)sizeof(unsigned int) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<signed int>() {
int e = (int)sizeof(signed int) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uint1>() {
int e = (int)sizeof(unsigned int) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<int1>() {
int e = (int)sizeof(signed int) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uint2>() {
int e = (int)sizeof(unsigned int) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<int2>() {
int e = (int)sizeof(signed int) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
}
#ifndef __GNUC__
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uint3>() {
int e = (int)sizeof(unsigned int) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<int3>() {
int e = (int)sizeof(signed int) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
}
#endif
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<uint4>() {
int e = (int)sizeof(unsigned int) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<int4>() {
int e = (int)sizeof(signed int) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<float>() {
int e = (int)sizeof(float) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<float1>() {
int e = (int)sizeof(float) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindFloat);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<float2>() {
int e = (int)sizeof(float) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindFloat);
}
#ifndef __GNUC__
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<float3>() {
int e = (int)sizeof(float) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindFloat);
}
#endif
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<float4>() {
int e = (int)sizeof(float) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindFloat);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<unsigned long>() {
int e = (int)sizeof(unsigned long) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<signed long>() {
int e = (int)sizeof(signed long) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ulong1>() {
int e = (int)sizeof(unsigned long) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<long1>() {
int e = (int)sizeof(signed long) * 8;
return hipCreateChannelDesc(e, 0, 0, 0, hipChannelFormatKindSigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ulong2>() {
int e = (int)sizeof(unsigned long) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<long2>() {
int e = (int)sizeof(signed long) * 8;
return hipCreateChannelDesc(e, e, 0, 0, hipChannelFormatKindSigned);
}
#ifndef __GNUC__
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ulong3>() {
int e = (int)sizeof(unsigned long) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<long3>() {
int e = (int)sizeof(signed long) * 8;
return hipCreateChannelDesc(e, e, e, 0, hipChannelFormatKindSigned);
}
#endif
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<ulong4>() {
int e = (int)sizeof(unsigned long) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindUnsigned);
}
template <>
inline hipChannelFormatDesc hipCreateChannelDesc<long4>() {
int e = (int)sizeof(signed long) * 8;
return hipCreateChannelDesc(e, e, e, e, hipChannelFormatKindSigned);
}
#else
struct hipChannelFormatDesc hipCreateChannelDesc(int x, int y, int z, int w,
enum hipChannelFormatKind f);
#endif
#endif
projects/hip/src/hipamd/include/hip/amd_detail/amd_device_functions.h
+1369
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/amd_detail/amd_driver_types.h
+478
Fájl megtekintése
@@ -0,0 +1,478 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DRIVER_TYPES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_DRIVER_TYPES_H
// The follow macro should be removed after upstream updation.
// It's defined here for workarround of rocThrust building failure.
#define HIP_INCLUDE_HIP_HCC_DETAIL_DRIVER_TYPES_H
#if !defined(__HIPCC_RTC__)
#ifndef __cplusplus
#include <stdbool.h>
#endif
#endif // !defined(__HIPCC_RTC__)
typedef void* hipDeviceptr_t;
typedef enum hipChannelFormatKind {
hipChannelFormatKindSigned = 0,
hipChannelFormatKindUnsigned = 1,
hipChannelFormatKindFloat = 2,
hipChannelFormatKindNone = 3
}hipChannelFormatKind;
typedef struct hipChannelFormatDesc {
int x;
int y;
int z;
int w;
enum hipChannelFormatKind f;
}hipChannelFormatDesc;
#define HIP_TRSA_OVERRIDE_FORMAT 0x01
#define HIP_TRSF_READ_AS_INTEGER 0x01
#define HIP_TRSF_NORMALIZED_COORDINATES 0x02
#define HIP_TRSF_SRGB 0x10
typedef enum hipArray_Format {
HIP_AD_FORMAT_UNSIGNED_INT8 = 0x01,
HIP_AD_FORMAT_UNSIGNED_INT16 = 0x02,
HIP_AD_FORMAT_UNSIGNED_INT32 = 0x03,
HIP_AD_FORMAT_SIGNED_INT8 = 0x08,
HIP_AD_FORMAT_SIGNED_INT16 = 0x09,
HIP_AD_FORMAT_SIGNED_INT32 = 0x0a,
HIP_AD_FORMAT_HALF = 0x10,
HIP_AD_FORMAT_FLOAT = 0x20
}hipArray_Format;
typedef struct HIP_ARRAY_DESCRIPTOR {
size_t Width;
size_t Height;
enum hipArray_Format Format;
unsigned int NumChannels;
}HIP_ARRAY_DESCRIPTOR;
typedef struct HIP_ARRAY3D_DESCRIPTOR {
size_t Width;
size_t Height;
size_t Depth;
enum hipArray_Format Format;
unsigned int NumChannels;
unsigned int Flags;
}HIP_ARRAY3D_DESCRIPTOR;
typedef struct hipArray {
void* data; // FIXME: generalize this
struct hipChannelFormatDesc desc;
unsigned int type;
unsigned int width;
unsigned int height;
unsigned int depth;
enum hipArray_Format Format;
unsigned int NumChannels;
bool isDrv;
unsigned int textureType;
}hipArray;
#if !defined(__HIPCC_RTC__)
typedef struct hip_Memcpy2D {
size_t srcXInBytes;
size_t srcY;
hipMemoryType srcMemoryType;
const void* srcHost;
hipDeviceptr_t srcDevice;
hipArray* srcArray;
size_t srcPitch;
size_t dstXInBytes;
size_t dstY;
hipMemoryType dstMemoryType;
void* dstHost;
hipDeviceptr_t dstDevice;
hipArray* dstArray;
size_t dstPitch;
size_t WidthInBytes;
size_t Height;
} hip_Memcpy2D;
#endif // !defined(__HIPCC_RTC__)
typedef struct hipArray* hipArray_t;
typedef hipArray_t hiparray;
typedef const struct hipArray* hipArray_const_t;
typedef struct hipMipmappedArray {
void* data;
struct hipChannelFormatDesc desc;
unsigned int type;
unsigned int width;
unsigned int height;
unsigned int depth;
unsigned int min_mipmap_level;
unsigned int max_mipmap_level;
unsigned int flags;
enum hipArray_Format format;
} hipMipmappedArray;
typedef struct hipMipmappedArray* hipMipmappedArray_t;
typedef const struct hipMipmappedArray* hipMipmappedArray_const_t;
/**
* hip resource types
*/
typedef enum hipResourceType {
hipResourceTypeArray = 0x00,
hipResourceTypeMipmappedArray = 0x01,
hipResourceTypeLinear = 0x02,
hipResourceTypePitch2D = 0x03
}hipResourceType;
typedef enum HIPresourcetype_enum {
HIP_RESOURCE_TYPE_ARRAY = 0x00, /**< Array resoure */
HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY = 0x01, /**< Mipmapped array resource */
HIP_RESOURCE_TYPE_LINEAR = 0x02, /**< Linear resource */
HIP_RESOURCE_TYPE_PITCH2D = 0x03 /**< Pitch 2D resource */
} HIPresourcetype;
/**
* hip address modes
*/
typedef enum HIPaddress_mode_enum {
HIP_TR_ADDRESS_MODE_WRAP = 0,
HIP_TR_ADDRESS_MODE_CLAMP = 1,
HIP_TR_ADDRESS_MODE_MIRROR = 2,
HIP_TR_ADDRESS_MODE_BORDER = 3
} HIPaddress_mode;
/**
* hip filter modes
*/
typedef enum HIPfilter_mode_enum {
HIP_TR_FILTER_MODE_POINT = 0,
HIP_TR_FILTER_MODE_LINEAR = 1
} HIPfilter_mode;
/**
* Texture descriptor
*/
typedef struct HIP_TEXTURE_DESC_st {
HIPaddress_mode addressMode[3]; /**< Address modes */
HIPfilter_mode filterMode; /**< Filter mode */
unsigned int flags; /**< Flags */
unsigned int maxAnisotropy; /**< Maximum anisotropy ratio */
HIPfilter_mode mipmapFilterMode; /**< Mipmap filter mode */
float mipmapLevelBias; /**< Mipmap level bias */
float minMipmapLevelClamp; /**< Mipmap minimum level clamp */
float maxMipmapLevelClamp; /**< Mipmap maximum level clamp */
float borderColor[4]; /**< Border Color */
int reserved[12];
} HIP_TEXTURE_DESC;
/**
* hip texture resource view formats
*/
typedef enum hipResourceViewFormat {
hipResViewFormatNone = 0x00,
hipResViewFormatUnsignedChar1 = 0x01,
hipResViewFormatUnsignedChar2 = 0x02,
hipResViewFormatUnsignedChar4 = 0x03,
hipResViewFormatSignedChar1 = 0x04,
hipResViewFormatSignedChar2 = 0x05,
hipResViewFormatSignedChar4 = 0x06,
hipResViewFormatUnsignedShort1 = 0x07,
hipResViewFormatUnsignedShort2 = 0x08,
hipResViewFormatUnsignedShort4 = 0x09,
hipResViewFormatSignedShort1 = 0x0a,
hipResViewFormatSignedShort2 = 0x0b,
hipResViewFormatSignedShort4 = 0x0c,
hipResViewFormatUnsignedInt1 = 0x0d,
hipResViewFormatUnsignedInt2 = 0x0e,
hipResViewFormatUnsignedInt4 = 0x0f,
hipResViewFormatSignedInt1 = 0x10,
hipResViewFormatSignedInt2 = 0x11,
hipResViewFormatSignedInt4 = 0x12,
hipResViewFormatHalf1 = 0x13,
hipResViewFormatHalf2 = 0x14,
hipResViewFormatHalf4 = 0x15,
hipResViewFormatFloat1 = 0x16,
hipResViewFormatFloat2 = 0x17,
hipResViewFormatFloat4 = 0x18,
hipResViewFormatUnsignedBlockCompressed1 = 0x19,
hipResViewFormatUnsignedBlockCompressed2 = 0x1a,
hipResViewFormatUnsignedBlockCompressed3 = 0x1b,
hipResViewFormatUnsignedBlockCompressed4 = 0x1c,
hipResViewFormatSignedBlockCompressed4 = 0x1d,
hipResViewFormatUnsignedBlockCompressed5 = 0x1e,
hipResViewFormatSignedBlockCompressed5 = 0x1f,
hipResViewFormatUnsignedBlockCompressed6H = 0x20,
hipResViewFormatSignedBlockCompressed6H = 0x21,
hipResViewFormatUnsignedBlockCompressed7 = 0x22
}hipResourceViewFormat;
typedef enum HIPresourceViewFormat_enum
{
HIP_RES_VIEW_FORMAT_NONE = 0x00, /**< No resource view format (use underlying resource format) */
HIP_RES_VIEW_FORMAT_UINT_1X8 = 0x01, /**< 1 channel unsigned 8-bit integers */
HIP_RES_VIEW_FORMAT_UINT_2X8 = 0x02, /**< 2 channel unsigned 8-bit integers */
HIP_RES_VIEW_FORMAT_UINT_4X8 = 0x03, /**< 4 channel unsigned 8-bit integers */
HIP_RES_VIEW_FORMAT_SINT_1X8 = 0x04, /**< 1 channel signed 8-bit integers */
HIP_RES_VIEW_FORMAT_SINT_2X8 = 0x05, /**< 2 channel signed 8-bit integers */
HIP_RES_VIEW_FORMAT_SINT_4X8 = 0x06, /**< 4 channel signed 8-bit integers */
HIP_RES_VIEW_FORMAT_UINT_1X16 = 0x07, /**< 1 channel unsigned 16-bit integers */
HIP_RES_VIEW_FORMAT_UINT_2X16 = 0x08, /**< 2 channel unsigned 16-bit integers */
HIP_RES_VIEW_FORMAT_UINT_4X16 = 0x09, /**< 4 channel unsigned 16-bit integers */
HIP_RES_VIEW_FORMAT_SINT_1X16 = 0x0a, /**< 1 channel signed 16-bit integers */
HIP_RES_VIEW_FORMAT_SINT_2X16 = 0x0b, /**< 2 channel signed 16-bit integers */
HIP_RES_VIEW_FORMAT_SINT_4X16 = 0x0c, /**< 4 channel signed 16-bit integers */
HIP_RES_VIEW_FORMAT_UINT_1X32 = 0x0d, /**< 1 channel unsigned 32-bit integers */
HIP_RES_VIEW_FORMAT_UINT_2X32 = 0x0e, /**< 2 channel unsigned 32-bit integers */
HIP_RES_VIEW_FORMAT_UINT_4X32 = 0x0f, /**< 4 channel unsigned 32-bit integers */
HIP_RES_VIEW_FORMAT_SINT_1X32 = 0x10, /**< 1 channel signed 32-bit integers */
HIP_RES_VIEW_FORMAT_SINT_2X32 = 0x11, /**< 2 channel signed 32-bit integers */
HIP_RES_VIEW_FORMAT_SINT_4X32 = 0x12, /**< 4 channel signed 32-bit integers */
HIP_RES_VIEW_FORMAT_FLOAT_1X16 = 0x13, /**< 1 channel 16-bit floating point */
HIP_RES_VIEW_FORMAT_FLOAT_2X16 = 0x14, /**< 2 channel 16-bit floating point */
HIP_RES_VIEW_FORMAT_FLOAT_4X16 = 0x15, /**< 4 channel 16-bit floating point */
HIP_RES_VIEW_FORMAT_FLOAT_1X32 = 0x16, /**< 1 channel 32-bit floating point */
HIP_RES_VIEW_FORMAT_FLOAT_2X32 = 0x17, /**< 2 channel 32-bit floating point */
HIP_RES_VIEW_FORMAT_FLOAT_4X32 = 0x18, /**< 4 channel 32-bit floating point */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC1 = 0x19, /**< Block compressed 1 */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC2 = 0x1a, /**< Block compressed 2 */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC3 = 0x1b, /**< Block compressed 3 */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC4 = 0x1c, /**< Block compressed 4 unsigned */
HIP_RES_VIEW_FORMAT_SIGNED_BC4 = 0x1d, /**< Block compressed 4 signed */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC5 = 0x1e, /**< Block compressed 5 unsigned */
HIP_RES_VIEW_FORMAT_SIGNED_BC5 = 0x1f, /**< Block compressed 5 signed */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC6H = 0x20, /**< Block compressed 6 unsigned half-float */
HIP_RES_VIEW_FORMAT_SIGNED_BC6H = 0x21, /**< Block compressed 6 signed half-float */
HIP_RES_VIEW_FORMAT_UNSIGNED_BC7 = 0x22 /**< Block compressed 7 */
} HIPresourceViewFormat;
/**
* HIP resource descriptor
*/
typedef struct hipResourceDesc {
enum hipResourceType resType;
union {
struct {
hipArray_t array;
} array;
struct {
hipMipmappedArray_t mipmap;
} mipmap;
struct {
void* devPtr;
struct hipChannelFormatDesc desc;
size_t sizeInBytes;
} linear;
struct {
void* devPtr;
struct hipChannelFormatDesc desc;
size_t width;
size_t height;
size_t pitchInBytes;
} pitch2D;
} res;
}hipResourceDesc;
typedef struct HIP_RESOURCE_DESC_st
{
HIPresourcetype resType; /**< Resource type */
union {
struct {
hipArray_t hArray; /**< HIP array */
} array;
struct {
hipMipmappedArray_t hMipmappedArray; /**< HIP mipmapped array */
} mipmap;
struct {
hipDeviceptr_t devPtr; /**< Device pointer */
hipArray_Format format; /**< Array format */
unsigned int numChannels; /**< Channels per array element */
size_t sizeInBytes; /**< Size in bytes */
} linear;
struct {
hipDeviceptr_t devPtr; /**< Device pointer */
hipArray_Format format; /**< Array format */
unsigned int numChannels; /**< Channels per array element */
size_t width; /**< Width of the array in elements */
size_t height; /**< Height of the array in elements */
size_t pitchInBytes; /**< Pitch between two rows in bytes */
} pitch2D;
struct {
int reserved[32];
} reserved;
} res;
unsigned int flags; /**< Flags (must be zero) */
} HIP_RESOURCE_DESC;
/**
* hip resource view descriptor
*/
struct hipResourceViewDesc {
enum hipResourceViewFormat format;
size_t width;
size_t height;
size_t depth;
unsigned int firstMipmapLevel;
unsigned int lastMipmapLevel;
unsigned int firstLayer;
unsigned int lastLayer;
};
/**
* Resource view descriptor
*/
typedef struct HIP_RESOURCE_VIEW_DESC_st
{
HIPresourceViewFormat format; /**< Resource view format */
size_t width; /**< Width of the resource view */
size_t height; /**< Height of the resource view */
size_t depth; /**< Depth of the resource view */
unsigned int firstMipmapLevel; /**< First defined mipmap level */
unsigned int lastMipmapLevel; /**< Last defined mipmap level */
unsigned int firstLayer; /**< First layer index */
unsigned int lastLayer; /**< Last layer index */
unsigned int reserved[16];
} HIP_RESOURCE_VIEW_DESC;
/**
* Memory copy types
*
*/
#if !defined(__HIPCC_RTC__)
typedef enum hipMemcpyKind {
hipMemcpyHostToHost = 0, ///< Host-to-Host Copy
hipMemcpyHostToDevice = 1, ///< Host-to-Device Copy
hipMemcpyDeviceToHost = 2, ///< Device-to-Host Copy
hipMemcpyDeviceToDevice = 3, ///< Device-to-Device Copy
hipMemcpyDefault =
4 ///< Runtime will automatically determine copy-kind based on virtual addresses.
} hipMemcpyKind;
typedef struct hipPitchedPtr {
void* ptr;
size_t pitch;
size_t xsize;
size_t ysize;
}hipPitchedPtr;
typedef struct hipExtent {
size_t width; // Width in elements when referring to array memory, in bytes when referring to
// linear memory
size_t height;
size_t depth;
}hipExtent;
typedef struct hipPos {
size_t x;
size_t y;
size_t z;
}hipPos;
typedef struct hipMemcpy3DParms {
hipArray_t srcArray;
struct hipPos srcPos;
struct hipPitchedPtr srcPtr;
hipArray_t dstArray;
struct hipPos dstPos;
struct hipPitchedPtr dstPtr;
struct hipExtent extent;
enum hipMemcpyKind kind;
} hipMemcpy3DParms;
typedef struct HIP_MEMCPY3D {
unsigned int srcXInBytes;
unsigned int srcY;
unsigned int srcZ;
unsigned int srcLOD;
hipMemoryType srcMemoryType;
const void* srcHost;
hipDeviceptr_t srcDevice;
hipArray_t srcArray;
unsigned int srcPitch;
unsigned int srcHeight;
unsigned int dstXInBytes;
unsigned int dstY;
unsigned int dstZ;
unsigned int dstLOD;
hipMemoryType dstMemoryType;
void* dstHost;
hipDeviceptr_t dstDevice;
hipArray_t dstArray;
unsigned int dstPitch;
unsigned int dstHeight;
unsigned int WidthInBytes;
unsigned int Height;
unsigned int Depth;
} HIP_MEMCPY3D;
static inline struct hipPitchedPtr make_hipPitchedPtr(void* d, size_t p, size_t xsz,
size_t ysz) {
struct hipPitchedPtr s;
s.ptr = d;
s.pitch = p;
s.xsize = xsz;
s.ysize = ysz;
return s;
}
static inline struct hipPos make_hipPos(size_t x, size_t y, size_t z) {
struct hipPos p;
p.x = x;
p.y = y;
p.z = z;
return p;
}
static inline struct hipExtent make_hipExtent(size_t w, size_t h, size_t d) {
struct hipExtent e;
e.width = w;
e.height = h;
e.depth = d;
return e;
}
typedef enum hipFunction_attribute {
HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK,
HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES,
HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES,
HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES,
HIP_FUNC_ATTRIBUTE_NUM_REGS,
HIP_FUNC_ATTRIBUTE_PTX_VERSION,
HIP_FUNC_ATTRIBUTE_BINARY_VERSION,
HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA,
HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES,
HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT,
HIP_FUNC_ATTRIBUTE_MAX
}hipFunction_attribute;
#endif // !defined(__HIPCC_RTC__)
#endif
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_atomic.h
+691
Fájl megtekintése
@@ -0,0 +1,691 @@
#include "amd_device_functions.h"
#if __has_builtin(__hip_atomic_compare_exchange_strong)
#if !__HIP_DEVICE_COMPILE__
//TODO: Remove this after compiler pre-defines the following Macros.
#define __HIP_MEMORY_SCOPE_SINGLETHREAD 1
#define __HIP_MEMORY_SCOPE_WAVEFRONT 2
#define __HIP_MEMORY_SCOPE_WORKGROUP 3
#define __HIP_MEMORY_SCOPE_AGENT 4
#define __HIP_MEMORY_SCOPE_SYSTEM 5
#endif
__device__
inline
int atomicCAS(int* address, int compare, int val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_AGENT);
return compare;
}
__device__
inline
int atomicCAS_system(int* address, int compare, int val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_SYSTEM);
return compare;
}
__device__
inline
unsigned int atomicCAS(unsigned int* address, unsigned int compare, unsigned int val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_AGENT);
return compare;
}
__device__
inline
unsigned int atomicCAS_system(unsigned int* address, unsigned int compare, unsigned int val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_SYSTEM);
return compare;
}
__device__
inline
unsigned long long atomicCAS(unsigned long long* address, unsigned long long compare,
unsigned long long val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_AGENT);
return compare;
}
__device__
inline
unsigned long long atomicCAS_system(unsigned long long* address, unsigned long long compare,
unsigned long long val) {
__hip_atomic_compare_exchange_strong(address, &compare, val, __ATOMIC_RELAXED, __ATOMIC_RELAXED,
__HIP_MEMORY_SCOPE_SYSTEM);
return compare;
}
__device__
inline
int atomicAdd(int* address, int val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicAdd_system(int* address, int val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicAdd(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicAdd_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicAdd(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicAdd_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
float atomicAdd(float* address, float val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
float atomicAdd_system(float* address, float val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
#if !defined(__HIPCC_RTC__)
DEPRECATED("use atomicAdd instead")
#endif // !defined(__HIPCC_RTC__)
__device__
inline
void atomicAddNoRet(float* address, float val)
{
__ockl_atomic_add_noret_f32(address, val);
}
__device__
inline
double atomicAdd(double* address, double val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
double atomicAdd_system(double* address, double val) {
return __hip_atomic_fetch_add(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicSub(int* address, int val) {
return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicSub_system(int* address, int val) {
return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicSub(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicSub_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_add(address, -val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicExch(int* address, int val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicExch_system(int* address, int val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicExch(unsigned int* address, unsigned int val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicExch_system(unsigned int* address, unsigned int val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicExch(unsigned long long* address, unsigned long long val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicExch_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
float atomicExch(float* address, float val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
float atomicExch_system(float* address, float val) {
return __hip_atomic_exchange(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicMin(int* address, int val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicMin_system(int* address, int val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicMin(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicMin_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicMin(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicMin_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_min(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicMax(int* address, int val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicMax_system(int* address, int val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicMax(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicMax_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicMax(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicMax_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_max(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicInc(unsigned int* address, unsigned int val)
{
__device__
extern
unsigned int __builtin_amdgcn_atomic_inc(
unsigned int*,
unsigned int,
unsigned int,
unsigned int,
bool) __asm("llvm.amdgcn.atomic.inc.i32.p0i32");
return __builtin_amdgcn_atomic_inc(
address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false);
}
__device__
inline
unsigned int atomicDec(unsigned int* address, unsigned int val)
{
__device__
extern
unsigned int __builtin_amdgcn_atomic_dec(
unsigned int*,
unsigned int,
unsigned int,
unsigned int,
bool) __asm("llvm.amdgcn.atomic.dec.i32.p0i32");
return __builtin_amdgcn_atomic_dec(
address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false);
}
__device__
inline
int atomicAnd(int* address, int val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicAnd_system(int* address, int val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicAnd(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicAnd_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicAnd(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicAnd_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_and(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicOr(int* address, int val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicOr_system(int* address, int val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicOr(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicOr_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicOr(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicOr_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_or(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
int atomicXor(int* address, int val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
int atomicXor_system(int* address, int val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned int atomicXor(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned int atomicXor_system(unsigned int* address, unsigned int val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
__device__
inline
unsigned long long atomicXor(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_AGENT);
}
__device__
inline
unsigned long long atomicXor_system(unsigned long long* address, unsigned long long val) {
return __hip_atomic_fetch_xor(address, val, __ATOMIC_RELAXED, __HIP_MEMORY_SCOPE_SYSTEM);
}
#else
__device__
inline
int atomicCAS(int* address, int compare, int val)
{
__atomic_compare_exchange_n(
address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
return compare;
}
__device__
inline
unsigned int atomicCAS(
unsigned int* address, unsigned int compare, unsigned int val)
{
__atomic_compare_exchange_n(
address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
return compare;
}
__device__
inline
unsigned long long atomicCAS(
unsigned long long* address,
unsigned long long compare,
unsigned long long val)
{
__atomic_compare_exchange_n(
address, &compare, val, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
return compare;
}
__device__
inline
int atomicAdd(int* address, int val)
{
return __atomic_fetch_add(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicAdd(unsigned int* address, unsigned int val)
{
return __atomic_fetch_add(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicAdd(
unsigned long long* address, unsigned long long val)
{
return __atomic_fetch_add(address, val, __ATOMIC_RELAXED);
}
__device__
inline
float atomicAdd(float* address, float val)
{
return __atomic_fetch_add(address, val, __ATOMIC_RELAXED);
}
#if !defined(__HIPCC_RTC__)
DEPRECATED("use atomicAdd instead")
#endif // !defined(__HIPCC_RTC__)
__device__
inline
void atomicAddNoRet(float* address, float val)
{
__ockl_atomic_add_noret_f32(address, val);
}
__device__
inline
double atomicAdd(double* address, double val)
{
return __atomic_fetch_add(address, val, __ATOMIC_RELAXED);
}
__device__
inline
int atomicSub(int* address, int val)
{
return __atomic_fetch_sub(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicSub(unsigned int* address, unsigned int val)
{
return __atomic_fetch_sub(address, val, __ATOMIC_RELAXED);
}
__device__
inline
int atomicExch(int* address, int val)
{
return __atomic_exchange_n(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicExch(unsigned int* address, unsigned int val)
{
return __atomic_exchange_n(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicExch(unsigned long long* address, unsigned long long val)
{
return __atomic_exchange_n(address, val, __ATOMIC_RELAXED);
}
__device__
inline
float atomicExch(float* address, float val)
{
return __uint_as_float(__atomic_exchange_n(
reinterpret_cast<unsigned int*>(address),
__float_as_uint(val),
__ATOMIC_RELAXED));
}
__device__
inline
int atomicMin(int* address, int val)
{
return __atomic_fetch_min(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicMin(unsigned int* address, unsigned int val)
{
return __atomic_fetch_min(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicMin(
unsigned long long* address, unsigned long long val)
{
unsigned long long tmp{__atomic_load_n(address, __ATOMIC_RELAXED)};
while (val < tmp) {
const auto tmp1 = __atomic_load_n(address, __ATOMIC_RELAXED);
if (tmp1 != tmp) { tmp = tmp1; continue; }
tmp = atomicCAS(address, tmp, val);
}
return tmp;
}
__device__
inline
int atomicMax(int* address, int val)
{
return __atomic_fetch_max(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicMax(unsigned int* address, unsigned int val)
{
return __atomic_fetch_max(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicMax(
unsigned long long* address, unsigned long long val)
{
unsigned long long tmp{__atomic_load_n(address, __ATOMIC_RELAXED)};
while (tmp < val) {
const auto tmp1 = __atomic_load_n(address, __ATOMIC_RELAXED);
if (tmp1 != tmp) { tmp = tmp1; continue; }
tmp = atomicCAS(address, tmp, val);
}
return tmp;
}
__device__
inline
unsigned int atomicInc(unsigned int* address, unsigned int val)
{
__device__
extern
unsigned int __builtin_amdgcn_atomic_inc(
unsigned int*,
unsigned int,
unsigned int,
unsigned int,
bool) __asm("llvm.amdgcn.atomic.inc.i32.p0i32");
return __builtin_amdgcn_atomic_inc(
address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false);
}
__device__
inline
unsigned int atomicDec(unsigned int* address, unsigned int val)
{
__device__
extern
unsigned int __builtin_amdgcn_atomic_dec(
unsigned int*,
unsigned int,
unsigned int,
unsigned int,
bool) __asm("llvm.amdgcn.atomic.dec.i32.p0i32");
return __builtin_amdgcn_atomic_dec(
address, val, __ATOMIC_RELAXED, 1 /* Device scope */, false);
}
__device__
inline
int atomicAnd(int* address, int val)
{
return __atomic_fetch_and(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicAnd(unsigned int* address, unsigned int val)
{
return __atomic_fetch_and(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicAnd(
unsigned long long* address, unsigned long long val)
{
return __atomic_fetch_and(address, val, __ATOMIC_RELAXED);
}
__device__
inline
int atomicOr(int* address, int val)
{
return __atomic_fetch_or(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicOr(unsigned int* address, unsigned int val)
{
return __atomic_fetch_or(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicOr(
unsigned long long* address, unsigned long long val)
{
return __atomic_fetch_or(address, val, __ATOMIC_RELAXED);
}
__device__
inline
int atomicXor(int* address, int val)
{
return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned int atomicXor(unsigned int* address, unsigned int val)
{
return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED);
}
__device__
inline
unsigned long long atomicXor(
unsigned long long* address, unsigned long long val)
{
return __atomic_fetch_xor(address, val, __ATOMIC_RELAXED);
}
#endif
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_common.h
+32
Fájl megtekintése
@@ -0,0 +1,32 @@
/*
Copyright (c) 2019 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
#if defined(__clang__) && defined(__HIP__)
#define __HIP_CLANG_ONLY__ 1
#else
#define __HIP_CLANG_ONLY__ 0
#endif
#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMMON_H
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_complex.h
+309
Fájl megtekintése
@@ -0,0 +1,309 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
#include "hip/amd_detail/amd_hip_vector_types.h"
#if defined(__HIPCC_RTC__)
#define __HOST_DEVICE__ __device__
#else
#define __HOST_DEVICE__ __host__ __device__
// TODO: Clang has a bug which allows device functions to call std functions
// when std functions are introduced into default namespace by using statement.
// math.h may be included after this bug is fixed.
#if __cplusplus
#include <cmath>
#else
#include "math.h"
#endif
#endif // !defined(__HIPCC_RTC__)
#if __cplusplus
#define COMPLEX_NEG_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type operator-(const type& op) { \
type ret; \
ret.x = -op.x; \
ret.y = -op.y; \
return ret; \
}
#define COMPLEX_EQ_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline bool operator==(const type& lhs, const type& rhs) { \
return lhs.x == rhs.x && lhs.y == rhs.y; \
}
#define COMPLEX_NE_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline bool operator!=(const type& lhs, const type& rhs) { \
return !(lhs == rhs); \
}
#define COMPLEX_ADD_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type operator+(const type& lhs, const type& rhs) { \
type ret; \
ret.x = lhs.x + rhs.x; \
ret.y = lhs.y + rhs.y; \
return ret; \
}
#define COMPLEX_SUB_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type operator-(const type& lhs, const type& rhs) { \
type ret; \
ret.x = lhs.x - rhs.x; \
ret.y = lhs.y - rhs.y; \
return ret; \
}
#define COMPLEX_MUL_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type operator*(const type& lhs, const type& rhs) { \
type ret; \
ret.x = lhs.x * rhs.x - lhs.y * rhs.y; \
ret.y = lhs.x * rhs.y + lhs.y * rhs.x; \
return ret; \
}
#define COMPLEX_DIV_OP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type operator/(const type& lhs, const type& rhs) { \
type ret; \
ret.x = (lhs.x * rhs.x + lhs.y * rhs.y); \
ret.y = (rhs.x * lhs.y - lhs.x * rhs.y); \
ret.x = ret.x / (rhs.x * rhs.x + rhs.y * rhs.y); \
ret.y = ret.y / (rhs.x * rhs.x + rhs.y * rhs.y); \
return ret; \
}
#define COMPLEX_ADD_PREOP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type& operator+=(type& lhs, const type& rhs) { \
lhs.x += rhs.x; \
lhs.y += rhs.y; \
return lhs; \
}
#define COMPLEX_SUB_PREOP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type& operator-=(type& lhs, const type& rhs) { \
lhs.x -= rhs.x; \
lhs.y -= rhs.y; \
return lhs; \
}
#define COMPLEX_MUL_PREOP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type& operator*=(type& lhs, const type& rhs) { \
lhs = lhs * rhs; \
return lhs; \
}
#define COMPLEX_DIV_PREOP_OVERLOAD(type) \
__HOST_DEVICE__ static inline type& operator/=(type& lhs, const type& rhs) { \
lhs = lhs / rhs; \
return lhs; \
}
#define COMPLEX_SCALAR_PRODUCT(type, type1) \
__HOST_DEVICE__ static inline type operator*(const type& lhs, type1 rhs) { \
type ret; \
ret.x = lhs.x * rhs; \
ret.y = lhs.y * rhs; \
return ret; \
}
#endif
typedef float2 hipFloatComplex;
__HOST_DEVICE__ static inline float hipCrealf(hipFloatComplex z) { return z.x; }
__HOST_DEVICE__ static inline float hipCimagf(hipFloatComplex z) { return z.y; }
__HOST_DEVICE__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) {
hipFloatComplex z;
z.x = a;
z.y = b;
return z;
}
__HOST_DEVICE__ static inline hipFloatComplex hipConjf(hipFloatComplex z) {
hipFloatComplex ret;
ret.x = z.x;
ret.y = -z.y;
return ret;
}
__HOST_DEVICE__ static inline float hipCsqabsf(hipFloatComplex z) {
return z.x * z.x + z.y * z.y;
}
__HOST_DEVICE__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) {
return make_hipFloatComplex(p.x + q.x, p.y + q.y);
}
__HOST_DEVICE__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) {
return make_hipFloatComplex(p.x - q.x, p.y - q.y);
}
__HOST_DEVICE__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) {
return make_hipFloatComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
}
__HOST_DEVICE__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) {
float sqabs = hipCsqabsf(q);
hipFloatComplex ret;
ret.x = (p.x * q.x + p.y * q.y) / sqabs;
ret.y = (p.y * q.x - p.x * q.y) / sqabs;
return ret;
}
__HOST_DEVICE__ static inline float hipCabsf(hipFloatComplex z) { return sqrtf(hipCsqabsf(z)); }
typedef double2 hipDoubleComplex;
__HOST_DEVICE__ static inline double hipCreal(hipDoubleComplex z) { return z.x; }
__HOST_DEVICE__ static inline double hipCimag(hipDoubleComplex z) { return z.y; }
__HOST_DEVICE__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) {
hipDoubleComplex z;
z.x = a;
z.y = b;
return z;
}
__HOST_DEVICE__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) {
hipDoubleComplex ret;
ret.x = z.x;
ret.y = -z.y;
return ret;
}
__HOST_DEVICE__ static inline double hipCsqabs(hipDoubleComplex z) {
return z.x * z.x + z.y * z.y;
}
__HOST_DEVICE__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) {
return make_hipDoubleComplex(p.x + q.x, p.y + q.y);
}
__HOST_DEVICE__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) {
return make_hipDoubleComplex(p.x - q.x, p.y - q.y);
}
__HOST_DEVICE__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) {
return make_hipDoubleComplex(p.x * q.x - p.y * q.y, p.y * q.x + p.x * q.y);
}
__HOST_DEVICE__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) {
double sqabs = hipCsqabs(q);
hipDoubleComplex ret;
ret.x = (p.x * q.x + p.y * q.y) / sqabs;
ret.y = (p.y * q.x - p.x * q.y) / sqabs;
return ret;
}
__HOST_DEVICE__ static inline double hipCabs(hipDoubleComplex z) { return sqrt(hipCsqabs(z)); }
#if __cplusplus
COMPLEX_NEG_OP_OVERLOAD(hipFloatComplex)
COMPLEX_EQ_OP_OVERLOAD(hipFloatComplex)
COMPLEX_NE_OP_OVERLOAD(hipFloatComplex)
COMPLEX_ADD_OP_OVERLOAD(hipFloatComplex)
COMPLEX_SUB_OP_OVERLOAD(hipFloatComplex)
COMPLEX_MUL_OP_OVERLOAD(hipFloatComplex)
COMPLEX_DIV_OP_OVERLOAD(hipFloatComplex)
COMPLEX_ADD_PREOP_OVERLOAD(hipFloatComplex)
COMPLEX_SUB_PREOP_OVERLOAD(hipFloatComplex)
COMPLEX_MUL_PREOP_OVERLOAD(hipFloatComplex)
COMPLEX_DIV_PREOP_OVERLOAD(hipFloatComplex)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned short)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed short)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned int)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed int)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, float)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned long)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed long)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, double)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, signed long long)
COMPLEX_SCALAR_PRODUCT(hipFloatComplex, unsigned long long)
COMPLEX_NEG_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_EQ_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_NE_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_ADD_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_SUB_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_MUL_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_DIV_OP_OVERLOAD(hipDoubleComplex)
COMPLEX_ADD_PREOP_OVERLOAD(hipDoubleComplex)
COMPLEX_SUB_PREOP_OVERLOAD(hipDoubleComplex)
COMPLEX_MUL_PREOP_OVERLOAD(hipDoubleComplex)
COMPLEX_DIV_PREOP_OVERLOAD(hipDoubleComplex)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned short)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed short)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned int)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed int)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, float)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned long)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed long)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, double)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, signed long long)
COMPLEX_SCALAR_PRODUCT(hipDoubleComplex, unsigned long long)
#endif
typedef hipFloatComplex hipComplex;
__HOST_DEVICE__ static inline hipComplex make_hipComplex(float x, float y) {
return make_hipFloatComplex(x, y);
}
__HOST_DEVICE__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) {
return make_hipFloatComplex((float)z.x, (float)z.y);
}
__HOST_DEVICE__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) {
return make_hipDoubleComplex((double)z.x, (double)z.y);
}
__HOST_DEVICE__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) {
float real = (p.x * q.x) + r.x;
float imag = (q.x * p.y) + r.y;
real = -(p.y * q.y) + real;
imag = (p.x * q.y) + imag;
return make_hipComplex(real, imag);
}
__HOST_DEVICE__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q,
hipDoubleComplex r) {
double real = (p.x * q.x) + r.x;
double imag = (q.x * p.y) + r.y;
real = -(p.y * q.y) + real;
imag = (p.x * q.y) + imag;
return make_hipDoubleComplex(real, imag);
}
#endif //HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COMPLEX_H
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_cooperative_groups.h
+510
Fájl megtekintése
@@ -0,0 +1,510 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/hip_cooperative_groups.h
*
* @brief Device side implementation of `Cooperative Group` feature.
*
* Defines new types and device API wrappers related to `Cooperative Group`
* feature, which the programmer can directly use in his kernel(s) in order to
* make use of this feature.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H
#if __cplusplus
#include <hip/amd_detail/hip_cooperative_groups_helper.h>
namespace cooperative_groups {
/** \brief The base type of all cooperative group types
*
* \details Holds the key properties of a constructed cooperative group types
* object, like the group type, its size, etc
*/
class thread_group {
protected:
uint32_t _type; // thread_group type
uint32_t _size; // total number of threads in the tread_group
uint64_t _mask; // Lanemask for coalesced and tiled partitioned group types,
// LSB represents lane 0, and MSB represents lane 63
// Construct a thread group, and set thread group type and other essential
// thread group properties. This generic thread group is directly constructed
// only when the group is supposed to contain only the calling the thread
// (throurh the API - `this_thread()`), and in all other cases, this thread
// group object is a sub-object of some other derived thread group object
__CG_QUALIFIER__ thread_group(internal::group_type type, uint32_t size,
uint64_t mask = (uint64_t)0) {
_type = type;
_size = size;
_mask = mask;
}
struct _tiled_info {
bool is_tiled;
unsigned int size;
} tiled_info;
friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent,
unsigned int tile_size);
friend class thread_block;
public:
// Total number of threads in the thread group, and this serves the purpose
// for all derived cooperative group types since their `size` is directly
// saved during the construction
__CG_QUALIFIER__ uint32_t size() const { return _size; }
__CG_QUALIFIER__ unsigned int cg_type() const { return _type; }
// Rank of the calling thread within [0, size())
__CG_QUALIFIER__ uint32_t thread_rank() const;
// Is this cooperative group type valid?
__CG_QUALIFIER__ bool is_valid() const;
// synchronize the threads in the thread group
__CG_QUALIFIER__ void sync() const;
};
/** \brief The multi-grid cooperative group type
*
* \details Represents an inter-device cooperative group type where the
* participating threads within the group spans across multple
* devices, running the (same) kernel on these devices
*/
class multi_grid_group : public thread_group {
// Only these friend functions are allowed to construct an object of this class
// and access its resources
friend __CG_QUALIFIER__ multi_grid_group this_multi_grid();
protected:
// Construct mutli-grid thread group (through the API this_multi_grid())
explicit __CG_QUALIFIER__ multi_grid_group(uint32_t size)
: thread_group(internal::cg_multi_grid, size) {}
public:
// Number of invocations participating in this multi-grid group. In other
// words, the number of GPUs
__CG_QUALIFIER__ uint32_t num_grids() { return internal::multi_grid::num_grids(); }
// Rank of this invocation. In other words, an ID number within the range
// [0, num_grids()) of the GPU, this kernel is running on
__CG_QUALIFIER__ uint32_t grid_rank() { return internal::multi_grid::grid_rank(); }
__CG_QUALIFIER__ uint32_t thread_rank() const { return internal::multi_grid::thread_rank(); }
__CG_QUALIFIER__ bool is_valid() const { return internal::multi_grid::is_valid(); }
__CG_QUALIFIER__ void sync() const { internal::multi_grid::sync(); }
};
/** \brief User exposed API interface to construct multi-grid cooperative
* group type object - `multi_grid_group`
*
* \details User is not allowed to directly construct an object of type
* `multi_grid_group`. Instead, he should construct it through this
* API function
*/
__CG_QUALIFIER__ multi_grid_group this_multi_grid() {
return multi_grid_group(internal::multi_grid::size());
}
/** \brief The grid cooperative group type
*
* \details Represents an inter-workgroup cooperative group type where the
* participating threads within the group spans across multiple
* workgroups running the (same) kernel on the same device
*/
class grid_group : public thread_group {
// Only these friend functions are allowed to construct an object of this class
// and access its resources
friend __CG_QUALIFIER__ grid_group this_grid();
protected:
// Construct grid thread group (through the API this_grid())
explicit __CG_QUALIFIER__ grid_group(uint32_t size) : thread_group(internal::cg_grid, size) {}
public:
__CG_QUALIFIER__ uint32_t thread_rank() const { return internal::grid::thread_rank(); }
__CG_QUALIFIER__ bool is_valid() const { return internal::grid::is_valid(); }
__CG_QUALIFIER__ void sync() const { internal::grid::sync(); }
};
/** \brief User exposed API interface to construct grid cooperative group type
* object - `grid_group`
*
* \details User is not allowed to directly construct an object of type
* `multi_grid_group`. Instead, he should construct it through this
* API function
*/
__CG_QUALIFIER__ grid_group this_grid() { return grid_group(internal::grid::size()); }
/** \brief The workgroup (thread-block in CUDA terminology) cooperative group
* type
*
* \details Represents an intra-workgroup cooperative group type where the
* participating threads within the group are exactly the same threads
* which are participated in the currently executing `workgroup`
*/
class thread_block : public thread_group {
// Only these friend functions are allowed to construct an object of thi
// class and access its resources
friend __CG_QUALIFIER__ thread_block this_thread_block();
friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent,
unsigned int tile_size);
friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_block& parent,
unsigned int tile_size);
protected:
// Construct a workgroup thread group (through the API this_thread_block())
explicit __CG_QUALIFIER__ thread_block(uint32_t size)
: thread_group(internal::cg_workgroup, size) {}
__CG_QUALIFIER__ thread_group new_tiled_group(unsigned int tile_size) const {
const bool pow2 = ((tile_size & (tile_size - 1)) == 0);
// Invalid tile size, assert
if (!tile_size || (tile_size > WAVEFRONT_SIZE) || !pow2) {
assert(false && "invalid tile size");
}
thread_group tiledGroup = thread_group(internal::cg_tiled_group, tile_size);
tiledGroup.tiled_info.size = tile_size;
tiledGroup.tiled_info.is_tiled = true;
return tiledGroup;
}
public:
// 3-dimensional block index within the grid
__CG_QUALIFIER__ dim3 group_index() { return internal::workgroup::group_index(); }
// 3-dimensional thread index within the block
__CG_QUALIFIER__ dim3 thread_index() { return internal::workgroup::thread_index(); }
__CG_QUALIFIER__ uint32_t thread_rank() const { return internal::workgroup::thread_rank(); }
__CG_QUALIFIER__ bool is_valid() const { return internal::workgroup::is_valid(); }
__CG_QUALIFIER__ void sync() const { internal::workgroup::sync(); }
};
/** \brief User exposed API interface to construct workgroup cooperative
* group type object - `thread_block`.
*
* \details User is not allowed to directly construct an object of type
* `thread_block`. Instead, he should construct it through this API
* function.
*/
__CG_QUALIFIER__ thread_block this_thread_block() {
return thread_block(internal::workgroup::size());
}
/** \brief The tiled_group cooperative group type
*
* \details Represents one tiled thread group in a wavefront.
* This group type also supports sub-wave level intrinsics.
*/
class tiled_group : public thread_group {
private:
friend __CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent,
unsigned int tile_size);
friend __CG_QUALIFIER__ tiled_group tiled_partition(const tiled_group& parent,
unsigned int tile_size);
__CG_QUALIFIER__ tiled_group new_tiled_group(unsigned int tile_size) const {
const bool pow2 = ((tile_size & (tile_size - 1)) == 0);
if (!tile_size || (tile_size > WAVEFRONT_SIZE) || !pow2) {
assert(false && "invalid tile size");
}
if (size() <= tile_size) {
return (*this);
}
tiled_group tiledGroup = tiled_group(tile_size);
tiledGroup.tiled_info.is_tiled = true;
return tiledGroup;
}
protected:
explicit __CG_QUALIFIER__ tiled_group(unsigned int tileSize)
: thread_group(internal::cg_tiled_group, tileSize) {
tiled_info.size = tileSize;
tiled_info.is_tiled = true;
}
public:
__CG_QUALIFIER__ unsigned int size() const { return (tiled_info.size); }
__CG_QUALIFIER__ unsigned int thread_rank() const {
return (internal::workgroup::thread_rank() & (tiled_info.size - 1));
}
__CG_QUALIFIER__ void sync() const {
// enforce memory ordering for memory instructions.
__builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent");
}
};
/**
* Implemenation of all publicly exposed base class APIs
*/
__CG_QUALIFIER__ uint32_t thread_group::thread_rank() const {
switch (this->_type) {
case internal::cg_multi_grid: {
return (static_cast<const multi_grid_group*>(this)->thread_rank());
}
case internal::cg_grid: {
return (static_cast<const grid_group*>(this)->thread_rank());
}
case internal::cg_workgroup: {
return (static_cast<const thread_block*>(this)->thread_rank());
}
case internal::cg_tiled_group: {
return (static_cast<const tiled_group*>(this)->thread_rank());
}
default: {
assert(false && "invalid cooperative group type");
return -1;
}
}
}
__CG_QUALIFIER__ bool thread_group::is_valid() const {
switch (this->_type) {
case internal::cg_multi_grid: {
return (static_cast<const multi_grid_group*>(this)->is_valid());
}
case internal::cg_grid: {
return (static_cast<const grid_group*>(this)->is_valid());
}
case internal::cg_workgroup: {
return (static_cast<const thread_block*>(this)->is_valid());
}
case internal::cg_tiled_group: {
return (static_cast<const tiled_group*>(this)->is_valid());
}
default: {
assert(false && "invalid cooperative group type");
return false;
}
}
}
__CG_QUALIFIER__ void thread_group::sync() const {
switch (this->_type) {
case internal::cg_multi_grid: {
static_cast<const multi_grid_group*>(this)->sync();
break;
}
case internal::cg_grid: {
static_cast<const grid_group*>(this)->sync();
break;
}
case internal::cg_workgroup: {
static_cast<const thread_block*>(this)->sync();
break;
}
case internal::cg_tiled_group: {
static_cast<const tiled_group*>(this)->sync();
break;
}
default: {
assert(false && "invalid cooperative group type");
}
}
}
/**
* Implemenation of publicly exposed `wrapper` APIs on top of basic cooperative
* group type APIs
*/
template <class CGTy> __CG_QUALIFIER__ uint32_t group_size(CGTy const& g) { return g.size(); }
template <class CGTy> __CG_QUALIFIER__ uint32_t thread_rank(CGTy const& g) {
return g.thread_rank();
}
template <class CGTy> __CG_QUALIFIER__ bool is_valid(CGTy const& g) { return g.is_valid(); }
template <class CGTy> __CG_QUALIFIER__ void sync(CGTy const& g) { g.sync(); }
template <unsigned int tileSize> class tile_base {
protected:
_CG_STATIC_CONST_DECL_ unsigned int numThreads = tileSize;
public:
// Rank of the thread within this tile
_CG_STATIC_CONST_DECL_ unsigned int thread_rank() {
return (internal::workgroup::thread_rank() & (numThreads - 1));
}
// Number of threads within this tile
__CG_STATIC_QUALIFIER__ unsigned int size() { return numThreads; }
};
template <unsigned int size> class thread_block_tile_base : public tile_base<size> {
static_assert(is_valid_tile_size<size>::value,
"Tile size is either not a power of 2 or greater than the wavefront size");
using tile_base<size>::numThreads;
public:
__CG_STATIC_QUALIFIER__ void sync() {
// enforce ordering for memory instructions
__builtin_amdgcn_fence(__ATOMIC_ACQ_REL, "agent");
}
template <class T> __CG_QUALIFIER__ T shfl(T var, int srcRank) const {
static_assert(is_valid_type<T>::value, "Neither an integer or float type.");
return (__shfl(var, srcRank, numThreads));
}
template <class T> __CG_QUALIFIER__ T shfl_down(T var, unsigned int lane_delta) const {
static_assert(is_valid_type<T>::value, "Neither an integer or float type.");
return (__shfl_down(var, lane_delta, numThreads));
}
template <class T> __CG_QUALIFIER__ T shfl_up(T var, unsigned int lane_delta) const {
static_assert(is_valid_type<T>::value, "Neither an integer or float type.");
return (__shfl_up(var, lane_delta, numThreads));
}
template <class T> __CG_QUALIFIER__ T shfl_xor(T var, unsigned int laneMask) const {
static_assert(is_valid_type<T>::value, "Neither an integer or float type.");
return (__shfl_xor(var, laneMask, numThreads));
}
};
/** \brief Group type - thread_block_tile
*
* \details Represents one tile of thread group.
*/
template <unsigned int tileSize, class ParentCGTy = void>
class thread_block_tile_type : public thread_block_tile_base<tileSize>, public tiled_group {
_CG_STATIC_CONST_DECL_ unsigned int numThreads = tileSize;
friend class thread_block_tile_type<tileSize, ParentCGTy>;
typedef thread_block_tile_base<numThreads> tbtBase;
protected:
__CG_QUALIFIER__ thread_block_tile_type() : tiled_group(numThreads) {
tiled_info.size = numThreads;
tiled_info.is_tiled = true;
}
public:
using tbtBase::size;
using tbtBase::sync;
using tbtBase::thread_rank;
};
/** \brief User exposed API to partition groups.
*
* \details A collective operation that partitions the parent group into a one-dimensional,
* row-major, tiling of subgroups.
*/
__CG_QUALIFIER__ thread_group tiled_partition(const thread_group& parent, unsigned int tile_size) {
if (parent.cg_type() == internal::cg_tiled_group) {
const tiled_group* cg = static_cast<const tiled_group*>(&parent);
return cg->new_tiled_group(tile_size);
} else {
const thread_block* tb = static_cast<const thread_block*>(&parent);
return tb->new_tiled_group(tile_size);
}
}
// Thread block type overload
__CG_QUALIFIER__ thread_group tiled_partition(const thread_block& parent, unsigned int tile_size) {
return (parent.new_tiled_group(tile_size));
}
// Coalesced group type overload
__CG_QUALIFIER__ tiled_group tiled_partition(const tiled_group& parent, unsigned int tile_size) {
return (parent.new_tiled_group(tile_size));
}
template <unsigned int size, class ParentCGTy> class thread_block_tile;
namespace impl {
template <unsigned int size, class ParentCGTy> class thread_block_tile_internal;
template <unsigned int size, class ParentCGTy>
class thread_block_tile_internal : public thread_block_tile_type<size, ParentCGTy> {
protected:
template <unsigned int tbtSize, class tbtParentT>
__CG_QUALIFIER__ thread_block_tile_internal(
const thread_block_tile_internal<tbtSize, tbtParentT>& g)
: thread_block_tile_type<size, ParentCGTy>() {}
__CG_QUALIFIER__ thread_block_tile_internal(const thread_block& g)
: thread_block_tile_type<size, ParentCGTy>() {}
};
} // namespace impl
template <unsigned int size, class ParentCGTy>
class thread_block_tile : public impl::thread_block_tile_internal<size, ParentCGTy> {
protected:
__CG_QUALIFIER__ thread_block_tile(const ParentCGTy& g)
: impl::thread_block_tile_internal<size, ParentCGTy>(g) {}
public:
__CG_QUALIFIER__ operator thread_block_tile<size, void>() const {
return thread_block_tile<size, void>(*this);
}
};
template <unsigned int size>
class thread_block_tile<size, void> : public impl::thread_block_tile_internal<size, void> {
template <unsigned int, class ParentCGTy> friend class thread_block_tile;
protected:
public:
template <class ParentCGTy>
__CG_QUALIFIER__ thread_block_tile(const thread_block_tile<size, ParentCGTy>& g)
: impl::thread_block_tile_internal<size, void>(g) {}
};
template <unsigned int size, class ParentCGTy = void> class thread_block_tile;
namespace impl {
template <unsigned int size, class ParentCGTy = void> struct tiled_partition_internal;
template <unsigned int size>
struct tiled_partition_internal<size, thread_block> : public thread_block_tile<size, thread_block> {
__CG_QUALIFIER__ tiled_partition_internal(const thread_block& g)
: thread_block_tile<size, thread_block>(g) {}
};
} // namespace impl
/** \brief User exposed API to partition groups.
*
* \details This constructs a templated class derieved from thread_group.
* The template defines tile size of the new thread group at compile time.
*/
template <unsigned int size, class ParentCGTy>
__CG_QUALIFIER__ thread_block_tile<size, ParentCGTy> tiled_partition(const ParentCGTy& g) {
static_assert(is_valid_tile_size<size>::value,
"Tiled partition with size > wavefront size. Currently not supported ");
return impl::tiled_partition_internal<size, ParentCGTy>(g);
}
} // namespace cooperative_groups
#endif // __cplusplus
#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_H
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_fp16.h
+1663
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_runtime.h
+417
Fájl megtekintése
@@ -0,0 +1,417 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/hip_runtime.h
* @brief Contains definitions of APIs for HIP runtime.
*/
//#pragma once
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_H
#include <hip/amd_detail/amd_hip_common.h>
//---
// Top part of file can be compiled with any compiler
#if !defined(__HIPCC_RTC__)
//#include <cstring>
#if __cplusplus
#include <cmath>
#include <cstdint>
#else
#include <math.h>
#include <string.h>
#include <stddef.h>
#endif // __cplusplus
#endif // !defined(__HIPCC_RTC__)
// __hip_malloc is not working. Disable it by default.
#ifndef __HIP_ENABLE_DEVICE_MALLOC__
#define __HIP_ENABLE_DEVICE_MALLOC__ 0
#endif
#if __HIP_CLANG_ONLY__
#if !defined(__align__)
#define __align__(x) __attribute__((aligned(x)))
#endif
#define CUDA_SUCCESS hipSuccess
#if !defined(__HIPCC_RTC__)
#include <hip/hip_runtime_api.h>
extern int HIP_TRACE_API;
#endif // !defined(__HIPCC_RTC__)
#ifdef __cplusplus
#include <hip/amd_detail/hip_ldg.h>
#endif
#include <hip/amd_detail/amd_hip_atomic.h>
#include <hip/amd_detail/host_defines.h>
#include <hip/amd_detail/amd_device_functions.h>
#include <hip/amd_detail/amd_surface_functions.h>
#include <hip/amd_detail/texture_fetch_functions.h>
#include <hip/amd_detail/texture_indirect_functions.h>
// TODO-HCC remove old definitions ; ~1602 hcc supports __HCC_ACCELERATOR__ define.
#if defined(__KALMAR_ACCELERATOR__) && !defined(__HCC_ACCELERATOR__)
#define __HCC_ACCELERATOR__ __KALMAR_ACCELERATOR__
#endif
// Feature tests:
#if (defined(__HCC_ACCELERATOR__) && (__HCC_ACCELERATOR__ != 0)) || __HIP_DEVICE_COMPILE__
// Device compile and not host compile:
// 32-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (1)
#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (1)
#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (1)
#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (1)
#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (1)
// 64-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (1)
#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (1)
// Doubles
#define __HIP_ARCH_HAS_DOUBLES__ (1)
// warp cross-lane operations:
#define __HIP_ARCH_HAS_WARP_VOTE__ (1)
#define __HIP_ARCH_HAS_WARP_BALLOT__ (1)
#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (1)
#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (0)
// sync
#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (1)
#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (0)
// misc
#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (0)
#define __HIP_ARCH_HAS_3DGRID__ (1)
#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (0)
#endif /* Device feature flags */
#define launch_bounds_impl0(requiredMaxThreadsPerBlock) \
__attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock)))
#define launch_bounds_impl1(requiredMaxThreadsPerBlock, minBlocksPerMultiprocessor) \
__attribute__((amdgpu_flat_work_group_size(1, requiredMaxThreadsPerBlock), \
amdgpu_waves_per_eu(minBlocksPerMultiprocessor)))
#define select_impl_(_1, _2, impl_, ...) impl_
#define __launch_bounds__(...) \
select_impl_(__VA_ARGS__, launch_bounds_impl1, launch_bounds_impl0)(__VA_ARGS__)
#if !defined(__HIPCC_RTC__)
__host__ inline void* __get_dynamicgroupbaseptr() { return nullptr; }
#endif // !defined(__HIPCC_RTC__)
#if __HIP_ARCH_GFX701__ == 0
__device__ unsigned __hip_ds_bpermute(int index, unsigned src);
__device__ float __hip_ds_bpermutef(int index, float src);
__device__ unsigned __hip_ds_permute(int index, unsigned src);
__device__ float __hip_ds_permutef(int index, float src);
template <int pattern>
__device__ unsigned __hip_ds_swizzle_N(unsigned int src);
template <int pattern>
__device__ float __hip_ds_swizzlef_N(float src);
template <int dpp_ctrl, int row_mask, int bank_mask, bool bound_ctrl>
__device__ int __hip_move_dpp_N(int src);
#endif //__HIP_ARCH_GFX803__ == 1
#ifndef __OPENMP_AMDGCN__
#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#if __HIP_ENABLE_DEVICE_MALLOC__
extern "C" __device__ void* __hip_malloc(size_t);
extern "C" __device__ void* __hip_free(void* ptr);
static inline __device__ void* malloc(size_t size) { return __hip_malloc(size); }
static inline __device__ void* free(void* ptr) { return __hip_free(ptr); }
#else
static inline __device__ void* malloc(size_t size) { __builtin_trap(); return nullptr; }
static inline __device__ void* free(void* ptr) { __builtin_trap(); return nullptr; }
#endif
#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#endif // !__OPENMP_AMDGCN__
// End doxygen API:
/**
* @}
*/
//
// hip-clang functions
//
#if !defined(__HIPCC_RTC__)
#define HIP_KERNEL_NAME(...) __VA_ARGS__
#define HIP_SYMBOL(X) X
typedef int hipLaunchParm;
template <std::size_t n, typename... Ts,
typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
void pArgs(const std::tuple<Ts...>&, void*) {}
template <std::size_t n, typename... Ts,
typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
void pArgs(const std::tuple<Ts...>& formals, void** _vargs) {
using T = typename std::tuple_element<n, std::tuple<Ts...> >::type;
static_assert(!std::is_reference<T>{},
"A __global__ function cannot have a reference as one of its "
"arguments.");
#if defined(HIP_STRICT)
static_assert(std::is_trivially_copyable<T>{},
"Only TriviallyCopyable types can be arguments to a __global__ "
"function");
#endif
_vargs[n] = const_cast<void*>(reinterpret_cast<const void*>(&std::get<n>(formals)));
return pArgs<n + 1>(formals, _vargs);
}
template <typename... Formals, typename... Actuals>
std::tuple<Formals...> validateArgsCountType(void (*kernel)(Formals...), std::tuple<Actuals...>(actuals)) {
static_assert(sizeof...(Formals) == sizeof...(Actuals), "Argument Count Mismatch");
std::tuple<Formals...> to_formals{std::move(actuals)};
return to_formals;
}
#if defined(HIP_TEMPLATE_KERNEL_LAUNCH)
template <typename... Args, typename F = void (*)(Args...)>
void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
std::uint32_t sharedMemBytes, hipStream_t stream, Args... args) {
constexpr size_t count = sizeof...(Args);
auto tup_ = std::tuple<Args...>{args...};
auto tup = validateArgsCountType(kernel, tup_);
void* _Args[count];
pArgs<0>(tup, _Args);
auto k = reinterpret_cast<void*>(kernel);
hipLaunchKernel(k, numBlocks, dimBlocks, _Args, sharedMemBytes, stream);
}
#else
#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...) \
do { \
kernelName<<<(numBlocks), (numThreads), (memPerBlock), (streamId)>>>(__VA_ARGS__); \
} while (0)
#define hipLaunchKernelGGL(kernelName, ...) hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__)
#endif
#include <hip/hip_runtime_api.h>
#endif // !defined(__HIPCC_RTC__)
extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(uint);
extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(uint);
extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(uint);
extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(uint);
struct __HIP_BlockIdx {
__device__
std::uint32_t operator()(std::uint32_t x) const noexcept { return __ockl_get_group_id(x); }
};
struct __HIP_BlockDim {
__device__
std::uint32_t operator()(std::uint32_t x) const noexcept {
return __ockl_get_local_size(x);
}
};
struct __HIP_GridDim {
__device__
std::uint32_t operator()(std::uint32_t x) const noexcept {
return __ockl_get_num_groups(x);
}
};
struct __HIP_ThreadIdx {
__device__
std::uint32_t operator()(std::uint32_t x) const noexcept {
return __ockl_get_local_id(x);
}
};
#if defined(__HIPCC_RTC__)
typedef struct dim3 {
uint32_t x; ///< x
uint32_t y; ///< y
uint32_t z; ///< z
#ifdef __cplusplus
constexpr __device__ dim3(uint32_t _x = 1, uint32_t _y = 1, uint32_t _z = 1) : x(_x), y(_y), z(_z){};
#endif
} dim3;
#endif // !defined(__HIPCC_RTC__)
template <typename F>
struct __HIP_Coordinates {
using R = decltype(F{}(0));
struct __X { __device__ operator R() const noexcept { return F{}(0); } };
struct __Y { __device__ operator R() const noexcept { return F{}(1); } };
struct __Z { __device__ operator R() const noexcept { return F{}(2); } };
static constexpr __X x{};
static constexpr __Y y{};
static constexpr __Z z{};
#ifdef __cplusplus
__device__ operator dim3() const { return dim3(x, y, z); }
#endif
};
template <typename F>
#if !defined(_MSC_VER)
__attribute__((weak))
#endif
constexpr typename __HIP_Coordinates<F>::__X __HIP_Coordinates<F>::x;
template <typename F>
#if !defined(_MSC_VER)
__attribute__((weak))
#endif
constexpr typename __HIP_Coordinates<F>::__Y __HIP_Coordinates<F>::y;
template <typename F>
#if !defined(_MSC_VER)
__attribute__((weak))
#endif
constexpr typename __HIP_Coordinates<F>::__Z __HIP_Coordinates<F>::z;
extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_size(uint);
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__X,
__HIP_Coordinates<__HIP_BlockDim>::__X) noexcept {
return __ockl_get_global_size(0);
}
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__X,
__HIP_Coordinates<__HIP_GridDim>::__X) noexcept {
return __ockl_get_global_size(0);
}
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__Y,
__HIP_Coordinates<__HIP_BlockDim>::__Y) noexcept {
return __ockl_get_global_size(1);
}
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__Y,
__HIP_Coordinates<__HIP_GridDim>::__Y) noexcept {
return __ockl_get_global_size(1);
}
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_GridDim>::__Z,
__HIP_Coordinates<__HIP_BlockDim>::__Z) noexcept {
return __ockl_get_global_size(2);
}
inline
__device__
std::uint32_t operator*(__HIP_Coordinates<__HIP_BlockDim>::__Z,
__HIP_Coordinates<__HIP_GridDim>::__Z) noexcept {
return __ockl_get_global_size(2);
}
static constexpr __HIP_Coordinates<__HIP_BlockDim> blockDim{};
static constexpr __HIP_Coordinates<__HIP_BlockIdx> blockIdx{};
static constexpr __HIP_Coordinates<__HIP_GridDim> gridDim{};
static constexpr __HIP_Coordinates<__HIP_ThreadIdx> threadIdx{};
extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_id(uint);
#define hipThreadIdx_x (__ockl_get_local_id(0))
#define hipThreadIdx_y (__ockl_get_local_id(1))
#define hipThreadIdx_z (__ockl_get_local_id(2))
extern "C" __device__ __attribute__((const)) size_t __ockl_get_group_id(uint);
#define hipBlockIdx_x (__ockl_get_group_id(0))
#define hipBlockIdx_y (__ockl_get_group_id(1))
#define hipBlockIdx_z (__ockl_get_group_id(2))
extern "C" __device__ __attribute__((const)) size_t __ockl_get_local_size(uint);
#define hipBlockDim_x (__ockl_get_local_size(0))
#define hipBlockDim_y (__ockl_get_local_size(1))
#define hipBlockDim_z (__ockl_get_local_size(2))
extern "C" __device__ __attribute__((const)) size_t __ockl_get_num_groups(uint);
#define hipGridDim_x (__ockl_get_num_groups(0))
#define hipGridDim_y (__ockl_get_num_groups(1))
#define hipGridDim_z (__ockl_get_num_groups(2))
#include <hip/amd_detail/amd_math_functions.h>
#if __HIP_HCC_COMPAT_MODE__
// Define HCC work item functions in terms of HIP builtin variables.
#pragma push_macro("__DEFINE_HCC_FUNC")
#define __DEFINE_HCC_FUNC(hc_fun,hip_var) \
inline __device__ __attribute__((always_inline)) uint hc_get_##hc_fun(uint i) { \
if (i==0) \
return hip_var.x; \
else if(i==1) \
return hip_var.y; \
else \
return hip_var.z; \
}
__DEFINE_HCC_FUNC(workitem_id, threadIdx)
__DEFINE_HCC_FUNC(group_id, blockIdx)
__DEFINE_HCC_FUNC(group_size, blockDim)
__DEFINE_HCC_FUNC(num_groups, gridDim)
#pragma pop_macro("__DEFINE_HCC_FUNC")
extern "C" __device__ __attribute__((const)) size_t __ockl_get_global_id(uint);
inline __device__ __attribute__((always_inline)) uint
hc_get_workitem_absolute_id(int dim)
{
return (uint)__ockl_get_global_id(dim);
}
#endif
#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#if !defined(__HIPCC_RTC__)
// Support std::complex.
#if !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
#pragma push_macro("__CUDA__")
#define __CUDA__
#include <__clang_cuda_math_forward_declares.h>
#include <__clang_cuda_complex_builtins.h>
// Workaround for using libc++ with HIP-Clang.
// The following headers requires clang include path before standard C++ include path.
// However libc++ include path requires to be before clang include path.
// To workaround this, we pass -isystem with the parent directory of clang include
// path instead of the clang include path itself.
#include <include/cuda_wrappers/algorithm>
#include <include/cuda_wrappers/complex>
#include <include/cuda_wrappers/new>
#undef __CUDA__
#pragma pop_macro("__CUDA__")
#endif // !_OPENMP || __HIP_ENABLE_CUDA_WRAPPER_FOR_OPENMP__
#endif // !defined(__HIPCC_RTC__)
#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#endif // __HIP_CLANG_ONLY__
#include <hip/amd_detail/hip_memory.h>
#endif // HIP_AMD_DETAIL_RUNTIME_H
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_surface_types.h
+54
Fájl megtekintése
@@ -0,0 +1,54 @@
/*
Copyright (c) 2015- present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/hip_surface_types.h
* @brief Defines surface types for HIP runtime.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H
#include <hip/amd_detail/amd_driver_types.h>
/**
* An opaque value that represents a hip surface object
*/
typedef unsigned long long hipSurfaceObject_t;
/**
* hip surface reference
*/
struct surfaceReference {
hipSurfaceObject_t surfaceObject;
};
/**
* hip surface boundary modes
*/
enum hipSurfaceBoundaryMode {
hipBoundaryModeZero = 0,
hipBoundaryModeTrap = 1,
hipBoundaryModeClamp = 2
};
#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_HIP_SURFACE_TYPES_H */
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_texture_types.h
+97
Fájl megtekintése
@@ -0,0 +1,97 @@
/*
Copyright (c) 2015-2016 Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/hip_texture_types.h
* @brief Defines the different newt vector types for HIP runtime.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H
/*******************************************************************************
* *
* *
* *
*******************************************************************************/
#if !defined(__HIPCC_RTC__)
#include <limits.h>
#include <hip/amd_detail/amd_channel_descriptor.h>
#endif // !defined(__HIPCC_RTC__)
#include <hip/amd_detail/amd_texture_types.h>
#if __cplusplus
/*******************************************************************************
* *
* *
* *
*******************************************************************************/
#if __HIP__
#define __HIP_TEXTURE_ATTRIB __attribute__((device_builtin_texture_type))
#else
#define __HIP_TEXTURE_ATTRIB
#endif
typedef textureReference* hipTexRef;
template <class T, int texType = hipTextureType1D,
enum hipTextureReadMode mode = hipReadModeElementType>
struct __HIP_TEXTURE_ATTRIB texture : public textureReference {
texture(int norm = 0, enum hipTextureFilterMode fMode = hipFilterModePoint,
enum hipTextureAddressMode aMode = hipAddressModeClamp) {
normalized = norm;
readMode = mode;
filterMode = fMode;
addressMode[0] = aMode;
addressMode[1] = aMode;
addressMode[2] = aMode;
channelDesc = hipCreateChannelDesc<T>();
sRGB = 0;
textureObject = nullptr;
maxAnisotropy = 0;
mipmapLevelBias = 0;
minMipmapLevelClamp = 0;
maxMipmapLevelClamp = 0;
}
texture(int norm, enum hipTextureFilterMode fMode, enum hipTextureAddressMode aMode,
struct hipChannelFormatDesc desc) {
normalized = norm;
readMode = mode;
filterMode = fMode;
addressMode[0] = aMode;
addressMode[1] = aMode;
addressMode[2] = aMode;
channelDesc = desc;
sRGB = 0;
textureObject = nullptr;
maxAnisotropy = 0;
mipmapLevelBias = 0;
minMipmapLevelClamp = 0;
maxMipmapLevelClamp = 0;
}
};
#endif /* __cplusplus */
#endif /* !HIP_INCLUDE_HIP_AMD_DETAIL_HIP_TEXTURE_TYPES_H */
projects/hip/src/hipamd/include/hip/amd_detail/amd_hip_vector_types.h
+1598
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/amd_detail/amd_hiprtc.h
+94
Fájl megtekintése
@@ -0,0 +1,94 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIPRTC_H
#define HIPRTC_H
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
#include <stdlib.h>
#if !defined(_WIN32)
#pragma GCC visibility push (default)
#endif
enum hiprtcResult {
HIPRTC_SUCCESS = 0,
HIPRTC_ERROR_OUT_OF_MEMORY = 1,
HIPRTC_ERROR_PROGRAM_CREATION_FAILURE = 2,
HIPRTC_ERROR_INVALID_INPUT = 3,
HIPRTC_ERROR_INVALID_PROGRAM = 4,
HIPRTC_ERROR_INVALID_OPTION = 5,
HIPRTC_ERROR_COMPILATION = 6,
HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7,
HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8,
HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9,
HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10,
HIPRTC_ERROR_INTERNAL_ERROR = 11
};
const char* hiprtcGetErrorString(hiprtcResult result);
hiprtcResult hiprtcVersion(int* major, int* minor);
typedef struct _hiprtcProgram* hiprtcProgram;
hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog,
const char* name_expression);
hiprtcResult hiprtcCompileProgram(hiprtcProgram prog,
int numOptions,
const char** options);
hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog,
const char* src,
const char* name,
int numHeaders,
const char** headers,
const char** includeNames);
hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog);
hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog,
const char* name_expression,
const char** lowered_name);
hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* log);
hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog,
size_t* logSizeRet);
hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* code);
hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* codeSizeRet);
#if !defined(_WIN32)
#pragma GCC visibility pop
#endif
#ifdef __cplusplus
}
#endif /* __cplusplus */
#endif //HIPRTC_H
projects/hip/src/hipamd/include/hip/amd_detail/amd_library_types.h
+41
Fájl megtekintése
@@ -0,0 +1,41 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_LIBRARY_TYPES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_LIBRARY_TYPES_H
typedef enum hipDataType {
HIP_R_16F = 2,
HIP_R_32F = 0,
HIP_R_64F = 1,
HIP_C_16F = 6,
HIP_C_32F = 4,
HIP_C_64F = 5
} hipDataType;
typedef enum hipLibraryPropertyType {
HIP_LIBRARY_MAJOR_VERSION,
HIP_LIBRARY_MINOR_VERSION,
HIP_LIBRARY_PATCH_LEVEL
} hipLibraryPropertyType;
#endif
projects/hip/src/hipamd/include/hip/amd_detail/amd_math_functions.h
+1502
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/amd_detail/amd_surface_functions.h
+59
Fájl megtekintése
@@ -0,0 +1,59 @@
/*
Copyright (c) 2018 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_SURFACE_FUNCTIONS_H
#include <hip/amd_detail/amd_hip_surface_types.h>
#define __SURFACE_FUNCTIONS_DECL__ static inline __device__
template <class T>
__SURFACE_FUNCTIONS_DECL__ void surf2Dread(T* data, hipSurfaceObject_t surfObj, int x, int y,
int boundaryMode = hipBoundaryModeZero) {
hipArray* arrayPtr = (hipArray*)surfObj;
size_t width = arrayPtr->width;
size_t height = arrayPtr->height;
int32_t xOffset = x / sizeof(T);
T* dataPtr = (T*)arrayPtr->data;
if ((xOffset > width) || (xOffset < 0) || (y > height) || (y < 0)) {
if (boundaryMode == hipBoundaryModeZero) {
*data = 0;
}
} else {
*data = *(dataPtr + y * width + xOffset);
}
}
template <class T>
__SURFACE_FUNCTIONS_DECL__ void surf2Dwrite(T data, hipSurfaceObject_t surfObj, int x, int y,
int boundaryMode = hipBoundaryModeZero) {
hipArray* arrayPtr = (hipArray*)surfObj;
size_t width = arrayPtr->width;
size_t height = arrayPtr->height;
int32_t xOffset = x / sizeof(T);
T* dataPtr = (T*)arrayPtr->data;
if (!((xOffset > width) || (xOffset < 0) || (y > height) || (y < 0))) {
*(dataPtr + y * width + xOffset) = data;
}
}
#endif
projects/hip/src/hipamd/include/hip/amd_detail/amd_texture_types.h
+109
Fájl megtekintése
@@ -0,0 +1,109 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_TYPES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_TEXTURE_TYPES_H
#include <hip/amd_detail/amd_driver_types.h>
#define hipTextureType1D 0x01
#define hipTextureType2D 0x02
#define hipTextureType3D 0x03
#define hipTextureTypeCubemap 0x0C
#define hipTextureType1DLayered 0xF1
#define hipTextureType2DLayered 0xF2
#define hipTextureTypeCubemapLayered 0xFC
/**
* Should be same as HSA_IMAGE_OBJECT_SIZE_DWORD/HSA_SAMPLER_OBJECT_SIZE_DWORD
*/
#define HIP_IMAGE_OBJECT_SIZE_DWORD 12
#define HIP_SAMPLER_OBJECT_SIZE_DWORD 8
#define HIP_SAMPLER_OBJECT_OFFSET_DWORD HIP_IMAGE_OBJECT_SIZE_DWORD
#define HIP_TEXTURE_OBJECT_SIZE_DWORD (HIP_IMAGE_OBJECT_SIZE_DWORD + HIP_SAMPLER_OBJECT_SIZE_DWORD)
/**
* An opaque value that represents a hip texture object
*/
struct __hip_texture;
typedef struct __hip_texture* hipTextureObject_t;
/**
* hip texture address modes
*/
enum hipTextureAddressMode {
hipAddressModeWrap = 0,
hipAddressModeClamp = 1,
hipAddressModeMirror = 2,
hipAddressModeBorder = 3
};
/**
* hip texture filter modes
*/
enum hipTextureFilterMode { hipFilterModePoint = 0, hipFilterModeLinear = 1 };
/**
* hip texture read modes
*/
enum hipTextureReadMode { hipReadModeElementType = 0, hipReadModeNormalizedFloat = 1 };
/**
* hip texture reference
*/
typedef struct textureReference {
int normalized;
enum hipTextureReadMode readMode;// used only for driver API's
enum hipTextureFilterMode filterMode;
enum hipTextureAddressMode addressMode[3]; // Texture address mode for up to 3 dimensions
struct hipChannelFormatDesc channelDesc;
int sRGB; // Perform sRGB->linear conversion during texture read
unsigned int maxAnisotropy; // Limit to the anisotropy ratio
enum hipTextureFilterMode mipmapFilterMode;
float mipmapLevelBias;
float minMipmapLevelClamp;
float maxMipmapLevelClamp;
hipTextureObject_t textureObject;
int numChannels;
enum hipArray_Format format;
}textureReference;
/**
* hip texture descriptor
*/
typedef struct hipTextureDesc {
enum hipTextureAddressMode addressMode[3]; // Texture address mode for up to 3 dimensions
enum hipTextureFilterMode filterMode;
enum hipTextureReadMode readMode;
int sRGB; // Perform sRGB->linear conversion during texture read
float borderColor[4];
int normalizedCoords;
unsigned int maxAnisotropy;
enum hipTextureFilterMode mipmapFilterMode;
float mipmapLevelBias;
float minMipmapLevelClamp;
float maxMipmapLevelClamp;
}hipTextureDesc;
#endif
projects/hip/src/hipamd/include/hip/amd_detail/concepts.hpp
+30
Fájl megtekintése
@@ -0,0 +1,30 @@
/*
Copyright (c) 2015-present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
namespace hip_impl // Documentation only.
{
#define requires(...)
#define FunctionalProcedure typename
} // namespace hip_impl
projects/hip/src/hipamd/include/hip/amd_detail/device_library_decls.h
+118
Fájl megtekintése
@@ -0,0 +1,118 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/device_library_decls.h
* @brief Contains declarations for types and functions in device library.
* Uses int64_t and uint64_t instead of long, long long, unsigned
* long and unsigned long long types for device library API
* declarations.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_DEVICE_LIBRARY_DECLS_H
#include "hip/amd_detail/host_defines.h"
typedef unsigned char uchar;
typedef unsigned short ushort;
typedef unsigned int uint;
typedef unsigned long ulong;
typedef unsigned long long ullong;
extern "C" __device__ __attribute__((const)) bool __ockl_wfany_i32(int);
extern "C" __device__ __attribute__((const)) bool __ockl_wfall_i32(int);
extern "C" __device__ uint __ockl_activelane_u32(void);
extern "C" __device__ __attribute__((const)) uint __ockl_mul24_u32(uint, uint);
extern "C" __device__ __attribute__((const)) int __ockl_mul24_i32(int, int);
extern "C" __device__ __attribute__((const)) uint __ockl_mul_hi_u32(uint, uint);
extern "C" __device__ __attribute__((const)) int __ockl_mul_hi_i32(int, int);
extern "C" __device__ __attribute__((const)) uint __ockl_sadd_u32(uint, uint, uint);
extern "C" __device__ __attribute__((const)) uchar __ockl_clz_u8(uchar);
extern "C" __device__ __attribute__((const)) ushort __ockl_clz_u16(ushort);
extern "C" __device__ __attribute__((const)) uint __ockl_clz_u32(uint);
extern "C" __device__ __attribute__((const)) uint64_t __ockl_clz_u64(uint64_t);
extern "C" __device__ __attribute__((const)) float __ocml_floor_f32(float);
extern "C" __device__ __attribute__((const)) float __ocml_rint_f32(float);
extern "C" __device__ __attribute__((const)) float __ocml_ceil_f32(float);
extern "C" __device__ __attribute__((const)) float __ocml_trunc_f32(float);
extern "C" __device__ __attribute__((const)) float __ocml_fmin_f32(float, float);
extern "C" __device__ __attribute__((const)) float __ocml_fmax_f32(float, float);
extern "C" __device__ __attribute__((convergent)) void __ockl_gws_init(uint nwm1, uint rid);
extern "C" __device__ __attribute__((convergent)) void __ockl_gws_barrier(uint nwm1, uint rid);
extern "C" __device__ __attribute__((const)) uint32_t __ockl_lane_u32();
extern "C" __device__ __attribute__((const)) int __ockl_grid_is_valid(void);
extern "C" __device__ __attribute__((convergent)) void __ockl_grid_sync(void);
extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_num_grids(void);
extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_grid_rank(void);
extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_size(void);
extern "C" __device__ __attribute__((const)) uint __ockl_multi_grid_thread_rank(void);
extern "C" __device__ __attribute__((const)) int __ockl_multi_grid_is_valid(void);
extern "C" __device__ __attribute__((convergent)) void __ockl_multi_grid_sync(void);
extern "C" __device__ void __ockl_atomic_add_noret_f32(float*, float);
extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_add_i32(int a);
extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_and_i32(int a);
extern "C" __device__ __attribute__((convergent)) int __ockl_wgred_or_i32(int a);
// Introduce local address space
#define __local __attribute__((address_space(3)))
#ifdef __HIP_DEVICE_COMPILE__
__device__ inline static __local void* __to_local(unsigned x) { return (__local void*)x; }
#endif //__HIP_DEVICE_COMPILE__
// Using hip.amdgcn.bc - sync threads
#define __CLK_LOCAL_MEM_FENCE 0x01
typedef unsigned __cl_mem_fence_flags;
typedef enum __memory_scope {
__memory_scope_work_item = __OPENCL_MEMORY_SCOPE_WORK_ITEM,
__memory_scope_work_group = __OPENCL_MEMORY_SCOPE_WORK_GROUP,
__memory_scope_device = __OPENCL_MEMORY_SCOPE_DEVICE,
__memory_scope_all_svm_devices = __OPENCL_MEMORY_SCOPE_ALL_SVM_DEVICES,
__memory_scope_sub_group = __OPENCL_MEMORY_SCOPE_SUB_GROUP
} __memory_scope;
// enum values aligned with what clang uses in EmitAtomicExpr()
typedef enum __memory_order
{
__memory_order_relaxed = __ATOMIC_RELAXED,
__memory_order_acquire = __ATOMIC_ACQUIRE,
__memory_order_release = __ATOMIC_RELEASE,
__memory_order_acq_rel = __ATOMIC_ACQ_REL,
__memory_order_seq_cst = __ATOMIC_SEQ_CST
} __memory_order;
// Linked from hip.amdgcn.bc
extern "C" __device__ void
__atomic_work_item_fence(__cl_mem_fence_flags, __memory_order, __memory_scope);
#endif
projects/hip/src/hipamd/include/hip/amd_detail/functional_grid_launch.hpp
+218
Fájl megtekintése
@@ -0,0 +1,218 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include "concepts.hpp"
#include "helpers.hpp"
#include "program_state.hpp"
#include "hip_runtime_api.h"
#include <cstdint>
#include <cstring>
#include <stdexcept>
#include <tuple>
#include <type_traits>
#include <utility>
hipError_t ihipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList, int numDevices,
unsigned int flags, hip_impl::program_state& ps);
hipError_t hipLaunchCooperativeKernel(const void* f, dim3 gridDim,
dim3 blockDim, void** args,
size_t sharedMem, hipStream_t stream,
hip_impl::program_state& ps);
hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices,
unsigned int flags,
hip_impl::program_state& ps);
#pragma GCC visibility push(hidden)
namespace hip_impl {
template <typename T, typename std::enable_if<std::is_integral<T>{}>::type* = nullptr>
inline T round_up_to_next_multiple_nonnegative(T x, T y) {
T tmp = x + y - 1;
return tmp - tmp % y;
}
template <
std::size_t n,
typename... Ts,
typename std::enable_if<n == sizeof...(Ts)>::type* = nullptr>
inline hip_impl::kernarg make_kernarg(
const std::tuple<Ts...>&,
const kernargs_size_align&,
hip_impl::kernarg kernarg) {
return kernarg;
}
template <
std::size_t n,
typename... Ts,
typename std::enable_if<n != sizeof...(Ts)>::type* = nullptr>
inline hip_impl::kernarg make_kernarg(
const std::tuple<Ts...>& formals,
const kernargs_size_align& size_align,
hip_impl::kernarg kernarg) {
using T = typename std::tuple_element<n, std::tuple<Ts...>>::type;
static_assert(
!std::is_reference<T>{},
"A __global__ function cannot have a reference as one of its "
"arguments.");
#if defined(HIP_STRICT)
static_assert(
std::is_trivially_copyable<T>{},
"Only TriviallyCopyable types can be arguments to a __global__ "
"function");
#endif
kernarg.resize(round_up_to_next_multiple_nonnegative(
kernarg.size(), size_align.alignment(n)) + size_align.size(n));
std::memcpy(
kernarg.data() + kernarg.size() - size_align.size(n),
&std::get<n>(formals),
size_align.size(n));
return make_kernarg<n + 1>(formals, size_align, std::move(kernarg));
}
template <typename... Formals, typename... Actuals>
inline hip_impl::kernarg make_kernarg(
void (*kernel)(Formals...), std::tuple<Actuals...> actuals) {
static_assert(sizeof...(Formals) == sizeof...(Actuals),
"The count of formal arguments must match the count of actuals.");
if (sizeof...(Formals) == 0) return {};
std::tuple<Formals...> to_formals{std::move(actuals)};
hip_impl::kernarg kernarg;
kernarg.reserve(sizeof(to_formals));
auto& ps = hip_impl::get_program_state();
return make_kernarg<0>(to_formals,
ps.get_kernargs_size_align(
reinterpret_cast<std::uintptr_t>(kernel)),
std::move(kernarg));
}
HIP_INTERNAL_EXPORTED_API hsa_agent_t target_agent(hipStream_t stream);
inline
__attribute__((visibility("hidden")))
void hipLaunchKernelGGLImpl(
std::uintptr_t function_address,
const dim3& numBlocks,
const dim3& dimBlocks,
std::uint32_t sharedMemBytes,
hipStream_t stream,
void** kernarg) {
const auto& kd = hip_impl::get_program_state().kernel_descriptor(function_address,
target_agent(stream));
hipModuleLaunchKernel(kd, numBlocks.x, numBlocks.y, numBlocks.z,
dimBlocks.x, dimBlocks.y, dimBlocks.z, sharedMemBytes,
stream, nullptr, kernarg);
}
} // Namespace hip_impl.
template <class T>
inline
hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0) {
using namespace hip_impl;
hip_impl::hip_init();
auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
target_agent(0));
return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
dynSharedMemPerBlk, blockSizeLimit);
}
template <class T>
inline
hipError_t hipOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize,
T kernel, size_t dynSharedMemPerBlk = 0, int blockSizeLimit = 0, unsigned int flags = 0 ) {
using namespace hip_impl;
hip_impl::hip_init();
if(flags != hipOccupancyDefault) return hipErrorNotSupported;
auto f = get_program_state().kernel_descriptor(reinterpret_cast<std::uintptr_t>(kernel),
target_agent(0));
return hipModuleOccupancyMaxPotentialBlockSize(gridSize, blockSize, f,
dynSharedMemPerBlk, blockSizeLimit);
}
template <typename... Args, typename F = void (*)(Args...)>
inline
void hipLaunchKernelGGL(F kernel, const dim3& numBlocks, const dim3& dimBlocks,
std::uint32_t sharedMemBytes, hipStream_t stream,
Args... args) {
hip_impl::hip_init();
auto kernarg = hip_impl::make_kernarg(kernel, std::tuple<Args...>{std::move(args)...});
std::size_t kernarg_size = kernarg.size();
void* config[]{
HIP_LAUNCH_PARAM_BUFFER_POINTER,
kernarg.data(),
HIP_LAUNCH_PARAM_BUFFER_SIZE,
&kernarg_size,
HIP_LAUNCH_PARAM_END};
hip_impl::hipLaunchKernelGGLImpl(reinterpret_cast<std::uintptr_t>(kernel),
numBlocks, dimBlocks, sharedMemBytes,
stream, &config[0]);
}
template <typename F>
inline
__attribute__((visibility("hidden")))
hipError_t hipLaunchCooperativeKernel(F f, dim3 gridDim, dim3 blockDim,
void** args, size_t sharedMem,
hipStream_t stream) {
hip_impl::hip_init();
auto& ps = hip_impl::get_program_state();
return hipLaunchCooperativeKernel(reinterpret_cast<void*>(f), gridDim,
blockDim, args, sharedMem, stream, ps);
}
inline
__attribute__((visibility("hidden")))
hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices,
unsigned int flags) {
hip_impl::hip_init();
auto& ps = hip_impl::get_program_state();
return hipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, ps);
}
#pragma GCC visibility pop
projects/hip/src/hipamd/include/hip/amd_detail/grid_launch.h
+67
Fájl megtekintése
@@ -0,0 +1,67 @@
#pragma once
#include <stdint.h>
#include <hc_defines.h>
#define GRID_LAUNCH_VERSION 20
// Extern definitions
namespace hc{
class completion_future;
class accelerator_view;
}
// 3 dim structure for groups and grids.
typedef struct gl_dim3
{
int x,y,z;
gl_dim3(uint32_t _x=1, uint32_t _y=1, uint32_t _z=1) : x(_x), y(_y), z(_z) {};
} gl_dim3;
typedef enum gl_barrier_bit {
barrier_bit_queue_default,
barrier_bit_none,
barrier_bit_wait,
} gl_barrier_bit;
// grid_launch_parm contains information used to launch the kernel.
typedef struct grid_launch_parm
{
//! Grid dimensions
gl_dim3 grid_dim;
//! Group dimensions
gl_dim3 group_dim;
//! Amount of dynamic group memory to use with the kernel launch.
//! This memory is in addition to the amount used statically in the kernel.
unsigned int dynamic_group_mem_bytes;
//! Control setting of barrier bit on per-packet basis:
//! See gl_barrier_bit description.
//! Placeholder, is not used to control packet dispatch yet
enum gl_barrier_bit barrier_bit;
//! Value of packet fences to apply to launch.
//! The correspond to the value of bits 9:14 in the AQL packet,
//! see HSA_PACKET_HEADER_ACQUIRE_FENCE_SCOPE and hsa_fence_scope_t.
unsigned int launch_fence;
//! Pointer to the accelerator_view where the kernel should execute.
//! If NULL, the default view on the default accelerator is used.
hc::accelerator_view *av;
//! Pointer to the completion_future used to track the status of the command.
//! If NULL, the command does not write status. In this case,
//! synchronization can be enforced with queue-level waits or
//! waiting on younger commands.
hc::completion_future *cf;
grid_launch_parm() = default;
} grid_launch_parm;
extern void init_grid_launch(grid_launch_parm *gl);
projects/hip/src/hipamd/include/hip/amd_detail/grid_launch.hpp
+50
Fájl megtekintése
@@ -0,0 +1,50 @@
#pragma once
#include "grid_launch.h"
#include "hc.hpp"
class grid_launch_parm_cxx : public grid_launch_parm
{
public:
grid_launch_parm_cxx() = default;
// customized serialization: don't need av and cf in kernel
__attribute__((annotate("serialize")))
void __cxxamp_serialize(Kalmar::Serialize& s) const {
s.Append(sizeof(int), &grid_dim.x);
s.Append(sizeof(int), &grid_dim.y);
s.Append(sizeof(int), &grid_dim.z);
s.Append(sizeof(int), &group_dim.x);
s.Append(sizeof(int), &group_dim.y);
s.Append(sizeof(int), &group_dim.z);
}
__attribute__((annotate("user_deserialize")))
grid_launch_parm_cxx(int grid_dim_x, int grid_dim_y, int grid_dim_z,
int group_dim_x, int group_dim_y, int group_dim_z) {
grid_dim.x = grid_dim_x;
grid_dim.y = grid_dim_y;
grid_dim.z = grid_dim_z;
group_dim.x = group_dim_x;
group_dim.y = group_dim_y;
group_dim.z = group_dim_z;
}
};
extern inline void grid_launch_init(grid_launch_parm *lp) {
lp->grid_dim.x = lp->grid_dim.y = lp->grid_dim.z = 1;
lp->group_dim.x = lp->group_dim.y = lp->group_dim.z = 1;
lp->dynamic_group_mem_bytes = 0;
lp->barrier_bit = barrier_bit_queue_default;
lp->launch_fence = -1;
// TODO - set to NULL?
static hc::accelerator_view av = hc::accelerator().get_default_view();
lp->av = &av;
lp->cf = NULL;
}
projects/hip/src/hipamd/include/hip/amd_detail/grid_launch_GGL.hpp
+26
Fájl megtekintése
@@ -0,0 +1,26 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#if GENERIC_GRID_LAUNCH == 1
#include "macro_based_grid_launch.hpp"
#endif // GENERIC_GRID_LAUNCH
projects/hip/src/hipamd/include/hip/amd_detail/helpers.hpp
+137
Fájl megtekintése
@@ -0,0 +1,137 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include "concepts.hpp"
#include <type_traits> // For std::conditional, std::decay, std::enable_if,
// std::false_type, std result_of and std::true_type.
#include <utility> // For std::declval.
#ifdef __has_include // Check if __has_include is present
# if __has_include(<version>) // Check for version header
# include <version>
# if defined(__cpp_lib_is_invocable) && !defined(HIP_HAS_INVOCABLE)
# define HIP_HAS_INVOCABLE __cpp_lib_is_invocable
# endif
# if defined(__cpp_lib_result_of_sfinae) && !defined(HIP_HAS_RESULT_OF_SFINAE)
# define HIP_HAS_RESULT_OF_SFINAE __cpp_lib_result_of_sfinae
# endif
# endif
#endif
#ifndef HIP_HAS_INVOCABLE
#define HIP_HAS_INVOCABLE 0
#endif
#ifndef HIP_HAS_RESULT_OF_SFINAE
#define HIP_HAS_RESULT_OF_SFINAE 0
#endif
namespace std { // TODO: these should be removed as soon as possible.
#if (__cplusplus < 201406L)
#if (__cplusplus < 201402L)
template <bool cond, typename T = void>
using enable_if_t = typename enable_if<cond, T>::type;
template <bool cond, typename T, typename U>
using conditional_t = typename conditional<cond, T, U>::type;
template <typename T>
using decay_t = typename decay<T>::type;
template <FunctionalProcedure F, typename... Ts>
using result_of_t = typename result_of<F(Ts...)>::type;
template <typename T>
using remove_reference_t = typename remove_reference<T>::type;
#endif
#endif
} // namespace std
namespace hip_impl {
template <typename...>
using void_t_ = void;
#if HIP_HAS_INVOCABLE
template <typename, typename = void>
struct is_callable_impl;
template <FunctionalProcedure F, typename... Ts>
struct is_callable_impl<F(Ts...)> : std::is_invocable<F, Ts...> {};
#elif HIP_HAS_RESULT_OF_SFINAE
template <typename, typename = void>
struct is_callable_impl : std::false_type {};
template <FunctionalProcedure F, typename... Ts>
struct is_callable_impl<F(Ts...), void_t_<typename std::result_of<F(Ts...)>::type > > : std::true_type {};
#else
template <class Base, class T, class Derived>
auto simple_invoke(T Base::*pmd, Derived&& ref)
-> decltype(static_cast<Derived&&>(ref).*pmd);
template <class PMD, class Pointer>
auto simple_invoke(PMD&& pmd, Pointer&& ptr)
-> decltype((*static_cast<Pointer&&>(ptr)).*static_cast<PMD&&>(pmd));
template <class Base, class T, class Derived>
auto simple_invoke(T Base::*pmd, const std::reference_wrapper<Derived>& ref)
-> decltype(ref.get().*pmd);
template <class Base, class T, class Derived, class... Args>
auto simple_invoke(T Base::*pmf, Derived&& ref, Args&&... args)
-> decltype((static_cast<Derived&&>(ref).*pmf)(static_cast<Args&&>(args)...));
template <class PMF, class Pointer, class... Args>
auto simple_invoke(PMF&& pmf, Pointer&& ptr, Args&&... args)
-> decltype(((*static_cast<Pointer&&>(ptr)).*static_cast<PMF&&>(pmf))(static_cast<Args&&>(args)...));
template <class Base, class T, class Derived, class... Args>
auto simple_invoke(T Base::*pmf, const std::reference_wrapper<Derived>& ref, Args&&... args)
-> decltype((ref.get().*pmf)(static_cast<Args&&>(args)...));
template<class F, class... Ts>
auto simple_invoke(F&& f, Ts&&... xs)
-> decltype(f(static_cast<Ts&&>(xs)...));
template <typename, typename = void>
struct is_callable_impl : std::false_type {};
template <FunctionalProcedure F, typename... Ts>
struct is_callable_impl<F(Ts...), void_t_<decltype(simple_invoke(std::declval<F>(), std::declval<Ts>()...))> >
: std::true_type {};
#endif
template <typename Call>
struct is_callable : is_callable_impl<Call> {};
#define count_macro_args_impl_hip_(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \
_14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, _25, \
_26, _27, _28, _29, _30, _31, _n, ...) \
_n
#define count_macro_args_hip_(...) \
count_macro_args_impl_hip_(, ##__VA_ARGS__, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, \
19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, \
0)
#define overloaded_macro_expand_hip_(macro, arg_cnt) macro##arg_cnt
#define overload_macro_impl_hip_(macro, arg_cnt) overloaded_macro_expand_hip_(macro, arg_cnt)
#define overload_macro_hip_(macro, ...) \
overload_macro_impl_hip_(macro, count_macro_args_hip_(__VA_ARGS__))(__VA_ARGS__)
} // namespace hip_impl
projects/hip/src/hipamd/include/hip/amd_detail/hip_cooperative_groups_helper.h
+179
Fájl megtekintése
@@ -0,0 +1,179 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/hip_cooperative_groups_helper.h
*
* @brief Device side implementation of cooperative group feature.
*
* Defines helper constructs and APIs which aid the types and device API
* wrappers defined within `amd_detail/hip_cooperative_groups.h`.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
#if __cplusplus
#include <hip/amd_detail/amd_device_functions.h>
#if !defined(__align__)
#define __align__(x) __attribute__((aligned(x)))
#endif
#if !defined(__CG_QUALIFIER__)
#define __CG_QUALIFIER__ __device__ __forceinline__
#endif
#if !defined(__CG_STATIC_QUALIFIER__)
#define __CG_STATIC_QUALIFIER__ __device__ static __forceinline__
#endif
#if !defined(_CG_STATIC_CONST_DECL_)
#define _CG_STATIC_CONST_DECL_ static constexpr
#endif
#if !defined(WAVEFRONT_SIZE)
#if __gfx1010__ || __gfx1011__ || __gfx1012__ || __gfx1030__ || __gfx1031__
#define WAVEFRONT_SIZE 32
#else
#define WAVEFRONT_SIZE 64
#endif
namespace cooperative_groups {
/* Global scope */
template <unsigned int size>
using is_power_of_2 = std::integral_constant<bool, (size & (size - 1)) == 0>;
template <unsigned int size>
using is_valid_wavefront = std::integral_constant<bool, (size <= WAVEFRONT_SIZE)>;
template <unsigned int size>
using is_valid_tile_size =
std::integral_constant<bool, is_power_of_2<size>::value && is_valid_wavefront<size>::value>;
template <typename T>
using is_valid_type =
std::integral_constant<bool, std::is_integral<T>::value || std::is_floating_point<T>::value>;
namespace internal {
/** \brief Enums representing different cooperative group types
*/
typedef enum {
cg_invalid,
cg_multi_grid,
cg_grid,
cg_workgroup,
cg_tiled_group
} group_type;
/**
* Functionalities related to multi-grid cooperative group type
*/
namespace multi_grid {
__CG_STATIC_QUALIFIER__ uint32_t num_grids() { return (uint32_t)__ockl_multi_grid_num_grids(); }
__CG_STATIC_QUALIFIER__ uint32_t grid_rank() { return (uint32_t)__ockl_multi_grid_grid_rank(); }
__CG_STATIC_QUALIFIER__ uint32_t size() { return (uint32_t)__ockl_multi_grid_size(); }
__CG_STATIC_QUALIFIER__ uint32_t thread_rank() { return (uint32_t)__ockl_multi_grid_thread_rank(); }
__CG_STATIC_QUALIFIER__ bool is_valid() { return (bool)__ockl_multi_grid_is_valid(); }
__CG_STATIC_QUALIFIER__ void sync() { __ockl_multi_grid_sync(); }
} // namespace multi_grid
/**
* Functionalities related to grid cooperative group type
*/
namespace grid {
__CG_STATIC_QUALIFIER__ uint32_t size() {
return (uint32_t)((hipBlockDim_z * hipGridDim_z) * (hipBlockDim_y * hipGridDim_y) *
(hipBlockDim_x * hipGridDim_x));
}
__CG_STATIC_QUALIFIER__ uint32_t thread_rank() {
// Compute global id of the workgroup to which the current thread belongs to
uint32_t blkIdx = (uint32_t)((hipBlockIdx_z * hipGridDim_y * hipGridDim_x) +
(hipBlockIdx_y * hipGridDim_x) + (hipBlockIdx_x));
// Compute total number of threads being passed to reach current workgroup
// within grid
uint32_t num_threads_till_current_workgroup =
(uint32_t)(blkIdx * (hipBlockDim_x * hipBlockDim_y * hipBlockDim_z));
// Compute thread local rank within current workgroup
uint32_t local_thread_rank = (uint32_t)((hipThreadIdx_z * hipBlockDim_y * hipBlockDim_x) +
(hipThreadIdx_y * hipBlockDim_x) + (hipThreadIdx_x));
return (num_threads_till_current_workgroup + local_thread_rank);
}
__CG_STATIC_QUALIFIER__ bool is_valid() { return (bool)__ockl_grid_is_valid(); }
__CG_STATIC_QUALIFIER__ void sync() { __ockl_grid_sync(); }
} // namespace grid
/**
* Functionalities related to `workgroup` (thread_block in CUDA terminology)
* cooperative group type
*/
namespace workgroup {
__CG_STATIC_QUALIFIER__ dim3 group_index() {
return (dim3((uint32_t)hipBlockIdx_x, (uint32_t)hipBlockIdx_y, (uint32_t)hipBlockIdx_z));
}
__CG_STATIC_QUALIFIER__ dim3 thread_index() {
return (dim3((uint32_t)hipThreadIdx_x, (uint32_t)hipThreadIdx_y, (uint32_t)hipThreadIdx_z));
}
__CG_STATIC_QUALIFIER__ uint32_t size() {
return ((uint32_t)(hipBlockDim_x * hipBlockDim_y * hipBlockDim_z));
}
__CG_STATIC_QUALIFIER__ uint32_t thread_rank() {
return ((uint32_t)((hipThreadIdx_z * hipBlockDim_y * hipBlockDim_x) +
(hipThreadIdx_y * hipBlockDim_x) + (hipThreadIdx_x)));
}
__CG_STATIC_QUALIFIER__ bool is_valid() {
// TODO(mahesha) any functionality need to be added here? I believe not
return true;
}
__CG_STATIC_QUALIFIER__ void sync() { __syncthreads(); }
} // namespace workgroup
} // namespace internal
} // namespace cooperative_groups
#endif // __cplusplus
#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_COOPERATIVE_GROUPS_HELPER_H
#endif
projects/hip/src/hipamd/include/hip/amd_detail/hip_fp16_gcc.h
+254
Fájl megtekintése
@@ -0,0 +1,254 @@
#pragma once
#if defined(__cplusplus)
#include <cstring>
#endif
struct __half_raw {
unsigned short x;
};
struct __half2_raw {
unsigned short x;
unsigned short y;
};
#if defined(__cplusplus)
struct __half;
__half __float2half(float);
float __half2float(__half);
// BEGIN STRUCT __HALF
struct __half {
protected:
unsigned short __x;
public:
// CREATORS
__half() = default;
__half(const __half_raw& x) : __x{x.x} {}
#if !defined(__HIP_NO_HALF_CONVERSIONS__)
__half(float x) : __x{__float2half(x).__x} {}
__half(double x) : __x{__float2half(x).__x} {}
#endif
__half(const __half&) = default;
__half(__half&&) = default;
~__half() = default;
// MANIPULATORS
__half& operator=(const __half&) = default;
__half& operator=(__half&&) = default;
__half& operator=(const __half_raw& x) { __x = x.x; return *this; }
#if !defined(__HIP_NO_HALF_CONVERSIONS__)
__half& operator=(float x)
{
__x = __float2half(x).__x;
return *this;
}
__half& operator=(double x)
{
return *this = static_cast<float>(x);
}
#endif
// ACCESSORS
operator float() const { return __half2float(*this); }
operator __half_raw() const { return __half_raw{__x}; }
};
// END STRUCT __HALF
// BEGIN STRUCT __HALF2
struct __half2 {
public:
__half x;
__half y;
// CREATORS
__half2() = default;
__half2(const __half2_raw& ix)
:
x{reinterpret_cast<const __half&>(ix.x)},
y{reinterpret_cast<const __half&>(ix.y)}
{}
__half2(const __half& ix, const __half& iy) : x{ix}, y{iy} {}
__half2(const __half2&) = default;
__half2(__half2&&) = default;
~__half2() = default;
// MANIPULATORS
__half2& operator=(const __half2&) = default;
__half2& operator=(__half2&&) = default;
__half2& operator=(const __half2_raw& ix)
{
x = reinterpret_cast<const __half_raw&>(ix.x);
y = reinterpret_cast<const __half_raw&>(ix.y);
return *this;
}
// ACCESSORS
operator __half2_raw() const
{
return __half2_raw{
reinterpret_cast<const unsigned short&>(x),
reinterpret_cast<const unsigned short&>(y)};
}
};
// END STRUCT __HALF2
inline
unsigned short __internal_float2half(
float flt, unsigned int& sgn, unsigned int& rem)
{
unsigned int x{};
std::memcpy(&x, &flt, sizeof(flt));
unsigned int u = (x & 0x7fffffffU);
sgn = ((x >> 16) & 0x8000U);
// NaN/+Inf/-Inf
if (u >= 0x7f800000U) {
rem = 0;
return static_cast<unsigned short>(
(u == 0x7f800000U) ? (sgn | 0x7c00U) : 0x7fffU);
}
// Overflows
if (u > 0x477fefffU) {
rem = 0x80000000U;
return static_cast<unsigned short>(sgn | 0x7bffU);
}
// Normal numbers
if (u >= 0x38800000U) {
rem = u << 19;
u -= 0x38000000U;
return static_cast<unsigned short>(sgn | (u >> 13));
}
// +0/-0
if (u < 0x33000001U) {
rem = u;
return static_cast<unsigned short>(sgn);
}
// Denormal numbers
unsigned int exponent = u >> 23;
unsigned int mantissa = (u & 0x7fffffU);
unsigned int shift = 0x7eU - exponent;
mantissa |= 0x800000U;
rem = mantissa << (32 - shift);
return static_cast<unsigned short>(sgn | (mantissa >> shift));
}
inline
__half __float2half(float x)
{
__half_raw r;
unsigned int sgn{};
unsigned int rem{};
r.x = __internal_float2half(x, sgn, rem);
if (rem > 0x80000000U || (rem == 0x80000000U && (r.x & 0x1))) ++r.x;
return r;
}
inline
__half __float2half_rn(float x) { return __float2half(x); }
inline
__half __float2half_rz(float x)
{
__half_raw r;
unsigned int sgn{};
unsigned int rem{};
r.x = __internal_float2half(x, sgn, rem);
return r;
}
inline
__half __float2half_rd(float x)
{
__half_raw r;
unsigned int sgn{};
unsigned int rem{};
r.x = __internal_float2half(x, sgn, rem);
if (rem && sgn) ++r.x;
return r;
}
inline
__half __float2half_ru(float x)
{
__half_raw r;
unsigned int sgn{};
unsigned int rem{};
r.x = __internal_float2half(x, sgn, rem);
if (rem && !sgn) ++r.x;
return r;
}
inline
__half2 __float2half2_rn(float x)
{
return __half2{__float2half_rn(x), __float2half_rn(x)};
}
inline
__half2 __floats2half2_rn(float x, float y)
{
return __half2{__float2half_rn(x), __float2half_rn(y)};
}
inline
float __internal_half2float(unsigned short x)
{
unsigned int sign = ((x >> 15) & 1);
unsigned int exponent = ((x >> 10) & 0x1f);
unsigned int mantissa = ((x & 0x3ff) << 13);
if (exponent == 0x1fU) { /* NaN or Inf */
mantissa = (mantissa ? (sign = 0, 0x7fffffU) : 0);
exponent = 0xffU;
} else if (!exponent) { /* Denorm or Zero */
if (mantissa) {
unsigned int msb;
exponent = 0x71U;
do {
msb = (mantissa & 0x400000U);
mantissa <<= 1; /* normalize */
--exponent;
} while (!msb);
mantissa &= 0x7fffffU; /* 1.mantissa is implicit */
}
} else {
exponent += 0x70U;
}
unsigned int u = ((sign << 31) | (exponent << 23) | mantissa);
float f;
memcpy(&f, &u, sizeof(u));
return f;
}
inline
float __half2float(__half x)
{
return __internal_half2float(static_cast<__half_raw>(x).x);
}
inline
float __low2float(__half2 x)
{
return __internal_half2float(static_cast<__half2_raw>(x).x);
}
inline
float __high2float(__half2 x)
{
return __internal_half2float(static_cast<__half2_raw>(x).y);
}
#if !defined(HIP_NO_HALF)
using half = __half;
using half2 = __half2;
#endif
#endif // defined(__cplusplus)
projects/hip/src/hipamd/include/hip/amd_detail/hip_fp16_math_fwd.h
+86
Fájl megtekintése
@@ -0,0 +1,86 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
// /*
// Half Math Functions
// */
#include "host_defines.h"
#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
extern "C"
{
__device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
__device__ _Float16 __ocml_cos_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_exp_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_exp10_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_exp2_f16(_Float16);
__device__ __attribute__((const)) _Float16 __ocml_floor_f16(_Float16);
__device__ __attribute__((const))
_Float16 __ocml_fma_f16(_Float16, _Float16, _Float16);
__device__ __attribute__((const)) _Float16 __ocml_fabs_f16(_Float16);
__device__ __attribute__((const)) int __ocml_isinf_f16(_Float16);
__device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
__device__ __attribute__((pure)) _Float16 __ocml_pown_f16(_Float16, int);
__device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16);
__device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
__device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
__device__ _Float16 __ocml_sin_f16(_Float16);
__device__ __attribute__((const)) _Float16 __ocml_sqrt_f16(_Float16);
__device__ __attribute__((const)) _Float16 __ocml_trunc_f16(_Float16);
typedef _Float16 __2f16 __attribute__((ext_vector_type(2)));
typedef short __2i16 __attribute__((ext_vector_type(2)));
#if __HIP_CLANG_ONLY__
__device__ __attribute__((const)) float __ockl_fdot2(__2f16 a, __2f16 b, float c, bool s);
#endif
__device__ __attribute__((const)) __2f16 __ocml_ceil_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_fabs_2f16(__2f16);
__device__ __2f16 __ocml_cos_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_exp_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_exp10_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_exp2_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_floor_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_fma_2f16(__2f16, __2f16, __2f16);
__device__ __attribute__((const)) __2i16 __ocml_isinf_2f16(__2f16);
__device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
__device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
__device__ inline
__2f16 __llvm_amdgcn_rcp_2f16(__2f16 x) // Not currently exposed by ROCDL.
{
return __2f16{__llvm_amdgcn_rcp_f16(x.x), __llvm_amdgcn_rcp_f16(x.y)};
}
__device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
__device__ __2f16 __ocml_sin_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_sqrt_2f16(__2f16);
__device__ __attribute__((const)) __2f16 __ocml_trunc_2f16(__2f16);
}
#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
projects/hip/src/hipamd/include/hip/amd_detail/hip_ldg.h
+100
Fájl megtekintése
@@ -0,0 +1,100 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_LDG_H
#if __HIP_CLANG_ONLY__
#include "amd_hip_vector_types.h"
#include "host_defines.h"
__device__ inline static char __ldg(const char* ptr) { return *ptr; }
__device__ inline static char2 __ldg(const char2* ptr) { return *ptr; }
__device__ inline static char4 __ldg(const char4* ptr) { return *ptr; }
__device__ inline static signed char __ldg(const signed char* ptr) { return ptr[0]; }
__device__ inline static unsigned char __ldg(const unsigned char* ptr) { return ptr[0]; }
__device__ inline static short __ldg(const short* ptr) { return ptr[0]; }
__device__ inline static short2 __ldg(const short2* ptr) { return ptr[0]; }
__device__ inline static short4 __ldg(const short4* ptr) { return ptr[0]; }
__device__ inline static unsigned short __ldg(const unsigned short* ptr) { return ptr[0]; }
__device__ inline static int __ldg(const int* ptr) { return ptr[0]; }
__device__ inline static int2 __ldg(const int2* ptr) { return ptr[0]; }
__device__ inline static int4 __ldg(const int4* ptr) { return ptr[0]; }
__device__ inline static unsigned int __ldg(const unsigned int* ptr) { return ptr[0]; }
__device__ inline static long __ldg(const long* ptr) { return ptr[0]; }
__device__ inline static unsigned long __ldg(const unsigned long* ptr) { return ptr[0]; }
__device__ inline static long long __ldg(const long long* ptr) { return ptr[0]; }
__device__ inline static longlong2 __ldg(const longlong2* ptr) { return ptr[0]; }
__device__ inline static unsigned long long __ldg(const unsigned long long* ptr) { return ptr[0]; }
__device__ inline static uchar2 __ldg(const uchar2* ptr) { return ptr[0]; }
__device__ inline static uchar4 __ldg(const uchar4* ptr) { return ptr[0]; }
__device__ inline static ushort2 __ldg(const ushort2* ptr) { return ptr[0]; }
__device__ inline static uint2 __ldg(const uint2* ptr) { return ptr[0]; }
__device__ inline static uint4 __ldg(const uint4* ptr) { return ptr[0]; }
__device__ inline static ulonglong2 __ldg(const ulonglong2* ptr) { return ptr[0]; }
__device__ inline static float __ldg(const float* ptr) { return ptr[0]; }
__device__ inline static float2 __ldg(const float2* ptr) { return ptr[0]; }
__device__ inline static float4 __ldg(const float4* ptr) { return ptr[0]; }
__device__ inline static double __ldg(const double* ptr) { return ptr[0]; }
__device__ inline static double2 __ldg(const double2* ptr) { return ptr[0]; }
#endif // __HIP_CLANG_ONLY__
#endif // HIP_LDG_H
projects/hip/src/hipamd/include/hip/amd_detail/hip_memory.h
+114
Fájl megtekintése
@@ -0,0 +1,114 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H
// Implementation of malloc and free device functions.
// HIP heap is implemented as a global array with fixed size. Users may define
// __HIP_SIZE_OF_PAGE and __HIP_NUM_PAGES to have a larger heap.
#if __HIP__ && __HIP_ENABLE_DEVICE_MALLOC__
// Size of page in bytes.
#ifndef __HIP_SIZE_OF_PAGE
#define __HIP_SIZE_OF_PAGE 64
#endif
// Total number of pages
#ifndef __HIP_NUM_PAGES
#define __HIP_NUM_PAGES (16 * 64 * 64)
#endif
#define __HIP_SIZE_OF_HEAP (__HIP_NUM_PAGES * __HIP_SIZE_OF_PAGE)
#if __HIP_DEVICE_COMPILE__
__attribute__((weak)) __device__ char __hip_device_heap[__HIP_SIZE_OF_HEAP];
__attribute__((weak)) __device__
uint32_t __hip_device_page_flag[__HIP_NUM_PAGES];
#else
extern __device__ char __hip_device_heap[];
extern __device__ uint32_t __hip_device_page_flag[];
#endif
extern "C" inline __device__ void* __hip_malloc(size_t size) {
char* heap = (char*)__hip_device_heap;
if (size > __HIP_SIZE_OF_HEAP) {
return (void*)nullptr;
}
uint32_t totalThreads =
hipBlockDim_x * hipGridDim_x * hipBlockDim_y
* hipGridDim_y * hipBlockDim_z * hipGridDim_z;
uint32_t currentWorkItem = hipThreadIdx_x + hipBlockDim_x * hipBlockIdx_x
+ (hipThreadIdx_y + hipBlockDim_y * hipBlockIdx_y) * hipBlockDim_x
+ (hipThreadIdx_z + hipBlockDim_z * hipBlockIdx_z) * hipBlockDim_x
* hipBlockDim_y;
uint32_t numHeapsPerWorkItem = __HIP_NUM_PAGES / totalThreads;
uint32_t heapSizePerWorkItem = __HIP_SIZE_OF_HEAP / totalThreads;
uint32_t stride = size / __HIP_SIZE_OF_PAGE;
uint32_t start = numHeapsPerWorkItem * currentWorkItem;
uint32_t k = 0;
while (__hip_device_page_flag[k] > 0) {
k++;
}
for (uint32_t i = 0; i < stride - 1; i++) {
__hip_device_page_flag[i + start + k] = 1;
}
__hip_device_page_flag[start + stride - 1 + k] = 2;
void* ptr = (void*)(heap
+ heapSizePerWorkItem * currentWorkItem + k * __HIP_SIZE_OF_PAGE);
return ptr;
}
extern "C" inline __device__ void* __hip_free(void* ptr) {
if (ptr == nullptr) {
return nullptr;
}
uint32_t offsetByte = (uint64_t)ptr - (uint64_t)__hip_device_heap;
uint32_t offsetPage = offsetByte / __HIP_SIZE_OF_PAGE;
while (__hip_device_page_flag[offsetPage] != 0) {
if (__hip_device_page_flag[offsetPage] == 2) {
__hip_device_page_flag[offsetPage] = 0;
offsetPage++;
break;
} else {
__hip_device_page_flag[offsetPage] = 0;
offsetPage++;
}
}
return nullptr;
}
#endif
#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_MEMORY_H
projects/hip/src/hipamd/include/hip/amd_detail/hip_runtime_prof.h
+77
Fájl megtekintése
@@ -0,0 +1,77 @@
/*
Copyright (c) 2019 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
// HIP ROCclr Op IDs enumeration
enum HipVdiOpId {
kHipVdiOpIdDispatch = 0,
kHipVdiOpIdCopy = 1,
kHipVdiOpIdBarrier = 2,
kHipVdiOpIdNumber = 3
};
// Types of ROCclr commands
enum HipVdiCommandKind {
kHipVdiCommandKernel = 0x11F0,
kHipVdiMemcpyDeviceToHost = 0x11F3,
kHipHipVdiMemcpyHostToDevice = 0x11F4,
kHipVdiMemcpyDeviceToDevice = 0x11F5,
kHipVidMemcpyDeviceToHostRect = 0x1201,
kHipVdiMemcpyHostToDeviceRect = 0x1202,
kHipVdiMemcpyDeviceToDeviceRect = 0x1203,
kHipVdiFillMemory = 0x1207,
};
/**
* @brief Initializes activity callback
*
* @param [input] id_callback Event ID callback function
* @param [input] op_callback Event operation callback function
* @param [input] arg Arguments passed into callback
*
* @returns None
*/
void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg);
/**
* @brief Enables activity callback
*
* @param [input] op Operation, which will trigger a callback (@see HipVdiOpId)
* @param [input] enable Enable state for the callback
*
* @returns True if successful
*/
bool hipEnableActivityCallback(uint32_t op, bool enable);
/**
* @brief Returns the description string for the operation kind
*
* @param [input] id Command kind id (@see HipVdiCommandKind)
*
* @returns A pointer to a const string with the command description
*/
const char* hipGetCmdName(uint32_t id);
#endif // HIP_INCLUDE_HIP_AMD_DETAIL_HIP_RUNTIME_PROF_H
projects/hip/src/hipamd/include/hip/amd_detail/host_defines.h
+72
Fájl megtekintése
@@ -0,0 +1,72 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/host_defines.h
* @brief TODO-doc
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_HOST_DEFINES_H
// The follow macro should be removed after upstream updation.
// It's defined here for workarround of rocThrust building failure.
#define HIP_INCLUDE_HIP_HCC_DETAIL_HOST_DEFINES_H
// Add guard to Generic Grid Launch method
#ifndef GENERIC_GRID_LAUNCH
#define GENERIC_GRID_LAUNCH 1
#endif
#if defined(__clang__) && defined(__HIP__)
#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#define __host__ __attribute__((host))
#define __device__ __attribute__((device))
#define __global__ __attribute__((global))
#define __shared__ __attribute__((shared))
#define __constant__ __attribute__((constant))
#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#define __noinline__ __attribute__((noinline))
#define __forceinline__ inline __attribute__((always_inline))
#else
// Non-HCC compiler
/**
* Function and kernel markers
*/
#define __host__
#define __device__
#define __global__
#define __noinline__
#define __forceinline__ inline
#define __shared__
#define __constant__
#endif
#endif
projects/hip/src/hipamd/include/hip/amd_detail/hsa_helpers.hpp
+102
Fájl megtekintése
@@ -0,0 +1,102 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include <hsa/hsa.h>
#include <cstdint>
#include <functional>
#include <string>
namespace hip_impl {
inline void* address(hsa_executable_symbol_t x) {
void* r = nullptr;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_ADDRESS, &r);
return r;
}
inline hsa_agent_t agent(hsa_executable_symbol_t x) {
hsa_agent_t r = {};
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_AGENT, &r);
return r;
}
inline std::uint32_t group_size(hsa_executable_symbol_t x) {
std::uint32_t r = 0u;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_GROUP_SEGMENT_SIZE, &r);
return r;
}
inline hsa_isa_t isa(hsa_agent_t x) {
hsa_isa_t r = {};
hsa_agent_iterate_isas(x,
[](hsa_isa_t i, void* o) {
*static_cast<hsa_isa_t*>(o) = i; // Pick the first.
return HSA_STATUS_INFO_BREAK;
},
&r);
return r;
}
inline std::uint64_t kernel_object(hsa_executable_symbol_t x) {
std::uint64_t r = 0u;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_OBJECT, &r);
return r;
}
inline std::string name(hsa_executable_symbol_t x) {
std::uint32_t sz = 0u;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME_LENGTH, &sz);
std::string r(sz, '\0');
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_NAME, &r.front());
return r;
}
inline std::uint32_t private_size(hsa_executable_symbol_t x) {
std::uint32_t r = 0u;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_KERNEL_PRIVATE_SEGMENT_SIZE, &r);
return r;
}
inline std::uint32_t size(hsa_executable_symbol_t x) {
std::uint32_t r = 0;
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_VARIABLE_SIZE, &r);
return r;
}
inline hsa_symbol_kind_t type(hsa_executable_symbol_t x) {
hsa_symbol_kind_t r = {};
hsa_executable_symbol_get_info(x, HSA_EXECUTABLE_SYMBOL_INFO_TYPE, &r);
return r;
}
} // namespace hip_impl
projects/hip/src/hipamd/include/hip/amd_detail/llvm_intrinsics.h
+41
Fájl megtekintése
@@ -0,0 +1,41 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
/**
* @file amd_detail/llvm_intrinsics.h
* @brief Contains declarations for wrapper functions for llvm intrinsics
* like llvm.amdgcn.s.barrier.
*/
#ifndef HIP_INCLUDE_HIP_AMD_DETAIL_LLVM_INTRINSICS_H
#define HIP_INCLUDE_HIP_AMD_DETAIL_LLVM_INTRINSICS_H
#include "hip/amd_detail/host_defines.h"
// FIXME: These should all be removed and proper builtins used.
__device__
unsigned __llvm_amdgcn_groupstaticsize() __asm("llvm.amdgcn.groupstaticsize");
__device__
int __llvm_amdgcn_ds_swizzle(int index, int pattern) __asm("llvm.amdgcn.ds.swizzle");
#endif
projects/hip/src/hipamd/include/hip/amd_detail/macro_based_grid_launch.hpp
+798
Fájl megtekintése
@@ -0,0 +1,798 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include "concepts.hpp"
#include "helpers.hpp"
#include "hc.hpp"
#include "hip/hip_ext.h"
#include "hip_runtime.h"
#include <functional>
#include <iostream>
#include <stdexcept>
#include <type_traits>
#include <utility>
namespace hip_impl {
namespace {
struct New_grid_launch_tag {};
struct Old_grid_launch_tag {};
template <typename C, typename D>
class RAII_guard {
D dtor_;
public:
RAII_guard() = default;
RAII_guard(const C& ctor, D dtor) : dtor_{std::move(dtor)} { ctor(); }
RAII_guard(const RAII_guard&) = default;
RAII_guard(RAII_guard&&) = default;
RAII_guard& operator=(const RAII_guard&) = default;
RAII_guard& operator=(RAII_guard&&) = default;
~RAII_guard() { dtor_(); }
};
template <typename C, typename D>
RAII_guard<C, D> make_RAII_guard(const C& ctor, D dtor) {
return RAII_guard<C, D>{ctor, std::move(dtor)};
}
template <FunctionalProcedure F, typename... Ts>
using is_new_grid_launch_t = typename std::conditional<is_callable<F(Ts...)>{}, New_grid_launch_tag,
Old_grid_launch_tag>::type;
} // namespace
// TODO: - dispatch rank should be derived from the domain dimensions passed
// in, and not always assumed to be 3;
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> ==
{Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag, dim3 num_blocks,
dim3 dim_blocks, int group_mem_bytes,
const hc::accelerator_view& acc_v, K k) {
const auto d =
hc::extent<3>{num_blocks.z * dim_blocks.z, num_blocks.y * dim_blocks.y,
num_blocks.x * dim_blocks.x}
.tile_with_dynamic(dim_blocks.z, dim_blocks.y, dim_blocks.x, group_mem_bytes);
try {
hc::parallel_for_each(acc_v, d, k);
} catch (std::exception& ex) {
std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
hip_throw(ex);
}
}
// TODO: these are workarounds, they should be removed.
hc::accelerator_view lock_stream_hip_(hipStream_t&, void*&);
void print_prelaunch_trace_(const char*, dim3, dim3, int, hipStream_t);
void unlock_stream_hip_(hipStream_t, void*, const char*, hc::accelerator_view*);
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> == {Ts...}) inline void grid_launch_hip_impl_(New_grid_launch_tag,
dim3 num_blocks, dim3 dim_blocks,
int group_mem_bytes,
hipStream_t stream,
const char* kernel_name, K k) {
void* lck_stream = nullptr;
auto acc_v = lock_stream_hip_(stream, lck_stream);
auto stream_guard =
make_RAII_guard(std::bind(print_prelaunch_trace_, kernel_name, num_blocks, dim_blocks,
group_mem_bytes, stream),
std::bind(unlock_stream_hip_, stream, lck_stream, kernel_name, &acc_v));
try {
grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
group_mem_bytes, acc_v, std::move(k));
} catch (std::exception& ex) {
std::cerr << "Failed in " << __func__ << ", with exception: " << ex.what() << std::endl;
hip_throw(ex);
}
}
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> ==
{hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(Old_grid_launch_tag,
dim3 num_blocks, dim3 dim_blocks,
int group_mem_bytes,
hipStream_t stream, K k) {
grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
group_mem_bytes, std::move(stream), std::move(k));
}
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> == {hipLaunchParm, Ts...}) inline void grid_launch_hip_impl_(
Old_grid_launch_tag, dim3 num_blocks, dim3 dim_blocks, int group_mem_bytes, hipStream_t stream,
const char* kernel_name, K k) {
grid_launch_hip_impl_(New_grid_launch_tag{}, std::move(num_blocks), std::move(dim_blocks),
group_mem_bytes, std::move(stream), kernel_name, std::move(k));
}
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> == {Ts...}) inline std::enable_if_t<
!std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
int group_mem_bytes, hipStream_t stream,
const char* kernel_name, K k) {
grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
std::move(dim_blocks), group_mem_bytes, std::move(stream), kernel_name,
std::move(k));
}
template <FunctionalProcedure K, typename... Ts>
requires(Domain<K> == {Ts...}) inline std::enable_if_t<
!std::is_function<K>::value> grid_launch_hip_(dim3 num_blocks, dim3 dim_blocks,
int group_mem_bytes, hipStream_t stream, K k) {
grid_launch_hip_impl_(is_new_grid_launch_t<K, Ts...>{}, std::move(num_blocks),
std::move(dim_blocks), group_mem_bytes, std::move(stream), std::move(k));
}
// TODO: these are temporary and purposefully noisy and disruptive.
#define make_kernel_name_hip(k, n) \
HIP_kernel_functor_name_begin##_##k##_##HIP_kernel_functor_name_end##_##n
#define make_kernel_functor_hip_30(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22, p23, p24, p25, p26, p27) \
struct make_kernel_name_hip(function_name, 28) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
std::decay_t<decltype(p23)> _p23_; \
std::decay_t<decltype(p24)> _p24_; \
std::decay_t<decltype(p25)> _p25_; \
std::decay_t<decltype(p26)> _p26_; \
std::decay_t<decltype(p27)> _p27_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_, _p23_, _p24_, _p25_, _p26_, _p27_); \
} \
}
#define make_kernel_functor_hip_29(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22, p23, p24, p25, p26) \
struct make_kernel_name_hip(function_name, 27) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
std::decay_t<decltype(p23)> _p23_; \
std::decay_t<decltype(p24)> _p24_; \
std::decay_t<decltype(p25)> _p25_; \
std::decay_t<decltype(p26)> _p26_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_, _p23_, _p24_, _p25_, _p26_); \
} \
}
#define make_kernel_functor_hip_28(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22, p23, p24, p25) \
struct make_kernel_name_hip(function_name, 26) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
std::decay_t<decltype(p23)> _p23_; \
std::decay_t<decltype(p24)> _p24_; \
std::decay_t<decltype(p25)> _p25_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_, _p23_, _p24_, _p25_); \
} \
}
#define make_kernel_functor_hip_27(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22, p23, p24) \
struct make_kernel_name_hip(function_name, 25) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
std::decay_t<decltype(p23)> _p23_; \
std::decay_t<decltype(p24)> _p24_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_, _p23_, _p24_); \
} \
}
#define make_kernel_functor_hip_26(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22, p23) \
struct make_kernel_name_hip(function_name, 24) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
std::decay_t<decltype(p23)> _p23_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_, _p23_); \
} \
}
#define make_kernel_functor_hip_25(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21, \
p22) \
struct make_kernel_name_hip(function_name, 23) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
std::decay_t<decltype(p22)> _p22_; \
__attribute__((used, flatten)) void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_, \
_p22_); \
} \
}
#define make_kernel_functor_hip_24(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20, p21) \
struct make_kernel_name_hip(function_name, 22) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
std::decay_t<decltype(p21)> _p21_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_, _p21_); \
} \
}
#define make_kernel_functor_hip_23(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19, p20) \
struct make_kernel_name_hip(function_name, 21) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
std::decay_t<decltype(p20)> _p20_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_, _p20_); \
} \
}
#define make_kernel_functor_hip_22(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, p19) \
struct make_kernel_name_hip(function_name, 20) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
std::decay_t<decltype(p19)> _p19_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_, _p19_); \
} \
}
#define make_kernel_functor_hip_21(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17, p18) \
struct make_kernel_name_hip(function_name, 19) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
std::decay_t<decltype(p18)> _p18_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_, _p18_); \
} \
}
#define make_kernel_functor_hip_20(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16, p17) \
struct make_kernel_name_hip(function_name, 18) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
std::decay_t<decltype(p17)> _p17_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_, _p17_); \
} \
}
#define make_kernel_functor_hip_19(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15, p16) \
struct make_kernel_name_hip(function_name, 17) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
std::decay_t<decltype(p16)> _p16_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_, _p16_); \
} \
}
#define make_kernel_functor_hip_18(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14, p15) \
struct make_kernel_name_hip(function_name, 16) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
std::decay_t<decltype(p15)> _p15_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_, _p15_); \
} \
}
#define make_kernel_functor_hip_17(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13, p14) \
struct make_kernel_name_hip(function_name, 15) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
std::decay_t<decltype(p14)> _p14_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_, _p14_); \
} \
}
#define make_kernel_functor_hip_16(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12, p13) \
struct make_kernel_name_hip(function_name, 14) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
std::decay_t<decltype(p13)> _p13_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_, _p13_); \
} \
}
#define make_kernel_functor_hip_15(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11, p12) \
struct make_kernel_name_hip(function_name, 13) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
std::decay_t<decltype(p12)> _p12_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_, \
_p12_); \
} \
}
#define make_kernel_functor_hip_14(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10, p11) \
struct make_kernel_name_hip(function_name, 12) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
std::decay_t<decltype(p11)> _p11_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_, _p11_); \
} \
}
#define make_kernel_functor_hip_13(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9, p10) \
struct make_kernel_name_hip(function_name, 11) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
std::decay_t<decltype(p10)> _p10_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { \
kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_, _p10_); \
} \
}
#define make_kernel_functor_hip_12(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8, \
p9) \
struct make_kernel_name_hip(function_name, 10) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
std::decay_t<decltype(p9)> _p9_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_, _p9_); } \
}
#define make_kernel_functor_hip_11(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7, p8) \
struct make_kernel_name_hip(function_name, 9) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
std::decay_t<decltype(p8)> _p8_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_, _p8_); } \
}
#define make_kernel_functor_hip_10(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6, p7) \
struct make_kernel_name_hip(function_name, 8) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
std::decay_t<decltype(p7)> _p7_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_, _p7_); } \
}
#define make_kernel_functor_hip_9(function_name, kernel_name, p0, p1, p2, p3, p4, p5, p6) \
struct make_kernel_name_hip(function_name, 7) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
std::decay_t<decltype(p6)> _p6_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_, _p6_); } \
}
#define make_kernel_functor_hip_8(function_name, kernel_name, p0, p1, p2, p3, p4, p5) \
struct make_kernel_name_hip(function_name, 6) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
std::decay_t<decltype(p5)> _p5_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_, _p5_); } \
}
#define make_kernel_functor_hip_7(function_name, kernel_name, p0, p1, p2, p3, p4) \
struct make_kernel_name_hip(function_name, 5) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
std::decay_t<decltype(p4)> _p4_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_, _p4_); } \
}
#define make_kernel_functor_hip_6(function_name, kernel_name, p0, p1, p2, p3) \
struct make_kernel_name_hip(function_name, 4) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
std::decay_t<decltype(p3)> _p3_; \
void operator()(const hc::tiled_index<3>&) const \
[[hc]] { kernel_name(_p0_, _p1_, _p2_, _p3_); } \
}
#define make_kernel_functor_hip_5(function_name, kernel_name, p0, p1, p2) \
struct make_kernel_name_hip(function_name, 3) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
std::decay_t<decltype(p2)> _p2_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_, _p2_); } \
}
#define make_kernel_functor_hip_4(function_name, kernel_name, p0, p1) \
struct make_kernel_name_hip(function_name, 2) { \
std::decay_t<decltype(p0)> _p0_; \
std::decay_t<decltype(p1)> _p1_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_, _p1_); } \
}
#define fofo(f, n) kernel_prefix_hip##f##kernel_suffix_hip##n
#define make_kernel_functor_hip_3(function_name, kernel_name, p0) \
struct make_kernel_name_hip(function_name, 1) { \
std::decay_t<decltype(p0)> _p0_; \
void operator()(const hc::tiled_index<3>&) const [[hc]] { kernel_name(_p0_); } \
}
#define make_kernel_functor_hip_2(function_name, kernel_name) \
struct make_kernel_name_hip(function_name, 0) { \
void operator()(const hc::tiled_index<3>&)[[hc]] { return kernel_name(hipLaunchParm{}); } \
}
#define make_kernel_functor_hip_1(...)
#define make_kernel_functor_hip_0(...)
#define make_kernel_functor_hip_(...) overload_macro_hip_(make_kernel_functor_hip_, __VA_ARGS__)
#define hipLaunchNamedKernelGGL(function_name, kernel_name, num_blocks, dim_blocks, \
group_mem_bytes, stream, ...) \
do { \
make_kernel_functor_hip_(function_name, kernel_name, __VA_ARGS__) \
hip_kernel_functor_impl_{__VA_ARGS__}; \
hip_impl::grid_launch_hip_(num_blocks, dim_blocks, group_mem_bytes, stream, #kernel_name, \
hip_kernel_functor_impl_); \
} while (0)
#define hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...) \
do { \
hipLaunchNamedKernelGGL(unnamed, kernel_name, num_blocks, dim_blocks, group_mem_bytes, \
stream, ##__VA_ARGS__); \
} while (0)
#define hipLaunchKernel(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, ...) \
do { \
hipLaunchKernelGGL(kernel_name, num_blocks, dim_blocks, group_mem_bytes, stream, \
hipLaunchParm{}, ##__VA_ARGS__); \
} while (0)
} // namespace hip_impl
projects/hip/src/hipamd/include/hip/amd_detail/math_fwd.h
+714
Fájl megtekintése
@@ -0,0 +1,714 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include "host_defines.h"
#if defined(__cplusplus)
extern "C" {
#endif
// DOT FUNCTIONS
#if __HIP_CLANG_ONLY__
__device__
__attribute__((const))
int __ockl_sdot2(
HIP_vector_base<short, 2>::Native_vec_,
HIP_vector_base<short, 2>::Native_vec_,
int, bool);
__device__
__attribute__((const))
unsigned int __ockl_udot2(
HIP_vector_base<unsigned short, 2>::Native_vec_,
HIP_vector_base<unsigned short, 2>::Native_vec_,
unsigned int, bool);
__device__
__attribute__((const))
int __ockl_sdot4(
HIP_vector_base<char, 4>::Native_vec_,
HIP_vector_base<char, 4>::Native_vec_,
int, bool);
__device__
__attribute__((const))
unsigned int __ockl_udot4(
HIP_vector_base<unsigned char, 4>::Native_vec_,
HIP_vector_base<unsigned char, 4>::Native_vec_,
unsigned int, bool);
__device__
__attribute__((const))
int __ockl_sdot8(int, int, int, bool);
__device__
__attribute__((const))
unsigned int __ockl_udot8(unsigned int, unsigned int, unsigned int, bool);
#endif
#if !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
// BEGIN FLOAT
__device__
__attribute__((const))
float __ocml_acos_f32(float);
__device__
__attribute__((pure))
float __ocml_acosh_f32(float);
__device__
__attribute__((const))
float __ocml_asin_f32(float);
__device__
__attribute__((pure))
float __ocml_asinh_f32(float);
__device__
__attribute__((const))
float __ocml_atan2_f32(float, float);
__device__
__attribute__((const))
float __ocml_atan_f32(float);
__device__
__attribute__((pure))
float __ocml_atanh_f32(float);
__device__
__attribute__((pure))
float __ocml_cbrt_f32(float);
__device__
__attribute__((const))
float __ocml_ceil_f32(float);
__device__
__attribute__((const))
__device__
float __ocml_copysign_f32(float, float);
__device__
float __ocml_cos_f32(float);
__device__
float __ocml_native_cos_f32(float);
__device__
__attribute__((pure))
__device__
float __ocml_cosh_f32(float);
__device__
float __ocml_cospi_f32(float);
__device__
float __ocml_i0_f32(float);
__device__
float __ocml_i1_f32(float);
__device__
__attribute__((pure))
float __ocml_erfc_f32(float);
__device__
__attribute__((pure))
float __ocml_erfcinv_f32(float);
__device__
__attribute__((pure))
float __ocml_erfcx_f32(float);
__device__
__attribute__((pure))
float __ocml_erf_f32(float);
__device__
__attribute__((pure))
float __ocml_erfinv_f32(float);
__device__
__attribute__((pure))
float __ocml_exp10_f32(float);
__device__
__attribute__((pure))
float __ocml_native_exp10_f32(float);
__device__
__attribute__((pure))
float __ocml_exp2_f32(float);
__device__
__attribute__((pure))
float __ocml_exp_f32(float);
__device__
__attribute__((pure))
float __ocml_native_exp_f32(float);
__device__
__attribute__((pure))
float __ocml_expm1_f32(float);
__device__
__attribute__((const))
float __ocml_fabs_f32(float);
__device__
__attribute__((const))
float __ocml_fdim_f32(float, float);
__device__
__attribute__((const))
float __ocml_floor_f32(float);
__device__
__attribute__((const))
float __ocml_fma_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_fmax_f32(float, float);
__device__
__attribute__((const))
float __ocml_fmin_f32(float, float);
__device__
__attribute__((const))
__device__
float __ocml_fmod_f32(float, float);
__device__
float __ocml_frexp_f32(float, __attribute__((address_space(5))) int*);
__device__
__attribute__((const))
float __ocml_hypot_f32(float, float);
__device__
__attribute__((const))
int __ocml_ilogb_f32(float);
__device__
__attribute__((const))
int __ocml_isfinite_f32(float);
__device__
__attribute__((const))
int __ocml_isinf_f32(float);
__device__
__attribute__((const))
int __ocml_isnan_f32(float);
__device__
float __ocml_j0_f32(float);
__device__
float __ocml_j1_f32(float);
__device__
__attribute__((const))
float __ocml_ldexp_f32(float, int);
__device__
float __ocml_lgamma_f32(float);
__device__
__attribute__((pure))
float __ocml_log10_f32(float);
__device__
__attribute__((pure))
float __ocml_native_log10_f32(float);
__device__
__attribute__((pure))
float __ocml_log1p_f32(float);
__device__
__attribute__((pure))
float __ocml_log2_f32(float);
__device__
__attribute__((pure))
float __ocml_native_log2_f32(float);
__device__
__attribute__((const))
float __ocml_logb_f32(float);
__device__
__attribute__((pure))
float __ocml_log_f32(float);
__device__
__attribute__((pure))
float __ocml_native_log_f32(float);
__device__
float __ocml_modf_f32(float, __attribute__((address_space(5))) float*);
__device__
__attribute__((const))
float __ocml_nearbyint_f32(float);
__device__
__attribute__((const))
float __ocml_nextafter_f32(float, float);
__device__
__attribute__((const))
float __ocml_len3_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_len4_f32(float, float, float, float);
__device__
__attribute__((pure))
float __ocml_ncdf_f32(float);
__device__
__attribute__((pure))
float __ocml_ncdfinv_f32(float);
__device__
__attribute__((pure))
float __ocml_pow_f32(float, float);
__device__
__attribute__((pure))
float __ocml_pown_f32(float, int);
__device__
__attribute__((pure))
float __ocml_rcbrt_f32(float);
__device__
__attribute__((const))
float __ocml_remainder_f32(float, float);
__device__
float __ocml_remquo_f32(float, float, __attribute__((address_space(5))) int*);
__device__
__attribute__((const))
float __ocml_rhypot_f32(float, float);
__device__
__attribute__((const))
float __ocml_rint_f32(float);
__device__
__attribute__((const))
float __ocml_rlen3_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_rlen4_f32(float, float, float, float);
__device__
__attribute__((const))
float __ocml_round_f32(float);
__device__
__attribute__((pure))
float __ocml_rsqrt_f32(float);
__device__
__attribute__((const))
float __ocml_scalb_f32(float, float);
__device__
__attribute__((const))
float __ocml_scalbn_f32(float, int);
__device__
__attribute__((const))
int __ocml_signbit_f32(float);
__device__
float __ocml_sincos_f32(float, __attribute__((address_space(5))) float*);
__device__
float __ocml_sincospi_f32(float, __attribute__((address_space(5))) float*);
__device__
float __ocml_sin_f32(float);
__device__
float __ocml_native_sin_f32(float);
__device__
__attribute__((pure))
float __ocml_sinh_f32(float);
__device__
float __ocml_sinpi_f32(float);
__device__
__attribute__((const))
float __ocml_sqrt_f32(float);
__device__
__attribute__((const))
float __ocml_native_sqrt_f32(float);
__device__
float __ocml_tan_f32(float);
__device__
__attribute__((pure))
float __ocml_tanh_f32(float);
__device__
float __ocml_tgamma_f32(float);
__device__
__attribute__((const))
float __ocml_trunc_f32(float);
__device__
float __ocml_y0_f32(float);
__device__
float __ocml_y1_f32(float);
// BEGIN INTRINSICS
__device__
__attribute__((const))
float __ocml_add_rte_f32(float, float);
__device__
__attribute__((const))
float __ocml_add_rtn_f32(float, float);
__device__
__attribute__((const))
float __ocml_add_rtp_f32(float, float);
__device__
__attribute__((const))
float __ocml_add_rtz_f32(float, float);
__device__
__attribute__((const))
float __ocml_sub_rte_f32(float, float);
__device__
__attribute__((const))
float __ocml_sub_rtn_f32(float, float);
__device__
__attribute__((const))
float __ocml_sub_rtp_f32(float, float);
__device__
__attribute__((const))
float __ocml_sub_rtz_f32(float, float);
__device__
__attribute__((const))
float __ocml_mul_rte_f32(float, float);
__device__
__attribute__((const))
float __ocml_mul_rtn_f32(float, float);
__device__
__attribute__((const))
float __ocml_mul_rtp_f32(float, float);
__device__
__attribute__((const))
float __ocml_mul_rtz_f32(float, float);
__device__
__attribute__((const))
float __ocml_div_rte_f32(float, float);
__device__
__attribute__((const))
float __ocml_div_rtn_f32(float, float);
__device__
__attribute__((const))
float __ocml_div_rtp_f32(float, float);
__device__
__attribute__((const))
float __ocml_div_rtz_f32(float, float);
__device__
__attribute__((const))
float __ocml_sqrt_rte_f32(float);
__device__
__attribute__((const))
float __ocml_sqrt_rtn_f32(float);
__device__
__attribute__((const))
float __ocml_sqrt_rtp_f32(float);
__device__
__attribute__((const))
float __ocml_sqrt_rtz_f32(float);
__device__
__attribute__((const))
float __ocml_fma_rte_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_fma_rtn_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_fma_rtp_f32(float, float, float);
__device__
__attribute__((const))
float __ocml_fma_rtz_f32(float, float, float);
__device__
__attribute__((const))
float __llvm_amdgcn_cos_f32(float) __asm("llvm.amdgcn.cos.f32");
__device__
__attribute__((const))
float __llvm_amdgcn_rcp_f32(float) __asm("llvm.amdgcn.rcp.f32");
__device__
__attribute__((const))
float __llvm_amdgcn_rsq_f32(float) __asm("llvm.amdgcn.rsq.f32");
__device__
__attribute__((const))
float __llvm_amdgcn_sin_f32(float) __asm("llvm.amdgcn.sin.f32");
// END INTRINSICS
// END FLOAT
// BEGIN DOUBLE
__device__
__attribute__((const))
double __ocml_acos_f64(double);
__device__
__attribute__((pure))
double __ocml_acosh_f64(double);
__device__
__attribute__((const))
double __ocml_asin_f64(double);
__device__
__attribute__((pure))
double __ocml_asinh_f64(double);
__device__
__attribute__((const))
double __ocml_atan2_f64(double, double);
__device__
__attribute__((const))
double __ocml_atan_f64(double);
__device__
__attribute__((pure))
double __ocml_atanh_f64(double);
__device__
__attribute__((pure))
double __ocml_cbrt_f64(double);
__device__
__attribute__((const))
double __ocml_ceil_f64(double);
__device__
__attribute__((const))
double __ocml_copysign_f64(double, double);
__device__
double __ocml_cos_f64(double);
__device__
__attribute__((pure))
double __ocml_cosh_f64(double);
__device__
double __ocml_cospi_f64(double);
__device__
double __ocml_i0_f64(double);
__device__
double __ocml_i1_f64(double);
__device__
__attribute__((pure))
double __ocml_erfc_f64(double);
__device__
__attribute__((pure))
double __ocml_erfcinv_f64(double);
__device__
__attribute__((pure))
double __ocml_erfcx_f64(double);
__device__
__attribute__((pure))
double __ocml_erf_f64(double);
__device__
__attribute__((pure))
double __ocml_erfinv_f64(double);
__device__
__attribute__((pure))
double __ocml_exp10_f64(double);
__device__
__attribute__((pure))
double __ocml_exp2_f64(double);
__device__
__attribute__((pure))
double __ocml_exp_f64(double);
__device__
__attribute__((pure))
double __ocml_expm1_f64(double);
__device__
__attribute__((const))
double __ocml_fabs_f64(double);
__device__
__attribute__((const))
double __ocml_fdim_f64(double, double);
__device__
__attribute__((const))
double __ocml_floor_f64(double);
__device__
__attribute__((const))
double __ocml_fma_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_fmax_f64(double, double);
__device__
__attribute__((const))
double __ocml_fmin_f64(double, double);
__device__
__attribute__((const))
double __ocml_fmod_f64(double, double);
__device__
double __ocml_frexp_f64(double, __attribute__((address_space(5))) int*);
__device__
__attribute__((const))
double __ocml_hypot_f64(double, double);
__device__
__attribute__((const))
int __ocml_ilogb_f64(double);
__device__
__attribute__((const))
int __ocml_isfinite_f64(double);
__device__
__attribute__((const))
int __ocml_isinf_f64(double);
__device__
__attribute__((const))
int __ocml_isnan_f64(double);
__device__
double __ocml_j0_f64(double);
__device__
double __ocml_j1_f64(double);
__device__
__attribute__((const))
double __ocml_ldexp_f64(double, int);
__device__
double __ocml_lgamma_f64(double);
__device__
__attribute__((pure))
double __ocml_log10_f64(double);
__device__
__attribute__((pure))
double __ocml_log1p_f64(double);
__device__
__attribute__((pure))
double __ocml_log2_f64(double);
__device__
__attribute__((const))
double __ocml_logb_f64(double);
__device__
__attribute__((pure))
double __ocml_log_f64(double);
__device__
double __ocml_modf_f64(double, __attribute__((address_space(5))) double*);
__device__
__attribute__((const))
double __ocml_nearbyint_f64(double);
__device__
__attribute__((const))
double __ocml_nextafter_f64(double, double);
__device__
__attribute__((const))
double __ocml_len3_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_len4_f64(double, double, double, double);
__device__
__attribute__((pure))
double __ocml_ncdf_f64(double);
__device__
__attribute__((pure))
double __ocml_ncdfinv_f64(double);
__device__
__attribute__((pure))
double __ocml_pow_f64(double, double);
__device__
__attribute__((pure))
double __ocml_pown_f64(double, int);
__device__
__attribute__((pure))
double __ocml_rcbrt_f64(double);
__device__
__attribute__((const))
double __ocml_remainder_f64(double, double);
__device__
double __ocml_remquo_f64(
double, double, __attribute__((address_space(5))) int*);
__device__
__attribute__((const))
double __ocml_rhypot_f64(double, double);
__device__
__attribute__((const))
double __ocml_rint_f64(double);
__device__
__attribute__((const))
double __ocml_rlen3_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_rlen4_f64(double, double, double, double);
__device__
__attribute__((const))
double __ocml_round_f64(double);
__device__
__attribute__((pure))
double __ocml_rsqrt_f64(double);
__device__
__attribute__((const))
double __ocml_scalb_f64(double, double);
__device__
__attribute__((const))
double __ocml_scalbn_f64(double, int);
__device__
__attribute__((const))
int __ocml_signbit_f64(double);
__device__
double __ocml_sincos_f64(double, __attribute__((address_space(5))) double*);
__device__
double __ocml_sincospi_f64(double, __attribute__((address_space(5))) double*);
__device__
double __ocml_sin_f64(double);
__device__
__attribute__((pure))
double __ocml_sinh_f64(double);
__device__
double __ocml_sinpi_f64(double);
__device__
__attribute__((const))
double __ocml_sqrt_f64(double);
__device__
double __ocml_tan_f64(double);
__device__
__attribute__((pure))
double __ocml_tanh_f64(double);
__device__
double __ocml_tgamma_f64(double);
__device__
__attribute__((const))
double __ocml_trunc_f64(double);
__device__
double __ocml_y0_f64(double);
__device__
double __ocml_y1_f64(double);
// BEGIN INTRINSICS
__device__
__attribute__((const))
double __ocml_add_rte_f64(double, double);
__device__
__attribute__((const))
double __ocml_add_rtn_f64(double, double);
__device__
__attribute__((const))
double __ocml_add_rtp_f64(double, double);
__device__
__attribute__((const))
double __ocml_add_rtz_f64(double, double);
__device__
__attribute__((const))
double __ocml_sub_rte_f64(double, double);
__device__
__attribute__((const))
double __ocml_sub_rtn_f64(double, double);
__device__
__attribute__((const))
double __ocml_sub_rtp_f64(double, double);
__device__
__attribute__((const))
double __ocml_sub_rtz_f64(double, double);
__device__
__attribute__((const))
double __ocml_mul_rte_f64(double, double);
__device__
__attribute__((const))
double __ocml_mul_rtn_f64(double, double);
__device__
__attribute__((const))
double __ocml_mul_rtp_f64(double, double);
__device__
__attribute__((const))
double __ocml_mul_rtz_f64(double, double);
__device__
__attribute__((const))
double __ocml_div_rte_f64(double, double);
__device__
__attribute__((const))
double __ocml_div_rtn_f64(double, double);
__device__
__attribute__((const))
double __ocml_div_rtp_f64(double, double);
__device__
__attribute__((const))
double __ocml_div_rtz_f64(double, double);
__device__
__attribute__((const))
double __ocml_sqrt_rte_f64(double);
__device__
__attribute__((const))
double __ocml_sqrt_rtn_f64(double);
__device__
__attribute__((const))
double __ocml_sqrt_rtp_f64(double);
__device__
__attribute__((const))
double __ocml_sqrt_rtz_f64(double);
__device__
__attribute__((const))
double __ocml_fma_rte_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_fma_rtn_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_fma_rtp_f64(double, double, double);
__device__
__attribute__((const))
double __ocml_fma_rtz_f64(double, double, double);
__device__
__attribute__((const))
double __llvm_amdgcn_rcp_f64(double) __asm("llvm.amdgcn.rcp.f64");
__device__
__attribute__((const))
double __llvm_amdgcn_rsq_f64(double) __asm("llvm.amdgcn.rsq.f64");
// END INTRINSICS
// END DOUBLE
#endif // !__CLANG_HIP_RUNTIME_WRAPPER_INCLUDED__
#if defined(__cplusplus)
} // extern "C"
#endif
projects/hip/src/hipamd/include/hip/amd_detail/ockl_image.h
+135
Fájl megtekintése
@@ -0,0 +1,135 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include <hip/hip_vector_types.h>
extern "C" {
#define ADDRESS_SPACE_CONSTANT __attribute__((address_space(4)))
__device__ float4::Native_vec_ __ockl_image_load_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
__device__ float4::Native_vec_ __ockl_image_load_1Db(unsigned int ADDRESS_SPACE_CONSTANT*i, int c);
__device__ float4::Native_vec_ __ockl_image_load_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_load_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_load_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_load_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_load_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f);
__device__ float4::Native_vec_ __ockl_image_load_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f);
__device__ float4::Native_vec_ __ockl_image_load_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int f, int l);
__device__ float4::Native_vec_ __ockl_image_load_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int f, int l);
__device__ void __ockl_image_store_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, float4::Native_vec_ p);
__device__ void __ockl_image_store_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, int c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, int2::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ void __ockl_image_store_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, int4::Native_vec_ c, int l, float4::Native_vec_ p);
__device__ float4::Native_vec_ __ockl_image_sample_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c);
__device__ float4::Native_vec_ __ockl_image_sample_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_sample_grad_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float dx, float dy);
__device__ float4::Native_vec_ __ockl_image_sample_grad_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float dx, float dy);
__device__ float4::Native_vec_ __ockl_image_sample_grad_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
__device__ float4::Native_vec_ __ockl_image_sample_grad_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float2::Native_vec_ dx, float2::Native_vec_ dy);
__device__ float4::Native_vec_ __ockl_image_sample_grad_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float4::Native_vec_ dx, float4::Native_vec_ dy);
__device__ float4::Native_vec_ __ockl_image_sample_lod_1D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_1Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_2Da(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_3D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_CM(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_sample_lod_CMa(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float4::Native_vec_ c, float l);
__device__ float4::Native_vec_ __ockl_image_gather4r_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_gather4g_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_gather4b_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
__device__ float4::Native_vec_ __ockl_image_gather4a_2D(unsigned int ADDRESS_SPACE_CONSTANT*i, unsigned int ADDRESS_SPACE_CONSTANT*s, float2::Native_vec_ c);
};
projects/hip/src/hipamd/include/hip/amd_detail/program_state.hpp
+107
Fájl megtekintése
@@ -0,0 +1,107 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include <hsa/amd_hsa_kernel_code.h>
#include <hsa/hsa.h>
#include <hsa/hsa_ext_amd.h>
#include <hsa/hsa_ven_amd_loader.h>
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <hip/hip_common.h>
struct ihipModuleSymbol_t;
using hipFunction_t = ihipModuleSymbol_t*;
namespace hip_impl {
// This section contains internal APIs that
// needs to be exported
#ifdef __GNUC__
#pragma GCC visibility push (default)
#endif
struct kernarg_impl;
class kernarg {
public:
kernarg();
kernarg(kernarg&&);
~kernarg();
std::uint8_t* data();
std::size_t size();
void reserve(std::size_t);
void resize(std::size_t);
private:
kernarg_impl* impl;
};
class kernargs_size_align;
class program_state_impl;
class program_state {
public:
program_state();
~program_state();
program_state(const program_state&) = delete;
hipFunction_t kernel_descriptor(std::uintptr_t,
hsa_agent_t);
kernargs_size_align get_kernargs_size_align(std::uintptr_t);
hsa_executable_t load_executable(const char*, const size_t,
hsa_executable_t,
hsa_agent_t);
hsa_executable_t load_executable_no_copy(const char*, const size_t,
hsa_executable_t,
hsa_agent_t);
void* global_addr_by_name(const char* name);
private:
friend class agent_globals_impl;
program_state_impl* impl;
};
class kernargs_size_align {
public:
std::size_t size(std::size_t n) const;
std::size_t alignment(std::size_t n) const;
const void* getHandle() const {return handle;};
private:
const void* handle;
friend kernargs_size_align program_state::get_kernargs_size_align(std::uintptr_t);
};
#ifdef __GNUC__
#pragma GCC visibility pop
#endif
inline
__attribute__((visibility("hidden")))
program_state& get_program_state() {
static program_state ps;
return ps;
}
} // Namespace hip_impl.
projects/hip/src/hipamd/include/hip/amd_detail/texture_fetch_functions.h
+388
Fájl megtekintése
@@ -0,0 +1,388 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#if defined(__cplusplus)
#include <hip/hip_vector_types.h>
#include <hip/hip_texture_types.h>
#include <hip/amd_detail/ockl_image.h>
#if !defined(__HIPCC_RTC__)
#include <type_traits>
#endif // !defined(__HIPCC_RTC__)
#define TEXTURE_PARAMETERS_INIT \
unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)t.textureObject; \
unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
template<typename T>
struct __hip_is_tex_channel_type
{
static constexpr bool value =
std::is_same<T, char>::value ||
std::is_same<T, unsigned char>::value ||
std::is_same<T, short>::value ||
std::is_same<T, unsigned short>::value ||
std::is_same<T, int>::value ||
std::is_same<T, unsigned int>::value ||
std::is_same<T, float>::value;
};
template<
typename T,
unsigned int rank>
struct __hip_is_tex_channel_type<HIP_vector_type<T, rank>>
{
static constexpr bool value =
__hip_is_tex_channel_type<T>::value &&
((rank == 1) ||
(rank == 2) ||
(rank == 4));
};
template<typename T>
struct __hip_is_tex_normalized_channel_type
{
static constexpr bool value =
std::is_same<T, char>::value ||
std::is_same<T, unsigned char>::value ||
std::is_same<T, short>::value ||
std::is_same<T, unsigned short>::value;
};
template<
typename T,
unsigned int rank>
struct __hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>
{
static constexpr bool value =
__hip_is_tex_normalized_channel_type<T>::value &&
((rank == 1) ||
(rank == 2) ||
(rank == 4));
};
template <
typename T,
hipTextureReadMode readMode,
typename Enable = void>
struct __hip_tex_ret
{
static_assert(std::is_same<Enable, void>::value, "Invalid channel type!");
};
template <
typename T,
hipTextureReadMode readMode>
using __hip_tex_ret_t = typename __hip_tex_ret<T, readMode, bool>::type;
template <typename T>
struct __hip_tex_ret<
T,
hipReadModeElementType,
typename std::enable_if<__hip_is_tex_channel_type<T>::value, bool>::type>
{
using type = T;
};
template<
typename T,
unsigned int rank>
struct __hip_tex_ret<
HIP_vector_type<T, rank>,
hipReadModeElementType,
typename std::enable_if<__hip_is_tex_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
{
using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeElementType>, rank>;
};
template<typename T>
struct __hip_tex_ret<
T,
hipReadModeNormalizedFloat,
typename std::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
{
using type = float;
};
template<
typename T,
unsigned int rank>
struct __hip_tex_ret<
HIP_vector_type<T, rank>,
hipReadModeNormalizedFloat,
typename std::enable_if<__hip_is_tex_normalized_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
{
using type = HIP_vector_type<__hip_tex_ret_t<T, hipReadModeNormalizedFloat>, rank>;
};
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1Dfetch(texture<T, hipTextureType1D, readMode> t, int x)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_load_1Db(i, x);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1D(texture<T, hipTextureType1D, readMode> t, float x)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_1D(i, s, x);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2D(texture<T, hipTextureType2D, readMode> t, float x, float y)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1DLayered(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2DLayered(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex3D(texture<T, hipTextureType3D, readMode> t, float x, float y, float z)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemap(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1DLod(texture<T, hipTextureType1D, readMode> t, float x, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2DLod(texture<T, hipTextureType2D, readMode> t, float x, float y, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1DLayeredLod(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_1Da(i, s, float2(x, layer).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2DLayeredLod(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_2Da(i, s, float4(x, y, layer, 0.0f).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex3DLod(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemapLod(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemapLayered(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemapLayeredLod(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float level)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemapGrad(texture<T, hipTextureTypeCubemap, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
{
TEXTURE_PARAMETERS_INIT;
// TODO missing in device libs.
// auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
// return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
return {};
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> texCubemapLayeredGrad(texture<T, hipTextureTypeCubemapLayered, readMode> t, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
{
TEXTURE_PARAMETERS_INIT;
// TODO missing in device libs.
// auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
// return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
return {};
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1DGrad(texture<T, hipTextureType1D, readMode> t, float x, float dPdx, float dPdy)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2DGrad(texture<T, hipTextureType2D, readMode> t, float x, float y, float2 dPdx, float2 dPdy)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex1DLayeredGrad(texture<T, hipTextureType1DLayered, readMode> t, float x, int layer, float dPdx, float dPdy)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex2DLayeredGrad(texture<T, hipTextureType2DLayered, readMode> t, float x, float y, int layer, float2 dPdx, float2 dPdy)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex_ret_t<T, readMode> tex3DGrad(texture<T, hipTextureType3D, readMode> t, float x, float y, float z, float4 dPdx, float4 dPdy)
{
TEXTURE_PARAMETERS_INIT;
auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
return *reinterpret_cast<__hip_tex_ret_t<T, readMode>*>(&tmp);
}
template <
typename T,
hipTextureReadMode readMode,
typename Enable = void>
struct __hip_tex2dgather_ret
{
static_assert(std::is_same<Enable, void>::value, "Invalid channel type!");
};
template <
typename T,
hipTextureReadMode readMode>
using __hip_tex2dgather_ret_t = typename __hip_tex2dgather_ret<T, readMode, bool>::type;
template <typename T>
struct __hip_tex2dgather_ret<
T,
hipReadModeElementType,
typename std::enable_if<__hip_is_tex_channel_type<T>::value, bool>::type>
{
using type = HIP_vector_type<T, 4>;
};
template<
typename T,
unsigned int rank>
struct __hip_tex2dgather_ret<
HIP_vector_type<T, rank>,
hipReadModeElementType,
typename std::enable_if<__hip_is_tex_channel_type<HIP_vector_type<T, rank>>::value, bool>::type>
{
using type = HIP_vector_type<T, 4>;
};
template <typename T>
struct __hip_tex2dgather_ret<
T,
hipReadModeNormalizedFloat,
typename std::enable_if<__hip_is_tex_normalized_channel_type<T>::value, bool>::type>
{
using type = float4;
};
template <typename T, hipTextureReadMode readMode>
static __forceinline__ __device__ __hip_tex2dgather_ret_t<T, readMode> tex2Dgather(texture<T, hipTextureType2D, readMode> t, float x, float y, int comp=0)
{
TEXTURE_PARAMETERS_INIT;
switch (comp) {
case 1: {
auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data);
return *reinterpret_cast<__hip_tex2dgather_ret_t<T, readMode>*>(&tmp);
}
case 2: {
auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data);
return *reinterpret_cast<__hip_tex2dgather_ret_t<T, readMode>*>(&tmp);
}
case 3: {
auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data);
return *reinterpret_cast<__hip_tex2dgather_ret_t<T, readMode>*>(&tmp);
}
default: {
auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data);
return *reinterpret_cast<__hip_tex2dgather_ret_t<T, readMode>*>(&tmp);
}
}
return {};
}
#endif
projects/hip/src/hipamd/include/hip/amd_detail/texture_functions.h
+11102
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/amd_detail/texture_indirect_functions.h
+503
Fájl megtekintése
@@ -0,0 +1,503 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#if defined(__cplusplus)
#include <hip/hip_vector_types.h>
#include <hip/hip_texture_types.h>
#include <hip/amd_detail/ockl_image.h>
#if !defined(__HIPCC_RTC__)
#include <type_traits>
#endif // !defined(__HIPCC_RTC__)
#define TEXTURE_OBJECT_PARAMETERS_INIT \
unsigned int ADDRESS_SPACE_CONSTANT* i = (unsigned int ADDRESS_SPACE_CONSTANT*)textureObject; \
unsigned int ADDRESS_SPACE_CONSTANT* s = i + HIP_SAMPLER_OBJECT_OFFSET_DWORD;
template<typename T>
struct __hip_is_itex_channel_type
{
static constexpr bool value =
std::is_same<T, char>::value ||
std::is_same<T, unsigned char>::value ||
std::is_same<T, short>::value ||
std::is_same<T, unsigned short>::value ||
std::is_same<T, int>::value ||
std::is_same<T, unsigned int>::value ||
std::is_same<T, float>::value;
};
template<
typename T,
unsigned int rank>
struct __hip_is_itex_channel_type<HIP_vector_type<T, rank>>
{
static constexpr bool value =
__hip_is_itex_channel_type<T>::value &&
((rank == 1) ||
(rank == 2) ||
(rank == 4));
};
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1Dfetch(hipTextureObject_t textureObject, int x)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_load_1Db(i, x);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1Dfetch(T *ptr, hipTextureObject_t textureObject, int x)
{
*ptr = tex1Dfetch<T>(textureObject, x);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1D(hipTextureObject_t textureObject, float x)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_1D(i, s, x);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1D(T *ptr, hipTextureObject_t textureObject, float x)
{
*ptr = tex1D<T>(textureObject, x);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2D(hipTextureObject_t textureObject, float x, float y)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_2D(i, s, float2(x, y).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2D(T *ptr, hipTextureObject_t textureObject, float x, float y)
{
*ptr = tex2D<T>(textureObject, x, y);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex3D(hipTextureObject_t textureObject, float x, float y, float z)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_3D(i, s, float4(x, y, z, 0.0f).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex3D(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
{
*ptr = tex3D<T>(textureObject, x, y, z);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1DLayered(hipTextureObject_t textureObject, float x, int layer)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1DLayered(T *ptr, hipTextureObject_t textureObject, float x, int layer)
{
*ptr = tex1DLayered<T>(textureObject, x, layer);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2DLayered(hipTextureObject_t textureObject, float x, float y, int layer)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2DLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer)
{
*ptr = tex1DLayered<T>(textureObject, x, y, layer);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemap(hipTextureObject_t textureObject, float x, float y, float z)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_CM(i, s, float4(x, y, z, 0.0f).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemap(T *ptr, hipTextureObject_t textureObject, float x, float y, float z)
{
*ptr = texCubemap<T>(textureObject, x, y, z);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemapLayered(hipTextureObject_t textureObject, float x, float y, float z, int layer)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_CMa(i, s, float4(x, y, z, layer).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemapLayered(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer)
{
*ptr = texCubemapLayered<T>(textureObject, x, y, z, layer);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2Dgather(hipTextureObject_t textureObject, float x, float y, int comp = 0)
{
TEXTURE_OBJECT_PARAMETERS_INIT
switch (comp) {
case 1: {
auto tmp = __ockl_image_gather4r_2D(i, s, float2(x, y).data);
return *reinterpret_cast<T*>(&tmp);
break;
}
case 2: {
auto tmp = __ockl_image_gather4g_2D(i, s, float2(x, y).data);
return *reinterpret_cast<T*>(&tmp);
break;
}
case 3: {
auto tmp = __ockl_image_gather4b_2D(i, s, float2(x, y).data);
return *reinterpret_cast<T*>(&tmp);
break;
}
default: {
auto tmp = __ockl_image_gather4a_2D(i, s, float2(x, y).data);
return *reinterpret_cast<T*>(&tmp);
break;
}
};
return {};
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2Dgather(T *ptr, hipTextureObject_t textureObject, float x, float y, int comp = 0)
{
*ptr = texCubemapLayered<T>(textureObject, x, y, comp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1DLod(hipTextureObject_t textureObject, float x, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_lod_1D(i, s, x, level);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1DLod(T *ptr, hipTextureObject_t textureObject, float x, float level)
{
*ptr = tex1DLod<T>(textureObject, x, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2DLod(hipTextureObject_t textureObject, float x, float y, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_lod_2D(i, s, float2(x, y).data, level);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float level)
{
*ptr = tex2DLod<T>(textureObject, x, y, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex3DLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_lod_3D(i, s, float4(x, y, z, 0.0f).data, level);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex3DLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
{
*ptr = tex3DLod<T>(textureObject, x, y, z, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1DLayeredLod(hipTextureObject_t textureObject, float x, int layer, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_1Da(i, s, float2(x, layer).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, int layer, float level)
{
*ptr = tex1DLayeredLod<T>(textureObject, x, layer, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2DLayeredLod(hipTextureObject_t textureObject, float x, float y, int layer, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_2Da(i, s, float4(x, y, layer, 0.0f).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2DLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float level)
{
*ptr = tex2DLayeredLod<T>(textureObject, x, y, layer, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemapLod(hipTextureObject_t textureObject, float x, float y, float z, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_lod_CM(i, s, float4(x, y, z, 0.0f).data, level);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemapLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float level)
{
*ptr = texCubemapLod<T>(textureObject, x, y, z, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemapGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
// TODO missing in device libs.
// auto tmp = __ockl_image_sample_grad_CM(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
// return *reinterpret_cast<T*>(&tmp);
return {};
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemapGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
{
*ptr = texCubemapGrad<T>(textureObject, x, y, z, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemapLayeredLod(hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_lod_CMa(i, s, float4(x, y, z, layer).data, level);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemapLayeredLod(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float level)
{
*ptr = texCubemapLayeredLod<T>(textureObject, x, y, z, layer, level);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1DGrad(hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_grad_1D(i, s, x, dPdx, dPdy);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1DGrad(T *ptr, hipTextureObject_t textureObject, float x, float dPdx, float dPdy)
{
*ptr = tex1DGrad<T>(textureObject, x, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2DGrad(hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_grad_2D(i, s, float2(x, y).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float2 dPdx, float2 dPdy)
{
*ptr = tex2DGrad<T>(textureObject, x, y, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex3DGrad(hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_grad_3D(i, s, float4(x, y, z, 0.0f).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex3DGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, float4 dPdx, float4 dPdy)
{
*ptr = tex3DGrad<T>(textureObject, x, y, z, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex1DLayeredGrad(hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_grad_1Da(i, s, float2(x, layer).data, dPdx, dPdy);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex1DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, int layer, float dPdx, float dPdy)
{
*ptr = tex1DLayeredGrad<T>(textureObject, x, layer, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T tex2DLayeredGrad(hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
auto tmp = __ockl_image_sample_grad_2Da(i, s, float4(x, y, layer, 0.0f).data, float2(dPdx.x, dPdx.y).data, float2(dPdy.x, dPdy.y).data);
return *reinterpret_cast<T*>(&tmp);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void tex2DLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, int layer, float2 dPdx, float2 dPdy)
{
*ptr = tex2DLayeredGrad<T>(textureObject, x, y, layer, dPdx, dPdy);
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ T texCubemapLayeredGrad(hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
{
TEXTURE_OBJECT_PARAMETERS_INIT
// TODO missing in device libs.
// auto tmp = __ockl_image_sample_grad_CMa(i, s, float4(x, y, z, layer).data, float4(dPdx.x, dPdx.y, dPdx.z, 0.0f).data, float4(dPdy.x, dPdy.y, dPdy.z, 0.0f).data);
// return *reinterpret_cast<T*>(&tmp);
return {};
}
template <
typename T,
typename std::enable_if<__hip_is_itex_channel_type<T>::value>::type* = nullptr>
static __device__ void texCubemapLayeredGrad(T *ptr, hipTextureObject_t textureObject, float x, float y, float z, int layer, float4 dPdx, float4 dPdy)
{
*ptr = texCubemapLayeredGrad<T>(textureObject, x, y, z, layer, dPdx, dPdy);
}
#endif
projects/hip/src/hipamd/include/hip/hcc_detail
Symlink
+1
Fájl megtekintése
@@ -0,0 +1 @@
amd_detail
projects/hip/src/hipamd/include/hip/nvcc_detail
Symlink
+1
Fájl megtekintése
@@ -0,0 +1 @@
nvidia_detail
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_channel_descriptor.h
+28
Fájl megtekintése
@@ -0,0 +1,28 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_CHANNEL_DESCRIPTOR_H
#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_CHANNEL_DESCRIPTOR_H
#include "channel_descriptor.h"
#endif
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hip_complex.h
+119
Fájl megtekintése
@@ -0,0 +1,119 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COMPLEX_H
#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COMPLEX_H
#include "cuComplex.h"
typedef cuFloatComplex hipFloatComplex;
__device__ __host__ static inline float hipCrealf(hipFloatComplex z) { return cuCrealf(z); }
__device__ __host__ static inline float hipCimagf(hipFloatComplex z) { return cuCimagf(z); }
__device__ __host__ static inline hipFloatComplex make_hipFloatComplex(float a, float b) {
return make_cuFloatComplex(a, b);
}
__device__ __host__ static inline hipFloatComplex hipConjf(hipFloatComplex z) { return cuConjf(z); }
__device__ __host__ static inline float hipCsqabsf(hipFloatComplex z) {
return cuCabsf(z) * cuCabsf(z);
}
__device__ __host__ static inline hipFloatComplex hipCaddf(hipFloatComplex p, hipFloatComplex q) {
return cuCaddf(p, q);
}
__device__ __host__ static inline hipFloatComplex hipCsubf(hipFloatComplex p, hipFloatComplex q) {
return cuCsubf(p, q);
}
__device__ __host__ static inline hipFloatComplex hipCmulf(hipFloatComplex p, hipFloatComplex q) {
return cuCmulf(p, q);
}
__device__ __host__ static inline hipFloatComplex hipCdivf(hipFloatComplex p, hipFloatComplex q) {
return cuCdivf(p, q);
}
__device__ __host__ static inline float hipCabsf(hipFloatComplex z) { return cuCabsf(z); }
typedef cuDoubleComplex hipDoubleComplex;
__device__ __host__ static inline double hipCreal(hipDoubleComplex z) { return cuCreal(z); }
__device__ __host__ static inline double hipCimag(hipDoubleComplex z) { return cuCimag(z); }
__device__ __host__ static inline hipDoubleComplex make_hipDoubleComplex(double a, double b) {
return make_cuDoubleComplex(a, b);
}
__device__ __host__ static inline hipDoubleComplex hipConj(hipDoubleComplex z) { return cuConj(z); }
__device__ __host__ static inline double hipCsqabs(hipDoubleComplex z) {
return cuCabs(z) * cuCabs(z);
}
__device__ __host__ static inline hipDoubleComplex hipCadd(hipDoubleComplex p, hipDoubleComplex q) {
return cuCadd(p, q);
}
__device__ __host__ static inline hipDoubleComplex hipCsub(hipDoubleComplex p, hipDoubleComplex q) {
return cuCsub(p, q);
}
__device__ __host__ static inline hipDoubleComplex hipCmul(hipDoubleComplex p, hipDoubleComplex q) {
return cuCmul(p, q);
}
__device__ __host__ static inline hipDoubleComplex hipCdiv(hipDoubleComplex p, hipDoubleComplex q) {
return cuCdiv(p, q);
}
__device__ __host__ static inline double hipCabs(hipDoubleComplex z) { return cuCabs(z); }
typedef cuFloatComplex hipComplex;
__device__ __host__ static inline hipComplex make_Complex(float x, float y) {
return make_cuComplex(x, y);
}
__device__ __host__ static inline hipFloatComplex hipComplexDoubleToFloat(hipDoubleComplex z) {
return cuComplexDoubleToFloat(z);
}
__device__ __host__ static inline hipDoubleComplex hipComplexFloatToDouble(hipFloatComplex z) {
return cuComplexFloatToDouble(z);
}
__device__ __host__ static inline hipComplex hipCfmaf(hipComplex p, hipComplex q, hipComplex r) {
return cuCfmaf(p, q, r);
}
__device__ __host__ static inline hipDoubleComplex hipCfma(hipDoubleComplex p, hipDoubleComplex q,
hipDoubleComplex r) {
return cuCfma(p, q, r);
}
#endif
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hip_cooperative_groups.h
+12
Fájl megtekintése
@@ -0,0 +1,12 @@
#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H
#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H
// Include CUDA headers
#include <cuda_runtime.h>
#include <cooperative_groups.h>
// Include HIP wrapper headers around CUDA
#include <hip/hip_runtime.h>
#include <hip/hip_runtime_api.h>
#endif // HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_COOPERATIVE_GROUPS_H
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime.h
+122
Fájl megtekintése
@@ -0,0 +1,122 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_H
#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_RUNTIME_H
#include <cuda_runtime.h>
#include <hip/hip_runtime_api.h>
#define HIP_KERNEL_NAME(...) __VA_ARGS__
typedef int hipLaunchParm;
#define hipLaunchKernelGGLInternal(kernelName, numBlocks, numThreads, memPerBlock, streamId, ...) \
do { \
kernelName<<<numBlocks, numThreads, memPerBlock, streamId>>>(__VA_ARGS__); \
} while (0)
#define hipLaunchKernelGGL(kernelName, ...) hipLaunchKernelGGLInternal((kernelName), __VA_ARGS__)
#define hipReadModeElementType cudaReadModeElementType
#ifdef __CUDA_ARCH__
// 32-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT32_ATOMICS__ (__CUDA_ARCH__ >= 110)
#define __HIP_ARCH_HAS_GLOBAL_FLOAT_ATOMIC_EXCH__ (__CUDA_ARCH__ >= 110)
#define __HIP_ARCH_HAS_SHARED_INT32_ATOMICS__ (__CUDA_ARCH__ >= 120)
#define __HIP_ARCH_HAS_SHARED_FLOAT_ATOMIC_EXCH__ (__CUDA_ARCH__ >= 120)
#define __HIP_ARCH_HAS_FLOAT_ATOMIC_ADD__ (__CUDA_ARCH__ >= 200)
// 64-bit Atomics:
#define __HIP_ARCH_HAS_GLOBAL_INT64_ATOMICS__ (__CUDA_ARCH__ >= 200)
#define __HIP_ARCH_HAS_SHARED_INT64_ATOMICS__ (__CUDA_ARCH__ >= 120)
// Doubles
#define __HIP_ARCH_HAS_DOUBLES__ (__CUDA_ARCH__ >= 120)
// warp cross-lane operations:
#define __HIP_ARCH_HAS_WARP_VOTE__ (__CUDA_ARCH__ >= 120)
#define __HIP_ARCH_HAS_WARP_BALLOT__ (__CUDA_ARCH__ >= 200)
#define __HIP_ARCH_HAS_WARP_SHUFFLE__ (__CUDA_ARCH__ >= 300)
#define __HIP_ARCH_HAS_WARP_FUNNEL_SHIFT__ (__CUDA_ARCH__ >= 350)
// sync
#define __HIP_ARCH_HAS_THREAD_FENCE_SYSTEM__ (__CUDA_ARCH__ >= 200)
#define __HIP_ARCH_HAS_SYNC_THREAD_EXT__ (__CUDA_ARCH__ >= 200)
// misc
#define __HIP_ARCH_HAS_SURFACE_FUNCS__ (__CUDA_ARCH__ >= 200)
#define __HIP_ARCH_HAS_3DGRID__ (__CUDA_ARCH__ >= 200)
#define __HIP_ARCH_HAS_DYNAMIC_PARALLEL__ (__CUDA_ARCH__ >= 350)
#endif
#ifdef __CUDACC__
#define hipThreadIdx_x threadIdx.x
#define hipThreadIdx_y threadIdx.y
#define hipThreadIdx_z threadIdx.z
#define hipBlockIdx_x blockIdx.x
#define hipBlockIdx_y blockIdx.y
#define hipBlockIdx_z blockIdx.z
#define hipBlockDim_x blockDim.x
#define hipBlockDim_y blockDim.y
#define hipBlockDim_z blockDim.z
#define hipGridDim_x gridDim.x
#define hipGridDim_y gridDim.y
#define hipGridDim_z gridDim.z
#define HIP_SYMBOL(X) &X
/**
* Map HIP_DYNAMIC_SHARED to "extern __shared__" for compatibility with old HIP applications
* To be removed in a future release.
*/
#define HIP_DYNAMIC_SHARED(type, var) extern __shared__ type var[];
#define HIP_DYNAMIC_SHARED_ATTRIBUTE
#ifdef __HIP_DEVICE_COMPILE__
#define abort_() \
{ asm("trap;"); }
#undef assert
#define assert(COND) \
{ \
if (!COND) { \
abort_(); \
} \
}
#endif
#define __clock() clock()
#define __clock64() clock64()
#endif
#endif
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hip_runtime_api.h
+2195
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hip_texture_types.h
+6
Fájl megtekintése
@@ -0,0 +1,6 @@
#ifndef HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_TEXTURE_TYPES_H
#define HIP_INCLUDE_HIP_NVIDIA_DETAIL_HIP_TEXTURE_TYPES_H
#include <texture_types.h>
#endif
projects/hip/src/hipamd/include/hip/nvidia_detail/nvidia_hiprtc.h
+168
Fájl megtekintése
@@ -0,0 +1,168 @@
/*
Copyright (c) 2021 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIPRTC_H
#define HIPRTC_H
#include <cuda.h>
#include <nvrtc.h>
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
#include <stdlib.h>
#if !defined(_WIN32)
#pragma GCC visibility push(default)
#endif
typedef enum hiprtcResult {
HIPRTC_SUCCESS = 0,
HIPRTC_ERROR_OUT_OF_MEMORY = 1,
HIPRTC_ERROR_PROGRAM_CREATION_FAILURE = 2,
HIPRTC_ERROR_INVALID_INPUT = 3,
HIPRTC_ERROR_INVALID_PROGRAM = 4,
HIPRTC_ERROR_INVALID_OPTION = 5,
HIPRTC_ERROR_COMPILATION = 6,
HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE = 7,
HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION = 8,
HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION = 9,
HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID = 10,
HIPRTC_ERROR_INTERNAL_ERROR = 11
} hiprtcResult;
inline static nvrtcResult hiprtcResultTonvrtcResult(hiprtcResult result) {
switch (result) {
case HIPRTC_SUCCESS:
return NVRTC_SUCCESS;
case HIPRTC_ERROR_OUT_OF_MEMORY:
return NVRTC_ERROR_OUT_OF_MEMORY;
case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE:
return NVRTC_ERROR_PROGRAM_CREATION_FAILURE;
case HIPRTC_ERROR_INVALID_INPUT:
return NVRTC_ERROR_INVALID_INPUT;
case HIPRTC_ERROR_INVALID_PROGRAM:
return NVRTC_ERROR_INVALID_PROGRAM;
case HIPRTC_ERROR_INVALID_OPTION:
return NVRTC_ERROR_INVALID_OPTION;
case HIPRTC_ERROR_COMPILATION:
return NVRTC_ERROR_COMPILATION;
case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE:
return NVRTC_ERROR_BUILTIN_OPERATION_FAILURE;
case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION:
return NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION;
case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION:
return NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION;
case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID:
return NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID;
case HIPRTC_ERROR_INTERNAL_ERROR:
return NVRTC_ERROR_INTERNAL_ERROR;
}
}
inline static hiprtcResult nvrtcResultTohiprtcResult(nvrtcResult result) {
switch (result) {
case NVRTC_SUCCESS:
return HIPRTC_SUCCESS;
case NVRTC_ERROR_OUT_OF_MEMORY:
return HIPRTC_ERROR_OUT_OF_MEMORY;
case NVRTC_ERROR_PROGRAM_CREATION_FAILURE:
return HIPRTC_ERROR_PROGRAM_CREATION_FAILURE;
case NVRTC_ERROR_INVALID_INPUT:
return HIPRTC_ERROR_INVALID_INPUT;
case NVRTC_ERROR_INVALID_PROGRAM:
return HIPRTC_ERROR_INVALID_PROGRAM;
case NVRTC_ERROR_INVALID_OPTION:
return HIPRTC_ERROR_INVALID_OPTION;
case NVRTC_ERROR_COMPILATION:
return HIPRTC_ERROR_COMPILATION;
case NVRTC_ERROR_BUILTIN_OPERATION_FAILURE:
return HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE;
case NVRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION:
return HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION;
case NVRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION:
return HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION;
case NVRTC_ERROR_NAME_EXPRESSION_NOT_VALID:
return HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID;
case NVRTC_ERROR_INTERNAL_ERROR:
return HIPRTC_ERROR_INTERNAL_ERROR;
}
}
const char* hiprtcGetErrorString(hiprtcResult result) {
return nvrtcGetErrorString(hiprtcResultTonvrtcResult(result));
}
hiprtcResult hiprtcVersion(int* major, int* minor) {
return nvrtcResultTohiprtcResult(nvrtcVersion(major, minor));
}
typedef nvrtcProgram hiprtcProgram;
hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, const char* name_expression) {
return nvrtcResultTohiprtcResult(nvrtcAddNameExpression(prog, name_expression));
}
hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char** options) {
return nvrtcResultTohiprtcResult(nvrtcCompileProgram(prog, numOptions, options));
}
hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const char* name,
int numHeaders, const char** headers, const char** includeNames) {
return nvrtcResultTohiprtcResult(
nvrtcCreateProgram(prog, src, name, numHeaders, headers, includeNames));
}
hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) {
return nvrtcResultTohiprtcResult(nvrtcDestroyProgram(prog));
}
hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, const char* name_expression,
const char** lowered_name) {
return nvrtcResultTohiprtcResult(nvrtcGetLoweredName(prog, name_expression, lowered_name));
}
hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* log) {
return nvrtcResultTohiprtcResult(nvrtcGetProgramLog(prog, log));
}
hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) {
return nvrtcResultTohiprtcResult(nvrtcGetProgramLogSize(prog, logSizeRet));
}
hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* code) {
return nvrtcResultTohiprtcResult(nvrtcGetPTX(prog, code));
}
hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* codeSizeRet) {
return nvrtcResultTohiprtcResult(nvrtcGetPTXSize(prog, codeSizeRet));
}
#if !defined(_WIN32)
#pragma GCC visibility pop
#endif
#ifdef __cplusplus
}
#endif /* __cplusplus */
#endif // HIPRTC_H
projects/hip/src/hipamd/src/amd_hsa_elf.hpp
+118
Fájl megtekintése
@@ -0,0 +1,118 @@
/*
Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
// AMDGPU OS for HSA compatible compute kernels.
enum { ELFOSABI_AMDGPU_HSA = 64, ELFOSABI_AMDGPU_PAL = 65, ELFOSABI_AMDGPU_MESA3D = 66 };
enum {
ELFABIVERSION_AMDGPU_HSA_V2 = 0,
ELFABIVERSION_AMDGPU_HSA_V3 = 1,
ELFABIVERSION_AMDGPU_HSA_V4 = 2
};
// AMDGPU specific e_flags
enum : unsigned {
EF_AMDGPU_MACH = 0x0ff,
// AMDGPU processors
EF_AMDGPU_MACH_NONE = 0x000,
EF_AMDGPU_MACH_R600_R600 = 0x001,
EF_AMDGPU_MACH_R600_R630 = 0x002,
EF_AMDGPU_MACH_R600_RS880 = 0x003,
EF_AMDGPU_MACH_R600_RV670 = 0x004,
EF_AMDGPU_MACH_R600_RV710 = 0x005,
EF_AMDGPU_MACH_R600_RV730 = 0x006,
EF_AMDGPU_MACH_R600_RV770 = 0x007,
EF_AMDGPU_MACH_R600_CEDAR = 0x008,
EF_AMDGPU_MACH_R600_CYPRESS = 0x009,
EF_AMDGPU_MACH_R600_JUNIPER = 0x00a,
EF_AMDGPU_MACH_R600_REDWOOD = 0x00b,
EF_AMDGPU_MACH_R600_SUMO = 0x00c,
EF_AMDGPU_MACH_R600_BARTS = 0x00d,
EF_AMDGPU_MACH_R600_CAICOS = 0x00e,
EF_AMDGPU_MACH_R600_CAYMAN = 0x00f,
EF_AMDGPU_MACH_R600_TURKS = 0x010,
EF_AMDGPU_MACH_R600_RESERVED_FIRST = 0x011,
EF_AMDGPU_MACH_R600_RESERVED_LAST = 0x01f,
EF_AMDGPU_MACH_R600_FIRST = EF_AMDGPU_MACH_R600_R600,
EF_AMDGPU_MACH_R600_LAST = EF_AMDGPU_MACH_R600_TURKS,
EF_AMDGPU_MACH_AMDGCN_GFX600 = 0x020,
EF_AMDGPU_MACH_AMDGCN_GFX601 = 0x021,
EF_AMDGPU_MACH_AMDGCN_GFX700 = 0x022,
EF_AMDGPU_MACH_AMDGCN_GFX701 = 0x023,
EF_AMDGPU_MACH_AMDGCN_GFX702 = 0x024,
EF_AMDGPU_MACH_AMDGCN_GFX703 = 0x025,
EF_AMDGPU_MACH_AMDGCN_GFX704 = 0x026,
EF_AMDGPU_MACH_AMDGCN_RESERVED_0X027 = 0x027,
EF_AMDGPU_MACH_AMDGCN_GFX801 = 0x028,
EF_AMDGPU_MACH_AMDGCN_GFX802 = 0x029,
EF_AMDGPU_MACH_AMDGCN_GFX803 = 0x02a,
EF_AMDGPU_MACH_AMDGCN_GFX810 = 0x02b,
EF_AMDGPU_MACH_AMDGCN_GFX900 = 0x02c,
EF_AMDGPU_MACH_AMDGCN_GFX902 = 0x02d,
EF_AMDGPU_MACH_AMDGCN_GFX904 = 0x02e,
EF_AMDGPU_MACH_AMDGCN_GFX906 = 0x02f,
EF_AMDGPU_MACH_AMDGCN_GFX908 = 0x030,
EF_AMDGPU_MACH_AMDGCN_GFX909 = 0x031,
EF_AMDGPU_MACH_AMDGCN_GFX90C = 0x032,
EF_AMDGPU_MACH_AMDGCN_GFX1010 = 0x033,
EF_AMDGPU_MACH_AMDGCN_GFX1011 = 0x034,
EF_AMDGPU_MACH_AMDGCN_GFX1012 = 0x035,
EF_AMDGPU_MACH_AMDGCN_GFX1030 = 0x036,
EF_AMDGPU_MACH_AMDGCN_GFX1031 = 0x037,
EF_AMDGPU_MACH_AMDGCN_GFX1032 = 0x038,
EF_AMDGPU_MACH_AMDGCN_GFX1033 = 0x039,
EF_AMDGPU_MACH_AMDGCN_GFX602 = 0x03a,
EF_AMDGPU_MACH_AMDGCN_GFX705 = 0x03b,
EF_AMDGPU_MACH_AMDGCN_GFX805 = 0x03c,
EF_AMDGPU_MACH_AMDGCN_GFX90A = 0x03f,
EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600,
EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX90A,
// Indicates if the "xnack" target feature is enabled for all code contained
// in the object.
//
// Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3.
EF_AMDGPU_FEATURE_XNACK_V3 = 0x100,
// Indicates if the "sramecc" target feature is enabled for all code
// contained in the object.
//
// Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V3.
EF_AMDGPU_FEATURE_SRAMECC_V3 = 0x200,
// Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4.
EF_AMDGPU_FEATURE_XNACK_V4 = 0x300,
EF_AMDGPU_FEATURE_XNACK_UNSUPPORTED_V4 = 0x000,
EF_AMDGPU_FEATURE_XNACK_ANY_V4 = 0x100,
EF_AMDGPU_FEATURE_XNACK_OFF_V4 = 0x200,
EF_AMDGPU_FEATURE_XNACK_ON_V4 = 0x300,
// SRAMECC selection mask for EF_AMDGPU_FEATURE_SRAMECC_* values.
// Only valid for ELFOSABI_AMDGPU_HSA and ELFABIVERSION_AMDGPU_HSA_V4.
EF_AMDGPU_FEATURE_SRAMECC_V4 = 0xc00,
EF_AMDGPU_FEATURE_SRAMECC_UNSUPPORTED_V4 = 0x000,
EF_AMDGPU_FEATURE_SRAMECC_ANY_V4 = 0x400,
EF_AMDGPU_FEATURE_SRAMECC_OFF_V4 = 0x800,
EF_AMDGPU_FEATURE_SRAMECC_ON_V4 = 0xc00,
};
projects/hip/src/hipamd/src/cl_d3d10.cpp
+1451
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/src/cl_d3d11.cpp
+1571
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/src/cl_d3d9.cpp
+855
Fájl megtekintése
@@ -0,0 +1,855 @@
/* Copyright (c) 2012-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifdef _WIN32
#include "top.hpp"
#include "cl_d3d9_amd.hpp"
#include "platform/command.hpp"
#include <cstring>
#include <utility>
#define D3DFMT_NV_12 static_cast<D3DFORMAT>(MAKEFOURCC('N', 'V', '1', '2'))
#define D3DFMT_P010 static_cast<D3DFORMAT>(MAKEFOURCC('P', '0', '1', '0'))
#define D3DFMT_YV_12 static_cast<D3DFORMAT>(MAKEFOURCC('Y', 'V', '1', '2'))
#define D3DFMT_YUY2 static_cast<D3DFORMAT>(MAKEFOURCC('Y', 'U', 'Y', '2'))
RUNTIME_ENTRY(cl_int, clGetDeviceIDsFromDX9MediaAdapterKHR,
(cl_platform_id platform, cl_uint num_media_adapters,
cl_dx9_media_adapter_type_khr* media_adapters_type, void* media_adapters,
cl_dx9_media_adapter_set_khr media_adapter_set, cl_uint num_entries,
cl_device_id* devices, cl_uint* num_devices)) {
cl_int errcode;
// Accept an array of DX9 devices here as the spec mention of array of num_media_adapters size.
IDirect3DDevice9Ex** d3d9_device = static_cast<IDirect3DDevice9Ex**>(media_adapters);
cl_device_id* gpu_devices = NULL;
cl_uint num_gpu_devices = 0;
static const bool VALIDATE_ONLY = true;
if (platform != NULL && platform != AMD_PLATFORM) {
LogWarning("\"platrform\" is not a valid AMD platform");
return CL_INVALID_PLATFORM;
}
// check if input parameter are correct
if ((num_media_adapters == 0) || (media_adapters_type == NULL) || (media_adapters == NULL) ||
(media_adapter_set != CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR &&
media_adapter_set != CL_ALL_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR) ||
(num_entries == 0 && devices != NULL)) {
return CL_INVALID_VALUE;
}
// Get GPU devices
errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, 0, NULL, &num_gpu_devices);
if (errcode != CL_SUCCESS && errcode != CL_DEVICE_NOT_FOUND) {
return CL_INVALID_VALUE;
}
if (!num_gpu_devices) {
*not_null(num_devices) = 0;
return CL_DEVICE_NOT_FOUND;
}
switch (media_adapter_set) {
case CL_PREFERRED_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR:
case CL_ALL_DEVICES_FOR_DX9_MEDIA_ADAPTER_KHR: {
gpu_devices = new cl_device_id[num_gpu_devices];
errcode = clGetDeviceIDs(NULL, CL_DEVICE_TYPE_GPU, num_gpu_devices, gpu_devices, NULL);
if (errcode != CL_SUCCESS) {
break;
}
std::vector<amd::Device*> compatible_devices;
for (cl_uint i = 0; i < num_gpu_devices; ++i) {
cl_device_id device = gpu_devices[i];
amd::Context::Flags context_flag;
amd::Context::DeviceFlagIdx devIdx;
switch (media_adapters_type[i]) {
case CL_ADAPTER_D3D9_KHR:
context_flag = amd::Context::Flags::D3D9DeviceKhr;
devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceKhrIdx;
break;
case CL_ADAPTER_D3D9EX_KHR:
context_flag = amd::Context::Flags::D3D9DeviceEXKhr;
devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceEXKhrIdx;
break;
case CL_ADAPTER_DXVA_KHR:
context_flag = amd::Context::Flags::D3D9DeviceVAKhr;
devIdx = amd::Context::DeviceFlagIdx::D3D9DeviceVAKhrIdx;
break;
}
for (cl_uint j = 0; j < num_media_adapters; ++j) {
// Since there can be multiple DX9 adapters passed in the array we need to validate
// interopability with each.
void* external_device[amd::Context::DeviceFlagIdx::LastDeviceFlagIdx] = {};
external_device[devIdx] = d3d9_device[j];
if (is_valid(device) && (media_adapters_type[j] == CL_ADAPTER_D3D9EX_KHR) &&
as_amd(device)->bindExternalDevice(context_flag, external_device, NULL,
VALIDATE_ONLY)) {
compatible_devices.push_back(as_amd(device));
}
}
}
if (compatible_devices.size() == 0) {
*not_null(num_devices) = 0;
errcode = CL_DEVICE_NOT_FOUND;
break;
}
auto it = compatible_devices.cbegin();
cl_uint compatible_count = std::min(num_entries, (cl_uint)compatible_devices.size());
while (compatible_count--) {
*devices++ = as_cl(*it++);
--num_entries;
}
while (num_entries--) {
*devices++ = (cl_device_id)0;
}
*not_null(num_devices) = (cl_uint)compatible_devices.size();
} break;
default:
LogWarning("\"d3d9_device_set\" is invalid");
errcode = CL_INVALID_VALUE;
}
delete[] gpu_devices;
return errcode;
}
RUNTIME_EXIT
RUNTIME_ENTRY_RET(cl_mem, clCreateFromDX9MediaSurfaceKHR,
(cl_context context, cl_mem_flags flags,
cl_dx9_media_adapter_type_khr adapter_type, void* surface_info, cl_uint plane,
cl_int* errcode_ret)) {
cl_mem clMemObj = NULL;
cl_dx9_surface_info_khr* cl_surf_info = NULL;
if (!is_valid(context)) {
*not_null(errcode_ret) = CL_INVALID_CONTEXT;
LogWarning("invalid parameter \"context\"");
return clMemObj;
}
if (!flags) flags = CL_MEM_READ_WRITE;
if (!(((flags & CL_MEM_READ_ONLY) == CL_MEM_READ_ONLY) ||
((flags & CL_MEM_WRITE_ONLY) == CL_MEM_WRITE_ONLY) ||
((flags & CL_MEM_READ_WRITE) == CL_MEM_READ_WRITE))) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
LogWarning("invalid parameter \"flags\"");
return clMemObj;
}
if ((adapter_type != CL_ADAPTER_D3D9_KHR) && (adapter_type != CL_ADAPTER_D3D9EX_KHR) &&
(adapter_type != CL_ADAPTER_DXVA_KHR)) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
return clMemObj;
}
if (!surface_info) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
LogWarning("parameter \"pD3DResource\" is a NULL pointer");
return clMemObj;
}
cl_surf_info = (cl_dx9_surface_info_khr*)surface_info;
IDirect3DSurface9* pD3D9Resource = cl_surf_info->resource;
HANDLE shared_handle = cl_surf_info->shared_handle;
if (!pD3D9Resource) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
LogWarning("parameter \"surface_info\" is a NULL pointer");
return clMemObj;
}
D3DSURFACE_DESC Desc;
pD3D9Resource->GetDesc(&Desc);
if ((Desc.Format != D3DFMT_NV_12) &&
(Desc.Format != D3DFMT_P010) &&
(Desc.Format != D3DFMT_YV_12) && (plane != 0)) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
LogWarning("The plane has to be Zero if the surface format is non-planar !");
return clMemObj;
}
// Check for image support
const std::vector<amd::Device*>& devices = as_amd(context)->devices();
bool supportPass = false;
bool sizePass = false;
for (const auto& it : devices) {
if (it->info().imageSupport_) {
supportPass = true;
}
}
if (!supportPass) {
*not_null(errcode_ret) = CL_INVALID_OPERATION;
LogWarning("there are no devices in context to support images");
return (cl_mem)0;
}
// Verify the resource is a 2D image
return amd::clCreateImage2DFromD3D9ResourceAMD(*as_amd(context), flags, adapter_type,
cl_surf_info, plane, errcode_ret);
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clEnqueueAcquireDX9MediaSurfacesKHR,
(cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects,
cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) {
return amd::clEnqueueAcquireExtObjectsAMD(command_queue, num_objects, mem_objects,
num_events_in_wait_list, event_wait_list, event,
CL_COMMAND_ACQUIRE_DX9_MEDIA_SURFACES_KHR);
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clEnqueueReleaseDX9MediaSurfacesKHR,
(cl_command_queue command_queue, cl_uint num_objects, const cl_mem* mem_objects,
cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) {
return amd::clEnqueueReleaseExtObjectsAMD(command_queue, num_objects, mem_objects,
num_events_in_wait_list, event_wait_list, event,
CL_COMMAND_RELEASE_DX9_MEDIA_SURFACES_KHR);
}
RUNTIME_EXIT
//
//
// namespace amd
//
//
namespace amd {
/*! @}
* \addtogroup CL-D3D9 interop helper functions
* @{
*/
//
// Class D3D9Object implementation
//
std::vector<std::pair<TD3D9RESINFO, TD3D9RESINFO>> D3D9Object::resources_;
Monitor D3D9Object::resLock_;
//
// clCreateImage2DFromD3D9ResourceAMD
//
cl_mem clCreateImage2DFromD3D9ResourceAMD(Context& amdContext, cl_mem_flags flags,
cl_dx9_media_adapter_type_khr adapter_type,
cl_dx9_surface_info_khr* surface_info, cl_uint plane,
int* errcode_ret) {
cl_dx9_surface_info_khr* cl_surf_info = reinterpret_cast<cl_dx9_surface_info_khr*>(surface_info);
IDirect3DSurface9* pD3D9Resource = cl_surf_info->resource;
HANDLE shared_handle = cl_surf_info->shared_handle;
D3D9Object obj;
cl_int errcode = D3D9Object::initD3D9Object(amdContext, adapter_type, surface_info, plane, obj);
if (CL_SUCCESS != errcode) {
*not_null(errcode_ret) = errcode;
return (cl_mem)0;
}
Image2DD3D9* pImage2DD3D9 = new (amdContext) Image2DD3D9(amdContext, flags, obj);
if (!pImage2DD3D9) {
*not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY;
return (cl_mem)0;
}
if (!pImage2DD3D9->create()) {
*not_null(errcode_ret) = CL_MEM_OBJECT_ALLOCATION_FAILURE;
pImage2DD3D9->release();
return (cl_mem)0;
}
*not_null(errcode_ret) = CL_SUCCESS;
return as_cl<Memory>(pImage2DD3D9);
}
//
// Helper function SyncD3D9Objects
//
void SyncD3D9Objects(std::vector<amd::Memory*>& memObjects) {
Memory*& mem = memObjects.front();
if (!mem) {
LogWarning("\nNULL memory object\n");
return;
}
InteropObject* interop = mem->getInteropObj();
if (!interop) {
LogWarning("\nNULL interop object\n");
return;
}
D3D9Object* d3d9Obj = interop->asD3D9Object();
if (!d3d9Obj) {
LogWarning("\nNULL D3D9 object\n");
return;
}
IDirect3DQuery9* query = d3d9Obj->getQuery();
if (!query) {
LogWarning("\nNULL IDirect3DQuery9\n");
return;
}
ScopedLock sl(d3d9Obj->getResLock());
query->Issue(D3DISSUE_END);
BOOL data = FALSE;
while (S_OK != query->GetData(&data, sizeof(BOOL), D3DGETDATA_FLUSH)) {
}
}
//
// Class D3D10Object implementation
//
size_t D3D9Object::getElementBytes(D3DFORMAT d3d9Format, cl_uint plane) {
size_t bytesPerPixel;
switch (d3d9Format) {
case D3DFMT_UNKNOWN:
case D3DFMT_UYVY:
case D3DFMT_DXT1:
case D3DFMT_DXT2:
case D3DFMT_DXT3:
case D3DFMT_DXT4:
case D3DFMT_DXT5:
case D3DFMT_VERTEXDATA:
case D3DFMT_D32:
case D3DFMT_D15S1:
case D3DFMT_D24S8:
case D3DFMT_D24X8:
case D3DFMT_D24X4S4:
case D3DFMT_D16:
case D3DFMT_INDEX16:
case D3DFMT_INDEX32:
case D3DFMT_MULTI2_ARGB8:
case D3DFMT_CxV8U8:
// Less than 1 byte per pixel - needs special consideration
bytesPerPixel = 0;
break;
case D3DFMT_R3G3B2:
case D3DFMT_P8:
case D3DFMT_A8:
case D3DFMT_L8:
case D3DFMT_A4L4:
bytesPerPixel = 1;
break;
case D3DFMT_R16F:
case D3DFMT_R5G6B5:
case D3DFMT_X1R5G5B5:
case D3DFMT_A1R5G5B5:
case D3DFMT_A4R4G4B4:
case D3DFMT_A8R3G3B2:
case D3DFMT_X4R4G4B4:
case D3DFMT_A8P8:
case D3DFMT_A8L8:
case D3DFMT_V8U8:
case D3DFMT_L6V5U5:
case D3DFMT_D16_LOCKABLE:
case D3DFMT_L16:
bytesPerPixel = 2;
break;
case D3DFMT_R8G8B8:
case D3DFMT_D24FS8:
bytesPerPixel = 3;
break;
case D3DFMT_D32F_LOCKABLE:
case D3DFMT_A8R8G8B8:
case D3DFMT_R32F:
case D3DFMT_X8R8G8B8:
case D3DFMT_A2B10G10R10:
case D3DFMT_A8B8G8R8:
case D3DFMT_X8B8G8R8:
case D3DFMT_G16R16:
case D3DFMT_A2R10G10B10:
case D3DFMT_Q8W8V8U8:
case D3DFMT_X8L8V8U8:
case D3DFMT_V16U16:
case D3DFMT_A2W10V10U10:
case D3DFMT_R8G8_B8G8:
case D3DFMT_G8R8_G8B8:
case D3DFMT_G16R16F:
case D3DFMT_YUY2:
bytesPerPixel = 4;
break;
case D3DFMT_G32R32F:
case D3DFMT_A16B16G16R16:
case D3DFMT_A16B16G16R16F:
case D3DFMT_Q16W16V16U16:
bytesPerPixel = 8;
break;
case D3DFMT_A32B32G32R32F:
bytesPerPixel = 16;
break;
//#if !defined(D3D_DISABLE_9EX)
// case D3DFMT_D32_LOCKABLE:
// case D3DFMT_S8_LOCKABLE:
//#endif // !D3D_DISABLE_9EX
case D3DFMT_NV_12:
if (plane == 0) {
bytesPerPixel = 1;
} else if (plane == 1) {
bytesPerPixel = 2;
} // plane != 0 or != 1 shouldn't happen here
break;
case D3DFMT_P010:
if (plane == 0) {
bytesPerPixel = 2;
} else if (plane == 1) {
bytesPerPixel = 4;
} // plane != 0 or != 1 shouldn't happen here
break;
case D3DFMT_YV_12:
bytesPerPixel = 1;
break;
default:
bytesPerPixel = 0;
_ASSERT(FALSE);
break;
}
return bytesPerPixel;
}
void setObjDesc(amd::D3D9ObjDesc_t& objDesc, D3DSURFACE_DESC& resDesc, cl_uint plane) {
objDesc.d3dPool_ = resDesc.Pool;
objDesc.resType_ = resDesc.Type;
objDesc.usage_ = resDesc.Usage;
objDesc.d3dFormat_ = resDesc.Format;
switch (resDesc.Format) {
case D3DFMT_NV_12:
case D3DFMT_P010:
objDesc.surfRect_.left = 0;
objDesc.surfRect_.top = 0;
if (plane == 0) {
objDesc.objSize_.Height = resDesc.Height;
objDesc.objSize_.Width = resDesc.Width;
objDesc.surfRect_.right = resDesc.Width; // resDesc.Width/2-1;
objDesc.surfRect_.bottom = 3 * resDesc.Height / 2;
; // 3*resDesc.Height/2-1;
} else if (plane == 1) {
objDesc.objSize_.Height = resDesc.Height / 2;
objDesc.objSize_.Width = resDesc.Width / 2;
objDesc.surfRect_.right = resDesc.Width; // resDesc.Width/2-1;
objDesc.surfRect_.bottom = 3 * resDesc.Height / 2;
; // 3*resDesc.Height/2-1;
} // plane != 0 or != 1 shouldn't happen here
break;
case D3DFMT_YV_12:
objDesc.surfRect_.left = 0;
if (plane == 0) {
objDesc.objSize_.Height = resDesc.Height;
objDesc.objSize_.Width = resDesc.Width;
objDesc.surfRect_.top = 0;
objDesc.surfRect_.right = resDesc.Width - 1;
objDesc.surfRect_.bottom = resDesc.Height - 1;
} else if (plane == 1) {
objDesc.objSize_.Height = resDesc.Height / 2;
objDesc.objSize_.Width = resDesc.Width / 2;
objDesc.surfRect_.top = resDesc.Height;
objDesc.surfRect_.right = resDesc.Width / 2 - 1;
objDesc.surfRect_.bottom = 3 * resDesc.Height / 2 - 1;
} else if (plane == 2) {
objDesc.objSize_.Height = resDesc.Height / 2;
objDesc.objSize_.Width = resDesc.Width / 2;
objDesc.surfRect_.top = 3 * resDesc.Height / 2;
objDesc.surfRect_.right = resDesc.Width / 2 - 1;
objDesc.surfRect_.bottom = 2 * resDesc.Height - 1;
} // plane > 0 or > 2 shouldn't happen here
break;
default:
objDesc.objSize_.Height = resDesc.Height;
objDesc.objSize_.Width = resDesc.Width;
objDesc.surfRect_.left = 0;
objDesc.surfRect_.top = 0;
objDesc.surfRect_.right = resDesc.Width - 1;
objDesc.surfRect_.bottom = resDesc.Height - 1;
if (resDesc.Format == D3DFMT_YUY2) {
objDesc.objSize_.Width >>= 1;
}
break;
}
}
int D3D9Object::initD3D9Object(const Context& amdContext,
cl_dx9_media_adapter_type_khr adapter_type,
cl_dx9_surface_info_khr* cl_surf_info, cl_uint plane,
D3D9Object& obj) {
ScopedLock sl(resLock_);
IDirect3DDevice9Ex* pDev9Ex = NULL;
cl_int errcode = CL_SUCCESS;
// Check if this ressource has already been used for interop
IDirect3DSurface9* pD3D9res = cl_surf_info->resource;
HANDLE shared_handle = cl_surf_info->shared_handle;
if ((adapter_type == CL_ADAPTER_D3D9_KHR) || (adapter_type == CL_ADAPTER_DXVA_KHR)) {
return CL_INVALID_DX9_MEDIA_ADAPTER_KHR; // Not supported yet
}
for (const auto& it : resources_) {
if (it.first.surfInfo.resource == cl_surf_info->resource && it.first.surfPlane == plane) {
return CL_INVALID_D3D9_RESOURCE_KHR;
}
}
HRESULT hr;
D3DQUERYTYPE desc = D3DQUERYTYPE_EVENT;
D3DSURFACE_DESC resDesc;
if (D3D_OK != pD3D9res->GetDesc(&resDesc)) {
return CL_INVALID_D3D9_RESOURCE_KHR;
}
hr = pD3D9res->GetContainer(IID_IDirect3DDevice9Ex, (void**)&pDev9Ex);
if (hr == D3D_OK) {
pDev9Ex->CreateQuery(desc, &(obj.pQuery_));
} else {
return CL_INVALID_D3D9_RESOURCE_KHR; // d3d9ex should be supported
}
obj.handleShared_ = shared_handle;
obj.surfPlane_ = plane;
obj.surfInfo_ = *cl_surf_info;
obj.adapterType_ = adapter_type;
// Init defaults
setObjDesc(obj.objDescOrig_, resDesc, plane);
obj.objDesc_ = obj.objDescOrig_;
// shared handle cases if the shared_handle is NULL
// first check if the format is NV12 or YV12, which we need special handling
if (NULL == shared_handle) {
bool found = false;
for (const auto& it : resources_) {
if (it.first.surfInfo.resource == cl_surf_info->resource &&
it.first.surfPlane != plane) {
obj.handleShared_ = it.second.surfInfo.shared_handle;
obj.pD3D9Res_ = it.second.surfInfo.resource;
obj.pD3D9Res_->AddRef();
obj.objDesc_ = obj.objDescOrig_;
found = true;
break;
}
}
if (!found) {
obj.handleShared_ = 0;
hr = pDev9Ex->CreateOffscreenPlainSurface(resDesc.Width, resDesc.Height, resDesc.Format,
resDesc.Pool, &obj.pD3D9Res_, &obj.handleShared_);
if (D3D_OK != hr) {
errcode = CL_INVALID_D3D9_RESOURCE_KHR;
}
}
// put the original info into the obj
obj.pD3D9ResOrig_ = pD3D9res;
obj.pD3D9ResOrig_->AddRef(); // addRef in case lost the resource
} else {
// Share the original resource
obj.pD3D9ResOrig_ = NULL;
obj.pD3D9Res_ = pD3D9res;
obj.pD3D9Res_->AddRef();
}
// Release the Ex interface
if (pDev9Ex) pDev9Ex->Release();
// Check for CL format compatibilty
if (obj.objDesc_.resType_ == D3DRTYPE_SURFACE) {
cl_image_format clFmt = obj.getCLFormatFromD3D9(obj.objDesc_.d3dFormat_, plane);
amd::Image::Format imageFormat(clFmt);
if (!imageFormat.isSupported(amdContext)) {
return CL_INVALID_IMAGE_FORMAT_DESCRIPTOR;
}
}
TD3D9RESINFO d3d9ObjOri = {*cl_surf_info, plane};
TD3D9RESINFO d3d9ObjShared = {{obj.pD3D9Res_, obj.handleShared_}, plane};
if (errcode == CL_SUCCESS) {
resources_.push_back({d3d9ObjOri, d3d9ObjShared});
}
return errcode;
}
cl_uint D3D9Object::getMiscFlag() {
switch (objDescOrig_.d3dFormat_) {
case D3DFMT_NV_12:
case D3DFMT_P010:
return 1;
break;
case D3DFMT_YV_12:
return 2;
break;
case D3DFMT_YUY2:
return 3;
break;
default:
return 0;
break;
}
}
cl_image_format D3D9Object::getCLFormatFromD3D9() {
return getCLFormatFromD3D9(objDesc_.d3dFormat_, surfPlane_);
}
cl_image_format D3D9Object::getCLFormatFromD3D9(D3DFORMAT d3d9Fmt, cl_uint plane) {
cl_image_format fmt;
fmt.image_channel_order = 0; // CL_RGBA;
fmt.image_channel_data_type = 0; // CL_UNSIGNED_INT8;
switch (d3d9Fmt) {
case D3DFMT_R32F:
fmt.image_channel_order = CL_R;
fmt.image_channel_data_type = CL_FLOAT;
break;
case D3DFMT_R16F:
fmt.image_channel_order = CL_R;
fmt.image_channel_data_type = CL_HALF_FLOAT;
break;
case D3DFMT_L16:
fmt.image_channel_order = CL_R;
fmt.image_channel_data_type = CL_UNORM_INT16;
break;
case D3DFMT_A8:
fmt.image_channel_order = CL_A;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_L8:
fmt.image_channel_order = CL_R;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_G32R32F:
fmt.image_channel_order = CL_RG;
fmt.image_channel_data_type = CL_FLOAT;
break;
case D3DFMT_G16R16F:
fmt.image_channel_order = CL_RG;
fmt.image_channel_data_type = CL_HALF_FLOAT;
break;
case D3DFMT_G16R16:
fmt.image_channel_order = CL_RG;
fmt.image_channel_data_type = CL_UNORM_INT16;
break;
case D3DFMT_A8L8:
fmt.image_channel_order = CL_RG;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_A32B32G32R32F:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_FLOAT;
break;
case D3DFMT_A16B16G16R16F:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_HALF_FLOAT;
break;
case D3DFMT_A16B16G16R16:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_UNORM_INT16;
break;
case D3DFMT_A8B8G8R8:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_X8B8G8R8:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_A8R8G8B8:
fmt.image_channel_order = CL_BGRA;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_X8R8G8B8:
fmt.image_channel_order = CL_BGRA;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_NV_12:
fmt.image_channel_data_type = CL_UNORM_INT8;
if (plane == 0) {
fmt.image_channel_order = CL_R;
} else if (plane == 1) {
fmt.image_channel_order = CL_RG;
}
break;
case D3DFMT_P010:
fmt.image_channel_data_type = CL_UNORM_INT16;
if (plane == 0) {
fmt.image_channel_order = CL_R;
} else if (plane == 1) {
fmt.image_channel_order = CL_RG;
}
break;
case D3DFMT_YV_12:
fmt.image_channel_order = CL_R;
fmt.image_channel_data_type = CL_UNORM_INT8;
break;
case D3DFMT_YUY2:
fmt.image_channel_order = CL_RGBA;
fmt.image_channel_data_type = CL_UNSIGNED_INT8;
break;
case D3DFMT_UNKNOWN:
case D3DFMT_R8G8B8:
case D3DFMT_R5G6B5:
case D3DFMT_X1R5G5B5:
case D3DFMT_A1R5G5B5:
case D3DFMT_A4R4G4B4:
case D3DFMT_R3G3B2:
case D3DFMT_A8R3G3B2:
case D3DFMT_X4R4G4B4:
case D3DFMT_A2B10G10R10:
case D3DFMT_A2R10G10B10:
case D3DFMT_A8P8:
case D3DFMT_P8:
case D3DFMT_A4L4:
case D3DFMT_V8U8:
case D3DFMT_L6V5U5:
case D3DFMT_X8L8V8U8:
case D3DFMT_Q8W8V8U8:
case D3DFMT_V16U16:
case D3DFMT_A2W10V10U10:
case D3DFMT_UYVY:
case D3DFMT_R8G8_B8G8:
case D3DFMT_G8R8_G8B8:
case D3DFMT_DXT1:
case D3DFMT_DXT2:
case D3DFMT_DXT3:
case D3DFMT_DXT4:
case D3DFMT_DXT5:
case D3DFMT_D16_LOCKABLE:
case D3DFMT_D32:
case D3DFMT_D15S1:
case D3DFMT_D24S8:
case D3DFMT_D24X8:
case D3DFMT_D24X4S4:
case D3DFMT_D16:
case D3DFMT_D32F_LOCKABLE:
case D3DFMT_D24FS8:
//#if !defined(D3D_DISABLE_9EX)
case D3DFMT_D32_LOCKABLE:
case D3DFMT_S8_LOCKABLE:
//#endif // !D3D_DISABLE_9EX
case D3DFMT_VERTEXDATA:
case D3DFMT_INDEX16:
case D3DFMT_INDEX32:
case D3DFMT_Q16W16V16U16:
case D3DFMT_MULTI2_ARGB8:
case D3DFMT_CxV8U8:
//#if !defined(D3D_DISABLE_9EX)
case D3DFMT_A1:
case D3DFMT_A2B10G10R10_XR_BIAS:
case D3DFMT_BINARYBUFFER:
_ASSERT(FALSE); // NOT SURPPORTED
break;
//#endif // !D3D_DISABLE_9EX
default:
_ASSERT(FALSE);
break;
}
return fmt;
}
bool D3D9Object::copyOrigToShared() {
// Don't copy if there is no orig
if (NULL == getD3D9ResOrig()) return true;
IDirect3DDevice9Ex* d3dDev;
HRESULT hr;
ScopedLock sl(getResLock());
IDirect3DSurface9* srcSurf = getD3D9ResOrig();
IDirect3DSurface9* dstSurf = getD3D9Resource();
hr = getD3D9Resource()->GetContainer(IID_IDirect3DDevice9Ex, (void**)&d3dDev);
if (hr != D3D_OK || !d3dDev) {
LogError("\nCannot get D3D9 device from D3D9 surface\n");
return false;
}
hr = d3dDev->StretchRect(srcSurf, NULL, dstSurf, NULL, D3DTEXF_NONE);
if (hr != D3D_OK) {
LogError("\ncopy original surface to shared surface failed\n");
return false;
}
// Flush D3D queues and make sure D3D stuff is finished
pQuery_->Issue(D3DISSUE_END);
BOOL data;
while ((D3D_OK != pQuery_->GetData(&data, sizeof(BOOL), D3DGETDATA_FLUSH)) && (data != TRUE)) {
}
if (d3dDev) d3dDev->Release();
return true;
}
bool D3D9Object::copySharedToOrig() {
// Don't copy if there is no orig
if (NULL == getD3D9ResOrig()) return true;
IDirect3DDevice9Ex* d3dDev;
HRESULT hr;
ScopedLock sl(getResLock());
hr = getD3D9Resource()->GetContainer(IID_IDirect3DDevice9Ex, (void**)&d3dDev);
if (hr != D3D_OK || !d3dDev) {
LogError("\nCannot get D3D9 device from D3D9 surface\n");
return false;
}
hr = d3dDev->StretchRect(getD3D9Resource(), NULL, getD3D9ResOrig(), NULL, D3DTEXF_NONE);
if (hr != D3D_OK) {
LogError("\ncopy shared surface to original surface failed\n");
return false;
}
if (d3dDev) d3dDev->Release();
return true;
}
void Image2DD3D9::initDeviceMemory() {
deviceMemories_ =
reinterpret_cast<DeviceMemory*>(reinterpret_cast<char*>(this) + sizeof(Image2DD3D9));
memset(deviceMemories_, 0, context_().devices().size() * sizeof(DeviceMemory));
}
} // namespace amd
#endif //_WIN32
projects/hip/src/hipamd/src/cl_gl.cpp
+2432
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/src/cl_gl_amd.hpp
+379
Fájl megtekintése
@@ -0,0 +1,379 @@
/* Copyright (c) 2010-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef CL_GL_AMD_HPP_
#define CL_GL_AMD_HPP_
#ifdef _WIN32
#include <windows.h>
#else //!_WIN32
#include <dlfcn.h>
#endif //!_WIN32
#include <GL/gl.h>
#include <GL/glext.h>
#include "CL/cl_gl.h"
#ifndef _WIN32
#include <GL/glx.h>
#endif //!_WIN32
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <EGL/eglplatform.h>
#include "platform/context.hpp"
#include "platform/command.hpp"
namespace amd
{
//! Class GLObject keeps all the info about the GL object
//! from which the CL object is created
class GLObject : public InteropObject
{
protected:
cl_gl_object_type clGLType_; //!< CL GL object type
GLenum glTarget_;
GLuint gluiName_;
GLint gliMipLevel_;
GLenum glInternalFormat_;
GLint gliWidth_;
GLint gliHeight_;
GLint gliDepth_;
GLenum glCubemapFace_;
GLsizei glNumSamples_;
public:
//! GLObject constructor initializes member variables
GLObject(
GLenum glTarget,
GLuint gluiName,
GLint gliMipLevel,
GLenum glInternalFormat,
GLint gliWidth,
GLint gliHeight,
GLint gliDepth,
cl_gl_object_type clGLType,
GLenum glCubemapFace,
GLsizei glNumSamples
): // Initialization of member variables
clGLType_(clGLType),
glTarget_(glTarget),
gluiName_(gluiName),
gliMipLevel_(gliMipLevel),
glInternalFormat_(glInternalFormat),
gliWidth_(gliWidth),
gliHeight_(gliHeight),
gliDepth_(gliDepth),
glCubemapFace_(glCubemapFace),
glNumSamples_(glNumSamples)
{
}
virtual ~GLObject() {}
virtual GLObject* asGLObject() {return this;}
//! GLObject query functions to get GL info from member variables
GLenum getGLTarget() const {return glTarget_;}
GLuint getGLName() const {return gluiName_;}
GLint getGLMipLevel() const {return gliMipLevel_;}
GLenum getGLInternalFormat() const {return glInternalFormat_;}
GLint getGLSize() const {return gliWidth_;}
GLint getGLWidth() const {return gliWidth_;}
GLint getGLHeight() const {return gliHeight_;}
GLint getGLDepth() const {return gliDepth_;}
cl_gl_object_type getCLGLObjectType() const { return clGLType_; }
GLenum getCubemapFace() const {return glCubemapFace_;}
GLsizei getNumSamples() const { return glNumSamples_;}
};
//! Class BufferGL is drived from classes Buffer and GLObject
//! where the former keeps all data for CL object and
//! the latter keeps all data for GL object
class BufferGL : public Buffer, public GLObject
{
protected:
//! Initializes the device memory array which is nested
// after'BufferGL' object in memory layout.
virtual void initDeviceMemory();
public:
//! BufferGL constructor just calls constructors of base classes
//! to pass down the parameters
BufferGL(
Context& amdContext,
cl_mem_flags clFlags,
size_t uiSizeInBytes,
GLenum glTarget,
GLuint gluiName)
: // Call base classes constructors
Buffer(
amdContext,
clFlags,
uiSizeInBytes
),
GLObject(
glTarget,
gluiName,
0, // Mipmap level default
GL_ARRAY_BUFFER, // Just init to some value
(GLint) uiSizeInBytes,
1,
1,
CL_GL_OBJECT_BUFFER,
0,
0
)
{
setInteropObj(this);
}
virtual ~BufferGL() {}
virtual BufferGL* asBufferGL() { return this; }
};
//! Class ImageGL is derived from classes Image and GLObject
//! where the former keeps all data for CL object and
//! the latter keeps all data for GL object
class ImageGL : public Image, public GLObject
{
public:
//! ImageGL constructor just calls constructors of base classes
//! to pass down the parameters
ImageGL(
Context& amdContext,
cl_mem_object_type clType,
cl_mem_flags clFlags,
const Format& format,
size_t width,
size_t height,
size_t depth,
GLenum glTarget,
GLuint gluiName,
GLint gliMipLevel,
GLenum glInternalFormat,
cl_gl_object_type clGLType,
GLsizei numSamples,
GLenum glCubemapFace = 0)
: Image(amdContext, clType, clFlags, format, width, height, depth,
Format(format).getElementSize() * width,
Format(format).getElementSize() * width * depth)
, GLObject(glTarget, gluiName, gliMipLevel, glInternalFormat,
static_cast<GLint>(width), static_cast<GLint>(height),
static_cast<GLint>(depth), clGLType, glCubemapFace,numSamples)
{
setInteropObj(this);
}
virtual ~ImageGL() {}
protected:
//! Initializes the device memory array which is nested
// after'BufferGL' object in memory layout.
virtual void initDeviceMemory();
};
#ifdef _WIN32
#define APICALL WINAPI
#define GETPROCADDRESS GetProcAddress
#define API_GETPROCADDR "wglGetProcAddress"
#define FCN_STR_TYPE LPCSTR
typedef PROC (WINAPI* PFN_xxxGetProcAddress) (LPCSTR fcnName);
typedef HGLRC (APICALL* PFN_wglCreateContext) (HDC hdc);
typedef HGLRC (APICALL* PFN_wglGetCurrentContext) (void);
typedef HDC (APICALL* PFN_wglGetCurrentDC) (void);
typedef BOOL (APICALL* PFN_wglDeleteContext) (HGLRC hglrc);
typedef BOOL (APICALL* PFN_wglMakeCurrent) (HDC hdc, HGLRC hglrc);
typedef BOOL (APICALL* PFN_wglShareLists) (HGLRC hglrc1, HGLRC hglrc2);
#else //!_WIN32
#define APICALL // __stdcall //??? todo odintsov
#define API_GETPROCADDR "glXGetProcAddress"
#define GETPROCADDRESS dlsym
#define FCN_STR_TYPE const GLubyte*
#define WINAPI
#define PROC void*
typedef void* (*PFN_xxxGetProcAddress) (const GLubyte* procName);
// X11 typedef
typedef Display* (*PFNXOpenDisplay)(_Xconst char* display_name );
typedef int (*PFNXCloseDisplay)(Display* display );
//glx typedefs
typedef GLXDrawable (*PFNglXGetCurrentDrawable)();
typedef Display* (*PFNglXGetCurrentDisplay)();
typedef GLXContext (*PFNglXGetCurrentContext)( void );
typedef XVisualInfo* (*PFNglXChooseVisual)(Display *dpy, int screen, int *attribList);
typedef GLXContext(*PFNglXCreateContext)(Display* dpy,XVisualInfo* vis,GLXContext shareList,Bool direct);
typedef void(*PFNglXDestroyContext)(Display* dpy, GLXContext ctx);
typedef Bool(*PFNglXMakeCurrent)( Display* dpy, GLXDrawable drawable, GLXContext ctx);
typedef void* HMODULE;
#endif //!_WIN32
#define GLPREFIX(rtype, fcn, dclargs) \
typedef rtype (APICALL* PFN_##fcn) dclargs;
// Declare prototypes for GL functions
#include "gl_functions.hpp"
class GLFunctions
{
public:
//! Locks any access to the virtual GPUs
class SetIntEnv : public amd::StackObject {
public:
//! Default constructor
SetIntEnv(GLFunctions* env);
//! Destructor
~SetIntEnv();
//! Checks if the environment setup was successful
bool isValid() const { return isValid_; }
private:
GLFunctions* env_; //!< GL environment
bool isValid_; //!< If TRUE, then it's a valid setup
};
private:
HMODULE libHandle_;
int missed_; // Indicates how many GL functions not init'ed, if any
amd::Monitor lock_;
EGLDisplay eglDisplay_;
EGLContext eglOriginalContext_;
EGLContext eglInternalContext_;
EGLContext eglTempContext_;
bool isEGL_;
#ifdef _WIN32
HGLRC hOrigGLRC_;
HDC hDC_;
HGLRC hIntGLRC_; // handle for internal GLRC to access shared context
HDC tempDC_;
HGLRC tempGLRC_;
PFN_wglCreateContext wglCreateContext_;
PFN_wglGetCurrentContext wglGetCurrentContext_;
PFN_wglGetCurrentDC wglGetCurrentDC_;
PFN_wglDeleteContext wglDeleteContext_;
PFN_wglMakeCurrent wglMakeCurrent_;
PFN_wglShareLists wglShareLists_;
#else
public:
Display* Dpy_;
GLXDrawable Drawable_;
GLXContext origCtx_;
Display* intDpy_;
Window intDrawable_;
GLXContext intCtx_;
Display* tempDpy_;
GLXDrawable tempDrawable_;
GLXContext tempCtx_;
//pointers to X11 functions
PFNXOpenDisplay XOpenDisplay_;
PFNXCloseDisplay XCloseDisplay_;
//pointers to GLX functions
PFNglXGetCurrentDrawable glXGetCurrentDrawable_;
PFNglXGetCurrentDisplay glXGetCurrentDisplay_;
PFNglXGetCurrentContext glXGetCurrentContext_;
PFNglXChooseVisual glXChooseVisual_;
PFNglXCreateContext glXCreateContext_;
PFNglXDestroyContext glXDestroyContext_;
PFNglXMakeCurrent glXMakeCurrent_;
#endif
public:
GLFunctions(HMODULE h, bool isEGL);
~GLFunctions();
// Query CL-GL context association
bool isAssociated() const
{
if (isEGL_ && eglDisplay_ && eglOriginalContext_) return true;
#ifdef _WIN32
if(hDC_ && hOrigGLRC_) return true;
#else //!_WIN32
if(Dpy_ && origCtx_) return true;
#endif //!_WIN32
return false;
}
bool isEGL() const
{
return isEGL_;
}
// Accessor methods
#ifdef _WIN32
HGLRC getOrigGLRC() const {return hOrigGLRC_;}
HDC getDC() const {return hDC_;}
HGLRC getIntGLRC() const {return hIntGLRC_;}
#else //!_WIN32
Display* getDpy() const {return Dpy_;}
GLXDrawable getDrawable() const {return Drawable_;}
GLXContext getOrigCtx() const {return origCtx_;}
Display* getIntDpy() const {return intDpy_;}
GLXDrawable getIntDrawable() const {return intDrawable_;}
GLXContext getIntCtx() const {return intCtx_;}
EGLDisplay getEglDpy() const { return eglDisplay_; }
EGLContext getEglOrigCtx() const { return eglOriginalContext_; }
#endif //!_WIN32
// Initialize GL dynamic library and function pointers
bool init(intptr_t hdc, intptr_t hglrc);
// Return true if successful, false - if error occurred
bool setIntEnv();
bool restoreEnv();
amd::Monitor& getLock() { return lock_; }
PFN_xxxGetProcAddress GetProcAddress_;
#define GLPREFIX(rtype, fcn, dclargs) \
PFN_##fcn fcn##_;
// Declare pointers to GL functions
#include "gl_functions.hpp"
};
//! Functions for executing the GL related stuff
cl_mem clCreateFromGLBufferAMD(Context& amdContext, cl_mem_flags flags,
GLuint bufobj, cl_int* errcode_ret);
cl_mem clCreateFromGLTextureAMD(Context& amdContext, cl_mem_flags flags,
GLenum target, GLint miplevel, GLuint texture, int* errcode_ret);
cl_mem clCreateFromGLRenderbufferAMD(Context& amdContext, cl_mem_flags flags,
GLuint renderbuffer, int* errcode_ret);
bool
getCLFormatFromGL(
const Context& amdContext,
GLint gliInternalFormat,
cl_image_format* pclImageFormat,
int* piBytesPerPixel,
cl_mem_flags flags
);
} //namespace amd
#endif //CL_GL_AMD_HPP_
projects/hip/src/hipamd/src/cl_lqdflash_amd.cpp
+312
Fájl megtekintése
@@ -0,0 +1,312 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "cl_common.hpp"
#include <CL/cl_ext.h>
#include "platform/object.hpp"
#include "cl_lqdflash_amd.h"
#ifndef WITH_LIQUID_FLASH
#if (!defined(BUILD_HSA_TARGET) && defined(WITH_HSA_DEVICE) && \
defined(WITH_AMDGPU_PRO)) || defined(_WIN32)
#define WITH_LIQUID_FLASH 1
#endif // _WIN32
#endif
#if WITH_LIQUID_FLASH
#include "lf.h"
#include <locale>
#include <codecvt>
#endif // WITH_LIQUID_FLASH
namespace amd {
LiquidFlashFile::~LiquidFlashFile() { close(); }
bool LiquidFlashFile::open() {
#if WITH_LIQUID_FLASH
lf_status err;
lf_file_flags flags = 0;
switch (flags_) {
case CL_FILE_READ_ONLY_AMD:
flags = LF_READ;
break;
case CL_FILE_WRITE_ONLY_AMD:
flags = LF_WRITE;
break;
case CL_FILE_READ_WRITE_AMD:
flags = LF_READ | LF_WRITE;
break;
}
#ifdef ATI_OS_LINUX
assert(sizeof(wchar_t) != sizeof(lf_char));
std::string name_char;
std::wstring_convert<std::codecvt_utf8<wchar_t>, wchar_t> cv;
name_char = cv.to_bytes(name_);
handle_ = lfOpenFile(name_char.c_str(), flags, &err);
#else
handle_ = lfOpenFile(name_.c_str(), flags, &err);
#endif
if (err != lf_success) {
return false;
}
if (lfGetFileBlockSize((lf_file)handle_, &blockSize_) != lf_success) {
return false;
}
if (lfGetFileSize((lf_file)handle_, &fileSize_) != lf_success) {
return false;
}
return true;
#else
return false;
#endif // WITH_LIQUID_FLASH
}
void LiquidFlashFile::close() {
#if WITH_LIQUID_FLASH
if (handle_ != NULL) {
lfReleaseFile((lf_file)handle_);
handle_ = NULL;
}
#endif // WITH_LIQUID_FLASH
}
bool LiquidFlashFile::transferBlock(bool writeBuffer, void* srcDst, uint64_t bufferSize,
uint64_t fileOffset, uint64_t bufferOffset,
uint64_t size) const {
#if WITH_LIQUID_FLASH
lf_status status;
lf_region_descriptor region = {fileOffset / blockSize(), bufferOffset / blockSize(),
size / blockSize()};
if (writeBuffer) {
status = lfReadFile(srcDst, bufferSize, (lf_file)handle_, 1, &region, NULL);
} else {
status = lfWriteFile(srcDst, bufferSize, (lf_file)handle_, 1, &region, NULL);
}
if (lf_success == status) {
return true;
} else {
return false;
}
#else
return false;
#endif // WITH_LIQUID_FLASH
}
} // namespace amd
/*! \addtogroup API
* @{
*
* \addtogroup AMD_Extensions
* @{
*
*/
RUNTIME_ENTRY_RET(cl_file_amd, clCreateSsgFileObjectAMD,
(cl_context context, cl_file_flags_amd flags, const wchar_t* file_name,
cl_int* errcode_ret)) {
#if WITH_LIQUID_FLASH && defined ATI_OS_LINUX
if (!is_valid(context)) {
*not_null(errcode_ret) = CL_INVALID_CONTEXT;
LogWarning("invalid parameter \"context\"");
return (cl_file_amd)0;
}
const std::vector<amd::Device*>& devices = as_amd(context)->devices();
bool supportPass = false;
for (auto& dev : devices) {
if (lf_success == lfCheckExtensionSupportForDevice(dev->info().pcieDeviceId_,
dev->info().pcieRevisionId_)) {
supportPass = true;
break;
}
}
if (!supportPass) {
*not_null(errcode_ret) = CL_INVALID_DEVICE;
LogWarning("SSG isn't supported");
return (cl_file_amd)0;
}
#endif
amd::LiquidFlashFile* file = new amd::LiquidFlashFile(file_name, flags);
if (file == NULL) {
*not_null(errcode_ret) = CL_OUT_OF_HOST_MEMORY;
return (cl_file_amd)0;
}
if (!file->open()) {
*not_null(errcode_ret) = CL_INVALID_VALUE;
delete file;
return (cl_file_amd)0;
}
*not_null(errcode_ret) = CL_SUCCESS;
return as_cl(file);
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clGetSsgFileObjectInfoAMD,
(cl_file_amd file, cl_file_info_amd param_name, size_t param_value_size,
void* param_value, size_t* param_value_size_ret)) {
if (!is_valid(file)) {
return CL_INVALID_FILE_OBJECT_AMD;
}
switch (param_name) {
case CL_FILE_BLOCK_SIZE_AMD: {
cl_uint blockSize = as_amd(file)->blockSize();
return amd::clGetInfo(blockSize, param_value_size, param_value, param_value_size_ret);
}
case CL_FILE_SIZE_AMD: {
cl_ulong fileSize = as_amd(file)->fileSize();
return amd::clGetInfo(fileSize, param_value_size, param_value, param_value_size_ret);
}
default:
break;
}
return CL_INVALID_VALUE;
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clRetainSsgFileObjectAMD, (cl_file_amd file)) {
if (!is_valid(file)) {
return CL_INVALID_FILE_OBJECT_AMD;
}
as_amd(file)->retain();
return CL_SUCCESS;
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clReleaseSsgFileObjectAMD, (cl_file_amd file)) {
if (!is_valid(file)) {
return CL_INVALID_FILE_OBJECT_AMD;
}
as_amd(file)->release();
return CL_SUCCESS;
}
RUNTIME_EXIT
static cl_int EnqueueTransferBufferFromSsgFileAMD(
cl_bool isWrite, cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write,
size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset,
cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event) {
if (!is_valid(command_queue)) {
return CL_INVALID_COMMAND_QUEUE;
}
if (!is_valid(buffer)) {
return CL_INVALID_MEM_OBJECT;
}
amd::Buffer* pBuffer = as_amd(buffer)->asBuffer();
if (pBuffer == NULL) {
return CL_INVALID_MEM_OBJECT;
}
if (pBuffer->getMemFlags() & (CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS)) {
return CL_INVALID_OPERATION;
}
amd::HostQueue* queue = as_amd(command_queue)->asHostQueue();
if (NULL == queue) {
return CL_INVALID_COMMAND_QUEUE;
}
amd::HostQueue& hostQueue = *queue;
if (hostQueue.context() != pBuffer->getContext()) {
return CL_INVALID_CONTEXT;
}
if (!is_valid(file)) {
return CL_INVALID_FILE_OBJECT_AMD;
}
amd::LiquidFlashFile* amdFile = as_amd(file);
amd::Coord3D bufferOffset(buffer_offset, 0, 0);
amd::Coord3D bufferSize(cb, 1, 1);
if ((!pBuffer->validateRegion(bufferOffset, bufferSize)) ||
// LF library supports aligned sizes only
((buffer_offset % amdFile->blockSize()) != 0) || ((cb % amdFile->blockSize()) != 0) ||
((file_offset % amdFile->blockSize()) != 0)) {
return CL_INVALID_VALUE;
}
amd::Command::EventWaitList eventWaitList;
cl_int err = amd::clSetEventWaitList(eventWaitList, hostQueue, num_events_in_wait_list,
event_wait_list);
if (err != CL_SUCCESS) {
return err;
}
amd::TransferBufferFileCommand* command;
command = new amd::TransferBufferFileCommand(
isWrite ? CL_COMMAND_READ_SSG_FILE_AMD : CL_COMMAND_WRITE_SSG_FILE_AMD, hostQueue,
eventWaitList, *pBuffer, bufferOffset, bufferSize, amdFile, file_offset);
if (command == NULL) {
return CL_OUT_OF_HOST_MEMORY;
}
// Make sure we have memory for the command execution
if (!command->validateMemory()) {
delete command;
return CL_MEM_OBJECT_ALLOCATION_FAILURE;
}
command->enqueue();
if (blocking_write) {
command->awaitCompletion();
}
*not_null(event) = as_cl(&command->event());
if (event == NULL) {
command->release();
}
return CL_SUCCESS;
}
RUNTIME_ENTRY(cl_int, clEnqueueReadSsgFileAMD,
(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write,
size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset,
cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) {
return EnqueueTransferBufferFromSsgFileAMD(CL_TRUE, command_queue, buffer, blocking_write,
buffer_offset, cb, file, file_offset,
num_events_in_wait_list, event_wait_list, event);
}
RUNTIME_EXIT
RUNTIME_ENTRY(cl_int, clEnqueueWriteSsgFileAMD,
(cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write,
size_t buffer_offset, size_t cb, cl_file_amd file, size_t file_offset,
cl_uint num_events_in_wait_list, const cl_event* event_wait_list, cl_event* event)) {
return EnqueueTransferBufferFromSsgFileAMD(CL_FALSE, command_queue, buffer, blocking_write,
buffer_offset, cb, file, file_offset,
num_events_in_wait_list, event_wait_list, event);
}
RUNTIME_EXIT
projects/hip/src/hipamd/src/cl_lqdflash_amd.h
+58
Fájl megtekintése
@@ -0,0 +1,58 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef __CL_LQDFLASH_AMD_H
#define __CL_LQDFLASH_AMD_H
#include "CL/cl_ext.h"
#ifdef __cplusplus
extern "C" {
#endif /*__cplusplus*/
extern CL_API_ENTRY cl_file_amd CL_API_CALL
clCreateSsgFileObjectAMD(cl_context context, cl_file_flags_amd flags, const wchar_t* file_name,
cl_int* errcode_ret) CL_EXT_SUFFIX__VERSION_1_2;
extern CL_API_ENTRY cl_int CL_API_CALL clGetSsgFileObjectInfoAMD(
cl_file_amd file, cl_file_info_amd param_name, size_t param_value_size, void* param_value,
size_t* param_value_size_ret) CL_EXT_SUFFIX__VERSION_1_2;
extern CL_API_ENTRY cl_int CL_API_CALL clRetainSsgFileObjectAMD(cl_file_amd file)
CL_EXT_SUFFIX__VERSION_1_2;
extern CL_API_ENTRY cl_int CL_API_CALL clReleaseSsgFileObjectAMD(cl_file_amd file)
CL_EXT_SUFFIX__VERSION_1_2;
extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueReadSsgFileAMD(
cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, size_t buffer_offset,
size_t cb, cl_file_amd file, size_t file_offset, cl_uint num_events_in_wait_list,
const cl_event* event_wait_list, cl_event* event) CL_EXT_SUFFIX__VERSION_1_2;
extern CL_API_ENTRY cl_int CL_API_CALL clEnqueueWriteSsgFileAMD(
cl_command_queue command_queue, cl_mem buffer, cl_bool blocking_write, size_t buffer_offset,
size_t cb, cl_file_amd file, size_t file_offset, cl_uint num_events_in_wait_list,
const cl_event* event_wait_list, cl_event* event) CL_EXT_SUFFIX__VERSION_1_2;
#ifdef __cplusplus
} /*extern "C"*/
#endif /*__cplusplus*/
#endif
projects/hip/src/hipamd/src/fixme.cpp
+32
Fájl megtekintése
@@ -0,0 +1,32 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "vdi_common.hpp"
#include <icd/loader/icd_dispatch.h>
cl_icd_dispatch amd::ICDDispatchedObject::icdVendorDispatch_[] = {0};
amd::PlatformIDS amd::PlatformID::Platform = {amd::ICDDispatchedObject::icdVendorDispatch_};
RUNTIME_ENTRY(cl_int, clGetDeviceIDs,
(cl_platform_id platform, cl_device_type device_type, cl_uint num_entries,
cl_device_id* devices, cl_uint* num_devices)) {
return CL_SUCCESS;
}
RUNTIME_EXIT
projects/hip/src/hipamd/src/hip_activity.cpp
+35
Fájl megtekintése
@@ -0,0 +1,35 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "platform/activity.hpp"
extern "C" void hipInitActivityCallback(void* id_callback, void* op_callback, void* arg) {
activity_prof::CallbacksTable::init(reinterpret_cast<activity_prof::id_callback_fun_t>(id_callback),
reinterpret_cast<activity_prof::callback_fun_t>(op_callback),
arg);
}
extern "C" bool hipEnableActivityCallback(unsigned op, bool enable) {
return activity_prof::CallbacksTable::SetEnabled(op, enable);
}
extern "C" const char* hipGetCmdName(unsigned op) {
return getOclCommandKindString(static_cast<uint32_t>(op));
}
projects/hip/src/hipamd/src/hip_code_object.cpp
Executable
+850
Fájl megtekintése
@@ -0,0 +1,850 @@
/*
Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#include "hip_code_object.hpp"
#include "amd_hsa_elf.hpp"
#include <cstring>
#include <hip/amd_detail/amd_driver_types.h>
#include "hip/hip_runtime_api.h"
#include "hip/hip_runtime.h"
#include "hip_internal.hpp"
#include "platform/program.hpp"
#include <elf/elf.hpp>
hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind,
amd::HostQueue& queue, bool isAsync = false);
hipError_t ihipFree(void* ptr);
//forward declaration of methods required for managed variables
hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0);
namespace {
size_t constexpr strLiteralLength(char const* str) {
return *str ? 1 + strLiteralLength(str + 1) : 0;
}
constexpr char const* CLANG_OFFLOAD_BUNDLER_MAGIC_STR = "__CLANG_OFFLOAD_BUNDLE__";
constexpr char const* OFFLOAD_KIND_HIP = "hip";
constexpr char const* OFFLOAD_KIND_HIPV4 = "hipv4";
constexpr char const* OFFLOAD_KIND_HCC = "hcc";
constexpr char const* AMDGCN_TARGET_TRIPLE = "amdgcn-amd-amdhsa-";
// ClangOFFLOADBundle info.
static constexpr size_t bundle_magic_string_size =
strLiteralLength(CLANG_OFFLOAD_BUNDLER_MAGIC_STR);
// Clang Offload bundler description & Header.
struct __ClangOffloadBundleInfo {
uint64_t offset;
uint64_t size;
uint64_t bundleEntryIdSize;
const char bundleEntryId[1];
};
struct __ClangOffloadBundleHeader {
const char magic[bundle_magic_string_size - 1];
uint64_t numOfCodeObjects;
__ClangOffloadBundleInfo desc[1];
};
} // namespace
namespace hip {
uint64_t CodeObject::ElfSize(const void *emi) {
return amd::Elf::getElfSize(emi);
}
static bool getProcName(uint32_t EFlags, std::string& proc_name, bool& xnackSupported,
bool& sramEccSupported) {
switch (EFlags & EF_AMDGPU_MACH) {
case EF_AMDGPU_MACH_AMDGCN_GFX700:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx700";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX701:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx701";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX702:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx702";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX703:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx703";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX704:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx704";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX705:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx705";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX801:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx801";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX802:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx802";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX803:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx803";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX805:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx805";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX810:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx810";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX900:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx900";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX902:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx902";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX904:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx904";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX906:
xnackSupported = true;
sramEccSupported = true;
proc_name = "gfx906";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX908:
xnackSupported = true;
sramEccSupported = true;
proc_name = "gfx908";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX909:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx909";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX90A:
xnackSupported = true;
sramEccSupported = true;
proc_name = "gfx90a";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX90C:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx90c";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1010:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx1010";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1011:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx1011";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1012:
xnackSupported = true;
sramEccSupported = false;
proc_name = "gfx1012";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1030:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx1030";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1031:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx1031";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1032:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx1032";
break;
case EF_AMDGPU_MACH_AMDGCN_GFX1033:
xnackSupported = false;
sramEccSupported = false;
proc_name = "gfx1033";
break;
default:
return false;
}
return true;
}
static bool getTripleTargetIDFromCodeObject(const void* code_object, std::string& target_id,
unsigned& co_version) {
if (!code_object) return false;
const Elf64_Ehdr* ehdr = reinterpret_cast<const Elf64_Ehdr*>(code_object);
if (ehdr->e_machine != EM_AMDGPU) return false;
if (ehdr->e_ident[EI_OSABI] != ELFOSABI_AMDGPU_HSA) return false;
bool isXnackSupported{false}, isSramEccSupported{false};
std::string proc_name;
if (!getProcName(ehdr->e_flags, proc_name, isXnackSupported, isSramEccSupported)) return false;
target_id = std::string(AMDGCN_TARGET_TRIPLE) + '-' + proc_name;
switch (ehdr->e_ident[EI_ABIVERSION]) {
case ELFABIVERSION_AMDGPU_HSA_V2: {
co_version = 2;
return false;
}
case ELFABIVERSION_AMDGPU_HSA_V3: {
co_version = 3;
if (isSramEccSupported) {
if (ehdr->e_flags & EF_AMDGPU_FEATURE_SRAMECC_V3)
target_id += ":sramecc+";
else
target_id += ":sramecc-";
}
if (isXnackSupported) {
if (ehdr->e_flags & EF_AMDGPU_FEATURE_XNACK_V3)
target_id += ":xnack+";
else
target_id += ":xnack-";
}
break;
}
case ELFABIVERSION_AMDGPU_HSA_V4: {
co_version = 4;
unsigned co_sram_value = (ehdr->e_flags) & EF_AMDGPU_FEATURE_SRAMECC_V4;
if (co_sram_value == EF_AMDGPU_FEATURE_SRAMECC_OFF_V4)
target_id += ":sramecc-";
else if (co_sram_value == EF_AMDGPU_FEATURE_SRAMECC_ON_V4)
target_id += ":sramecc+";
unsigned co_xnack_value = (ehdr->e_flags) & EF_AMDGPU_FEATURE_XNACK_V4;
if (co_xnack_value == EF_AMDGPU_FEATURE_XNACK_OFF_V4)
target_id += ":xnack-";
else if (co_xnack_value == EF_AMDGPU_FEATURE_XNACK_ON_V4)
target_id += ":xnack+";
break;
}
default: {
return false;
}
}
return true;
}
// Consumes the string 'consume_' from the starting of the given input
// eg: input = amdgcn-amd-amdhsa--gfx908 and consume_ is amdgcn-amd-amdhsa--
// input will become gfx908.
static bool consume(std::string& input, std::string consume_) {
if (input.substr(0, consume_.size()) != consume_) {
return false;
}
input = input.substr(consume_.size());
return true;
}
// Trim String till character, will be used to get gpuname
// example: input is gfx908:sram-ecc+ and trim char is :
// input will become sram-ecc+.
static std::string trimName(std::string& input, char trim) {
auto pos_ = input.find(trim);
auto res = input;
if (pos_ == std::string::npos) {
input = "";
} else {
res = input.substr(0, pos_);
input = input.substr(pos_);
}
return res;
}
static char getFeatureValue(std::string& input, std::string feature) {
char res = ' ';
if (consume(input, std::move(feature))) {
res = input[0];
input = input.substr(1);
}
return res;
}
static bool getTargetIDValue(std::string& input, std::string& processor, char& sramecc_value,
char& xnack_value) {
processor = trimName(input, ':');
sramecc_value = getFeatureValue(input, std::string(":sramecc"));
if (sramecc_value != ' ' && sramecc_value != '+' && sramecc_value != '-') return false;
xnack_value = getFeatureValue(input, std::string(":xnack"));
if (xnack_value != ' ' && xnack_value != '+' && xnack_value != '-') return false;
return true;
}
static bool getTripleTargetID(std::string bundled_co_entry_id, const void* code_object,
std::string& co_triple_target_id, unsigned& co_version) {
std::string offload_kind = trimName(bundled_co_entry_id, '-');
if (offload_kind != OFFLOAD_KIND_HIPV4 && offload_kind != OFFLOAD_KIND_HIP &&
offload_kind != OFFLOAD_KIND_HCC)
return false;
if (offload_kind != OFFLOAD_KIND_HIPV4)
return getTripleTargetIDFromCodeObject(code_object, co_triple_target_id, co_version);
// For code object V4 onwards the bundled code object entry ID correctly
// specifies the target tripple.
co_version = 4;
co_triple_target_id = bundled_co_entry_id.substr(1);
return true;
}
static bool isCodeObjectCompatibleWithDevice(std::string co_triple_target_id,
std::string agent_triple_target_id) {
// Primitive Check
if (co_triple_target_id == agent_triple_target_id) return true;
// Parse code object triple target id
if (!consume(co_triple_target_id, std::string(AMDGCN_TARGET_TRIPLE) + '-')) {
return false;
}
std::string co_processor;
char co_sram_ecc, co_xnack;
if (!getTargetIDValue(co_triple_target_id, co_processor, co_sram_ecc, co_xnack)) {
return false;
}
if (!co_triple_target_id.empty()) return false;
// Parse agent isa triple target id
if (!consume(agent_triple_target_id, std::string(AMDGCN_TARGET_TRIPLE) + '-')) {
return false;
}
std::string agent_isa_processor;
char isa_sram_ecc, isa_xnack;
if (!getTargetIDValue(agent_triple_target_id, agent_isa_processor, isa_sram_ecc, isa_xnack)) {
return false;
}
if (!agent_triple_target_id.empty()) return false;
// Check for compatibility
if (agent_isa_processor != co_processor) return false;
if (co_sram_ecc != ' ') {
if (co_sram_ecc != isa_sram_ecc) return false;
}
if (co_xnack != ' ') {
if (co_xnack != isa_xnack) return false;
}
return true;
}
// This will be moved to COMGR eventually
hipError_t CodeObject::ExtractCodeObjectFromFile(amd::Os::FileDesc fdesc, size_t fsize,
const void ** image, const std::vector<std::string>& device_names,
std::vector<std::pair<const void*, size_t>>& code_objs) {
hipError_t hip_error = hipSuccess;
if (fdesc < 0) {
return hipErrorFileNotFound;
}
// Map the file to memory, with offset 0.
//file will be unmapped in ModuleUnload
//const void* image = nullptr;
if (!amd::Os::MemoryMapFileDesc(fdesc, fsize, 0, image)) {
return hipErrorInvalidValue;
}
// retrieve code_objs{binary_image, binary_size} for devices
hip_error = extractCodeObjectFromFatBinary(*image, device_names, code_objs);
return hip_error;
}
// This will be moved to COMGR eventually
hipError_t CodeObject::ExtractCodeObjectFromMemory(const void* data,
const std::vector<std::string>& device_names,
std::vector<std::pair<const void*, size_t>>& code_objs,
std::string& uri) {
// Get the URI from memory
if (!amd::Os::GetURIFromMemory(data, 0, uri)) {
return hipErrorInvalidValue;
}
return extractCodeObjectFromFatBinary(data, device_names, code_objs);
}
// This will be moved to COMGR eventually
hipError_t CodeObject::extractCodeObjectFromFatBinary(const void* data,
const std::vector<std::string>& agent_triple_target_ids,
std::vector<std::pair<const void*, size_t>>& code_objs) {
std::string magic((const char*)data, bundle_magic_string_size);
if (magic.compare(CLANG_OFFLOAD_BUNDLER_MAGIC_STR)) {
return hipErrorInvalidKernelFile;
}
// Initialize Code objects
code_objs.reserve(agent_triple_target_ids.size());
for (size_t i = 0; i < agent_triple_target_ids.size(); i++) {
code_objs.push_back(std::make_pair(nullptr, 0));
}
const auto obheader = reinterpret_cast<const __ClangOffloadBundleHeader*>(data);
const auto* desc = &obheader->desc[0];
size_t num_code_objs = code_objs.size();
for (uint64_t i = 0; i < obheader->numOfCodeObjects; ++i,
desc = reinterpret_cast<const __ClangOffloadBundleInfo*>(
reinterpret_cast<uintptr_t>(&desc->bundleEntryId[0]) +
desc->bundleEntryIdSize)) {
const void* image =
reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(obheader) + desc->offset);
const size_t image_size = desc->size;
if (num_code_objs == 0) break;
std::string bundleEntryId{desc->bundleEntryId, desc->bundleEntryIdSize};
unsigned co_version = 0;
std::string co_triple_target_id;
if (!getTripleTargetID(bundleEntryId, image, co_triple_target_id, co_version)) continue;
for (size_t dev = 0; dev < agent_triple_target_ids.size(); ++dev) {
if (code_objs[dev].first) continue;
if (isCodeObjectCompatibleWithDevice(co_triple_target_id, agent_triple_target_ids[dev])) {
code_objs[dev] = std::make_pair(image, image_size);
--num_code_objs;
}
}
}
if (num_code_objs == 0) {
return hipSuccess;
} else {
LogPrintfError("%s",
"hipErrorNoBinaryForGpu: Unable to find code object for all current devices!");
LogPrintfError("%s", " Devices:");
for (size_t i = 0; i < agent_triple_target_ids.size(); i++) {
LogPrintfError(" %s - [%s]", agent_triple_target_ids[i].c_str(),
((code_objs[i].first) ? "Found" : "Not Found"));
}
const auto obheader = reinterpret_cast<const __ClangOffloadBundleHeader*>(data);
const auto* desc = &obheader->desc[0];
LogPrintfError("%s", " Bundled Code Objects:");
for (uint64_t i = 0; i < obheader->numOfCodeObjects; ++i,
desc = reinterpret_cast<const __ClangOffloadBundleInfo*>(
reinterpret_cast<uintptr_t>(&desc->bundleEntryId[0]) +
desc->bundleEntryIdSize)) {
std::string bundleEntryId{desc->bundleEntryId, desc->bundleEntryIdSize};
const void* image =
reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(obheader) + desc->offset);
unsigned co_version = 0;
std::string co_triple_target_id;
bool valid_co = getTripleTargetID(bundleEntryId, image, co_triple_target_id, co_version);
if (valid_co) {
LogPrintfError(" %s - [code object v%u is %s]", bundleEntryId.c_str(), co_version,
co_triple_target_id.c_str());
} else {
LogPrintfError(" %s - [Unsupported]", bundleEntryId.c_str());
}
}
guarantee(false, "hipErrorNoBinaryForGpu: Unable to find code object for all current devices!");
return hipErrorNoBinaryForGpu;
}
}
hipError_t DynCO::loadCodeObject(const char* fname, const void* image) {
amd::ScopedLock lock(dclock_);
// Number of devices = 1 in dynamic code object
fb_info_ = new FatBinaryInfo(fname, image);
std::vector<hip::Device*> devices = { g_devices[ihipGetDevice()] };
IHIP_RETURN_ONFAIL(fb_info_->ExtractFatBinary(devices));
// No Lazy loading for DynCO
IHIP_RETURN_ONFAIL(fb_info_->BuildProgram(ihipGetDevice()));
// Define Global variables
IHIP_RETURN_ONFAIL(populateDynGlobalVars());
// Define Global functions
IHIP_RETURN_ONFAIL(populateDynGlobalFuncs());
return hipSuccess;
}
//Dynamic Code Object
DynCO::~DynCO() {
amd::ScopedLock lock(dclock_);
for (auto& elem : vars_) {
if(elem.second->getVarKind() == Var::DVK_Managed) {
ihipFree(elem.second->getManagedVarPtr());
}
delete elem.second;
}
vars_.clear();
for (auto& elem : functions_) {
delete elem.second;
}
functions_.clear();
delete fb_info_;
}
hipError_t DynCO::getDeviceVar(DeviceVar** dvar, std::string var_name) {
amd::ScopedLock lock(dclock_);
CheckDeviceIdMatch();
auto it = vars_.find(var_name);
if (it == vars_.end()) {
LogPrintfError("Cannot find the Var: %s ", var_name.c_str());
return hipErrorNotFound;
}
it->second->getDeviceVar(dvar, device_id_, module());
return hipSuccess;
}
hipError_t DynCO::getDynFunc(hipFunction_t* hfunc, std::string func_name) {
amd::ScopedLock lock(dclock_);
CheckDeviceIdMatch();
if(hfunc == nullptr) {
return hipErrorInvalidValue;
}
auto it = functions_.find(func_name);
if (it == functions_.end()) {
LogPrintfError("Cannot find the function: %s ", func_name.c_str());
return hipErrorNotFound;
}
/* See if this could be solved */
return it->second->getDynFunc(hfunc, module());
}
hipError_t DynCO::initDynManagedVars(const std::string& managedVar) {
amd::ScopedLock lock(dclock_);
DeviceVar* dvar;
void* pointer = nullptr;
hipError_t status = hipSuccess;
// To get size of the managed variable
status = getDeviceVar(&dvar, managedVar + ".managed");
if (status != hipSuccess) {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to get .managed device variable:%s",
status, managedVar.c_str());
return status;
}
// Allocate managed memory for these symbols
status = ihipMallocManaged(&pointer, dvar->size());
if (status != hipSuccess) {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to allocate managed memory", status);
guarantee(false, "Error during allocation of managed memory!");
}
// update as manager variable and set managed memory pointer and size
auto it = vars_.find(managedVar);
it->second->setManagedVarInfo(pointer, dvar->size());
// copy initial value to the managed variable to the managed memory allocated
amd::HostQueue* queue = hip::getNullStream();
if (queue != nullptr) {
status = ihipMemcpy(pointer, reinterpret_cast<address>(dvar->device_ptr()), dvar->size(),
hipMemcpyDeviceToDevice, *queue);
if (status != hipSuccess) {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to copy device ptr:%s", status,
managedVar.c_str());
return status;
}
} else {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL");
return hipErrorInvalidResourceHandle;
}
// Get deivce ptr to initialize with managed memory pointer
status = getDeviceVar(&dvar, managedVar);
if (status != hipSuccess) {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to get managed device variable:%s",
status, managedVar.c_str());
return status;
}
// copy managed memory pointer to the managed device variable
status = ihipMemcpy(reinterpret_cast<address>(dvar->device_ptr()), &pointer, dvar->size(),
hipMemcpyHostToDevice, *queue);
if (status != hipSuccess) {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Status %d, failed to copy device ptr:%s", status,
managedVar.c_str());
return status;
}
return status;
}
hipError_t DynCO::populateDynGlobalVars() {
amd::ScopedLock lock(dclock_);
std::vector<std::string> var_names;
std::vector<std::string> undef_var_names;
std::string managedVarExt = ".managed";
// For Dynamic Modules there is only one hipFatBinaryDevInfo_
device::Program* dev_program = fb_info_->GetProgram(ihipGetDevice())
->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
if (!dev_program->getGlobalVarFromCodeObj(&var_names)) {
LogPrintfError("Could not get Global vars from Code Obj for Module: 0x%x \n", module());
return hipErrorSharedObjectSymbolNotFound;
}
for (auto& elem : var_names) {
vars_.insert(
std::make_pair(elem, new Var(elem, Var::DeviceVarKind::DVK_Variable, 0, 0, 0, nullptr)));
}
for (auto& elem : var_names) {
if (elem.find(managedVarExt) != std::string::npos) {
std::string managedVar = elem;
managedVar.erase(managedVar.length() - managedVarExt.length(), managedVarExt.length());
initDynManagedVars(managedVar);
}
}
return hipSuccess;
}
hipError_t DynCO::populateDynGlobalFuncs() {
amd::ScopedLock lock(dclock_);
std::vector<std::string> func_names;
device::Program* dev_program
= fb_info_->GetProgram(ihipGetDevice())->getDeviceProgram(
*hip::getCurrentDevice()->devices()[0]);
// Get all the global func names from COMGR
if (!dev_program->getGlobalFuncFromCodeObj(&func_names)) {
LogPrintfError("Could not get Global Funcs from Code Obj for Module: 0x%x \n", module());
return hipErrorSharedObjectSymbolNotFound;
}
for (auto& elem : func_names) {
functions_.insert(std::make_pair(elem, new Function(elem)));
}
return hipSuccess;
}
//Static Code Object
StatCO::StatCO() {
}
StatCO::~StatCO() {
amd::ScopedLock lock(sclock_);
for (auto& elem : functions_) {
delete elem.second;
}
functions_.clear();
for (auto& elem : vars_) {
delete elem.second;
}
vars_.clear();
}
hipError_t StatCO::digestFatBinary(const void* data, FatBinaryInfo*& programs) {
amd::ScopedLock lock(sclock_);
if (programs != nullptr) {
return hipSuccess;
}
// Create a new fat binary object and extract the fat binary for all devices.
programs = new FatBinaryInfo(nullptr, data);
IHIP_RETURN_ONFAIL(programs->ExtractFatBinary(g_devices));
return hipSuccess;
}
FatBinaryInfo** StatCO::addFatBinary(const void* data, bool initialized) {
amd::ScopedLock lock(sclock_);
if (initialized) {
digestFatBinary(data, modules_[data]);
}
return &modules_[data];
}
hipError_t StatCO::removeFatBinary(FatBinaryInfo** module) {
amd::ScopedLock lock(sclock_);
auto vit = vars_.begin();
while (vit != vars_.end()) {
if (vit->second->moduleInfo() == module) {
delete vit->second;
vit = vars_.erase(vit);
} else {
++vit;
}
}
auto it = managedVars_.begin();
while (it != managedVars_.end()) {
if ((*it)->moduleInfo() == module) {
ihipFree((*it)->getManagedVarPtr());
delete *it;
managedVars_.erase(it);
} else {
++it;
}
}
auto fit = functions_.begin();
while (fit != functions_.end()) {
if (fit->second->moduleInfo() == module) {
delete fit->second;
fit = functions_.erase(fit);
} else {
++fit;
}
}
auto mit = modules_.begin();
while (mit != modules_.end()) {
if (&mit->second == module) {
delete mit->second;
mit = modules_.erase(mit);
} else {
++mit;
}
}
return hipSuccess;
}
hipError_t StatCO::registerStatFunction(const void* hostFunction, Function* func) {
amd::ScopedLock lock(sclock_);
if (functions_.find(hostFunction) != functions_.end()) {
DevLogPrintfError("hostFunctionPtr: 0x%x already exists", hostFunction);
}
functions_.insert(std::make_pair(hostFunction, func));
return hipSuccess;
}
hipError_t StatCO::getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId) {
amd::ScopedLock lock(sclock_);
const auto it = functions_.find(hostFunction);
if (it == functions_.end()) {
return hipErrorInvalidSymbol;
}
return it->second->getStatFunc(hfunc, deviceId);
}
hipError_t StatCO::getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId) {
amd::ScopedLock lock(sclock_);
const auto it = functions_.find(hostFunction);
if (it == functions_.end()) {
return hipErrorInvalidSymbol;
}
return it->second->getStatFuncAttr(func_attr, deviceId);
}
hipError_t StatCO::registerStatGlobalVar(const void* hostVar, Var* var) {
amd::ScopedLock lock(sclock_);
if (vars_.find(hostVar) != vars_.end()) {
return hipErrorInvalidSymbol;
}
vars_.insert(std::make_pair(hostVar, var));
return hipSuccess;
}
hipError_t StatCO::getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr,
size_t* size_ptr) {
amd::ScopedLock lock(sclock_);
const auto it = vars_.find(hostVar);
if (it == vars_.end()) {
return hipErrorInvalidSymbol;
}
DeviceVar* dvar = nullptr;
IHIP_RETURN_ONFAIL(it->second->getStatDeviceVar(&dvar, deviceId));
*dev_ptr = dvar->device_ptr();
*size_ptr = dvar->size();
return hipSuccess;
}
hipError_t StatCO::registerStatManagedVar(Var* var) {
managedVars_.emplace_back(var);
return hipSuccess;
}
hipError_t StatCO::initStatManagedVarDevicePtr(int deviceId) {
amd::ScopedLock lock(sclock_);
if (managedVarsDevicePtrInitalized_.find(deviceId) == managedVarsDevicePtrInitalized_.end() ||
!managedVarsDevicePtrInitalized_[deviceId]) {
for (auto var : managedVars_) {
DeviceVar* dvar = nullptr;
IHIP_RETURN_ONFAIL(var->getStatDeviceVar(&dvar, deviceId));
amd::HostQueue* queue = hip::getNullStream();
if(queue != nullptr) {
ihipMemcpy(reinterpret_cast<address>(dvar->device_ptr()), var->getManagedVarPtr(),
dvar->size(), hipMemcpyHostToDevice, *queue);
} else {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL");
return hipErrorInvalidResourceHandle;
}
}
managedVarsDevicePtrInitalized_[deviceId] = true;
}
return hipSuccess;
}
}; //namespace: hip
projects/hip/src/hipamd/src/hip_code_object.hpp
Executable
+165
Fájl megtekintése
@@ -0,0 +1,165 @@
/*
Copyright (c) 2015-2020 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#ifndef HIP_CODE_OBJECT_HPP
#define HIP_CODE_OBJECT_HPP
#include "hip_global.hpp"
#include <cstring>
#include <unordered_map>
#include "hip/hip_runtime.h"
#include "hip/hip_runtime_api.h"
#include "hip_internal.hpp"
#include "device/device.hpp"
#include "platform/program.hpp"
//Forward Declaration for friend usage
class PlatformState;
namespace hip {
//Code Object base class
class CodeObject {
public:
virtual ~CodeObject() {}
// Functions to add_dev_prog and build
static hipError_t add_program(int deviceId, hipModule_t hmod, const void* binary_ptr,
size_t binary_size);
static hipError_t build_module(hipModule_t hmod, const std::vector<amd::Device*>& devices);
// Given an file desc and file size, extracts to code object for corresponding devices,
// return code_objs{binary_ptr, binary_size}, which could be used to determine foffset
static hipError_t ExtractCodeObjectFromFile(amd::Os::FileDesc fdesc, size_t fsize,
const void ** image, const std::vector<std::string>& device_names,
std::vector<std::pair<const void*, size_t>>& code_objs);
// Given an ptr to memory, extracts to code object for corresponding devices,
// returns code_objs{binary_ptr, binary_size} and uniform resource indicator
static hipError_t ExtractCodeObjectFromMemory(const void* data,
const std::vector<std::string>& device_names,
std::vector<std::pair<const void*, size_t>>& code_objs,
std::string& uri);
static uint64_t ElfSize(const void* emi);
protected:
//Given an ptr to image or file, extracts to code object
//for corresponding devices
static hipError_t extractCodeObjectFromFatBinary(const void*,
const std::vector<std::string>&,
std::vector<std::pair<const void*, size_t>>&);
CodeObject() {}
private:
friend const std::vector<hipModule_t>& modules();
};
//Dynamic Code Object
class DynCO : public CodeObject {
amd::Monitor dclock_{"Guards Dynamic Code object", true};
public:
DynCO() : device_id_(ihipGetDevice()) {}
virtual ~DynCO();
//LoadsCodeObject and its data
hipError_t loadCodeObject(const char* fname, const void* image=nullptr);
hipModule_t module() { return fb_info_->Module(ihipGetDevice()); };
//Gets GlobalVar/Functions from a dynamically loaded code object
hipError_t getDynFunc(hipFunction_t* hfunc, std::string func_name);
hipError_t getDeviceVar(DeviceVar** dvar, std::string var_name);
hipError_t getManagedVarPointer(std::string name, void** pointer, size_t* size_ptr) const {
auto it = vars_.find(name);
if (it != vars_.end() && it->second->getVarKind() == Var::DVK_Managed) {
*pointer = it->second->getManagedVarPtr();
*size_ptr = it->second->getSize();
}
return hipSuccess;
}
// Device ID Check to check if module is launched in the same device it was loaded.
inline void CheckDeviceIdMatch() {
if (device_id_ != ihipGetDevice()) {
guarantee(false, "Device mismatch from where this module is loaded");
}
}
private:
int device_id_;
FatBinaryInfo* fb_info_;
//Maps for vars/funcs, could be keyed in with std::string name
std::unordered_map<std::string, Function*> functions_;
std::unordered_map<std::string, Var*> vars_;
//Populate Global Vars/Funcs from an code object(@ module_load)
hipError_t populateDynGlobalFuncs();
hipError_t populateDynGlobalVars();
hipError_t initDynManagedVars(const std::string& managedVar);
};
//Static Code Object
class StatCO: public CodeObject {
amd::Monitor sclock_{"Guards Static Code object", true};
public:
StatCO();
virtual ~StatCO();
//Add/Remove/Digest Fat Binaries passed to us from "__hipRegisterFatBinary"
FatBinaryInfo** addFatBinary(const void* data, bool initialized);
hipError_t removeFatBinary(FatBinaryInfo** module);
hipError_t digestFatBinary(const void* data, FatBinaryInfo*& programs);
//Register vars/funcs given to use from __hipRegister[Var/Func/ManagedVar]
hipError_t registerStatFunction(const void* hostFunction, Function* func);
hipError_t registerStatGlobalVar(const void* hostVar, Var* var);
hipError_t registerStatManagedVar(Var *var);
//Retrive Vars/Funcs for a given hostSidePtr(const void*), unless stated otherwise.
hipError_t getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId);
hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId);
hipError_t getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr,
size_t* size_ptr);
//Managed variable is a defined symbol in code object
//pointer to the alocated managed memory has to be copied to the address of symbol
hipError_t initStatManagedVarDevicePtr(int deviceId);
private:
friend class ::PlatformState;
//Populated during __hipRegisterFatBinary
std::unordered_map<const void*, FatBinaryInfo*> modules_;
//Populated during __hipRegisterFuncs
std::unordered_map<const void*, Function*> functions_;
//Populated during __hipRegisterVars
std::unordered_map<const void*, Var*> vars_;
//Populated during __hipRegisterManagedVar
std::vector<Var*> managedVars_;
std::unordered_map<int, bool> managedVarsDevicePtrInitalized_;
};
}; // namespace hip
#endif /* HIP_CODE_OBJECT_HPP */
projects/hip/src/hipamd/src/hip_context.cpp
Executable
+377
Fájl megtekintése
@@ -0,0 +1,377 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
#include "hip_platform.hpp"
#include "platform/runtime.hpp"
#include "utils/flags.hpp"
#include "utils/versions.hpp"
std::vector<hip::Device*> g_devices;
namespace hip {
thread_local Device* g_device = nullptr;
thread_local std::stack<Device*> g_ctxtStack;
thread_local hipError_t g_lastError = hipSuccess;
std::once_flag g_ihipInitialized;
Device* host_device = nullptr;
void init() {
if (!amd::Runtime::initialized()) {
amd::IS_HIP = true;
GPU_NUM_MEM_DEPENDENCY = 0;
AMD_DIRECT_DISPATCH = flagIsDefault(AMD_DIRECT_DISPATCH) ? false : AMD_DIRECT_DISPATCH;
amd::Runtime::init();
}
const std::vector<amd::Device*>& devices = amd::Device::getDevices(CL_DEVICE_TYPE_GPU, false);
for (unsigned int i=0; i<devices.size(); i++) {
const std::vector<amd::Device*> device(1, devices[i]);
amd::Context* context = new amd::Context(device, amd::Context::Info());
if (!context) return;
// Enable active wait on the device by default
devices[i]->SetActiveWait(true);
if (context && CL_SUCCESS != context->create(nullptr)) {
context->release();
} else {
g_devices.push_back(new Device(context, i));
}
}
amd::Context* hContext = new amd::Context(devices, amd::Context::Info());
if (!hContext) return;
if (CL_SUCCESS != hContext->create(nullptr)) {
hContext->release();
}
host_device = new Device(hContext, -1);
PlatformState::instance().init();
}
Device* getCurrentDevice() {
return g_device;
}
void setCurrentDevice(unsigned int index) {
assert(index<g_devices.size());
g_device = g_devices[index];
}
amd::HostQueue* getQueue(hipStream_t stream) {
if (stream == nullptr) {
return getNullStream();
} else {
constexpr bool WaitNullStreamOnly = true;
amd::HostQueue* queue = reinterpret_cast<hip::Stream*>(stream)->asHostQueue();
if (!(reinterpret_cast<hip::Stream*>(stream)->Flags() & hipStreamNonBlocking)) {
iHipWaitActiveStreams(queue, WaitNullStreamOnly);
}
return queue;
}
}
// ================================================================================================
amd::HostQueue* getNullStream(amd::Context& ctx) {
for (auto& it : g_devices) {
if (it->asContext() == &ctx) {
return it->NullStream();
}
}
// If it's a pure SVM allocation with system memory access, then it shouldn't matter which device
// runtime selects by default
if (hip::host_device->asContext() == &ctx) {
// Return current...
return getNullStream();
}
return nullptr;
}
// ================================================================================================
amd::HostQueue* getNullStream() {
Device* device = getCurrentDevice();
return device ? device->NullStream() : nullptr;
}
};
using namespace hip;
hipError_t hipInit(unsigned int flags) {
HIP_INIT_API(hipInit, flags);
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxCreate(hipCtx_t *ctx, unsigned int flags, hipDevice_t device) {
HIP_INIT_API(hipCtxCreate, ctx, flags, device);
if (static_cast<size_t>(device) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidValue);
}
*ctx = reinterpret_cast<hipCtx_t>(g_devices[device]);
// Increment ref count for device primary context
g_devices[device]->retain();
g_ctxtStack.push(g_devices[device]);
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxSetCurrent(hipCtx_t ctx) {
HIP_INIT_API(hipCtxSetCurrent, ctx);
if (ctx == nullptr) {
if(!g_ctxtStack.empty()) {
g_ctxtStack.pop();
}
} else {
hip::g_device = reinterpret_cast<hip::Device*>(ctx);
if(!g_ctxtStack.empty()) {
g_ctxtStack.pop();
}
g_ctxtStack.push(hip::getCurrentDevice());
}
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxGetCurrent(hipCtx_t* ctx) {
HIP_INIT_API(hipCtxGetCurrent, ctx);
*ctx = reinterpret_cast<hipCtx_t>(hip::getCurrentDevice());
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxGetSharedMemConfig(hipSharedMemConfig* pConfig) {
HIP_INIT_API(hipCtxGetSharedMemConfig, pConfig);
*pConfig = hipSharedMemBankSizeFourByte;
HIP_RETURN(hipSuccess);
}
hipError_t hipRuntimeGetVersion(int *runtimeVersion) {
HIP_INIT_API(hipRuntimeGetVersion, runtimeVersion);
if (!runtimeVersion) {
HIP_RETURN(hipErrorInvalidValue);
}
// HIP_VERSION = HIP_VERSION_MAJOR*100 + HIP_MINOR_VERSION
*runtimeVersion = HIP_VERSION;
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxDestroy(hipCtx_t ctx) {
HIP_INIT_API(hipCtxDestroy, ctx);
hip::Device* dev = reinterpret_cast<hip::Device*>(ctx);
if (dev == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
// Need to remove the ctx of calling thread if its the top one
if (!g_ctxtStack.empty() && g_ctxtStack.top() == dev) {
g_ctxtStack.pop();
}
// Remove context from global context list
for (unsigned int i = 0; i < g_devices.size(); i++) {
if (g_devices[i] == dev) {
// Decrement ref count for device primary context
dev->release();
}
}
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxPopCurrent(hipCtx_t* ctx) {
HIP_INIT_API(hipCtxPopCurrent, ctx);
hip::Device** dev = reinterpret_cast<hip::Device**>(ctx);
if (!g_ctxtStack.empty()) {
if (dev != nullptr) {
*dev = g_ctxtStack.top();
}
g_ctxtStack.pop();
} else {
DevLogError("Context Stack empty \n");
HIP_RETURN(hipErrorInvalidContext);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxPushCurrent(hipCtx_t ctx) {
HIP_INIT_API(hipCtxPushCurrent, ctx);
hip::Device* dev = reinterpret_cast<hip::Device*>(ctx);
if (dev == nullptr) {
HIP_RETURN(hipErrorInvalidContext);
}
hip::g_device = dev;
g_ctxtStack.push(hip::getCurrentDevice());
HIP_RETURN(hipSuccess);
}
hipError_t hipDriverGetVersion(int* driverVersion) {
HIP_INIT_API(hipDriverGetVersion, driverVersion);
if (!driverVersion) {
HIP_RETURN(hipErrorInvalidValue);
}
// HIP_VERSION = HIP_VERSION_MAJOR*100 + HIP_MINOR_VERSION
*driverVersion = HIP_VERSION;
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxGetDevice(hipDevice_t* device) {
HIP_INIT_API(hipCtxGetDevice, device);
if (device != nullptr) {
*device = hip::getCurrentDevice()->deviceId();
HIP_RETURN(hipSuccess);
} else {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipErrorInvalidContext);
}
hipError_t hipCtxGetApiVersion(hipCtx_t ctx, int* apiVersion) {
HIP_INIT_API(hipCtxGetApiVersion, apiVersion);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipCtxGetCacheConfig(hipFuncCache_t* cacheConfig) {
HIP_INIT_API(hipCtxGetCacheConfig, cacheConfig);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipCtxSetCacheConfig(hipFuncCache_t cacheConfig) {
HIP_INIT_API(hipCtxSetCacheConfig, cacheConfig);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipCtxSetSharedMemConfig(hipSharedMemConfig config) {
HIP_INIT_API(hipCtxSetSharedMemConfig, config);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipCtxSynchronize(void) {
HIP_INIT_API(hipCtxSynchronize, 1);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipCtxGetFlags(unsigned int* flags) {
HIP_INIT_API(hipCtxGetFlags, flags);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipDevicePrimaryCtxGetState(hipDevice_t dev, unsigned int* flags, int* active) {
HIP_INIT_API(hipDevicePrimaryCtxGetState, dev, flags, active);
if (static_cast<unsigned int>(dev) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (flags != nullptr) {
*flags = 0;
}
if (active != nullptr) {
*active = (g_devices[dev] == hip::getCurrentDevice())? 1 : 0;
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDevicePrimaryCtxRelease(hipDevice_t dev) {
HIP_INIT_API(hipDevicePrimaryCtxRelease, dev);
if (static_cast<unsigned int>(dev) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDevicePrimaryCtxRetain(hipCtx_t* pctx, hipDevice_t dev) {
HIP_INIT_API(hipDevicePrimaryCtxRetain, pctx, dev);
if (static_cast<unsigned int>(dev) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (pctx == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
*pctx = reinterpret_cast<hipCtx_t>(g_devices[dev]);
HIP_RETURN(hipSuccess);
}
hipError_t hipDevicePrimaryCtxReset(hipDevice_t dev) {
HIP_INIT_API(hipDevicePrimaryCtxReset, dev);
HIP_RETURN(hipSuccess);
}
hipError_t hipDevicePrimaryCtxSetFlags(hipDevice_t dev, unsigned int flags) {
HIP_INIT_API(hipDevicePrimaryCtxSetFlags, dev, flags);
if (static_cast<unsigned int>(dev) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
} else {
HIP_RETURN(hipErrorContextAlreadyInUse);
}
}
projects/hip/src/hipamd/src/hip_conversions.hpp
+903
Fájl megtekintése
@@ -0,0 +1,903 @@
/*
Copyright (c) 2015 - present Advanced Micro Devices, Inc. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
#pragma once
#include <hip/amd_detail/amd_driver_types.h>
#include <hip/amd_detail/amd_texture_types.h>
namespace hip
{
inline
cl_channel_type getCLChannelType(const hipArray_Format hipFormat,
const hipTextureReadMode hipReadMode) {
if (hipReadMode == hipReadModeElementType) {
switch (hipFormat) {
case HIP_AD_FORMAT_UNSIGNED_INT8:
return CL_UNSIGNED_INT8;
case HIP_AD_FORMAT_SIGNED_INT8:
return CL_SIGNED_INT8;
case HIP_AD_FORMAT_UNSIGNED_INT16:
return CL_UNSIGNED_INT16;
case HIP_AD_FORMAT_SIGNED_INT16:
return CL_SIGNED_INT16;
case HIP_AD_FORMAT_UNSIGNED_INT32:
return CL_UNSIGNED_INT32;
case HIP_AD_FORMAT_SIGNED_INT32:
return CL_SIGNED_INT32;
case HIP_AD_FORMAT_HALF:
return CL_HALF_FLOAT;
case HIP_AD_FORMAT_FLOAT:
return CL_FLOAT;
}
} else if (hipReadMode == hipReadModeNormalizedFloat) {
switch (hipFormat) {
case HIP_AD_FORMAT_UNSIGNED_INT8:
return CL_UNORM_INT8;
case HIP_AD_FORMAT_SIGNED_INT8:
return CL_SNORM_INT8;
case HIP_AD_FORMAT_UNSIGNED_INT16:
return CL_UNORM_INT16;
case HIP_AD_FORMAT_SIGNED_INT16:
return CL_SNORM_INT16;
case HIP_AD_FORMAT_UNSIGNED_INT32:
return CL_UNSIGNED_INT32;
case HIP_AD_FORMAT_SIGNED_INT32:
return CL_SIGNED_INT32;
case HIP_AD_FORMAT_HALF:
return CL_HALF_FLOAT;
case HIP_AD_FORMAT_FLOAT:
return CL_FLOAT;
}
}
ShouldNotReachHere();
return {};
}
inline
cl_channel_order getCLChannelOrder(const unsigned int hipNumChannels,
const int sRGB) {
switch (hipNumChannels) {
case 1:
return CL_R;
case 2:
return CL_RG;
case 4:
return (sRGB == 1) ? CL_sRGBA : CL_RGBA;
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
cl_mem_object_type getCLMemObjectType(const unsigned int hipWidth,
const unsigned int hipHeight,
const unsigned int hipDepth,
const unsigned int flags) {
if (flags == hipArrayDefault) {
if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth == 0)) {
return CL_MEM_OBJECT_IMAGE1D;
} else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth == 0)) {
return CL_MEM_OBJECT_IMAGE2D;
} else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) {
return CL_MEM_OBJECT_IMAGE3D;
}
} else if (flags == hipArrayLayered) {
if ((hipWidth != 0) && (hipHeight == 0) && (hipDepth != 0)) {
return CL_MEM_OBJECT_IMAGE1D_ARRAY;
} else if ((hipWidth != 0) && (hipHeight != 0) && (hipDepth != 0)) {
return CL_MEM_OBJECT_IMAGE2D_ARRAY;
}
}
ShouldNotReachHere();
return {};
}
inline
cl_addressing_mode getCLAddressingMode(const hipTextureAddressMode hipAddressMode) {
switch (hipAddressMode) {
case hipAddressModeWrap:
return CL_ADDRESS_REPEAT;
case hipAddressModeClamp:
return CL_ADDRESS_CLAMP_TO_EDGE;
case hipAddressModeMirror:
return CL_ADDRESS_MIRRORED_REPEAT;
case hipAddressModeBorder:
return CL_ADDRESS_CLAMP;
}
ShouldNotReachHere();
return {};
}
inline
cl_filter_mode getCLFilterMode(const hipTextureFilterMode hipFilterMode) {
switch (hipFilterMode) {
case hipFilterModePoint:
return CL_FILTER_NEAREST;
case hipFilterModeLinear:
return CL_FILTER_LINEAR;
}
ShouldNotReachHere();
return {};
}
inline
cl_mem_object_type getCLMemObjectType(const hipResourceType hipResType) {
switch (hipResType) {
case hipResourceTypeLinear:
return CL_MEM_OBJECT_IMAGE1D_BUFFER;
case hipResourceTypePitch2D:
return CL_MEM_OBJECT_IMAGE2D;
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
size_t getElementSize(const hipArray_const_t array) {
switch (array->Format) {
case HIP_AD_FORMAT_UNSIGNED_INT8:
case HIP_AD_FORMAT_SIGNED_INT8:
return 1 * array->NumChannels;
case HIP_AD_FORMAT_UNSIGNED_INT16:
case HIP_AD_FORMAT_SIGNED_INT16:
case HIP_AD_FORMAT_HALF:
return 2 * array->NumChannels;
case HIP_AD_FORMAT_UNSIGNED_INT32:
case HIP_AD_FORMAT_SIGNED_INT32:
case HIP_AD_FORMAT_FLOAT:
return 4 * array->NumChannels;
}
ShouldNotReachHere();
return {};
}
inline
hipChannelFormatDesc getChannelFormatDesc(int numChannels,
hipArray_Format arrayFormat) {
switch (arrayFormat) {
case HIP_AD_FORMAT_UNSIGNED_INT8:
switch (numChannels) {
case 1:
return {8, 0, 0, 0, hipChannelFormatKindUnsigned};
case 2:
return {8, 8, 0, 0, hipChannelFormatKindUnsigned};
case 4:
return {8, 8, 8, 8, hipChannelFormatKindUnsigned};
}
case HIP_AD_FORMAT_SIGNED_INT8:
switch (numChannels) {
case 1:
return {8, 0, 0, 0, hipChannelFormatKindSigned};
case 2:
return {8, 8, 0, 0, hipChannelFormatKindSigned};
case 4:
return {8, 8, 8, 8, hipChannelFormatKindSigned};
}
case HIP_AD_FORMAT_UNSIGNED_INT16:
switch (numChannels) {
case 1:
return {16, 0, 0, 0, hipChannelFormatKindUnsigned};
case 2:
return {16, 16, 0, 0, hipChannelFormatKindUnsigned};
case 4:
return {16, 16, 16, 16, hipChannelFormatKindUnsigned};
}
case HIP_AD_FORMAT_SIGNED_INT16:
switch (numChannels) {
case 1:
return {16, 0, 0, 0, hipChannelFormatKindSigned};
case 2:
return {16, 16, 0, 0, hipChannelFormatKindSigned};
case 4:
return {16, 16, 16, 16, hipChannelFormatKindSigned};
}
case HIP_AD_FORMAT_UNSIGNED_INT32:
switch (numChannels) {
case 1:
return {32, 0, 0, 0, hipChannelFormatKindUnsigned};
case 2:
return {32, 32, 0, 0, hipChannelFormatKindUnsigned};
case 4:
return {32, 32, 32, 32, hipChannelFormatKindUnsigned};
}
case HIP_AD_FORMAT_SIGNED_INT32:
switch (numChannels) {
case 1:
return {32, 0, 0, 0, hipChannelFormatKindSigned};
case 2:
return {32, 32, 0, 0, hipChannelFormatKindSigned};
case 4:
return {32, 32, 32, 32, hipChannelFormatKindSigned};
}
case HIP_AD_FORMAT_HALF:
switch (numChannels) {
case 1:
return {16, 0, 0, 0, hipChannelFormatKindFloat};
case 2:
return {16, 16, 0, 0, hipChannelFormatKindFloat};
case 4:
return {16, 16, 16, 16, hipChannelFormatKindFloat};
}
case HIP_AD_FORMAT_FLOAT:
switch (numChannels) {
case 1:
return {32, 0, 0, 0, hipChannelFormatKindFloat};
case 2:
return {32, 32, 0, 0, hipChannelFormatKindFloat};
case 4:
return {32, 32, 32, 32, hipChannelFormatKindFloat};
}
}
ShouldNotReachHere();
return {};
}
inline
unsigned int getNumChannels(const hipChannelFormatDesc& desc) {
return ((desc.x != 0) + (desc.y != 0) + (desc.z != 0) + (desc.w != 0));
}
inline
hipArray_Format getArrayFormat(const hipChannelFormatDesc& desc) {
switch (desc.f) {
case hipChannelFormatKindUnsigned:
switch (desc.x) {
case 8:
return HIP_AD_FORMAT_UNSIGNED_INT8;
case 16:
return HIP_AD_FORMAT_UNSIGNED_INT16;
case 32:
return HIP_AD_FORMAT_UNSIGNED_INT32;
}
case hipChannelFormatKindSigned:
switch (desc.x) {
case 8:
return HIP_AD_FORMAT_SIGNED_INT8;
case 16:
return HIP_AD_FORMAT_SIGNED_INT16;
case 32:
return HIP_AD_FORMAT_SIGNED_INT32;
}
case hipChannelFormatKindFloat:
switch (desc.x) {
case 16:
return HIP_AD_FORMAT_HALF;
case 32:
return HIP_AD_FORMAT_FLOAT;
}
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
int getNumChannels(const hipResourceViewFormat hipFormat) {
switch (hipFormat) {
case hipResViewFormatUnsignedChar1:
case hipResViewFormatSignedChar1:
case hipResViewFormatUnsignedShort1:
case hipResViewFormatSignedShort1:
case hipResViewFormatUnsignedInt1:
case hipResViewFormatSignedInt1:
case hipResViewFormatHalf1:
case hipResViewFormatFloat1:
return 1;
case hipResViewFormatUnsignedChar2:
case hipResViewFormatSignedChar2:
case hipResViewFormatUnsignedShort2:
case hipResViewFormatSignedShort2:
case hipResViewFormatUnsignedInt2:
case hipResViewFormatSignedInt2:
case hipResViewFormatHalf2:
case hipResViewFormatFloat2:
return 2;
case hipResViewFormatUnsignedChar4:
case hipResViewFormatSignedChar4:
case hipResViewFormatUnsignedShort4:
case hipResViewFormatSignedShort4:
case hipResViewFormatUnsignedInt4:
case hipResViewFormatSignedInt4:
case hipResViewFormatHalf4:
case hipResViewFormatFloat4:
return 4;
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
hipArray_Format getArrayFormat(const hipResourceViewFormat hipFormat) {
switch (hipFormat) {
case hipResViewFormatUnsignedChar1:
case hipResViewFormatUnsignedChar2:
case hipResViewFormatUnsignedChar4:
return HIP_AD_FORMAT_UNSIGNED_INT8;
case hipResViewFormatSignedChar1:
case hipResViewFormatSignedChar2:
case hipResViewFormatSignedChar4:
return HIP_AD_FORMAT_SIGNED_INT8;
case hipResViewFormatUnsignedShort1:
case hipResViewFormatUnsignedShort2:
case hipResViewFormatUnsignedShort4:
return HIP_AD_FORMAT_UNSIGNED_INT16;
case hipResViewFormatSignedShort1:
case hipResViewFormatSignedShort2:
case hipResViewFormatSignedShort4:
return HIP_AD_FORMAT_SIGNED_INT16;
case hipResViewFormatUnsignedInt1:
case hipResViewFormatUnsignedInt2:
case hipResViewFormatUnsignedInt4:
return HIP_AD_FORMAT_UNSIGNED_INT32;
case hipResViewFormatSignedInt1:
case hipResViewFormatSignedInt2:
case hipResViewFormatSignedInt4:
return HIP_AD_FORMAT_SIGNED_INT32;
case hipResViewFormatHalf1:
case hipResViewFormatHalf2:
case hipResViewFormatHalf4:
return HIP_AD_FORMAT_HALF;
case hipResViewFormatFloat1:
case hipResViewFormatFloat2:
case hipResViewFormatFloat4:
return HIP_AD_FORMAT_FLOAT;
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
hipResourceViewFormat getResourceViewFormat(const hipChannelFormatDesc& desc) {
switch (desc.f) {
case hipChannelFormatKindUnsigned:
switch (getNumChannels(desc)) {
case 1:
switch (desc.x) {
case 8:
return hipResViewFormatUnsignedChar1;
case 16:
return hipResViewFormatUnsignedShort1;
case 32:
return hipResViewFormatUnsignedInt1;
}
case 2:
switch (desc.x) {
case 8:
return hipResViewFormatUnsignedChar2;
case 16:
return hipResViewFormatUnsignedShort2;
case 32:
return hipResViewFormatUnsignedInt2;
}
case 4:
switch (desc.x) {
case 8:
return hipResViewFormatUnsignedChar4;
case 16:
return hipResViewFormatUnsignedShort4;
case 32:
return hipResViewFormatUnsignedInt4;
}
}
case hipChannelFormatKindSigned:
switch (getNumChannels(desc)) {
case 1:
switch (desc.x) {
case 8:
return hipResViewFormatSignedChar1;
case 16:
return hipResViewFormatSignedShort1;
case 32:
return hipResViewFormatSignedInt1;
}
case 2:
switch (desc.x) {
case 8:
return hipResViewFormatSignedChar2;
case 16:
return hipResViewFormatSignedShort2;
case 32:
return hipResViewFormatSignedInt2;
}
case 4:
switch (desc.x) {
case 8:
return hipResViewFormatSignedChar4;
case 16:
return hipResViewFormatSignedShort4;
case 32:
return hipResViewFormatSignedInt4;
}
}
case hipChannelFormatKindFloat:
switch (getNumChannels(desc)) {
case 1:
switch (desc.x) {
case 16:
return hipResViewFormatHalf1;
case 32:
return hipResViewFormatFloat1;
}
case 2:
switch (desc.x) {
case 16:
return hipResViewFormatHalf2;
case 32:
return hipResViewFormatFloat2;
}
case 4:
switch (desc.x) {
case 16:
return hipResViewFormatHalf4;
case 32:
return hipResViewFormatFloat4;
}
}
default:
break;
}
ShouldNotReachHere();
return {};
}
inline
hipTextureDesc getTextureDesc(const textureReference* texRef) {
hipTextureDesc texDesc = {};
std::memcpy(texDesc.addressMode, texRef->addressMode, sizeof(texDesc.addressMode));
texDesc.filterMode = texRef->filterMode;
texDesc.readMode = texRef->readMode;
texDesc.sRGB = texRef->sRGB;
texDesc.normalizedCoords = texRef->normalized;
texDesc.maxAnisotropy = texRef->maxAnisotropy;
texDesc.mipmapFilterMode = texRef->mipmapFilterMode;
texDesc.mipmapLevelBias = texRef->mipmapLevelBias;
texDesc.minMipmapLevelClamp = texRef->minMipmapLevelClamp;
texDesc.maxMipmapLevelClamp = texRef->maxMipmapLevelClamp;
return texDesc;
}
inline
hipResourceViewDesc getResourceViewDesc(hipArray_const_t array,
const hipResourceViewFormat format) {
hipResourceViewDesc resViewDesc = {};
resViewDesc.format = format;
resViewDesc.width = array->width;
resViewDesc.height = array->height;
resViewDesc.depth = array->depth;
resViewDesc.firstMipmapLevel = 0;
resViewDesc.lastMipmapLevel = 0;
resViewDesc.firstLayer = 0;
resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */
return resViewDesc;
}
inline
hipResourceViewDesc getResourceViewDesc(hipMipmappedArray_const_t array,
const hipResourceViewFormat format) {
hipResourceViewDesc resViewDesc = {};
resViewDesc.format = format;
resViewDesc.width = array->width;
resViewDesc.height = array->height;
resViewDesc.depth = array->depth;
resViewDesc.firstMipmapLevel = 0;
resViewDesc.lastMipmapLevel = 0; /* TODO add hipMipmappedArray::numMipLevels */
resViewDesc.firstLayer = 0;
resViewDesc.lastLayer = 0; /* TODO add hipArray::numLayers */
return resViewDesc;
}
inline
std::pair<hipMemoryType, hipMemoryType> getMemoryType(const hipMemcpyKind kind) {
switch (kind) {
case hipMemcpyHostToHost:
return {hipMemoryTypeHost, hipMemoryTypeHost};
case hipMemcpyHostToDevice:
return {hipMemoryTypeHost, hipMemoryTypeDevice};
case hipMemcpyDeviceToHost:
return {hipMemoryTypeDevice, hipMemoryTypeHost};
case hipMemcpyDeviceToDevice:
return {hipMemoryTypeDevice, hipMemoryTypeDevice};
case hipMemcpyDefault:
return {hipMemoryTypeUnified, hipMemoryTypeUnified};
}
ShouldNotReachHere();
return {};
}
inline
HIP_MEMCPY3D getDrvMemcpy3DDesc(const hip_Memcpy2D& desc2D) {
HIP_MEMCPY3D desc3D = {};
desc3D.srcXInBytes = desc2D.srcXInBytes;
desc3D.srcY = desc2D.srcY;
desc3D.srcZ = 0;
desc3D.srcLOD = 0;
desc3D.srcMemoryType = desc2D.srcMemoryType;
desc3D.srcHost = desc2D.srcHost;
desc3D.srcDevice = desc2D.srcDevice;
desc3D.srcArray = desc2D.srcArray;
desc3D.srcPitch = desc2D.srcPitch;
desc3D.srcHeight = 0;
desc3D.dstXInBytes = desc2D.dstXInBytes;
desc3D.dstY = desc2D.dstY;
desc3D.dstZ = 0;
desc3D.dstLOD = 0;
desc3D.dstMemoryType = desc2D.dstMemoryType;
desc3D.dstHost = desc2D.dstHost;
desc3D.dstDevice = desc2D.dstDevice;
desc3D.dstArray = desc2D.dstArray;
desc3D.dstPitch = desc2D.dstPitch;
desc3D.dstHeight = 0;
desc3D.WidthInBytes = desc2D.WidthInBytes;
desc3D.Height = desc2D.Height;
desc3D.Depth = 1;
return desc3D;
}
inline
HIP_MEMCPY3D getDrvMemcpy3DDesc(const hipMemcpy3DParms& desc) {
HIP_MEMCPY3D descDrv = {};
descDrv.WidthInBytes = desc.extent.width;
descDrv.Height = desc.extent.height;
descDrv.Depth = desc.extent.depth;
descDrv.srcXInBytes = desc.srcPos.x;
descDrv.srcY = desc.srcPos.y;
descDrv.srcZ = desc.srcPos.z;
descDrv.srcLOD = 0;
descDrv.dstXInBytes = desc.dstPos.x;
descDrv.dstY = desc.dstPos.y;
descDrv.dstZ = desc.dstPos.z;
descDrv.dstLOD = 0;
if (desc.srcArray != nullptr) {
descDrv.srcMemoryType = hipMemoryTypeArray;
descDrv.srcArray = desc.srcArray;
// When reffering to array memory, hipPos::x is in elements.
descDrv.srcXInBytes *= getElementSize(desc.srcArray);
}
if (desc.srcPtr.ptr != nullptr) {
descDrv.srcMemoryType = std::get<0>(hip::getMemoryType(desc.kind));
descDrv.srcHost = desc.srcPtr.ptr;
descDrv.srcDevice = desc.srcPtr.ptr;
descDrv.srcPitch = desc.srcPtr.pitch;
descDrv.srcHeight = desc.srcPtr.ysize;
}
if (desc.dstArray != nullptr) {
descDrv.dstMemoryType = hipMemoryTypeArray;
descDrv.dstArray = desc.dstArray;
// When reffering to array memory, hipPos::x is in elements.
descDrv.dstXInBytes *= getElementSize(desc.dstArray);
}
if (desc.dstPtr.ptr != nullptr) {
descDrv.dstMemoryType = std::get<1>(getMemoryType(desc.kind));
descDrv.dstHost = desc.dstPtr.ptr;
descDrv.dstDevice = desc.dstPtr.ptr;
descDrv.dstPitch = desc.dstPtr.pitch;
descDrv.dstHeight = desc.dstPtr.ysize;
}
// If a HIP array is participating in the copy, the extent is defined in terms of that array's elements.
if ((desc.srcArray != nullptr) && (desc.dstArray == nullptr)) {
descDrv.WidthInBytes *= getElementSize(desc.srcArray);
} else if ((desc.srcArray == nullptr) && (desc.dstArray != nullptr)) {
descDrv.WidthInBytes *= getElementSize(desc.dstArray);
} else if ((desc.srcArray != nullptr) && (desc.dstArray != nullptr)) {
descDrv.WidthInBytes *= getElementSize(desc.dstArray);
}
return descDrv;
}
inline
hipResourceType getResourceType(const HIPresourcetype resType) {
// These two enums should be isomorphic.
return static_cast<hipResourceType>(resType);
}
inline
HIPresourcetype getResourceType(const hipResourceType resType) {
// These two enums should be isomorphic.
return static_cast<HIPresourcetype>(resType);
}
inline
hipResourceDesc getResourceDesc(const HIP_RESOURCE_DESC& resDesc) {
hipResourceDesc desc;
desc.resType = getResourceType(resDesc.resType);
switch (desc.resType) {
case hipResourceTypeArray:
desc.res.array.array = resDesc.res.array.hArray;
break;
case hipResourceTypeMipmappedArray:
desc.res.mipmap.mipmap = resDesc.res.mipmap.hMipmappedArray;
break;
case hipResourceTypeLinear:
desc.res.linear.devPtr = resDesc.res.linear.devPtr;
desc.res.linear.desc = getChannelFormatDesc(resDesc.res.linear.numChannels, resDesc.res.linear.format);
desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes;
break;
case hipResourceTypePitch2D:
desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr;
desc.res.pitch2D.desc = getChannelFormatDesc(resDesc.res.pitch2D.numChannels, resDesc.res.pitch2D.format);
desc.res.pitch2D.width = resDesc.res.pitch2D.width;
desc.res.pitch2D.height = resDesc.res.pitch2D.height;
desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes;
break;
default:
break;
}
return desc;
}
inline
HIP_RESOURCE_DESC getResourceDesc(const hipResourceDesc& resDesc) {
HIP_RESOURCE_DESC desc;
desc.resType = getResourceType(resDesc.resType);
switch (desc.resType) {
case HIP_RESOURCE_TYPE_ARRAY:
desc.res.array.hArray = resDesc.res.array.array;
break;
case HIP_RESOURCE_TYPE_MIPMAPPED_ARRAY:
desc.res.mipmap.hMipmappedArray = resDesc.res.mipmap.mipmap;
break;
case HIP_RESOURCE_TYPE_LINEAR:
desc.res.linear.devPtr = resDesc.res.linear.devPtr;
desc.res.linear.numChannels = getNumChannels(resDesc.res.linear.desc);
desc.res.linear.format = getArrayFormat(resDesc.res.linear.desc);
desc.res.linear.sizeInBytes = resDesc.res.linear.sizeInBytes;
break;
case HIP_RESOURCE_TYPE_PITCH2D:
desc.res.pitch2D.devPtr = resDesc.res.pitch2D.devPtr;
desc.res.pitch2D.numChannels = getNumChannels(resDesc.res.pitch2D.desc);
desc.res.pitch2D.format = getArrayFormat(resDesc.res.pitch2D.desc);
desc.res.pitch2D.width = resDesc.res.pitch2D.width;
desc.res.pitch2D.height = resDesc.res.pitch2D.height;
desc.res.pitch2D.pitchInBytes = resDesc.res.pitch2D.pitchInBytes;
break;
default:
break;
}
return desc;
}
inline
hipTextureAddressMode getAddressMode(const HIPaddress_mode mode) {
// These two enums should be isomorphic.
return static_cast<hipTextureAddressMode>(mode);
}
inline
HIPaddress_mode getAddressMode(const hipTextureAddressMode mode) {
// These two enums should be isomorphic.
return static_cast<HIPaddress_mode>(mode);
}
inline
hipTextureFilterMode getFilterMode(const HIPfilter_mode mode) {
// These two enums should be isomorphic.
return static_cast<hipTextureFilterMode>(mode);
}
inline
HIPfilter_mode getFilterMode(const hipTextureFilterMode mode) {
// These two enums should be isomorphic.
return static_cast<HIPfilter_mode>(mode);
}
inline
hipTextureReadMode getReadMode(const unsigned int flags) {
if (flags & HIP_TRSF_READ_AS_INTEGER) {
return hipReadModeElementType;
} else {
return hipReadModeNormalizedFloat;
}
}
inline
unsigned int getReadMode(const hipTextureReadMode mode) {
if (mode == hipReadModeElementType) {
return HIP_TRSF_READ_AS_INTEGER;
} else {
return 0;
}
}
inline
int getsRGB(const unsigned int flags) {
if (flags & HIP_TRSF_SRGB) {
return 1;
} else {
return 0;
}
}
inline
unsigned int getsRGB(const int sRGB) {
if (sRGB == 1) {
return HIP_TRSF_SRGB;
} else {
return 0;
}
}
inline
int getNormalizedCoords(const unsigned int flags) {
if (flags & HIP_TRSF_NORMALIZED_COORDINATES) {
return 1;
} else {
return 0;
}
}
inline
unsigned int getNormalizedCoords(const int normalizedCoords) {
if (normalizedCoords == 1) {
return HIP_TRSF_NORMALIZED_COORDINATES;
} else {
return 0;
}
}
inline
hipTextureDesc getTextureDesc(const HIP_TEXTURE_DESC& texDesc) {
hipTextureDesc desc;
desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]);
desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]);
desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]);
desc.filterMode = getFilterMode(texDesc.filterMode);
desc.readMode = getReadMode(texDesc.flags);
desc.sRGB = getsRGB(texDesc.flags);
std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor));
desc.normalizedCoords = getNormalizedCoords(texDesc.flags);
desc.maxAnisotropy = texDesc.maxAnisotropy;
desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode);
desc.mipmapLevelBias = texDesc.mipmapLevelBias;
desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp;
desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp;
return desc;
}
inline
HIP_TEXTURE_DESC getTextureDesc(const hipTextureDesc& texDesc) {
HIP_TEXTURE_DESC desc;
desc.addressMode[0] = getAddressMode(texDesc.addressMode[0]);
desc.addressMode[1] = getAddressMode(texDesc.addressMode[1]);
desc.addressMode[2] = getAddressMode(texDesc.addressMode[2]);
desc.filterMode = getFilterMode(texDesc.filterMode);
desc.flags = 0;
desc.flags |= getReadMode(texDesc.readMode);
desc.flags |= getsRGB(texDesc.sRGB);
desc.flags |= getNormalizedCoords(texDesc.normalizedCoords);
desc.maxAnisotropy = texDesc.maxAnisotropy;
desc.mipmapFilterMode = getFilterMode(texDesc.mipmapFilterMode);
desc.mipmapLevelBias = texDesc.mipmapLevelBias;
desc.minMipmapLevelClamp = texDesc.minMipmapLevelClamp;
desc.maxMipmapLevelClamp = texDesc.maxMipmapLevelClamp;
std::memcpy(desc.borderColor, texDesc.borderColor, sizeof(desc.borderColor));
return desc;
}
inline
hipResourceViewFormat getResourceViewFormat(const HIPresourceViewFormat format) {
// These two enums should be isomorphic.
return static_cast<hipResourceViewFormat>(format);
}
inline
HIPresourceViewFormat getResourceViewFormat(const hipResourceViewFormat format) {
// These two enums should be isomorphic.
return static_cast<HIPresourceViewFormat>(format);
}
inline
hipResourceViewDesc getResourceViewDesc(const HIP_RESOURCE_VIEW_DESC& resViewDesc) {
hipResourceViewDesc desc;
desc.format = getResourceViewFormat(resViewDesc.format);
desc.width = resViewDesc.width;
desc.height = resViewDesc.height;
desc.depth = resViewDesc.depth;
desc.firstMipmapLevel = resViewDesc.firstMipmapLevel;
desc.lastMipmapLevel = resViewDesc.lastMipmapLevel;
desc.firstLayer = resViewDesc.firstLayer;
desc.lastLayer = resViewDesc.lastLayer;
return desc;
}
inline
HIP_RESOURCE_VIEW_DESC getResourceViewDesc(const hipResourceViewDesc& resViewDesc) {
HIP_RESOURCE_VIEW_DESC desc;
desc.format = getResourceViewFormat(resViewDesc.format);
desc.width = resViewDesc.width;
desc.height = resViewDesc.height;
desc.depth = resViewDesc.depth;
desc.firstMipmapLevel = resViewDesc.firstMipmapLevel;
desc.lastMipmapLevel = resViewDesc.lastMipmapLevel;
desc.firstLayer = resViewDesc.firstLayer;
desc.lastLayer = resViewDesc.lastLayer;
return desc;
}
inline
size_t getElementSize(const hipChannelFormatDesc &desc) {
return (desc.x / 8) * getNumChannels(desc);
}
};
projects/hip/src/hipamd/src/hip_device.cpp
+240
Fájl megtekintése
@@ -0,0 +1,240 @@
/* Copyright (c) 2018-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
namespace hip {
// ================================================================================================
amd::HostQueue* Device::NullStream(bool skip_alloc) {
amd::HostQueue* null_queue = null_stream_.asHostQueue(skip_alloc);
if (null_queue == nullptr) {
return nullptr;
}
// Wait for all active streams before executing commands on the default
iHipWaitActiveStreams(null_queue);
return null_queue;
}
}
hipError_t hipDeviceGet(hipDevice_t *device, int deviceId) {
HIP_INIT_API(hipDeviceGet, device, deviceId);
if (deviceId < 0 ||
static_cast<size_t>(deviceId) >= g_devices.size() ||
device == nullptr) {
HIP_RETURN(hipErrorInvalidDevice);
}
*device = deviceId;
HIP_RETURN(hipSuccess);
};
hipError_t hipDeviceTotalMem (size_t *bytes, hipDevice_t device) {
HIP_INIT_API(hipDeviceTotalMem, bytes, device);
if (device < 0 || static_cast<size_t>(device) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (bytes == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
auto* deviceHandle = g_devices[device]->devices()[0];
const auto& info = deviceHandle->info();
*bytes = info.globalMemSize_;
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceComputeCapability(int *major, int *minor, hipDevice_t device) {
HIP_INIT_API(hipDeviceComputeCapability, major, minor, device);
if (device < 0 || static_cast<size_t>(device) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (major == nullptr || minor == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
auto* deviceHandle = g_devices[device]->devices()[0];
const auto& isa = deviceHandle->isa();
*major = isa.versionMajor();
*minor = isa.versionMinor();
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetCount(int* count) {
HIP_INIT_API(hipDeviceGetCount, count);
HIP_RETURN(ihipDeviceGetCount(count));
}
hipError_t ihipDeviceGetCount(int* count) {
if (count == nullptr) {
return hipErrorInvalidValue;
}
// Get all available devices
*count = g_devices.size();
if (*count < 1) {
return hipErrorNoDevice;
}
return hipSuccess;
}
hipError_t hipDeviceGetName(char *name, int len, hipDevice_t device) {
HIP_INIT_API(hipDeviceGetName, (void*)name, len, device);
if (device < 0 || static_cast<size_t>(device) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (name == nullptr || len <= 0) {
HIP_RETURN(hipErrorInvalidValue);
}
auto* deviceHandle = g_devices[device]->devices()[0];
const auto& info = deviceHandle->info();
const auto nameLen = ::strlen(info.boardName_);
// Make sure that the size of `dest` is big enough to hold `src` including
// trailing zero byte
if (nameLen > (cl_uint)(len - 1)) {
HIP_RETURN(hipErrorInvalidValue);
}
::strncpy(name, info.boardName_, (nameLen + 1));
HIP_RETURN(hipSuccess);
}
hipError_t hipGetDeviceProperties ( hipDeviceProp_t* props, hipDevice_t device ) {
HIP_INIT_API(hipGetDeviceProperties, props, device);
if (props == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if (unsigned(device) >= g_devices.size()) {
HIP_RETURN(hipErrorInvalidDevice);
}
auto* deviceHandle = g_devices[device]->devices()[0];
hipDeviceProp_t deviceProps = {0};
const auto& info = deviceHandle->info();
const auto& isa = deviceHandle->isa();
::strncpy(deviceProps.name, info.boardName_, 128);
deviceProps.totalGlobalMem = info.globalMemSize_;
deviceProps.sharedMemPerBlock = info.localMemSizePerCU_;
deviceProps.regsPerBlock = info.availableRegistersPerCU_;
deviceProps.warpSize = info.wavefrontWidth_;
deviceProps.maxThreadsPerBlock = info.maxWorkGroupSize_;
deviceProps.maxThreadsDim[0] = info.maxWorkItemSizes_[0];
deviceProps.maxThreadsDim[1] = info.maxWorkItemSizes_[1];
deviceProps.maxThreadsDim[2] = info.maxWorkItemSizes_[2];
deviceProps.maxGridSize[0] = INT32_MAX;
deviceProps.maxGridSize[1] = INT32_MAX;
deviceProps.maxGridSize[2] = INT32_MAX;
deviceProps.clockRate = info.maxEngineClockFrequency_ * 1000;
deviceProps.memoryClockRate = info.maxMemoryClockFrequency_ * 1000;
deviceProps.memoryBusWidth = info.globalMemChannels_;
deviceProps.totalConstMem = info.maxConstantBufferSize_;
deviceProps.major = isa.versionMajor();
deviceProps.minor = isa.versionMinor();
deviceProps.multiProcessorCount = info.maxComputeUnits_;
deviceProps.l2CacheSize = info.l2CacheSize_;
deviceProps.maxThreadsPerMultiProcessor = info.maxThreadsPerCU_;
deviceProps.computeMode = 0;
deviceProps.clockInstructionRate = info.timeStampFrequency_;
deviceProps.arch.hasGlobalInt32Atomics = 1;
deviceProps.arch.hasGlobalFloatAtomicExch = 1;
deviceProps.arch.hasSharedInt32Atomics = 1;
deviceProps.arch.hasSharedFloatAtomicExch = 1;
deviceProps.arch.hasFloatAtomicAdd = 1;
deviceProps.arch.hasGlobalInt64Atomics = 1;
deviceProps.arch.hasSharedInt64Atomics = 1;
deviceProps.arch.hasDoubles = 1;
deviceProps.arch.hasWarpVote = 1;
deviceProps.arch.hasWarpBallot = 1;
deviceProps.arch.hasWarpShuffle = 1;
deviceProps.arch.hasFunnelShift = 0;
deviceProps.arch.hasThreadFenceSystem = 1;
deviceProps.arch.hasSyncThreadsExt = 0;
deviceProps.arch.hasSurfaceFuncs = 0;
deviceProps.arch.has3dGrid = 1;
deviceProps.arch.hasDynamicParallelism = 0;
deviceProps.concurrentKernels = 1;
deviceProps.pciDomainID = info.pciDomainID;
deviceProps.pciBusID = info.deviceTopology_.pcie.bus;
deviceProps.pciDeviceID = info.deviceTopology_.pcie.device;
deviceProps.maxSharedMemoryPerMultiProcessor = info.localMemSizePerCU_;
deviceProps.canMapHostMemory = 1;
//FIXME: This should be removed, targets can have character names as well.
deviceProps.gcnArch = isa.versionMajor() * 100 + isa.versionMinor() * 10 + isa.versionStepping();
sprintf(deviceProps.gcnArchName, "%s", isa.targetId());
deviceProps.cooperativeLaunch = info.cooperativeGroups_;
deviceProps.cooperativeMultiDeviceLaunch = info.cooperativeMultiDeviceGroups_;
deviceProps.cooperativeMultiDeviceUnmatchedFunc = info.cooperativeMultiDeviceGroups_;
deviceProps.cooperativeMultiDeviceUnmatchedGridDim = info.cooperativeMultiDeviceGroups_;
deviceProps.cooperativeMultiDeviceUnmatchedBlockDim = info.cooperativeMultiDeviceGroups_;
deviceProps.cooperativeMultiDeviceUnmatchedSharedMem = info.cooperativeMultiDeviceGroups_;
deviceProps.maxTexture1DLinear = 16 * info.imageMaxBufferSize_; // Max pixel size is 16 bytes
deviceProps.maxTexture1D = info.image1DMaxWidth_;
deviceProps.maxTexture2D[0] = info.image2DMaxWidth_;
deviceProps.maxTexture2D[1] = info.image2DMaxHeight_;
deviceProps.maxTexture3D[0] = info.image3DMaxWidth_;
deviceProps.maxTexture3D[1] = info.image3DMaxHeight_;
deviceProps.maxTexture3D[2] = info.image3DMaxDepth_;
deviceProps.hdpMemFlushCntl = info.hdpMemFlushCntl;
deviceProps.hdpRegFlushCntl = info.hdpRegFlushCntl;
deviceProps.memPitch = info.maxMemAllocSize_;
deviceProps.textureAlignment = info.imageBaseAddressAlignment_;
deviceProps.texturePitchAlignment = info.imagePitchAlignment_;
deviceProps.kernelExecTimeoutEnabled = 0;
deviceProps.ECCEnabled = info.errorCorrectionSupport_? 1:0;
deviceProps.isLargeBar = info.largeBar_ ? 1 : 0;
deviceProps.asicRevision = info.asicRevision_;
// HMM capabilities
deviceProps.managedMemory = info.hmmSupported_;
deviceProps.concurrentManagedAccess = info.hmmSupported_;
deviceProps.directManagedMemAccessFromHost = info.hmmDirectHostAccess_;
deviceProps.pageableMemoryAccess = info.hmmCpuMemoryAccessible_;
deviceProps.pageableMemoryAccessUsesHostPageTables = info.hostUnifiedMemory_;
*props = deviceProps;
HIP_RETURN(hipSuccess);
}
projects/hip/src/hipamd/src/hip_device_runtime.cpp
Executable
+563
Fájl megtekintése
@@ -0,0 +1,563 @@
/* Copyright (c) 2018-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
hipError_t hipChooseDevice(int* device, const hipDeviceProp_t* properties) {
HIP_INIT_API(hipChooseDevice, device, properties);
if (device == nullptr || properties == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
*device = 0;
cl_uint maxMatchedCount = 0;
int count = 0;
ihipDeviceGetCount(&count);
for (cl_int i = 0; i< count; ++i) {
hipDeviceProp_t currentProp = {0};
cl_uint validPropCount = 0;
cl_uint matchedCount = 0;
hipError_t err = hipGetDeviceProperties(&currentProp, i);
if (properties->major != 0) {
validPropCount++;
if(currentProp.major >= properties->major) {
matchedCount++;
}
}
if (properties->minor != 0) {
validPropCount++;
if(currentProp.minor >= properties->minor) {
matchedCount++;
}
}
if(properties->totalGlobalMem != 0) {
validPropCount++;
if(currentProp.totalGlobalMem >= properties->totalGlobalMem) {
matchedCount++;
}
}
if(properties->sharedMemPerBlock != 0) {
validPropCount++;
if(currentProp.sharedMemPerBlock >= properties->sharedMemPerBlock) {
matchedCount++;
}
}
if(properties->maxThreadsPerBlock != 0) {
validPropCount++;
if(currentProp.maxThreadsPerBlock >= properties->maxThreadsPerBlock ) {
matchedCount++;
}
}
if(properties->totalConstMem != 0) {
validPropCount++;
if(currentProp.totalConstMem >= properties->totalConstMem ) {
matchedCount++;
}
}
if(properties->multiProcessorCount != 0) {
validPropCount++;
if(currentProp.multiProcessorCount >=
properties->multiProcessorCount ) {
matchedCount++;
}
}
if(properties->maxThreadsPerMultiProcessor != 0) {
validPropCount++;
if(currentProp.maxThreadsPerMultiProcessor >=
properties->maxThreadsPerMultiProcessor ) {
matchedCount++;
}
}
if(properties->memoryClockRate != 0) {
validPropCount++;
if(currentProp.memoryClockRate >= properties->memoryClockRate ) {
matchedCount++;
}
}
if(properties->memoryBusWidth != 0) {
validPropCount++;
if(currentProp.memoryBusWidth >= properties->memoryBusWidth ) {
matchedCount++;
}
}
if(properties->l2CacheSize != 0) {
validPropCount++;
if(currentProp.l2CacheSize >= properties->l2CacheSize ) {
matchedCount++;
}
}
if(properties->regsPerBlock != 0) {
validPropCount++;
if(currentProp.regsPerBlock >= properties->regsPerBlock ) {
matchedCount++;
}
}
if(properties->maxSharedMemoryPerMultiProcessor != 0) {
validPropCount++;
if(currentProp.maxSharedMemoryPerMultiProcessor >=
properties->maxSharedMemoryPerMultiProcessor ) {
matchedCount++;
}
}
if(properties->warpSize != 0) {
validPropCount++;
if(currentProp.warpSize >= properties->warpSize ) {
matchedCount++;
}
}
if(validPropCount == matchedCount) {
*device = matchedCount > maxMatchedCount ? i : *device;
maxMatchedCount = std::max(matchedCount, maxMatchedCount);
}
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetAttribute(int* pi, hipDeviceAttribute_t attr, int device) {
HIP_INIT_API(hipDeviceGetAttribute, pi, attr, device);
if (pi == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
int count = 0;
ihipDeviceGetCount(&count);
if (device < 0 || device >= count) {
HIP_RETURN(hipErrorInvalidDevice);
}
//FIXME: should we cache the props, or just select from deviceHandle->info_?
hipDeviceProp_t prop = {0};
hipError_t err = hipGetDeviceProperties(&prop, device);
if (err != hipSuccess) {
HIP_RETURN(err);
}
switch (attr) {
case hipDeviceAttributeMaxThreadsPerBlock:
*pi = prop.maxThreadsPerBlock;
break;
case hipDeviceAttributeMaxBlockDimX:
*pi = prop.maxThreadsDim[0];
break;
case hipDeviceAttributeMaxBlockDimY:
*pi = prop.maxThreadsDim[1];
break;
case hipDeviceAttributeMaxBlockDimZ:
*pi = prop.maxThreadsDim[2];
break;
case hipDeviceAttributeMaxGridDimX:
*pi = prop.maxGridSize[0];
break;
case hipDeviceAttributeMaxGridDimY:
*pi = prop.maxGridSize[1];
break;
case hipDeviceAttributeMaxGridDimZ:
*pi = prop.maxGridSize[2];
break;
case hipDeviceAttributeMaxSharedMemoryPerBlock:
*pi = prop.sharedMemPerBlock;
break;
case hipDeviceAttributeTotalConstantMemory:
*pi = prop.totalConstMem;
break;
case hipDeviceAttributeWarpSize:
*pi = prop.warpSize;
break;
case hipDeviceAttributeMaxRegistersPerBlock:
*pi = prop.regsPerBlock;
break;
case hipDeviceAttributeClockRate:
*pi = prop.clockRate;
break;
case hipDeviceAttributeMemoryClockRate:
*pi = prop.memoryClockRate;
break;
case hipDeviceAttributeMemoryBusWidth:
*pi = prop.memoryBusWidth;
break;
case hipDeviceAttributeMultiprocessorCount:
*pi = prop.multiProcessorCount;
break;
case hipDeviceAttributeComputeMode:
*pi = prop.computeMode;
break;
case hipDeviceAttributeL2CacheSize:
*pi = prop.l2CacheSize;
break;
case hipDeviceAttributeMaxThreadsPerMultiProcessor:
*pi = prop.maxThreadsPerMultiProcessor;
break;
case hipDeviceAttributeComputeCapabilityMajor:
*pi = prop.major;
break;
case hipDeviceAttributeComputeCapabilityMinor:
*pi = prop.minor;
break;
case hipDeviceAttributePciBusId:
*pi = prop.pciBusID;
break;
case hipDeviceAttributeConcurrentKernels:
*pi = prop.concurrentKernels;
break;
case hipDeviceAttributePciDeviceId:
*pi = prop.pciDeviceID;
break;
case hipDeviceAttributeMaxSharedMemoryPerMultiprocessor:
*pi = prop.maxSharedMemoryPerMultiProcessor;
break;
case hipDeviceAttributeIsMultiGpuBoard:
*pi = prop.isMultiGpuBoard;
break;
case hipDeviceAttributeCooperativeLaunch:
*pi = prop.cooperativeLaunch;
break;
case hipDeviceAttributeCooperativeMultiDeviceLaunch:
*pi = prop.cooperativeMultiDeviceLaunch;
break;
case hipDeviceAttributeIntegrated:
*pi = prop.integrated;
break;
case hipDeviceAttributeMaxTexture1DWidth:
*pi = prop.maxTexture1D;
break;
case hipDeviceAttributeMaxTexture2DWidth:
*pi = prop.maxTexture2D[0];
break;
case hipDeviceAttributeMaxTexture2DHeight:
*pi = prop.maxTexture2D[1];
break;
case hipDeviceAttributeMaxTexture3DWidth:
*pi = prop.maxTexture3D[0];
break;
case hipDeviceAttributeMaxTexture3DHeight:
*pi = prop.maxTexture3D[1];
break;
case hipDeviceAttributeMaxTexture3DDepth:
*pi = prop.maxTexture3D[2];
break;
case hipDeviceAttributeHdpMemFlushCntl:
*reinterpret_cast<unsigned int**>(pi) = prop.hdpMemFlushCntl;
break;
case hipDeviceAttributeHdpRegFlushCntl:
*reinterpret_cast<unsigned int**>(pi) = prop.hdpRegFlushCntl;
break;
case hipDeviceAttributeMaxPitch:
*pi = prop.memPitch;
break;
case hipDeviceAttributeTextureAlignment:
*pi = prop.textureAlignment;
break;
case hipDeviceAttributeTexturePitchAlignment:
*pi = prop.texturePitchAlignment;
break;
case hipDeviceAttributeKernelExecTimeout:
*pi = prop.kernelExecTimeoutEnabled;
break;
case hipDeviceAttributeCanMapHostMemory:
*pi = prop.canMapHostMemory;
break;
case hipDeviceAttributeEccEnabled:
*pi = prop.ECCEnabled;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedFunc:
*pi = prop.cooperativeMultiDeviceUnmatchedFunc;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedGridDim:
*pi = prop.cooperativeMultiDeviceUnmatchedGridDim;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedBlockDim:
*pi = prop.cooperativeMultiDeviceUnmatchedBlockDim;
break;
case hipDeviceAttributeCooperativeMultiDeviceUnmatchedSharedMem:
*pi = prop.cooperativeMultiDeviceUnmatchedSharedMem;
break;
case hipDeviceAttributeAsicRevision:
*pi = prop.asicRevision;
break;
case hipDeviceAttributeManagedMemory:
*pi = prop.managedMemory;
break;
case hipDeviceAttributeDirectManagedMemAccessFromHost:
*pi = prop.directManagedMemAccessFromHost;
break;
case hipDeviceAttributeConcurrentManagedAccess:
*pi = prop.concurrentManagedAccess;
break;
case hipDeviceAttributePageableMemoryAccess:
*pi = prop.pageableMemoryAccess;
break;
case hipDeviceAttributePageableMemoryAccessUsesHostPageTables:
*pi = prop.pageableMemoryAccessUsesHostPageTables;
break;
case hipDeviceAttributeCanUseStreamWaitValue:
// hipStreamWaitValue64() and hipStreamWaitValue32() support
*pi = g_devices[device]->devices()[0]->info().aqlBarrierValue_;
break;
default:
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetByPCIBusId(int* device, const char*pciBusIdstr) {
HIP_INIT_API(hipDeviceGetByPCIBusId, device, pciBusIdstr);
if (device == nullptr || pciBusIdstr == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
int pciBusID = -1;
int pciDeviceID = -1;
int pciDomainID = -1;
bool found = false;
if (sscanf (pciBusIdstr, "%04x:%02x:%02x", &pciDomainID, &pciBusID, &pciDeviceID) == 0x3) {
int count = 0;
ihipDeviceGetCount(&count);
for (cl_int i = 0; i < count; i++) {
hipDevice_t dev;
hipDeviceGet(&dev, i);
hipDeviceProp_t prop;
hipGetDeviceProperties(&prop, dev);
if ((pciBusID == prop.pciBusID) && (pciDomainID == prop.pciDomainID)
&& (pciDeviceID == prop.pciDeviceID)) {
*device = i;
found = true;
break;
}
}
}
if (!found) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetCacheConfig ( hipFuncCache_t * cacheConfig ) {
HIP_INIT_API(hipDeviceGetCacheConfig, cacheConfig);
if(cacheConfig == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
*cacheConfig = hipFuncCache_t();
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetLimit ( size_t* pValue, hipLimit_t limit ) {
HIP_INIT_API(hipDeviceGetLimit, pValue, limit);
if(pValue == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if(limit == hipLimitMallocHeapSize) {
hipDeviceProp_t prop;
hipGetDeviceProperties(&prop, ihipGetDevice());
*pValue = prop.totalGlobalMem;
HIP_RETURN(hipSuccess);
} else {
HIP_RETURN(hipErrorUnsupportedLimit);
}
}
hipError_t hipDeviceGetPCIBusId ( char* pciBusId, int len, int device ) {
HIP_INIT_API(hipDeviceGetPCIBusId, (void*)pciBusId, len, device);
int count;
ihipDeviceGetCount(&count);
if (device < 0 || device >= count) {
HIP_RETURN(hipErrorInvalidDevice);
}
if (pciBusId == nullptr || len <= 0) {
HIP_RETURN(hipErrorInvalidValue);
}
hipDeviceProp_t prop;
hipGetDeviceProperties(&prop, device);
snprintf (pciBusId, len, "%04x:%02x:%02x.0",
prop.pciDomainID,
prop.pciBusID,
prop.pciDeviceID);
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetSharedMemConfig ( hipSharedMemConfig * pConfig ) {
HIP_INIT_API(hipDeviceGetSharedMemConfig, pConfig);
if (pConfig == nullptr) {
return HIP_RETURN(hipErrorInvalidValue);
}
*pConfig = hipSharedMemBankSizeFourByte;
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceReset ( void ) {
HIP_INIT_API(hipDeviceReset);
/* FIXME */
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceSetCacheConfig ( hipFuncCache_t cacheConfig ) {
HIP_INIT_API(hipDeviceSetCacheConfig, cacheConfig);
// No way to set cache config yet.
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceSetLimit ( hipLimit_t limit, size_t value ) {
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipDeviceSetSharedMemConfig ( hipSharedMemConfig config ) {
HIP_INIT_API(hipDeviceSetSharedMemConfig, config);
// No way to set cache config yet.
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceSynchronize ( void ) {
HIP_INIT_API(hipDeviceSynchronize);
amd::HostQueue* queue = hip::getNullStream();
if (!queue) {
HIP_RETURN(hipErrorOutOfMemory);
}
queue->finish();
hip::Stream::syncNonBlockingStreams();
HIP_RETURN(hipSuccess);
}
int ihipGetDevice() {
hip::Device* device = hip::getCurrentDevice();
if(device == nullptr){
return -1;
}
return device->deviceId();
}
hipError_t hipGetDevice ( int* deviceId ) {
HIP_INIT_API(hipGetDevice, deviceId);
if (deviceId != nullptr) {
int dev = ihipGetDevice();
if (dev == -1) {
HIP_RETURN(hipErrorNoDevice);
}
*deviceId = dev;
HIP_RETURN(hipSuccess);
} else {
HIP_RETURN(hipErrorInvalidValue);
}
}
hipError_t hipGetDeviceCount ( int* count ) {
HIP_INIT_API(hipGetDeviceCount, count);
HIP_RETURN(ihipDeviceGetCount(count));
}
hipError_t hipGetDeviceFlags ( unsigned int* flags ) {
HIP_INIT_API(hipGetDeviceFlags, flags);
if (flags == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
*flags = hip::getCurrentDevice()->getFlags();
HIP_RETURN(hipSuccess);
}
hipError_t hipSetDevice ( int device ) {
HIP_INIT_API(hipSetDevice, device);
if (static_cast<unsigned int>(device) < g_devices.size()) {
hip::setCurrentDevice(device);
HIP_RETURN(hipSuccess);
}
HIP_RETURN(hipErrorInvalidDevice);
}
hipError_t hipSetDeviceFlags ( unsigned int flags ) {
HIP_INIT_API(hipSetDeviceFlags, flags);
constexpr uint32_t supportedFlags =
hipDeviceScheduleMask | hipDeviceMapHost | hipDeviceLmemResizeToMax;
if (flags & ~supportedFlags) {
HIP_RETURN(hipErrorInvalidValue);
}
amd::Device* device = hip::getCurrentDevice()->devices()[0];
switch (flags & hipDeviceScheduleMask) {
case hipDeviceScheduleAuto:
// Current behavior is different from the spec, due to MT usage in runtime
if (hip::host_device->devices().size() >= std::thread::hardware_concurrency()) {
device->SetActiveWait(false);
break;
}
// Fall through for active wait...
case hipDeviceScheduleSpin:
case hipDeviceScheduleYield:
// The both options falls into yield, because MT usage in runtime
device->SetActiveWait(true);
break;
case hipDeviceScheduleBlockingSync:
device->SetActiveWait(false);
break;
default:
break;
}
hip::getCurrentDevice()->setFlags(flags & hipDeviceScheduleMask);
HIP_RETURN(hipSuccess);
}
hipError_t hipSetValidDevices ( int* device_arr, int len ) {
HIP_INIT_API(hipSetValidDevices, device_arr, len);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
projects/hip/src/hipamd/src/hip_error.cpp
+176
Fájl megtekintése
@@ -0,0 +1,176 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
hipError_t hipGetLastError()
{
HIP_INIT_API(hipGetLastError);
hipError_t err = hip::g_lastError;
hip::g_lastError = hipSuccess;
return err;
}
hipError_t hipPeekAtLastError()
{
HIP_INIT_API(hipPeekAtLastError);
hipError_t err = hip::g_lastError;
HIP_RETURN(err);
}
const char *hipGetErrorName(hipError_t hip_error)
{
switch (hip_error) {
case hipSuccess:
return "hipSuccess";
case hipErrorInvalidValue:
return "hipErrorInvalidValue";
case hipErrorOutOfMemory:
return "hipErrorOutOfMemory";
case hipErrorNotInitialized:
return "hipErrorNotInitialized";
case hipErrorDeinitialized:
return "hipErrorDeinitialized";
case hipErrorProfilerDisabled:
return "hipErrorProfilerDisabled";
case hipErrorProfilerNotInitialized:
return "hipErrorProfilerNotInitialized";
case hipErrorProfilerAlreadyStarted:
return "hipErrorProfilerAlreadyStarted";
case hipErrorProfilerAlreadyStopped:
return "hipErrorProfilerAlreadyStopped";
case hipErrorInvalidConfiguration:
return "hipErrorInvalidConfiguration";
case hipErrorInvalidSymbol:
return "hipErrorInvalidSymbol";
case hipErrorInvalidDevicePointer:
return "hipErrorInvalidDevicePointer";
case hipErrorInvalidMemcpyDirection:
return "hipErrorInvalidMemcpyDirection";
case hipErrorInsufficientDriver:
return "hipErrorInsufficientDriver";
case hipErrorMissingConfiguration:
return "hipErrorMissingConfiguration";
case hipErrorPriorLaunchFailure:
return "hipErrorPriorLaunchFailure";
case hipErrorInvalidDeviceFunction:
return "hipErrorInvalidDeviceFunction";
case hipErrorNoDevice:
return "hipErrorNoDevice";
case hipErrorInvalidDevice:
return "hipErrorInvalidDevice";
case hipErrorInvalidPitchValue:
return "hipErrorInvalidPitchValue";
case hipErrorInvalidImage:
return "hipErrorInvalidImage";
case hipErrorInvalidContext:
return "hipErrorInvalidContext";
case hipErrorContextAlreadyCurrent:
return "hipErrorContextAlreadyCurrent";
case hipErrorMapFailed:
return "hipErrorMapFailed";
case hipErrorUnmapFailed:
return "hipErrorUnmapFailed";
case hipErrorArrayIsMapped:
return "hipErrorArrayIsMapped";
case hipErrorAlreadyMapped:
return "hipErrorAlreadyMapped";
case hipErrorNoBinaryForGpu:
return "hipErrorNoBinaryForGpu";
case hipErrorAlreadyAcquired:
return "hipErrorAlreadyAcquired";
case hipErrorNotMapped:
return "hipErrorNotMapped";
case hipErrorNotMappedAsArray:
return "hipErrorNotMappedAsArray";
case hipErrorNotMappedAsPointer:
return "hipErrorNotMappedAsPointer";
case hipErrorECCNotCorrectable:
return "hipErrorECCNotCorrectable";
case hipErrorUnsupportedLimit:
return "hipErrorUnsupportedLimit";
case hipErrorContextAlreadyInUse:
return "hipErrorContextAlreadyInUse";
case hipErrorPeerAccessUnsupported:
return "hipErrorPeerAccessUnsupported";
case hipErrorInvalidKernelFile:
return "hipErrorInvalidKernelFile";
case hipErrorInvalidGraphicsContext:
return "hipErrorInvalidGraphicsContext";
case hipErrorInvalidSource:
return "hipErrorInvalidSource";
case hipErrorFileNotFound:
return "hipErrorFileNotFound";
case hipErrorSharedObjectSymbolNotFound:
return "hipErrorSharedObjectSymbolNotFound";
case hipErrorSharedObjectInitFailed:
return "hipErrorSharedObjectInitFailed";
case hipErrorOperatingSystem:
return "hipErrorOperatingSystem";
case hipErrorInvalidHandle:
return "hipErrorInvalidHandle";
case hipErrorNotFound:
return "hipErrorNotFound";
case hipErrorNotReady:
return "hipErrorNotReady";
case hipErrorIllegalAddress:
return "hipErrorIllegalAddress";
case hipErrorLaunchOutOfResources:
return "hipErrorLaunchOutOfResources";
case hipErrorLaunchTimeOut:
return "hipErrorLaunchTimeOut";
case hipErrorPeerAccessAlreadyEnabled:
return "hipErrorPeerAccessAlreadyEnabled";
case hipErrorPeerAccessNotEnabled:
return "hipErrorPeerAccessNotEnabled";
case hipErrorSetOnActiveProcess:
return "hipErrorSetOnActiveProcess";
case hipErrorAssert:
return "hipErrorAssert";
case hipErrorHostMemoryAlreadyRegistered:
return "hipErrorHostMemoryAlreadyRegistered";
case hipErrorHostMemoryNotRegistered:
return "hipErrorHostMemoryNotRegistered";
case hipErrorLaunchFailure:
return "hipErrorLaunchFailure";
case hipErrorNotSupported:
return "hipErrorNotSupported";
case hipErrorUnknown:
return "hipErrorUnknown";
case hipErrorRuntimeMemory:
return "hipErrorRuntimeMemory";
case hipErrorRuntimeOther:
return "hipErrorRuntimeOther";
case hipErrorCooperativeLaunchTooLarge:
return "hipErrorCooperativeLaunchTooLarge";
case hipErrorTbd:
return "hipErrorTbd";
default:
return "hipErrorUnknown";
};
}
const char *hipGetErrorString(hipError_t hip_error)
{
return hipGetErrorName(hip_error);
}
projects/hip/src/hipamd/src/hip_event.cpp
Executable
+431
Fájl megtekintése
@@ -0,0 +1,431 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_event.hpp"
void ipcEventCallback(hipStream_t stream, hipError_t status, void* user_data)
{
std::atomic<int> *signal = reinterpret_cast<std::atomic<int>*>(user_data);
signal->store(0);
return;
}
namespace hip {
bool Event::ready() {
if (event_->status() != CL_COMPLETE) {
event_->notifyCmdQueue();
}
return (event_->status() == CL_COMPLETE);
}
hipError_t Event::query() {
amd::ScopedLock lock(lock_);
// If event is not recorded, event_ is null, hence return hipSuccess
if (event_ == nullptr) {
return hipSuccess;
}
return ready() ? hipSuccess : hipErrorNotReady;
}
hipError_t Event::synchronize() {
amd::ScopedLock lock(lock_);
// If event is not recorded, event_ is null, hence return hipSuccess
if (event_ == nullptr) {
return hipSuccess;
}
event_->awaitCompletion();
return hipSuccess;
}
hipError_t Event::elapsedTime(Event& eStop, float& ms) {
amd::ScopedLock startLock(lock_);
if (this == &eStop) {
if (event_ == nullptr) {
return hipErrorInvalidHandle;
}
if (flags & hipEventDisableTiming) {
return hipErrorInvalidHandle;
}
if (!ready()) {
return hipErrorNotReady;
}
ms = 0.f;
return hipSuccess;
}
amd::ScopedLock stopLock(eStop.lock_);
if (event_ == nullptr ||
eStop.event_ == nullptr) {
return hipErrorInvalidHandle;
}
if ((flags | eStop.flags) & hipEventDisableTiming) {
return hipErrorInvalidHandle;
}
if (!ready() || !eStop.ready()) {
return hipErrorNotReady;
}
if (event_ == eStop.event_ && recorded_ && eStop.recorded_) {
// Events are the same, which indicates the stream is empty and likely
// eventRecord is called on another stream. For such cases insert and measure a
// marker.
amd::Command* command = new amd::Marker(*event_->command().queue(), kMarkerDisableFlush);
command->enqueue();
command->awaitCompletion();
ms = static_cast<float>(static_cast<int64_t>(command->event().profilingInfo().end_) - time())/1000000.f;
command->release();
} else {
ms = static_cast<float>(eStop.time() - time())/1000000.f;
}
return hipSuccess;
}
int64_t Event::time() const {
assert(event_ != nullptr);
if (recorded_) {
return static_cast<int64_t>(event_->profilingInfo().end_);
} else {
return static_cast<int64_t>(event_->profilingInfo().start_);
}
}
hipError_t Event::streamWait(amd::HostQueue* hostQueue, uint flags) {
if ((event_ == nullptr) || (event_->command().queue() == hostQueue)) {
return hipSuccess;
}
amd::ScopedLock lock(lock_);
bool retain = false;
if (!event_->notifyCmdQueue()) {
return hipErrorLaunchOutOfResources;
}
amd::Command::EventWaitList eventWaitList;
eventWaitList.push_back(event_);
amd::Command* command = new amd::Marker(*hostQueue, kMarkerDisableFlush, eventWaitList);
if (command == NULL) {
return hipErrorOutOfMemory;
}
command->enqueue();
command->release();
return hipSuccess;
}
void Event::addMarker(amd::HostQueue* queue, amd::Command* command, bool record) {
// Keep the lock always at the beginning of this to avoid a race. SWDEV-277847
amd::ScopedLock lock(lock_);
if (command == nullptr) {
bool recordExplicitGpuTs = !queue->properties().test(CL_QUEUE_PROFILING_ENABLE) &&
!(flags & hipEventDisableTiming);
// Always submit a EventMarker. This would submit a NOP with a signal.
command = new hip::EventMarker(*queue, !kMarkerDisableFlush, recordExplicitGpuTs);
command->enqueue();
}
if (event_ == &command->event()) return;
if (event_ != nullptr) {
event_->release();
}
event_ = &command->event();
recorded_ = record;
}
}
hipError_t ihipEventCreateWithFlags(hipEvent_t* event, unsigned flags) {
if (event == nullptr) {
return hipErrorInvalidValue;
}
#if !defined(_MSC_VER)
unsigned supportedFlags = hipEventDefault | hipEventBlockingSync | hipEventDisableTiming |
hipEventReleaseToDevice | hipEventReleaseToSystem | hipEventInterprocess;
#else
unsigned supportedFlags = hipEventDefault | hipEventBlockingSync | hipEventDisableTiming |
hipEventReleaseToDevice | hipEventReleaseToSystem;
#endif
const unsigned releaseFlags = (hipEventReleaseToDevice | hipEventReleaseToSystem);
const bool illegalFlags =
(flags & ~supportedFlags) || // can't set any unsupported flags.
(flags & releaseFlags) == releaseFlags; // can't set both release flags
if (!illegalFlags) {
hip::Event* e = new hip::Event(flags);
if (e == nullptr) {
return hipErrorOutOfMemory;
}
*event = reinterpret_cast<hipEvent_t>(e);
} else {
return hipErrorInvalidValue;
}
return hipSuccess;
}
hipError_t ihipEventQuery(hipEvent_t event) {
if (event == nullptr) {
return hipErrorInvalidHandle;
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) {
int prev_read_idx = e->ipc_evt_.ipc_shmem_->read_index;
int offset = (prev_read_idx % IPC_SIGNALS_PER_EVENT);
if (e->ipc_evt_.ipc_shmem_->read_index < prev_read_idx+IPC_SIGNALS_PER_EVENT && e->ipc_evt_.ipc_shmem_->signal[offset] != 0) {
return hipErrorNotReady;
}
return hipSuccess;
} else {
return e->query();
}
}
hipError_t hipEventCreateWithFlags(hipEvent_t* event, unsigned flags) {
HIP_INIT_API(hipEventCreateWithFlags, event, flags);
HIP_RETURN(ihipEventCreateWithFlags(event, flags), *event);
}
hipError_t hipEventCreate(hipEvent_t* event) {
HIP_INIT_API(hipEventCreate, event);
HIP_RETURN(ihipEventCreateWithFlags(event, 0), *event);
}
hipError_t hipEventDestroy(hipEvent_t event) {
HIP_INIT_API(hipEventDestroy, event);
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) {
int owners = -- e->ipc_evt_.ipc_shmem_->owners;
// Make sure event is synchronized
hipEventSynchronize(event);
if (!amd::Os::MemoryUnmapFile(e->ipc_evt_.ipc_shmem_,sizeof(hip::ihipIpcEventShmem_t))) {
HIP_RETURN(hipErrorInvalidHandle);
}
}
delete e;
HIP_RETURN(hipSuccess);
}
hipError_t hipEventElapsedTime(float *ms, hipEvent_t start, hipEvent_t stop) {
HIP_INIT_API(hipEventElapsedTime, ms, start, stop);
if (start == nullptr || stop == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
if (ms == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hip::Event* eStart = reinterpret_cast<hip::Event*>(start);
hip::Event* eStop = reinterpret_cast<hip::Event*>(stop);
if (eStart->deviceId() != eStop->deviceId()) {
HIP_RETURN(hipErrorInvalidHandle);
}
HIP_RETURN(eStart->elapsedTime(*eStop, *ms), "Elapsed Time = ", *ms);
}
// ================================================================================================
bool createIpcEventShmemIfNeeded(hip::Event::ihipIpcEvent_t& ipc_evt) {
#if !defined(_MSC_VER)
if (ipc_evt.ipc_shmem_) {
// ipc_shmem_ already created, no need to create it again
return true;
}
char name_template[] = "/tmp/eventXXXXXX";
int temp_fd = mkstemp(name_template);
ipc_evt.ipc_name_ = name_template;
ipc_evt.ipc_name_.replace(0, 5, "/hip_");
if (!amd::Os::MemoryMapFileTruncated(ipc_evt.ipc_name_.c_str(),
const_cast<const void**> (reinterpret_cast<void**>(&(ipc_evt.ipc_shmem_))), sizeof(hip::ihipIpcEventShmem_t))) {
return false;
}
ipc_evt.ipc_shmem_->owners = 1;
ipc_evt.ipc_shmem_->read_index = -1;
ipc_evt.ipc_shmem_->write_index = 0;
for (uint32_t sig_idx = 0; sig_idx < IPC_SIGNALS_PER_EVENT; ++sig_idx) {
ipc_evt.ipc_shmem_->signal[sig_idx] = 0;
}
close(temp_fd);
return true;
#else
return false;
#endif
}
hipError_t hipEventRecord(hipEvent_t event, hipStream_t stream) {
HIP_INIT_API(hipEventRecord, event, stream);
STREAM_CAPTURE(hipEventRecord, stream, event);
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
amd::HostQueue* queue = hip::getQueue(stream);
if (g_devices[e->deviceId()]->devices()[0] != &queue->device()) {
HIP_RETURN(hipErrorInvalidHandle);
}
bool isRecorded = e->isRecorded();
if ((e->flags & hipEventInterprocess) && !isRecorded) {
amd::Command* command = new amd::Marker(*queue, kMarkerDisableFlush);
amd::Event& tEvent = command->event();
createIpcEventShmemIfNeeded(e->ipc_evt_);
int write_index = e->ipc_evt_.ipc_shmem_->write_index++;
int offset = write_index % IPC_SIGNALS_PER_EVENT;
while (e->ipc_evt_.ipc_shmem_->signal[offset] != 0) {
amd::Os::sleep(1);
}
// Lock signal.
e->ipc_evt_.ipc_shmem_->signal[offset] = 1;
e->ipc_evt_.ipc_shmem_->owners_device_id = e->deviceId();
std::atomic<int> *signal = &e->ipc_evt_.ipc_shmem_->signal[offset];
StreamCallback* cbo = new StreamCallback(stream,
reinterpret_cast<hipStreamCallback_t> (ipcEventCallback), signal, command);
if (!tEvent.setCallback(CL_COMPLETE, ihipStreamCallback,cbo)) {
command->release();
return hipErrorInvalidHandle;
}
command->enqueue();
tEvent.notifyCmdQueue();
// Update read index to indicate new signal.
int expected = write_index - 1;
while (!e->ipc_evt_.ipc_shmem_->read_index.compare_exchange_weak(expected, write_index)) {
amd::Os::sleep(1);
}
} else {
e->addMarker(queue, nullptr, true);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipEventSynchronize(hipEvent_t event) {
HIP_INIT_API(hipEventSynchronize, event);
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if ((e->flags & hipEventInterprocess) && (e->ipc_evt_.ipc_shmem_)) {
int prev_read_idx = e->ipc_evt_.ipc_shmem_->read_index;
if (prev_read_idx >= 0) {
int offset = (prev_read_idx % IPC_SIGNALS_PER_EVENT);
while ((e->ipc_evt_.ipc_shmem_->read_index < prev_read_idx + IPC_SIGNALS_PER_EVENT)
&& (e->ipc_evt_.ipc_shmem_->signal[offset] != 0)) {
amd::Os::sleep(1);
}
}
HIP_RETURN(hipSuccess);
} else {
HIP_RETURN(e->synchronize());
}
}
hipError_t hipEventQuery(hipEvent_t event) {
HIP_INIT_API(hipEventQuery, event);
HIP_RETURN(ihipEventQuery(event));
}
hipError_t hipIpcGetEventHandle(hipIpcEventHandle_t* handle, hipEvent_t event) {
HIP_INIT_API(hipIpcGetEventHandle, handle, event);
#if !defined(_MSC_VER)
if (handle == nullptr || event == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if (!(e->flags & hipEventInterprocess)) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
if (!createIpcEventShmemIfNeeded(e->ipc_evt_)) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
ihipIpcEventHandle_t* iHandle = reinterpret_cast<ihipIpcEventHandle_t*>(handle);
memset(iHandle->shmem_name, 0, HIP_IPC_HANDLE_SIZE);
e->ipc_evt_.ipc_name_.copy(iHandle->shmem_name, std::string::npos);
HIP_RETURN(hipSuccess);
#else
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
#endif
}
hipError_t hipIpcOpenEventHandle(hipEvent_t* event, hipIpcEventHandle_t handle) {
HIP_INIT_API(NONE, event, handle);
#if !defined(_MSC_VER)
hipError_t hip_err = hipSuccess;
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hip_err = ihipEventCreateWithFlags(event, hipEventDisableTiming | hipEventInterprocess);
if (hip_err != hipSuccess) {
HIP_RETURN(hip_err);
}
hip::Event* e = reinterpret_cast<hip::Event*>(*event);
ihipIpcEventHandle_t* iHandle = reinterpret_cast<ihipIpcEventHandle_t*>(&handle);
hip::Event::ihipIpcEvent_t& ipc_evt = e->ipc_evt_;
ipc_evt.ipc_name_ = iHandle->shmem_name;
if (!amd::Os::MemoryMapFileTruncated(ipc_evt.ipc_name_.c_str(),
(const void**) &(ipc_evt.ipc_shmem_), sizeof(hip::ihipIpcEventShmem_t))) {
HIP_RETURN(hipErrorInvalidValue);
}
ipc_evt.ipc_shmem_->owners += 1;
e->setDeviceId(ipc_evt.ipc_shmem_->owners_device_id.load());
HIP_RETURN(hipSuccess);
#else
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
#endif
}
projects/hip/src/hipamd/src/hip_event.hpp
+151
Fájl megtekintése
@@ -0,0 +1,151 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef HIP_EVENT_H
#define HIP_EVENT_H
#include "hip_internal.hpp"
#include "thread/monitor.hpp"
// Internal structure for stream callback handler
class StreamCallback {
public:
StreamCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData,
amd::Command* command)
: stream_(stream), callBack_(callback),
userData_(userData), command_(command) {
};
hipStream_t stream_;
hipStreamCallback_t callBack_;
void* userData_;
amd::Command* command_;
};
void CL_CALLBACK ihipStreamCallback(cl_event event, cl_int command_exec_status, void* user_data);
namespace hip {
class EventMarker: public amd::Marker {
public:
EventMarker(amd::HostQueue& queue, bool disableFlush, bool markerTs = false)
: amd::Marker(queue, disableFlush) {
profilingInfo_.enabled_ = true;
profilingInfo_.callback_ = nullptr;
profilingInfo_.marker_ts_ = markerTs;
profilingInfo_.clear();
}
};
#define IPC_SIGNALS_PER_EVENT 32
typedef struct ihipIpcEventShmem_s {
std::atomic<int> owners;
std::atomic<int> owners_device_id;
std::atomic<int> read_index;
std::atomic<int> write_index;
std::atomic<int> signal[IPC_SIGNALS_PER_EVENT];
} ihipIpcEventShmem_t;
class Event {
/// event recorded on stream where capture is active
bool onCapture_;
/// capture stream where event is recorded
hipStream_t captureStream_;
/// Previous captured nodes before event record
std::vector<hipGraphNode_t> nodesPrevToRecorded_;
public:
Event(unsigned int flags) : flags(flags), lock_("hipEvent_t", true),
event_(nullptr), recorded_(false) {
// No need to init event_ here as addMarker does that
onCapture_ = false;
device_id_ = hip::getCurrentDevice()->deviceId(); // Created in current device ctx
}
~Event() {
if (event_ != nullptr) {
event_->release();
}
}
unsigned int flags;
hipError_t query();
hipError_t synchronize();
hipError_t elapsedTime(Event& stop, float& ms);
hipError_t streamWait(amd::HostQueue* queue, uint flags);
void addMarker(amd::HostQueue* queue, amd::Command* command, bool record);
bool isRecorded() { return recorded_; }
amd::Monitor& lock() { return lock_; }
const int deviceId() { return device_id_; }
void setDeviceId(int id) { device_id_ = id; }
/// End capture on this event
void EndCapture() {
onCapture_ = false;
captureStream_ = nullptr;
}
/// Start capture when waited on this event
void StartCapture(hipStream_t stream) {
onCapture_ = true;
captureStream_ = stream;
}
/// Get capture status of the graph
bool GetCaptureStatus() { return onCapture_; }
/// Get capture stream where event is recorded
hipStream_t GetCaptureStream() { return captureStream_; }
/// Set capture stream where event is recorded
void SetCaptureStream(hipStream_t stream) { captureStream_ = stream; }
/// Returns previous captured nodes before event record
std::vector<hipGraphNode_t> GetNodesPrevToRecorded() const { return nodesPrevToRecorded_; }
/// Set last captured graph node before event record
void SetNodesPrevToRecorded(std::vector<hipGraphNode_t>& graphNode) {
nodesPrevToRecorded_ = graphNode;
}
// IPC Events
struct ihipIpcEvent_t {
std::string ipc_name_;
int ipc_fd_;
ihipIpcEventShmem_t* ipc_shmem_;
ihipIpcEvent_t(): ipc_name_("dummy"), ipc_fd_(0), ipc_shmem_(nullptr) {
}
void setipcname(const char* name) {
ipc_name_ = std::string(name);
}
};
ihipIpcEvent_t ipc_evt_;
private:
amd::Monitor lock_;
amd::HostQueue* stream_;
amd::Event* event_;
int device_id_;
//! Flag to indicate hipEventRecord has been called. This is needed except for
//! hip*ModuleLaunchKernel API which takes start and stop events so no
//! hipEventRecord is called. Cleanup needed once those APIs are deprecated.
bool recorded_;
bool ready();
int64_t time() const;
};
};
#endif // HIP_EVEMT_H
projects/hip/src/hipamd/src/hip_fatbin.cpp
Executable
+158
Fájl megtekintése
@@ -0,0 +1,158 @@
#include "hip_fatbin.hpp"
#include "hip_code_object.hpp"
namespace hip {
FatBinaryDeviceInfo::~FatBinaryDeviceInfo() {
if (program_ != nullptr) {
program_->release();
program_ = nullptr;
}
}
FatBinaryInfo::FatBinaryInfo(const char* fname, const void* image)
: fdesc_(amd::Os::FDescInit()), fsize_(0), image_(image), uri_(std::string()) {
if (fname != nullptr) {
fname_ = std::string(fname);
} else {
fname_ = std::string();
}
fatbin_dev_info_.resize(g_devices.size());
}
FatBinaryInfo::~FatBinaryInfo() {
for (auto& fbd: fatbin_dev_info_) {
delete fbd;
}
if (fdesc_ > 0) {
if (fsize_ && !amd::Os::MemoryUnmapFile(image_, fsize_)) {
guarantee(false, "Cannot unmap file");
}
if (!amd::Os::CloseFileHandle(fdesc_)) {
guarantee(false, "Cannot close file");
}
}
fname_ = std::string();
fdesc_ = amd::Os::FDescInit();
fsize_ = 0;
image_ = nullptr;
uri_ = std::string();
}
hipError_t FatBinaryInfo::ExtractFatBinary(const std::vector<hip::Device*>& devices) {
hipError_t hip_error = hipSuccess;
std::vector<std::pair<const void*, size_t>> code_objs;
// Copy device names for Extract Code object File
std::vector<std::string> device_names;
device_names.reserve(devices.size());
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
device_names.push_back(devices[dev_idx]->devices()[0]->isa().isaName());
}
// We are given file name, get the file desc and file size
if (fname_.size() > 0) {
// Get File Handle & size of the file.
if (!amd::Os::GetFileHandle(fname_.c_str(), &fdesc_, &fsize_)) {
return hipErrorFileNotFound;
}
if (fsize_ == 0) {
return hipErrorInvalidKernelFile;
}
// Extract the code object from file
hip_error = CodeObject::ExtractCodeObjectFromFile(fdesc_, fsize_, &image_,
device_names, code_objs);
} else if (image_ != nullptr) {
// We are directly given image pointer directly, try to extract file desc & file Size
hip_error = CodeObject::ExtractCodeObjectFromMemory(image_,
device_names, code_objs, uri_);
} else {
return hipErrorInvalidValue;
}
if (hip_error == hipErrorNoBinaryForGpu) {
guarantee(false, "hipErrorNoBinaryForGpu: Couldn't find binary for current devices!");
return hip_error;
}
if (hip_error == hipErrorInvalidKernelFile) {
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
// the image type is no CLANG_OFFLOAD_BUNDLER, image for current device directly passed
fatbin_dev_info_[devices[dev_idx]->deviceId()]
= new FatBinaryDeviceInfo(image_, CodeObject::ElfSize(image_), 0);
}
} else if(hip_error == hipSuccess) {
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
// Calculate the offset wrt binary_image and the original image
size_t offset_l
= (reinterpret_cast<address>(const_cast<void*>(code_objs[dev_idx].first))
- reinterpret_cast<address>(const_cast<void*>(image_)));
fatbin_dev_info_[devices[dev_idx]->deviceId()]
= new FatBinaryDeviceInfo(code_objs[dev_idx].first, code_objs[dev_idx].second, offset_l);
}
}
for (size_t dev_idx = 0; dev_idx < devices.size(); ++dev_idx) {
fatbin_dev_info_[devices[dev_idx]->deviceId()]->program_
= new amd::Program(*devices[dev_idx]->asContext());
if (fatbin_dev_info_[devices[dev_idx]->deviceId()]->program_ == NULL) {
return hipErrorOutOfMemory;
}
}
return hipSuccess;
}
hipError_t FatBinaryInfo::AddDevProgram(const int device_id) {
// Device Id bounds Check
DeviceIdCheck(device_id);
FatBinaryDeviceInfo* fbd_info = fatbin_dev_info_[device_id];
// If fat binary was already added, skip this step and return success
if (fbd_info->add_dev_prog_ == false) {
amd::Context* ctx = g_devices[device_id]->asContext();
if (CL_SUCCESS != fbd_info->program_->addDeviceProgram(*ctx->devices()[0],
fbd_info->binary_image_,
fbd_info->binary_size_, false,
nullptr, nullptr, fdesc_,
fbd_info->binary_offset_, uri_)) {
return hipErrorInvalidKernelFile;
}
fbd_info->add_dev_prog_ = true;
}
return hipSuccess;
}
hipError_t FatBinaryInfo::BuildProgram(const int device_id) {
// Device Id Check and Add DeviceProgram if not added so far
DeviceIdCheck(device_id);
IHIP_RETURN_ONFAIL(AddDevProgram(device_id));
// If Program was already built skip this step and return success
FatBinaryDeviceInfo* fbd_info = fatbin_dev_info_[device_id];
if (fbd_info->prog_built_ == false) {
if(CL_SUCCESS != fbd_info->program_->build(g_devices[device_id]->devices(),
nullptr, nullptr, nullptr,
kOptionChangeable, kNewDevProg)) {
return hipErrorSharedObjectInitFailed;
}
fbd_info->prog_built_ = true;
}
if (!fbd_info->program_->load()) {
return hipErrorSharedObjectInitFailed;
}
return hipSuccess;
}
} //namespace : hip
projects/hip/src/hipamd/src/hip_fatbin.hpp
Executable
+87
Fájl megtekintése
@@ -0,0 +1,87 @@
#ifndef HIP_FAT_BINARY_HPP
#define HIP_FAT_BINARY_HPP
#include "hip/hip_runtime.h"
#include "hip/hip_runtime_api.h"
#include "hip_internal.hpp"
#include "platform/program.hpp"
namespace hip {
//Fat Binary Per Device info
class FatBinaryDeviceInfo {
public:
FatBinaryDeviceInfo (const void* binary_image, size_t binary_size, size_t binary_offset)
: binary_image_(binary_image), binary_size_(binary_size),
binary_offset_(binary_offset), program_(nullptr),
add_dev_prog_(false), prog_built_(false) {}
~FatBinaryDeviceInfo();
private:
const void* binary_image_; // binary image ptr
size_t binary_size_; // binary image size
size_t binary_offset_; // image offset from original
amd::Program* program_; // reinterpreted as hipModule_t
friend class FatBinaryInfo;
//Control Variables
bool add_dev_prog_;
bool prog_built_;
};
// Fat Binary Info
class FatBinaryInfo {
public:
FatBinaryInfo(const char* fname, const void* image);
~FatBinaryInfo();
// Loads Fat binary from file or image, unbundles COs for devices.
hipError_t ExtractFatBinary(const std::vector<hip::Device*>& devices);
hipError_t AddDevProgram(const int device_id);
hipError_t BuildProgram(const int device_id);
// Device Id bounds check
inline void DeviceIdCheck(const int device_id) const {
guarantee(device_id >= 0, "Invalid DeviceId less than 0");
guarantee(static_cast<size_t>(device_id) < fatbin_dev_info_.size(), "Invalid DeviceId, greater than no of fatbin device info!");
}
// Getter Methods
amd::Program* GetProgram(int device_id) {
DeviceIdCheck(device_id);
return fatbin_dev_info_[device_id]->program_;
}
hipModule_t Module(int device_id) const {
DeviceIdCheck(device_id);
return reinterpret_cast<hipModule_t>(as_cl(fatbin_dev_info_[device_id]->program_));
}
hipError_t GetModule(int device_id, hipModule_t* hmod) const {
DeviceIdCheck(device_id);
*hmod = reinterpret_cast<hipModule_t>(as_cl(fatbin_dev_info_[device_id]->program_));
return hipSuccess;
}
private:
std::string fname_; // File name
amd::Os::FileDesc fdesc_; // File descriptor
size_t fsize_; // Total file size
// Even when file is passed image will be mmapped till ~desctructor.
const void* image_; // Image
// Only used for FBs where image is directly passed
std::string uri_; // Uniform resource indicator
// Per Device Info, like corresponding binary ptr, size.
std::vector<FatBinaryDeviceInfo*> fatbin_dev_info_;
};
}; /* namespace hip */
#endif /* HIP_FAT_BINARY_HPP */
projects/hip/src/hipamd/src/hip_formatting.hpp
+853
Fájl megtekintése
@@ -0,0 +1,853 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hiprtc.h>
#include <hip/hip_runtime_api.h>
inline std::ostream& operator<<(std::ostream& os, const hipTextureFilterMode& s) {
switch (s) {
case hipFilterModePoint:
os << "hipFilterModePoint";
break;
case hipFilterModeLinear:
os << "hipFilterModeLinear";
break;
default:
os << "hipFilterModePoint";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipTextureReadMode& s) {
switch (s) {
case hipReadModeElementType:
os << "hipReadModeElementType";
break;
case hipReadModeNormalizedFloat:
os << "hipReadModeNormalizedFloat";
break;
default:
os << "hipReadModeElementType";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipTextureAddressMode& s) {
switch (s) {
case hipAddressModeWrap:
os << "hipAddressModeWrap";
break;
case hipAddressModeClamp:
os << "hipAddressModeClamp";
break;
case hipAddressModeMirror:
os << "hipAddressModeMirror";
break;
case hipAddressModeBorder:
os << "hipAddressModeBorder";
break;
default:
os << "hipAddressModeWrap";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipMemcpyKind& s) {
switch (s) {
case hipMemcpyHostToHost:
os << "hipMemcpyHostToHost";
break;
case hipMemcpyHostToDevice:
os << "hipMemcpyHostToDevice";
break;
case hipMemcpyDeviceToHost:
os << "hipMemcpyDeviceToHost";
break;
case hipMemcpyDeviceToDevice:
os << "hipMemcpyDeviceToDevice";
break;
case hipMemcpyDefault:
os << "hipMemcpyDefault";
break;
default:
os << "hipMemcpyDefault";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatKind& s) {
switch (s) {
case hipChannelFormatKindSigned:
os << "hipChannelFormatKindSigned";
break;
case hipChannelFormatKindUnsigned:
os << "hipMemcpyHostToDevice";
break;
case hipChannelFormatKindFloat:
os << "hipChannelFormatKindFloat";
break;
case hipChannelFormatKindNone:
os << "hipChannelFormatKindNone";
break;
default:
os << "hipChannelFormatKindNone";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipArray_Format& s) {
switch (s) {
case HIP_AD_FORMAT_UNSIGNED_INT8:
os << "HIP_AD_FORMAT_UNSIGNED_INT8";
break;
case HIP_AD_FORMAT_UNSIGNED_INT16:
os << "HIP_AD_FORMAT_UNSIGNED_INT16";
break;
case HIP_AD_FORMAT_UNSIGNED_INT32:
os << "HIP_AD_FORMAT_UNSIGNED_INT32";
break;
case HIP_AD_FORMAT_SIGNED_INT8:
os << "HIP_AD_FORMAT_SIGNED_INT8";
break;
case HIP_AD_FORMAT_SIGNED_INT16:
os << "HIP_AD_FORMAT_SIGNED_INT16";
break;
case HIP_AD_FORMAT_SIGNED_INT32:
os << "HIP_AD_FORMAT_SIGNED_INT32";
break;
case HIP_AD_FORMAT_HALF:
os << "HIP_AD_FORMAT_HALF";
break;
case HIP_AD_FORMAT_FLOAT:
os << "HIP_AD_FORMAT_FLOAT";
break;
default:
os << "HIP_AD_FORMAT_FLOAT";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipResourceViewFormat& s) {
switch (s) {
case hipResViewFormatNone:
os << "hipResViewFormatNone";
break;
case hipResViewFormatUnsignedChar1:
os << "hipResViewFormatUnsignedChar1";
break;
case hipResViewFormatUnsignedChar2:
os << "hipResViewFormatUnsignedChar2";
break;
case hipResViewFormatUnsignedChar4:
os << "hipResViewFormatUnsignedChar4";
break;
case hipResViewFormatSignedChar1:
os << "hipResViewFormatSignedChar1";
break;
case hipResViewFormatSignedChar2:
os << "hipResViewFormatSignedChar2";
break;
case hipResViewFormatSignedChar4:
os << "hipResViewFormatSignedChar4";
break;
case hipResViewFormatUnsignedShort1:
os << "hipResViewFormatUnsignedShort1";
break;
case hipResViewFormatUnsignedShort2:
os << "hipResViewFormatUnsignedShort2";
break;
case hipResViewFormatUnsignedShort4:
os << "hipResViewFormatUnsignedShort4";
break;
case hipResViewFormatSignedShort1:
os << "hipResViewFormatSignedShort1";
break;
case hipResViewFormatSignedShort2:
os << "hipResViewFormatSignedShort2";
break;
case hipResViewFormatSignedShort4:
os << "hipResViewFormatSignedShort4";
break;
case hipResViewFormatUnsignedInt1:
os << "hipResViewFormatUnsignedInt1";
break;
case hipResViewFormatUnsignedInt2:
os << "hipResViewFormatUnsignedInt2";
break;
case hipResViewFormatUnsignedInt4:
os << "hipResViewFormatUnsignedInt4";
break;
case hipResViewFormatSignedInt1:
os << "hipResViewFormatSignedInt1";
break;
case hipResViewFormatSignedInt2:
os << "hipResViewFormatSignedInt2";
break;
case hipResViewFormatSignedInt4:
os << "hipResViewFormatSignedInt4";
break;
case hipResViewFormatHalf1:
os << "hipResViewFormatHalf1";
break;
case hipResViewFormatHalf2:
os << "hipResViewFormatHalf2";
break;
case hipResViewFormatHalf4:
os << "hipResViewFormatHalf4";
break;
case hipResViewFormatFloat1:
os << "hipResViewFormatFloat1";
break;
case hipResViewFormatFloat2:
os << "hipResViewFormatFloat2";
break;
case hipResViewFormatFloat4:
os << "hipResViewFormatFloat4";
break;
case hipResViewFormatUnsignedBlockCompressed1:
os << "hipResViewFormatUnsignedBlockCompressed1";
break;
case hipResViewFormatUnsignedBlockCompressed2:
os << "hipResViewFormatUnsignedBlockCompressed2";
break;
case hipResViewFormatUnsignedBlockCompressed3:
os << "hipResViewFormatUnsignedBlockCompressed3";
break;
case hipResViewFormatUnsignedBlockCompressed4:
os << "hipResViewFormatUnsignedBlockCompressed4";
break;
case hipResViewFormatSignedBlockCompressed4:
os << "hipResViewFormatSignedBlockCompressed4";
break;
case hipResViewFormatUnsignedBlockCompressed5:
os << "hipResViewFormatUnsignedBlockCompressed5";
break;
case hipResViewFormatSignedBlockCompressed5:
os << "hipResViewFormatSignedBlockCompressed5";
break;
case hipResViewFormatUnsignedBlockCompressed6H:
os << "hipResViewFormatUnsignedBlockCompressed6H";
break;
case hipResViewFormatSignedBlockCompressed6H:
os << "hipResViewFormatSignedBlockCompressed6H";
break;
case hipResViewFormatUnsignedBlockCompressed7:
os << "hipResViewFormatUnsignedBlockCompressed7";
break;
default:
os << "hipResViewFormatNone";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipFunction_attribute& s) {
switch (s) {
case HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK:
os << "HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK";
break;
case HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES:
os << "HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES";
break;
case HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES:
os << "HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES";
break;
case HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES:
os << "HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES";
break;
case HIP_FUNC_ATTRIBUTE_NUM_REGS:
os << "HIP_FUNC_ATTRIBUTE_NUM_REGS";
break;
case HIP_FUNC_ATTRIBUTE_PTX_VERSION:
os << "HIP_FUNC_ATTRIBUTE_PTX_VERSION";
break;
case HIP_FUNC_ATTRIBUTE_BINARY_VERSION:
os << "HIP_FUNC_ATTRIBUTE_BINARY_VERSION";
break;
case HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA:
os << "HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA";
break;
case HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES:
os << "HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES";
break;
case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT:
os << "HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT";
break;
case HIP_FUNC_ATTRIBUTE_MAX:
os << "HIP_FUNC_ATTRIBUTE_MAX";
break;
default:
os << "HIP_FUNC_ATTRIBUTE_MAX";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hiprtcResult& s) {
switch (s) {
case HIPRTC_SUCCESS:
os << "HIPRTC_SUCCESS";
break;
case HIPRTC_ERROR_OUT_OF_MEMORY:
os << "HIPRTC_ERROR_OUT_OF_MEMORY";
break;
case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE:
os << "HIPRTC_ERROR_PROGRAM_CREATION_FAILURE";
break;
case HIPRTC_ERROR_INVALID_INPUT:
os << "HIPRTC_ERROR_INVALID_INPUT";
break;
case HIPRTC_ERROR_INVALID_PROGRAM:
os << "HIPRTC_ERROR_INVALID_PROGRAM";
break;
case HIPRTC_ERROR_INVALID_OPTION:
os << "HIPRTC_ERROR_INVALID_OPTION";
break;
case HIPRTC_ERROR_COMPILATION:
os << "HIPRTC_ERROR_COMPILATION";
break;
case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE:
os << "HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE";
break;
case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION:
os << "HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION";
break;
case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION:
os << "IPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION";
break;
case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID:
os << "HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID";
break;
case HIPRTC_ERROR_INTERNAL_ERROR:
os << "HIPRTC_ERROR_INTERNAL_ERROR";
break;
default:
os << "HIPRTC_ERROR_INTERNAL_ERROR";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipJitOption& s) {
switch (s) {
case hipJitOptionMaxRegisters:
os << "hipJitOptionMaxRegisters";
break;
case hipJitOptionThreadsPerBlock:
os << "hipJitOptionThreadsPerBlock";
break;
case hipJitOptionWallTime:
os << "hipJitOptionWallTime";
break;
case hipJitOptionInfoLogBuffer:
os << "hipJitOptionInfoLogBuffer";
break;
case hipJitOptionInfoLogBufferSizeBytes:
os << "hipJitOptionInfoLogBufferSizeBytes";
break;
case hipJitOptionErrorLogBuffer:
os << "hipJitOptionErrorLogBuffer";
break;
case hipJitOptionErrorLogBufferSizeBytes:
os << "hipJitOptionErrorLogBufferSizeBytes";
break;
case hipJitOptionOptimizationLevel:
os << "hipJitOptionOptimizationLevel";
break;
case hipJitOptionTargetFromContext:
os << "hipJitOptionTargetFromContext";
break;
case hipJitOptionTarget:
os << "hipJitOptionTarget";
break;
case hipJitOptionFallbackStrategy:
os << "hipJitOptionFallbackStrategy";
break;
case hipJitOptionGenerateDebugInfo:
os << "hipJitOptionGenerateDebugInfo";
break;
case hipJitOptionCacheMode:
os << "hipJitOptionCacheMode";
break;
case hipJitOptionSm3xOpt:
os << "hipJitOptionSm3xOpt";
break;
case hipJitOptionFastCompile:
os << "hipJitOptionFastCompile";
break;
case hipJitOptionNumOptions:
os << "hipJitOptionNumOptions";
break;
default:
os << "hipJitOptionMaxRegisters";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipFuncCache_t& s) {
switch (s) {
case hipFuncCachePreferNone:
os << "hipFuncCachePreferNone";
break;
case hipFuncCachePreferShared:
os << "hipFuncCachePreferShared";
break;
case hipFuncCachePreferL1:
os << "hipFuncCachePreferL1";
break;
case hipFuncCachePreferEqual:
os << "hipFuncCachePreferEqual";
break;
default:
os << "hipFuncCachePreferNone";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipSharedMemConfig& s) {
switch (s) {
case hipSharedMemBankSizeDefault:
os << "hipSharedMemBankSizeDefault";
break;
case hipSharedMemBankSizeFourByte:
os << "hipSharedMemBankSizeFourByte";
break;
case hipSharedMemBankSizeEightByte:
os << "hipSharedMemBankSizeEightByte";
break;
default:
os << "hipSharedMemBankSizeDefault";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipDataType& s) {
switch (s) {
case HIP_R_16F:
os << "HIP_R_16F";
break;
case HIP_R_32F:
os << "HIP_R_32F";
break;
case HIP_R_64F:
os << "HIP_R_64F";
break;
case HIP_C_16F:
os << "HIP_C_16F";
break;
case HIP_C_32F:
os << "HIP_C_32F";
break;
case HIP_C_64F:
os << "HIP_C_64F";
break;
default:
os << "HIP_R_16F";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipLibraryPropertyType& s) {
switch (s) {
case HIP_LIBRARY_MAJOR_VERSION:
os << "HIP_LIBRARY_MAJOR_VERSION";
break;
case HIP_LIBRARY_MINOR_VERSION:
os << "HIP_LIBRARY_MINOR_VERSION";
break;
case HIP_LIBRARY_PATCH_LEVEL:
os << "HIP_LIBRARY_PATCH_LEVEL";
break;
default:
os << "HIP_LIBRARY_MAJOR_VERSION";
};
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t& s) {
os << hip_api_name(s);
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hip_api_id_t* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc& s) {
os << '{'
<< '{'
<< s.addressMode[0]
<< ','
<< s.addressMode[1]
<< ','
<< s.addressMode[2]
<< '}'
<< ','
<< s.filterMode
<< ','
<< s.readMode
<< ','
<< s.sRGB
<< ','
<< '{'
<< s.borderColor[0]
<< ','
<< s.borderColor[1]
<< ','
<< s.borderColor[2]
<< ','
<< s.borderColor[3]
<< '}'
<< ','
<< s.normalizedCoords
<< ','
<< s.mipmapFilterMode
<< ','
<< s.mipmapLevelBias
<< ','
<< s.minMipmapLevelClamp
<< ','
<< s.maxMipmapLevelClamp
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipTextureDesc* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const dim3& s) {
os << '{'
<< s.x
<< ','
<< s.y
<< ','
<< s.z
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const dim3* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc& s) {
os << '{'
<< s.x
<< ','
<< s.y
<< ','
<< s.z
<< ','
<< s.w
<< ','
<< s.f
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipChannelFormatDesc* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray& s) {
os << '{'
<< s.data
<< ','
<< s.desc
<< ','
<< s.width
<< ','
<< s.height
<< ','
<< s.depth
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipMipmappedArray* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc& s) {
os << '{'
<< s.resType
<< ','
<< '{';
switch (s.resType) {
case hipResourceTypeLinear:
os << s.res.linear.devPtr
<< ','
<< s.res.linear.desc
<< ','
<< s.res.linear.sizeInBytes;
break;
case hipResourceTypePitch2D:
os << s.res.pitch2D.devPtr
<< ','
<< s.res.pitch2D.desc
<< ','
<< s.res.pitch2D.width
<< ','
<< s.res.pitch2D.height
<< ','
<< s.res.pitch2D.pitchInBytes;
break;
case hipResourceTypeArray:
os << s.res.array.array;
break;
case hipResourceTypeMipmappedArray:
os <<s.res.mipmap.mipmap;
break;
default:
break;
}
os << '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipResourceDesc* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipArray& s) {
os << '{'
<< s.data
<< ','
<< s.desc
<< ','
<< s.type
<< ','
<< s.width
<< ','
<< s.height
<< ','
<< s.depth
<< ','
<< s.Format
<< ','
<< s.NumChannels
<< ','
<< s.isDrv
<< ','
<< s.textureType
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipArray* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const textureReference& s) {
os << '{'
<< s.normalized
<< ','
<< s.readMode
<< ','
<< s.filterMode
<< ','
<< '{'
<< s.addressMode[0]
<< ','
<< s.addressMode[1]
<< ','
<< s.addressMode[2]
<< '}'
<< ','
<< s.channelDesc
<< ','
<< s.sRGB
<< ','
<< s.maxAnisotropy
<< ','
<< s.mipmapFilterMode
<< ','
<< s.mipmapLevelBias
<< ','
<< s.minMipmapLevelClamp
<< ','
<< s.maxMipmapLevelClamp
<< ','
<< s.textureObject
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const textureReference* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipError_t& s) {
os << hipGetErrorName(s);
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipError_t* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipResourceViewDesc& s) {
os << '{'
<< s.format
<< ','
<< s.width
<< ','
<< s.height
<< ','
<< s.depth
<< ','
<< s.firstMipmapLevel
<< ','
<< s.lastMipmapLevel
<< ','
<< s.firstLayer
<< ','
<< s.lastLayer
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipResourceViewDesc* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY_DESCRIPTOR& s) {
os << '{'
<< s.Width
<< ','
<< s.Height
<< ','
<< s.Format
<< ','
<< s.NumChannels
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY_DESCRIPTOR* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY3D_DESCRIPTOR& s) {
os << '{'
<< s.Width
<< ','
<< s.Height
<< ','
<< s.Depth
<< ','
<< s.Format
<< ','
<< s.NumChannels
<< ','
<< s.Flags
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const HIP_ARRAY3D_DESCRIPTOR* s) {
if (s) {
os << *s;
} else {
os << "nullptr";
}
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipExtent& s) {
os << '{'
<< s.width
<< ','
<< s.height
<< ','
<< s.depth
<< '}';
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipIpcEventHandle_t& s) {
//TODO fill in later
return os;
}
inline std::ostream& operator<<(std::ostream& os, const hipIpcEventHandle_t* s) {
//TODO fill in later
return os;
}
projects/hip/src/hipamd/src/hip_global.cpp
Executable
+196
Fájl megtekintése
@@ -0,0 +1,196 @@
#include "hip_global.hpp"
#include "hip/hip_runtime.h"
#include "hip_internal.hpp"
#include "hip_code_object.hpp"
#include "platform/program.hpp"
namespace hip {
//Device Vars
DeviceVar::DeviceVar(std::string name, hipModule_t hmod) : shadowVptr(nullptr), name_(name),
amd_mem_obj_(nullptr), device_ptr_(nullptr),
size_(0) {
amd::Program* program = as_amd(reinterpret_cast<cl_program>(hmod));
device::Program* dev_program = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
if (dev_program == nullptr) {
LogPrintfError("Cannot get Device Program for module: 0x%x \n", hmod);
guarantee(false, "Cannot get Device Program");
}
if(!dev_program->createGlobalVarObj(&amd_mem_obj_, &device_ptr_, &size_, name.c_str())) {
LogPrintfError("Cannot create Global Var obj for symbol: %s \n", name.c_str());
guarantee(false, "Cannot create GlobalVar Obj");
}
// Handle size 0 symbols
if (size_ != 0) {
if (amd_mem_obj_ == nullptr || device_ptr_ == nullptr) {
LogPrintfError("Cannot get memory for creating device Var: %s", name.c_str());
guarantee(false, "Cannot get memory for creating device var");
}
amd::MemObjMap::AddMemObj(device_ptr_, amd_mem_obj_);
}
}
DeviceVar::~DeviceVar() {
if (amd_mem_obj_ != nullptr) {
amd::MemObjMap::RemoveMemObj(device_ptr_);
amd_mem_obj_->release();
}
if (shadowVptr != nullptr) {
textureReference* texRef = reinterpret_cast<textureReference*>(shadowVptr);
delete texRef;
shadowVptr = nullptr;
}
device_ptr_ = nullptr;
size_ = 0;
}
//Device Functions
DeviceFunc::DeviceFunc(std::string name, hipModule_t hmod) : dflock_("function lock"),
name_(name), kernel_(nullptr) {
amd::Program* program = as_amd(reinterpret_cast<cl_program>(hmod));
const amd::Symbol *symbol = program->findSymbol(name.c_str());
if (symbol == nullptr) {
LogPrintfError("Cannot find Symbol with name: %s \n", name.c_str());
guarantee(false, "Cannot find Symbol");
}
kernel_ = new amd::Kernel(*program, *symbol, name);
if (kernel_ == nullptr) {
LogPrintfError("Cannot create kernel with name: %s \n", name.c_str());
guarantee(false, "Cannot Create kernel");
}
}
DeviceFunc::~DeviceFunc() {
if (kernel_ != nullptr) {
kernel_->release();
}
}
//Abstract functions
Function::Function(std::string name, FatBinaryInfo** modules)
: name_(name), modules_(modules) {
dFunc_.resize(g_devices.size());
}
Function::~Function() {
for (auto& elem : dFunc_) {
delete elem;
}
name_ = "";
modules_ = nullptr;
}
hipError_t Function::getDynFunc(hipFunction_t* hfunc, hipModule_t hmod) {
guarantee((dFunc_.size() == g_devices.size()), "dFunc Size mismatch");
if (dFunc_[ihipGetDevice()] == nullptr) {
dFunc_[ihipGetDevice()] = new DeviceFunc(name_, hmod);
}
*hfunc = dFunc_[ihipGetDevice()]->asHipFunction();
return hipSuccess;
}
hipError_t Function::getStatFunc(hipFunction_t* hfunc, int deviceId) {
guarantee(modules_ != nullptr, "Module not initialized");
hipModule_t hmod = nullptr;
IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId));
IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod));
if (dFunc_[deviceId] == nullptr) {
dFunc_[deviceId] = new DeviceFunc(name_, hmod);
}
*hfunc = dFunc_[deviceId]->asHipFunction();
return hipSuccess;
}
hipError_t Function::getStatFuncAttr(hipFuncAttributes* func_attr, int deviceId) {
guarantee((modules_ != nullptr), "Module not initialized");
hipModule_t hmod = nullptr;
IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId));
IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod));
if (dFunc_[deviceId] == nullptr) {
dFunc_[deviceId] = new DeviceFunc(name_, hmod);
}
const std::vector<amd::Device*>& devices = amd::Device::getDevices(CL_DEVICE_TYPE_GPU, false);
amd::Kernel* kernel = dFunc_[deviceId]->kernel();
const device::Kernel::WorkGroupInfo* wginfo = kernel->getDeviceKernel(*devices[deviceId])->workGroupInfo();
func_attr->sharedSizeBytes = static_cast<int>(wginfo->localMemSize_);
func_attr->binaryVersion = static_cast<int>(kernel->signature().version());
func_attr->cacheModeCA = 0;
func_attr->constSizeBytes = 0;
func_attr->localSizeBytes = wginfo->privateMemSize_;
func_attr->maxDynamicSharedSizeBytes = static_cast<int>(wginfo->availableLDSSize_
- wginfo->localMemSize_);
func_attr->maxThreadsPerBlock = static_cast<int>(wginfo->size_);
func_attr->numRegs = static_cast<int>(wginfo->usedVGPRs_);
func_attr->preferredShmemCarveout = 0;
func_attr->ptxVersion = 30;
return hipSuccess;
}
//Abstract Vars
Var::Var(std::string name, DeviceVarKind dVarKind, size_t size, int type, int norm,
FatBinaryInfo** modules) : name_(name), dVarKind_(dVarKind), size_(size),
type_(type), norm_(norm), modules_(modules) {
dVar_.resize(g_devices.size());
}
Var::Var(std::string name, DeviceVarKind dVarKind, void *pointer, size_t size,
unsigned align, FatBinaryInfo** modules) : name_(name), dVarKind_(dVarKind),
size_(size), modules_(modules), managedVarPtr_(pointer), align_(align) {
dVar_.resize(g_devices.size());
}
Var::~Var() {
for (auto& elem : dVar_) {
delete elem;
}
modules_ = nullptr;
}
hipError_t Var::getDeviceVar(DeviceVar** dvar, int deviceId, hipModule_t hmod) {
guarantee((deviceId >= 0), "Invalid DeviceId, less than zero");
guarantee((static_cast<size_t>(deviceId) < g_devices.size()),
"Invalid DeviceId, greater than no of code objects");
guarantee((dVar_.size() == g_devices.size()),
"Device Var not initialized to size");
if (dVar_[deviceId] == nullptr) {
dVar_[deviceId] = new DeviceVar(name_, hmod);
}
*dvar = dVar_[deviceId];
return hipSuccess;
}
hipError_t Var::getStatDeviceVar(DeviceVar** dvar, int deviceId) {
guarantee((deviceId >= 0) , "Invalid DeviceId, less than zero");
guarantee((static_cast<size_t>(deviceId) < g_devices.size()),
"Invalid DeviceId, greater than no of code objects");
if (dVar_[deviceId] == nullptr) {
hipModule_t hmod = nullptr;
IHIP_RETURN_ONFAIL((*modules_)->BuildProgram(deviceId));
IHIP_RETURN_ONFAIL((*modules_)->GetModule(deviceId, &hmod));
dVar_[deviceId] = new DeviceVar(name_, hmod);
}
*dvar = dVar_[deviceId];
return hipSuccess;
}
}; //namespace: hip
projects/hip/src/hipamd/src/hip_global.hpp
Executable
+127
Fájl megtekintése
@@ -0,0 +1,127 @@
#ifndef HIP_GLOBAL_HPP
#define HIP_GLOBAL_HPP
#include <vector>
#include <string>
#include "hip/hip_runtime_api.h"
#include "hip/hip_runtime.h"
#include "hip_internal.hpp"
#include "hip_fatbin.hpp"
#include "platform/program.hpp"
namespace hip {
//Forward Declaration
class CodeObject;
//Device Structures
class DeviceVar {
public:
DeviceVar(std::string name, hipModule_t hmod);
~DeviceVar();
//Accessors for device ptr and size, populated during constructor.
hipDeviceptr_t device_ptr() const { return device_ptr_; }
size_t size() const { return size_; }
std::string name() const { return name_; }
void* shadowVptr;
private:
std::string name_; //Name of the var
amd::Memory* amd_mem_obj_; //amd_mem_obj abstraction
hipDeviceptr_t device_ptr_; //Device Pointer
size_t size_; //Size of the var
};
class DeviceFunc {
public:
DeviceFunc(std::string name, hipModule_t hmod);
~DeviceFunc();
amd::Monitor dflock_;
//Converts DeviceFunc to hipFunction_t(used by app) and vice versa.
hipFunction_t asHipFunction() { return reinterpret_cast<hipFunction_t>(this); }
static DeviceFunc* asFunction(hipFunction_t f) { return reinterpret_cast<DeviceFunc*>(f); }
//Accessor for kernel_ and name_ populated during constructor.
std::string name() const { return name_; }
amd::Kernel* kernel() const { return kernel_; }
private:
std::string name_; //name of the func(not unique identifier)
amd::Kernel* kernel_; //Kernel ptr referencing to ROCclr Symbol
};
//Abstract Structures
class Function {
public:
Function(std::string name, FatBinaryInfo** modules=nullptr);
~Function();
//Return DeviceFunc for this this dynamically loaded module
hipError_t getDynFunc(hipFunction_t* hfunc, hipModule_t hmod);
//Return Device Func & attr . Generate/build if not already done so.
hipError_t getStatFunc(hipFunction_t *hfunc, int deviceId);
hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, int deviceId);
void resize_dFunc(size_t size) { dFunc_.resize(size); }
FatBinaryInfo** moduleInfo() { return modules_; };
private:
std::vector<DeviceFunc*> dFunc_; //DeviceFuncObj per Device
std::string name_; //name of the func(not unique identifier)
FatBinaryInfo** modules_; // static module where it is referenced
};
class Var {
public:
//Types of variable
enum DeviceVarKind {
DVK_Variable = 0,
DVK_Surface,
DVK_Texture,
DVK_Managed
};
Var(std::string name, DeviceVarKind dVarKind, size_t size, int type, int norm,
FatBinaryInfo** modules = nullptr);
Var(std::string name, DeviceVarKind dVarKind, void *pointer, size_t size, unsigned align,
FatBinaryInfo** modules = nullptr);
~Var();
//Return DeviceVar for this dynamically loaded module
hipError_t getDeviceVar(DeviceVar** dvar, int deviceId, hipModule_t hmod);
//Return DeviceVar for module Generate/build if not already done so.
hipError_t getStatDeviceVar(DeviceVar** dvar, int deviceId);
void resize_dVar(size_t size) { dVar_.resize(size); }
FatBinaryInfo** moduleInfo() { return modules_; };
DeviceVarKind getVarKind() const { return dVarKind_; }
size_t getSize() const { return size_; }
void* getManagedVarPtr() { return managedVarPtr_; };
void setManagedVarInfo(void* pointer, size_t size) {
managedVarPtr_ = pointer;
size_ = size;
dVarKind_ = DVK_Managed;
}
private:
std::vector<DeviceVar*> dVar_; // DeviceVarObj per Device
std::string name_; // Variable name (not unique identifier)
DeviceVarKind dVarKind_; // Variable kind
size_t size_; // Size of the variable
int type_; // Type(Textures/Surfaces only)
int norm_; // Type(Textures/Surfaces only)
FatBinaryInfo** modules_; // static module where it is referenced
void *managedVarPtr_; // Managed memory pointer with size_ & align_
unsigned int align_; // Managed memory alignment
};
}; //namespace: hip
#endif /* HIP_GLOBAL_HPP */
projects/hip/src/hipamd/src/hip_graph.cpp
+415
Fájl megtekintése
@@ -0,0 +1,415 @@
/* Copyright (c) 2021-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "hip_graph_internal.hpp"
#include "platform/command.hpp"
#include "hip_conversions.hpp"
#include "hip_platform.hpp"
#include "hip_event.hpp"
thread_local std::vector<hipStream_t> g_captureStreams;
std::unordered_map<amd::Command*, hipGraphExec_t> hipGraphExec::activeGraphExec_;
hipError_t ihipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipKernelNodeParams* pNodeParams) {
if (pGraphNode == nullptr || graph == nullptr ||
(numDependencies > 0 && pDependencies == nullptr) || pNodeParams == nullptr) {
return hipErrorInvalidValue;
}
hipFunction_t func = nullptr;
hipError_t status =
PlatformState::instance().getStatFunc(&func, pNodeParams->func, ihipGetDevice());
if ((status != hipSuccess) || (func == nullptr)) {
return hipErrorInvalidDeviceFunction;
}
size_t globalWorkSizeX = static_cast<size_t>(pNodeParams->gridDim.x) * pNodeParams->blockDim.x;
size_t globalWorkSizeY = static_cast<size_t>(pNodeParams->gridDim.y) * pNodeParams->blockDim.y;
size_t globalWorkSizeZ = static_cast<size_t>(pNodeParams->gridDim.z) * pNodeParams->blockDim.z;
if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
return hipErrorInvalidConfiguration;
}
status = ihipLaunchKernel_validate(
func, static_cast<uint32_t>(globalWorkSizeX), static_cast<uint32_t>(globalWorkSizeY),
static_cast<uint32_t>(globalWorkSizeZ), pNodeParams->blockDim.x, pNodeParams->blockDim.y,
pNodeParams->blockDim.z, pNodeParams->sharedMemBytes, pNodeParams->kernelParams,
pNodeParams->extra, ihipGetDevice(), 0);
if (status != hipSuccess) {
return status;
}
*pGraphNode = new hipGraphKernelNode(pNodeParams, func);
if (numDependencies == 0) {
graph->AddNode(*pGraphNode);
}
for (size_t i = 0; i < numDependencies; i++) {
if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) {
return hipErrorInvalidValue;
}
}
return hipSuccess;
}
hipError_t ihipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipMemcpy3DParms* pCopyParams) {
if (pGraphNode == nullptr || graph == nullptr ||
(numDependencies > 0 && pDependencies == nullptr) || pCopyParams == nullptr) {
return hipErrorInvalidValue;
}
ihipMemcpy3D_validate(pCopyParams);
*pGraphNode = new hipGraphMemcpyNode(pCopyParams);
if (numDependencies == 0) {
graph->AddNode(*pGraphNode);
}
for (size_t i = 0; i < numDependencies; i++) {
if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) {
return hipErrorInvalidValue;
}
}
return hipSuccess;
}
hipError_t ihipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipMemsetParams* pMemsetParams) {
if (pGraphNode == nullptr || graph == nullptr ||
(numDependencies > 0 && pDependencies == nullptr) || pMemsetParams == nullptr) {
return hipErrorInvalidValue;
}
if (pMemsetParams->height == 1) {
ihipMemset_validate(pMemsetParams->dst, pMemsetParams->value, pMemsetParams->elementSize,
pMemsetParams->width * pMemsetParams->elementSize);
} else {
auto sizeBytes = pMemsetParams->width * pMemsetParams->height * 1;
ihipMemset3D_validate(
{pMemsetParams->dst, pMemsetParams->pitch, pMemsetParams->width, pMemsetParams->height},
pMemsetParams->value, {pMemsetParams->width, pMemsetParams->height, 1}, sizeBytes);
}
*pGraphNode = new hipGraphMemsetNode(pMemsetParams);
if (numDependencies == 0) {
graph->AddNode(*pGraphNode);
}
for (size_t i = 0; i < numDependencies; i++) {
if (graph->AddEdge(*(pDependencies + i), *pGraphNode) != hipSuccess) {
return hipErrorInvalidValue;
}
}
return hipSuccess;
}
hipError_t capturehipLaunchKernel(hipStream_t& stream, const void*& hostFunction, dim3& gridDim,
dim3& blockDim, void**& args, size_t& sharedMemBytes) {
ClPrint(amd::LOG_INFO, amd::LOG_API,
"[hipGraph] current capture node kernel launch on stream : %p", stream);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hipKernelNodeParams nodeParams;
nodeParams.func = const_cast<void*>(hostFunction);
nodeParams.blockDim = blockDim;
nodeParams.extra = nullptr;
nodeParams.gridDim = gridDim;
nodeParams.kernelParams = args;
nodeParams.sharedMemBytes = sharedMemBytes;
hipGraphNode_t pGraphNode;
hipError_t status =
ihipGraphAddKernelNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(),
s->GetLastCapturedNodes().size(), &nodeParams);
if (status != hipSuccess) {
return status;
}
s->SetLastCapturedNode(pGraphNode);
return hipSuccess;
}
hipError_t capturehipMemcpy3DAsync(hipStream_t& stream, const hipMemcpy3DParms*& p) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memcpy3D on stream : %p",
stream);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hipGraphNode_t pGraphNode;
hipError_t status =
ihipGraphAddMemcpyNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(),
s->GetLastCapturedNodes().size(), p);
if (status != hipSuccess) {
return status;
}
s->SetLastCapturedNode(pGraphNode);
return hipSuccess;
}
hipError_t capturehipMemcpyAsync(hipStream_t& stream, void*& dst, const void*& src,
size_t& sizeBytes, hipMemcpyKind& kind) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memcpy1D on stream : %p",
stream);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hipGraphNode_t pGraphNode;
hipGraph_t graph = nullptr;
std::vector<hipGraphNode_t> pDependencies = s->GetLastCapturedNodes();
size_t numDependencies = s->GetLastCapturedNodes().size();
graph = s->GetCaptureGraph();
ihipMemcpy_validate(dst, src, sizeBytes, kind);
pGraphNode = new hipGraphMemcpyNode1D(dst, src, sizeBytes, kind);
if (numDependencies == 0) {
graph->AddNode(pGraphNode);
}
for (size_t i = 0; i < numDependencies; i++) {
if (graph->AddEdge(pDependencies[i], pGraphNode) != hipSuccess) {
return hipErrorInvalidValue;
}
}
s->SetLastCapturedNode(pGraphNode);
return hipSuccess;
}
hipError_t capturehipMemcpyFromSymbolAsync(hipStream_t& stream, void*& dst, const void*& symbol,
size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind) {
ClPrint(amd::LOG_INFO, amd::LOG_API,
"[hipGraph] current capture node MemcpyFromSymbolNode on stream : %p", stream);
return hipSuccess;
}
hipError_t capturehipMemcpyToSymbolAsync(hipStream_t& stream, const void*& symbol, const void*& src,
size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind) {
ClPrint(amd::LOG_INFO, amd::LOG_API,
"[hipGraph] current capture node MemcpyToSymbolNode on stream : %p", stream);
return hipSuccess;
}
hipError_t capturehipMemsetAsync(hipStream_t& stream, void*& dst, int& value, size_t& valueSize,
size_t& sizeBytes) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset1D on stream : %p",
stream);
hipMemsetParams memsetParams = {0};
memsetParams.dst = dst;
memsetParams.value = value;
memsetParams.elementSize = valueSize;
memsetParams.width = sizeBytes / valueSize;
memsetParams.height = 1;
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hipGraphNode_t pGraphNode;
hipError_t status =
ihipGraphAddMemsetNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(),
s->GetLastCapturedNodes().size(), &memsetParams);
if (status != hipSuccess) {
return status;
}
s->SetLastCapturedNode(pGraphNode);
return hipSuccess;
}
hipError_t capturehipMemset2DAsync(hipStream_t& stream, void*& dst, size_t& pitch, int& value,
size_t& width, size_t& height) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset2D on stream : %p",
stream);
hipMemsetParams memsetParams = {0};
memsetParams.dst = dst;
memsetParams.value = value;
memsetParams.width = width;
memsetParams.height = height;
memsetParams.pitch = pitch;
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hipGraphNode_t pGraphNode;
hipError_t status =
ihipGraphAddMemsetNode(&pGraphNode, s->GetCaptureGraph(), s->GetLastCapturedNodes().data(),
s->GetLastCapturedNodes().size(), &memsetParams);
if (status != hipSuccess) {
return status;
}
s->SetLastCapturedNode(pGraphNode);
return hipSuccess;
}
hipError_t capturehipMemset3DAsync(hipStream_t& stream, hipPitchedPtr& pitchedDevPtr, int& value,
hipExtent& extent) {
ClPrint(amd::LOG_INFO, amd::LOG_API, "[hipGraph] current capture node Memset3D on stream : %p",
stream);
return hipSuccess;
}
hipError_t capturehipEventRecord(hipStream_t& stream, hipEvent_t& event) {
ClPrint(amd::LOG_INFO, amd::LOG_API,
"[hipGraph] current capture node EventRecord on stream : %p, Event %p", stream, event);
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
hip::Event* e = reinterpret_cast<hip::Event*>(event);
e->StartCapture(stream);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
std::vector<hipGraphNode_t> lastCapturedNodes = s->GetLastCapturedNodes();
if (!lastCapturedNodes.empty()) {
e->SetNodesPrevToRecorded(lastCapturedNodes);
}
return hipSuccess;
}
hipError_t capturehipStreamWaitEvent(hipEvent_t& event, hipStream_t& stream, unsigned int& flags) {
ClPrint(amd::LOG_INFO, amd::LOG_API,
"[hipGraph] current capture node StreamWaitEvent on stream : %p, Event %p", stream,
event);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if (event == nullptr || stream == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if (!s->IsOriginStream()) {
s->SetCaptureGraph(reinterpret_cast<hip::Stream*>(e->GetCaptureStream())->GetCaptureGraph());
s->SetCaptureMode(reinterpret_cast<hip::Stream*>(e->GetCaptureStream())->GetCaptureMode());
s->SetParentStream(e->GetCaptureStream());
}
s->AddCrossCapturedNode(e->GetNodesPrevToRecorded());
g_captureStreams.push_back(stream);
return hipSuccess;
}
hipError_t hipStreamIsCapturing(hipStream_t stream, hipStreamCaptureStatus** pCaptureStatus) {
HIP_INIT_API(hipStreamIsCapturing, stream, pCaptureStatus);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hipStreamCaptureStatus captureStatus = reinterpret_cast<hip::Stream*>(stream)->GetCaptureStatus();
*pCaptureStatus = &captureStatus;
HIP_RETURN(hipSuccess);
}
hipError_t hipStreamBeginCapture(hipStream_t stream, hipStreamCaptureMode mode) {
HIP_INIT_API(hipStreamBeginCapture, stream, mode);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
// capture cannot be initiated on legacy stream
// It can be initiated if the stream is not already in capture mode
if (stream == nullptr || s->GetCaptureStatus() == hipStreamCaptureStatusActive) {
HIP_RETURN(hipErrorInvalidValue);
}
s->SetCaptureGraph(new hipGraph());
s->SetCaptureMode(mode);
s->SetOriginStream();
g_captureStreams.push_back(stream);
HIP_RETURN_DURATION(hipSuccess);
}
hipError_t hipStreamEndCapture(hipStream_t stream, hipGraph_t* pGraph) {
HIP_INIT_API(hipStreamEndCapture, stream, pGraph);
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
// Capture must be ended on the same stream in which it was initiated
if (!s->IsOriginStream()) {
HIP_RETURN(hipErrorStreamCaptureUnmatched);
}
// If mode is not hipStreamCaptureModeRelaxed, hipStreamEndCapture must be called on the stream
// from the same thread
if (s->GetCaptureMode() != hipStreamCaptureModeRelaxed &&
std::find(g_captureStreams.begin(), g_captureStreams.end(), stream) ==
g_captureStreams.end()) {
HIP_RETURN(hipErrorStreamCaptureWrongThread);
}
// If capture was invalidated, due to a violation of the rules of stream capture
if (s->GetCaptureStatus() == hipStreamCaptureStatusInvalidated) {
*pGraph = nullptr;
HIP_RETURN(hipErrorStreamCaptureInvalidated);
}
// check if all parallel streams have joined
if (s->GetCaptureGraph()->GetLeafNodeCount() != 1) {
return hipErrorStreamCaptureUnjoined;
}
*pGraph = s->GetCaptureGraph();
// end capture on all streams/events part of graph capture
HIP_RETURN_DURATION(s->EndCapture());
}
hipError_t hipGraphCreate(hipGraph_t* pGraph, unsigned int flags) {
HIP_INIT_API(hipGraphCreate, pGraph, flags);
*pGraph = new hipGraph();
HIP_RETURN(hipSuccess);
}
hipError_t hipGraphDestroy(hipGraph_t graph) {
HIP_INIT_API(hipGraphDestroy, graph);
delete graph;
HIP_RETURN(hipSuccess);
}
hipError_t hipGraphAddKernelNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipKernelNodeParams* pNodeParams) {
HIP_INIT_API(hipGraphAddKernelNode, pGraphNode, graph, pDependencies, numDependencies,
pNodeParams);
HIP_RETURN_DURATION(
ihipGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams););
}
hipError_t hipGraphAddMemcpyNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipMemcpy3DParms* pCopyParams) {
HIP_INIT_API(hipGraphAddMemcpyNode, pGraphNode, graph, pDependencies, numDependencies,
pCopyParams);
HIP_RETURN_DURATION(
ihipGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams););
}
hipError_t hipGraphAddMemsetNode(hipGraphNode_t* pGraphNode, hipGraph_t graph,
const hipGraphNode_t* pDependencies, size_t numDependencies,
const hipMemsetParams* pMemsetParams) {
HIP_INIT_API(hipGraphAddMemsetNode, pGraphNode, graph, pDependencies, numDependencies,
pMemsetParams);
HIP_RETURN_DURATION(
ihipGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams););
}
hipError_t ihipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph,
hipGraphNode_t* pErrorNode, char* pLogBuffer, size_t bufferSize) {
std::vector<std::vector<Node>> parallelLists;
std::unordered_map<Node, std::vector<Node>> nodeWaitLists;
graph->GetRunList(parallelLists, nodeWaitLists);
std::vector<Node> levelOrder;
graph->LevelOrder(levelOrder);
*pGraphExec = new hipGraphExec(levelOrder, parallelLists, nodeWaitLists);
if (*pGraphExec != nullptr) {
return (*pGraphExec)->Init();
} else {
return hipErrorOutOfMemory;
}
}
hipError_t hipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph,
hipGraphNode_t* pErrorNode, char* pLogBuffer, size_t bufferSize) {
HIP_INIT_API(hipGraphInstantiate, pGraphExec, graph);
HIP_RETURN_DURATION(ihipGraphInstantiate(pGraphExec, graph, pErrorNode, pLogBuffer, bufferSize));
}
hipError_t hipGraphExecDestroy(hipGraphExec_t pGraphExec) {
HIP_INIT_API(hipGraphExecDestroy, pGraphExec);
delete pGraphExec;
HIP_RETURN(hipSuccess);
}
hipError_t ihipGraphlaunch(hipGraphExec_t graphExec, hipStream_t stream) {
return graphExec->Run(stream);
}
hipError_t hipGraphLaunch(hipGraphExec_t graphExec, hipStream_t stream) {
HIP_INIT_API(hipGraphLaunch, graphExec, stream);
HIP_RETURN_DURATION(ihipGraphlaunch(graphExec, stream));
}
projects/hip/src/hipamd/src/hip_graph_capture.hpp
+48
Fájl megtekintése
@@ -0,0 +1,48 @@
/* Copyright (c) 2021-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#pragma once
// forward declaration of capture methods
hipError_t capturehipLaunchKernel(hipStream_t& stream, const void*& hostFunction, dim3& gridDim,
dim3& blockDim, void**& args, size_t& sharedMemBytes);
hipError_t capturehipMemcpy3DAsync(hipStream_t& stream, const hipMemcpy3DParms*& p);
hipError_t capturehipMemcpyAsync(hipStream_t& stream, void*& dst, const void*& src,
size_t& sizeBytes, hipMemcpyKind& kind);
hipError_t capturehipMemcpyFromSymbolAsync(hipStream_t& stream, void*& dst, const void*& symbol,
size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind);
hipError_t capturehipMemcpyToSymbolAsync(hipStream_t& stream, const void*& symbol, const void*& src,
size_t& sizeBytes, size_t& offset, hipMemcpyKind& kind);
hipError_t capturehipMemsetAsync(hipStream_t& stream, void*& dst, int& value, size_t& valueSize,
size_t& sizeBytes);
hipError_t capturehipMemset2DAsync(hipStream_t& stream, void*& dst, size_t& pitch, int& value,
size_t& width, size_t& height);
hipError_t capturehipMemset3DAsync(hipStream_t& stream, hipPitchedPtr& pitchedDevPtr, int& value,
hipExtent& extent);
hipError_t capturehipEventRecord(hipStream_t& stream, hipEvent_t& event);
hipError_t capturehipStreamWaitEvent(hipEvent_t& event, hipStream_t& stream, unsigned int& flags);
projects/hip/src/hipamd/src/hip_graph_helper.hpp
+35
Fájl megtekintése
@@ -0,0 +1,35 @@
hipError_t ihipMemcpy3D_validate(const hipMemcpy3DParms* p);
hipError_t ihipMemcpy_validate(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind);
hipError_t ihipMemcpyCommand(amd::Command*& command, void* dst, const void* src, size_t sizeBytes,
hipMemcpyKind kind, amd::HostQueue& queue);
hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
uint32_t sharedMemBytes, void** kernelParams, void** extra,
int deviceId, uint32_t params);
hipError_t ihipMemset_validate(void* dst, int64_t value, size_t valueSize, size_t sizeBytes);
hipError_t ihipMemset3D_validate(hipPitchedPtr pitchedDevPtr, int value, hipExtent extent,
size_t sizeBytes);
hipError_t ihipLaunchKernelCommand(amd::Command*& command, hipFunction_t f,
uint32_t globalWorkSizeX, uint32_t globalWorkSizeY,
uint32_t globalWorkSizeZ, uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, uint32_t sharedMemBytes,
amd::HostQueue* queue, void** kernelParams, void** extra,
hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags,
uint32_t params, uint32_t gridId, uint32_t numGrids,
uint64_t prevGridSum, uint64_t allGridSum, uint32_t firstDevice);
hipError_t ihipMemcpy3DCommand(amd::Command*& command, const hipMemcpy3DParms* p,
amd::HostQueue* queue);
hipError_t ihipMemsetCommand(std::vector<amd::Command*>& commands, void* dst, int64_t value,
size_t valueSize, size_t sizeBytes, amd::HostQueue* queue);
hipError_t ihipMemset3DCommand(std::vector<amd::Command*>& commands, hipPitchedPtr pitchedDevPtr,
int value, hipExtent extent, amd::HostQueue* queue);
projects/hip/src/hipamd/src/hip_graph_internal.cpp
+364
Fájl megtekintése
@@ -0,0 +1,364 @@
/* Copyright (c) 2021-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "hip_graph_internal.hpp"
#include <queue>
#define CASE_STRING(X, C) \
case X: \
case_string = #C; \
break;
const char* GetGraphNodeTypeString(uint32_t op) {
const char* case_string;
switch (static_cast<hipGraphNodeType>(op)) {
CASE_STRING(hipGraphNodeTypeKernel, KernelNode)
CASE_STRING(hipGraphNodeTypeMemcpy, Memcpy3DNode)
CASE_STRING(hipGraphNodeTypeMemset, MemsetNode)
CASE_STRING(hipGraphNodeTypeHost, HostNode)
CASE_STRING(hipGraphNodeTypeGraph, GraphNode)
CASE_STRING(hipGraphNodeTypeEmpty, EmptyNode)
CASE_STRING(hipGraphNodeTypeWaitEvent, WaitEventNode)
CASE_STRING(hipGraphNodeTypeEventRecord, EventRecordNode)
CASE_STRING(hipGraphNodeTypeMemcpy1D, Memcpy1DNode)
CASE_STRING(hipGraphNodeTypeMemcpyFromSymbol, MemcpyFromSymbolNode)
CASE_STRING(hipGraphNodeTypeMemcpyToSymbol, MemcpyToSymbolNode)
default:
case_string = "Unknown node type";
};
return case_string;
};
hipError_t hipGraph::AddNode(const Node& node) {
vertices_.emplace_back(node);
nodeOutDegree_[node] = 0;
nodeInDegree_[node] = 0;
node->SetLevel(0);
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Add %s(%p)\n",
GetGraphNodeTypeString(node->GetType()), node);
return hipSuccess;
}
hipError_t hipGraph::AddEdge(const Node& parentNode, const Node& childNode) {
// if vertice doesn't exist, add it to the graph
if (std::find(vertices_.begin(), vertices_.end(), parentNode) == vertices_.end()) {
AddNode(parentNode);
}
if (std::find(vertices_.begin(), vertices_.end(), childNode) == vertices_.end()) {
AddNode(childNode);
}
// Check if edge already exists
auto connectedEdges = edges_.find(parentNode);
if (connectedEdges != edges_.end()) {
if (std::find(connectedEdges->second.begin(), connectedEdges->second.end(), childNode) !=
connectedEdges->second.end()) {
return hipSuccess;
}
connectedEdges->second.emplace_back(childNode);
} else {
edges_[parentNode] = {childNode};
}
nodeOutDegree_[parentNode]++;
nodeInDegree_[childNode]++;
childNode->SetLevel(std::max(childNode->GetLevel(), parentNode->GetLevel() + 1));
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Add edge btwn %s(%p) - %s(%p)\n",
GetGraphNodeTypeString(parentNode->GetType()), parentNode,
GetGraphNodeTypeString(childNode->GetType()), childNode);
return hipSuccess;
}
// root nodes are all vertices with 0 in-degrees
std::vector<Node> hipGraph::GetRootNodes() const {
std::vector<Node> roots;
for (auto entry : vertices_) {
if (nodeInDegree_.at(entry) == 0) {
roots.push_back(entry);
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] root node: %s(%p)\n",
GetGraphNodeTypeString(entry->GetType()), entry);
}
}
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "\n");
return roots;
}
// leaf nodes are all vertices with 0 out-degrees
std::vector<Node> hipGraph::GetLeafNodes() const {
std::vector<Node> leafNodes;
for (auto entry : vertices_) {
if (nodeOutDegree_.at(entry) == 0) {
leafNodes.push_back(entry);
}
}
return leafNodes;
}
size_t hipGraph::GetLeafNodeCount() const {
int numLeafNodes = 0;
for (auto entry : vertices_) {
if (nodeOutDegree_.at(entry) == 0) {
numLeafNodes++;
}
}
return numLeafNodes;
}
std::vector<std::pair<Node, Node>> hipGraph::GetEdges() const {
std::vector<std::pair<Node, Node>> edges;
for (const auto& i : edges_) {
for (const auto& j : i.second) {
edges.push_back(std::make_pair(i.first, j));
}
}
return edges;
}
void hipGraph::GetRunListUtil(Node v, std::unordered_map<Node, bool>& visited,
std::vector<Node>& singleList,
std::vector<std::vector<Node>>& parallelLists,
std::unordered_map<Node, std::vector<Node>>& dependencies) {
// Mark the current node as visited.
visited[v] = true;
singleList.push_back(v);
// Recurse for all the vertices adjacent to this vertex
for (auto& adjNode : edges_[v]) {
if (!visited[adjNode]) {
// For the parallel list nodes add parent as the dependency
if (singleList.empty()) {
ClPrint(amd::LOG_INFO, amd::LOG_CODE,
"[hipGraph] For %s(%p)- add parent as dependency %s(%p)\n",
GetGraphNodeTypeString(adjNode->GetType()), adjNode,
GetGraphNodeTypeString(v->GetType()), v);
dependencies[adjNode].push_back(v);
}
GetRunListUtil(adjNode, visited, singleList, parallelLists, dependencies);
} else {
for (auto& list : parallelLists) {
// Merge singleList when adjNode matches with the first element of the list in existing
// lists
if (adjNode == list[0]) {
for (auto k = singleList.rbegin(); k != singleList.rend(); ++k) {
list.insert(list.begin(), *k);
}
singleList.erase(singleList.begin(), singleList.end());
}
}
// If the list cannot be merged with the existing list add as dependancy
if (!singleList.empty()) {
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] For %s(%p)- add dependency %s(%p)\n",
GetGraphNodeTypeString(adjNode->GetType()), adjNode,
GetGraphNodeTypeString(v->GetType()), v);
dependencies[adjNode].push_back(v);
}
}
}
if (!singleList.empty()) {
parallelLists.push_back(singleList);
singleList.erase(singleList.begin(), singleList.end());
}
}
// The function to do Topological Sort.
// It uses recursive GetRunListUtil()
void hipGraph::GetRunList(std::vector<std::vector<Node>>& parallelLists,
std::unordered_map<Node, std::vector<Node>>& dependencies) {
std::vector<Node> singleList;
// Mark all the vertices as not visited
std::unordered_map<Node, bool> visited;
for (auto node : vertices_) visited[node] = false;
// Call the recursive helper function for all vertices one by one
for (auto node : vertices_) {
if (visited[node] == false) {
GetRunListUtil(node, visited, singleList, parallelLists, dependencies);
}
}
for (size_t i = 0; i < parallelLists.size(); i++) {
for (size_t j = 0; j < parallelLists[i].size(); j++) {
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] list %d - %s(%p)\n", i + 1,
GetGraphNodeTypeString(parallelLists[i][j]->GetType()), parallelLists[i][j]);
}
}
}
hipError_t hipGraph::LevelOrder(std::vector<Node>& levelOrder) {
std::vector<Node> roots = GetRootNodes();
std::unordered_map<Node, bool> visited;
std::queue<Node> q;
for (auto it = roots.begin(); it != roots.end(); it++) {
q.push(*it);
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] %s(%p) level:%d \n",
GetGraphNodeTypeString((*it)->GetType()), *it, (*it)->GetLevel());
}
while (!q.empty()) {
Node& node = q.front();
q.pop();
levelOrder.push_back(node);
for (const auto& i : edges_[node]) {
if (visited.find(i) == visited.end() && i->GetLevel() == (node->GetLevel() + 1)) {
q.push(i);
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] %s(%p) level:%d \n",
GetGraphNodeTypeString(i->GetType()), i, i->GetLevel());
visited[i] = true;
}
}
}
return hipSuccess;
}
hipError_t hipGraphExec::CreateQueues() {
parallelQueues_.reserve(parallelLists_.size());
for (size_t i = 0; i < parallelLists_.size(); i++) {
amd::HostQueue* queue;
cl_command_queue_properties properties =
(callbacks_table.is_enabled() || HIP_FORCE_QUEUE_PROFILING) ? CL_QUEUE_PROFILING_ENABLE : 0;
queue = new amd::HostQueue(*hip::getCurrentDevice()->asContext(),
*hip::getCurrentDevice()->devices()[0], properties);
bool result = (queue != nullptr) ? queue->create() : false;
// Create a host queue
if (result) {
parallelQueues_.push_back(queue);
} else {
ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] Failed to create host queue\n");
return hipErrorOutOfMemory;
}
}
return hipSuccess;
}
hipError_t hipGraphExec::FillCommands() {
// Create commands
int i = 0;
hipError_t status;
for (const auto& list : parallelLists_) {
for (auto& node : list) {
status = node->CreateCommand(parallelQueues_[i]);
if (status != hipSuccess) return status;
}
i++;
}
// Add waitlists for all the commands
for (auto& node : levelOrder_) {
auto nodeWaitList = nodeWaitLists_.find(node);
if (nodeWaitList != nodeWaitLists_.end()) {
amd::Command::EventWaitList waitList;
for (auto depNode : nodeWaitList->second) {
for (auto command : depNode->GetCommands()) {
waitList.push_back(command);
}
}
for (auto command : nodeWaitList->first->GetCommands()) {
command->updateEventWaitList(waitList);
}
}
}
return status;
}
hipError_t hipGraphExec::Init() {
hipError_t status;
status = CreateQueues();
if (status != hipSuccess) {
return status;
}
status = FillCommands();
if (status != hipSuccess) {
return status;
}
rootCommand_ = nullptr;
/// stream should execute next command after graph finishes
/// Add marker to the stream that waits for all the last commands in parallel queues of graph
for (auto& singleList : parallelLists_) {
graphLastCmdWaitList_.push_back(singleList.back()->GetCommands().back());
}
return status;
}
void hipGraphExec::ResetGraph(cl_event event, cl_int command_exec_status, void* user_data) {
ClPrint(amd::LOG_INFO, amd::LOG_CODE, "[hipGraph] Inside resetGraph!\n");
hipGraphExec_t graphExec =
hipGraphExec::activeGraphExec_[reinterpret_cast<amd::Command*>(user_data)];
if (graphExec != nullptr) {
for (auto& node : graphExec->levelOrder_) {
for (auto& command : node->GetCommands()) {
command->resetStatus(CL_INT_MAX);
}
}
graphExec->rootCommand_->resetStatus(CL_INT_MAX);
graphExec->bExecPending_.store(false);
} else {
ClPrint(amd::LOG_ERROR, amd::LOG_CODE, "[hipGraph] graphExec is nullptr during resetGraph!\n");
}
}
hipError_t hipGraphExec::UpdateGraphToWaitOnRoot() {
for (auto& singleList : parallelLists_) {
amd::Command::EventWaitList waitList;
waitList.push_back(rootCommand_);
if (!singleList.empty()) {
auto commands = singleList[0]->GetCommands();
if (!commands.empty()) {
commands[0]->updateEventWaitList(waitList);
}
}
}
return hipSuccess;
}
hipError_t hipGraphExec::Run(hipStream_t stream) {
if (bExecPending_.load() == true) {
ClPrint(
amd::LOG_INFO, amd::LOG_CODE,
"[hipGraph] Same graph launched while previous one is active, wait for it to finish!\n");
lastEnqueuedGraphCmd_->awaitCompletion();
}
amd::HostQueue* queue = hip::getQueue(stream);
if (queue == nullptr) {
return hipErrorInvalidResourceHandle;
}
if (rootCommand_ == nullptr || rootCommand_->queue() != queue) {
if (rootCommand_ != nullptr) {
rootCommand_->release();
}
rootCommand_ = new amd::Marker(*queue, false, {});
UpdateGraphToWaitOnRoot();
}
rootCommand_->enqueue();
for (auto& node : levelOrder_) {
for (auto& command : node->GetCommands()) {
command->enqueue();
}
}
amd::Command* command = new amd::Marker(*queue, false, graphLastCmdWaitList_);
if (command == nullptr) {
return hipErrorOutOfMemory;
}
amd::Event& event = command->event();
if (!event.setCallback(CL_COMPLETE, hipGraphExec::ResetGraph, command)) {
return hipErrorInvalidHandle;
}
hipGraphExec::activeGraphExec_[command] = this;
lastEnqueuedGraphCmd_ = command;
bExecPending_.store(true);
command->enqueue();
command->release();
return hipSuccess;
}
projects/hip/src/hipamd/src/hip_graph_internal.hpp
+355
Fájl megtekintése
@@ -0,0 +1,355 @@
/* Copyright (c) 2021-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#pragma once
#include <algorithm>
#include <queue>
#include <stack>
#include <iostream>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "hip/hip_runtime.h"
#include "hip_internal.hpp"
#include "hip_graph_helper.hpp"
typedef hipGraphNode* Node;
class hipGraphNode {
protected:
uint32_t level_;
hipGraphNodeType type_;
std::vector<amd::Command*> commands_;
bool visited_;
public:
hipGraphNode(hipGraphNodeType type) {
type_ = type;
level_ = 0;
visited_ = false;
}
virtual ~hipGraphNode() {
for (auto command : commands_) {
delete command;
}
}
virtual hipError_t CreateCommand(amd::HostQueue* queue) { return hipSuccess; }
std::vector<amd::Command*>& GetCommands() { return commands_; }
hipGraphNodeType GetType() { return type_; }
uint32_t GetLevel() { return level_; }
void SetLevel(uint32_t level) { level_ = level; }
};
class hipGraph {
std::unordered_map<Node, size_t> nodeInDegree_; // count of in coming edges for every vertex
std::unordered_map<Node, size_t> nodeOutDegree_; // count of outgoing edges for every vertex
std::vector<Node> vertices_;
std::unordered_map<Node, std::vector<Node>> edges_;
public:
hipGraph() {}
~hipGraph(){};
/// add node to the graph
hipError_t AddNode(const Node& node);
/// add edge to the graph
hipError_t AddEdge(const Node& parentNode, const Node& childNode);
/// Returns root nodes, all vertices with 0 in-degrees
std::vector<Node> GetRootNodes() const;
/// Returns leaf nodes, all vertices with 0 out-degrees
std::vector<Node> GetLeafNodes() const;
/// Returns number of leaf nodes
size_t GetLeafNodeCount() const;
/// Returns total numbers of nodes in the graph
size_t GetNodeCount() const { return vertices_.size(); }
/// returns all the nodes in the graph
std::vector<Node> GetNodes() const { return vertices_; }
/// returns all the edges in the graph
std::vector<std::pair<Node, Node>> GetEdges() const;
void GetRunListUtil(Node v, std::unordered_map<Node, bool>& visited,
std::vector<Node>& singleList, std::vector<std::vector<Node>>& parallelList,
std::unordered_map<Node, std::vector<Node>>& dependencies);
void GetRunList(std::vector<std::vector<Node>>& parallelList,
std::unordered_map<Node, std::vector<Node>>& dependencies);
hipError_t LevelOrder(std::vector<Node>& levelOrder);
};
class hipGraphKernelNode : public hipGraphNode {
hipKernelNodeParams* pKernelParams_;
hipFunction_t func_;
public:
hipGraphKernelNode(const hipKernelNodeParams* pNodeParams, const hipFunction_t func)
: hipGraphNode(hipGraphNodeTypeKernel) {
pKernelParams_ = new hipKernelNodeParams(*pNodeParams);
func_ = func;
}
~hipGraphKernelNode() { delete pKernelParams_; }
hipError_t CreateCommand(amd::HostQueue* queue) {
commands_.reserve(1);
amd::Command* command;
hipError_t status = ihipLaunchKernelCommand(
command, func_, pKernelParams_->gridDim.x * pKernelParams_->blockDim.x,
pKernelParams_->gridDim.y * pKernelParams_->blockDim.y,
pKernelParams_->gridDim.z * pKernelParams_->blockDim.z, pKernelParams_->blockDim.x,
pKernelParams_->blockDim.y, pKernelParams_->blockDim.z, pKernelParams_->sharedMemBytes,
queue, pKernelParams_->kernelParams, pKernelParams_->extra, nullptr, nullptr, 0, 0, 0, 0, 0,
0, 0);
commands_.emplace_back(command);
return status;
}
void GetParams(hipKernelNodeParams* params) {
std::memcpy(params, pKernelParams_, sizeof(hipKernelNodeParams));
}
void SetParams(hipKernelNodeParams* params) {
std::memcpy(pKernelParams_, params, sizeof(hipKernelNodeParams));
}
};
class hipGraphMemcpyNode : public hipGraphNode {
hipMemcpy3DParms* pCopyParams_;
public:
hipGraphMemcpyNode(const hipMemcpy3DParms* pCopyParams) : hipGraphNode(hipGraphNodeTypeMemcpy) {
pCopyParams_ = new hipMemcpy3DParms(*pCopyParams);
}
~hipGraphMemcpyNode() { delete pCopyParams_; }
hipError_t CreateCommand(amd::HostQueue* queue) {
commands_.reserve(1);
amd::Command* command;
hipError_t status = ihipMemcpy3DCommand(command, pCopyParams_, queue);
commands_.emplace_back(command);
return status;
}
void GetParams(hipMemcpy3DParms* params) {
std::memcpy(params, pCopyParams_, sizeof(hipMemcpy3DParms));
}
void SetParams(hipMemcpy3DParms* params) {
std::memcpy(pCopyParams_, params, sizeof(hipMemcpy3DParms));
}
};
class hipGraphMemcpyNode1D : public hipGraphNode {
void* dst_;
const void* src_;
size_t count_;
hipMemcpyKind kind_;
public:
hipGraphMemcpyNode1D(void* dst, const void* src, size_t count, hipMemcpyKind kind)
: hipGraphNode(hipGraphNodeTypeMemcpy1D), dst_(dst), src_(src), count_(count), kind_(kind) {}
~hipGraphMemcpyNode1D() {}
hipError_t CreateCommand(amd::HostQueue* queue) {
commands_.reserve(1);
amd::Command* command = nullptr;
hipError_t status = ihipMemcpyCommand(command, dst_, src_, count_, kind_, *queue);
commands_.emplace_back(command);
return status;
}
void SetParams(void* dst, const void* src, size_t count, hipMemcpyKind kind) {
dst_ = dst;
src_ = src;
count_ = count;
kind_ = kind;
}
};
template <class T> class hipGraphMemcpyNodeFromSymbol : public hipGraphNode {
void* dst_;
const T& symbol_;
size_t count_;
size_t offset_;
hipMemcpyKind kind_;
public:
hipGraphMemcpyNodeFromSymbol(void* dst, const void* symbol, size_t count, size_t offset,
hipMemcpyKind kind)
: hipGraphNode(hipGraphNodeTypeMemcpyFromSymbol),
dst_(dst),
symbol_(symbol),
count_(count),
offset_(offset),
kind_(kind) {}
~hipGraphMemcpyNodeFromSymbol() {}
hipError_t CreateCommand(amd::HostQueue* queue);
void SetParams(void* dst, const void* symbol, size_t count, size_t offset, hipMemcpyKind kind) {
dst_ = dst;
symbol_ = symbol;
count_ = count;
offset_ = offset;
kind_ = kind;
}
};
template <class T> class hipGraphMemcpyNodeToSymbol : public hipGraphNode {
const T& symbol_;
const void* src_;
size_t count_;
size_t offset_;
hipMemcpyKind kind_;
public:
hipGraphMemcpyNodeToSymbol(const T& symbol, void* src, size_t count, size_t offset,
hipMemcpyKind kind)
: hipGraphNode(hipGraphNodeTypeMemcpyToSymbol),
symbol_(symbol),
src_(src),
count_(count),
offset_(offset),
kind_(kind) {}
~hipGraphMemcpyNodeToSymbol() {}
hipError_t CreateCommand(amd::HostQueue* queue);
void SetParams(const T& symbol, void* src, size_t count, size_t offset, hipMemcpyKind kind) {
symbol_ = symbol;
src_ = src;
count_ = count;
offset_ = offset;
kind_ = kind;
}
};
class hipGraphMemsetNode : public hipGraphNode {
hipMemsetParams* pMemsetParams_;
public:
hipGraphMemsetNode(const hipMemsetParams* pMemsetParams) : hipGraphNode(hipGraphNodeTypeMemset) {
pMemsetParams_ = new hipMemsetParams(*pMemsetParams);
}
~hipGraphMemsetNode() { delete pMemsetParams_; }
hipError_t CreateCommand(amd::HostQueue* queue) {
if (pMemsetParams_->height == 1) {
return ihipMemsetCommand(commands_, pMemsetParams_->dst, pMemsetParams_->value,
pMemsetParams_->elementSize,
pMemsetParams_->width * pMemsetParams_->elementSize, queue);
} else {
return ihipMemset3DCommand(commands_,
{pMemsetParams_->dst, pMemsetParams_->pitch, pMemsetParams_->width,
pMemsetParams_->height},
pMemsetParams_->elementSize,
{pMemsetParams_->width, pMemsetParams_->height, 1}, queue);
}
return hipSuccess;
}
void GetParams(hipMemsetParams* params) {
std::memcpy(params, pMemsetParams_, sizeof(hipMemsetParams));
}
void SetParams(hipMemsetParams* params) {
std::memcpy(pMemsetParams_, params, sizeof(hipMemsetParams));
}
};
class hipGraphEventRecordNode : public hipGraphNode {
hipEvent_t event_;
public:
hipGraphEventRecordNode(hipEvent_t event)
: hipGraphNode(hipGraphNodeTypeEventRecord), event_(event) {}
~hipGraphEventRecordNode() {}
hipError_t CreateCommand(amd::HostQueue* queue);
void GetParams(hipEvent_t* event) { *event = event_; }
void SetParams(hipEvent_t event) { event_ = event; }
};
class hipGraphEventWaitNode : public hipGraphNode {
hipEvent_t event_;
public:
hipGraphEventWaitNode(hipEvent_t event)
: hipGraphNode(hipGraphNodeTypeWaitEvent), event_(event) {}
~hipGraphEventWaitNode() {}
hipError_t CreateCommand(amd::HostQueue* queue);
void GetParams(hipEvent_t* event) { *event = event_; }
void SetParams(hipEvent_t event) { event_ = event; }
};
class hipGraphHostNode : public hipGraphNode {
hipHostNodeParams* pNodeParams_;
public:
hipGraphHostNode(const hipHostNodeParams* pNodeParams) : hipGraphNode(hipGraphNodeTypeHost) {
pNodeParams_ = new hipHostNodeParams(*pNodeParams);
}
~hipGraphHostNode() { delete pNodeParams_; }
hipError_t CreateCommand(amd::HostQueue* queue);
void GetParams(hipHostNodeParams* params) {
std::memcpy(params, pNodeParams_, sizeof(hipHostNodeParams));
}
void SetParams(hipHostNodeParams* params) {
std::memcpy(pNodeParams_, params, sizeof(hipHostNodeParams));
}
};
class hipGraphExec {
std::vector<std::vector<Node>> parallelLists_;
std::vector<Node> levelOrder_;
std::unordered_map<Node, std::vector<Node>> nodeWaitLists_;
std::vector<amd::HostQueue*> parallelQueues_;
static std::unordered_map<amd::Command*, hipGraphExec_t> activeGraphExec_;
amd::Command::EventWaitList graphLastCmdWaitList_;
amd::Command* lastEnqueuedGraphCmd_;
std::atomic<bool> bExecPending_;
amd::Command* rootCommand_;
public:
hipGraphExec(std::vector<Node>& levelOrder, std::vector<std::vector<Node>>& lists,
std::unordered_map<Node, std::vector<Node>>& nodeWaitLists)
: parallelLists_(lists),
levelOrder_(levelOrder),
nodeWaitLists_(nodeWaitLists),
lastEnqueuedGraphCmd_(nullptr),
rootCommand_(nullptr) {
bExecPending_.store(false);
}
~hipGraphExec() {
for (auto queue : parallelQueues_) {
queue->release();
}
for (auto node : levelOrder_) {
delete node;
}
}
hipError_t CreateQueues();
hipError_t FillCommands();
hipError_t Init();
hipError_t UpdateGraphToWaitOnRoot();
hipError_t Run(hipStream_t stream);
static void ResetGraph(cl_event event, cl_int command_exec_status, void* user_data);
};
projects/hip/src/hipamd/src/hip_hcc.def.in
Executable
+293
Fájl megtekintése
@@ -0,0 +1,293 @@
EXPORTS
hipChooseDevice
hipCtxCreate
hipCtxDestroy
hipCtxDisablePeerAccess
hipCtxEnablePeerAccess
hipCtxGetApiVersion
hipCtxGetCacheConfig
hipCtxGetCurrent
hipCtxGetDevice
hipCtxGetFlags
hipCtxGetSharedMemConfig
hipCtxPopCurrent
hipCtxPushCurrent
hipCtxSetCacheConfig
hipCtxSetCurrent
hipCtxSetSharedMemConfig
hipCtxSynchronize
hipDeviceCanAccessPeer
hipDeviceComputeCapability
hipDeviceDisablePeerAccess
hipDeviceEnablePeerAccess
hipDeviceGet
hipDeviceGetAttribute
hipDeviceGetByPCIBusId
hipDeviceGetCacheConfig
hipDeviceGetStreamPriorityRange
hipDeviceGetLimit
hipDeviceGetName
hipDeviceGetPCIBusId
hipDeviceGetSharedMemConfig
hipDeviceGetP2PAttribute
hipDevicePrimaryCtxGetState
hipDevicePrimaryCtxRelease
hipDevicePrimaryCtxReset
hipDevicePrimaryCtxRetain
hipDevicePrimaryCtxSetFlags
hipDeviceReset
hipDeviceSetCacheConfig
hipDeviceSetSharedMemConfig
hipDeviceSynchronize
hipDeviceTotalMem
hipDriverGetVersion
hipEventCreate
hipEventCreateWithFlags
hipEventDestroy
hipEventElapsedTime
hipEventQuery
hipEventRecord
hipEventSynchronize
hipExtGetLinkTypeAndHopCount
hipExtLaunchMultiKernelMultiDevice
hipExtMallocWithFlags
hipExtModuleLaunchKernel
hipExtLaunchKernel
hipFree
hipFreeArray
hipFuncSetAttribute
hipFuncSetCacheConfig
hipFuncSetSharedMemConfig
hipGetDevice
hipGetDeviceCount
hipGetDeviceProperties
hipGetErrorName
hipGetErrorString
hipGetLastError
hipMemAllocHost
hipHostAlloc
hipHostFree
hipHostGetDevicePointer
hipHostGetFlags
hipHostMalloc
hipHostRegister
hipHostUnregister
hipInit
hipIpcCloseMemHandle
hipIpcGetMemHandle
hipIpcOpenMemHandle
hipIpcGetEventHandle
hipIpcOpenEventHandle
hipMalloc
hipMalloc3D
hipMalloc3DArray
hipMallocManaged
hipArrayCreate
hipArray3DCreate
hipArrayDestroy
hipMallocArray
hipMemAdvise
hipMemAllocPitch
hipMallocPitch
hipMemcpy
hipMemcpyWithStream
hipMemcpyParam2D
hipMemcpy2D
hipMemcpy2DAsync
hipMemcpy2DToArray
hipMemcpy2DToArrayAsync
hipMemcpy3D
hipMemcpy3DAsync
hipDrvMemcpy3D
hipDrvMemcpy3DAsync
hipMemcpyAsync
hipMemcpyDtoD
hipMemcpyDtoDAsync
hipMemcpyDtoH
hipMemcpyDtoHAsync
hipMemcpyFromSymbol
hipMemcpyFromSymbolAsync
hipMemcpyHtoD
hipMemcpyHtoDAsync
hipMemcpyPeer
hipMemcpyPeerAsync
hipMemcpyToArray
hipMemcpyFromArray
hipMemcpyToSymbol
hipMemcpyToSymbolAsync
hipMemGetAddressRange
hipGetSymbolAddress
hipGetSymbolSize
hipMemGetInfo
hipMemPrefetchAsync
hipMemPtrGetInfo
hipMemRangeGetAttribute
hipMemRangeGetAttributes
hipMemset
hipMemsetAsync
hipMemsetD8
hipMemsetD8Async
hipMemsetD16
hipMemsetD16Async
hipMemsetD32
hipMemsetD32Async
hipMemset2D
hipMemset2DAsync
hipMemset3D
hipMemset3DAsync
hipModuleGetFunction
hipModuleGetGlobal
hipModuleGetTexRef
hipModuleLaunchKernel
hipModuleLaunchKernelExt
hipLaunchCooperativeKernel
hipLaunchCooperativeKernelMultiDevice
hipHccModuleLaunchKernel
hipModuleLoad
hipModuleLoadData
hipModuleLoadDataEx
hipModuleUnload
hipModuleOccupancyMaxPotentialBlockSize
hipModuleOccupancyMaxPotentialBlockSizeWithFlags
hipModuleOccupancyMaxActiveBlocksPerMultiprocessor
hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags
hipOccupancyMaxPotentialBlockSize
hipOccupancyMaxActiveBlocksPerMultiprocessor
hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags
hipFuncGetAttribute
hipFuncGetAttributes
hipPeekAtLastError
hipPointerGetAttributes
hipProfilerStart
hipProfilerStop
hipRuntimeGetVersion
hipGetDeviceFlags
hipSetDevice
hipSetDeviceFlags
hipStreamAddCallback
hipStreamAttachMemAsync
hipStreamCreate
hipStreamCreateWithFlags
hipStreamCreateWithPriority
hipStreamDestroy
hipStreamGetFlags
hipStreamQuery
hipStreamSynchronize
hipStreamWaitEvent
__hipPopCallConfiguration
__hipPushCallConfiguration
__hipRegisterFatBinary
__hipRegisterFunction
__hipRegisterVar
__hipRegisterSurface
__hipRegisterTexture
__hipRegisterManagedVar
__hipUnregisterFatBinary
hipConfigureCall
hipSetupArgument
hipLaunchByPtr
hipLaunchKernel
hipRegisterApiCallback
hipRemoveApiCallback
hipRegisterActivityCallback
hipRemoveActivityCallback
hipApiName
hipKernelNameRef
hipBindTexture
hipBindTexture2D
hipBindTextureToArray
hipBindTextureToMipmappedArray
hipGetTextureAlignmentOffset
hipGetTextureReference
hipUnbindTexture
hipCreateChannelDesc
hipCreateTextureObject
hipDestroyTextureObject
hipGetChannelDesc
hipGetTextureObjectResourceDesc
hipGetTextureObjectResourceViewDesc
hipGetTextureObjectTextureDesc
hipTexRefGetAddress
hipTexRefGetAddressMode
hipTexRefGetArray
hipTexRefGetBorderColor
hipTexRefGetFilterMode
hipTexRefGetFlags
hipTexRefGetFormat
hipTexRefGetMaxAnisotropy
hipTexRefGetMipmapFilterMode
hipTexRefGetMipmapLevelBias
hipTexRefGetMipmapLevelClamp
hipTexRefGetMipmappedArray
hipTexRefSetAddress
hipTexRefSetAddress2D
hipTexRefSetAddressMode
hipTexRefSetArray
hipTexRefSetBorderColor
hipTexRefSetFilterMode
hipTexRefSetFlags
hipTexRefSetFormat
hipTexRefSetMaxAnisotropy
hipTexRefSetMipmapFilterMode
hipTexRefSetMipmapLevelBias
hipTexRefSetMipmapLevelClamp
hipTexRefSetMipmappedArray
hipProfilerStart
hipProfilerStop
hipCreateSurfaceObject
hipDestroySurfaceObject
hipInitActivityCallback
hipEnableActivityCallback
hipGetCmdName
hiprtcAddNameExpression
hiprtcCompileProgram
hiprtcCreateProgram
hiprtcDestroyProgram
hiprtcGetLoweredName
hiprtcGetProgramLog
hiprtcGetProgramLogSize
hiprtcGetCode
hiprtcGetCodeSize
hiprtcGetErrorString
hipMipmappedArrayCreate
hipMallocMipmappedArray
hipMipmappedArrayDestroy
hipFreeMipmappedArray
hipMipmappedArrayGetLevel
hipGetMipmappedArrayLevel
hipMallocHost
hipFreeHost
hipTexObjectCreate
hipTexObjectDestroy
hipTexObjectGetResourceDesc
hipTexObjectGetResourceViewDesc
hipTexObjectGetTextureDesc
hipExtStreamCreateWithCUMask
hipStreamGetPriority
hipMemcpy2DFromArray
hipMemcpy2DFromArrayAsync
hipDrvMemcpy2DUnaligned
hipMemcpyAtoH
hipMemcpyHtoA
hipMemcpyParam2DAsync
__gnu_h2f_ieee
__gnu_f2h_ieee
hipExtStreamGetCUMask
hipImportExternalMemory
hipExternalMemoryGetMappedBuffer
hipDestroyExternalMemory
hipGraphCreate
hipGraphDestroy
hipGraphAddKernelNode
hipGraphAddMemsetNode
hipGraphAddMemcpyNode
hipGraphInstantiate
hipGraphLaunch
hipStreamIsCapturing
hipStreamBeginCapture
hipStreamEndCapture
hipGraphExecDestroy
hipImportExternalSemaphore
hipSignalExternalSemaphoresAsync
hipWaitExternalSemaphoresAsync
hipDestroyExternalSemaphore
projects/hip/src/hipamd/src/hip_hcc.map.in
Executable
+310
Fájl megtekintése
@@ -0,0 +1,310 @@
{
global:
hipChooseDevice;
hipCtxCreate;
hipCtxDestroy;
hipCtxDisablePeerAccess;
hipCtxEnablePeerAccess;
hipCtxGetApiVersion;
hipCtxGetCacheConfig;
hipCtxGetCurrent;
hipCtxGetDevice;
hipCtxGetFlags;
hipCtxGetSharedMemConfig;
hipCtxPopCurrent;
hipCtxPushCurrent;
hipCtxSetCacheConfig;
hipCtxSetCurrent;
hipCtxSetSharedMemConfig;
hipCtxSynchronize;
hipDeviceCanAccessPeer;
hipDeviceComputeCapability;
hipDeviceDisablePeerAccess;
hipDeviceEnablePeerAccess;
hipDeviceGet;
hipDeviceGetAttribute;
hipDeviceGetByPCIBusId;
hipDeviceGetCacheConfig;
hipDeviceGetStreamPriorityRange;
hipDeviceGetLimit;
hipDeviceGetName;
hipDeviceGetPCIBusId;
hipDeviceGetSharedMemConfig;
hipDeviceGetP2PAttribute;
hipDevicePrimaryCtxGetState;
hipDevicePrimaryCtxRelease;
hipDevicePrimaryCtxReset;
hipDevicePrimaryCtxRetain;
hipDevicePrimaryCtxSetFlags;
hipDeviceReset;
hipDeviceSetCacheConfig;
hipDeviceSetSharedMemConfig;
hipDeviceSynchronize;
hipDeviceTotalMem;
hipDriverGetVersion;
hipEventCreate;
hipEventCreateWithFlags;
hipEventDestroy;
hipEventElapsedTime;
hipEventQuery;
hipEventRecord;
hipEventSynchronize;
hipExtGetLinkTypeAndHopCount;
hipExtLaunchMultiKernelMultiDevice;
hipExtMallocWithFlags;
hipExtModuleLaunchKernel;
hipExtLaunchKernel;
hipFree;
hipFreeArray;
hipFuncSetAttribute;
hipFuncSetCacheConfig;
hipFuncSetSharedMemConfig;
hipGetDevice;
hipGetDeviceCount;
hipGetDeviceProperties;
hipGetErrorName;
hipGetErrorString;
hipGetLastError;
hipMemAdvise;
hipMemAllocHost;
hipHostAlloc;
hipHostFree;
hipHostGetDevicePointer;
hipHostGetFlags;
hipHostMalloc;
hipHostRegister;
hipHostUnregister;
hipInit;
hipIpcCloseMemHandle;
hipIpcGetMemHandle;
hipIpcOpenMemHandle;
hipIpcGetEventHandle;
hipIpcOpenEventHandle;
hipMalloc;
hipMalloc3D;
hipMalloc3DArray;
hipMallocManaged;
hipArrayCreate;
hipArrayDestroy;
hipArray3DCreate;
hipMallocArray;
hipMallocPitch;
hipMemAllocPitch;
hipMemcpy;
hipMemcpyWithStream;
hipMemcpyParam2D;
hipMemcpy2D;
hipMemcpy2DAsync;
hipMemcpy2DToArray;
hipMemcpy2DToArrayAsync;
hipDrvMemcpy2DUnaligned;
hipMemcpy3D;
hipMemcpy3DAsync;
hipDrvMemcpy3D;
hipDrvMemcpy3DAsync;
hipMemcpyAsync;
hipMemcpyDtoD;
hipMemcpyDtoDAsync;
hipMemcpyDtoH;
hipMemcpyDtoHAsync;
hipMemcpyFromSymbol;
hipMemcpyFromSymbolAsync;
hipMemcpyHtoD;
hipMemcpyHtoDAsync;
hipMemcpyPeer;
hipMemcpyPeerAsync;
hipMemcpyToArray;
hipMemcpyFromArray;
hipMemcpyToSymbol;
hipMemcpyToSymbolAsync;
hipMemGetAddressRange;
hipGetSymbolAddress;
hipGetSymbolSize;
hipMemGetInfo;
hipMemPrefetchAsync;
hipMemPtrGetInfo;
hipMemRangeGetAttribute;
hipMemRangeGetAttributes;
hipMemset;
hipMemsetAsync;
hipMemsetD8;
hipMemsetD8Async;
hipMemsetD16;
hipMemsetD16Async;
hipMemsetD32;
hipMemsetD32Async;
hipMemset2D;
hipMemset2DAsync;
hipMemset3D;
hipMemset3DAsync;
hipModuleGetFunction;
hipModuleGetGlobal;
hipModuleGetTexRef;
hipModuleLaunchKernel;
hipModuleLaunchKernelExt;
hipLaunchCooperativeKernel;
hipLaunchCooperativeKernelMultiDevice;
hipModuleLoad;
hipModuleLoadData;
hipModuleLoadDataEx;
hipModuleUnload;
hipModuleOccupancyMaxPotentialBlockSize;
hipModuleOccupancyMaxPotentialBlockSizeWithFlags;
hipModuleOccupancyMaxActiveBlocksPerMultiprocessor;
hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags;
hipOccupancyMaxPotentialBlockSize;
hipOccupancyMaxActiveBlocksPerMultiprocessor;
hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags;
hipFuncGetAttribute;
hipFuncGetAttributes;
hipPeekAtLastError;
hipPointerGetAttributes;
hipProfilerStart;
hipProfilerStop;
hipRuntimeGetVersion;
hipGetDeviceFlags;
hipSetDevice;
hipSetDeviceFlags;
hipStreamAddCallback;
hipStreamAttachMemAsync;
hipStreamCreate;
hipStreamCreateWithFlags;
hipStreamCreateWithPriority;
hipStreamDestroy;
hipStreamGetFlags;
hipStreamQuery;
hipStreamSynchronize;
hipStreamWaitEvent;
hipStreamWaitValue32;
hipStreamWaitValue64;
hipStreamWriteValue32;
hipStreamWriteValue64;
__hipPopCallConfiguration;
__hipPushCallConfiguration;
__hipRegisterFatBinary;
__hipRegisterFunction;
__hipRegisterVar;
__hipRegisterSurface;
__hipRegisterTexture;
__hipRegisterManagedVar;
__hipUnregisterFatBinary;
__gnu_h2f_ieee;
__gnu_f2h_ieee;
hipConfigureCall;
hipSetupArgument;
hipLaunchByPtr;
hipLaunchKernel;
hipRegisterApiCallback;
hipRemoveApiCallback;
hipRegisterActivityCallback;
hipRemoveActivityCallback;
hipApiName;
hipKernelNameRef;
hipKernelNameRefByPtr;
hipGetStreamDeviceId;
hipProfilerStart;
hipProfilerStop;
hiprtcCompileProgram;
hiprtcCreateProgram;
hiprtcDestroyProgram;
hiprtcGetLoweredName;
hiprtcGetProgramLog;
hiprtcGetProgramLogSize;
hiprtcGetCode;
hiprtcGetCodeSize;
hiprtcGetErrorString;
hiprtcAddNameExpression;
hiprtcVersion;
hipBindTexture;
hipBindTexture2D;
hipBindTextureToArray;
hipBindTextureToMipmappedArray;
hipGetTextureAlignmentOffset;
hipGetTextureReference;
hipUnbindTexture;
hipCreateChannelDesc;
hipCreateTextureObject;
hipDestroyTextureObject;
hipGetChannelDesc;
hipGetTextureObjectResourceDesc;
hipGetTextureObjectResourceViewDesc;
hipGetTextureObjectTextureDesc;
hipTexRefGetAddress;
hipTexRefGetAddressMode;
hipTexRefGetArray;
hipTexRefGetBorderColor;
hipTexRefGetFilterMode;
hipTexRefGetFlags;
hipTexRefGetFormat;
hipTexRefGetMaxAnisotropy;
hipTexRefGetMipmapFilterMode;
hipTexRefGetMipmapLevelBias;
hipTexRefGetMipmapLevelClamp;
hipTexRefGetMipmappedArray;
hipTexRefSetAddress;
hipTexRefSetAddress2D;
hipTexRefSetAddressMode;
hipTexRefSetArray;
hipTexRefSetBorderColor;
hipTexRefSetFilterMode;
hipTexRefSetFlags;
hipTexRefSetFormat;
hipTexRefSetMaxAnisotropy;
hipTexRefSetMipmapFilterMode;
hipTexRefSetMipmapLevelBias;
hipTexRefSetMipmapLevelClamp;
hipTexRefSetMipmappedArray;
hipMipmappedArrayCreate;
hipMallocMipmappedArray;
hipMipmappedArrayDestroy;
hipFreeMipmappedArray;
hipMipmappedArrayGetLevel;
hipGetMipmappedArrayLevel;
hipMallocHost;
hipFreeHost;
hipTexObjectCreate;
hipTexObjectDestroy;
hipTexObjectGetResourceDesc;
hipTexObjectGetResourceViewDesc;
hipTexObjectGetTextureDesc;
hipImportExternalMemory;
hipExternalMemoryGetMappedBuffer;
hipDestroyExternalMemory;
hipGraphCreate;
hipGraphDestroy;
hipGraphAddKernelNode;
hipGraphAddMemsetNode;
hipGraphAddMemcpyNode;
hipGraphInstantiate;
hipGraphLaunch;
hipStreamIsCapturing;
hipStreamBeginCapture;
hipStreamEndCapture;
hipGraphExecDestroy;
hipImportExternalSemaphore;
hipSignalExternalSemaphoresAsync;
hipWaitExternalSemaphoresAsync;
hipDestroyExternalSemaphore;
extern "C++" {
hip_impl::hipLaunchKernelGGLImpl*;
hip_impl::demangle*;
hipCreateSurfaceObject*;
hipDestroySurfaceObject*;
hipHccModuleLaunchKernel*;
hipExtModuleLaunchKernel*;
hipInitActivityCallback*;
hipEnableActivityCallback*;
hipGetCmdName*;
hipExtStreamCreateWithCUMask;
hipStreamGetPriority;
hipMemcpy2DFromArray;
hipMemcpy2DFromArrayAsync;
hipMemcpyAtoH;
hipMemcpyHtoA;
hipMemcpyParam2DAsync;
__hipGetPCH;
hipExtStreamGetCUMask;
};
local:
*;
};
projects/hip/src/hipamd/src/hip_hcc.rc
+75
Fájl megtekintése
@@ -0,0 +1,75 @@
#define STR(__macro__) #__macro__
#define XSTR(__macro__) STR(__macro__)
#if defined(_DEBUG)
#define DEBUG_ONLY(x) x
#else
#define DEBUG_ONLY(x)
#endif
#define VERSION_PREFIX_MAJOR 2
#define VERSION_PREFIX_MINOR 0
#define APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
//
// Generated from the TEXTINCLUDE 2 resource.
//
#include "winresrc.h"
#include "utils/versions.hpp"
/////////////////////////////////////////////////////////////////////////////
#undef APSTUDIO_READONLY_SYMBOLS
/////////////////////////////////////////////////////////////////////////////
// English (U.S.) resources
#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
#ifdef _WIN32
LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
#pragma code_page(1252)
#endif //_WIN32
/////////////////////////////////////////////////////////////////////////////
//
// Version
//
VS_VERSION_INFO VERSIONINFO
FILEVERSION 10,0,AMD_PLATFORM_BUILD_NUMBER,AMD_PLATFORM_REVISION_NUMBER
PRODUCTVERSION 10,0,AMD_PLATFORM_BUILD_NUMBER,AMD_PLATFORM_REVISION_NUMBER
FILEFLAGSMASK 0x3fL
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x40004L
FILETYPE 0x2L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "Comments", " \0"
VALUE "CompanyName", "Advanced Micro Devices Inc.\0"
VALUE "FileDescription", AMD_PLATFORM_NAME " OpenCL " XSTR(VERSION_PREFIX_MAJOR) "." XSTR(VERSION_PREFIX_MINOR) " Runtime\0"
VALUE "FileVersion", "10.0." XSTR(AMD_PLATFORM_BUILD_NUMBER) "." XSTR(AMD_PLATFORM_REVISION_NUMBER)
VALUE "InternalName", "OpenCL"
VALUE "LegalCopyright", "Copyright (C) 2011 Advanced Micro Devices Inc.\0"
VALUE "OriginalFilename", "OpenCL.dll"
VALUE "ProductName", "OpenCL " XSTR(VERSION_PREFIX_MAJOR) "." XSTR(VERSION_PREFIX_MINOR) " " AMD_PLATFORM_INFO "\0"
VALUE "ProductVersion", "10.0." XSTR(AMD_PLATFORM_BUILD_NUMBER) "." XSTR(AMD_PLATFORM_REVISION_NUMBER)
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END
#endif // English (U.S.) resources
/////////////////////////////////////////////////////////////////////////////
projects/hip/src/hipamd/src/hip_hmm.cpp
+224
Fájl megtekintése
@@ -0,0 +1,224 @@
/* Copyright (c) 2020-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
#include "hip_conversions.hpp"
#include "platform/context.hpp"
#include "platform/command.hpp"
#include "platform/memory.hpp"
// Forward declaraiton of a function
hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0);
// Make sure HIP defines match ROCclr to avoid double conversion
static_assert(hipCpuDeviceId == amd::CpuDeviceId, "CPU device ID mismatch with ROCclr!");
static_assert(hipInvalidDeviceId == amd::InvalidDeviceId,
"Invalid device ID mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseSetReadMostly) ==
amd::MemoryAdvice::SetReadMostly, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseUnsetReadMostly) ==
amd::MemoryAdvice::UnsetReadMostly, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseSetPreferredLocation) ==
amd::MemoryAdvice::SetPreferredLocation, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseUnsetPreferredLocation) ==
amd::MemoryAdvice::UnsetPreferredLocation, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseSetAccessedBy) ==
amd::MemoryAdvice::SetAccessedBy, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseUnsetAccessedBy) ==
amd::MemoryAdvice::UnsetAccessedBy, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseSetCoarseGrain) ==
amd::MemoryAdvice::SetCoarseGrain, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemAdviseUnsetCoarseGrain) ==
amd::MemoryAdvice::UnsetCoarseGrain, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemRangeAttributeReadMostly) ==
amd::MemRangeAttribute::ReadMostly, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemRangeAttributePreferredLocation) ==
amd::MemRangeAttribute::PreferredLocation, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemRangeAttributeAccessedBy) ==
amd::MemRangeAttribute::AccessedBy, "Enum mismatch with ROCclr!");
static_assert(static_cast<uint32_t>(hipMemRangeAttributeLastPrefetchLocation) ==
amd::MemRangeAttribute::LastPrefetchLocation, "Enum mismatch with ROCclr!");
// ================================================================================================
hipError_t hipMallocManaged(void** dev_ptr, size_t size, unsigned int flags) {
HIP_INIT_API(hipMallocManaged, dev_ptr, size, flags);
if ((dev_ptr == nullptr) || (size == 0) ||
((flags != hipMemAttachGlobal) && (flags != hipMemAttachHost))) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(ihipMallocManaged(dev_ptr, size), *dev_ptr);
}
// ================================================================================================
hipError_t hipMemPrefetchAsync(const void* dev_ptr, size_t count, int device,
hipStream_t stream) {
HIP_INIT_API(hipMemPrefetchAsync, dev_ptr, count, device, stream);
if ((dev_ptr == nullptr) || (count == 0)) {
HIP_RETURN(hipErrorInvalidValue);
}
amd::HostQueue* queue = nullptr;
bool cpu_access = (device == hipCpuDeviceId) ? true : false;
// Pick the specified stream or Null one from the provided device
if (stream != nullptr) {
queue = hip::getQueue(stream);
} else {
if (!cpu_access) {
queue = g_devices[device]->NullStream();
} else {
queue = hip::getCurrentDevice()->NullStream();
}
}
if (queue == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
amd::Command::EventWaitList waitList;
amd::SvmPrefetchAsyncCommand* command =
new amd::SvmPrefetchAsyncCommand(*queue, waitList, dev_ptr, count, cpu_access);
if (command == nullptr) {
return hipErrorOutOfMemory;
}
if (!command->validateMemory()) {
delete command;
HIP_RETURN(hipErrorInvalidValue);
}
command->enqueue();
command->release();
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipMemAdvise(const void* dev_ptr, size_t count, hipMemoryAdvise advice, int device) {
HIP_INIT_API(hipMemAdvise, dev_ptr, count, advice, device);
if ((dev_ptr == nullptr) || (count == 0) ||
((device != hipCpuDeviceId) && (static_cast<size_t>(device) >= g_devices.size()))) {
HIP_RETURN(hipErrorInvalidValue);
}
amd::Device* dev = (device == hipCpuDeviceId) ?
g_devices[0]->devices()[0] : g_devices[device]->devices()[0];
bool use_cpu = (device == hipCpuDeviceId) ? true : false;
// Set the allocation attributes in AMD HMM
if (!dev->SetSvmAttributes(dev_ptr, count, static_cast<amd::MemoryAdvice>(advice), use_cpu)) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipMemRangeGetAttribute(void* data, size_t data_size, hipMemRangeAttribute attribute,
const void* dev_ptr, size_t count) {
HIP_INIT_API(hipMemRangeGetAttribute, data, data_size, attribute, dev_ptr, count);
if ((data == nullptr) || (data_size == 0) || (dev_ptr == nullptr) || (count == 0)) {
HIP_RETURN(hipErrorInvalidValue);
}
// Shouldn't matter for which device the interface is called
amd::Device* dev = g_devices[0]->devices()[0];
// Get the allocation attribute from AMD HMM
if (!dev->GetSvmAttributes(&data, &data_size, reinterpret_cast<int*>(&attribute), 1,
dev_ptr, count)) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipMemRangeGetAttributes(void** data, size_t* data_sizes,
hipMemRangeAttribute* attributes, size_t num_attributes,
const void* dev_ptr, size_t count) {
HIP_INIT_API(hipMemRangeGetAttributes, data, data_sizes,
attributes, num_attributes, dev_ptr, count);
if ((data == nullptr) || (data_sizes == nullptr) || (attributes == nullptr) ||
(num_attributes == 0) || (dev_ptr == nullptr) || (count == 0)) {
HIP_RETURN(hipErrorInvalidValue);
}
// Shouldn't matter for which device the interface is called
amd::Device* dev = g_devices[0]->devices()[0];
// Get the allocation attributes from AMD HMM
if (!dev->GetSvmAttributes(data, data_sizes, reinterpret_cast<int*>(attributes),
num_attributes, dev_ptr, count)) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipStreamAttachMemAsync(hipStream_t stream, hipDeviceptr_t* dev_ptr,
size_t length, unsigned int flags) {
HIP_INIT_API(hipStreamAttachMemAsync, stream, dev_ptr, length, flags);
if ((stream == nullptr) || (dev_ptr == nullptr) || (length == 0)) {
HIP_RETURN(hipErrorInvalidValue);
}
// Unclear what should be done for this interface in AMD HMM, since it's generic SVM alloc
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align) {
if (ptr == nullptr) {
return hipErrorInvalidValue;
} else if (size == 0) {
*ptr = nullptr;
return hipSuccess;
}
assert((hip::host_device->asContext()!= nullptr) && "Current host context must be valid");
amd::Context& ctx = *hip::host_device->asContext();
const amd::Device& dev = *ctx.devices()[0];
// For now limit to the max allocation size on the device.
// The apps should be able to go over the limit in the future
if (dev.info().maxMemAllocSize_ < size) {
return hipErrorMemoryAllocation;
}
// Allocate SVM fine grain buffer with the forced host pointer, avoiding explicit memory
// allocation in the device driver
*ptr = amd::SvmBuffer::malloc(ctx, CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_ALLOC_HOST_PTR,
size, (align == 0) ? dev.info().memBaseAddrAlign_ : align);
if (*ptr == nullptr) {
return hipErrorMemoryAllocation;
}
ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] ihipMallocManaged ptr=0x%zx", getpid(),
std::this_thread::get_id(), *ptr);
return hipSuccess;
}
projects/hip/src/hipamd/src/hip_intercept.cpp
Executable
+81
Fájl megtekintése
@@ -0,0 +1,81 @@
/* Copyright (c) 2019-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include "hip/hip_runtime.h"
#include "hip_internal.hpp"
#include "hip_platform.hpp"
#include "hip_prof_api.h"
// HIP API callback/activity
api_callbacks_table_t callbacks_table;
extern const std::string& FunctionName(const hipFunction_t f);
const char* hipKernelNameRef(const hipFunction_t f) { return FunctionName(f).c_str(); }
int hipGetStreamDeviceId(hipStream_t stream) {
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
return (s != nullptr)? s->DeviceId() : ihipGetDevice();
}
const char* hipKernelNameRefByPtr(const void* hostFunction, hipStream_t stream) {
if (hostFunction == NULL) {
return NULL;
}
int deviceId = hipGetStreamDeviceId(stream);
if (deviceId == -1) {
LogPrintfError("Wrong Device Id: %d \n", deviceId);
return NULL;
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId);
if (hip_error != hipSuccess) {
return NULL;
}
return hipKernelNameRef(func);
}
hipError_t hipRegisterApiCallback(uint32_t id, void* fun, void* arg) {
return callbacks_table.set_callback(id, reinterpret_cast<api_callbacks_table_t::fun_t>(fun), arg) ?
hipSuccess : hipErrorInvalidValue;
}
hipError_t hipRemoveApiCallback(uint32_t id) {
return callbacks_table.set_callback(id, NULL, NULL) ? hipSuccess : hipErrorInvalidValue;
}
hipError_t hipRegisterActivityCallback(uint32_t id, void* fun, void* arg) {
return callbacks_table.set_activity(id, reinterpret_cast<api_callbacks_table_t::act_t>(fun), arg) ?
hipSuccess : hipErrorInvalidValue;
}
hipError_t hipRemoveActivityCallback(uint32_t id) {
return callbacks_table.set_activity(id, NULL, NULL) ? hipSuccess : hipErrorInvalidValue;
}
hipError_t hipEnableTracing(bool enabled) {
callbacks_table.set_enabled(enabled);
return hipSuccess;
}
const char* hipApiName(uint32_t id) {
return hip_api_name(id);
}
projects/hip/src/hipamd/src/hip_internal.hpp
Executable
+353
Fájl megtekintése
@@ -0,0 +1,353 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef HIP_SRC_HIP_INTERNAL_H
#define HIP_SRC_HIP_INTERNAL_H
#include "vdi_common.hpp"
#include "hip_prof_api.h"
#include "trace_helper.h"
#include "utils/debug.hpp"
#include "hip_formatting.hpp"
#include "hip_graph_capture.hpp"
#include <unordered_set>
#include <thread>
#include <stack>
#include <mutex>
#include <iterator>
#ifdef _WIN32
#include <process.h>
#else
#include <unistd.h>
#endif
#define KNRM "\x1B[0m"
#define KRED "\x1B[31m"
#define KGRN "\x1B[32m"
#define KYEL "\x1B[33m"
#define KBLU "\x1B[34m"
#define KMAG "\x1B[35m"
#define KCYN "\x1B[36m"
#define KWHT "\x1B[37m"
/*! IHIP IPC MEMORY Structure */
#define IHIP_IPC_MEM_HANDLE_SIZE 32
#define IHIP_IPC_MEM_RESERVED_SIZE LP64_SWITCH(24,16)
typedef struct ihipIpcMemHandle_st {
char ipc_handle[IHIP_IPC_MEM_HANDLE_SIZE]; ///< ipc memory handle on ROCr
size_t psize;
size_t poffset;
char reserved[IHIP_IPC_MEM_RESERVED_SIZE];
} ihipIpcMemHandle_t;
#define IHIP_IPC_EVENT_HANDLE_SIZE 32
#define IHIP_IPC_EVENT_RESERVED_SIZE LP64_SWITCH(28,24)
typedef struct ihipIpcEventHandle_st {
//hsa_amd_ipc_signal_t ipc_handle; ///< ipc signal handle on ROCr
//char ipc_handle[IHIP_IPC_EVENT_HANDLE_SIZE];
//char reserved[IHIP_IPC_EVENT_RESERVED_SIZE];
char shmem_name[IHIP_IPC_EVENT_HANDLE_SIZE];
}ihipIpcEventHandle_t;
#ifdef _WIN32
inline int getpid() { return _getpid(); }
#endif
#define HIP_INIT() \
std::call_once(hip::g_ihipInitialized, hip::init); \
if (hip::g_device == nullptr && g_devices.size() > 0) { \
hip::g_device = g_devices[0]; \
}
#define HIP_API_PRINT(...) \
uint64_t startTimeUs=0 ; HIPPrintDuration(amd::LOG_INFO, amd::LOG_API, &startTimeUs, "%-5d: [%zx] %s%s ( %s )%s", getpid(), std::this_thread::get_id(), KGRN, \
__func__, ToString( __VA_ARGS__ ).c_str(),KNRM);
#define HIP_ERROR_PRINT(err, ...) \
ClPrint(amd::LOG_INFO, amd::LOG_API, "%-5d: [%zx] %s: Returned %s : %s", getpid(), std::this_thread::get_id(), \
__func__, hipGetErrorName(err), ToString( __VA_ARGS__ ).c_str());
// This macro should be called at the beginning of every HIP API.
#define HIP_INIT_API(cid, ...) \
HIP_API_PRINT(__VA_ARGS__) \
amd::Thread* thread = amd::Thread::current(); \
if (!VDI_CHECK_THREAD(thread)) { \
HIP_RETURN(hipErrorOutOfMemory); \
} \
HIP_INIT() \
HIP_CB_SPAWNER_OBJECT(cid);
#define HIP_RETURN_DURATION(ret, ...) \
hip::g_lastError = ret; \
HIPPrintDuration(amd::LOG_INFO, amd::LOG_API, &startTimeUs, "%-5d: [%zx] %s: Returned %s : %s", getpid(), std::this_thread::get_id(), \
__func__, hipGetErrorName(hip::g_lastError), ToString( __VA_ARGS__ ).c_str()); \
return hip::g_lastError;
#define HIP_RETURN(ret, ...) \
hip::g_lastError = ret; \
HIP_ERROR_PRINT(hip::g_lastError, __VA_ARGS__) \
return hip::g_lastError;
#define HIP_RETURN_ONFAIL(func) \
do { \
hipError_t herror = (func); \
if (herror != hipSuccess) { \
HIP_RETURN(herror); \
} \
} while (0);
// Cannot be use in place of HIP_RETURN.
// Refrain from using for external HIP APIs
#define IHIP_RETURN_ONFAIL(func) \
do { \
hipError_t herror = (func); \
if (herror != hipSuccess) { \
return herror; \
} \
} while (0);
#define STREAM_CAPTURE(name, stream, ...) \
if (stream != nullptr && \
reinterpret_cast<hip::Stream*>(stream)->GetCaptureStatus() == \
hipStreamCaptureStatusActive) { \
hipError_t status = capture##name(stream, ##__VA_ARGS__); \
HIP_RETURN(status); \
}
#define EVENT_CAPTURE(name, event, ...) \
if (event != nullptr && reinterpret_cast<hip::Event*>(event)->GetCaptureStatus() == true) { \
hipError_t status = capture##name(event, ##__VA_ARGS__); \
HIP_RETURN(status); \
}
namespace hc {
class accelerator;
class accelerator_view;
};
namespace hip {
class Device;
class Stream {
public:
enum Priority : int { High = -1, Normal = 0, Low = 1 };
private:
amd::HostQueue* queue_;
mutable amd::Monitor lock_;
Device* device_;
Priority priority_;
unsigned int flags_;
bool null_;
const std::vector<uint32_t> cuMask_;
/// Stream capture related parameters
/// Current capture status of the stream
hipStreamCaptureStatus captureStatus_;
/// Graph that is constructed with capture
hipGraph_t pCaptureGraph_;
/// Based on mode stream capture places restrictions on API calls that can be made within or
/// concurrently
hipStreamCaptureMode captureMode_;
bool originStream_;
/// Origin sream has no parent. Parent stream for the derived captured streams with event
/// dependencies
hipStream_t parentStream_;
/// Last graph node captured in the stream
std::vector<hipGraphNode_t> lastCapturedNodes_;
/// Derived streams/Paralell branches from the origin stream
std::vector<hipStream_t> parallelCaptureStreams_;
/// Capture events
std::vector<hipEvent_t> captureEvents_;
public:
Stream(Device* dev, Priority p = Priority::Normal, unsigned int f = 0, bool null_stream = false,
const std::vector<uint32_t>& cuMask = {},
hipStreamCaptureStatus captureStatus = hipStreamCaptureStatusNone);
~Stream();
/// Creates the hip stream object, including AMD host queue
bool Create();
/// Get device AMD host queue object. The method can allocate the queue
amd::HostQueue* asHostQueue(bool skip_alloc = false);
void Finish() const;
/// Get device ID associated with the current stream;
int DeviceId() const;
/// Get device ID associated with a stream;
static int DeviceId(const hipStream_t hStream);
/// Returns if stream is null stream
bool Null() const { return null_; }
/// Returns the lock object for the current stream
amd::Monitor& Lock() const { return lock_; }
/// Returns the creation flags for the current stream
unsigned int Flags() const { return flags_; }
/// Returns the priority for the current stream
Priority GetPriority() const { return priority_; }
/// Returns the CU mask for the current stream
const std::vector<uint32_t> GetCUMask() const { return cuMask_; }
/// Sync all non-blocking streams
static void syncNonBlockingStreams();
/// Returns capture status of the current stream
hipStreamCaptureStatus GetCaptureStatus() const { return captureStatus_; }
/// Returns capture mode of the current stream
hipStreamCaptureMode GetCaptureMode() const { return captureMode_; }
/// Returns if stream is origin stream
bool IsOriginStream() const { return originStream_; }
void SetOriginStream() { originStream_ = true; }
/// Returns captured graph
hipGraph_t GetCaptureGraph() const { return pCaptureGraph_; }
/// Returns last captured graph node
std::vector<hipGraphNode_t> GetLastCapturedNodes() const { return lastCapturedNodes_; }
/// Set last captured graph node
void SetLastCapturedNode(hipGraphNode_t graphNode) {
lastCapturedNodes_.clear();
lastCapturedNodes_.push_back(graphNode);
}
/// Append captured node via the wait event cross stream
void AddCrossCapturedNode(std::vector<hipGraphNode_t> graphNodes) {
for (auto node : graphNodes) {
lastCapturedNodes_.push_back(node);
}
}
/// Set graph that is being captured
void SetCaptureGraph(hipGraph_t pGraph) {
pCaptureGraph_ = pGraph;
captureStatus_ = hipStreamCaptureStatusActive;
}
/// reset capture parameters
hipError_t EndCapture();
/// Set capture status
void SetCaptureStatus(hipStreamCaptureStatus captureStatus) { captureStatus_ = captureStatus; }
/// Set capture mode
void SetCaptureMode(hipStreamCaptureMode captureMode) { captureMode_ = captureMode; }
/// Set parent stream
void SetParentStream(hipStream_t parentStream) { parentStream_ = parentStream; }
/// Get parent stream
hipStream_t GetParentStream() { return parentStream_; }
};
/// HIP Device class
class Device {
amd::Monitor lock_{"Device lock"};
/// ROCclr context
amd::Context* context_;
/// Device's ID
/// Store it here so we don't have to loop through the device list every time
int deviceId_;
/// ROCclr host queue for default streams
Stream null_stream_;
/// Store device flags
unsigned int flags_;
/// Maintain list of user enabled peers
std::list<int> userEnabledPeers;
public:
Device(amd::Context* ctx, int devId):
context_(ctx), deviceId_(devId), null_stream_(this, Stream::Priority::Normal, 0, true), flags_(hipDeviceScheduleSpin)
{ assert(ctx != nullptr); }
~Device() {}
amd::Context* asContext() const { return context_; }
int deviceId() const { return deviceId_; }
void retain() const { context_->retain(); }
void release() const { context_->release(); }
const std::vector<amd::Device*>& devices() const { return context_->devices(); }
hipError_t EnablePeerAccess(int peerDeviceId){
amd::ScopedLock lock(lock_);
bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
if (found) {
return hipErrorPeerAccessAlreadyEnabled;
}
userEnabledPeers.push_back(peerDeviceId);
return hipSuccess;
}
hipError_t DisablePeerAccess(int peerDeviceId) {
amd::ScopedLock lock(lock_);
bool found = (std::find(userEnabledPeers.begin(), userEnabledPeers.end(), peerDeviceId) != userEnabledPeers.end());
if (found) {
userEnabledPeers.remove(peerDeviceId);
return hipSuccess;
} else {
return hipErrorPeerAccessNotEnabled;
}
}
unsigned int getFlags() const { return flags_; }
void setFlags(unsigned int flags) { flags_ = flags; }
amd::HostQueue* NullStream(bool skip_alloc = false);
};
extern std::once_flag g_ihipInitialized;
/// Current thread's device
extern thread_local Device* g_device;
extern thread_local hipError_t g_lastError;
/// Device representing the host - for pinned memory
extern Device* host_device;
extern void init();
extern Device* getCurrentDevice();
extern void setCurrentDevice(unsigned int index);
/// Get ROCclr queue associated with hipStream
/// Note: This follows the CUDA spec to sync with default streams
/// and Blocking streams
extern amd::HostQueue* getQueue(hipStream_t s);
/// Get default stream associated with the ROCclr context
extern amd::HostQueue* getNullStream(amd::Context&);
/// Get default stream of the thread
extern amd::HostQueue* getNullStream();
/// Check if stream is valid
extern bool isValid(hipStream_t stream);
};
struct ihipExec_t {
dim3 gridDim_;
dim3 blockDim_;
size_t sharedMem_;
hipStream_t hStream_;
std::vector<char> arguments_;
};
/// Wait all active streams on the blocking queue. The method enqueues a wait command and
/// doesn't stall the current thread
extern void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream = false);
extern std::vector<hip::Device*> g_devices;
extern hipError_t ihipDeviceGetCount(int* count);
extern int ihipGetDevice();
extern hipError_t ihipMalloc(void** ptr, size_t sizeBytes, unsigned int flags);
extern amd::Memory* getMemoryObject(const void* ptr, size_t& offset);
extern amd::Memory* getMemoryObjectWithOffset(const void* ptr, const size_t size);
constexpr bool kOptionChangeable = true;
constexpr bool kNewDevProg = false;
constexpr bool kMarkerDisableFlush = true; //!< Avoids command batch flush in ROCclr
#endif // HIP_SRC_HIP_INTERNAL_H
projects/hip/src/hipamd/src/hip_memory.cpp
Executable
+2852
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/src/hip_module.cpp
Executable
+675
Fájl megtekintése
@@ -0,0 +1,675 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include <elf/elf.hpp>
#include <fstream>
#include "hip_internal.hpp"
#include "platform/program.hpp"
#include "hip_event.hpp"
#include "hip_platform.hpp"
hipError_t ihipModuleLoadData(hipModule_t* module, const void* mmap_ptr, size_t mmap_size);
extern hipError_t ihipLaunchKernel(const void* hostFunction,
dim3 gridDim,
dim3 blockDim,
void** args,
size_t sharedMemBytes,
hipStream_t stream,
hipEvent_t startEvent,
hipEvent_t stopEvent,
int flags);
const std::string& FunctionName(const hipFunction_t f) {
return hip::DeviceFunc::asFunction(f)->kernel()->name();
}
static uint64_t ElfSize(const void *emi)
{
return amd::Elf::getElfSize(emi);
}
hipError_t hipModuleUnload(hipModule_t hmod) {
HIP_INIT_API(hipModuleUnload, hmod);
HIP_RETURN(PlatformState::instance().unloadModule(hmod));
}
hipError_t hipModuleLoad(hipModule_t* module, const char* fname) {
HIP_INIT_API(hipModuleLoad, module, fname);
HIP_RETURN(PlatformState::instance().loadModule(module, fname));
}
hipError_t hipModuleLoadData(hipModule_t *module, const void *image)
{
HIP_INIT_API(hipModuleLoadData, module, image);
HIP_RETURN(PlatformState::instance().loadModule(module, 0, image));
}
hipError_t hipModuleLoadDataEx(hipModule_t *module, const void *image,
unsigned int numOptions, hipJitOption* options,
void** optionsValues)
{
/* TODO: Pass options to Program */
HIP_INIT_API(hipModuleLoadDataEx, module, image);
HIP_RETURN(PlatformState::instance().loadModule(module, 0, image));
}
extern hipError_t __hipExtractCodeObjectFromFatBinary(const void* data,
const std::vector<std::string>& devices,
std::vector<std::pair<const void*, size_t>>& code_objs);
hipError_t hipModuleGetFunction(hipFunction_t *hfunc, hipModule_t hmod, const char *name) {
HIP_INIT_API(hipModuleGetFunction, hfunc, hmod, name);
if(hfunc == nullptr || name == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if (hipSuccess != PlatformState::instance().getDynFunc(hfunc, hmod, name)) {
LogPrintfError("Cannot find the function: %s for module: 0x%x \n", name, hmod);
HIP_RETURN(hipErrorNotFound);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes, hipModule_t hmod, const char* name)
{
HIP_INIT_API(hipModuleGetGlobal, dptr, bytes, hmod, name);
if(dptr == nullptr || bytes == nullptr || name == nullptr) {
return hipErrorInvalidValue;
}
/* Get address and size for the global symbol */
if (hipSuccess != PlatformState::instance().getDynGlobalVar(name, hmod, dptr, bytes)) {
LogPrintfError("Cannot find global Var: %s for module: 0x%x at device: %d \n", name, hmod,
ihipGetDevice());
HIP_RETURN(hipErrorNotFound);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipFuncGetAttribute(int* value, hipFunction_attribute attrib, hipFunction_t hfunc) {
HIP_INIT_API(hipFuncGetAttribute, value, attrib, hfunc);
if ((value == nullptr) || (hfunc == nullptr)) {
HIP_RETURN(hipErrorInvalidValue);
}
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(hfunc);
if (function == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
amd::Kernel* kernel = function->kernel();
if (kernel == nullptr) {
HIP_RETURN(hipErrorInvalidDeviceFunction);
}
const device::Kernel::WorkGroupInfo* wrkGrpInfo
= kernel->getDeviceKernel(*(hip::getCurrentDevice()->devices()[0]))->workGroupInfo();
if (wrkGrpInfo == nullptr) {
HIP_RETURN(hipErrorMissingConfiguration);
}
switch(attrib) {
case HIP_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES:
*value = static_cast<int>(wrkGrpInfo->localMemSize_);
break;
case HIP_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK:
*value = static_cast<int>(wrkGrpInfo->size_);
break;
case HIP_FUNC_ATTRIBUTE_CONST_SIZE_BYTES:
*value = 0;
break;
case HIP_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES:
*value = static_cast<int>(wrkGrpInfo->privateMemSize_);
break;
case HIP_FUNC_ATTRIBUTE_NUM_REGS:
*value = static_cast<int>(wrkGrpInfo->usedVGPRs_);
break;
case HIP_FUNC_ATTRIBUTE_PTX_VERSION:
*value = 30; // Defaults to 3.0 as HCC
break;
case HIP_FUNC_ATTRIBUTE_BINARY_VERSION:
*value = static_cast<int>(kernel->signature().version());
break;
case HIP_FUNC_ATTRIBUTE_CACHE_MODE_CA:
*value = 0;
break;
case HIP_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES:
*value = static_cast<int>(wrkGrpInfo->availableLDSSize_ - wrkGrpInfo->localMemSize_);
break;
case HIP_FUNC_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT:
*value = 0;
break;
default:
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipFuncGetAttributes(hipFuncAttributes* attr, const void* func)
{
HIP_INIT_API(hipFuncGetAttributes, attr, func);
HIP_RETURN_ONFAIL(PlatformState::instance().getStatFuncAttr(attr, func, ihipGetDevice()));
HIP_RETURN(hipSuccess);
}
hipError_t hipFuncSetAttribute ( const void* func, hipFuncAttribute attr, int value ) {
HIP_INIT_API(hipFuncSetAttribute, func, attr, value);
// No way to set function attribute yet.
HIP_RETURN(hipSuccess);
}
hipError_t hipFuncSetCacheConfig (const void* func, hipFuncCache_t cacheConfig) {
HIP_INIT_API(hipFuncSetCacheConfig, cacheConfig);
// No way to set cache config yet.
HIP_RETURN(hipSuccess);
}
hipError_t hipFuncSetSharedMemConfig ( const void* func, hipSharedMemConfig config) {
HIP_INIT_API(hipFuncSetSharedMemConfig, func, config);
// No way to set Shared Memory config function yet.
HIP_RETURN(hipSuccess);
}
hipError_t ihipLaunchKernel_validate(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, uint32_t sharedMemBytes,
void** kernelParams, void** extra, int deviceId,
uint32_t params = 0) {
if (f == nullptr) {
LogPrintfError("%s", "Function passed is null");
return hipErrorInvalidImage;
}
if ((kernelParams != nullptr) && (extra != nullptr)) {
LogPrintfError("%s",
"Both, kernelParams and extra Params are provided, only one should be provided");
return hipErrorInvalidValue;
}
if (globalWorkSizeX == 0 || globalWorkSizeY == 0 || globalWorkSizeZ == 0 || blockDimX == 0 ||
blockDimY == 0 || blockDimZ == 0) {
return hipErrorInvalidValue;
}
if (extra != nullptr) {
if (extra[0] != HIP_LAUNCH_PARAM_BUFFER_POINTER || extra[2] != HIP_LAUNCH_PARAM_BUFFER_SIZE ||
extra[4] != HIP_LAUNCH_PARAM_END) {
return hipErrorNotInitialized;
}
}
const amd::Device* device = g_devices[deviceId]->devices()[0];
// Make sure dispatch doesn't exceed max workgroup size limit
if (blockDimX * blockDimY * blockDimZ > device->info().maxWorkGroupSize_) {
return hipErrorInvalidConfiguration;
}
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f);
amd::Kernel* kernel = function->kernel();
// Make sure the launch params are not larger than if specified launch_bounds
// If it exceeds, then return a failure
if (blockDimX * blockDimY * blockDimZ >
kernel->getDeviceKernel(*device)->workGroupInfo()->size_) {
LogPrintfError("Launch params (%u, %u, %u) are larger than launch bounds (%lu) for kernel %s",
blockDimX, blockDimY, blockDimZ,
kernel->getDeviceKernel(*device)->workGroupInfo()->size_,
function->name().c_str());
return hipErrorLaunchFailure;
}
if (params & amd::NDRangeKernelCommand::CooperativeGroups) {
if (!device->info().cooperativeGroups_) {
return hipErrorLaunchFailure;
}
int num_blocks = 0;
int max_blocks_per_grid = 0;
int best_block_size = 0;
int block_size = blockDimX * blockDimY * blockDimZ;
hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(&num_blocks, &max_blocks_per_grid,
&best_block_size, *device, f,
block_size, sharedMemBytes, true);
if (((globalWorkSizeX * globalWorkSizeY * globalWorkSizeZ) / block_size) >
unsigned(max_blocks_per_grid)) {
return hipErrorCooperativeLaunchTooLarge;
}
}
if (params & amd::NDRangeKernelCommand::CooperativeMultiDeviceGroups) {
if (!device->info().cooperativeMultiDeviceGroups_) {
return hipErrorLaunchFailure;
}
}
address kernargs = nullptr;
// 'extra' is a struct that contains the following info: {
// HIP_LAUNCH_PARAM_BUFFER_POINTER, kernargs,
// HIP_LAUNCH_PARAM_BUFFER_SIZE, &kernargs_size,
// HIP_LAUNCH_PARAM_END }
if (extra != nullptr) {
if (extra[0] != HIP_LAUNCH_PARAM_BUFFER_POINTER || extra[2] != HIP_LAUNCH_PARAM_BUFFER_SIZE ||
extra[4] != HIP_LAUNCH_PARAM_END) {
return hipErrorNotInitialized;
}
kernargs = reinterpret_cast<address>(extra[1]);
}
const amd::KernelSignature& signature = kernel->signature();
for (size_t i = 0; i < signature.numParameters(); ++i) {
const amd::KernelParameterDescriptor& desc = signature.at(i);
if (kernelParams == nullptr) {
assert(kernargs != nullptr);
kernel->parameters().set(i, desc.size_, kernargs + desc.offset_,
desc.type_ == T_POINTER /*svmBound*/);
} else {
assert(extra == nullptr);
kernel->parameters().set(i, desc.size_, kernelParams[i],
desc.type_ == T_POINTER /*svmBound*/);
}
}
return hipSuccess;
}
hipError_t ihipLaunchKernelCommand(amd::Command*& command, hipFunction_t f,
uint32_t globalWorkSizeX, uint32_t globalWorkSizeY,
uint32_t globalWorkSizeZ, uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, uint32_t sharedMemBytes,
amd::HostQueue* queue, void** kernelParams, void** extra,
hipEvent_t startEvent = nullptr, hipEvent_t stopEvent = nullptr,
uint32_t flags = 0, uint32_t params = 0, uint32_t gridId = 0,
uint32_t numGrids = 0, uint64_t prevGridSum = 0,
uint64_t allGridSum = 0, uint32_t firstDevice = 0) {
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f);
amd::Kernel* kernel = function->kernel();
size_t globalWorkOffset[3] = {0};
size_t globalWorkSize[3] = {globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ};
size_t localWorkSize[3] = {blockDimX, blockDimY, blockDimZ};
amd::NDRangeContainer ndrange(3, globalWorkOffset, globalWorkSize, localWorkSize);
amd::Command::EventWaitList waitList;
bool profileNDRange = false;
address kernargs = nullptr;
profileNDRange = (startEvent != nullptr || stopEvent != nullptr);
// Flag set to 1 signifies that kernel can be launched in anyorder
if (flags & hipExtAnyOrderLaunch) {
params |= amd::NDRangeKernelCommand::AnyOrderLaunch;
}
amd::NDRangeKernelCommand* kernelCommand = new amd::NDRangeKernelCommand(
*queue, waitList, *kernel, ndrange, sharedMemBytes, params, gridId, numGrids, prevGridSum,
allGridSum, firstDevice, profileNDRange);
if (!kernelCommand) {
return hipErrorOutOfMemory;
}
// Capture the kernel arguments
if (CL_SUCCESS != kernelCommand->captureAndValidate()) {
delete kernelCommand;
return hipErrorOutOfMemory;
}
command = kernelCommand;
return hipSuccess;
}
hipError_t ihipModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
uint32_t sharedMemBytes, hipStream_t hStream, void** kernelParams,
void** extra, hipEvent_t startEvent, hipEvent_t stopEvent,
uint32_t flags = 0, uint32_t params = 0, uint32_t gridId = 0,
uint32_t numGrids = 0, uint64_t prevGridSum = 0,
uint64_t allGridSum = 0, uint32_t firstDevice = 0) {
HIP_INIT_API(ihipModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ,
blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra,
startEvent, stopEvent, flags, params);
int deviceId = hip::Stream::DeviceId(hStream);
HIP_RETURN_ONFAIL(PlatformState::instance().initStatManagedVarDevicePtr(deviceId));
if (f == nullptr) {
LogPrintfError("%s", "Function passed is null");
return hipErrorInvalidImage;
}
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(f);
amd::Kernel* kernel = function->kernel();
amd::ScopedLock lock(function->dflock_);
hipError_t status =
ihipLaunchKernel_validate(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX,
blockDimY, blockDimZ, sharedMemBytes, kernelParams, extra, deviceId, params);
if (status != hipSuccess) {
return status;
}
amd::Command* command = nullptr;
amd::HostQueue* queue = hip::getQueue(hStream);
status = ihipLaunchKernelCommand(command, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ,
blockDimX, blockDimY, blockDimZ, sharedMemBytes, queue,
kernelParams, extra, startEvent, stopEvent, flags, params,
gridId, numGrids, prevGridSum, allGridSum, firstDevice);
if (status != hipSuccess) {
return status;
}
hip::Event* eStart = reinterpret_cast<hip::Event*>(startEvent);
hip::Event* eStop = reinterpret_cast<hip::Event*>(stopEvent);
command->enqueue();
if (startEvent != nullptr) {
eStart->addMarker(queue, command, false);
command->retain();
}
if (stopEvent != nullptr) {
eStop->addMarker(queue, command, true);
command->retain();
}
command->release();
return hipSuccess;
}
hipError_t hipModuleLaunchKernel(hipFunction_t f,
uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ,
uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
uint32_t sharedMemBytes, hipStream_t hStream,
void **kernelParams, void **extra)
{
HIP_INIT_API(hipModuleLaunchKernel, f, gridDimX, gridDimY, gridDimZ,
blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream,
kernelParams, extra);
size_t globalWorkSizeX = static_cast<size_t>(gridDimX) * blockDimX;
size_t globalWorkSizeY = static_cast<size_t>(gridDimY) * blockDimY;
size_t globalWorkSizeZ = static_cast<size_t>(gridDimZ) * blockDimZ;
if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
HIP_RETURN(ihipModuleLaunchKernel(f, static_cast<uint32_t>(globalWorkSizeX),
static_cast<uint32_t>(globalWorkSizeY),
static_cast<uint32_t>(globalWorkSizeZ),
blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream, kernelParams, extra, nullptr, nullptr));
}
hipError_t hipExtModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t localWorkSizeX, uint32_t localWorkSizeY,
uint32_t localWorkSizeZ, size_t sharedMemBytes,
hipStream_t hStream, void** kernelParams, void** extra,
hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags)
{
HIP_INIT_API(hipExtModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ,
localWorkSizeX, localWorkSizeY, localWorkSizeZ,
sharedMemBytes, hStream,
kernelParams, extra, startEvent, stopEvent, flags);
HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, localWorkSizeX, localWorkSizeY,
localWorkSizeZ, sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent, flags));
}
hipError_t hipHccModuleLaunchKernel(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, size_t sharedMemBytes,
hipStream_t hStream, void** kernelParams, void** extra,
hipEvent_t startEvent,
hipEvent_t stopEvent)
{
HIP_INIT_API(hipHccModuleLaunchKernel, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ,
blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream,
kernelParams, extra, startEvent, stopEvent);
HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent));
}
hipError_t hipModuleLaunchKernelExt(hipFunction_t f, uint32_t globalWorkSizeX,
uint32_t globalWorkSizeY, uint32_t globalWorkSizeZ,
uint32_t blockDimX, uint32_t blockDimY,
uint32_t blockDimZ, size_t sharedMemBytes,
hipStream_t hStream, void** kernelParams, void** extra,
hipEvent_t startEvent,
hipEvent_t stopEvent)
{
HIP_INIT_API(hipModuleLaunchKernelExt, f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ,
blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream,
kernelParams, extra, startEvent, stopEvent);
HIP_RETURN(ihipModuleLaunchKernel(f, globalWorkSizeX, globalWorkSizeY, globalWorkSizeZ, blockDimX, blockDimY, blockDimZ,
sharedMemBytes, hStream, kernelParams, extra, startEvent, stopEvent));
}
extern "C" hipError_t hipLaunchKernel(const void *hostFunction,
dim3 gridDim,
dim3 blockDim,
void** args,
size_t sharedMemBytes,
hipStream_t stream)
{
HIP_INIT_API(hipLaunchKernel, hostFunction, gridDim, blockDim, args, sharedMemBytes, stream);
STREAM_CAPTURE(hipLaunchKernel, stream, hostFunction, gridDim, blockDim, args, sharedMemBytes);
HIP_RETURN(ihipLaunchKernel(hostFunction, gridDim, blockDim, args, sharedMemBytes, stream,
nullptr, nullptr, 0));
}
extern "C" hipError_t hipExtLaunchKernel(const void* hostFunction,
dim3 gridDim,
dim3 blockDim,
void** args,
size_t sharedMemBytes,
hipStream_t stream,
hipEvent_t startEvent,
hipEvent_t stopEvent,
int flags)
{
HIP_INIT_API(hipExtLaunchKernel, hostFunction, gridDim, blockDim, args, sharedMemBytes, stream);
HIP_RETURN(ihipLaunchKernel(hostFunction, gridDim, blockDim, args, sharedMemBytes, stream, startEvent, stopEvent, flags));
}
hipError_t hipLaunchCooperativeKernel(const void* f,
dim3 gridDim, dim3 blockDim,
void **kernelParams, uint32_t sharedMemBytes, hipStream_t hStream)
{
HIP_INIT_API(hipLaunchCooperativeKernel, f, gridDim, blockDim,
sharedMemBytes, hStream);
hipFunction_t func = nullptr;
int deviceId = hip::Stream::DeviceId(hStream);
HIP_RETURN_ONFAIL(PlatformState::instance().getStatFunc(&func, f, deviceId));
size_t globalWorkSizeX = static_cast<size_t>(gridDim.x) * blockDim.x;
size_t globalWorkSizeY = static_cast<size_t>(gridDim.y) * blockDim.y;
size_t globalWorkSizeZ = static_cast<size_t>(gridDim.z) * blockDim.z;
if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
HIP_RETURN(ihipModuleLaunchKernel(func, static_cast<uint32_t>(globalWorkSizeX),
static_cast<uint32_t>(globalWorkSizeY),
static_cast<uint32_t>(globalWorkSizeZ),
blockDim.x, blockDim.y, blockDim.z,
sharedMemBytes, hStream, kernelParams, nullptr, nullptr, nullptr, 0,
amd::NDRangeKernelCommand::CooperativeGroups));
}
hipError_t ihipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices, unsigned int flags, uint32_t extFlags)
{
int numActiveGPUs = 0;
ihipDeviceGetCount(&numActiveGPUs);
if ((numDevices > numActiveGPUs) || (launchParamsList == nullptr)) {
return hipErrorInvalidValue;
}
hipError_t result = hipErrorUnknown;
uint64_t allGridSize = 0;
uint32_t blockDims = 0;
std::vector<const amd::Device*> mgpu_list(numDevices);
for (int i = 0; i < numDevices; ++i) {
const hipLaunchParams& launch = launchParamsList[i];
blockDims = launch.blockDim.x * launch.blockDim.y * launch.blockDim.z;
allGridSize += launch.gridDim.x * launch.gridDim.y * launch.gridDim.z *
blockDims;
// Make sure block dimensions are valid
if (0 == blockDims) {
return hipErrorInvalidConfiguration;
}
if (launch.stream != nullptr) {
// Validate devices to make sure it dosn't have duplicates
amd::HostQueue* queue = reinterpret_cast<hip::Stream*>(launch.stream)->asHostQueue();
auto device = &queue->vdev()->device();
for (int j = 0; j < numDevices; ++j) {
if (mgpu_list[j] == device) {
return hipErrorInvalidDevice;
}
}
mgpu_list[i] = device;
} else {
return hipErrorInvalidResourceHandle;
}
}
uint64_t prevGridSize = 0;
uint32_t firstDevice = 0;
// Sync the execution streams on all devices
if ((flags & hipCooperativeLaunchMultiDeviceNoPreSync) == 0) {
for (int i = 0; i < numDevices; ++i) {
amd::HostQueue* queue =
reinterpret_cast<hip::Stream*>(launchParamsList[i].stream)->asHostQueue();
queue->finish();
}
}
for (int i = 0; i < numDevices; ++i) {
const hipLaunchParams& launch = launchParamsList[i];
amd::HostQueue* queue = reinterpret_cast<hip::Stream*>(launch.stream)->asHostQueue();
hipFunction_t func = nullptr;
// The order of devices in the launch may not match the order in the global array
for (size_t dev = 0; dev < g_devices.size(); ++dev) {
// Find the matching device and request the kernel function
if (&queue->vdev()->device() == g_devices[dev]->devices()[0]) {
IHIP_RETURN_ONFAIL(PlatformState::instance().getStatFunc(&func, launch.func, dev));
// Save ROCclr index of the first device in the launch
if (i == 0) {
firstDevice = queue->vdev()->device().index();
}
break;
}
}
if (func == nullptr) {
result = hipErrorInvalidDeviceFunction;
HIP_RETURN(result);
}
size_t globalWorkSizeX = static_cast<size_t>(launch.gridDim.x) * launch.blockDim.x;
size_t globalWorkSizeY = static_cast<size_t>(launch.gridDim.y) * launch.blockDim.y;
size_t globalWorkSizeZ = static_cast<size_t>(launch.gridDim.z) * launch.blockDim.z;
if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
result = ihipModuleLaunchKernel(func, static_cast<uint32_t>(globalWorkSizeX),
static_cast<uint32_t>(globalWorkSizeY), static_cast<uint32_t>(globalWorkSizeZ),
launch.blockDim.x, launch.blockDim.y, launch.blockDim.z,
launch.sharedMem, launch.stream, launch.args, nullptr, nullptr, nullptr,
flags, extFlags, i, numDevices, prevGridSize, allGridSize, firstDevice);
if (result != hipSuccess) {
break;
}
prevGridSize += globalWorkSizeX * globalWorkSizeY * globalWorkSizeZ;
}
// Sync the execution streams on all devices
if ((flags & hipCooperativeLaunchMultiDeviceNoPostSync) == 0) {
for (int i = 0; i < numDevices; ++i) {
amd::HostQueue* queue =
reinterpret_cast<hip::Stream*>(launchParamsList[i].stream)->asHostQueue();
queue->finish();
}
}
return result;
}
hipError_t hipLaunchCooperativeKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices, unsigned int flags)
{
HIP_INIT_API(hipLaunchCooperativeKernelMultiDevice, launchParamsList, numDevices, flags);
return HIP_RETURN(ihipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags,
(amd::NDRangeKernelCommand::CooperativeGroups |
amd::NDRangeKernelCommand::CooperativeMultiDeviceGroups)));
}
hipError_t hipExtLaunchMultiKernelMultiDevice(hipLaunchParams* launchParamsList,
int numDevices, unsigned int flags) {
HIP_INIT_API(hipExtLaunchMultiKernelMultiDevice, launchParamsList, numDevices, flags);
return HIP_RETURN(ihipLaunchCooperativeKernelMultiDevice(launchParamsList, numDevices, flags, 0));
}
hipError_t hipModuleGetTexRef(textureReference** texRef, hipModule_t hmod, const char* name) {
HIP_INIT_API(hipModuleGetTexRef, texRef, hmod, name);
/* input args check */
if ((texRef == nullptr) || (name == nullptr)) {
HIP_RETURN(hipErrorInvalidValue);
}
/* Get address and size for the global symbol */
if (hipSuccess != PlatformState::instance().getDynTexRef(name, hmod, texRef)) {
LogPrintfError("Cannot get texRef for name: %s at module:0x%x \n", name, hmod);
HIP_RETURN(hipErrorNotFound);
}
// Texture references created by HIP driver API
// have the default read mode set to normalized float.
(*texRef)->readMode = hipReadModeNormalizedFloat;
PlatformState::instance().registerTexRef(*texRef, hmod, std::string(name));
HIP_RETURN(hipSuccess);
}
projects/hip/src/hipamd/src/hip_peer.cpp
Executable
+250
Fájl megtekintése
@@ -0,0 +1,250 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
hipError_t hipDeviceCanAccessPeer(int* canAccessPeer, hipCtx_t thisCtx, hipCtx_t peerCtx) {
HIP_INIT_API(NONE, canAccessPeer, thisCtx, peerCtx);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipMemcpyPeer(void* dst, hipCtx_t dstCtx, const void* src, hipCtx_t srcCtx,
size_t sizeBytes) {
HIP_INIT_API(NONE, dst, dstCtx, src, srcCtx, sizeBytes);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipMemcpyPeerAsync(void* dst, hipCtx_t dstDevice, const void* src, hipCtx_t srcDevice,
size_t sizeBytes, hipStream_t stream) {
HIP_INIT_API(NONE, dst, dstDevice, src, srcDevice, sizeBytes, stream);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t canAccessPeer(int* canAccessPeer, int deviceId, int peerDeviceId){
amd::Device* device = nullptr;
amd::Device* peer_device = nullptr;
if (canAccessPeer == nullptr) {
return hipErrorInvalidValue;
}
/* Peer cannot be self */
if (deviceId == peerDeviceId) {
*canAccessPeer = 0;
return hipSuccess;
}
/* Cannot exceed the max number of devices */
if (static_cast<size_t>(deviceId) >= g_devices.size()
|| static_cast<size_t>(peerDeviceId) >= g_devices.size()) {
return hipErrorInvalidDevice;
}
device = g_devices[deviceId]->devices()[0];
peer_device = g_devices[peerDeviceId]->devices()[0];
*canAccessPeer = static_cast<int>(std::find(device->p2pDevices_.begin(),
device->p2pDevices_.end(), as_cl(peer_device))
!= device->p2pDevices_.end());
return hipSuccess;
}
hipError_t findLinkInfo(int device1, int device2,
std::vector<amd::Device::LinkAttrType>* link_attrs) {
amd::Device* amd_dev_obj1 = nullptr;
amd::Device* amd_dev_obj2 = nullptr;
const int numDevices = static_cast<int>(g_devices.size());
if ((device1 < 0) || (device1 >= numDevices) || (device2 < 0) || (device2 >= numDevices)) {
return hipErrorInvalidDevice;
}
amd_dev_obj1 = g_devices[device1]->devices()[0];
amd_dev_obj2 = g_devices[device2]->devices()[0];
if (!amd_dev_obj1->findLinkInfo(*amd_dev_obj2, link_attrs)) {
return hipErrorInvalidHandle;
}
return hipSuccess;
}
hipError_t hipExtGetLinkTypeAndHopCount(int device1, int device2,
uint32_t* linktype, uint32_t* hopcount) {
HIP_INIT_API(hipExtGetLinkTypeAndHopCount, device1, device2, linktype, hopcount);
if (linktype == nullptr || hopcount == nullptr ||
device1 == device2 || device1 < 0 || device2 < 0) {
HIP_RETURN(hipErrorInvalidValue);
}
// Fill out the list of LinkAttributes
std::vector<amd::Device::LinkAttrType> link_attrs;
link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkHopCount, 0));
HIP_RETURN_ONFAIL(findLinkInfo(device1, device2, &link_attrs));
*linktype = static_cast<uint32_t>(link_attrs[0].second);
*hopcount = static_cast<uint32_t>(link_attrs[1].second);
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceGetP2PAttribute(int* value, hipDeviceP2PAttr attr,
int srcDevice, int dstDevice) {
HIP_INIT_API(hipDeviceGetP2PAttribute, value, attr, srcDevice, dstDevice);
if (value == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
if (srcDevice == dstDevice || srcDevice >= static_cast<int>(g_devices.size())
|| dstDevice >= static_cast<int>(g_devices.size())) {
HIP_RETURN(hipErrorInvalidDevice);
}
std::vector<amd::Device::LinkAttrType> link_attrs;
switch (attr) {
case hipDevP2PAttrPerformanceRank : {
link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
break;
}
case hipDevP2PAttrAccessSupported : {
HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice));
break;
}
case hipDevP2PAttrNativeAtomicSupported : {
link_attrs.push_back(std::make_pair(amd::Device::LinkAttribute::kLinkLinkType, 0));
break;
}
case hipDevP2PAttrHipArrayAccessSupported : {
hipDeviceProp_t srcDeviceProp;
hipDeviceProp_t dstDeviceProp;
HIP_RETURN_ONFAIL(hipGetDeviceProperties(&srcDeviceProp, srcDevice));
HIP_RETURN_ONFAIL(hipGetDeviceProperties(&dstDeviceProp, dstDevice));
// Linear layout access is supported if P2P is enabled
// Opaque Images are supported only on homogeneous systems
// Might have more conditions to check, in future.
if (srcDeviceProp.gcnArch == dstDeviceProp.gcnArch) {
HIP_RETURN_ONFAIL(canAccessPeer(value, srcDevice, dstDevice));
} else {
value = 0;
}
break;
}
default : {
LogPrintfError("Invalid attribute attr: %d ", attr);
HIP_RETURN(hipErrorInvalidValue);
break;
}
}
if ((attr != hipDevP2PAttrAccessSupported) && (attr != hipDevP2PAttrHipArrayAccessSupported)) {
HIP_RETURN_ONFAIL(findLinkInfo(srcDevice, dstDevice, &link_attrs));
*value = static_cast<int>(link_attrs[0].second);
}
HIP_RETURN(hipSuccess);
}
hipError_t hipDeviceCanAccessPeer(int* canAccess, int deviceId, int peerDeviceId) {
HIP_INIT_API(hipDeviceCanAccessPeer, canAccess, deviceId, peerDeviceId);
HIP_RETURN(canAccessPeer(canAccess, deviceId, peerDeviceId));
}
hipError_t hipDeviceDisablePeerAccess(int peerDeviceId) {
HIP_INIT_API(hipDeviceDisablePeerAccess, peerDeviceId);
int deviceId = hip::getCurrentDevice()->deviceId();
int canAccess = 0;
if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) {
HIP_RETURN(hipErrorInvalidDevice);
}
amd::Device* device = g_devices[deviceId]->devices()[0];
amd::Device* peer_device = g_devices[peerDeviceId]->devices()[0];
peer_device->disableP2P(device);
HIP_RETURN(hip::getCurrentDevice()->DisablePeerAccess(peerDeviceId));
}
hipError_t hipDeviceEnablePeerAccess(int peerDeviceId, unsigned int flags) {
HIP_INIT_API(hipDeviceEnablePeerAccess, peerDeviceId, flags);
int deviceId = hip::getCurrentDevice()->deviceId();
int canAccess = 0;
if (flags != 0) {
HIP_RETURN(hipErrorInvalidValue);
}
if ((hipSuccess != canAccessPeer(&canAccess, deviceId, peerDeviceId)) || (canAccess == 0)) {
HIP_RETURN(hipErrorInvalidDevice);
}
amd::Device* device = g_devices[deviceId]->asContext()->devices()[0];
amd::Device* peer_device = g_devices[peerDeviceId]->asContext()->devices()[0];
peer_device->enableP2P(device);
HIP_RETURN(hip::getCurrentDevice()->EnablePeerAccess(peerDeviceId));
}
hipError_t hipMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice,
size_t sizeBytes) {
HIP_INIT_API(hipMemcpyPeer, dst, dstDevice, src, srcDevice, sizeBytes);
if (srcDevice >= static_cast<int>(g_devices.size()) ||
dstDevice >= static_cast<int>(g_devices.size()) ||
srcDevice < 0 || dstDevice < 0) {
HIP_RETURN(hipErrorInvalidDevice);
}
HIP_RETURN(hipMemcpy(dst, src, sizeBytes, hipMemcpyDeviceToDevice));
}
hipError_t hipMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice,
size_t sizeBytes, hipStream_t stream) {
HIP_INIT_API(hipMemcpyPeerAsync, dst, dstDevice, src, srcDevice, sizeBytes, stream);
if (srcDevice >= static_cast<int>(g_devices.size()) ||
dstDevice >= static_cast<int>(g_devices.size()) ||
srcDevice < 0 || dstDevice < 0) {
HIP_RETURN(hipErrorInvalidDevice);
}
HIP_RETURN(hipMemcpyAsync(dst, src, sizeBytes, hipMemcpyDeviceToDevice, stream));
}
hipError_t hipCtxEnablePeerAccess(hipCtx_t peerCtx, unsigned int flags) {
HIP_INIT_API(hipCtxEnablePeerAccess, peerCtx, flags);
HIP_RETURN(hipSuccess);
}
hipError_t hipCtxDisablePeerAccess(hipCtx_t peerCtx) {
HIP_INIT_API(hipCtxDisablePeerAccess, peerCtx);
HIP_RETURN(hipSuccess);
}
projects/hip/src/hipamd/src/hip_platform.cpp
Executable
+945
Fájl megtekintése
@@ -0,0 +1,945 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include <hip/amd_detail/amd_texture_types.h>
#include "hip_platform.hpp"
#include "hip_internal.hpp"
#include "platform/program.hpp"
#include "platform/runtime.hpp"
#include <unordered_map>
constexpr unsigned __hipFatMAGIC2 = 0x48495046; // "HIPF"
thread_local std::stack<ihipExec_t> execStack_;
PlatformState* PlatformState::platform_; // Initiaized as nullptr by default
//forward declaration of methods required for __hipRegisrterManagedVar
hipError_t ihipMallocManaged(void** ptr, size_t size, unsigned int align = 0);
hipError_t ihipMemcpy(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind,
amd::HostQueue& queue, bool isAsync = false);
struct __CudaFatBinaryWrapper {
unsigned int magic;
unsigned int version;
void* binary;
void* dummy1;
};
hipError_t hipModuleGetGlobal(hipDeviceptr_t* dptr, size_t* bytes,
hipModule_t hmod, const char* name);
hipError_t ihipCreateGlobalVarObj(const char* name, hipModule_t hmod, amd::Memory** amd_mem_obj,
hipDeviceptr_t* dptr, size_t* bytes);
extern hipError_t ihipModuleLaunchKernel(hipFunction_t f,
uint32_t gridDimX, uint32_t gridDimY, uint32_t gridDimZ,
uint32_t blockDimX, uint32_t blockDimY, uint32_t blockDimZ,
uint32_t sharedMemBytes, hipStream_t hStream,
void **kernelParams, void **extra,
hipEvent_t startEvent, hipEvent_t stopEvent, uint32_t flags = 0,
uint32_t params = 0, uint32_t gridId = 0, uint32_t numGrids = 0,
uint64_t prevGridSum = 0, uint64_t allGridSum = 0, uint32_t firstDevice = 0);
static bool isCompatibleCodeObject(const std::string& codeobj_target_id,
const char* device_name) {
// Workaround for device name mismatch.
// Device name may contain feature strings delimited by '+', e.g.
// gfx900+xnack. Currently HIP-Clang does not include feature strings
// in code object target id in fat binary. Therefore drop the feature
// strings from device name before comparing it with code object target id.
std::string short_name(device_name);
auto feature_loc = short_name.find('+');
if (feature_loc != std::string::npos) {
short_name.erase(feature_loc);
}
return codeobj_target_id == short_name;
}
extern "C" hip::FatBinaryInfo** __hipRegisterFatBinary(const void* data)
{
const __CudaFatBinaryWrapper* fbwrapper = reinterpret_cast<const __CudaFatBinaryWrapper*>(data);
if (fbwrapper->magic != __hipFatMAGIC2 || fbwrapper->version != 1) {
LogPrintfError("Cannot Register fat binary. FatMagic: %u version: %u ", fbwrapper->magic,
fbwrapper->version);
return nullptr;
}
return PlatformState::instance().addFatBinary(fbwrapper->binary);
}
extern "C" void __hipRegisterFunction(
hip::FatBinaryInfo** modules,
const void* hostFunction,
char* deviceFunction,
const char* deviceName,
unsigned int threadLimit,
uint3* tid,
uint3* bid,
dim3* blockDim,
dim3* gridDim,
int* wSize) {
static int enable_deferred_loading { []() {
char *var = getenv("HIP_ENABLE_DEFERRED_LOADING");
return var ? atoi(var) : 1;
}() };
hip::Function* func = new hip::Function(std::string(deviceName), modules);
PlatformState::instance().registerStatFunction(hostFunction, func);
if (!enable_deferred_loading) {
HIP_INIT();
hipFunction_t hfunc = nullptr;
hipError_t hip_error = hipSuccess;
for (size_t dev_idx = 0; dev_idx < g_devices.size(); ++dev_idx) {
hip_error = PlatformState::instance().getStatFunc(&hfunc, hostFunction, dev_idx);
guarantee((hip_error == hipSuccess), "Cannot Retrieve Static function");
}
}
}
// Registers a device-side global variable.
// For each global variable in device code, there is a corresponding shadow
// global variable in host code. The shadow host variable is used to keep
// track of the value of the device side global variable between kernel
// executions.
extern "C" void __hipRegisterVar(
hip::FatBinaryInfo** modules, // The device modules containing code object
void* var, // The shadow variable in host code
char* hostVar, // Variable name in host code
char* deviceVar, // Variable name in device code
int ext, // Whether this variable is external
size_t size, // Size of the variable
int constant, // Whether this variable is constant
int global) // Unknown, always 0
{
hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Variable, size, 0, 0, modules);
PlatformState::instance().registerStatGlobalVar(var, var_ptr);
}
extern "C" void __hipRegisterSurface(hip::FatBinaryInfo** modules, // The device modules containing code object
void* var, // The shadow variable in host code
char* hostVar, // Variable name in host code
char* deviceVar, // Variable name in device code
int type, int ext) {
hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Surface, sizeof(surfaceReference), 0, 0, modules);
PlatformState::instance().registerStatGlobalVar(var, var_ptr);
}
extern "C" void __hipRegisterManagedVar(void *hipModule, // Pointer to hip module returned from __hipRegisterFatbinary
void **pointer, // Pointer to a chunk of managed memory with size \p size and alignment \p align
// HIP runtime allocates such managed memory and assign it to \p pointer
void *init_value, // Initial value to be copied into \p pointer
const char *name, // Name of the variable in code object
size_t size,
unsigned align) {
HIP_INIT();
hipError_t status = ihipMallocManaged(pointer, size, align);
if( status == hipSuccess) {
amd::HostQueue* queue = hip::getNullStream();
if(queue != nullptr) {
ihipMemcpy(*pointer, init_value, size, hipMemcpyHostToDevice, *queue);
} else {
ClPrint(amd::LOG_ERROR, amd::LOG_API, "Host Queue is NULL");
}
} else {
guarantee(false, "Error during allocation of managed memory!");
}
hip::Var* var_ptr = new hip::Var(std::string(name), hip::Var::DeviceVarKind::DVK_Managed, pointer,
size, align, reinterpret_cast<hip::FatBinaryInfo**>(hipModule));
PlatformState::instance().registerStatManagedVar(var_ptr);
}
extern "C" void __hipRegisterTexture(hip::FatBinaryInfo** modules, // The device modules containing code object
void* var, // The shadow variable in host code
char* hostVar, // Variable name in host code
char* deviceVar, // Variable name in device code
int type, int norm, int ext) {
hip::Var* var_ptr = new hip::Var(std::string(hostVar), hip::Var::DeviceVarKind::DVK_Texture, sizeof(textureReference), 0, 0, modules);
PlatformState::instance().registerStatGlobalVar(var, var_ptr);
}
extern "C" void __hipUnregisterFatBinary(hip::FatBinaryInfo** modules)
{
PlatformState::instance().removeFatBinary(modules);
}
extern "C" hipError_t hipConfigureCall(
dim3 gridDim,
dim3 blockDim,
size_t sharedMem,
hipStream_t stream)
{
HIP_INIT_API(hipConfigureCall, gridDim, blockDim, sharedMem, stream);
PlatformState::instance().configureCall(gridDim, blockDim, sharedMem, stream);
HIP_RETURN(hipSuccess);
}
extern "C" hipError_t __hipPushCallConfiguration(
dim3 gridDim,
dim3 blockDim,
size_t sharedMem,
hipStream_t stream)
{
HIP_INIT_API(__hipPushCallConfiguration, gridDim, blockDim, sharedMem, stream);
PlatformState::instance().configureCall(gridDim, blockDim, sharedMem, stream);
HIP_RETURN(hipSuccess);
}
extern "C" hipError_t __hipPopCallConfiguration(dim3 *gridDim,
dim3 *blockDim,
size_t *sharedMem,
hipStream_t *stream) {
HIP_INIT_API(__hipPopCallConfiguration, gridDim, blockDim, sharedMem, stream);
ihipExec_t exec;
PlatformState::instance().popExec(exec);
*gridDim = exec.gridDim_;
*blockDim = exec.blockDim_;
*sharedMem = exec.sharedMem_;
*stream = exec.hStream_;
HIP_RETURN(hipSuccess);
}
extern "C" hipError_t hipSetupArgument(
const void *arg,
size_t size,
size_t offset)
{
HIP_INIT_API(hipSetupArgument, arg, size, offset);
PlatformState::instance().setupArgument(arg, size, offset);
HIP_RETURN(hipSuccess);
}
extern "C" hipError_t hipLaunchByPtr(const void *hostFunction)
{
HIP_INIT_API(hipLaunchByPtr, hostFunction);
ihipExec_t exec;
PlatformState::instance().popExec(exec);
hip::Stream* stream = reinterpret_cast<hip::Stream*>(exec.hStream_);
int deviceId = (stream != nullptr)? stream->DeviceId() : ihipGetDevice();
if (deviceId == -1) {
LogPrintfError("Wrong DeviceId: %d \n", deviceId);
HIP_RETURN(hipErrorNoDevice);
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId);
if ((hip_error != hipSuccess) || (func == nullptr)) {
LogPrintfError("Could not retrieve hostFunction: 0x%x \n", hostFunction);
HIP_RETURN(hipErrorInvalidDeviceFunction);
}
size_t size = exec.arguments_.size();
void *extra[] = {
HIP_LAUNCH_PARAM_BUFFER_POINTER, &exec.arguments_[0],
HIP_LAUNCH_PARAM_BUFFER_SIZE, &size,
HIP_LAUNCH_PARAM_END
};
HIP_RETURN(hipModuleLaunchKernel(func,
exec.gridDim_.x, exec.gridDim_.y, exec.gridDim_.z,
exec.blockDim_.x, exec.blockDim_.y, exec.blockDim_.z,
exec.sharedMem_, exec.hStream_, nullptr, extra));
}
hipError_t hipGetSymbolAddress(void** devPtr, const void* symbol) {
HIP_INIT_API(hipGetSymbolAddress, devPtr, symbol);
hipError_t hip_error = hipSuccess;
size_t sym_size = 0;
HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(symbol, ihipGetDevice(), devPtr, &sym_size));
HIP_RETURN(hipSuccess, *devPtr);
}
hipError_t hipGetSymbolSize(size_t* sizePtr, const void* symbol) {
HIP_INIT_API(hipGetSymbolSize, sizePtr, symbol);
hipDeviceptr_t device_ptr = nullptr;
HIP_RETURN_ONFAIL(PlatformState::instance().getStatGlobalVar(symbol, ihipGetDevice(), &device_ptr, sizePtr));
HIP_RETURN(hipSuccess, *sizePtr);
}
hipError_t ihipCreateGlobalVarObj(const char* name, hipModule_t hmod, amd::Memory** amd_mem_obj,
hipDeviceptr_t* dptr, size_t* bytes)
{
HIP_INIT();
amd::Program* program = nullptr;
device::Program* dev_program = nullptr;
/* Get Device Program pointer*/
program = as_amd(reinterpret_cast<cl_program>(hmod));
dev_program = program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
if (dev_program == nullptr) {
LogPrintfError("Cannot get Device Function for module: 0x%x \n", hmod);
HIP_RETURN(hipErrorInvalidDeviceFunction);
}
/* Find the global Symbols */
if (!dev_program->createGlobalVarObj(amd_mem_obj, dptr, bytes, name)) {
LogPrintfError("Cannot create Global Var obj for symbol: %s \n", name);
HIP_RETURN(hipErrorInvalidSymbol);
}
HIP_RETURN(hipSuccess);
}
namespace hip_impl {
hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(
int* maxBlocksPerCU, int* numBlocksPerGrid, int* bestBlockSize,
const amd::Device& device, hipFunction_t func, int inputBlockSize,
size_t dynamicSMemSize, bool bCalcPotentialBlkSz)
{
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(func);
const amd::Kernel& kernel = *function->kernel();
const device::Kernel::WorkGroupInfo* wrkGrpInfo = kernel.getDeviceKernel(device)->workGroupInfo();
if (bCalcPotentialBlkSz == false) {
if (inputBlockSize <= 0) {
return hipErrorInvalidValue;
}
*bestBlockSize = 0;
// Make sure the requested block size is smaller than max supported
if (inputBlockSize > int(device.info().maxWorkGroupSize_)) {
*maxBlocksPerCU = 0;
*numBlocksPerGrid = 0;
return hipSuccess;
}
}
else {
if (inputBlockSize > int(device.info().maxWorkGroupSize_) ||
inputBlockSize <= 0) {
// The user wrote the kernel to work with a workgroup size
// bigger than this hardware can support. Or they do not care
// about the size So just assume its maximum size is
// constrained by hardware
inputBlockSize = device.info().maxWorkGroupSize_;
}
}
// Find wave occupancy per CU => simd_per_cu * GPR usage
size_t MaxWavesPerSimd;
if (device.isa().versionMajor() <= 9) {
MaxWavesPerSimd = 8; // Limited by SPI 32 per CU, hence 8 per SIMD
} else {
MaxWavesPerSimd = 16;
}
size_t VgprWaves = MaxWavesPerSimd;
size_t maxVGPRs;
uint32_t VgprGranularity;
if (device.isa().versionMajor() <= 9) {
if (device.isa().versionMajor() == 9 &&
device.isa().versionMinor() == 0 &&
device.isa().versionStepping() == 10) {
maxVGPRs = 512;
VgprGranularity = 8;
}
else {
maxVGPRs = 256;
VgprGranularity = 4;
}
}
else {
maxVGPRs = 1024;
VgprGranularity = 8;
}
if (wrkGrpInfo->usedSGPRs_ > 0) {
VgprWaves = maxVGPRs / amd::alignUp(wrkGrpInfo->usedVGPRs_, VgprGranularity);
}
size_t GprWaves = VgprWaves;
if (wrkGrpInfo->usedSGPRs_ > 0) {
size_t maxSGPRs;
if (device.isa().versionMajor() < 8) {
maxSGPRs = 512;
}
else if (device.isa().versionMajor() < 10) {
maxSGPRs = 800;
}
else {
maxSGPRs = SIZE_MAX; // gfx10+ does not share SGPRs between waves
}
const size_t SgprWaves = maxSGPRs / amd::alignUp(wrkGrpInfo->usedSGPRs_, 16);
GprWaves = std::min(VgprWaves, SgprWaves);
}
const size_t alu_occupancy = device.info().simdPerCU_ * std::min(MaxWavesPerSimd, GprWaves);
const int alu_limited_threads = alu_occupancy * wrkGrpInfo->wavefrontSize_;
int lds_occupancy_wgs = INT_MAX;
const size_t total_used_lds = wrkGrpInfo->usedLDSSize_ + dynamicSMemSize;
if (total_used_lds != 0) {
lds_occupancy_wgs = static_cast<int>(device.info().localMemSize_ / total_used_lds);
}
// Calculate how many blocks of inputBlockSize we can fit per CU
// Need to align with hardware wavefront size. If they want 65 threads, but
// waves are 64, then we need 128 threads per block.
// So this calculates how many blocks we can fit.
*maxBlocksPerCU = alu_limited_threads / amd::alignUp(inputBlockSize, wrkGrpInfo->wavefrontSize_);
// Unless those blocks are further constrained by LDS size.
*maxBlocksPerCU = std::min(*maxBlocksPerCU, lds_occupancy_wgs);
// Some callers of this function want to return the block size, in threads, that
// leads to the maximum occupancy. In that case, inputBlockSize is the maximum
// workgroup size the user wants to allow, or that the hardware can allow.
// It is either the number of threads that we are limited to due to occupancy, or
// the maximum available block size for this kernel, which could have come from the
// user. e.g., if the user indicates the maximum block size is 64 threads, but we
// calculate that 128 threads can fit in each CU, we have to give up and return 64.
*bestBlockSize = std::min(alu_limited_threads, amd::alignUp(inputBlockSize, wrkGrpInfo->wavefrontSize_));
// If the best block size is smaller than the block size used to fit the maximum,
// then we need to make the grid bigger for full occupancy.
const int bestBlocksPerCU = alu_limited_threads / (*bestBlockSize);
// Unless those blocks are further constrained by LDS size.
*numBlocksPerGrid = device.info().maxComputeUnits_ * std::min(bestBlocksPerCU, lds_occupancy_wgs);
return hipSuccess;
}
}
extern "C" {
hipError_t hipOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
const void* f, size_t dynSharedMemPerBlk,
int blockSizeLimit)
{
HIP_INIT_API(hipOccupancyMaxPotentialBlockSize, f, dynSharedMemPerBlk, blockSizeLimit);
if ((gridSize == nullptr) || (blockSize == nullptr)) {
HIP_RETURN(hipErrorInvalidValue);
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice());
if ((hip_error != hipSuccess) || (func == nullptr)) {
return HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int max_blocks_per_grid = 0;
int num_blocks = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSizeLimit, dynSharedMemPerBlk,true);
if (ret == hipSuccess) {
*blockSize = best_block_size;
*gridSize = max_blocks_per_grid;
}
HIP_RETURN(ret);
}
hipError_t hipModuleOccupancyMaxPotentialBlockSize(int* gridSize, int* blockSize,
hipFunction_t f, size_t dynSharedMemPerBlk,
int blockSizeLimit)
{
HIP_INIT_API(hipModuleOccupancyMaxPotentialBlockSize, f, dynSharedMemPerBlk, blockSizeLimit);
if ((gridSize == nullptr) || (blockSize == nullptr)) {
HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int max_blocks_per_grid = 0;
int num_blocks = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSizeLimit, dynSharedMemPerBlk,true);
if (ret == hipSuccess) {
*blockSize = best_block_size;
*gridSize = max_blocks_per_grid;
}
HIP_RETURN(ret);
}
hipError_t hipModuleOccupancyMaxPotentialBlockSizeWithFlags(int* gridSize, int* blockSize,
hipFunction_t f, size_t dynSharedMemPerBlk,
int blockSizeLimit, unsigned int flags)
{
HIP_INIT_API(hipModuleOccupancyMaxPotentialBlockSizeWithFlags, f, dynSharedMemPerBlk, blockSizeLimit, flags);
if ((gridSize == nullptr) || (blockSize == nullptr)) {
HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int max_blocks_per_grid = 0;
int num_blocks = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSizeLimit, dynSharedMemPerBlk,true);
if (ret == hipSuccess) {
*blockSize = best_block_size;
*gridSize = max_blocks_per_grid;
}
HIP_RETURN(ret);
}
hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks,
hipFunction_t f, int blockSize, size_t dynSharedMemPerBlk)
{
HIP_INIT_API(hipModuleOccupancyMaxActiveBlocksPerMultiprocessor, f, blockSize, dynSharedMemPerBlk);
if (numBlocks == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int num_blocks = 0;
int max_blocks_per_grid = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSize, dynSharedMemPerBlk, false);
*numBlocks = num_blocks;
HIP_RETURN(ret);
}
hipError_t hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks,
hipFunction_t f, int blockSize,
size_t dynSharedMemPerBlk, unsigned int flags)
{
HIP_INIT_API(hipModuleOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, f, blockSize, dynSharedMemPerBlk, flags);
if (numBlocks == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int num_blocks = 0;
int max_blocks_per_grid = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, f, blockSize, dynSharedMemPerBlk, false);
*numBlocks = num_blocks;
HIP_RETURN(ret);
}
hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks,
const void* f, int blockSize, size_t dynamicSMemSize)
{
HIP_INIT_API(hipOccupancyMaxActiveBlocksPerMultiprocessor, f, blockSize, dynamicSMemSize);
if (numBlocks == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice());
if ((hip_error != hipSuccess) || (func == nullptr)) {
return HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int num_blocks = 0;
int max_blocks_per_grid = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSize, dynamicSMemSize, false);
*numBlocks = num_blocks;
HIP_RETURN(ret);
}
hipError_t hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks,
const void* f,
int blockSize, size_t dynamicSMemSize, unsigned int flags)
{
HIP_INIT_API(hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, f, blockSize, dynamicSMemSize, flags);
if (numBlocks == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, f, ihipGetDevice());
if ((hip_error != hipSuccess) || (func == nullptr)) {
return HIP_RETURN(hipErrorInvalidValue);
}
const amd::Device& device = *hip::getCurrentDevice()->devices()[0];
int num_blocks = 0;
int max_blocks_per_grid = 0;
int best_block_size = 0;
hipError_t ret = hip_impl::ihipOccupancyMaxActiveBlocksPerMultiprocessor(
&num_blocks, &max_blocks_per_grid, &best_block_size, device, func, blockSize, dynamicSMemSize, false);
*numBlocks = num_blocks;
HIP_RETURN(ret);
}
}
#if defined(ATI_OS_LINUX)
namespace hip_impl {
void hipLaunchKernelGGLImpl(
uintptr_t function_address,
const dim3& numBlocks,
const dim3& dimBlocks,
uint32_t sharedMemBytes,
hipStream_t stream,
void** kernarg)
{
HIP_INIT();
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice();
if (deviceId == -1) {
LogPrintfError("Wrong Device Id: %d \n", deviceId);
}
hipFunction_t func = nullptr;
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, reinterpret_cast<void*>(function_address), deviceId);
if ((hip_error != hipSuccess) || (func == nullptr)) {
LogPrintfError("Cannot find the static function: 0x%x", function_address);
}
hipModuleLaunchKernel(func,
numBlocks.x, numBlocks.y, numBlocks.z,
dimBlocks.x, dimBlocks.y, dimBlocks.z,
sharedMemBytes, stream, nullptr, kernarg);
}
void hipLaunchCooperativeKernelGGLImpl(
uintptr_t function_address,
const dim3& numBlocks,
const dim3& dimBlocks,
uint32_t sharedMemBytes,
hipStream_t stream,
void** kernarg)
{
HIP_INIT();
hipLaunchCooperativeKernel(reinterpret_cast<void*>(function_address),
numBlocks, dimBlocks, kernarg, sharedMemBytes, stream);
}
}
#endif // defined(ATI_OS_LINUX)
hipError_t ihipLaunchKernel(const void* hostFunction,
dim3 gridDim,
dim3 blockDim,
void** args,
size_t sharedMemBytes,
hipStream_t stream,
hipEvent_t startEvent,
hipEvent_t stopEvent,
int flags)
{
hipFunction_t func = nullptr;
int deviceId = hip::Stream::DeviceId(stream);
hipError_t hip_error = PlatformState::instance().getStatFunc(&func, hostFunction, deviceId);
if ((hip_error != hipSuccess) || (func == nullptr)) {
HIP_RETURN(hipErrorInvalidDeviceFunction);
}
size_t globalWorkSizeX = static_cast<size_t>(gridDim.x) * blockDim.x;
size_t globalWorkSizeY = static_cast<size_t>(gridDim.y) * blockDim.y;
size_t globalWorkSizeZ = static_cast<size_t>(gridDim.z) * blockDim.z;
if (globalWorkSizeX > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeY > std::numeric_limits<uint32_t>::max() ||
globalWorkSizeZ > std::numeric_limits<uint32_t>::max()) {
HIP_RETURN(hipErrorInvalidConfiguration);
}
HIP_RETURN(ihipModuleLaunchKernel(func, static_cast<uint32_t>(globalWorkSizeX),
static_cast<uint32_t>(globalWorkSizeY),
static_cast<uint32_t>(globalWorkSizeZ),
blockDim.x, blockDim.y, blockDim.z,
sharedMemBytes, stream, args, nullptr, startEvent, stopEvent,
flags));
}
// conversion routines between float and half precision
static inline std::uint32_t f32_as_u32(float f) { union { float f; std::uint32_t u; } v; v.f = f; return v.u; }
static inline float u32_as_f32(std::uint32_t u) { union { float f; std::uint32_t u; } v; v.u = u; return v.f; }
static inline int clamp_int(int i, int l, int h) { return std::min(std::max(i, l), h); }
// half float, the f16 is in the low 16 bits of the input argument
static inline float __convert_half_to_float(std::uint32_t a) noexcept {
std::uint32_t u = ((a << 13) + 0x70000000U) & 0x8fffe000U;
std::uint32_t v = f32_as_u32(u32_as_f32(u) * u32_as_f32(0x77800000U)/*0x1.0p+112f*/) + 0x38000000U;
u = (a & 0x7fff) != 0 ? v : u;
return u32_as_f32(u) * u32_as_f32(0x07800000U)/*0x1.0p-112f*/;
}
// float half with nearest even rounding
// The lower 16 bits of the result is the bit pattern for the f16
static inline std::uint32_t __convert_float_to_half(float a) noexcept {
std::uint32_t u = f32_as_u32(a);
int e = static_cast<int>((u >> 23) & 0xff) - 127 + 15;
std::uint32_t m = ((u >> 11) & 0xffe) | ((u & 0xfff) != 0);
std::uint32_t i = 0x7c00 | (m != 0 ? 0x0200 : 0);
std::uint32_t n = ((std::uint32_t)e << 12) | m;
std::uint32_t s = (u >> 16) & 0x8000;
int b = clamp_int(1-e, 0, 13);
std::uint32_t d = (0x1000 | m) >> b;
d |= (d << b) != (0x1000 | m);
std::uint32_t v = e < 1 ? d : n;
v = (v >> 2) + (((v & 0x7) == 3) | ((v & 0x7) > 5));
v = e > 30 ? 0x7c00 : v;
v = e == 143 ? i : v;
return s | v;
}
extern "C"
#if !defined(_MSC_VER)
__attribute__((weak))
#endif
float __gnu_h2f_ieee(unsigned short h){
return __convert_half_to_float((std::uint32_t) h);
}
extern "C"
#if !defined(_MSC_VER)
__attribute__((weak))
#endif
unsigned short __gnu_f2h_ieee(float f){
return (unsigned short)__convert_float_to_half(f);
}
void PlatformState::init()
{
amd::ScopedLock lock(lock_);
if(initialized_ || g_devices.empty()) {
return;
}
initialized_ = true;
for (auto& it : statCO_.modules_) {
digestFatBinary(it.first, it.second);
}
for (auto &it : statCO_.vars_) {
it.second->resize_dVar(g_devices.size());
}
for (auto &it : statCO_.functions_) {
it.second->resize_dFunc(g_devices.size());
}
}
hipError_t PlatformState::loadModule(hipModule_t *module, const char* fname, const void* image) {
amd::ScopedLock lock(lock_);
if(module == nullptr) {
return hipErrorInvalidValue;
}
hip::DynCO* dynCo = new hip::DynCO();
hipError_t hip_error = dynCo->loadCodeObject(fname, image);
if (hip_error != hipSuccess) {
delete dynCo;
return hip_error;
}
*module = dynCo->module();
assert(*module != nullptr);
if (dynCO_map_.find(*module) != dynCO_map_.end()) {
return hipErrorAlreadyMapped;
}
dynCO_map_.insert(std::make_pair(*module, dynCo));
return hipSuccess;
}
hipError_t PlatformState::unloadModule(hipModule_t hmod) {
amd::ScopedLock lock(lock_);
auto it = dynCO_map_.find(hmod);
if (it == dynCO_map_.end()) {
return hipErrorNotFound;
}
delete it->second;
dynCO_map_.erase(hmod);
auto tex_it = texRef_map_.begin();
while (tex_it != texRef_map_.end()) {
if (tex_it->second.first == hmod) {
tex_it = texRef_map_.erase(tex_it);
} else {
++tex_it;
}
}
return hipSuccess;
}
hipError_t PlatformState::getDynFunc(hipFunction_t* hfunc, hipModule_t hmod,
const char* func_name) {
amd::ScopedLock lock(lock_);
auto it = dynCO_map_.find(hmod);
if (it == dynCO_map_.end()) {
LogPrintfError("Cannot find the module: 0x%x", hmod);
return hipErrorNotFound;
}
if (0 == strlen(func_name)) {
return hipErrorNotFound;
}
return it->second->getDynFunc(hfunc, func_name);
}
hipError_t PlatformState::getDynGlobalVar(const char* hostVar, hipModule_t hmod,
hipDeviceptr_t* dev_ptr, size_t* size_ptr) {
amd::ScopedLock lock(lock_);
if(hostVar == nullptr || dev_ptr == nullptr || size_ptr == nullptr) {
return hipErrorInvalidValue;
}
auto it = dynCO_map_.find(hmod);
if (it == dynCO_map_.end()) {
LogPrintfError("Cannot find the module: 0x%x", hmod);
return hipErrorNotFound;
}
*dev_ptr = nullptr;
it->second->getManagedVarPointer(hostVar, dev_ptr, size_ptr);
// if dev_ptr is nullptr, hostvar is not in managed variable list
if (*dev_ptr == nullptr) {
hip::DeviceVar* dvar = nullptr;
IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar));
*dev_ptr = dvar->device_ptr();
*size_ptr = dvar->size();
}
return hipSuccess;
}
hipError_t PlatformState::registerTexRef(textureReference* texRef, hipModule_t hmod,
std::string name) {
amd::ScopedLock lock(lock_);
texRef_map_.insert(std::make_pair(texRef, std::make_pair(hmod, name)));
return hipSuccess;
}
hipError_t PlatformState::getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr,
size_t* size_ptr) {
amd::ScopedLock lock(lock_);
auto tex_it = texRef_map_.find(texRef);
if (tex_it == texRef_map_.end()) {
LogPrintfError("Cannot find the texRef Entry: 0x%x", texRef);
return hipErrorNotFound;
}
auto it = dynCO_map_.find(tex_it->second.first);
if (it == dynCO_map_.end()) {
LogPrintfError("Cannot find the module: 0x%x", tex_it->second.first);
return hipErrorNotFound;
}
hip::DeviceVar* dvar = nullptr;
IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, tex_it->second.second));
*dev_ptr = dvar->device_ptr();
*size_ptr = dvar->size();
return hipSuccess;
}
hipError_t PlatformState::getDynTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef) {
amd::ScopedLock lock(lock_);
auto it = dynCO_map_.find(hmod);
if (it == dynCO_map_.end()) {
LogPrintfError("Cannot find the module: 0x%x", hmod);
return hipErrorNotFound;
}
hip::DeviceVar* dvar = nullptr;
IHIP_RETURN_ONFAIL(it->second->getDeviceVar(&dvar, hostVar));
dvar->shadowVptr = new texture<char>();
*texRef = reinterpret_cast<textureReference*>(dvar->shadowVptr);
return hipSuccess;
}
hipError_t PlatformState::digestFatBinary(const void* data, hip::FatBinaryInfo*& programs) {
return statCO_.digestFatBinary(data, programs);
}
hip::FatBinaryInfo** PlatformState::addFatBinary(const void* data) {
return statCO_.addFatBinary(data, initialized_);
}
hipError_t PlatformState::removeFatBinary(hip::FatBinaryInfo** module) {
return statCO_.removeFatBinary(module);
}
hipError_t PlatformState::registerStatFunction(const void* hostFunction, hip::Function* func) {
return statCO_.registerStatFunction(hostFunction, func);
}
hipError_t PlatformState::registerStatGlobalVar(const void* hostVar, hip::Var* var) {
return statCO_.registerStatGlobalVar(hostVar, var);
}
hipError_t PlatformState::registerStatManagedVar(hip::Var* var) {
return statCO_.registerStatManagedVar(var);
}
hipError_t PlatformState::getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId) {
return statCO_.getStatFunc(hfunc, hostFunction, deviceId);
}
hipError_t PlatformState::getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId) {
if(func_attr == nullptr || hostFunction == nullptr) {
return hipErrorInvalidValue;
}
return statCO_.getStatFuncAttr(func_attr, hostFunction, deviceId);
}
hipError_t PlatformState::getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr,
size_t* size_ptr) {
return statCO_.getStatGlobalVar(hostVar, deviceId, dev_ptr, size_ptr);
}
hipError_t PlatformState::initStatManagedVarDevicePtr(int deviceId) {
return statCO_.initStatManagedVarDevicePtr(deviceId);
}
void PlatformState::setupArgument(const void *arg, size_t size, size_t offset) {
auto& arguments = execStack_.top().arguments_;
if (arguments.size() < offset + size) {
arguments.resize(offset + size);
}
::memcpy(&arguments[offset], arg, size);
}
void PlatformState::configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem,
hipStream_t stream) {
execStack_.push(ihipExec_t{gridDim, blockDim, sharedMem, stream});
}
void PlatformState::popExec(ihipExec_t& exec) {
exec = std::move(execStack_.top());
execStack_.pop();
}
projects/hip/src/hipamd/src/hip_platform.hpp
Executable
+97
Fájl megtekintése
@@ -0,0 +1,97 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#pragma once
#include "hip_internal.hpp"
#include "hip_fatbin.hpp"
#include "device/device.hpp"
#include "hip_code_object.hpp"
namespace hip_impl {
hipError_t ihipOccupancyMaxActiveBlocksPerMultiprocessor(
int* maxBlocksPerCU, int* numBlocksPerGrid, int* bestBlockSize,
const amd::Device& device, hipFunction_t func, int blockSize,
size_t dynamicSMemSize, bool bCalcPotentialBlkSz);
} /* namespace hip_impl*/
class PlatformState {
amd::Monitor lock_{"Guards PlatformState globals", true};
/* Singleton object */
static PlatformState* platform_;
PlatformState() {}
~PlatformState() {}
public:
void init();
//Dynamic Code Objects functions
hipError_t loadModule(hipModule_t* module, const char* fname, const void* image = nullptr);
hipError_t unloadModule(hipModule_t hmod);
hipError_t getDynFunc(hipFunction_t *hfunc, hipModule_t hmod, const char* func_name);
hipError_t getDynGlobalVar(const char* hostVar, hipModule_t hmod, hipDeviceptr_t* dev_ptr,
size_t* size_ptr);
hipError_t getDynTexRef(const char* hostVar, hipModule_t hmod, textureReference** texRef);
hipError_t registerTexRef(textureReference* texRef, hipModule_t hmod, std::string name);
hipError_t getDynTexGlobalVar(textureReference* texRef, hipDeviceptr_t* dev_ptr,
size_t* size_ptr);
/* Singleton instance */
static PlatformState& instance() {
if (platform_ == nullptr) {
// __hipRegisterFatBinary() will call this when app starts, thus
// there is no multiple entry issue here.
platform_ = new PlatformState();
}
return *platform_;
}
//Static Code Objects functions
hip::FatBinaryInfo** addFatBinary(const void* data);
hipError_t removeFatBinary(hip::FatBinaryInfo** module);
hipError_t digestFatBinary(const void* data, hip::FatBinaryInfo*& programs);
hipError_t registerStatFunction(const void* hostFunction, hip::Function* func);
hipError_t registerStatGlobalVar(const void* hostVar, hip::Var* var);
hipError_t registerStatManagedVar(hip::Var* var);
hipError_t getStatFunc(hipFunction_t* hfunc, const void* hostFunction, int deviceId);
hipError_t getStatFuncAttr(hipFuncAttributes* func_attr, const void* hostFunction, int deviceId);
hipError_t getStatGlobalVar(const void* hostVar, int deviceId, hipDeviceptr_t* dev_ptr,
size_t* size_ptr);
hipError_t initStatManagedVarDevicePtr(int deviceId);
//Exec Functions
void setupArgument(const void *arg, size_t size, size_t offset);
void configureCall(dim3 gridDim, dim3 blockDim, size_t sharedMem, hipStream_t stream);
void popExec(ihipExec_t& exec);
private:
//Dynamic Code Object map, keyin module to get the corresponding object
std::unordered_map<hipModule_t, hip::DynCO*> dynCO_map_;
hip::StatCO statCO_; //Static Code object var
bool initialized_{false};
std::unordered_map<textureReference*, std::pair<hipModule_t, std::string>> texRef_map_;
};
projects/hip/src/hipamd/src/hip_prof_api.h
+270
Fájl megtekintése
@@ -0,0 +1,270 @@
/* Copyright (c) 2019-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef HIP_SRC_HIP_PROF_API_H
#define HIP_SRC_HIP_PROF_API_H
#include <atomic>
#include <iostream>
#include <mutex>
#if USE_PROF_API
#include "hip/amd_detail/hip_prof_str.h"
#include "platform/prof_protocol.h"
// HIP API callbacks spawner object macro
#define HIP_CB_SPAWNER_OBJECT(CB_ID) \
api_callbacks_spawner_t<HIP_API_ID_##CB_ID> __api_tracer; \
{ \
hip_api_data_t* api_data = __api_tracer.get_api_data_ptr(); \
if (api_data != NULL) { \
hip_api_data_t& api_data_ref = *api_data; \
INIT_CB_ARGS_DATA(CB_ID, api_data_ref); \
__api_tracer.call(); \
} \
}
static const uint32_t HIP_DOMAIN_ID = ACTIVITY_DOMAIN_HIP_API;
typedef activity_record_t hip_api_record_t;
typedef activity_rtapi_callback_t hip_api_callback_t;
typedef activity_sync_callback_t hip_act_callback_t;
class api_callbacks_table_t {
public:
typedef std::mutex mutex_t;
typedef hip_api_record_t record_t;
typedef hip_api_callback_t fun_t;
typedef hip_act_callback_t act_t;
// HIP API callbacks table
struct hip_cb_table_entry_t {
volatile std::atomic<bool> sync;
volatile std::atomic<uint32_t> sem;
act_t act;
void* a_arg;
fun_t fun;
void* arg;
};
struct hip_cb_table_t {
hip_cb_table_entry_t arr[HIP_API_ID_NUMBER];
};
api_callbacks_table_t() {
memset(&callbacks_table_, 0, sizeof(callbacks_table_));
}
bool set_activity(uint32_t id, act_t fun, void* arg) {
std::lock_guard<mutex_t> lock(mutex_);
bool ret = true;
if (id < HIP_API_ID_NUMBER) {
cb_sync(id);
/*
'fun != nullptr' indicates it is activity register call,
increment should happen only once but client is free to call
register CB multiple times for same API id hence the check
'fun == nullptr' indicates it is de-register call and
decrement should happen only once hence the check
*/
if (fun != nullptr) {
if (callbacks_table_.arr[id].act == nullptr) {
enabled_api_count_++;
}
} else {
if (callbacks_table_.arr[id].act != nullptr) {
enabled_api_count_--;
}
}
if (enabled_api_count_ > 0) {
amd::IS_PROFILER_ON = true;
} else {
amd::IS_PROFILER_ON = false;
}
callbacks_table_.arr[id].act = fun;
callbacks_table_.arr[id].a_arg = arg;
cb_release(id);
} else {
ret = false;
}
return ret;
}
bool set_callback(uint32_t id, fun_t fun, void* arg) {
std::lock_guard<mutex_t> lock(mutex_);
bool ret = true;
if (id < HIP_API_ID_NUMBER) {
cb_sync(id);
callbacks_table_.arr[id].fun = fun;
callbacks_table_.arr[id].arg = arg;
cb_release(id);
} else {
ret = false;
}
return ret;
}
void set_enabled(const bool& enabled) {
amd::IS_PROFILER_ON = enabled;
}
inline hip_cb_table_entry_t& entry(const uint32_t& id) {
return callbacks_table_.arr[id];
}
inline void sem_sync(const uint32_t& id) {
sem_increment(id);
if (entry(id).sync.load() == true) sync_wait(id);
}
inline void sem_release(const uint32_t& id) {
sem_decrement(id);
}
inline bool is_enabled() const {
return amd::IS_PROFILER_ON;
}
private:
inline void cb_sync(const uint32_t& id) {
entry(id).sync.store(true);
while (entry(id).sem.load() != 0) {}
}
inline void cb_release(const uint32_t& id) {
entry(id).sync.store(false);
}
inline void sem_increment(const uint32_t& id) {
const uint32_t prev = entry(id).sem.fetch_add(1);
if (prev == UINT32_MAX) {
std::cerr << "sem overflow id = " << id << std::endl << std::flush;
abort();
}
}
inline void sem_decrement(const uint32_t& id) {
const uint32_t prev = entry(id).sem.fetch_sub(1);
if (prev == 0) {
std::cerr << "sem corrupted id = " << id << std::endl << std::flush;
abort();
}
}
void sync_wait(const uint32_t& id) {
sem_decrement(id);
while (entry(id).sync.load() == true) {}
sem_increment(id);
}
mutex_t mutex_;
hip_cb_table_t callbacks_table_;
uint32_t enabled_api_count_;
};
extern api_callbacks_table_t callbacks_table;
template <int cid_>
class api_callbacks_spawner_t {
public:
api_callbacks_spawner_t() :
api_data_(NULL)
{
if (!is_enabled()) return;
if (cid_ >= HIP_API_ID_NUMBER) {
fprintf(stderr, "HIP %s bad id %d\n", __FUNCTION__, cid_);
abort();
}
callbacks_table.sem_sync(cid_);
hip_act_callback_t act = entry(cid_).act;
if (act != NULL) api_data_ = (hip_api_data_t*) act(cid_, NULL, NULL, NULL);
}
void call() {
hip_api_callback_t fun = entry(cid_).fun;
void* arg = entry(cid_).arg;
if (fun != NULL) {
fun(HIP_DOMAIN_ID, cid_, api_data_, arg);
api_data_->phase = ACTIVITY_API_PHASE_EXIT;
}
}
~api_callbacks_spawner_t() {
if (!is_enabled()) return;
if (api_data_ != NULL) {
hip_api_callback_t fun = entry(cid_).fun;
void* arg = entry(cid_).arg;
hip_act_callback_t act = entry(cid_).act;
void* a_arg = entry(cid_).a_arg;
if (fun != NULL) fun(HIP_DOMAIN_ID, cid_, api_data_, arg);
if (act != NULL) act(cid_, NULL, NULL, a_arg);
}
callbacks_table.sem_release(cid_);
}
hip_api_data_t* get_api_data_ptr() {
return api_data_;
}
bool is_enabled() const {
return callbacks_table.is_enabled();
}
private:
inline api_callbacks_table_t::hip_cb_table_entry_t& entry(const uint32_t& id) {
return callbacks_table.entry(id);
}
hip_api_data_t* api_data_;
};
template <>
class api_callbacks_spawner_t<HIP_API_ID_NUMBER> {
public:
api_callbacks_spawner_t() {}
void call() {}
hip_api_data_t* get_api_data_ptr() { return NULL; }
bool is_enabled() const { return false; }
};
#else
#define HIP_CB_SPAWNER_OBJECT(x) do {} while(0)
class api_callbacks_table_t {
public:
typedef void* act_t;
typedef void* fun_t;
bool set_activity(uint32_t id, act_t fun, void* arg) { return false; }
bool set_callback(uint32_t id, fun_t fun, void* arg) { return false; }
};
#endif
#endif // HIP_SRC_HIP_PROF_API_H
projects/hip/src/hipamd/src/hip_prof_gen.py
Executable
+673
Fájl megtekintése
@@ -0,0 +1,673 @@
#!/usr/bin/python
# Copyright (c) 2019-present Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import os, sys, re
PROF_HEADER = "hip_prof_str.h"
OUTPUT = PROF_HEADER
REC_MAX_LEN = 1024
# Recursive sources processing
recursive_mode = 0
# HIP_INIT_API macro patching
hip_patch_mode = 0
# API matching types check
types_check_mode = 0
# Private API check
private_check_mode = 0
# Messages and errors controll
verbose = 0
errexit = 0
inp_file = 'none'
line_num = -1
# Verbose message
def message(msg):
if verbose: sys.stdout.write(msg + '\n')
# Fatal error termination
def error(msg):
if line_num != -1:
msg += ", file '" + inp_file + "', line (" + str(line_num) + ")"
if errexit:
msg = " Error: " + msg
else:
msg = " Warning: " + msg
sys.stdout.write(msg + '\n')
sys.stderr.write(sys.argv[0] + msg +'\n')
def fatal(msg):
error(msg)
sys.exit(1)
#############################################################
# Normalizing API name
def filtr_api_name(name):
name = re.sub(r'\s*$', r'', name);
return name
def filtr_api_decl(record):
record = re.sub("\s__dparm\([^\)]*\)", r'', record);
record = re.sub("\(void\*\)", r'', record);
return record
# Normalizing API arguments
def filtr_api_args(args_str):
args_str = re.sub(r'^\s*', r'', args_str);
args_str = re.sub(r'\s*$', r'', args_str);
args_str = re.sub(r'\s*,\s*', r',', args_str);
args_str = re.sub(r'\s+', r' ', args_str);
args_str = re.sub(r'\s*(\*+)\s*', r'\1 ', args_str);
args_str = re.sub(r'(enum|struct) ', '', args_str);
return args_str
# Normalizing types
def norm_api_types(type_str):
type_str = re.sub(r'uint32_t', r'unsigned int', type_str)
type_str = re.sub(r'^unsigned$', r'unsigned int', type_str)
return type_str
# Creating a list of arguments [(type, name), ...]
def list_api_args(args_str):
args_str = filtr_api_args(args_str)
args_list = []
if args_str != '':
for arg_pair in args_str.split(','):
if arg_pair == 'void': continue
arg_pair = re.sub(r'\s*=\s*\S+$','', arg_pair);
m = re.match("^(.*)\s(\S+)$", arg_pair);
if m:
arg_type = norm_api_types(m.group(1))
arg_name = m.group(2)
args_list.append((arg_type, arg_name))
else:
fatal("bad args: args_str: '" + args_str + "' arg_pair: '" + arg_pair + "'")
return args_list;
# Creating arguments string "type0, type1, ..."
def filtr_api_types(args_str):
args_list = list_api_args(args_str)
types_str = ''
for arg_tuple in args_list:
types_str += arg_tuple[0] + ', '
return types_str
# Creating options list [opt0, opt1, ...]
def filtr_api_opts(args_str):
args_list = list_api_args(args_str)
opts_list = []
for arg_tuple in args_list:
opts_list.append(arg_tuple[1])
return opts_list
# Checking for pointer non-void arg type
def pointer_ck(arg_type):
ptr_type = ''
m = re.match(r'(.*)\*$', arg_type)
if m:
ptr_type = m.group(1)
ptr_type = re.sub(r'const ', '', ptr_type)
if ptr_type == 'void': ptr_type = ''
return ptr_type
#############################################################
# Parsing API header
# hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset);
def parse_api(inp_file_p, out):
global inp_file
global line_num
inp_file = inp_file_p
beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(");
api_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)");
end_pattern = re.compile("Texture");
hidden_pattern = re.compile(r'__attribute__\(\(visibility\("hidden"\)\)\)')
nms_open_pattern = re.compile(r'namespace hip_impl {')
nms_close_pattern = re.compile(r'}')
inp = open(inp_file, 'r')
found = 0
hidden = 0
nms_level = 0;
record = ""
line_num = -1
for line in inp.readlines():
record += re.sub(r'^\s+', r' ', line[:-1])
line_num += 1
if len(record) > REC_MAX_LEN:
fatal("bad record \"" + record + "\"")
m = beg_pattern.match(line)
if m:
name = m.group(2)
if hidden != 0:
message("api: " + name + " - hidden")
elif nms_level != 0:
message("api: " + name + " - hip_impl")
else:
message("api: " + name)
found = 1
if found != 0:
record = re.sub("\s__dparm\([^\)]*\)", '', record);
m = api_pattern.match(record)
if m:
found = 0
if end_pattern.search(record): break
api_name = filtr_api_name(m.group(2))
api_args = m.group(3)
if not api_name in out:
out[api_name] = api_args
else: continue
hidden = 0
if hidden_pattern.match(line): hidden = 1
if nms_open_pattern.match(line): nms_level += 1
if (nms_level > 0) and nms_close_pattern.match(line): nms_level -= 1
if nms_level < 0:
fatal("nms level < 0")
record = ""
inp.close()
line_num = -1
#############################################################
# Parsing API implementation
# hipError_t hipSetupArgument(const void* arg, size_t size, size_t offset) {
# HIP_INIT_API(hipSetupArgument, arg, size, offset);
# inp_file - input implementation source file
# api_map - input public API map [<api name>] => <api args>
# out - output map [<api name>] => [opt0, opt1, ...]
def parse_content(inp_file_p, api_map, out):
global hip_patch_mode
global types_check_mode
global private_check_mode
global inp_file
global line_num
inp_file = inp_file_p
# API method begin pattern
beg_pattern = re.compile("^(hipError_t|const char\s*\*)\s+[^\(]+\(");
# API declaration pattern
decl_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)\s*;");
# API definition pattern
api_pattern = re.compile("^(hipError_t|const char\s*\*)\s+([^\(]+)\(([^\)]*)\)\s*{");
# API init macro pattern
init_pattern = re.compile("(^\s*HIP_INIT_API\s*)\((([^,]+)(,.*|)|)(\);|,)\s*$");
# Open input file
inp = open(inp_file, 'r')
# API name
api_name = ""
# Valid public API found flag
api_valid = 0
# API overload (parameters mismatch)
api_overload = 0
# Input file patched content
content = ''
# Sub content for found API defiition
sub_content = ''
# Current record, accumulating several API definition related lines
record = ''
# Current input file line number
line_num = -1
# API beginning found flag
found = 0
# Reading input file
for line in inp.readlines():
# Accumulating record
record += re.sub(r'^\s+', r' ', line[:-1])
line_num += 1
if len(record) > REC_MAX_LEN:
fatal("bad record \"" + record + "\"")
break;
# Looking for API begin
if found == 0:
record = re.sub(r'\s*extern\s+"C"\s+', r'', record);
if beg_pattern.match(record):
found = 1
record = filtr_api_decl(record)
# Matching API declaration
if found == 1:
if decl_pattern.match(record):
found = 0
# Matching API definition
if found == 1:
m = api_pattern.match(record)
# Checking if complete API matched
if m:
found = 2
api_valid = 0
api_overload = 0
api_name = filtr_api_name(m.group(2))
# Checking if API name is in the API map
if (private_check_mode == 0) or (api_name in api_map):
if not api_name in api_map: api_map[api_name] = ''
# Getting API arguments
api_args = m.group(3)
# Getting etalon arguments from the API map
eta_args = api_map[api_name]
if eta_args == '':
eta_args = api_args
api_map[api_name] = eta_args
# Normalizing API arguments
api_types = filtr_api_types(api_args)
# Normalizing etalon arguments
eta_types = filtr_api_types(eta_args)
if (api_types == eta_types) or ((types_check_mode == 0) and (not api_name in out)):
# API is already found and not is mismatched
if (api_name in out):
fatal("API redefined \"" + api_name + "\", record \"" + record + "\"")
# Set valid public API found flag
api_valid = 1
# Set output API map with API arguments list
out[api_name] = filtr_api_opts(api_args)
# Register missmatched API methods
else:
api_overload = 1
# Warning about mismatched API, possible non public overloaded version
api_diff = '\t\t' + inp_file + " line(" + str(line_num) + ")\n\t\tapi: " + api_types + "\n\t\teta: " + eta_types
message("\t" + api_name + ' args mismatch:\n' + api_diff + '\n')
# API found action
if found == 2:
if hip_patch_mode != 0:
# Looking for INIT macro
m = init_pattern.match(line)
if m:
init_name = api_name
if api_overload == 1: init_name = 'NONE'
init_args = m.group(4)
line = m.group(1) + '(' + init_name + init_args + m.group(5) + '\n'
m = init_pattern.match(line)
if m:
found = 0
if api_valid == 1: message("\t" + api_name)
# Ignore if it is initialized as NONE
init_name = m.group(3)
if init_name != 'NONE':
# Check if init name matching API name
if init_name != api_name:
fatal("init name mismatch: '" + init_name + "' <> '" + api_name + "'")
# Registering dummy API for non public API if the name in INIT is not NONE
if api_valid == 0:
# If init name is not in public API map then it is private API
# else it was not identified and will be checked on finish
if not init_name in api_map:
if init_name in out:
fatal("API reinit \"" + api_name + "\", record \"" + record + "\"")
out[init_name] = []
elif re.search('}', line):
found = 0
# Expect INIT macro for valid public API
# Removing and registering non-conformant APIs with missing HIP_INIT macro
if api_valid == 1:
if api_name in out:
del out[api_name]
del api_map[api_name]
# Registering non-conformant APIs
out['.' + api_name] = 1
else:
fatal("API is not in out \"" + api_name + "\", record \"" + record + "\"")
if found != 1: record = ""
content += line
inp.close()
line_num = -1
if len(out) != 0:
return content
else:
return ''
# src path walk
def parse_src(api_map, src_path, src_patt, out):
global recursive_mode
pattern = re.compile(src_patt)
src_path = re.sub(r'\s', '', src_path)
for src_dir in src_path.split(':'):
message("Parsing " + src_dir + " for '" + src_patt + "'")
for root, dirs, files in os.walk(src_dir):
for fnm in files:
if pattern.search(fnm):
file = root + '/' + fnm
message(file)
content = parse_content(file, api_map, out);
if (hip_patch_mode != 0) and (content != ''):
f = open(file, 'w')
f.write(content)
f.close()
if recursive_mode == 0: break
#############################################################
# Generating profiling primitives header
# api_map - public API map [<api name>] => [(type, name), ...]
# opts_map - opts map [<api name>] => [opt0, opt1, ...]
def generate_prof_header(f, api_map, opts_map):
# Private API list
priv_lst = []
f.write('// automatically generated sources\n')
f.write('#ifndef _HIP_PROF_STR_H\n');
f.write('#define _HIP_PROF_STR_H\n');
f.write('#define HIP_PROF_VER 1\n')
# Generating dummy macro for non-public API
f.write('\n// Dummy API primitives\n')
f.write('#define INIT_NONE_CB_ARGS_DATA(cb_data) {};\n')
for name in opts_map:
if not name in api_map:
opts_lst = opts_map[name]
if len(opts_lst) != 0:
fatal("bad dummy API \"" + name + "\", args: " + str(opts_lst))
f.write('#define INIT_'+ name + '_CB_ARGS_DATA(cb_data) {};\n')
priv_lst.append(name)
for name in priv_lst:
message("Private: " + name)
# Generating the callbacks ID enumaration
f.write('\n// HIP API callbacks ID enumaration\n')
f.write('enum hip_api_id_t {\n')
cb_id = 0
for name in api_map.keys():
f.write(' HIP_API_ID_' + name + ' = ' + str(cb_id) + ',\n')
cb_id += 1
f.write(' HIP_API_ID_NUMBER = ' + str(cb_id) + ',\n')
f.write('\n')
f.write(' HIP_API_ID_NONE = HIP_API_ID_NUMBER,\n')
for name in priv_lst:
f.write(' HIP_API_ID_' + name + ' = HIP_API_ID_NUMBER,\n')
f.write('};\n')
# Generating the method to return API name by ID
f.write('\n// Return HIP API string by given ID\n')
f.write('static inline const char* hip_api_name(const uint32_t id) {\n')
f.write(' switch(id) {\n')
for name in api_map.keys():
f.write(' case HIP_API_ID_' + name + ': return "' + name + '";\n')
f.write(' };\n')
f.write(' return "unknown";\n')
f.write('};\n')
# Generating the method for querying API ID by name
f.write('\n')
f.write('#include <string.h>\n');
f.write('// Return HIP API ID by given name\n')
f.write('static inline uint32_t hipApiIdByName(const char* name) {\n')
for name, args in api_map.items():
f.write(' if (strcmp("' + name + '", name) == 0) return HIP_API_ID_' + name + ';\n')
f.write(' return HIP_API_ID_NUMBER;\n')
f.write('}\n')
# Generating the callbacks data structure
f.write('\n// HIP API callbacks data structure\n')
f.write(
'typedef struct hip_api_data_s {\n' +
' uint64_t correlation_id;\n' +
' uint32_t phase;\n' +
' union {\n'
)
for name, args in api_map.items():
if len(args) != 0:
f.write(' struct {\n')
for arg_tuple in args:
arg_type = arg_tuple[0]
ptr_type = pointer_ck(arg_type)
arg_name = arg_tuple[1]
# Checking for enum type
if arg_type == "hipLimit_t": arg_type = 'enum ' + arg_type
# Structuer field code
f.write(' ' + arg_type + ' ' + arg_name + ';\n')
if ptr_type != '':
f.write(' ' + ptr_type + ' ' + arg_name + '__val;\n')
f.write(' } ' + name + ';\n')
f.write(
' } args;\n' +
'} hip_api_data_t;\n'
)
# Generating the callbacks args data filling macros
f.write('\n// HIP API callbacks args data filling macros\n')
for name, args in api_map.items():
f.write('// ' + name + str(args) + '\n')
f.write('#define INIT_' + name + '_CB_ARGS_DATA(cb_data) { \\\n')
if name in opts_map:
opts_list = opts_map[name]
if len(args) != len(opts_list):
fatal("\"" + name + "\" API args and opts mismatch, args: " + str(args) + ", opts: " + str(opts_list))
# API args iterating:
# type is args[<ind>][0]
# name is args[<ind>][1]
for ind in range(0, len(args)):
arg_tuple = args[ind]
arg_type = arg_tuple[0]
ptr_type = pointer_ck(arg_type)
fld_name = arg_tuple[1]
opt_name = opts_list[ind]
if arg_type == "const char*":
f.write(' cb_data.args.' + name + '.' + fld_name + ' = (' + opt_name + ') ? strdup(' + opt_name + ') : NULL; \\\n')
else:
f.write(' cb_data.args.' + name + '.' + fld_name + ' = (' + arg_type + ')' + opt_name + '; \\\n')
f.write('};\n')
f.write('#define INIT_CB_ARGS_DATA(cb_id, cb_data) INIT_##cb_id##_CB_ARGS_DATA(cb_data)\n')
f.write('#if HIP_PROF_HIP_API_STRING\n')
# Generating the method for the API args filling
f.write('\n')
f.write('// HIP API args filling method\n')
f.write('static inline void hipApiArgsInit(hip_api_id_t id, hip_api_data_t* data) {\n')
f.write(' switch (id) {\n')
for name, args in api_map.items():
f.write('// ' + name + str(args) + '\n')
f.write(' case HIP_API_ID_' + name + ':\n')
for ind in range(0, len(args)):
arg_tuple = args[ind]
arg_type = arg_tuple[0]
ptr_type = pointer_ck(arg_type)
fld_name = arg_tuple[1]
var_name = 'data->args.' + name + '.' + fld_name
if arg_type == "char*":
f.write(' ' + var_name + ' = (' + var_name + ') ? strdup(' + var_name + ') : NULL;\n')
else:
if ptr_type != '':
f.write(' if (' + var_name + ') ' + var_name + '__val = *(' + var_name + ');\n')
f.write(' break;\n')
f.write(' default: break;\n')
f.write(' };\n')
f.write('}\n')
# Generating the method for the API string, name and parameters
f.write('\n')
f.write('#include <sstream>\n');
f.write('#include <string>\n');
f.write('// HIP API string method, method name and parameters\n')
f.write('static inline const char* hipApiString(hip_api_id_t id, const hip_api_data_t* data) {\n')
f.write(' std::ostringstream oss;\n')
f.write(' switch (id) {\n')
for name, args in api_map.items():
f.write(' case HIP_API_ID_' + name + ':\n')
f.write(' oss << "' + name + '(";\n')
for ind in range(0, len(args)):
arg_tuple = args[ind]
arg_type = arg_tuple[0]
ptr_type = pointer_ck(arg_type)
arg_name = arg_tuple[1]
var_name = 'data->args.' + name + '.' + arg_name
delim = '' if ind == 0 else ', ';
oss_stream = 'oss << "' + delim + arg_name + '='
line_shift = ' '
f.write(line_shift)
if ptr_type != '':
f.write('if (' + var_name + ' == NULL) ' + oss_stream + 'NULL";\n' + line_shift + 'else ')
if pointer_ck(ptr_type) != '':
f.write(oss_stream + '" << (void*)' + var_name + '__val' + ';\n')
else:
f.write(oss_stream + '" << ' + var_name + '__val' + ';\n')
else:
f.write(oss_stream + '" << ' + var_name + ';\n')
f.write(' oss << ")";\n')
f.write(' break;\n')
f.write(' default: oss << "unknown";\n')
f.write(' };\n')
f.write(' return strdup(oss.str().c_str());\n')
f.write('}\n')
f.write('#endif // HIP_PROF_HIP_API_STRING\n')
f.write('#endif // _HIP_PROF_STR_H\n');
#############################################################
# main
while len(sys.argv) > 1:
if not re.match(r'-', sys.argv[1]): break
if (sys.argv[1] == '-v'):
verbose = 1
sys.argv.pop(1)
if (sys.argv[1] == '-r'):
recursive_mode = 1
sys.argv.pop(1)
if (sys.argv[1] == '-t'):
types_check_mode = 1
sys.argv.pop(1)
if (sys.argv[1] == '--priv'):
private_check_mode = 1
sys.argv.pop(1)
if (sys.argv[1] == '-e'):
errexit = 1
sys.argv.pop(1)
if (sys.argv[1] == '-p'):
hip_patch_mode = 1
sys.argv.pop(1)
# Usage
if (len(sys.argv) < 3):
fatal ("Usage: " + sys.argv[0] + " [-v] <input HIP API .h file> <patched srcs path> [<output>]\n" +
" -v - verbose messages\n" +
" -r - process source directory recursively\n" +
" -t - API types matching check\n" +
" --priv - private API check\n" +
" -e - on error exit mode\n" +
" -p - HIP_INIT_API macro patching mode\n" +
"\n" +
" Example:\n" +
" $ " + sys.argv[0] + " -v -p -t --priv ./api/hip/include/hip/amd_detail/hip_runtime_api.h ./api/hip ./api/hip/include/hip/amd_detail/hip_prof_str.h");
# API header file given as an argument
src_pat = "\.cpp$"
api_hfile = sys.argv[1]
if not os.path.isfile(api_hfile):
fatal("input file '" + api_hfile + "' not found")
# Srcs directory given as an argument
src_dir = sys.argv[2]
if not os.path.isdir(src_dir):
fatal("src directory " + src_dir + "' not found")
if len(sys.argv) > 3: OUTPUT = sys.argv[3]
# API declaration map
api_map = {
'hipSetupArgument': '',
'hipMalloc3DArray': '',
'hipFuncGetAttribute': '',
'hipMemset3DAsync': '',
'hipKernelNameRef': '',
'hipStreamGetPriority': '',
'hipLaunchByPtr': '',
'hipFreeHost': '',
'hipGetErrorName': '',
'hipMemcpy3DAsync': '',
'hipMemcpyParam2DAsync': '',
'hipArray3DCreate': '',
'hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags': '',
'hipOccupancyMaxPotentialBlockSize': '',
'hipMallocManaged': '',
'hipOccupancyMaxActiveBlocksPerMultiprocessor': '',
'hipGetErrorString': '',
'hipMallocHost': '',
'hipModuleLoadDataEx': '',
'hipGetDeviceProperties': '',
'hipConfigureCall': '',
'hipHccModuleLaunchKernel': '',
'hipExtModuleLaunchKernel': '',
}
# API options map
opts_map = {}
# Parsing API header
parse_api(api_hfile, api_map)
# Parsing sources
parse_src(api_map, src_dir, src_pat, opts_map)
# Checking for non-conformant APIs with missing HIP_INIT macro
for name in list(opts_map.keys()):
m = re.match(r'\.(\S*)', name)
if m:
message("Init missing: " + m.group(1))
del opts_map[name]
# Converting api map to map of lists
# Checking for not found APIs
not_found = 0
if len(opts_map) != 0:
for name in api_map.keys():
args_str = api_map[name];
api_map[name] = list_api_args(args_str)
if not name in opts_map:
error("implementation not found: " + name)
not_found += 1
if not_found != 0:
error(str(not_found) + " API calls missing in interception layer")
# The output subdirectory seems to exist or not depending on the
# version of cmake.
output_dir = os.path.dirname(OUTPUT)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Generating output header file
with open(OUTPUT, 'w') as f:
generate_prof_header(f, api_map, opts_map)
# Successfull exit
sys.exit(0)
projects/hip/src/hipamd/src/hip_profile.cpp
+40
Fájl megtekintése
@@ -0,0 +1,40 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
hipError_t hipProfilerStart() {
HIP_INIT_API(hipProfilerStart);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipProfilerStop() {
HIP_INIT_API(hipProfilerStop);
assert(0 && "Unimplemented");
HIP_RETURN(hipErrorNotSupported);
}
projects/hip/src/hipamd/src/hip_rtc.cpp
Executable
+419
Fájl megtekintése
@@ -0,0 +1,419 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hiprtc_internal.hpp"
#include <hip/hiprtc.h>
#include "platform/program.hpp"
#ifdef __HIP_ENABLE_PCH
extern const char __hip_pch[];
extern unsigned __hip_pch_size;
void __hipGetPCH(const char** pch, unsigned int *size) {
*pch = __hip_pch;
*size = __hip_pch_size;
}
#endif
namespace hiprtc {
thread_local hiprtcResult g_lastRtcError = HIPRTC_SUCCESS;
}
class ProgramState {
amd::Monitor lock_;
private:
static ProgramState* programState_;
ProgramState() : lock_("Guards program state") {}
~ProgramState() {}
public:
std::unordered_map<amd::Program*,
std::pair<std::vector<std::string>, std::vector<std::string>>> progHeaders_;
std::map<std::string, std::pair<std::string, std::string>> nameExpresssion_;
static ProgramState& instance();
uint32_t addNameExpression(const char* name_expression);
char* getLoweredName(const char* name_expression);
};
ProgramState* ProgramState::programState_ = nullptr;
ProgramState& ProgramState::instance() {
if (programState_ == nullptr) {
programState_ = new ProgramState;
}
return *programState_;
}
uint32_t ProgramState::addNameExpression(const char* name_expression) {
amd::ScopedLock lock(lock_);
// Strip clean of any '(' or ')' or '&'
std::string strippedName(name_expression);
if (strippedName.back() == ')') {
strippedName.pop_back();
strippedName.erase(0, strippedName.find('('));
}
if (strippedName.front() == '&') {
strippedName.erase(0, 1);
}
auto it = nameExpresssion_.find(name_expression);
if (it == nameExpresssion_.end()) {
nameExpresssion_.insert(std::pair<std::string, std::pair<std::string, std::string>>
(name_expression, std::make_pair(strippedName,"")));
}
return nameExpresssion_.size();
}
char* demangle(const char* loweredName) {
if (!loweredName) {
return nullptr;
}
#if __linux__
int status = 0;
char* demangledName = DEMANGLE(loweredName, nullptr, nullptr, &status);
if (status != 0) {
LogPrintfError("Cannot demangle loweredName: %s \n", loweredName);
return nullptr;
}
#elif defined(_WIN32)
char* demangledName = (char*)malloc(UNDECORATED_SIZE);
if (!UnDecorateSymbolName(loweredName, demangledName,
UNDECORATED_SIZE/ sizeof(*demangledName), UNDNAME_COMPLETE))
{
free(demangledName);
LogPrintfError("Cannot undecorate loweredName: %s demangledName: %s \n",
loweredName, demangledName);
return nullptr;
}
#else
#error "Only Linux and Windows are supported"
#endif // __linux__
return demangledName;
}
static std::string handleMangledName(std::string name) {
std::string loweredName;
char* demangled = demangle(name.c_str());
loweredName.assign(demangled == nullptr ? std::string() : demangled);
free(demangled);
if (loweredName.empty()) {
return name;
}
if (loweredName.find(".kd") != std::string::npos) {
return {};
}
if (loweredName.find("void ") == 0) {
loweredName.erase(0, strlen("void "));
}
auto dx{loweredName.find_first_of("(<")};
if (dx == std::string::npos) {
return loweredName;
}
if (loweredName[dx] == '<') {
uint32_t count = 1;
do {
++dx;
count += (loweredName[dx] == '<') ? 1 : ((loweredName[dx] == '>') ? -1 : 0);
} while (count);
loweredName.erase(++dx);
} else {
loweredName.erase(dx);
}
return loweredName;
}
static std::string getValueOf(const std::string& option) {
std::string res;
auto f = std::find(option.begin(), option.end(), '=');
if (f != option.end()) res = std::string(f + 1, option.end());
return res;
}
static void transformOptions(std::vector<std::string>& options, amd::Program* program) {
std::vector<const char*> t_option;
for (auto& i : options) {
#ifdef __HIP_ENABLE_PCH
// Use precompiled header for hip
if (i == "-hip-pch") {
const char* pch = nullptr;
unsigned int pch_size = 0;
__hipGetPCH(&pch, &pch_size);
program->addPreCompiledHeader(std::string(pch, pch_size));
i = "-nogpuinc";
continue;
}
#endif
// Some rtc samples use --gpu-architecture
if (i.rfind("--gpu-architecture=", 0) == 0) {
auto val = getValueOf(i);
i = "--offload-arch=" + val;
continue;
}
}
}
const char* hiprtcGetErrorString(hiprtcResult x) {
switch (x) {
case HIPRTC_SUCCESS:
return "HIPRTC_SUCCESS";
case HIPRTC_ERROR_OUT_OF_MEMORY:
return "HIPRTC_ERROR_OUT_OF_MEMORY";
case HIPRTC_ERROR_PROGRAM_CREATION_FAILURE:
return "HIPRTC_ERROR_PROGRAM_CREATION_FAILURE";
case HIPRTC_ERROR_INVALID_INPUT:
return "HIPRTC_ERROR_INVALID_INPUT";
case HIPRTC_ERROR_INVALID_PROGRAM:
return "HIPRTC_ERROR_INVALID_PROGRAM";
case HIPRTC_ERROR_INVALID_OPTION:
return "HIPRTC_ERROR_INVALID_OPTION";
case HIPRTC_ERROR_COMPILATION:
return "HIPRTC_ERROR_COMPILATION";
case HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE:
return "HIPRTC_ERROR_BUILTIN_OPERATION_FAILURE";
case HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION:
return "HIPRTC_ERROR_NO_NAME_EXPRESSIONS_AFTER_COMPILATION";
case HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION:
return "HIPRTC_ERROR_NO_LOWERED_NAMES_BEFORE_COMPILATION";
case HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID:
return "HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID";
case HIPRTC_ERROR_INTERNAL_ERROR:
return "HIPRTC_ERROR_INTERNAL_ERROR";
default:
LogPrintfError("Invalid HIPRTC error code: %d \n", x);
return nullptr;
};
ShouldNotReachHere();
return nullptr;
}
hiprtcResult hiprtcCreateProgram(hiprtcProgram* prog, const char* src, const char* name,
int numHeaders, const char** headers, const char** headerNames) {
HIPRTC_INIT_API(prog, src, name, numHeaders, headers, headerNames);
if (prog == nullptr) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_PROGRAM);
}
if (numHeaders < 0) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
if (numHeaders && (headers == nullptr || headerNames == nullptr)) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
amd::Program* program = new amd::Program(*hip::getCurrentDevice()->asContext(), src, amd::Program::HIP,
numHeaders, headers, headerNames);
if (program == NULL) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
if (CL_SUCCESS != program->addDeviceProgram(*hip::getCurrentDevice()->devices()[0])) {
program->release();
HIPRTC_RETURN(HIPRTC_ERROR_PROGRAM_CREATION_FAILURE);
}
*prog = reinterpret_cast<hiprtcProgram>(as_cl(program));
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcCompileProgram(hiprtcProgram prog, int numOptions, const char** options) {
// FIXME[skudchad] Add headers to amd::Program::build and device::Program::build,
// pass the saved from ProgramState to amd::Program::build
HIPRTC_INIT_API(prog, numOptions, options);
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
std::ostringstream ostrstr;
std::vector<std::string> oarr(&options[0], &options[numOptions]);
const std::string hipVerOpt{"--hip-version=" + std::to_string(HIP_VERSION_MAJOR) + '.' +
std::to_string(HIP_VERSION_MINOR) + '.' +
std::to_string(HIP_VERSION_PATCH)};
const std::string hipVerMajor{"-DHIP_VERSION_MAJOR=" + std::to_string(HIP_VERSION_MAJOR)};
const std::string hipVerMinor{"-DHIP_VERSION_MINOR=" + std::to_string(HIP_VERSION_MINOR)};
const std::string hipVerPatch{"-DHIP_VERSION_PATCH=" + std::to_string(HIP_VERSION_PATCH)};
oarr.push_back(hipVerOpt);
oarr.push_back(hipVerMajor);
oarr.push_back(hipVerMinor);
oarr.push_back(hipVerPatch);
transformOptions(oarr, program);
std::copy(oarr.begin(), oarr.end(), std::ostream_iterator<std::string>(ostrstr, " "));
std::vector<amd::Device*> devices{hip::getCurrentDevice()->devices()[0]};
if (CL_SUCCESS != program->build(devices, ostrstr.str().c_str(), nullptr, nullptr)) {
HIPRTC_RETURN(HIPRTC_ERROR_COMPILATION);
}
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcAddNameExpression(hiprtcProgram prog, const char* name_expression) {
HIPRTC_INIT_API(prog, name_expression);
if (name_expression == nullptr) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
uint32_t id = ProgramState::instance().addNameExpression(name_expression);
const auto var{"__hiprtc_" + std::to_string(id)};
const auto code{"\nextern \"C\" constexpr auto " + var + " = " + name_expression + ';'};
program->appendToSource(code.c_str());
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcGetLoweredName(hiprtcProgram prog, const char* name_expression,
const char** loweredName) {
HIPRTC_INIT_API(prog, name_expression, loweredName);
if (name_expression == nullptr || loweredName == nullptr) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
device::Program* dev_program
= program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
auto it = ProgramState::instance().nameExpresssion_.find(name_expression);
if (it == ProgramState::instance().nameExpresssion_.end()) {
return HIPRTC_ERROR_NAME_EXPRESSION_NOT_VALID;
}
std::string strippedName = it->second.first;
std::vector<std::string> mangledNames;
if (!dev_program->getLoweredNames(&mangledNames)) {
HIPRTC_RETURN(HIPRTC_ERROR_COMPILATION);
}
for (auto &name : mangledNames) {
std::string demangledName = handleMangledName(name);
if (demangledName == strippedName) {
it->second.second.assign(name);
}
}
*loweredName = it->second.second.c_str();
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcDestroyProgram(hiprtcProgram* prog) {
HIPRTC_INIT_API(prog);
if (prog == NULL) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
// Release program. hiprtcProgram is a double pointer so free *prog
amd::Program* program = as_amd(reinterpret_cast<cl_program>(*prog));
program->release();
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcGetCode(hiprtcProgram prog, char* binaryMem) {
HIPRTC_INIT_API(prog, binaryMem);
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
const device::Program::binary_t& binary =
program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0])->binary();
::memcpy(binaryMem, binary.first, binary.second);
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcGetCodeSize(hiprtcProgram prog, size_t* binarySizeRet) {
HIPRTC_INIT_API(prog, binarySizeRet);
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
*binarySizeRet =
program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0])->binary().second;
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcGetProgramLog(hiprtcProgram prog, char* dst) {
HIPRTC_INIT_API(prog, dst);
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
const device::Program* devProgram =
program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
auto log = program->programLog() + devProgram->buildLog().c_str();
log.copy(dst, log.size());
dst[log.size()] = '\0';
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcGetProgramLogSize(hiprtcProgram prog, size_t* logSizeRet) {
HIPRTC_INIT_API(prog, logSizeRet);
amd::Program* program = as_amd(reinterpret_cast<cl_program>(prog));
const device::Program* devProgram =
program->getDeviceProgram(*hip::getCurrentDevice()->devices()[0]);
auto log = program->programLog() + devProgram->buildLog().c_str();
*logSizeRet = log.size() + 1;
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
hiprtcResult hiprtcVersion(int* major, int* minor) {
HIPRTC_INIT_API(major, minor);
if (major == nullptr || minor == nullptr) {
HIPRTC_RETURN(HIPRTC_ERROR_INVALID_INPUT);
}
*major = 9;
*minor = 0;
HIPRTC_RETURN(HIPRTC_SUCCESS);
}
projects/hip/src/hipamd/src/hip_stream.cpp
Executable
+598
Fájl megtekintése
@@ -0,0 +1,598 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
#include "hip_event.hpp"
#include "thread/monitor.hpp"
#include "hip_prof_api.h"
extern api_callbacks_table_t callbacks_table;
static amd::Monitor streamSetLock{"Guards global stream set"};
static std::unordered_set<hip::Stream*> streamSet;
namespace hip {
// ================================================================================================
Stream::Stream(hip::Device* dev, Priority p, unsigned int f, bool null_stream,
const std::vector<uint32_t>& cuMask, hipStreamCaptureStatus captureStatus)
: queue_(nullptr),
lock_("Stream Callback lock"),
device_(dev),
priority_(p),
flags_(f),
null_(null_stream),
cuMask_(cuMask),
captureStatus_(captureStatus) {}
// ================================================================================================
Stream::~Stream() {
if (queue_ != nullptr) {
amd::ScopedLock lock(streamSetLock);
streamSet.erase(this);
queue_->release();
queue_ = nullptr;
}
}
hipError_t Stream::EndCapture() {
for (auto event : captureEvents_) {
hip::Event* e = reinterpret_cast<hip::Event*>(event);
e->EndCapture();
}
for (auto stream : parallelCaptureStreams_) {
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
s->EndCapture();
}
captureStatus_ = hipStreamCaptureStatusNone;
pCaptureGraph_ = nullptr;
originStream_ = false;
parentStream_ = nullptr;
lastCapturedNodes_.clear();
parallelCaptureStreams_.clear();
captureEvents_.clear();
return hipSuccess;
}
// ================================================================================================
bool Stream::Create() {
// Enable queue profiling if a profiler is attached which sets the callback_table flag
// or if we force it with env var. This would enable time stamp collection for every
// command submitted to the stream(queue).
bool isProfilerAttached = callbacks_table.is_enabled();
cl_command_queue_properties properties = (isProfilerAttached || HIP_FORCE_QUEUE_PROFILING) ?
CL_QUEUE_PROFILING_ENABLE : 0;
amd::CommandQueue::Priority p;
switch (priority_) {
case Priority::High:
p = amd::CommandQueue::Priority::High;
break;
case Priority::Low:
p = amd::CommandQueue::Priority::Low;
break;
case Priority::Normal:
default:
p = amd::CommandQueue::Priority::Normal;
break;
}
amd::HostQueue* queue = new amd::HostQueue(*device_->asContext(), *device_->devices()[0],
properties, amd::CommandQueue::RealTimeDisabled,
p, cuMask_);
// Create a host queue
bool result = (queue != nullptr) ? queue->create() : false;
// Insert just created stream into the list of the blocking queues
if (result) {
amd::ScopedLock lock(streamSetLock);
streamSet.insert(this);
queue_ = queue;
queue->vdev()->profilerAttach(isProfilerAttached);
} else if (queue != nullptr) {
queue->release();
}
return result;
}
// ================================================================================================
amd::HostQueue* Stream::asHostQueue(bool skip_alloc) {
if (queue_ != nullptr) {
return queue_;
}
// Access to the stream object is lock protected, because possible allocation
amd::ScopedLock l(Lock());
if (queue_ == nullptr) {
// Create the host queue for the first time
if (!skip_alloc) {
Create();
}
}
return queue_;
}
// ================================================================================================
void Stream::Finish() const {
if (queue_ != nullptr) {
queue_->finish();
}
}
// ================================================================================================
int Stream::DeviceId() const {
return device_->deviceId();
}
int Stream::DeviceId(const hipStream_t hStream) {
hip::Stream* s = reinterpret_cast<hip::Stream*>(hStream);
int deviceId = (s != nullptr)? s->DeviceId() : ihipGetDevice();
assert(deviceId >= 0 && deviceId < static_cast<int>(g_devices.size()));
return deviceId;
}
void Stream::syncNonBlockingStreams() {
amd::ScopedLock lock(streamSetLock);
for (auto& it : streamSet) {
if (it->Flags() & hipStreamNonBlocking) {
it->asHostQueue()->finish();
}
}
}
// ================================================================================================
bool isValid(hipStream_t stream) {
// NULL stream is always valid
if (stream == nullptr) {
return true;
}
hip::Stream* s = reinterpret_cast<hip::Stream*>(stream);
amd::ScopedLock lock(streamSetLock);
if (streamSet.find(s) == streamSet.end()) {
return false;
}
return true;
}
};// hip namespace
// ================================================================================================
void iHipWaitActiveStreams(amd::HostQueue* blocking_queue, bool wait_null_stream) {
amd::Command::EventWaitList eventWaitList;
{
amd::ScopedLock lock(streamSetLock);
for (const auto& stream : streamSet) {
amd::HostQueue* active_queue = stream->asHostQueue();
// If it's the current device
if ((&active_queue->device() == &blocking_queue->device()) &&
// Make sure it's a default stream
((stream->Flags() & hipStreamNonBlocking) == 0) &&
// and it's not the current stream
(active_queue != blocking_queue) &&
// check for a wait on the null stream
(stream->Null() == wait_null_stream)) {
// Get the last valid command
amd::Command* command = active_queue->getLastQueuedCommand(true);
if (command != nullptr) {
// Check the current active status
if (command->status() != CL_COMPLETE) {
command->notifyCmdQueue();
eventWaitList.push_back(command);
} else {
command->release();
}
}
// Nullstream, hence there is nothing else to wait
if (wait_null_stream) {
break;
}
}
}
}
// Check if we have to wait anything
if (eventWaitList.size() > 0) {
amd::Command* command = new amd::Marker(*blocking_queue, kMarkerDisableFlush, eventWaitList);
if (command != nullptr) {
command->enqueue();
command->release();
}
}
// Release all active commands. It's safe after the marker was enqueued
for (const auto& it : eventWaitList) {
it->release();
}
}
// ================================================================================================
void CL_CALLBACK ihipStreamCallback(cl_event event, cl_int command_exec_status, void* user_data) {
hipError_t status = hipSuccess;
StreamCallback* cbo = reinterpret_cast<StreamCallback*>(user_data);
cbo->callBack_(cbo->stream_, status, cbo->userData_);
cbo->command_->release();
delete cbo;
}
// ================================================================================================
static hipError_t ihipStreamCreate(hipStream_t* stream,
unsigned int flags, hip::Stream::Priority priority,
const std::vector<uint32_t>& cuMask = {}) {
if (flags != hipStreamDefault && flags != hipStreamNonBlocking) {
return hipErrorInvalidValue;
}
hip::Stream* hStream = new hip::Stream(hip::getCurrentDevice(), priority, flags, false, cuMask);
if (hStream == nullptr || !hStream->Create()) {
delete hStream;
return hipErrorOutOfMemory;
}
*stream = reinterpret_cast<hipStream_t>(hStream);
return hipSuccess;
}
// ================================================================================================
hipError_t hipStreamCreateWithFlags(hipStream_t *stream, unsigned int flags) {
HIP_INIT_API(hipStreamCreateWithFlags, stream, flags);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(ihipStreamCreate(stream, flags, hip::Stream::Priority::Normal), *stream);
}
// ================================================================================================
hipError_t hipStreamCreate(hipStream_t *stream) {
HIP_INIT_API(hipStreamCreate, stream);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(ihipStreamCreate(stream, hipStreamDefault, hip::Stream::Priority::Normal), *stream);
}
// ================================================================================================
hipError_t hipStreamCreateWithPriority(hipStream_t* stream, unsigned int flags, int priority) {
HIP_INIT_API(hipStreamCreateWithPriority, stream, flags, priority);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
hip::Stream::Priority streamPriority;
if (priority <= hip::Stream::Priority::High) {
streamPriority = hip::Stream::Priority::High;
} else if (priority >= hip::Stream::Priority::Low) {
streamPriority = hip::Stream::Priority::Low;
} else {
streamPriority = hip::Stream::Priority::Normal;
}
HIP_RETURN(ihipStreamCreate(stream, flags, streamPriority), *stream);
}
// ================================================================================================
hipError_t hipDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) {
HIP_INIT_API(hipDeviceGetStreamPriorityRange, leastPriority, greatestPriority);
if (leastPriority != nullptr) {
*leastPriority = hip::Stream::Priority::Low;
}
if (greatestPriority != nullptr) {
*greatestPriority = hip::Stream::Priority::High;
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipStreamGetFlags(hipStream_t stream, unsigned int* flags) {
HIP_INIT_API(hipStreamGetFlags, stream, flags);
if ((flags != nullptr) && (stream != nullptr)) {
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
*flags = reinterpret_cast<hip::Stream*>(stream)->Flags();
} else {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipStreamSynchronize(hipStream_t stream) {
HIP_INIT_API(hipStreamSynchronize, stream);
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
// Wait for the current host queue
hip::getQueue(stream)->finish();
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipStreamDestroy(hipStream_t stream) {
HIP_INIT_API(hipStreamDestroy, stream);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
delete reinterpret_cast<hip::Stream*>(stream);
HIP_RETURN(hipSuccess);
}
struct CallbackData {
int previous_read_index;
hip::ihipIpcEventShmem_t *shmem;
};
void WaitThenDecrementSignal(hipStream_t stream, hipError_t status, void* user_data){
CallbackData *data = (CallbackData *)user_data;
int offset = data->previous_read_index % IPC_SIGNALS_PER_EVENT;
while (data->shmem->read_index < data->previous_read_index+IPC_SIGNALS_PER_EVENT
&& data->shmem->signal[offset] != 0) {
}
delete data;
}
// ================================================================================================
hipError_t hipStreamWaitEvent(hipStream_t stream, hipEvent_t event, unsigned int flags) {
HIP_INIT_API(hipStreamWaitEvent, stream, event, flags);
EVENT_CAPTURE(hipStreamWaitEvent, event, stream, flags);
if (event == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
amd::HostQueue* queue = hip::getQueue(stream);
hip::Event* e = reinterpret_cast<hip::Event*>(event);
if (e->flags & hipEventInterprocess) {
amd::Command* command = new amd::Marker(*queue, false);
auto t{new CallbackData{e->ipc_evt_.ipc_shmem_->read_index, e->ipc_evt_.ipc_shmem_}};
StreamCallback* cbo = new StreamCallback(stream,
reinterpret_cast<hipStreamCallback_t> (WaitThenDecrementSignal), t, command);
if (!command->setCallback(CL_COMPLETE, ihipStreamCallback,cbo)) {
command->release();
return hipErrorInvalidHandle;
}
command->enqueue();
command->awaitCompletion();
HIP_RETURN(hipSuccess);
} else {
HIP_RETURN(e->streamWait(queue, flags));
}
}
// ================================================================================================
hipError_t hipStreamQuery(hipStream_t stream) {
HIP_INIT_API(hipStreamQuery, stream);
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
amd::HostQueue* hostQueue = hip::getQueue(stream);
amd::Command* command = hostQueue->getLastQueuedCommand(true);
if (command == nullptr) {
// Nothing was submitted to the queue
HIP_RETURN(hipSuccess);
}
amd::Event& event = command->event();
if (command->type() != 0) {
event.notifyCmdQueue();
}
hipError_t status = (command->status() == CL_COMPLETE) ? hipSuccess : hipErrorNotReady;
command->release();
HIP_RETURN(status);
}
// ================================================================================================
hipError_t hipStreamAddCallback(hipStream_t stream, hipStreamCallback_t callback, void* userData,
unsigned int flags) {
HIP_INIT_API(hipStreamAddCallback, stream, callback, userData, flags);
//flags - Reserved for future use, must be 0
if (callback == nullptr || flags != 0) {
HIP_RETURN(hipErrorInvalidValue);
}
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
amd::HostQueue* hostQueue = hip::getQueue(stream);
amd::Command* last_command = hostQueue->getLastQueuedCommand(true);
amd::Command::EventWaitList eventWaitList;
if (last_command != nullptr) {
eventWaitList.push_back(last_command);
}
amd::Command* command = new amd::Marker(*hostQueue, !kMarkerDisableFlush, eventWaitList);
if (command == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
StreamCallback* cbo = new StreamCallback(stream, callback, userData, command);
if ((cbo == nullptr) || !command->setCallback(CL_COMPLETE, ihipStreamCallback, cbo)) {
command->release();
if (last_command != nullptr) {
last_command->release();
}
return hipErrorInvalidHandle;
}
// Retain callback command for the blocking marker
command->retain();
command->enqueue();
// @note: don't release the command here, because it will be released after HIP callback
if (last_command != nullptr) {
last_command->release();
}
// Add the new barrier to stall the stream, until the callback is done
eventWaitList.clear();
eventWaitList.push_back(command);
amd::Command* block_command = new amd::Marker(*hostQueue, !kMarkerDisableFlush, eventWaitList);
if (block_command == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
block_command->enqueue();
block_command->release();
// Release the callback marker
command->release();
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipExtStreamCreateWithCUMask(hipStream_t* stream, uint32_t cuMaskSize,
const uint32_t* cuMask) {
HIP_INIT_API(hipExtStreamCreateWithCUMask, stream, cuMaskSize, cuMask);
if (stream == nullptr) {
HIP_RETURN(hipErrorInvalidHandle);
}
if (cuMaskSize == 0 || cuMask == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
const std::vector<uint32_t> cuMaskv(cuMask, cuMask + cuMaskSize);
HIP_RETURN(ihipStreamCreate(stream, hipStreamDefault, hip::Stream::Priority::Normal, cuMaskv), *stream);
}
// ================================================================================================
hipError_t hipStreamGetPriority(hipStream_t stream, int* priority) {
HIP_INIT_API(hipStreamGetPriority, stream, priority);
if ((priority != nullptr) && (stream != nullptr)) {
if (!hip::isValid(stream)) {
return HIP_RETURN(hipErrorContextIsDestroyed);
}
*priority = static_cast<int>(reinterpret_cast<hip::Stream*>(stream)->GetPriority());
} else {
HIP_RETURN(hipErrorInvalidValue);
}
HIP_RETURN(hipSuccess);
}
// ================================================================================================
hipError_t hipExtStreamGetCUMask(hipStream_t stream, uint32_t cuMaskSize, uint32_t* cuMask) {
HIP_INIT_API(hipExtStreamGetCUMask, stream, cuMaskSize, cuMask);
if (cuMask == nullptr) {
HIP_RETURN(hipErrorInvalidValue);
}
int deviceId = hip::getCurrentDevice()->deviceId();
auto* deviceHandle = g_devices[deviceId]->devices()[0];
const auto& info = deviceHandle->info();
// find the minimum cuMaskSize required to present the CU mask bit-array in a patch of 32 bits
// and return error if the cuMaskSize argument is less than cuMaskSizeRequired
uint32_t cuMaskSizeRequired = info.maxComputeUnits_ / 32 +
((info.maxComputeUnits_ % 32) ? 1 : 0);
if (cuMaskSize < cuMaskSizeRequired) {
HIP_RETURN(hipErrorInvalidValue);
}
// make a default CU mask bit-array where all CUs are active
// this default mask will be returned when there is no
// custom or global CU mask defined
std::vector<uint32_t> defaultCUMask;
uint32_t temp = 0;
uint32_t bit_index = 0;
for (uint32_t i = 0; i < info.maxComputeUnits_; i++) {
temp |= 1UL << bit_index;
if (bit_index >= 32) {
defaultCUMask.push_back(temp);
temp = 0;
bit_index = 0;
temp |= 1UL << bit_index;
}
bit_index += 1;
}
if (bit_index != 0) {
defaultCUMask.push_back(temp);
}
// if the stream is null then either return globalCUMask_ (if it is defined)
// or return defaultCUMask
if (stream == nullptr) {
if (info.globalCUMask_.size() != 0) {
std::copy(info.globalCUMask_.begin(), info.globalCUMask_.end(), cuMask);
} else {
std::copy(defaultCUMask.begin(), defaultCUMask.end(), cuMask);
}
} else {
// if the stream is not null then get the stream's CU mask and return one of the below cases
// case1 if globalCUMask_ is defined then return the AND of globalCUMask_ and stream's CU mask
// case2 if globalCUMask_ is not defined then retuen AND of defaultCUMask and stream's CU mask
// in both cases above if stream's CU mask is empty then either globalCUMask_ (for case1)
// or defaultCUMask(for case2) will be returned
std::vector<uint32_t> streamCUMask;
streamCUMask = reinterpret_cast<hip::Stream*>(stream)->GetCUMask();
std::vector<uint32_t> mask = {};
if (info.globalCUMask_.size() != 0) {
for (uint32_t i = 0; i < std::min(streamCUMask.size(), info.globalCUMask_.size()); i++) {
mask.push_back(streamCUMask[i] & info.globalCUMask_[i]);
}
} else {
for (uint32_t i = 0; i < std::min(streamCUMask.size(), defaultCUMask.size()); i++) {
mask.push_back(streamCUMask[i] & defaultCUMask[i]);
}
// check to make sure after ANDing streamCUMask (custom-defined) with global CU mask,
//we have non-zero mask, oterwise just return either globalCUMask_ or defaultCUMask
bool zeroCUMask = true;
for (auto m : mask) {
if (m != 0) {
zeroCUMask = false;
break;
}
}
if (zeroCUMask) {
mask = (info.globalCUMask_.size() != 0) ? info.globalCUMask_ : defaultCUMask;
}
std::copy(mask.begin(), mask.end(), cuMask);
}
}
HIP_RETURN(hipSuccess);
}
projects/hip/src/hipamd/src/hip_stream_ops.cpp
+129
Fájl megtekintése
@@ -0,0 +1,129 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
#include "platform/command_utils.hpp"
hipError_t ihipStreamOperation(hipStream_t stream, cl_command_type cmdType, void* ptr,
int64_t value, uint64_t mask, unsigned int flags, size_t sizeBytes) {
size_t offset = 0;
unsigned int outFlags = 0;
amd::Memory* memory = getMemoryObject(ptr, offset);
if (!memory) {
return hipErrorInvalidValue;
}
// NOTE: 'mask' is only used in Wait operation, 'sizeBytes' is only used in Write operation
// 'flags' for now used only for Wait, but in future there will usecases for Write too.
if (cmdType == ROCCLR_COMMAND_STREAM_WAIT_VALUE) {
// Wait is only supported on SignalMemory objects
if (!(memory->getMemFlags() & ROCCLR_MEM_HSA_SIGNAL_MEMORY)) {
return hipErrorInvalidValue;
}
switch (flags) {
case hipStreamWaitValueGte:
outFlags = ROCCLR_STREAM_WAIT_VALUE_GTE;
break;
case hipStreamWaitValueEq:
outFlags = ROCCLR_STREAM_WAIT_VALUE_EQ;
break;
case hipStreamWaitValueAnd:
outFlags = ROCCLR_STREAM_WAIT_VALUE_AND;
break;
case hipStreamWaitValueNor:
outFlags = ROCCLR_STREAM_WAIT_VALUE_NOR;
break;
default:
ShouldNotReachHere();
break;
}
} else if (cmdType != ROCCLR_COMMAND_STREAM_WRITE_VALUE) {
return hipErrorInvalidValue;
}
amd::HostQueue* queue = hip::getQueue(stream);
amd::Command::EventWaitList waitList;
amd::StreamOperationCommand* command =
new amd::StreamOperationCommand(*queue, cmdType, waitList, *memory->asBuffer(),
value, mask, outFlags, offset, sizeBytes);
if (command == nullptr) {
return hipErrorOutOfMemory;
}
command->enqueue();
command->release();
return hipSuccess;
}
hipError_t hipStreamWaitValue32(hipStream_t stream, void* ptr, int32_t value, unsigned int flags,
uint32_t mask) {
HIP_INIT_API(hipStreamWaitValue32, stream, ptr, value, mask, flags);
// NOTE: ptr corresponds to a HSA Signal memeory which is 64 bits.
// 32 bit value and mask are converted to 64-bit values.
HIP_RETURN_DURATION(ihipStreamOperation(
stream,
ROCCLR_COMMAND_STREAM_WAIT_VALUE,
ptr,
value,
mask,
flags,
0)); // sizeBytes un-used for wait, set it to 0
}
hipError_t hipStreamWaitValue64(hipStream_t stream, void* ptr, int64_t value, unsigned int flags,
uint64_t mask) {
HIP_INIT_API(hipStreamWaitValue64, stream, ptr, value, mask, flags);
HIP_RETURN_DURATION(ihipStreamOperation(
stream,
ROCCLR_COMMAND_STREAM_WAIT_VALUE,
ptr,
value,
mask,
flags,
0)); // sizeBytes un-used for wait, set it to 0
}
hipError_t hipStreamWriteValue32(hipStream_t stream, void* ptr, int32_t value, unsigned int flags) {
HIP_INIT_API(hipStreamWriteValue32, stream, ptr, value, flags);
HIP_RETURN_DURATION(ihipStreamOperation(
stream,
ROCCLR_COMMAND_STREAM_WRITE_VALUE,
ptr,
value,
0, // mask un-used set it to 0
0, // flags un-used for now set it to 0
4));
}
hipError_t hipStreamWriteValue64(hipStream_t stream, void* ptr, int64_t value, unsigned int flags) {
HIP_INIT_API(hipStreamWriteValue64, stream, ptr, value, flags);
HIP_RETURN_DURATION(ihipStreamOperation(
stream,
ROCCLR_COMMAND_STREAM_WRITE_VALUE,
ptr,
value,
0, // mask un-used set it to 0
0, // flags un-used for now set it to 0
8));
}
projects/hip/src/hipamd/src/hip_surface.cpp
+37
Fájl megtekintése
@@ -0,0 +1,37 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#include <hip/hip_runtime.h>
#include "hip_internal.hpp"
#include <hip/amd_detail/amd_hip_surface_types.h>
hipError_t hipCreateSurfaceObject(hipSurfaceObject_t* pSurfObject,
const hipResourceDesc* pResDesc) {
HIP_INIT_API(hipCreateSurfaceObject, pSurfObject, pResDesc);
HIP_RETURN(hipErrorNotSupported);
}
hipError_t hipDestroySurfaceObject(hipSurfaceObject_t surfaceObject) {
HIP_INIT_API(hipDestroySurfaceObject, surfaceObject);
HIP_RETURN(hipErrorNotSupported);
}
projects/hip/src/hipamd/src/hip_texture.cpp
Executable
+1303
Fájl megtekintése
File diff suppressed because it is too large. Load diff
projects/hip/src/hipamd/src/hiprtc_internal.hpp
+65
Fájl megtekintése
@@ -0,0 +1,65 @@
/* Copyright (c) 2015-present Advanced Micro Devices, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. */
#ifndef HIPRTC_SRC_HIP_INTERNAL_H
#define HIPRTC_SRC_HIP_INTERNAL_H
#include "hip_internal.hpp"
#if __linux__
#include <cstdlib>
#if HIPRTC_USE_CXXABI
#include <cxxabi.h>
#define DEMANGLE abi::__cxa_demangle
#else
extern "C" char * __cxa_demangle(const char *mangled_name, char *output_buffer,
size_t *length, int *status);
#define DEMANGLE __cxa_demangle
#endif //HIPRTC_USE_CXXABI
#elif defined(_WIN32)
#include <Windows.h>
#include <DbgHelp.h>
#define UNDECORATED_SIZE 4096
#endif // __linux__
// This macro should be called at the beginning of every HIP RTC API.
#define HIPRTC_INIT_API(...) \
ClPrint(amd::LOG_INFO, amd::LOG_API, "[%zx] %s ( %s )", std::this_thread::get_id(), __func__, ToString( __VA_ARGS__ ).c_str()); \
amd::Thread* thread = amd::Thread::current(); \
if (!VDI_CHECK_THREAD(thread)) { \
HIPRTC_RETURN(HIPRTC_ERROR_INTERNAL_ERROR); \
} \
HIP_INIT();
#define HIPRTC_RETURN(ret) \
hiprtc::g_lastRtcError = ret; \
ClPrint(amd::LOG_INFO, amd::LOG_API, "[%zx] %s: Returned %s", std::this_thread::get_id(), __func__, \
hiprtcGetErrorString(hiprtc::g_lastRtcError)); \
return hiprtc::g_lastRtcError;
#endif // HIPRTC_SRC_HIP_INTERNAL_H
Loaded 100 of 119 files, more files were not shown because too many files have changed in this diff. Show more